From 174bdd582b311a5fdbdba3542bad22eef95cacdc Mon Sep 17 00:00:00 2001 From: Nikita Smith Date: Wed, 26 Aug 2026 17:02:59 -0700 Subject: [PATCH 001/121] switch to a script based declarative model for linker tests; new COFF codec produces byte identical binaries for the tests --- src/base/base_core.h | 14 +- src/base/base_files.c | 14 + src/base/base_files.h | 2 + src/linker/tests/abs_vs_abs.tst | 20 + src/linker/tests/abs_vs_common.tst | 25 + src/linker/tests/abs_vs_regular.tst | 45 + src/linker/tests/abs_vs_weak.tst | 44 + src/linker/tests/alt_name.tst | 66 + src/linker/tests/base_relocs.tst | 81 + src/linker/tests/comdat_any.tst | 77 + src/linker/tests/comdat_associative.tst | 79 + src/linker/tests/comdat_associative_loop.tst | 53 + .../tests/comdat_associative_non_comdat.tst | 53 + .../comdat_associative_out_of_bounds.tst | 51 + src/linker/tests/comdat_exact_match.tst | 78 + ...mdat_external_symbol_at_nonzero_offset.tst | 68 + .../comdat_external_symbol_at_zero_offset.tst | 66 + ...dat_external_symbol_uses_leader_offset.tst | 73 + src/linker/tests/comdat_largest.tst | 83 + ...t_external_symbol_uses_selected_offset.tst | 92 + src/linker/tests/comdat_no_duplicates.tst | 72 + src/linker/tests/comdat_same_size.tst | 78 + src/linker/tests/comdat_with_offset.tst | 59 + src/linker/tests/common_block.tst | 119 + src/linker/tests/communal_var_vs_regular.tst | 42 + .../tests/communal_var_vs_regular_comdat.tst | 45 + src/linker/tests/cyclic_type.tst | 56 + .../tests/debug_p_and_debug_t_in_obj.tst | 81 + src/linker/tests/debug_p_sig_mismatch.tst | 126 + src/linker/tests/def_file_full.tst | 236 + src/linker/tests/defer_duplicate_imp_link.tst | 123 + src/linker/tests/defer_imp_link.tst | 55 + .../defer_impl_link_to_second_search_pass.tst | 48 + src/linker/tests/delay_import.tst | 231 + src/linker/tests/delay_import_user32.tst | 68 + src/linker/tests/determ_test.tst | 59 + src/linker/tests/empty_section.tst | 41 + src/linker/tests/fail_if_mismatch.tst | 40 + src/linker/tests/find_merged_pdata.tst | 54 + src/linker/tests/first_member_header.tst | 82 + src/linker/tests/flag_conf.tst | 51 + src/linker/tests/function_pad_min.tst | 30 + src/linker/tests/gen_rrt.tst | 202 + src/linker/tests/get_msf_stream_pages.tst | 4 + src/linker/tests/ghash_basic.tst | 114 + src/linker/tests/ghash_check_corrupt.tst | 40 + src/linker/tests/ghash_check_hash_alg.tst | 40 + src/linker/tests/ghash_check_magic.tst | 40 + src/linker/tests/ghash_check_version.tst | 40 + src/linker/tests/ghash_match_debug_t.tst | 48 + .../tests/guard_cf_pulls_load_config.tst | 42 + src/linker/tests/guardsym_directive.tst | 25 + .../icf_associative_child_prevents_fold.tst | 53 + .../tests/icf_comdat_reloc_targets_fold.tst | 108 + src/linker/tests/icf_comdat_symlink_chain.tst | 83 + .../icf_cpp_identical_functions_fold.tst | 49 + ...icf_cpp_multihop_functions_do_not_fold.tst | 55 + src/linker/tests/icf_cyclic_icf.tst | 37 + src/linker/tests/icf_fold_diamond.tst | 51 + src/linker/tests/icf_fold_two_funcs.tst | 81 + .../tests/icf_fold_with_largest_align.tst | 96 + ...tes_different_color_spaces_do_not_fold.tst | 49 + src/linker/tests/icf_llvm_addrsig.tst | 36 + .../icf_llvm_addrsig_external_symbol.tst | 46 + ...ltihop_reloc_target_colors_do_not_fold.tst | 69 + .../icf_pdata_differs_by_function_color.tst | 62 + src/linker/tests/icf_pdata_xdata_fold.tst | 91 + .../icf_readonly_non_vftable_data_policy.tst | 58 + ..._reloc_target_symbol_types_do_not_fold.tst | 109 + src/linker/tests/icf_same_but_different.tst | 97 + ...f_unresolved_reloc_targets_do_not_fold.tst | 91 + .../tests/icf_vftable_and_vbtable_policy.tst | 79 + ...able_external_symbol_at_nonzero_offset.tst | 63 + src/linker/tests/image_base.tst | 45 + src/linker/tests/import_export.tst | 293 + src/linker/tests/import_kernel32.tst | 118 + src/linker/tests/include.tst | 41 + src/linker/tests/infer_asan.tst | 52 + src/linker/tests/invalid_bss.tst | 49 + ...ber_imp_and_regular_symbol_queued_once.tst | 38 + ...b_member_reloc_apply_off_out_of_bounds.tst | 55 + src/linker/tests/link_undef.tst | 24 + src/linker/tests/link_unref_undef.tst | 20 + src/linker/tests/linker_tests.c | 9329 +---------------- src/linker/tests/long_section_name.tst | 33 + src/linker/tests/machine_compat_check.tst | 102 + src/linker/tests/map.tst | 65 + src/linker/tests/merge.tst | 84 + src/linker/tests/merge_duplicate_types.tst | 285 + .../ms_link_icf_section_flag_eligibility.tst | 110 + .../tests/ms_link_icfs_identical_comdats.tst | 85 + src/linker/tests/opt_ref_dangling_section.tst | 54 + .../tests/opt_ref_weak_alias_comdat.tst | 32 + .../tests/out_of_bounds_section_number.tst | 45 + src/linker/tests/patch_cv_symbol_tree.tst | 4 + src/linker/tests/pch_sig_fallback.tst | 252 + src/linker/tests/pdbstripped.tst | 161 + src/linker/tests/psi_addr_map_radix_sort.tst | 4 + .../tests/reloc_against_removed_comdat.tst | 71 + .../tests/reloc_apply_off_out_of_bounds.tst | 30 + .../relocate_undefined_section_symbol.tst | 150 + src/linker/tests/removed_section.tst | 45 + src/linker/tests/second_member_header.tst | 34 + src/linker/tests/sect_align.tst | 56 + src/linker/tests/sect_symbol.tst | 49 + ...ection_directive_align_grouped_section.tst | 36 + ...on_directive_read_only_grouped_section.tst | 35 + src/linker/tests/section_sort.tst | 43 + src/linker/tests/simple_lib_test.tst | 43 + src/linker/tests/simple_link_test.tst | 94 + .../tests/u64_array_radix_sort_parallel.tst | 4 + src/linker/tests/undef_reloc_section.tst | 29 + src/linker/tests/undef_section.tst | 52 + src/linker/tests/undef_weak_lib.tst | 27 + src/linker/tests/undef_weak_search_alias.tst | 37 + src/linker/tests/utf16_rsp.tst | 24 + src/linker/tests/validate_gsi.tst | 157 + .../tests/validate_gsi_procs_and_typedefs.tst | 176 + src/linker/tests/validate_info_stream.tst | 4 + src/linker/tests/validate_psi.tst | 85 + .../weak_alias_comdat_duplicate_fallback.tst | 493 + src/linker/tests/weak_alias_vs_weak_alias.tst | 30 + .../tests/weak_alias_vs_weak_antidep.tst | 29 + src/linker/tests/weak_alias_vs_weak_lib.tst | 29 + src/linker/tests/weak_alias_vs_weak_nolib.tst | 29 + .../tests/weak_antidep_vs_weak_alias.tst | 28 + .../tests/weak_antidep_vs_weak_antidep.tst | 39 + src/linker/tests/weak_antidep_vs_weak_lib.tst | 28 + .../tests/weak_antidep_vs_weak_nolib.tst | 28 + src/linker/tests/weak_cycle.tst | 44 + src/linker/tests/weak_lib_vs_weak_alias.tst | 31 + src/linker/tests/weak_lib_vs_weak_antidep.tst | 32 + src/linker/tests/weak_lib_vs_weak_lib.tst | 39 + src/linker/tests/weak_lib_vs_weak_nolib.tst | 32 + src/linker/tests/weak_nolib_vs_weak_alias.tst | 27 + .../tests/weak_nolib_vs_weak_antidep.tst | 28 + src/linker/tests/weak_nolib_vs_weak_lib.tst | 28 + src/linker/tests/weak_nolib_vs_weak_nolib.tst | 28 + src/linker/tests/weak_tag.tst | 28 + src/linker/tests/weak_vs_common.tst | 39 + src/linker/tests/whole_archive.tst | 45 + ...zero_length_comdat_referenced_by_reloc.tst | 52 + ...ngth_static_comdat_referenced_by_reloc.tst | 44 + src/mdesk/mdesk.h | 1 + src/torture/tests/torture_coff_tests.c | 464 + src/torture/tests/torture_linker_tests.c | 279 + src/torture/tests/torture_script_tests.c | 18 + src/torture/tests/torture_subprocess_tests.c | 147 + src/torture/torture.c | 315 +- src/torture/torture.h | 8 - src/torture/torture_base.c | 107 + src/torture/torture_build.c | 757 ++ src/torture/torture_build.h | 134 + src/torture/torture_coff.c | 931 ++ src/torture/torture_coff.h | 150 + src/torture/torture_controller.c | 317 + src/torture/torture_controller.h | 77 + src/torture/torture_linker.c | 1512 +++ src/torture/torture_linker.h | 26 + src/torture/torture_main.c | 40 +- src/torture/torture_script.c | 747 ++ src/torture/torture_script.h | 165 + 162 files changed, 15529 insertions(+), 9495 deletions(-) create mode 100644 src/linker/tests/abs_vs_abs.tst create mode 100644 src/linker/tests/abs_vs_common.tst create mode 100644 src/linker/tests/abs_vs_regular.tst create mode 100644 src/linker/tests/abs_vs_weak.tst create mode 100644 src/linker/tests/alt_name.tst create mode 100644 src/linker/tests/base_relocs.tst create mode 100644 src/linker/tests/comdat_any.tst create mode 100644 src/linker/tests/comdat_associative.tst create mode 100644 src/linker/tests/comdat_associative_loop.tst create mode 100644 src/linker/tests/comdat_associative_non_comdat.tst create mode 100644 src/linker/tests/comdat_associative_out_of_bounds.tst create mode 100644 src/linker/tests/comdat_exact_match.tst create mode 100644 src/linker/tests/comdat_external_symbol_at_nonzero_offset.tst create mode 100644 src/linker/tests/comdat_external_symbol_at_zero_offset.tst create mode 100644 src/linker/tests/comdat_external_symbol_uses_leader_offset.tst create mode 100644 src/linker/tests/comdat_largest.tst create mode 100644 src/linker/tests/comdat_largest_external_symbol_uses_selected_offset.tst create mode 100644 src/linker/tests/comdat_no_duplicates.tst create mode 100644 src/linker/tests/comdat_same_size.tst create mode 100644 src/linker/tests/comdat_with_offset.tst create mode 100644 src/linker/tests/common_block.tst create mode 100644 src/linker/tests/communal_var_vs_regular.tst create mode 100644 src/linker/tests/communal_var_vs_regular_comdat.tst create mode 100644 src/linker/tests/cyclic_type.tst create mode 100644 src/linker/tests/debug_p_and_debug_t_in_obj.tst create mode 100644 src/linker/tests/debug_p_sig_mismatch.tst create mode 100644 src/linker/tests/def_file_full.tst create mode 100644 src/linker/tests/defer_duplicate_imp_link.tst create mode 100644 src/linker/tests/defer_imp_link.tst create mode 100644 src/linker/tests/defer_impl_link_to_second_search_pass.tst create mode 100644 src/linker/tests/delay_import.tst create mode 100644 src/linker/tests/delay_import_user32.tst create mode 100644 src/linker/tests/determ_test.tst create mode 100644 src/linker/tests/empty_section.tst create mode 100644 src/linker/tests/fail_if_mismatch.tst create mode 100644 src/linker/tests/find_merged_pdata.tst create mode 100644 src/linker/tests/first_member_header.tst create mode 100644 src/linker/tests/flag_conf.tst create mode 100644 src/linker/tests/function_pad_min.tst create mode 100644 src/linker/tests/gen_rrt.tst create mode 100644 src/linker/tests/get_msf_stream_pages.tst create mode 100644 src/linker/tests/ghash_basic.tst create mode 100644 src/linker/tests/ghash_check_corrupt.tst create mode 100644 src/linker/tests/ghash_check_hash_alg.tst create mode 100644 src/linker/tests/ghash_check_magic.tst create mode 100644 src/linker/tests/ghash_check_version.tst create mode 100644 src/linker/tests/ghash_match_debug_t.tst create mode 100644 src/linker/tests/guard_cf_pulls_load_config.tst create mode 100644 src/linker/tests/guardsym_directive.tst create mode 100644 src/linker/tests/icf_associative_child_prevents_fold.tst create mode 100644 src/linker/tests/icf_comdat_reloc_targets_fold.tst create mode 100644 src/linker/tests/icf_comdat_symlink_chain.tst create mode 100644 src/linker/tests/icf_cpp_identical_functions_fold.tst create mode 100644 src/linker/tests/icf_cpp_multihop_functions_do_not_fold.tst create mode 100644 src/linker/tests/icf_cyclic_icf.tst create mode 100644 src/linker/tests/icf_fold_diamond.tst create mode 100644 src/linker/tests/icf_fold_two_funcs.tst create mode 100644 src/linker/tests/icf_fold_with_largest_align.tst create mode 100644 src/linker/tests/icf_identical_bytes_different_color_spaces_do_not_fold.tst create mode 100644 src/linker/tests/icf_llvm_addrsig.tst create mode 100644 src/linker/tests/icf_llvm_addrsig_external_symbol.tst create mode 100644 src/linker/tests/icf_multihop_reloc_target_colors_do_not_fold.tst create mode 100644 src/linker/tests/icf_pdata_differs_by_function_color.tst create mode 100644 src/linker/tests/icf_pdata_xdata_fold.tst create mode 100644 src/linker/tests/icf_readonly_non_vftable_data_policy.tst create mode 100644 src/linker/tests/icf_reloc_target_symbol_types_do_not_fold.tst create mode 100644 src/linker/tests/icf_same_but_different.tst create mode 100644 src/linker/tests/icf_unresolved_reloc_targets_do_not_fold.tst create mode 100644 src/linker/tests/icf_vftable_and_vbtable_policy.tst create mode 100644 src/linker/tests/icf_vftable_external_symbol_at_nonzero_offset.tst create mode 100644 src/linker/tests/image_base.tst create mode 100644 src/linker/tests/import_export.tst create mode 100644 src/linker/tests/import_kernel32.tst create mode 100644 src/linker/tests/include.tst create mode 100644 src/linker/tests/infer_asan.tst create mode 100644 src/linker/tests/invalid_bss.tst create mode 100644 src/linker/tests/lib_member_imp_and_regular_symbol_queued_once.tst create mode 100644 src/linker/tests/lib_member_reloc_apply_off_out_of_bounds.tst create mode 100644 src/linker/tests/link_undef.tst create mode 100644 src/linker/tests/link_unref_undef.tst create mode 100644 src/linker/tests/long_section_name.tst create mode 100644 src/linker/tests/machine_compat_check.tst create mode 100644 src/linker/tests/map.tst create mode 100644 src/linker/tests/merge.tst create mode 100644 src/linker/tests/merge_duplicate_types.tst create mode 100644 src/linker/tests/ms_link_icf_section_flag_eligibility.tst create mode 100644 src/linker/tests/ms_link_icfs_identical_comdats.tst create mode 100644 src/linker/tests/opt_ref_dangling_section.tst create mode 100644 src/linker/tests/opt_ref_weak_alias_comdat.tst create mode 100644 src/linker/tests/out_of_bounds_section_number.tst create mode 100644 src/linker/tests/patch_cv_symbol_tree.tst create mode 100644 src/linker/tests/pch_sig_fallback.tst create mode 100644 src/linker/tests/pdbstripped.tst create mode 100644 src/linker/tests/psi_addr_map_radix_sort.tst create mode 100644 src/linker/tests/reloc_against_removed_comdat.tst create mode 100644 src/linker/tests/reloc_apply_off_out_of_bounds.tst create mode 100644 src/linker/tests/relocate_undefined_section_symbol.tst create mode 100644 src/linker/tests/removed_section.tst create mode 100644 src/linker/tests/second_member_header.tst create mode 100644 src/linker/tests/sect_align.tst create mode 100644 src/linker/tests/sect_symbol.tst create mode 100644 src/linker/tests/section_directive_align_grouped_section.tst create mode 100644 src/linker/tests/section_directive_read_only_grouped_section.tst create mode 100644 src/linker/tests/section_sort.tst create mode 100644 src/linker/tests/simple_lib_test.tst create mode 100644 src/linker/tests/simple_link_test.tst create mode 100644 src/linker/tests/u64_array_radix_sort_parallel.tst create mode 100644 src/linker/tests/undef_reloc_section.tst create mode 100644 src/linker/tests/undef_section.tst create mode 100644 src/linker/tests/undef_weak_lib.tst create mode 100644 src/linker/tests/undef_weak_search_alias.tst create mode 100644 src/linker/tests/utf16_rsp.tst create mode 100644 src/linker/tests/validate_gsi.tst create mode 100644 src/linker/tests/validate_gsi_procs_and_typedefs.tst create mode 100644 src/linker/tests/validate_info_stream.tst create mode 100644 src/linker/tests/validate_psi.tst create mode 100644 src/linker/tests/weak_alias_comdat_duplicate_fallback.tst create mode 100644 src/linker/tests/weak_alias_vs_weak_alias.tst create mode 100644 src/linker/tests/weak_alias_vs_weak_antidep.tst create mode 100644 src/linker/tests/weak_alias_vs_weak_lib.tst create mode 100644 src/linker/tests/weak_alias_vs_weak_nolib.tst create mode 100644 src/linker/tests/weak_antidep_vs_weak_alias.tst create mode 100644 src/linker/tests/weak_antidep_vs_weak_antidep.tst create mode 100644 src/linker/tests/weak_antidep_vs_weak_lib.tst create mode 100644 src/linker/tests/weak_antidep_vs_weak_nolib.tst create mode 100644 src/linker/tests/weak_cycle.tst create mode 100644 src/linker/tests/weak_lib_vs_weak_alias.tst create mode 100644 src/linker/tests/weak_lib_vs_weak_antidep.tst create mode 100644 src/linker/tests/weak_lib_vs_weak_lib.tst create mode 100644 src/linker/tests/weak_lib_vs_weak_nolib.tst create mode 100644 src/linker/tests/weak_nolib_vs_weak_alias.tst create mode 100644 src/linker/tests/weak_nolib_vs_weak_antidep.tst create mode 100644 src/linker/tests/weak_nolib_vs_weak_lib.tst create mode 100644 src/linker/tests/weak_nolib_vs_weak_nolib.tst create mode 100644 src/linker/tests/weak_tag.tst create mode 100644 src/linker/tests/weak_vs_common.tst create mode 100644 src/linker/tests/whole_archive.tst create mode 100644 src/linker/tests/zero_length_comdat_referenced_by_reloc.tst create mode 100644 src/linker/tests/zero_length_static_comdat_referenced_by_reloc.tst create mode 100644 src/torture/tests/torture_coff_tests.c create mode 100644 src/torture/tests/torture_linker_tests.c create mode 100644 src/torture/tests/torture_script_tests.c create mode 100644 src/torture/tests/torture_subprocess_tests.c create mode 100644 src/torture/torture_base.c create mode 100644 src/torture/torture_build.c create mode 100644 src/torture/torture_build.h create mode 100644 src/torture/torture_coff.c create mode 100644 src/torture/torture_coff.h create mode 100644 src/torture/torture_controller.c create mode 100644 src/torture/torture_controller.h create mode 100644 src/torture/torture_linker.c create mode 100644 src/torture/torture_linker.h create mode 100644 src/torture/torture_script.c create mode 100644 src/torture/torture_script.h diff --git a/src/base/base_core.h b/src/base/base_core.h index 438316179..043726e3a 100644 --- a/src/base/base_core.h +++ b/src/base/base_core.h @@ -672,18 +672,8 @@ Compiler; # define Compiler_CURRENT Compiler_Null #endif -typedef enum Linker -{ - Linker_Null, - Linker_radlink, - Linker_msvc, - Linker_lld, - Linker_COUNT -} -Linker; - -//////////////////////////////// -//~ rjf: Access Flags +//////////////////////////////// +//~ rjf: Access Flags typedef U32 AccessFlags; enum diff --git a/src/base/base_files.c b/src/base/base_files.c index ca63d3900..4c24664f4 100644 --- a/src/base/base_files.c +++ b/src/base/base_files.c @@ -18,6 +18,20 @@ file_match(File a, File b) return result; } +internal B32 +file_pair_ok(FilePair f) +{ + return !file_match(f.read, file_zero()) && !file_match(f.write, file_zero()); +} + +internal void +file_pair_close(FilePair *p) +{ + file_close(p->read); + file_close(p->write); + MemoryZeroStruct(p); +} + //////////////////////////////// //~ rjf: Filesystem Helpers (Helpers, Implemented Once) diff --git a/src/base/base_files.h b/src/base/base_files.h index b06216c17..016115ec5 100644 --- a/src/base/base_files.h +++ b/src/base/base_files.h @@ -58,6 +58,8 @@ struct FileMap internal File file_zero(void); internal B32 file_match(File a, File b); +internal B32 file_pair_ok(FilePair p); +internal void file_pair_close(FilePair *p); //////////////////////////////// //~ rjf: Filesystem Helpers (Helpers, Implemented Once) diff --git a/src/linker/tests/abs_vs_abs.tst b/src/linker/tests/abs_vs_abs.tst new file mode 100644 index 000000000..518c298fe --- /dev/null +++ b/src/linker/tests/abs_vs_abs.tst @@ -0,0 +1,20 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { foo: { kind: absolute, name: "foo", value: 97, storage: external } } } } } + b_obj: { file_name: "b.obj", coff: { object: { machine: x64, symbols: { foo: { kind: absolute, name: "foo", value: 98, storage: external } } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:error.exe a.obj b.obj entry.obj", expect_exit: 24 } } + steps: {} +} diff --git a/src/linker/tests/abs_vs_common.tst b/src/linker/tests/abs_vs_common.tst new file mode 100644 index 000000000..a102dea25 --- /dev/null +++ b/src/linker/tests/abs_vs_common.tst @@ -0,0 +1,25 @@ +test: +{ + artifacts: + { + common_obj: { file_name: "common.obj", coff: { object: { machine: x64, symbols: { foo: { kind: common, name: "foo", size: 321 } } } } } + abs_obj: { file_name: "abs.obj", coff: { object: { machine: x64, symbols: { foo: { kind: absolute, name: "foo", value: 4660, storage: external } } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } } + } + } + build: + { + link: { args: "/subsystem:console /entry:my_entry /out:common_after_absolute_definition_first.exe abs.obj common.obj entry.obj", expect_exit: any } + // TODO: validate that linker issues multiply defined symbol error + link: { args: "/subsystem:console /entry:my_entry /out:common_after_absolute_common_first.exe common.obj abs.obj entry.obj", when_previous_exit: 0, expect_exit: 24 } + } + steps: {} +} diff --git a/src/linker/tests/abs_vs_regular.tst b/src/linker/tests/abs_vs_regular.tst new file mode 100644 index 000000000..fec4172f3 --- /dev/null +++ b/src/linker/tests/abs_vs_regular.tst @@ -0,0 +1,45 @@ +test: +{ + artifacts: + { + regular_obj: + { + file_name: "regular.obj" + coff: { object: { machine: x64, sections: { data: { name: ".data", permissions: (read, write), content: initialized_data, data: { hex: "c0ffee" } } }, symbols: { foo: { kind: external, name: "foo", section: data, value: 0 } } } } + } + abs_obj: { file_name: "abs.obj", coff: { object: { machine: x64, symbols: { foo: { kind: absolute, name: "foo", value: 4660, storage: external } } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { foo_ref: { type: Addr32Nb, offset: 3, symbol: foo } } + } } + symbols: + { + entry: { kind: external, name: "my_entry", section: text, value: 0 } + foo: { kind: undefined, name: "foo" } + } + } } + } + } + build: + { + // TODO: validate that linker issues multiply defined symbol error + link: { args: "/subsystem:console /entry:my_entry /out:abs_first.exe abs.obj regular.obj entry.obj", expect_exit: nonzero } + // linker should complain about multiply defined symbol + link: { args: "/subsystem:console /entry:my_entry /out:regular_first.exe regular.obj abs.obj entry.obj", expect_exit: nonzero } + // linker should complain even in case regular is before abs + } + steps: {} +} diff --git a/src/linker/tests/abs_vs_weak.tst b/src/linker/tests/abs_vs_weak.tst new file mode 100644 index 000000000..295008813 --- /dev/null +++ b/src/linker/tests/abs_vs_weak.tst @@ -0,0 +1,44 @@ +test: +{ + artifacts: + { + abs_obj: { file_name: "abs.obj", coff: { object: { machine: x64, symbols: { foo: { kind: absolute, name: "foo", value: 291, storage: external } } } } } + text_obj: + { + file_name: "text.obj" + coff: { object: + { + machine: x64 + sections: + { + mydata: { name: ".mydata", permissions: (read, execute), content: code, alignment: 1, data: { text: "mydata" } } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "48b80000000000000000c3" } + relocations: { foo_ref: { type: Addr64, offset: 2, symbol: foo } } + } + } + symbols: + { + mydata: { kind: external, name: "mydata", section: mydata, value: 0 } + foo: { kind: weak, name: "foo", fallback: mydata, search: no_library } + entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } } + } + image: { file_name: "text_first.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:my_entry /out:abs_first.exe abs.obj text.obj" } + link: { args: "/subsystem:console /entry:my_entry /out:text_first.exe text.obj abs.obj", artifact: image } + } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48b82301000000000000c3 } } } } } + } +} diff --git a/src/linker/tests/alt_name.tst b/src/linker/tests/alt_name.tst new file mode 100644 index 000000000..bceb77fa7 --- /dev/null +++ b/src/linker/tests/alt_name.tst @@ -0,0 +1,66 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: { machine: x64, sections: { data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "test" } } }, symbols: { test: { kind: external, name: "test", section: data, value: 0 } } } } + } + foo_obj: + { + file_name: "foo.obj" + coff: { object: { machine: x64, sections: { data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "foo" } } }, symbols: { foo: { kind: external, name: "foo", section: data, value: 0 } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { foo: { type: Addr32Nb, offset: 0, symbol: foo } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + foo: { kind: undefined, name: "foo" } + } + } } + } + } + build: + { + // basic alternate name test + link: { args: "/subsystem:console /entry:entry /out:alternate_basic.exe /alternatename:foo=test test.obj entry.obj" } + // linker should not chase alt name links + link: { args: "/subsystem:console /entry:entry /out:alternate_chain.exe /alternatename:foo=bar /alternatename:bar=test test.obj entry.obj", expect_exit: nonzero } + // alt name conflict + link: { args: "/subsystem:console /entry:entry /out:alternate_conflict.exe /alternatename:foo=test /alternatename:foo=qwe test.obj entry.obj", expect_exit: nonzero } + // syntax error + link: { args: "/subsystem:console /entry:entry /out:alternate_missing_equal.exe /alternatename:foo foo.obj entry.obj", expect_exit: nonzero } + // syntax error + link: { args: "/subsystem:console /entry:entry /out:alternate_wrong_separator.exe /alternatename:foo-oof foo.obj entry.obj", expect_exit: nonzero } + // syntax error + link: { args: "/subsystem:console /entry:entry /out:alternate_extra_equal.exe /alternatename:foo=test=bar foo.obj entry.obj", expect_exit: nonzero } + // syntax error + link: { args: "/subsystem:console /entry:entry /out:alternate_empty_target.exe /alternatename:foo= foo.obj entry.obj", expect_exit: nonzero } + // syntax error + link: { args: "/subsystem:console /entry:entry /out:alternate_empty_source.exe /alternatename:= foo.obj entry.obj", expect_exit: nonzero } + // syntax error + link: { args: "/subsystem:console /entry:entry /out:alternate_empty_option.exe /alternatename: foo.obj entry.obj", expect_exit: nonzero } + // TODO: check that RAD Linker prints these warnings + // warn about alt name to self alt name? + link: { args: "/subsystem:console /entry:entry /out:alternate_self.exe /alternatename:foo=foo foo.obj entry.obj" } + // warn about alt name to unknown symbol? + link: { args: "/subsystem:console /entry:entry /out:alternate_unknown.exe /alternatename:qwe=ewq foo.obj entry.obj" } + } + steps: {} +} diff --git a/src/linker/tests/base_relocs.tst b/src/linker/tests/base_relocs.tst new file mode 100644 index 000000000..fe2db7471 --- /dev/null +++ b/src/linker/tests/base_relocs.tst @@ -0,0 +1,81 @@ +test: +{ + artifacts: + { + // main.obj + main_obj: + { + file_name: "main.obj" + coff: { object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: + { + concat: + { + hex: "48b80000000000000000" // mov rax, func_name + hex: "ffd0" // call rax + hex: "4831c0" // xor rax, rax + hex: "b800000000" // mov eax, func_name + hex: "ffd0" // call rax + hex: "c3" // ret + } + } + relocations: + { + foo_64: { type: Addr64, offset: 2, symbol: foo } + foo_32: { type: Addr32, offset: 16, symbol: foo } + } + } + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { zero: 4 } + relocations: { absolute: { type: Addr32, offset: 0, symbol: abs } } + } + } + symbols: + { + foo: { kind: undefined, name: "foo" } + abs: { kind: absolute, name: "abs", value: 305419896, storage: static } + entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } } + } + + // func.obj + func_obj: + { + file_name: "func.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { foo: { kind: external, name: "foo", section: text, value: 0 } } + } } + } + } + + build: + { + // linker must not produce base relocations for absolute symbol + link: { args: "/subsystem:console /entry:my_entry /dynamicbase /largeaddressaware:no /out:a.exe main.obj func.obj", output: none } + + // it is illegal to merge .reloc with other sections + link: { args: "/subsystem:console /entry:my_entry /dynamicbase /largeaddressaware:no /out:a.exe /merge:.reloc=.rdata main.obj func.obj", output: none, expect_exit: 26 } + + // the other way around is illegal too + link: { args: "/subsystem:console /entry:my_entry /dynamicbase /largeaddressaware:no /out:a.exe /merge:.rdata=.reloc main.obj func.obj", output: none, expect_exit: 26 } + } + steps: {} +} diff --git a/src/linker/tests/comdat_any.tst b/src/linker/tests/comdat_any.tst new file mode 100644 index 000000000..add120426 --- /dev/null +++ b/src/linker/tests/comdat_any.tst @@ -0,0 +1,77 @@ +test: +{ + artifacts: + { + one_obj: + { + file_name: "1.obj" + coff: { object: + { + machine: x64 + sections: { test: { name: ".test$mn", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "1" } } } + symbols: + { + test_def: { kind: section_definition, section: test, selection: Any } + test: { kind: external_function, name: "TEST", section: test, value: 0 } + } + } } + } + two_obj: + { + file_name: "2.obj" + coff: { object: + { + machine: x64 + sections: { test: { name: ".test$mn", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "2" } } } + symbols: + { + test_def: { kind: section_definition, section: test, selection: Any } + test: { kind: external, name: "TEST", section: test, value: 0 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: + { + concat: + { + hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } + } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + one_image: { file_name: "1.exe", pe: {} } + two_image: { file_name: "2.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:1.exe 1.obj 2.obj entry.obj", artifact: one_image } + link: { args: "/subsystem:console /entry:entry /out:2.exe 2.obj 1.obj entry.obj", artifact: two_image } + } + steps: + { + expect_pe: { artifact: one_image, expected: { pe: { sections: { ".test": { data: 31 } } } } } + expect_pe: { artifact: two_image, expected: { pe: { sections: { ".test": { data: 32 } } } } } + } +} diff --git a/src/linker/tests/comdat_associative.tst b/src/linker/tests/comdat_associative.tst new file mode 100644 index 000000000..f3e936f6b --- /dev/null +++ b/src/linker/tests/comdat_associative.tst @@ -0,0 +1,79 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + a: { name: "a", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "a" } } + aa: { name: "aa", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "aa" } } + } + symbols: + { + a_def: { kind: section_definition, section: a, selection: Largest } + test: { kind: external, name: "TEST", section: a, value: 0 } + aa_def: { kind: section_definition, section: aa, selection: Associative, associate: a } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: + { + bb: { name: "bb", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "bb" } } + b: { name: "b", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "b" } } + bbb: { name: "bbb", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "bbb" } } + } + symbols: + { + bb_def: { kind: section_definition, section: bb, selection: Largest } + b_def: { kind: section_definition, section: b, selection: Associative, associate: bb } + bbb_def: { kind: section_definition, section: bbb, selection: Associative, associate: bb } + test: { kind: external, name: "TEST", section: bb, value: 0 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj b.obj", artifact: image } } + steps: + { + expect_pe: + { + artifact: image + expected: { pe: { sections: { @absent "a", @absent "aa", "b": { data: 62 }, "bb": { data: 6262 }, "bbb": { data: 626262 } } } } + } + } +} diff --git a/src/linker/tests/comdat_associative_loop.tst b/src/linker/tests/comdat_associative_loop.tst new file mode 100644 index 000000000..56b72b756 --- /dev/null +++ b/src/linker/tests/comdat_associative_loop.tst @@ -0,0 +1,53 @@ +test: +{ + artifacts: + { + loop_obj: + { + file_name: "loop.obj" + coff: { object: + { + machine: x64 + sections: + { + aaaa: { name: ".aaaa", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "aaaa" } } + aa: { name: ".aa", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "aa" } } + a: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "a" } } + aaa: { name: ".aaa", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "aaa" } } + } + symbols: + { + aaa_def: { kind: section_definition, section: aaa, selection: Associative, associate: aa } + aaaa_def: { kind: section_definition, section: aaaa, selection: Associative, associate: aaa } + a_def: { kind: section_definition, section: a, selection: Associative, associate: aa } + aa_def: { kind: section_definition, section: aa, selection: Associative, associate: a } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe loop.obj entry.obj", expect_exit: 29 } } + steps: {} +} diff --git a/src/linker/tests/comdat_associative_non_comdat.tst b/src/linker/tests/comdat_associative_non_comdat.tst new file mode 100644 index 000000000..f28612f9e --- /dev/null +++ b/src/linker/tests/comdat_associative_non_comdat.tst @@ -0,0 +1,53 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: + { + machine: x64 + sections: + { + a: { name: ".a", permissions: (read, write), content: initialized_data, data: { text: "a" } } + b: { name: ".b", permissions: (read, write), content: initialized_data, data: { text: "b" } } + } + symbols: + { + test: { kind: external, name: "TEST", section: a, value: 0 } + b_def: { kind: section_definition, section: b, selection: Associative, associate: a } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj test.obj", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".a": { data: 61 }, ".b": { data: 62 } } } } } + } +} diff --git a/src/linker/tests/comdat_associative_out_of_bounds.tst b/src/linker/tests/comdat_associative_out_of_bounds.tst new file mode 100644 index 000000000..b475081fc --- /dev/null +++ b/src/linker/tests/comdat_associative_out_of_bounds.tst @@ -0,0 +1,51 @@ +test: +{ + artifacts: + { + bad_obj: + { + file_name: "bad.obj" + bytes: + { + data: { concat: + { + hex: "6486020000000000670000000500000000000000" + hex: "2e6100000000000000000000000000000100000064000000000000000000000000000000401000c0" + hex: "2e6161000000000000000000000000000200000065000000000000000000000000000000401000c0" + text: "aaa" + hex: "2e6100000000000000000000010000000301" + hex: "010000000000000000000000000002000000" + hex: "544553540000000000000000010000000200" + hex: "2e6161000000000000000000020000000301" + hex: "020000000000000000000000410105000000" + hex: "04000000" + } } + } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj bad.obj", expect_exit: 7 } } + steps: {} +} diff --git a/src/linker/tests/comdat_exact_match.tst b/src/linker/tests/comdat_exact_match.tst new file mode 100644 index 000000000..f105b1984 --- /dev/null +++ b/src/linker/tests/comdat_exact_match.tst @@ -0,0 +1,78 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: { a: { name: ".a", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "a" } } } + symbols: + { + a_def: { kind: section_definition, section: a, selection: ExactMatch } + test: { kind: external, name: "TEST", section: a, value: 0 } + } + } } + } + a2_obj: + { + file_name: "a2.obj" + coff: { object: + { + machine: x64 + sections: { a2: { name: ".a2", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "a" } } } + symbols: + { + a2_def: { kind: section_definition, section: a2, selection: ExactMatch } + test: { kind: external, name: "TEST", section: a2, value: 0 } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: { b: { name: ".b", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "b" } } } + symbols: + { + b_def: { kind: section_definition, section: b, selection: ExactMatch } + test: { kind: external, name: "TEST", section: b, value: 0 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + image: { file_name: "b.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj b.obj", expect_exit: nonzero } + link: { args: "/subsystem:console /entry:entry /out:b.exe entry.obj a2.obj a.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".a2": { data: 61 } } } } } } +} diff --git a/src/linker/tests/comdat_external_symbol_at_nonzero_offset.tst b/src/linker/tests/comdat_external_symbol_at_nonzero_offset.tst new file mode 100644 index 000000000..0f4ea3a6b --- /dev/null +++ b/src/linker/tests/comdat_external_symbol_at_nonzero_offset.tst @@ -0,0 +1,68 @@ +// MSVC vftables use COMDAT sections whose public symbol can start past the +// section definition symbol; references to a replaced copy must target the winner. +test: +{ + artifacts: + { + leader_obj: + { + file_name: "leader.obj" + coff: { object: + { + machine: x64 + sections: { data: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } } + symbols: + { + data_def: { kind: section_definition, section: data, selection: Any } + foo: { kind: external, name: "foo", section: data, value: 8 } + } + } } + } + ref_obj: + { + file_name: "ref.obj" + coff: { object: + { + machine: x64 + sections: + { + data: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } + ptr: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 8 + data: { zero: 8 } + relocations: { foo_ref: { type: Addr64, offset: 0, symbol: foo } } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { concat: { hex: "488d0500000000" // lea rax, [rip + foo] + hex: "c3" + } } + relocations: { foo_ref: { type: Rel32, offset: 3, symbol: foo } } + } + } + symbols: + { + data_def: { kind: section_definition, section: data, selection: Any } + foo: { kind: external, name: "foo", section: data, value: 8 } + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe leader.obj ref.obj", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".rdata": {}, ".data": {}, ".text": {} } } } } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, target_section: ".rdata", target_offset: 8 } + expect_pe_word: { artifact: image, section: ".text", offset: 3, type: rel32, target_section: ".rdata", target_offset: 8 } + } +} diff --git a/src/linker/tests/comdat_external_symbol_at_zero_offset.tst b/src/linker/tests/comdat_external_symbol_at_zero_offset.tst new file mode 100644 index 000000000..4aba77468 --- /dev/null +++ b/src/linker/tests/comdat_external_symbol_at_zero_offset.tst @@ -0,0 +1,66 @@ +test: +{ + artifacts: + { + leader_obj: + { + file_name: "leader.obj" + coff: { object: + { + machine: x64 + sections: { data: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 8 } } } + symbols: + { + data_def: { kind: section_definition, section: data, selection: Any } + foo: { kind: external, name: "foo", section: data, value: 0 } + } + } } + } + ref_obj: + { + file_name: "ref.obj" + coff: { object: + { + machine: x64 + sections: + { + data: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 8 } } + ptr: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 8 + data: { zero: 8 } + relocations: { foo_ref: { type: Addr64, offset: 0, symbol: foo } } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { concat: { hex: "488d0500000000" // lea rax, [rip + foo] + hex: "c3" + } } + relocations: { foo_ref: { type: Rel32, offset: 3, symbol: foo } } + } + } + symbols: + { + data_def: { kind: section_definition, section: data, selection: Any } + foo: { kind: external, name: "foo", section: data, value: 0 } + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe leader.obj ref.obj", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".rdata": {}, ".data": {}, ".text": {} } } } } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, target_section: ".rdata", target_offset: 0 } + expect_pe_word: { artifact: image, section: ".text", offset: 3, type: rel32, target_section: ".rdata", target_offset: 0 } + } +} diff --git a/src/linker/tests/comdat_external_symbol_uses_leader_offset.tst b/src/linker/tests/comdat_external_symbol_uses_leader_offset.tst new file mode 100644 index 000000000..0e745e68b --- /dev/null +++ b/src/linker/tests/comdat_external_symbol_uses_leader_offset.tst @@ -0,0 +1,73 @@ +// Duplicate COMDAT sections can have identical bytes while their symbol tables +// disagree about where a same-named public symbol points inside the section. +// This mirrors MSVC vftable COMDATs: the selected copy may have leading RTTI data +// at offset 0 and the vftable symbol at offset 8, while a discarded copy's +// vftable symbol is at offset 0. Relocations against the discarded symbol must +// use the selected symbol's value, not just the selected section contribution +// plus the discarded symbol's original offset. +test: +{ + artifacts: + { + leader_obj: + { + file_name: "leader.obj" + coff: { object: + { + machine: x64 + sections: { vftable: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } } + symbols: + { + vftable_def: { kind: section_definition, section: vftable, selection: Any } + vftable: { kind: external, name: "??_7X@@6B@", section: vftable, value: 8 } + } + } } + } + ref_obj: + { + file_name: "ref.obj" + coff: { object: + { + machine: x64 + sections: + { + vftable: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } + ptr: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 8 + data: { zero: 8 } + relocations: { vftable_ref: { type: Addr64, offset: 0, symbol: vftable } } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { concat: { hex: "488d0500000000" // lea rax, [rip + ??_7X] + hex: "c3" + } } + relocations: { vftable_ref: { type: Rel32, offset: 3, symbol: vftable } } + } + } + symbols: + { + vftable_def: { kind: section_definition, section: vftable, selection: Any } + vftable: { kind: external, name: "??_7X@@6B@", section: vftable, value: 0 } + entry: { kind: external_function, name: "entry", section: text, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe leader.obj ref.obj", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".rdata": {}, ".data": {}, ".text": {} } } } } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, target_section: ".rdata", target_offset: 8 } + expect_pe_word: { artifact: image, section: ".text", offset: 3, type: rel32, target_section: ".rdata", target_offset: 8 } + } +} diff --git a/src/linker/tests/comdat_largest.tst b/src/linker/tests/comdat_largest.tst new file mode 100644 index 000000000..d6fca51d8 --- /dev/null +++ b/src/linker/tests/comdat_largest.tst @@ -0,0 +1,83 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: { a: { name: ".a", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "a" } } } + symbols: + { + a_def: { kind: section_definition, section: a, selection: Largest } + test: { kind: external, name: "TEST", section: a, value: 0 } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: { b: { name: ".b", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "bb" } } } + symbols: + { + b_def: { kind: section_definition, section: b, selection: Largest } + test: { kind: external, name: "TEST", section: b, value: 0 } + } + } } + } + c_obj: + { + file_name: "c.obj" + coff: { object: + { + machine: x64 + sections: { c: { name: ".c", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "c" } } } + symbols: + { + c_def: { kind: section_definition, section: c, selection: Largest } + test: { kind: external, name: "TEST", section: c, value: 0 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + a_image: { file_name: "a.exe", pe: {} } + b_image: { file_name: "b.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /out:a.exe /entry:entry entry.obj a.obj b.obj", artifact: a_image } + link: { args: "/subsystem:console /out:b.exe /entry:entry entry.obj c.obj a.obj", artifact: b_image } + } + steps: + { + expect_pe: { artifact: a_image, expected: { pe: { sections: { @absent ".a", ".b": { data: 6262 } } } } } + expect_pe: { artifact: b_image, expected: { pe: { sections: { ".c": { data: 63 } } } } } + } +} diff --git a/src/linker/tests/comdat_largest_external_symbol_uses_selected_offset.tst b/src/linker/tests/comdat_largest_external_symbol_uses_selected_offset.tst new file mode 100644 index 000000000..c5494f2a2 --- /dev/null +++ b/src/linker/tests/comdat_largest_external_symbol_uses_selected_offset.tst @@ -0,0 +1,92 @@ +// Chromium has duplicate vftable COMDATs where the referencing copy is an +// IMAGE_COMDAT_SELECT_ANY section with the public vftable at offset 0, while +// the selected IMAGE_COMDAT_SELECT_LARGEST copy has the same public symbol at +// offset 8. Relocations in the discarded object must resolve to the selected +// public symbol, not to the discarded section. +test: +{ + artifacts: + { + discarded_obj: + { + file_name: "discarded.obj" + coff: { object: + { + machine: x64 + sections: + { + vftable: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } + ptr: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 8 + data: { zero: 16 } + relocations: + { + vftable_ref: { type: Addr64, offset: 0, symbol: vftable } + force_selected_ref: { type: Addr64, offset: 8, symbol: force_selected } + } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { concat: { hex: "488d0500000000" // lea rax, [rip + ??_7X] + hex: "c3" + } } + relocations: { vftable_ref: { type: Rel32, offset: 3, symbol: vftable } } + } + } + symbols: + { + vftable_def: { kind: section_definition, section: vftable, selection: Any } + vftable: { kind: external, name: "??_7X@@6B@", section: vftable, value: 0 } + force_selected: { kind: undefined, name: "force_selected" } + entry: { kind: external_function, name: "entry", section: text, value: 0 } + } + } } + } + selected_lib: + { + file_name: "selected.lib" + coff: { library: + { + second_linker_member: true + members: + { + selected: + { + path: "selected.obj" + object: + { + machine: x64 + sections: + { + vftable: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 24 } } + force_text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + vftable_def: { kind: section_definition, section: vftable, selection: Largest } + vftable: { kind: external, name: "??_7X@@6B@", section: vftable, value: 8 } + force_selected: { kind: external_function, name: "force_selected", section: force_text, value: 0 } + } + } + } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe /opt:ref,noicf discarded.obj selected.lib", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".rdata": {}, ".data": {}, ".text": {} } } } } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, target_section: ".rdata", target_offset: 8 } + expect_pe_word: { artifact: image, section: ".text", offset: 3, type: rel32, target_section: ".rdata", target_offset: 8 } + } +} diff --git a/src/linker/tests/comdat_no_duplicates.tst b/src/linker/tests/comdat_no_duplicates.tst new file mode 100644 index 000000000..6032e4b85 --- /dev/null +++ b/src/linker/tests/comdat_no_duplicates.tst @@ -0,0 +1,72 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: + { + concat: + { + hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } + } + relocations: { a_ref: { type: Addr32Nb, offset: 0, symbol: a } } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + a: { kind: undefined, name: "a" } + } + } } + } + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: { test: { name: ".test", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "a" } } } + symbols: + { + test_def: { kind: section_definition, section: test, selection: NoDuplicates } + a: { kind: external, name: "a", section: test, value: 0 } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: { test: { name: ".test", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "a" } } } + symbols: + { + test_def: { kind: section_definition, section: test, selection: NoDuplicates } + a: { kind: external, name: "a", section: test, value: 0 } + } + } } + } + image: { file_name: "b.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj", expect_exit: 24 } + link: { args: "/subsystem:console /entry:entry /out:b.exe a.obj entry.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".test": { data: 61 } } } } } } +} diff --git a/src/linker/tests/comdat_same_size.tst b/src/linker/tests/comdat_same_size.tst new file mode 100644 index 000000000..bf4d5244a --- /dev/null +++ b/src/linker/tests/comdat_same_size.tst @@ -0,0 +1,78 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: { a: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "a" } } } + symbols: + { + a_def: { kind: section_definition, section: a, selection: SameSize } + test: { kind: external, name: "TEST", section: a, value: 0 } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: { b: { name: ".b", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "b" } } } + symbols: + { + b_def: { kind: section_definition, section: b, selection: SameSize } + test: { kind: external, name: "TEST", section: b, value: 0 } + } + } } + } + c_obj: + { + file_name: "c.obj" + coff: { object: + { + machine: x64 + sections: { c: { name: ".c", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "cc" } } } + symbols: + { + c_def: { kind: section_definition, section: c, selection: SameSize } + test: { kind: external, name: "TEST", section: c, value: 0 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj", artifact: image } + link: { args: "/subsystem:console /entry:entry /out:b.exe a.obj b.obj c.obj entry.obj", expect_exit: 24 } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".a": { data: 61 } } } } } } +} diff --git a/src/linker/tests/comdat_with_offset.tst b/src/linker/tests/comdat_with_offset.tst new file mode 100644 index 000000000..271af2494 --- /dev/null +++ b/src/linker/tests/comdat_with_offset.tst @@ -0,0 +1,59 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: { rdata: { name: ".rdata", permissions: (read), content: initialized_data, flags: (link_comdat), data: { concat: { text: "1Hello, World!", zero: 1 } } } } + symbols: + { + rdata_def: { kind: section_definition, section: rdata, selection: Largest } + test: { kind: external, name: "TEST", section: rdata, value: 1 } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: { rdata: { name: ".rdata", permissions: (read), content: initialized_data, flags: (link_comdat), data: { concat: { text: "Hello, World!", zero: 1 } } } } + symbols: + { + rdata_def: { kind: section_definition, section: rdata, selection: Largest } + test: { kind: external, name: "TEST", section: rdata, value: 1 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 3, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj" } } + steps: {} +} diff --git a/src/linker/tests/common_block.tst b/src/linker/tests/common_block.tst new file mode 100644 index 000000000..1d4bc93b6 --- /dev/null +++ b/src/linker/tests/common_block.tst @@ -0,0 +1,119 @@ +test: +{ + + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: + { + object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { zero: 6 } + relocations: { a_ref: { type: Addr32, offset: 0, symbol: a } } + } + // shift common block's initial position + bss: + { + name: ".bss" + permissions: (read, write) + content: uninitialized_data + data: { zero: 1 } + } + } + symbols: { a: { kind: common, name: "A", size: 3 } } + } + } + } + b_obj: + { + file_name: "b.obj" + coff: + { + object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { zero: 9 } + relocations: { b_ref: { type: Addr64, offset: 0, symbol: b } } + } + } + symbols: { b: { kind: common, name: "B", size: 6 } } + } + } + } + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "c3" } + } + } + symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } + } + } + image: { file_name: "common_block.exe", pe: {} } + } + + build: + { + link: + { + args: "/subsystem:console /entry:my_entry /out:common_block.exe /fixed /largeaddressaware:no /merge:.bss=.comm a.obj b.obj entry.obj" + artifact: image + } + } + + steps: + { + expect_pe: + { + artifact: image + expected: + { + pe: + { + sections: + { + // blocks must be sorted in descending order to reduce alignment padding + ".comm": { virtual_size: 19 } + ".data": {} + } + } + } + } + // ensure linker correctly patched addresses for symbols pointing into common block + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u32, target_section: ".comm", target_offset: 16, target_address: va } + expect_pe_word: { artifact: image, section: ".data", offset: 6, type: u64, target_section: ".comm", target_offset: 8, target_address: va } + } +} diff --git a/src/linker/tests/communal_var_vs_regular.tst b/src/linker/tests/communal_var_vs_regular.tst new file mode 100644 index 000000000..359cd05ab --- /dev/null +++ b/src/linker/tests/communal_var_vs_regular.tst @@ -0,0 +1,42 @@ +test: +{ + artifacts: + { + communal_obj: { file_name: "communal.obj", coff: { object: { machine: x64, symbols: { test: { kind: common, name: "TEST", size: 1 } } } } } + defn_obj: + { + file_name: "defn.obj" + coff: { object: { machine: x64, + sections: { data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "test" } } } + symbols: { test: { kind: external, name: "TEST", section: data, value: 0 } } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + data: { hex: "48c7c000000000c3" } + // ret + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } } } + } + a_image: { file_name: "a.exe", pe: {} } + b_image: { file_name: "b.exe", pe: {} } + } + build: + { + // linker should replace communal TEST with .data TEST + link: { args: "/subsystem:console /entry:entry /out:a.exe communal.obj defn.obj entry.obj", artifact: a_image } + link: { args: "/subsystem:console /entry:entry /out:b.exe defn.obj communal.obj entry.obj", artifact: b_image } + } + steps: + { + expect_pe: { artifact: a_image, expected: { pe: { sections: { ".data": { data: 74657374 } } } } } + expect_pe: { artifact: b_image, expected: { pe: { sections: { ".data": { data: 74657374 } } } } } + } +} diff --git a/src/linker/tests/communal_var_vs_regular_comdat.tst b/src/linker/tests/communal_var_vs_regular_comdat.tst new file mode 100644 index 000000000..db26283db --- /dev/null +++ b/src/linker/tests/communal_var_vs_regular_comdat.tst @@ -0,0 +1,45 @@ +test: +{ + artifacts: + { + communal_obj: { file_name: "communal.obj", coff: { object: { machine: x64, symbols: { test: { kind: common, name: "TEST", size: 1 } } } } } + large_obj: + { + file_name: "large.obj" + coff: { object: { machine: x64, + sections: { data: { name: ".data", permissions: (read, write), content: initialized_data, flags: (link_comdat), data: { text: "test" } } } + symbols: { + data_def: { kind: section_definition, section: data, selection: Largest } + test: { kind: external, name: "TEST", section: data, value: 0 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + data: { hex: "48c7c000000000c3" } + // ret + relocations: { test_ref: { type: Addr32Nb, offset: 0, symbol: test } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } } } + } + a_image: { file_name: "a.exe", pe: {} } + b_image: { file_name: "b.exe", pe: {} } + } + build: + { + // linker should replace communal TEST with .data TEST + link: { args: "/subsystem:console /entry:entry /out:a.exe communal.obj large.obj entry.obj", artifact: a_image } + link: { args: "/subsystem:console /entry:entry /out:b.exe large.obj communal.obj entry.obj", artifact: b_image } + } + steps: + { + expect_pe: { artifact: a_image, expected: { pe: { sections: { ".data": { data: 74657374 } } } } } + expect_pe: { artifact: b_image, expected: { pe: { sections: { ".data": { data: 74657374 } } } } } + } +} diff --git a/src/linker/tests/cyclic_type.tst b/src/linker/tests/cyclic_type.tst new file mode 100644 index 000000000..cd5eddafe --- /dev/null +++ b/src/linker/tests/cyclic_type.tst @@ -0,0 +1,56 @@ +test: +{ + artifacts: + { + cycle_obj: + { + file_name: "cycle.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { hex: "040000000a00021001100000000000000a0002100010000000000000" } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { hex: "04000000f10000002b000000290010110000000000000000000000000000000000000000000000000110000000000000000000666f6f0000" } + } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:full /rad_ignore:-43 cycle.obj entry.obj" + output: none + stderr_matches: "*Error(*): *: LF_POINTER(type_index: *) forward refs member type index * (leaf struct offset: *)*" + } + } + steps: {} +} diff --git a/src/linker/tests/debug_p_and_debug_t_in_obj.tst b/src/linker/tests/debug_p_and_debug_t_in_obj.tst new file mode 100644 index 000000000..c8976d173 --- /dev/null +++ b/src/linker/tests/debug_p_and_debug_t_in_obj.tst @@ -0,0 +1,81 @@ +test: +{ + artifacts: + { + pch_obj: + { + file_name: "pch.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_p: + { + name: ".debug$P" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + // signature + hex: "040000000a0002100300000000000000" + // PCH ender + hex: "06001400bebafeca" + } } + } + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + hex: "04000000160009150010000001000000bebafeca" + text: "pch.obj" + hex: "000a00021003000000000000000e000810001000000000000000000000" + } } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + hex: "04000000f1000000100000000e000111bebafeca" + text: "pch.obj" + hex: "00" + } } + } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:full pch.obj entry.obj" + output: none + stderr_matches: "*Warning(067): *pch.obj: multiple sections with debug types detected, obj must have either .debug$T or .debug$P; discarding both sections*" + } + } + steps: {} +} diff --git a/src/linker/tests/debug_p_sig_mismatch.tst b/src/linker/tests/debug_p_sig_mismatch.tst new file mode 100644 index 000000000..7152a7998 --- /dev/null +++ b/src/linker/tests/debug_p_sig_mismatch.tst @@ -0,0 +1,126 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + // ver_fe_major + // ver_fe_minor + // ver_fe_build + // ver_feqfe + // ver_major + // ver_minor + // ver_build + // ver_qfe + debug_p: + { + name: ".debug$P" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + // signature + hex: "04000000" + // duplicate in a.obj + hex: "0a0002100300000000000000" + // unique procedure type + hex: "0e000810001000000200000000000000" + // PCH ender + hex: "06001400bebafeca" + } } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + hex: "04000000f1000000300000000c000111bebafeca" + text: "a.obj" + hex: "0020003c1100000000d00000000000000000000e002400197f0000544f525455524500" + } } + } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: + { + // ver_fe_major + // ver_fe_minor + // ver_fe_build + // ver_feqfe + // ver_major + // ver_minor + // ver_build + // ver_qfe + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + hex: "04000000140009150010000002000000efbeadde" + text: "a.obj" + hex: "000a00021003000000000000000e000810001000000000000000000000" + } } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + flags: (discardable) + data: { concat: + { + hex: "04000000f1000000300000000c000111efbeadde" + text: "a.obj" + hex: "0020003c1100000000d00000000000000000000e002400197f0000544f525455524500" + } } + } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:full a.obj b.obj entry.obj" + output: none + stderr_matches: "*Error(048): *b.obj: PCH signature mismatch, expected 0xdeadbeef got 0xcafebabe; PCH obj *a.obj*" + } + } + steps: {} +} diff --git a/src/linker/tests/def_file_full.tst b/src/linker/tests/def_file_full.tst new file mode 100644 index 000000000..6aba9d648 --- /dev/null +++ b/src/linker/tests/def_file_full.tst @@ -0,0 +1,236 @@ +test: +{ + + artifacts: + { + def_full_obj: + { + file_name: "def_full.obj" + coff: + { + object: + { + machine: x64 + sections: + { + data: + { + name: ".rdata" + permissions: (read, write) + content: initialized_data + data: { text: "test" } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "c3" } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + foo: { kind: external, name: "foo", section: data, value: 0 } + } + } + } + } + + full_def: + { + file_name: "full.def" + text: { data: { concat: + { + text: "; leading comment" + hex: "0a" + text: "NAME " + hex: "22" + text: "def full.exe" + hex: "22" + text: " BASE=0x140020000" + hex: "0a" + text: "VERSION 7.8" + hex: "0a" + text: "HEAPSIZE 0x30000, 0x4000" + hex: "0a" + text: "STACKSIZE 0x50000,0x6000" + hex: "0a" + text: "SECTIONS .rdata READ" + hex: "0a" + text: "EXPORTS foo @ 2 DATA" + hex: "0a" + } } } + } + bad_base_space_def: + { + file_name: "bad_base_space.def" + text: { data: { concat: + { + text: "NAME bad_base_space BASE = 0x140020000" + hex: "0a" + text: "EXPORTS foo @2 DATA" + hex: "0a" + } } } + } + bad_base_colon_def: + { + file_name: "bad_base_colon.def" + text: { data: { concat: + { + text: "NAME bad_base_colon BASE:0x140020000" + hex: "0a" + text: "EXPORTS foo @2 DATA" + hex: "0a" + } } } + } + bad_section_align_def: + { + file_name: "bad_section_align.def" + text: { data: { concat: + { + text: "NAME bad_section_align" + hex: "0a" + text: "SECTIONS .rdata READ ALIGN=8192" + hex: "0a" + text: "EXPORTS foo @2 DATA" + hex: "0a" + } } } + } + exe_image: { file_name: "def full.exe", pe: {} } + + def_full_dll_obj: + { + file_name: "def_full_dll.obj" + coff: + { + object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { text: "test" } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "c3" } + } + } + symbols: + { + dll_entry: { kind: external, name: "_DllMainCRTStartup", section: text, value: 0 } + dll_foo: { kind: external, name: "dll_foo", section: data, value: 0 } + } + } + } + } + full_dll_def: + { + file_name: "full_dll.def" + text: { data: { concat: + { + text: "LIBRARY folded BASE=0x180020000" + hex: "0a" + text: "EXPORTS" + hex: "0a" + text: " dll_foo DATA" + hex: "0a" + } } } + } + dll_image: { file_name: "folded.dll", pe: {} } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /def:full.def def_full.obj" + artifact: exe_image + } + + link: + { + args: "/subsystem:console /entry:entry /def:bad_base_space.def /out:bad_base_space.exe def_full.obj" + expect_exit: nonzero + } + link: + { + args: "/subsystem:console /entry:entry /def:bad_base_colon.def /out:bad_base_colon.exe def_full.obj" + expect_exit: nonzero + } + link: + { + args: "/subsystem:console /entry:entry /def:bad_section_align.def /out:bad_section_align.exe def_full.obj" + expect_exit: nonzero + } + + link: + { + args: "/dll /subsystem:console /def:full_dll.def def_full_dll.obj" + artifact: dll_image + } + } + + steps: + { + expect_pe: + { + artifact: exe_image + expected: + { + pe: + { + optional: + { + image_base: 5368840192 + major_image_version: 7 + minor_image_version: 8 + sizeof_heap_reserve: 196608 + sizeof_heap_commit: 16384 + sizeof_stack_reserve: 327680 + sizeof_stack_commit: 24576 + } + sections: + { + ".rdata": { raw_flags: 1073741888 } + } + exports: + { + count: 1 + entries: + { + export_0: { name: "foo", ordinal: 2 } + } + } + } + } + } + expect_pe: + { + artifact: dll_image + expected: + { + pe: + { + optional: { image_base: 6442582016 } + exports: + { + count: 1 + entries: + { + export_0: { name: "dll_foo" } + } + } + } + } + } + } +} diff --git a/src/linker/tests/defer_duplicate_imp_link.tst b/src/linker/tests/defer_duplicate_imp_link.tst new file mode 100644 index 000000000..93f5c40ad --- /dev/null +++ b/src/linker/tests/defer_duplicate_imp_link.tst @@ -0,0 +1,123 @@ +test: +{ + + artifacts: + { + bar_lib: + { + file_name: "defer_duplicate_bar.lib" + coff: { library: + { + members: + { + bar_scaffold: { dll_import: { name: "bar.dll", machine: x64, timestamp: 0 } } + bar_import: { import: { dll: "bar.dll", name: "bar", machine: x64, timestamp: 0, type: code, lookup: name, hint: 0 } } + qwe_member: + { + path: "member_2.obj" + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "ff2500000000" } } + } + symbols: + { + imp_bar: { kind: undefined, name: "__imp_bar" } + qwe: { kind: external_function, name: "qwe", section: text, value: 0 } + } + } + } + } + } } + } + foo_lib: + { + file_name: "defer_duplicate_foo.lib" + coff: { library: + { + members: + { + foo_scaffold: { dll_import: { name: "foo.dll", machine: x64, timestamp: 0 } } + bar_import: { import: { dll: "foo.dll", name: "bar", machine: x64, timestamp: 0, type: code, lookup: name, hint: 0 } } + thunk_member: + { + path: "member_2.obj" + object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "ff2500000000" } + relocations: { bar_ref: { type: Rel32, offset: 2, symbol: bar } } + } + } + symbols: + { + bar: { kind: undefined, name: "bar" } + qwe: { kind: undefined, name: "qwe" } + thunk: { kind: external_function, name: "thunk", section: text, value: 0 } + } + } + } + } + } } + } + entry_obj: + { + file_name: "defer_duplicate_entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + image: { file_name: "defer_duplicate_imp_link.exe", pe: {} } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:defer_duplicate_imp_link.exe defer_duplicate_bar.lib defer_duplicate_foo.lib defer_duplicate_entry.obj /include:thunk" + artifact: image + } + } + + steps: + { + // Exactly one import descriptor remains, and it names foo.dll. + expect_pe_word: { artifact: image, section: ".data", offset: 16, type: u32, target_section: ".data", target_offset: 56, target_address: rva } + expect_pe_word: { artifact: image, section: ".data", offset: 28, type: u32, target_section: ".data", target_offset: 78, target_address: rva } + expect_pe_word: { artifact: image, section: ".data", offset: 32, type: u32, target_section: ".data", target_offset: 0, target_address: rva } + expect_pe_bytes: { artifact: image, section: ".data", offset: 36, hex: "0000000000000000000000000000000000000000" } + + // The sole name import is bar with hint zero. Its nonzero IAT and ILT words + // are equal and point inside .data at the same hint/name record. + expect_pe_word: + { + artifact: image + section: ".data" + offset: 0 + type: u64 + nonzero: true + target_section: ".data" + target_offset: 72 + target_address: rva + other_section: ".data" + other_offset: 56 + other_type: u64 + relation: equal + } + expect_pe_bytes: { artifact: image, section: ".data", offset: 8, hex: "0000000000000000" } + expect_pe_bytes: { artifact: image, section: ".data", offset: 64, hex: "0000000000000000" } + expect_pe_bytes: { artifact: image, section: ".data", offset: 72, hex: "000062617200666f6f2e646c6c00" } + } +} diff --git a/src/linker/tests/defer_imp_link.tst b/src/linker/tests/defer_imp_link.tst new file mode 100644 index 000000000..6efa25ae6 --- /dev/null +++ b/src/linker/tests/defer_imp_link.tst @@ -0,0 +1,55 @@ +test: +{ + artifacts: + { + bar_lib: + { + file_name: "bar.lib" + coff: { library: { members: { + import_descriptor: { dll_import: { name: "bar.dll", machine: x64, timestamp: 0 } } + import_symbol: { import: { dll: "bar.dll", name: "bar", machine: x64, timestamp: 0, type: code, lookup: name, hint: 0 } } + implementation: + { + path: "member_2.obj" + object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "ff2500000000" } } } + symbols: { + imp_bar: { kind: undefined, name: "__imp_bar" } + qwe: { kind: external_function, name: "qwe", section: text, value: 0 } + } + } + } + } } } + } + foo_lib: + { + file_name: "foo.lib" + coff: { library: { members: { + import_descriptor: { dll_import: { name: "foo.dll", machine: x64, timestamp: 0 } } + import_symbol: { import: { dll: "foo.dll", name: "bar", machine: x64, timestamp: 0, type: code, lookup: name, hint: 0 } } + thunk: + { + path: "member_2.obj" + object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "ff2500000000" } + relocations: { bar_ref: { type: Rel32, offset: 2, symbol: bar } } + } }, symbols: { + bar: { kind: undefined, name: "bar" } + qwe: { kind: undefined, name: "qwe" } + thunk: { kind: external_function, name: "thunk", section: text, value: 0 } + } } + } + } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe bar.lib foo.lib entry.obj /include:thunk" } } + steps: {} +} diff --git a/src/linker/tests/defer_impl_link_to_second_search_pass.tst b/src/linker/tests/defer_impl_link_to_second_search_pass.tst new file mode 100644 index 000000000..2735f83d2 --- /dev/null +++ b/src/linker/tests/defer_impl_link_to_second_search_pass.tst @@ -0,0 +1,48 @@ +test: +{ + artifacts: + { + imp_ref_obj: + { + file_name: "imp_ref.obj" + coff: { object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } } + symbols: { + entry: { kind: external_function, name: "entry", section: text, value: 0 } + imp_foo: { kind: undefined, name: "__imp_foo" } + func: { kind: undefined, name: "func" } + } + } } + } + impl_ref_lib: + { + file_name: "impl_ref.lib" + coff: { library: { second_linker_member: true, members: { implementation: { + path: "impl_ref.obj" + object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } } + symbols: { + func: { kind: external_function, name: "func", section: text, value: 0 } + foo: { kind: undefined, name: "foo" } + } + } + } } } } + } + foo_lib: + { + file_name: "foo.lib" + coff: { library: { second_linker_member: true, members: { imported: { import: { + dll: "foo.dll", name: "foo", machine: x64, timestamp: 4294967295, type: code, lookup: name, hint: 0 + } } } } } + } + foo2_lib: + { + file_name: "foo2.lib" + coff: { library: { second_linker_member: true, members: { imported: { import: { + dll: "foo.dll", name: "foo", machine: x64, timestamp: 4294967295, type: code, lookup: name, hint: 0 + } } } } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe foo.lib foo2.lib imp_ref.obj impl_ref.lib" } } + steps: {} +} diff --git a/src/linker/tests/delay_import.tst b/src/linker/tests/delay_import.tst new file mode 100644 index 000000000..4d6ee00db --- /dev/null +++ b/src/linker/tests/delay_import.tst @@ -0,0 +1,231 @@ +test: +{ + + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + return_0: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, 0 + // ret + data: { hex: "48c7c000000000c3" } + } + return_1: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, 1 + // ret + data: { hex: "48c7c001000000c3" } + } + return_2: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, 2 + // ret + data: { hex: "48c7c002000000c3" } + } + } + symbols: + { + return_1: { kind: external_function, name: "return_1", section: return_1, value: 0 } + return_2: { kind: external_function, name: "return_2", section: return_2, value: 0 } + } + } } + } + + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: + { + return_0: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, 0 + // ret + data: { hex: "48c7c000000000c3" } + } + return_123: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, 123 + // ret + data: { hex: "48c7c07b000000c3" } + } + return_321: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // mov rax, 321 + // ret + data: { hex: "48c7c041010000c3" } + } + } + symbols: + { + return_123: { kind: external_function, name: "return_123", section: return_123, value: 0 } + return_321: { kind: external_function, name: "return_321", section: return_321, value: 0 } + } + } } + } + + main_obj: + { + file_name: "main.obj" + coff: { object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // push rsi + // push rdi + // sub rsp,28h + // call return_1 + // mov esi,eax + // call return_2 + // mov edi,eax + // add edi,esi + // call return_123 + // mov esi,eax + // call return_321 + // add eax,esi + // add eax,edi + // add rsp,28h + // pop rdi + // pop rsi + // ret + data: { hex: "56574883ec28e80000000089c6e80000000089c701f7e80000000089c6e80000000001f001f84883c4285f5ec3" } + relocations: + { + return_1_ref: { type: Rel32, offset: 7, symbol: return_1 } + return_2_ref: { type: Rel32, offset: 14, symbol: return_2 } + return_123_ref: { type: Rel32, offset: 23, symbol: return_123 } + return_321_ref: { type: Rel32, offset: 30, symbol: return_321 } + } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + return_1: { kind: undefined, name: "return_1" } + return_2: { kind: undefined, name: "return_2" } + return_123: { kind: undefined, name: "return_123" } + return_321: { kind: undefined, name: "return_321" } + } + } } + } + + a_dll: { file_name: "a.dll", pe: {} } + b_dll: { file_name: "b.dll", pe: {} } + image: { file_name: "a.exe", pe: {} } + } + + build: + { + link: + { + args: "/dll /implib:a.lib /export:return_1 /export:return_2 /rad_time_stamp:0x69EB0E28 a.obj libcmt.lib" + artifact: a_dll + } + + link: + { + args: "/dll /implib:b.lib /export:return_123 /export:return_321 /rad_time_stamp:0x69EB0E28 b.obj libcmt.lib" + artifact: b_dll + } + + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /fixed /debug:full /rad_time_stamp:0x69EB0E28 main.obj a.lib b.lib kernel32.lib delayimp.lib libcmt.lib /delayload:a.dll /delayload:b.dll" + artifact: image + } + } + + steps: + { + expect_pe: + { + artifact: image + expected: + { + pe: + { + delay_imports: + { + count: 2 + dll_0: + { + name: "a.dll" + attributes: 1 + @range(1, 18446744073709551615) module_handle + @range(1, 18446744073709551615) import_name_table + @range(1, 18446744073709551615) bound_table + @range(1, 18446744073709551615) unload_table + timestamp: 0 + bound_count: 2 + unload_count: 2 + count: 2 + entries: + { + import_0: { type: name, name: "return_1", hint: 0 } + import_1: { type: name, name: "return_2", hint: 1 } + } + } + dll_1: + { + name: "b.dll" + attributes: 1 + @range(1, 18446744073709551615) module_handle + @range(1, 18446744073709551615) import_name_table + @range(1, 18446744073709551615) bound_table + @range(1, 18446744073709551615) unload_table + timestamp: 0 + bound_count: 2 + unload_count: 2 + count: 2 + entries: + { + import_0: { type: name, name: "return_123", hint: 0 } + import_1: { type: name, name: "return_321", hint: 1 } + } + } + } + } + } + } + } +} diff --git a/src/linker/tests/delay_import_user32.tst b/src/linker/tests/delay_import_user32.tst new file mode 100644 index 000000000..0a33d60fa --- /dev/null +++ b/src/linker/tests/delay_import_user32.tst @@ -0,0 +1,68 @@ +test: +{ + + artifacts: + { + delay_import_obj: + { + file_name: "delay_import.obj" + coff: + { + object: + { + machine: x64 + sections: + { + str: + { + name: ".str" + permissions: (read, write) + content: initialized_data + data: { hex: "7465737400666f6f00" } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // sub rsp,28h + // xor r9d,r9d + // lea r8,[msg] + // lea rdx,[caption] + // xor ecx,ecx + // call qword ptr [__imp_MessageBoxA] + // xor eax,eax + // add rsp,28h + // ret + data: { hex: "4883ec284533c94c8d0500000000488d150000000033c9ff150000000033c04883c428c3" } + relocations: + { + msg_ref: { type: Rel32, offset: 10, symbol: msg } + caption_ref: { type: Rel32, offset: 17, symbol: caption } + message_box_ref: { type: Rel32, offset: 25, symbol: message_box } + } + } + } + symbols: + { + msg: { kind: external, name: "msg", section: str, value: 0 } + caption: { kind: external, name: "caption", section: str, value: 5 } + entry: { kind: external, name: "entry", section: text, value: 0 } + message_box: { kind: undefined, name: "__imp_MessageBoxA" } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /out:a.exe /entry:entry /fixed /delayload:user32.dll kernel32.lib user32.lib libcmt.lib delayimp.lib delay_import.obj /debug:full" + } + } + + steps: {} +} diff --git a/src/linker/tests/determ_test.tst b/src/linker/tests/determ_test.tst new file mode 100644 index 000000000..343cd30fc --- /dev/null +++ b/src/linker/tests/determ_test.tst @@ -0,0 +1,59 @@ +test: +{ + build: + { + windows: + { + outputs: + { + object: "test.obj" + baseline_exe: "main.exe" + baseline_pdb: "main.pdb" + } + + // compile the test target (torture) + compile: + { + output: "%outputs.object%" + args: "/fsanitize=address /Z7 -DBUILD_GIT_HASH=Stringify(0000000000000000000000000000000000000000) -I%source_dir% /Zc:preprocessor %source_dir%/torture/torture_main.c" + } + + // single-threaded link + link: + { + output: "%outputs.baseline_exe%" + args: "%outputs.object% /debug:full /rad_time_stamp:0 /rad_workers:1 /pdbaltpath:main.pdb /rad_log:-all /rad_ignore:74" + produces: { "%outputs.baseline_pdb%" } + } + + // read b + + // multi-threaded links + link: + { + output: "%run%.exe" + args: "%outputs.object% /debug:full /rad_time_stamp:0 /rad_imagealtpath:main.exe /pdbaltpath:main.pdb /rad_log:-all /rad_ignore:74" + produces: { "%run%.pdb" } + repeat: 25 + index: run + parallel: true + } + + // wait for linkers + } + } + + steps: + { + repeat: + { + count: 25 + index: run + steps: + { + compare_file: { left: "%build.outputs.baseline_exe%", right: "%run%.exe" } + compare_file: { left: "%build.outputs.baseline_pdb%", right: "%run%.pdb" } + } + } + } +} diff --git a/src/linker/tests/empty_section.tst b/src/linker/tests/empty_section.tst new file mode 100644 index 000000000..07decdbc8 --- /dev/null +++ b/src/linker/tests/empty_section.tst @@ -0,0 +1,41 @@ +test: +{ + artifacts: + { + empty_obj: + { + file_name: "empty_section.obj" + coff: { object: + { + machine: x64 + // TEST is defined in a zero-sized code section and cannot satisfy a live relocation. + sections: { test: { name: ".test", permissions: (read, execute), content: code, data: { zero: 0 } } } + symbols: { test: { kind: external, name: "TEST", section: test, value: 0 } } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "48c7c000000000c3" } + relocations: { test: { type: Addr32Nb, offset: 3, symbol: test } } + } } + symbols: + { + test: { kind: undefined, name: "TEST" } + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + } + // A relocation cannot resolve to a definition in an empty section. + build: { link: { args: "/subsystem:console /entry:entry /out:empty_section.exe empty_section.obj entry.obj", expect_exit: nonzero } } + steps: {} +} diff --git a/src/linker/tests/fail_if_mismatch.tst b/src/linker/tests/fail_if_mismatch.tst new file mode 100644 index 000000000..e3463777c --- /dev/null +++ b/src/linker/tests/fail_if_mismatch.tst @@ -0,0 +1,40 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + a1_obj: { file_name: "a1.obj", coff: { object: { machine: x64, directives: { directive: "/FAILIFMISMATCH:a=1" } } } } + a2_obj: { file_name: "a2.obj", coff: { object: { machine: x64, directives: { directive: "/FAILIFMISMATCH:a=2" } } } } + a1_copy_obj: { file_name: "a1_copy.obj", coff: { object: { machine: x64, directives: { directive: "/FAILIFMISMATCH:a=1" } } } } + conf_dirs_obj: { file_name: "conf_dirs.obj", coff: { object: { machine: x64, directives: { directive: "/FAILIFMISMATCH:a=1 /FAILIFMISMATCH:a=2" } } } } + } + + build: + { + // ------------------------------------------------------------ + // try linking two objs with mismatching directives + link: { args: "entry.obj a1.obj a2.obj /entry:entry /subsystem:console /out:a2.exe", output: none, expect_exit: 32 } + + // ------------------------------------------------------------ + // happy case + link: { args: "entry.obj a1.obj a1_copy.obj /entry:entry /subsystem:console /out:a1.exe", output: none } + + // ------------------------------------------------------------ + // test conflicting directives in obj + link: { args: "entry.obj conf_dirs.obj /entry:entry /subsystem:console /out:conf_dirs.exe", output: none, expect_exit: 32 } + + // ------------------------------------------------------------ + // passing switch on command line + link: { args: "entry.obj a1.obj /FAILIFMISMATCH:a=2 /out:cmddir.exe", output: none, expect_exit: 32 } + } + steps: {} +} diff --git a/src/linker/tests/find_merged_pdata.tst b/src/linker/tests/find_merged_pdata.tst new file mode 100644 index 000000000..f15b339ac --- /dev/null +++ b/src/linker/tests/find_merged_pdata.tst @@ -0,0 +1,54 @@ +test: +{ + artifacts: + { + main_obj: + { + file_name: "main.obj" + coff: { object: + { + machine: x64 + sections: + { + xdata: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, data: { hex: "191b0300090140000270000000000000f00100000104010004420000" } } + pdata: + { + name: ".pdata" + permissions: (read) + content: initialized_data + alignment: 4 + data: { zero: 12 } + relocations: + { + unwind: { type: Addr32Nb, offset: 8, symbol: unwind_foobar } + first: { type: Addr32Nb, offset: 0, symbol: foobar } + one_past_last: { type: Addr32Nb, offset: 4, symbol: foobar } + } + } + foobar: + { + name: ".foobar" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "40574881ec00020000488b05000000004833c448898424f0010000488d0424488bf833c0b9ec010000f3aab804000000486bc0028b0404488b8c24f00100004833cce8000000004881c4000200005fc3cccccccccccccccccccccccccccccccc4883ec28e8000000004883c428c3" } + } + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + foobar: { kind: static, name: "foobar", section: foobar, value: 0 } + xdata_definition: { kind: section_definition, section: xdata, selection: Null } + unwind_foobar: { kind: static, name: "$unwind$foobar", section: xdata, value: 0 } + pdata_definition: { kind: section_definition, section: pdata, selection: Null } + pdata_foobar: { kind: static, name: "$pdata$foobar", section: pdata, value: 0 } + entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe main.obj /merge:.pdata=.rdata", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { data_directories: { exceptions: { file_size: 12 } } } } } } +} diff --git a/src/linker/tests/first_member_header.tst b/src/linker/tests/first_member_header.tst new file mode 100644 index 000000000..d4d887f31 --- /dev/null +++ b/src/linker/tests/first_member_header.tst @@ -0,0 +1,82 @@ +test: +{ + + artifacts: + { + test_lib: + { + file_name: "test.lib" + coff: + { + library: + { + second_linker_member: false + members: + { + symbols: + { + path: "obj.obj" + object: + { + machine: x64 + symbols: + { + eight: { kind: absolute, name: "8", value: 8, storage: external } + one: { kind: absolute, name: "1", value: 1, storage: external } + nine: { kind: absolute, name: "9", value: 9, storage: external } + seven: { kind: absolute, name: "7", value: 7, storage: external } + four: { kind: absolute, name: "4", value: 4, storage: external } + five: { kind: absolute, name: "5", value: 5, storage: external } + two: { kind: absolute, name: "2", value: 2, storage: external } + three: { kind: absolute, name: "3", value: 3, storage: external } + six: { kind: absolute, name: "6", value: 6, storage: external } + } + } + } + } + } + } + } + + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "c3" } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + } + + build: + { + link: + { + args: "/RAD_MEMORY_MAP_FILES:READ_WRITE /subsystem:console /entry:entry /out:a.exe test.lib entry.obj /include:1 /include:2 /include:3 /include:4 /include:5 /include:6 /include:7 /include:8 /include:9" + } + } + + steps: + { + expect_file: { path: "test.lib", equals_artifact: test_lib } + expect_file: { path: "entry.obj", equals_artifact: entry_obj } + } +} diff --git a/src/linker/tests/flag_conf.tst b/src/linker/tests/flag_conf.tst new file mode 100644 index 000000000..3a400df42 --- /dev/null +++ b/src/linker/tests/flag_conf.tst @@ -0,0 +1,51 @@ +test: +{ + artifacts: + { + conf_obj: + { + file_name: "conf.obj" + coff: { object: + { + machine: x64 + sections: + { + a: { name: ".mysect", permissions: (read, execute), content: initialized_data, data: { text: "one" } } + b: { name: ".mysect", permissions: (read, write), content: initialized_data, data: { text: "two" } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe conf.obj entry.obj", artifact: image } } + steps: + { + expect_pe: + { + artifact: image + expected: + { + pe: + { + section_count: 3 + @count(3) sections_by_index: + { + section_1: { name: ".text" } + section_2: { name: ".mysect", raw_flags: 1610612800 } + section_3: { name: ".mysect", raw_flags: 3221225536 } + } + } + } + } + } +} diff --git a/src/linker/tests/function_pad_min.tst b/src/linker/tests/function_pad_min.tst new file mode 100644 index 000000000..e9ec4fd7c --- /dev/null +++ b/src/linker/tests/function_pad_min.tst @@ -0,0 +1,30 @@ +test: +{ + artifacts: + { + funcs_obj: + { + file_name: "funcs.obj" + coff: { object: + { + machine: x64 + sections: + { + a: { name: ".text", permissions: (read, execute), content: code, alignment: 4, data: { hex: "c3" } } + b: { name: ".text", permissions: (read, execute), content: code, alignment: 4, data: { hex: "c3" } } + c: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + a: { kind: external_function, name: "A", section: a, value: 0 } + b: { kind: external_function, name: "B", section: b, value: 0 } + c: { kind: external_function, name: "C", section: c, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/subsystem:console /entry:A /functionpadmin:1 /out:a.exe funcs.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: ccccccccc3ccccccc3ccc3 } } } } } } +} diff --git a/src/linker/tests/gen_rrt.tst b/src/linker/tests/gen_rrt.tst new file mode 100644 index 000000000..8f3020a4b --- /dev/null +++ b/src/linker/tests/gen_rrt.tst @@ -0,0 +1,202 @@ +test: +{ + + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_p: + { + name: ".debug$P" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + concat: + { + // signature + hex: "04000000" + // duplicate in a.obj + hex: "0a0002100300000000000000" + // unique procedure type + hex: "0e000810001000000200000000000000" + // PCH ender + hex: "06001400bebafeca" + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + align4: + { + concat: + { + // C13 signature and symbols subsection kind. + hex: "04000000f1000000" + size32le: + { + concat: + { + // S_OBJNAME, signature 0xcafebabe, and the terminated a.obj work path. + size16le: + { + concat: + { + hex: "0111bebafeca" + work_path: "a.obj" + zero: 1 + } + } + + // S_COMPILE3: C/x64, front-end 0.0.0.0, compiler 14.36.32537.0, TORTURE. + size16le: { hex: "3c1100000000d00000000000000000000e002400197f0000544f525455524500" } + } + } + } + } + } + } + } + } + } + } + + b_obj: + { + file_name: "b.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + concat: + { + // signature + hex: "04000000" + + // PCH starter + hex: "280009150010000002000000bebafeca" + text: "corrupt-pch-file-path.obj" + zero: 1 + + // duplicate pointer type + hex: "0a0002100300000000000000" + + // unique procedure type + hex: "0e000810001000000000000000000000" + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + align4: + { + concat: + { + // C13 signature and symbols subsection kind. + hex: "04000000f1000000" + size32le: + { + concat: + { + // Preserve the original use of a.obj in b.obj's S_OBJNAME record. + size16le: + { + concat: + { + hex: "0111bebafeca" + work_path: "a.obj" + zero: 1 + } + } + + // S_COMPILE3: C/x64, front-end 0.0.0.0, compiler 14.36.32537.0, TORTURE. + size16le: { hex: "3c1100000000d00000000000000000000e002400197f0000544f525455524500" } + } + } + } + } + } + } + } + } + } + } + + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "c3" } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:full /rad_type_server:foo.rrt a.obj b.obj entry.obj" + } + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:ghash a.obj foo.rrt b.obj entry.obj" + } + } + + steps: {} +} diff --git a/src/linker/tests/get_msf_stream_pages.tst b/src/linker/tests/get_msf_stream_pages.tst new file mode 100644 index 000000000..3ea2bb73c --- /dev/null +++ b/src/linker/tests/get_msf_stream_pages.tst @@ -0,0 +1,4 @@ +test: +{ + steps: { get_msf_stream_pages: {} } +} diff --git a/src/linker/tests/ghash_basic.tst b/src/linker/tests/ghash_basic.tst new file mode 100644 index 000000000..af9017136 --- /dev/null +++ b/src/linker/tests/ghash_basic.tst @@ -0,0 +1,114 @@ +test: +{ + + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + + a_obj: + { + file_name: "a.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + raw_flags: 33554432 + data: { hex: "0400000012000515000080000000000000000000000000000a00061500008000000000000e000715000080000000000000000000" } + } + debug_h: + { + name: ".debug$H" + permissions: (read) + content: initialized_data + raw_flags: 33554432 + data: { hex: "c5c9330100000200010000000000000002000000000000000300000000000000" } + } + } + } + } + } + + b_obj: + { + file_name: "b.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + raw_flags: 33554432 + data: { hex: "0400000012000515000080000000000000000000000000000a00061500008000000000000e000715000080000000000000000000" } + } + debug_h: + { + name: ".debug$H" + permissions: (read) + content: initialized_data + raw_flags: 33554432 + data: { hex: "c5c9330100000200040000000000000005000000000000000600000000000000" } + } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:ghash.exe /debug:ghash entry.obj a.obj b.obj" + } + link: + { + args: "/subsystem:console /entry:entry /out:full.exe /debug:full entry.obj a.obj b.obj" + } + } + + steps: + { + expect_pdb: + { + path: "ghash.pdb" + expected: { pdb: { tpi: { leaf_count: 6 } } } + } + expect_pdb: + { + path: "full.pdb" + expected: { pdb: { tpi: { leaf_count: 3 } } } + } + } +} diff --git a/src/linker/tests/ghash_check_corrupt.tst b/src/linker/tests/ghash_check_corrupt.tst new file mode 100644 index 000000000..3d3ee80a9 --- /dev/null +++ b/src/linker/tests/ghash_check_corrupt.tst @@ -0,0 +1,40 @@ +test: +{ + artifacts: + { + debug_obj: + { + file_name: "debug.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_t: { name: ".debug$T", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "0400000012000515000080000000000000000000000000000a00061500008000000000000e000715000080000000000000000000" } } + debug_h: { name: ".debug$H", permissions: (read), content: initialized_data, flags: (discardable), data: { zero: 0 } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj" + output: none + stderr_matches: "*Warning(*): *: .debug$H section is too small to contain the header*" + } + } + steps: {} +} diff --git a/src/linker/tests/ghash_check_hash_alg.tst b/src/linker/tests/ghash_check_hash_alg.tst new file mode 100644 index 000000000..0c07419f7 --- /dev/null +++ b/src/linker/tests/ghash_check_hash_alg.tst @@ -0,0 +1,40 @@ +test: +{ + artifacts: + { + debug_obj: + { + file_name: "debug.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_t: { name: ".debug$T", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "0400000012000515000080000000000000000000000000000a00061500008000000000000e000715000080000000000000000000" } } + debug_h: { name: ".debug$H", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "c5c9330100000100a5fdf48ae0caba6e9a76f994668785c388229ba829a5a87e" } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj" + output: none + stderr_matches: "*Warning(*): *: mismatched .debug$H hash algorithm: got SHA1_8, expected *" + } + } + steps: {} +} diff --git a/src/linker/tests/ghash_check_magic.tst b/src/linker/tests/ghash_check_magic.tst new file mode 100644 index 000000000..527fa3492 --- /dev/null +++ b/src/linker/tests/ghash_check_magic.tst @@ -0,0 +1,40 @@ +test: +{ + artifacts: + { + debug_obj: + { + file_name: "debug.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_t: { name: ".debug$T", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "0400000012000515000080000000000000000000000000000a00061500008000000000000e000715000080000000000000000000" } } + debug_h: { name: ".debug$H", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "7b00000000000200a5fdf48ae0caba6e9a76f994668785c388229ba829a5a87e" } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj" + output: none + stderr_matches: "*Warning(*): *: .debug$H contains invalid magic: got 0x7b, expected 0x133c9c5*" + } + } + steps: {} +} diff --git a/src/linker/tests/ghash_check_version.tst b/src/linker/tests/ghash_check_version.tst new file mode 100644 index 000000000..78f97f470 --- /dev/null +++ b/src/linker/tests/ghash_check_version.tst @@ -0,0 +1,40 @@ +test: +{ + artifacts: + { + debug_obj: + { + file_name: "debug.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_t: { name: ".debug$T", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "0400000012000515000080000000000000000000000000000a00061500008000000000000e000715000080000000000000000000" } } + debug_h: { name: ".debug$H", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "c5c93301efbe0200a5fdf48ae0caba6e9a76f994668785c388229ba829a5a87e" } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj" + output: none + stderr_matches: "*Warning(*): *: mismatched .debug$H version: got 48879, expected 0*" + } + } + steps: {} +} diff --git a/src/linker/tests/ghash_match_debug_t.tst b/src/linker/tests/ghash_match_debug_t.tst new file mode 100644 index 000000000..c09659daa --- /dev/null +++ b/src/linker/tests/ghash_match_debug_t.tst @@ -0,0 +1,48 @@ +test: +{ + artifacts: + { + debug_obj: + { + file_name: "debug.obj" + coff: { object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + flags: (discardable) + //str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); + data: { hex: "0400000012000515000080000000000000000000000000000a0006150000800000000000" } + } + debug_h: { name: ".debug$H", permissions: (read), content: initialized_data, flags: (discardable), data: { hex: "c5c9330100000200a5fdf48ae0caba6e9a76f994668785c388229ba829a5a87e" } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj" + output: none + stderr_matches: "*Warning(*): *: mismatched .debug$H hash count and type count: got 3 hashes for 2 types*" + } + } + steps: {} +} diff --git a/src/linker/tests/guard_cf_pulls_load_config.tst b/src/linker/tests/guard_cf_pulls_load_config.tst new file mode 100644 index 000000000..a2572f258 --- /dev/null +++ b/src/linker/tests/guard_cf_pulls_load_config.tst @@ -0,0 +1,42 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + loadcfg_lib: + { + file_name: "loadcfg.lib" + coff: { library: + { + second_linker_member: true + members: { loadcfg_obj: + { + path: "loadcfg.obj" + object: + { + machine: x64 + sections: { loadcfg: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 8, data: { concat: { hex: "40000000", zero: 60 } } } } + symbols: { load_config_used: { kind: external, name: "_load_config_used", section: loadcfg, value: 0 } } + } + } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe /guard:cf entry.obj loadcfg.lib", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: + { + data_directories: { load_config: { file_size: 64 } } + optional: { @bits_clear(16384) dll_characteristics } + } } } } +} diff --git a/src/linker/tests/guardsym_directive.tst b/src/linker/tests/guardsym_directive.tst new file mode 100644 index 000000000..d748e8e18 --- /dev/null +++ b/src/linker/tests/guardsym_directive.tst @@ -0,0 +1,25 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + guardsym_obj: + { + file_name: "guardsym.obj" + // MSVC link accepts GUARDSYM without treating the named symbol as /INCLUDE. + coff: { object: { machine: x64, directives: { directive: "/GUARDSYM:missing,S" } } } + } + } + + build: { link: { args: "entry.obj guardsym.obj /entry:entry /subsystem:console /out:guardsym.exe", output: none } } + steps: {} +} diff --git a/src/linker/tests/icf_associative_child_prevents_fold.tst b/src/linker/tests/icf_associative_child_prevents_fold.tst new file mode 100644 index 000000000..fe0e2d04f --- /dev/null +++ b/src/linker/tests/icf_associative_child_prevents_fold.tst @@ -0,0 +1,53 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_associative_child.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + fn_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + fn_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + handler_a: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01020304" } } + handler_b: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "04030201" } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000000000000000000000000000" } + relocations: + { + a: { type: Addr64, offset: 0, symbol: fn_a_symbol } + b: { type: Addr64, offset: 8, symbol: fn_b_symbol } + } + } + } + symbols: + { + fn_a_definition: { kind: section_definition, section: fn_a, selection: NoDuplicates } + fn_b_definition: { kind: section_definition, section: fn_b, selection: NoDuplicates } + handler_a_definition: { kind: section_definition, section: handler_a, selection: Associative, associate: fn_a } + handler_b_definition: { kind: section_definition, section: handler_b, selection: Associative, associate: fn_b } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + fn_a_symbol: { kind: external_function, name: "fn_a", section: fn_a, value: 0 } + fn_b_symbol: { kind: external_function, name: "fn_b", section: fn_b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_associative_child.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true, other_section: ".data", other_offset: 0, relation: not_equal } + } +} diff --git a/src/linker/tests/icf_comdat_reloc_targets_fold.tst b/src/linker/tests/icf_comdat_reloc_targets_fold.tst new file mode 100644 index 000000000..74b7a3187 --- /dev/null +++ b/src/linker/tests/icf_comdat_reloc_targets_fold.tst @@ -0,0 +1,108 @@ +test: +{ + artifacts: + { + a_object: + { + file_name: "icf_comdat_reloc_a.obj" + coff: { object: + { + machine: x64 + sections: + { + fn_a: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + // mov rax, shared + // ret + data: { hex: "48b80000000000000000c3" } + relocations: { shared: { type: Addr64, offset: 2, symbol: shared_local_a } } + } + shared: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "00" } } + } + symbols: + { + fn_a_definition: { kind: section_definition, section: fn_a, selection: NoDuplicates } + shared_definition: { kind: section_definition, section: shared, selection: Any } + fn_a_symbol: { kind: external_function, name: "fn_a", section: fn_a, value: 0 } + shared_symbol: { kind: external, name: "shared", section: shared, value: 0 } + shared_local_a: { kind: static, name: "shared_local_a", section: shared, value: 0 } + } + } } + } + b_object: + { + file_name: "icf_comdat_reloc_b.obj" + coff: { object: + { + machine: x64 + sections: + { + fn_b: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + // mov rax, shared + // ret + data: { hex: "48b80000000000000000c3" } + relocations: { shared: { type: Addr64, offset: 2, symbol: shared_local_b } } + } + shared: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "00" } } + } + symbols: + { + fn_b_definition: { kind: section_definition, section: fn_b, selection: NoDuplicates } + shared_definition: { kind: section_definition, section: shared, selection: Any } + fn_b_symbol: { kind: external_function, name: "fn_b", section: fn_b, value: 0 } + shared_symbol: { kind: external, name: "shared", section: shared, value: 0 } + shared_local_b: { kind: static, name: "shared_local_b", section: shared, value: 0 } + } + } } + } + entry_object: + { + file_name: "icf_comdat_reloc_entry.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000000000000000000000000000" } + relocations: + { + a: { type: Addr64, offset: 0, symbol: fn_a } + b: { type: Addr64, offset: 8, symbol: fn_b } + } + } + } + symbols: + { + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + fn_a: { kind: undefined_function, name: "fn_a" } + fn_b: { kind: undefined_function, name: "fn_b" } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_comdat_reloc_entry.obj icf_comdat_reloc_a.obj icf_comdat_reloc_b.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, other_section: ".data", other_offset: 8, relation: equal } + } +} diff --git a/src/linker/tests/icf_comdat_symlink_chain.tst b/src/linker/tests/icf_comdat_symlink_chain.tst new file mode 100644 index 000000000..3fe2ef9e7 --- /dev/null +++ b/src/linker/tests/icf_comdat_symlink_chain.tst @@ -0,0 +1,83 @@ +test: +{ + artifacts: + { + leader_object: + { + file_name: "icf_chain_leader.obj" + coff: { object: + { + machine: x64 + sections: { leader: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c390" } } } + symbols: + { + leader_definition: { kind: section_definition, section: leader, selection: NoDuplicates } + leader: { kind: external_function, name: "leader", section: leader, value: 0 } + } + } } + } + duplicate_object: + { + file_name: "icf_chain_duplicate.obj" + coff: { object: + { + machine: x64 + sections: + { + duplicate: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000000000000000000000000000" } + relocations: + { + duplicate: { type: Addr64, offset: 0, symbol: local_duplicate } + leader: { type: Addr64, offset: 8, symbol: leader } + } + } + } + symbols: + { + duplicate_definition: { kind: section_definition, section: duplicate, selection: Largest } + duplicate: { kind: external_function, name: "dup", section: duplicate, value: 0 } + local_duplicate: { kind: static, name: "local_dup", section: duplicate, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + leader: { kind: undefined_function, name: "leader" } + } + } } + } + selected_object: + { + file_name: "icf_chain_selected.obj" + coff: { object: + { + machine: x64 + sections: { duplicate: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c390" } } } + symbols: + { + duplicate_definition: { kind: section_definition, section: duplicate, selection: Largest } + duplicate: { kind: external_function, name: "dup", section: duplicate, value: 0 } + } + } } + } + entry_object: + { + file_name: "icf_chain_entry.obj" + coff: { object: + { + machine: x64 + sections: { entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external_function, name: "entry", section: entry, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_chain_leader.obj icf_chain_duplicate.obj icf_chain_selected.obj icf_chain_entry.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, other_section: ".data", other_offset: 8, relation: equal } + } +} diff --git a/src/linker/tests/icf_cpp_identical_functions_fold.tst b/src/linker/tests/icf_cpp_identical_functions_fold.tst new file mode 100644 index 000000000..1f859180b --- /dev/null +++ b/src/linker/tests/icf_cpp_identical_functions_fold.tst @@ -0,0 +1,49 @@ +test: +{ + artifacts: + { + source: + { + file_name: "icf_cpp_fold.cpp" + text: { data: { concat: + { + text: "extern " + hex: "22" + text: "C" + hex: "22" + text: " __declspec(noinline) int a(void) { return 42; }" + hex: "0a" + text: "extern " + hex: "22" + text: "C" + hex: "22" + text: " __declspec(noinline) int b(void) { return 42; }" + hex: "0a" + text: "extern " + hex: "22" + text: "C" + hex: "22" + text: " int (* volatile pa)(void) = a;" + hex: "0a" + text: "extern " + hex: "22" + text: "C" + hex: "22" + text: " int (* volatile pb)(void) = b;" + hex: "0a" + text: "extern " + hex: "22" + text: "C" + hex: "22" + text: " int entry(void) { return pa == pb ? 0 : 1; }" + hex: "0a" + } } } + } + } + build: + { + compile: { tool: cl, output: none, args: "/nologo /c /O2 /Gy /Zc:preprocessor /Fo:icf_cpp_fold.obj icf_cpp_fold.cpp" } + link: { output: none, args: "/nodefaultlib /subsystem:console /entry:entry /out:icf_cpp_fold.exe /opt:ref,icf /include:pa /include:pb icf_cpp_fold.obj" } + } + steps: { run: { path: "icf_cpp_fold.exe" } } +} diff --git a/src/linker/tests/icf_cpp_multihop_functions_do_not_fold.tst b/src/linker/tests/icf_cpp_multihop_functions_do_not_fold.tst new file mode 100644 index 000000000..6a388163c --- /dev/null +++ b/src/linker/tests/icf_cpp_multihop_functions_do_not_fold.tst @@ -0,0 +1,55 @@ +test: +{ + artifacts: + { + source: + { + file_name: "icf_cpp_multihop.c" + text: { data: { concat: + { + text: "__declspec(noinline) int leaf_a(void) { return 1; }" + hex: "0a" + text: "__declspec(noinline) int leaf_b(void) { return 2; }" + hex: "0a" + text: "__declspec(noinline) int mid_a(void) { return leaf_a(); }" + hex: "0a" + text: "__declspec(noinline) int mid_b(void) { return leaf_b(); }" + hex: "0a" + text: "__declspec(noinline) int top_a(void) { return mid_a(); }" + hex: "0a" + text: "__declspec(noinline) int top_b(void) { return mid_b(); }" + hex: "0a" + text: "int (* volatile p_top_a)(void) = top_a;" + hex: "0a" + text: "int (* volatile p_top_b)(void) = top_b;" + hex: "0a" + text: "int (* volatile p_mid_a)(void) = mid_a;" + hex: "0a" + text: "int (* volatile p_mid_b)(void) = mid_b;" + hex: "0a" + text: "int (* volatile p_leaf_a)(void) = leaf_a;" + hex: "0a" + text: "int (* volatile p_leaf_b)(void) = leaf_b;" + hex: "0a" + text: "int entry(void) {" + hex: "0a" + text: " if (p_top_a == p_top_b) { return 1; }" + hex: "0a" + text: " if (p_mid_a == p_mid_b) { return 2; }" + hex: "0a" + text: " if (p_leaf_a == p_leaf_b) { return 3; }" + hex: "0a" + text: " return 0;" + hex: "0a" + text: "}" + hex: "0a" + } } } + } + } + build: + { + compile: { tool: cl, output: none, args: "/nologo /c /O2 /Gy /Zc:preprocessor /Fo:icf_cpp_multihop.obj icf_cpp_multihop.c" } + link: { output: none, args: "/nodefaultlib /subsystem:console /entry:entry /out:icf_cpp_multihop.exe /opt:ref,icf /include:p_top_a /include:p_top_b /include:p_mid_a /include:p_mid_b /include:p_leaf_a /include:p_leaf_b icf_cpp_multihop.obj" } + } + steps: { run: { path: "icf_cpp_multihop.exe" } } +} diff --git a/src/linker/tests/icf_cyclic_icf.tst b/src/linker/tests/icf_cyclic_icf.tst new file mode 100644 index 000000000..7e28f1529 --- /dev/null +++ b/src/linker/tests/icf_cyclic_icf.tst @@ -0,0 +1,37 @@ +test: +{ + artifacts: + { + object: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + a: { name: ".text", permissions: (read, execute), content: code, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call_b: { type: Rel32, offset: 1, symbol: b_symbol } } } + b: { name: ".text", permissions: (read, execute), content: code, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call_a: { type: Rel32, offset: 1, symbol: a_symbol } } } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: NoDuplicates } + b_definition: { kind: section_definition, section: b, selection: NoDuplicates } + a_symbol: { kind: external, name: "a", section: a, value: 0 } + b_symbol: { kind: static, name: "b", section: b, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /out:a.exe /entry:a /opt:icf a.obj", artifact: image } } + steps: + { + // validate output + expect_pe: { artifact: image, expected: { pe: { sections: { ".text": + { + // a and b folded into a self-call + data: e8fbffffffc3 + } } } } } + } +} diff --git a/src/linker/tests/icf_fold_diamond.tst b/src/linker/tests/icf_fold_diamond.tst new file mode 100644 index 000000000..74b0f31c5 --- /dev/null +++ b/src/linker/tests/icf_fold_diamond.tst @@ -0,0 +1,51 @@ +test: +{ + artifacts: + { + object: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + a: + { + name: ".text" + permissions: (read, execute) + content: code + flags: (link_comdat) + data: { hex: "e800000000e800000000c3" } + relocations: + { + call_b: { type: Rel32, offset: 1, symbol: b_symbol } + call_c: { type: Rel32, offset: 6, symbol: c_symbol } + } + } + b: { name: ".text", permissions: (read, execute), content: code, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call_d: { type: Rel32, offset: 1, symbol: d_symbol } } } + c: { name: ".text", permissions: (read, execute), content: code, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call_d: { type: Rel32, offset: 1, symbol: d_symbol } } } + d: { name: ".text", permissions: (read, execute), content: code, flags: (link_comdat), data: { hex: "4831c0c3" } } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: NoDuplicates } + b_definition: { kind: section_definition, section: b, selection: NoDuplicates } + c_definition: { kind: section_definition, section: c, selection: NoDuplicates } + d_definition: { kind: section_definition, section: d, selection: NoDuplicates } + a_symbol: { kind: external, name: "a", section: a, value: 0 } + b_symbol: { kind: external, name: "b", section: b, value: 0 } + c_symbol: { kind: external, name: "c", section: c, value: 0 } + d_symbol: { kind: external, name: "d", section: d, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:a /out:a.exe /opt:icf a.obj", artifact: image } } + steps: + { + // validate output + expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: e80b000000e806000000c3cccccccccce80b000000c3cccccccccccccccccccc4831c0c3 } } } } } + } +} diff --git a/src/linker/tests/icf_fold_two_funcs.tst b/src/linker/tests/icf_fold_two_funcs.tst new file mode 100644 index 000000000..8ce86de05 --- /dev/null +++ b/src/linker/tests/icf_fold_two_funcs.tst @@ -0,0 +1,81 @@ +test: +{ + artifacts: + { + object: + { + file_name: "ident_funcs.obj" + coff: { object: + { + machine: x64 + sections: + { + a: + { + name: ".text$mn" + permissions: (read, execute) + content: code + flags: (link_comdat) + // xor rax, rax + // ret + data: { hex: "4831c0c3" } + } + b: + { + name: ".text$mb" + permissions: (read, execute) + content: code + flags: (link_comdat) + // xor rax, rax + // ret + data: { hex: "4831c0c3" } + } + entry: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + // call a + // call b + // ret + data: { hex: "e800000000e800000000c3" } + relocations: + { + call_a: { type: Rel32, offset: 1, symbol: a_symbol } + call_b: { type: Rel32, offset: 6, symbol: b_symbol } + } + } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: NoDuplicates } + b_definition: { kind: section_definition, section: b, selection: NoDuplicates } + a_symbol: { kind: external_function, name: "a", section: a, value: 0 } + b_symbol: { kind: external_function, name: "b", section: b, value: 0 } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:icf ident_funcs.obj", artifact: image } } + steps: + { + expect_pe: + { + artifact: image + expected: { pe: { sections: { ".text": + { + // validate .text header + virtual_offset: 4096 + @range(20, 18446744073709551615) virtual_size + file_size: 512 + // entry + // pad + // a and b folded + @starts_with(e80b000000e806000000c3cccccccccc4831c0c3) data + } } } } + } + } +} diff --git a/src/linker/tests/icf_fold_with_largest_align.tst b/src/linker/tests/icf_fold_with_largest_align.tst new file mode 100644 index 000000000..ece99c150 --- /dev/null +++ b/src/linker/tests/icf_fold_with_largest_align.tst @@ -0,0 +1,96 @@ +test: +{ + artifacts: + { + a_object: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "e800000000e800000000c3" } + relocations: + { + call_a: { type: Rel32, offset: 1, symbol: a_symbol } + call_b: { type: Rel32, offset: 6, symbol: b_symbol } + } + } + a: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 4 + flags: (link_comdat) + // mov rax, 1 + data: { hex: "48c7c001000000c3" } + } + b: { name: ".text", permissions: (read, execute), content: code, alignment: 8, flags: (link_comdat), data: { hex: "48c7c001000000c3" } } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: NoDuplicates } + b_definition: { kind: section_definition, section: b, selection: NoDuplicates } + entry_symbol: { kind: external, name: "entry", section: entry, value: 0 } + a_symbol: { kind: static, name: "a", section: a, value: 0 } + b_symbol: { kind: static, name: "b", section: b, value: 0 } + } + } } + } + // swap sections for a and b + b_object: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "e800000000e800000000c3" } + relocations: + { + call_a: { type: Rel32, offset: 1, symbol: a_symbol } + call_b: { type: Rel32, offset: 6, symbol: b_symbol } + } + } + a: { name: ".text", permissions: (read, execute), content: code, alignment: 8, flags: (link_comdat), data: { hex: "48c7c001000000c3" } } + b: { name: ".text", permissions: (read, execute), content: code, alignment: 4, flags: (link_comdat), data: { hex: "48c7c001000000c3" } } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: NoDuplicates } + b_definition: { kind: section_definition, section: b, selection: NoDuplicates } + entry_symbol: { kind: external, name: "entry", section: entry, value: 0 } + a_symbol: { kind: static, name: "a", section: a, value: 0 } + b_symbol: { kind: static, name: "b", section: b, value: 0 } + } + } } + } + a_image: { file_name: "a.exe", pe: {} } + b_image: { file_name: "b.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:icf a.obj", artifact: a_image } + link: { args: "/subsystem:console /entry:entry /out:b.exe /opt:icf b.obj", artifact: b_image } + } + steps: + { + // validate output in a.exe + expect_pe: { artifact: a_image, expected: { pe: { sections: { ".text": { data: e80b000000e806000000c3cccccccccc48c7c001000000c3 } } } } } + // validate output in b.exe + expect_pe: { artifact: b_image, expected: { pe: { sections: { ".text": { data: e80b000000e806000000c3cccccccccc48c7c001000000c3 } } } } } + } +} diff --git a/src/linker/tests/icf_identical_bytes_different_color_spaces_do_not_fold.tst b/src/linker/tests/icf_identical_bytes_different_color_spaces_do_not_fold.tst new file mode 100644 index 000000000..661d5452f --- /dev/null +++ b/src/linker/tests/icf_identical_bytes_different_color_spaces_do_not_fold.tst @@ -0,0 +1,49 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_color_spaces.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + text: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + vftable: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000000000000000000000000000" } + relocations: + { + text: { type: Addr64, offset: 0, symbol: text_symbol } + vftable: { type: Addr64, offset: 8, symbol: vftable_symbol } + } + } + } + symbols: + { + text_definition: { kind: section_definition, section: text, selection: NoDuplicates } + vftable_definition: { kind: section_definition, section: vftable, selection: NoDuplicates } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + text_symbol: { kind: external_function, name: "text", section: text, value: 0 } + vftable_symbol: { kind: external, name: "??_7type@@6B@", section: vftable, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_color_spaces.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true, other_section: ".data", other_offset: 0, relation: not_equal } + } +} diff --git a/src/linker/tests/icf_llvm_addrsig.tst b/src/linker/tests/icf_llvm_addrsig.tst new file mode 100644 index 000000000..09a2ddb5d --- /dev/null +++ b/src/linker/tests/icf_llvm_addrsig.tst @@ -0,0 +1,36 @@ +test: +{ + artifacts: + { + source: + { + file_name: "main.c" + text: { data: { concat: + { + text: "int foo() { return 123; }" + hex: "0a" + text: "int bar() { return 123; }" + hex: "0a" + text: "int main() {" + hex: "0a" + text: "int (*fn)() = &foo;" + hex: "0a" + text: "return fn != bar;" + hex: "0a" + text: "}" + hex: "0a" + } } } + } + } + build: + { + compile: { tool: clang, output: "main.obj", args: "main.c -ffunction-sections -target x86_64-pc-windows-msvc" } + link: { output: none, args: "main.obj /opt:icf /out:a.exe libcmt.lib" } + link: { output: none, args: "main.obj /opt:icf /out:a_no_addrsig.exe libcmt.lib /llvm_addrsig:no" } + } + steps: + { + run: { path: "a.exe", expect_exit: 1 } + run: { path: "a_no_addrsig.exe" } + } +} diff --git a/src/linker/tests/icf_llvm_addrsig_external_symbol.tst b/src/linker/tests/icf_llvm_addrsig_external_symbol.tst new file mode 100644 index 000000000..bfac6d790 --- /dev/null +++ b/src/linker/tests/icf_llvm_addrsig_external_symbol.tst @@ -0,0 +1,46 @@ +// .llvm_addrsig can name an undefined external whose definition is in another +// object; ICF must parse and mark the resolved symbol's object, not the referrer. +test: +{ + artifacts: + { + ref_source: + { + file_name: "ref.c" + text: { data: { concat: + { + text: "extern int ext_sig();" + hex: "0a" + text: "int (*ext_sig_addr)() = &ext_sig;" + hex: "0a" + text: "int entry() { return ext_sig_addr(); }" + hex: "0a" + } } } + } + def_source: + { + file_name: "def.c" + text: { data: { concat: + { + text: "int dummy0() { return 0; }" + hex: "0a" + text: "int dummy1() { return 1; }" + hex: "0a" + text: "int dummy2() { return 2; }" + hex: "0a" + text: "int dummy3() { return 3; }" + hex: "0a" + text: "int dummy4() { return 4; }" + hex: "0a" + text: "int ext_sig() { return 0; }" + hex: "0a" + } } } + } + } + build: + { + compile: { tool: clang, output: "ref.obj", args: "ref.c -ffunction-sections -target x86_64-pc-windows-msvc" } + compile: { tool: clang, output: "def.obj", args: "def.c -ffunction-sections -target x86_64-pc-windows-msvc" } + link: { output: none, args: "ref.obj def.obj /subsystem:console /entry:entry /opt:icf /out:addrsig_ext.exe libcmt.lib" } + } +} diff --git a/src/linker/tests/icf_multihop_reloc_target_colors_do_not_fold.tst b/src/linker/tests/icf_multihop_reloc_target_colors_do_not_fold.tst new file mode 100644 index 000000000..ccaf7b253 --- /dev/null +++ b/src/linker/tests/icf_multihop_reloc_target_colors_do_not_fold.tst @@ -0,0 +1,69 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_multihop.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + top_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call: { type: Rel32, offset: 1, symbol: mid_a_symbol } } } + top_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call: { type: Rel32, offset: 1, symbol: mid_b_symbol } } } + mid_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call: { type: Rel32, offset: 1, symbol: leaf_a_symbol } } } + mid_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "e800000000c3" }, relocations: { call: { type: Rel32, offset: 1, symbol: leaf_b_symbol } } } + leaf_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "48c7c001000000c3" } } + leaf_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "48c7c002000000c3" } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" } + relocations: + { + top_a: { type: Addr64, offset: 0, symbol: top_a_symbol } + top_b: { type: Addr64, offset: 8, symbol: top_b_symbol } + mid_a: { type: Addr64, offset: 16, symbol: mid_a_symbol } + mid_b: { type: Addr64, offset: 24, symbol: mid_b_symbol } + leaf_a: { type: Addr64, offset: 32, symbol: leaf_a_symbol } + leaf_b: { type: Addr64, offset: 40, symbol: leaf_b_symbol } + } + } + } + symbols: + { + top_a_definition: { kind: section_definition, section: top_a, selection: NoDuplicates } + top_b_definition: { kind: section_definition, section: top_b, selection: NoDuplicates } + mid_a_definition: { kind: section_definition, section: mid_a, selection: NoDuplicates } + mid_b_definition: { kind: section_definition, section: mid_b, selection: NoDuplicates } + leaf_a_definition: { kind: section_definition, section: leaf_a, selection: NoDuplicates } + leaf_b_definition: { kind: section_definition, section: leaf_b, selection: NoDuplicates } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + top_a_symbol: { kind: external_function, name: "top_a", section: top_a, value: 0 } + top_b_symbol: { kind: external_function, name: "top_b", section: top_b, value: 0 } + mid_a_symbol: { kind: external_function, name: "mid_a", section: mid_a, value: 0 } + mid_b_symbol: { kind: external_function, name: "mid_b", section: mid_b, value: 0 } + leaf_a_symbol: { kind: external_function, name: "leaf_a", section: leaf_a, value: 0 } + leaf_b_symbol: { kind: external_function, name: "leaf_b", section: leaf_b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_multihop.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, other_section: ".data", other_offset: 8, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 16, type: u64, nonzero: true, other_section: ".data", other_offset: 24, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 24, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 32, type: u64, nonzero: true, other_section: ".data", other_offset: 40, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 40, type: u64, nonzero: true } + } +} diff --git a/src/linker/tests/icf_pdata_differs_by_function_color.tst b/src/linker/tests/icf_pdata_differs_by_function_color.tst new file mode 100644 index 000000000..1cf0ea0b6 --- /dev/null +++ b/src/linker/tests/icf_pdata_differs_by_function_color.tst @@ -0,0 +1,62 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_pdata_diff.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + fn_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "b801000000c3" } } + fn_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "b802000000c3" } } + xdata_a: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01000000" } } + xdata_b: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01000000" } } + pdata_a: { name: ".pdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "000000000000000000000000" }, relocations: { first: { type: Addr32Nb, offset: 0, symbol: fn_a_symbol }, last: { type: Addr32Nb, offset: 4, symbol: fn_a_symbol }, unwind: { type: Addr32Nb, offset: 8, symbol: unwind_a } } } + pdata_b: { name: ".pdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "000000000000000000000000" }, relocations: { first: { type: Addr32Nb, offset: 0, symbol: fn_b_symbol }, last: { type: Addr32Nb, offset: 4, symbol: fn_b_symbol }, unwind: { type: Addr32Nb, offset: 8, symbol: unwind_b } } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "000000000000000000000000000000000000000000000000" } + relocations: + { + pdata_a: { type: Addr64, offset: 0, symbol: pdata_a_symbol } + pdata_b: { type: Addr64, offset: 8, symbol: pdata_b_symbol } + unwind_a: { type: Addr64, offset: 16, symbol: unwind_a } + } + } + } + symbols: + { + fn_a_definition: { kind: section_definition, section: fn_a, selection: NoDuplicates } + fn_b_definition: { kind: section_definition, section: fn_b, selection: NoDuplicates } + xdata_a_definition: { kind: section_definition, section: xdata_a, selection: NoDuplicates } + xdata_b_definition: { kind: section_definition, section: xdata_b, selection: NoDuplicates } + pdata_a_definition: { kind: section_definition, section: pdata_a, selection: NoDuplicates } + pdata_b_definition: { kind: section_definition, section: pdata_b, selection: NoDuplicates } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + fn_a_symbol: { kind: external_function, name: "fn_a", section: fn_a, value: 0 } + fn_b_symbol: { kind: external_function, name: "fn_b", section: fn_b, value: 0 } + unwind_a: { kind: external, name: "$unwind$a", section: xdata_a, value: 0 } + unwind_b: { kind: external, name: "$unwind$b", section: xdata_b, value: 0 } + pdata_a_symbol: { kind: external, name: "$pdata$a", section: pdata_a, value: 0 } + pdata_b_symbol: { kind: external, name: "$pdata$b", section: pdata_b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_pdata_diff.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, other_section: ".data", other_offset: 8, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true } + } +} diff --git a/src/linker/tests/icf_pdata_xdata_fold.tst b/src/linker/tests/icf_pdata_xdata_fold.tst new file mode 100644 index 000000000..3dcc4453f --- /dev/null +++ b/src/linker/tests/icf_pdata_xdata_fold.tst @@ -0,0 +1,91 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_pdata_xdata_fold.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + fn_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + fn_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + xdata_a: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01000000" } } + xdata_b: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01000000" } } + pdata_a: + { + name: ".pdata" + permissions: (read) + content: initialized_data + alignment: 4 + flags: (link_comdat) + data: { hex: "000000000000000000000000" } + relocations: + { + first: { type: Addr32Nb, offset: 0, symbol: fn_a_symbol } + last: { type: Addr32Nb, offset: 4, symbol: fn_a_symbol } + unwind: { type: Addr32Nb, offset: 8, symbol: unwind_a } + } + } + pdata_b: + { + name: ".pdata" + permissions: (read) + content: initialized_data + alignment: 4 + flags: (link_comdat) + data: { hex: "000000000000000000000000" } + relocations: + { + first: { type: Addr32Nb, offset: 0, symbol: fn_b_symbol } + last: { type: Addr32Nb, offset: 4, symbol: fn_b_symbol } + unwind: { type: Addr32Nb, offset: 8, symbol: unwind_b } + } + } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "0000000000000000000000000000000000000000000000000000000000000000" } + relocations: + { + pdata_a: { type: Addr64, offset: 0, symbol: pdata_a_symbol } + pdata_b: { type: Addr64, offset: 8, symbol: pdata_b_symbol } + unwind_a: { type: Addr64, offset: 16, symbol: unwind_a } + unwind_b: { type: Addr64, offset: 24, symbol: unwind_b } + } + } + } + symbols: + { + fn_a_definition: { kind: section_definition, section: fn_a, selection: NoDuplicates } + fn_b_definition: { kind: section_definition, section: fn_b, selection: NoDuplicates } + xdata_a_definition: { kind: section_definition, section: xdata_a, selection: NoDuplicates } + xdata_b_definition: { kind: section_definition, section: xdata_b, selection: NoDuplicates } + pdata_a_definition: { kind: section_definition, section: pdata_a, selection: NoDuplicates } + pdata_b_definition: { kind: section_definition, section: pdata_b, selection: NoDuplicates } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + fn_a_symbol: { kind: external_function, name: "fn_a", section: fn_a, value: 0 } + fn_b_symbol: { kind: external_function, name: "fn_b", section: fn_b, value: 0 } + unwind_a: { kind: external, name: "$unwind$a", section: xdata_a, value: 0 } + unwind_b: { kind: external, name: "$unwind$b", section: xdata_b, value: 0 } + pdata_a_symbol: { kind: external, name: "$pdata$a", section: pdata_a, value: 0 } + pdata_b_symbol: { kind: external, name: "$pdata$b", section: pdata_b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_pdata_xdata_fold.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, other_section: ".data", other_offset: 8, relation: equal } + expect_pe_word: { artifact: image, section: ".data", offset: 16, type: u64, nonzero: true, other_section: ".data", other_offset: 24, relation: equal } + } +} diff --git a/src/linker/tests/icf_readonly_non_vftable_data_policy.tst b/src/linker/tests/icf_readonly_non_vftable_data_policy.tst new file mode 100644 index 000000000..9a24492d8 --- /dev/null +++ b/src/linker/tests/icf_readonly_non_vftable_data_policy.tst @@ -0,0 +1,58 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_rdata_policy.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + data_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "01020304" } } + data_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "01020304" } } + data_c: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "01020304" } } + data_d: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "01020304" } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "0000000000000000000000000000000000000000000000000000000000000000" } + relocations: + { + a: { type: Addr64, offset: 0, symbol: data_a_symbol } + b: { type: Addr64, offset: 8, symbol: data_b_symbol } + c: { type: Addr64, offset: 16, symbol: data_c_symbol } + d: { type: Addr64, offset: 24, symbol: data_d_symbol } + } + } + } + symbols: + { + data_a_definition: { kind: section_definition, section: data_a, selection: NoDuplicates } + data_b_definition: { kind: section_definition, section: data_b, selection: NoDuplicates } + data_c_definition: { kind: section_definition, section: data_c, selection: Any } + data_d_definition: { kind: section_definition, section: data_d, selection: Any } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + data_a_symbol: { kind: external, name: "data_a", section: data_a, value: 0 } + data_b_symbol: { kind: external, name: "data_b", section: data_b, value: 0 } + data_c_symbol: { kind: external, name: "data_c", section: data_c, value: 0 } + data_d_symbol: { kind: external, name: "data_d", section: data_d, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_rdata_policy.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, other_section: ".data", other_offset: 8, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 16, type: u64, nonzero: true, other_section: ".data", other_offset: 24, relation: equal } + } +} diff --git a/src/linker/tests/icf_reloc_target_symbol_types_do_not_fold.tst b/src/linker/tests/icf_reloc_target_symbol_types_do_not_fold.tst new file mode 100644 index 000000000..dab3f4f96 --- /dev/null +++ b/src/linker/tests/icf_reloc_target_symbol_types_do_not_fold.tst @@ -0,0 +1,109 @@ +// ICF must preserve identical sections with different symbol targets +test: +{ + artifacts: + { + entry_object: + { + file_name: "icf_interp_entry.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } // ret + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "000000000000000000000000000000000000000000000000" } + relocations: + { + regular: { type: Addr64, offset: 0, symbol: fn_regular } + common: { type: Addr64, offset: 8, symbol: fn_common } + absolute: { type: Addr64, offset: 16, symbol: fn_abs } + } + } + } + symbols: + { + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + fn_regular: { kind: undefined, name: "fn_regular" } + fn_common: { kind: undefined, name: "fn_common" } + fn_abs: { kind: undefined, name: "fn_abs" } + } + } } + } + regular_object: + { + file_name: "icf_interp_regular.obj" + coff: { object: + { + machine: x64 + sections: + { + fn: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + // mov rax, target + // ret + data: { hex: "48c7c000000000c3" } + relocations: { target: { type: Rel32, offset: 3, symbol: target } } + } + target: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, data: { hex: "00" } } + } + symbols: + { + fn_definition: { kind: section_definition, section: fn, selection: NoDuplicates } + fn: { kind: external_function, name: "fn_regular", section: fn, value: 0 } + target: { kind: external, name: "target_regular", section: target, value: 0 } + } + } } + } + common_object: + { + file_name: "icf_interp_common.obj" + coff: { object: + { + machine: x64 + sections: { fn: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "48c7c000000000c3" }, relocations: { target: { type: Rel32, offset: 3, symbol: target } } } } + symbols: + { + fn_definition: { kind: section_definition, section: fn, selection: NoDuplicates } + fn: { kind: external_function, name: "fn_common", section: fn, value: 0 } + target: { kind: common, name: "target_common", size: 8 } + } + } } + } + absolute_object: + { + file_name: "icf_interp_abs.obj" + coff: { object: + { + machine: x64 + sections: { fn: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "48c7c000000000c3" }, relocations: { target: { type: Rel32, offset: 3, symbol: target } } } } + symbols: + { + fn_definition: { kind: section_definition, section: fn, selection: NoDuplicates } + fn: { kind: external_function, name: "fn_abs", section: fn, value: 0 } + target: { kind: absolute, name: "target_abs", value: 4660, storage: external } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf icf_interp_entry.obj icf_interp_regular.obj icf_interp_common.obj icf_interp_abs.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, other_section: ".data", other_offset: 8, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, other_section: ".data", other_offset: 16, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, other_section: ".data", other_offset: 16, relation: not_equal } + } +} diff --git a/src/linker/tests/icf_same_but_different.tst b/src/linker/tests/icf_same_but_different.tst new file mode 100644 index 000000000..b91d85d61 --- /dev/null +++ b/src/linker/tests/icf_same_but_different.tst @@ -0,0 +1,97 @@ +test: +{ + artifacts: + { + object: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "e800000000e800000000c3" } + relocations: + { + call_a: { type: Rel32, offset: 1, symbol: a_symbol } + call_b: { type: Rel32, offset: 6, symbol: b_symbol } + } + } + a: + { + name: ".text" + permissions: (read, execute) + content: code + flags: (link_comdat) + // call $ + data: { hex: "e800000000c3" } + relocations: { call_c: { type: Rel32, offset: 1, symbol: c_symbol } } + } + b: + { + name: ".text" + permissions: (read, execute) + content: code + flags: (link_comdat) + // call $ + data: { hex: "e800000000c3" } + relocations: { call_d: { type: Rel32, offset: 1, symbol: d_symbol } } + } + c: + { + name: ".text" + permissions: (read, execute) + content: code + flags: (link_comdat) + // mov rax, 1 + // ret + data: { hex: "48c7c001000000c3" } + } + d: + { + name: ".text" + permissions: (read, execute) + content: code + flags: (link_comdat) + // mov rax, 2 + // ret + data: { hex: "48c7c002000000c3" } + } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: NoDuplicates } + b_definition: { kind: section_definition, section: b, selection: NoDuplicates } + c_definition: { kind: section_definition, section: c, selection: NoDuplicates } + d_definition: { kind: section_definition, section: d, selection: NoDuplicates } + entry_symbol: { kind: external, name: "entry", section: entry, value: 0 } + a_symbol: { kind: external, name: "a", section: a, value: 0 } + b_symbol: { kind: external, name: "b", section: b, value: 0 } + c_symbol: { kind: external, name: "c", section: c, value: 0 } + d_symbol: { kind: external, name: "d", section: d, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:icf a.obj", artifact: image } } + steps: + { + // validate output + expect_pe: { artifact: image, expected: { pe: { sections: { ".text": + { + // call a + // call b + // call c + // call d + // mov rax, 1 + // mov rax, 2 + data: e80b000000e816000000c3cccccccccce81b000000c3cccccccccccccccccccce81b000000c3cccccccccccccccccccc48c7c001000000c3cccccccccccccccc48c7c002000000c3 + } } } } } + } +} diff --git a/src/linker/tests/icf_unresolved_reloc_targets_do_not_fold.tst b/src/linker/tests/icf_unresolved_reloc_targets_do_not_fold.tst new file mode 100644 index 000000000..19a5c4520 --- /dev/null +++ b/src/linker/tests/icf_unresolved_reloc_targets_do_not_fold.tst @@ -0,0 +1,91 @@ +// ICF must preserve sections with unresolved target symbols (with /FORCE) +test: +{ + artifacts: + { + entry_object: + { + file_name: "icf_unresolved_entry.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } // ret + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000000000000000000000000000" } + relocations: + { + undef: { type: Addr64, offset: 0, symbol: fn_undef } + weak: { type: Addr64, offset: 8, symbol: fn_weak } + } + } + } + symbols: + { + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + fn_undef: { kind: undefined, name: "fn_undef" } + fn_weak: { kind: undefined, name: "fn_weak" } + } + } } + } + undef_object: + { + file_name: "icf_unresolved_undef.obj" + coff: { object: + { + machine: x64 + sections: + { + fn: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + // mov rax, target + // ret + data: { hex: "48c7c000000000c3" } + relocations: { target: { type: Rel32, offset: 3, symbol: target } } + } + } + symbols: + { + fn_definition: { kind: section_definition, section: fn, selection: NoDuplicates } + fn: { kind: external_function, name: "fn_undef", section: fn, value: 0 } + target: { kind: undefined, name: "target_undef" } + } + } } + } + weak_object: + { + file_name: "icf_unresolved_weak.obj" + coff: { object: + { + machine: x64 + sections: { fn: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "48c7c000000000c3" }, relocations: { target: { type: Rel32, offset: 3, symbol: target } } } } + symbols: + { + fn_definition: { kind: section_definition, section: fn, selection: NoDuplicates } + fn: { kind: external_function, name: "fn_weak", section: fn, value: 0 } + fallback: { kind: absolute, name: "target_weak_fallback", value: 0, storage: external } + target: { kind: weak, name: "target_weak", fallback: fallback, search: no_library } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/force /subsystem:console /entry:entry /out:a.exe /opt:ref,icf icf_unresolved_entry.obj icf_unresolved_undef.obj icf_unresolved_weak.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true, other_section: ".data", other_offset: 0, relation: not_equal } + } +} diff --git a/src/linker/tests/icf_vftable_and_vbtable_policy.tst b/src/linker/tests/icf_vftable_and_vbtable_policy.tst new file mode 100644 index 000000000..23252aaef --- /dev/null +++ b/src/linker/tests/icf_vftable_and_vbtable_policy.tst @@ -0,0 +1,79 @@ +test: +{ + artifacts: + { + object: + { + file_name: "icf_tables.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + vf_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" } } + vf_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" } } + vb_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" } } + vb_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" } } + rtti_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, data: { hex: "01" } } + rtti_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, data: { hex: "02" } } + vf_ref_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" }, relocations: { rtti: { type: Addr64, offset: 0, symbol: rtti_a_symbol } } } + vf_ref_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" }, relocations: { rtti: { type: Addr64, offset: 0, symbol: rtti_b_symbol } } } + vb_ref_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" }, relocations: { rtti: { type: Addr64, offset: 0, symbol: rtti_a_symbol } } } + vb_ref_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0000000000000000" }, relocations: { rtti: { type: Addr64, offset: 0, symbol: rtti_b_symbol } } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" } + relocations: + { + vf_a: { type: Addr64, offset: 0, symbol: vf_a_symbol } + vf_b: { type: Addr64, offset: 8, symbol: vf_b_symbol } + vb_a: { type: Addr64, offset: 16, symbol: vb_a_symbol } + vb_b: { type: Addr64, offset: 24, symbol: vb_b_symbol } + vf_ref_a: { type: Addr64, offset: 32, symbol: vf_ref_a_symbol } + vf_ref_b: { type: Addr64, offset: 40, symbol: vf_ref_b_symbol } + vb_ref_a: { type: Addr64, offset: 48, symbol: vb_ref_a_symbol } + vb_ref_b: { type: Addr64, offset: 56, symbol: vb_ref_b_symbol } + } + } + } + symbols: + { + vf_a_definition: { kind: section_definition, section: vf_a, selection: NoDuplicates } + vf_b_definition: { kind: section_definition, section: vf_b, selection: NoDuplicates } + vb_a_definition: { kind: section_definition, section: vb_a, selection: NoDuplicates } + vb_b_definition: { kind: section_definition, section: vb_b, selection: NoDuplicates } + vf_ref_a_definition: { kind: section_definition, section: vf_ref_a, selection: NoDuplicates } + vf_ref_b_definition: { kind: section_definition, section: vf_ref_b, selection: NoDuplicates } + vb_ref_a_definition: { kind: section_definition, section: vb_ref_a, selection: NoDuplicates } + vb_ref_b_definition: { kind: section_definition, section: vb_ref_b, selection: NoDuplicates } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + vf_a_symbol: { kind: external, name: "??_7a@@6B@", section: vf_a, value: 0 } + vf_b_symbol: { kind: external, name: "??_7b@@6B@", section: vf_b, value: 0 } + vb_a_symbol: { kind: external, name: "??_8a@@7B@", section: vb_a, value: 0 } + vb_b_symbol: { kind: external, name: "??_8b@@7B@", section: vb_b, value: 0 } + vf_ref_a_symbol: { kind: external, name: "??_7ra@@6B@", section: vf_ref_a, value: 0 } + vf_ref_b_symbol: { kind: external, name: "??_7rb@@6B@", section: vf_ref_b, value: 0 } + vb_ref_a_symbol: { kind: external, name: "??_8ra@@7B@", section: vb_ref_a, value: 0 } + vb_ref_b_symbol: { kind: external, name: "??_8rb@@7B@", section: vb_ref_b, value: 0 } + rtti_a_symbol: { kind: external, name: "rtti_a", section: rtti_a, value: 0 } + rtti_b_symbol: { kind: external, name: "rtti_b", section: rtti_b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_tables.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, other_section: ".data", other_offset: 8, relation: equal } + expect_pe_word: { artifact: image, section: ".data", offset: 16, type: u64, other_section: ".data", other_offset: 24, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 32, type: u64, other_section: ".data", other_offset: 40, relation: not_equal } + expect_pe_word: { artifact: image, section: ".data", offset: 48, type: u64, other_section: ".data", other_offset: 56, relation: not_equal } + } +} diff --git a/src/linker/tests/icf_vftable_external_symbol_at_nonzero_offset.tst b/src/linker/tests/icf_vftable_external_symbol_at_nonzero_offset.tst new file mode 100644 index 000000000..770f42219 --- /dev/null +++ b/src/linker/tests/icf_vftable_external_symbol_at_nonzero_offset.tst @@ -0,0 +1,63 @@ +test: +{ + artifacts: + { + vftable_obj: + { + file_name: "vftable.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { concat: { hex: "488d0500000000" // lea rax, [rip + ??_7B] + hex: "c3" // ret + } } + relocations: { vftable_b_ref: { type: Rel32, offset: 3, symbol: vftable_b } } + } + vftable_a: { name: ".rdata$vt", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } + vftable_b: { name: ".rdata$vt", permissions: (read), content: initialized_data, alignment: 8, flags: (link_comdat), data: { zero: 16 } } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 8 + data: { zero: 16 } + relocations: + { + vftable_a_ref: { type: Addr64, offset: 0, symbol: vftable_a } + vftable_b_ref: { type: Addr64, offset: 8, symbol: vftable_b } + } + } + } + symbols: + { + vftable_a_def: { kind: section_definition, section: vftable_a, selection: Any } + vftable_b_def: { kind: section_definition, section: vftable_b, selection: Any } + rtti_a: { kind: external, name: "??_R4A@@6B@", section: vftable_a, value: 0 } + vftable_a: { kind: external, name: "??_7A@@6B@", section: vftable_a, value: 8 } + rtti_b: { kind: external, name: "??_R4B@@6B@", section: vftable_b, value: 0 } + vftable_b: { kind: external, name: "??_7B@@6B@", section: vftable_b, value: 8 } + entry: { kind: external_function, name: "entry", section: entry, value: 0 } + addresses: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses vftable.obj", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".rdata": {}, ".data": {}, ".text": {} } } } } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true, modulo: 16, remainder: 8 } + expect_pe_word: { artifact: image, section: ".data", offset: 8, type: u64, nonzero: true, modulo: 16, remainder: 8 } + expect_pe_word: { artifact: image, section: ".text", offset: 3, type: rel32, other_section: ".data", other_offset: 8, other_type: u64, relation: equal } + } +} diff --git a/src/linker/tests/image_base.tst b/src/linker/tests/image_base.tst new file mode 100644 index 000000000..ff21821ce --- /dev/null +++ b/src/linker/tests/image_base.tst @@ -0,0 +1,45 @@ +test: +{ + artifacts: + { + image_base_obj: + { + file_name: "image_base.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: + { + concat: + { + hex: "488d0d00000000" // lea rcx, [__ImageBase] + hex: "48b80000000000000000" // mov rax, __ImageBase + hex: "b800000000" // mov eax, __ImageBase + hex: "c3" // ret + } + } + relocations: + { + relative: { type: Rel32, offset: 3, symbol: image_base } + absolute_64: { type: Addr64, offset: 9, symbol: image_base } + image_relative_32: { type: Addr32Nb, offset: 18, symbol: image_base } + } + } } + symbols: + { + image_base: { kind: undefined, name: "__ImageBase" } + entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/subsystem:console /entry:my_entry /base:0x2000000140000000 /out:a.exe image_base.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 488d0df9efffff48b80000004001000020b800000000c3 } } } } } } +} diff --git a/src/linker/tests/import_export.tst b/src/linker/tests/import_export.tst new file mode 100644 index 000000000..8030fb360 --- /dev/null +++ b/src/linker/tests/import_export.tst @@ -0,0 +1,293 @@ +test: +{ + // write objs + artifacts: + { + import_obj: + { + file_name: "import.obj" + coff: { object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { zero: 1024 } + relocations: + { + imp_foo_ref: { type: Addr32Nb, offset: 0, symbol: imp_foo } + imp_bar_ref: { type: Addr32Nb, offset: 4, symbol: imp_bar } + imp_baz_ref: { type: Addr32Nb, offset: 8, symbol: imp_baz } + imp_baf_ref: { type: Addr32Nb, offset: 12, symbol: imp_baf } + imp_ord_ref: { type: Addr32Nb, offset: 16, symbol: imp_ord } + bar_ref: { type: Addr32Nb, offset: 20, symbol: bar } + foo_ref: { type: Addr32Nb, offset: 24, symbol: foo } + ord_ref: { type: Addr32Nb, offset: 28, symbol: ord } + } + } + } + symbols: + { + imp_foo: { kind: undefined, name: "__imp_foo" } + imp_bar: { kind: undefined, name: "__imp_bar" } + imp_baz: { kind: undefined, name: "__imp_baz" } + imp_baf: { kind: undefined, name: "__imp_baf" } + imp_ord: { kind: undefined, name: "__imp_ord" } + bar: { kind: undefined, name: "bar" } + foo: { kind: undefined, name: "foo" } + ord: { kind: undefined, name: "ord" } + //"baf", + //"baz", + //"__imp_ord2", + //"__imp_ord4", + } + } } + } + + export_obj: + { + file_name: "export.obj" + coff: { object: + { + machine: x64 + sections: + { + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "test" } } + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "b801000000c3" } } + } + symbols: + { + entry: { kind: external, name: "_DllMainCRTStartup", section: text, value: 0 } + foo: { kind: external, name: "foo", section: data, value: 0 } + ord: { kind: external, name: "ord", section: data, value: 1 } + ord2: { kind: external, name: "ord2", section: data, value: 2 } + ord3: { kind: external, name: "ord3", section: data, value: 9 } + ord4: { kind: external, name: "ord4", section: data, value: 10 } + } + directives: + { + directive: "/export:foo=foo" + directive: "/export:bar=foo" + directive: "/export:ord,@5" + directive: "/export:ord2,@6,DATA" + directive: "/export:ord3,@7,NONAME,PRIVATE" + directive: "/export:ord4,@8,NONAME,DATA" + directive: "/export:baz=BAZ.qwe" + directive: "/export:baf=BAZ.#1" + } + } } + } + + baz_obj: + { + file_name: "baz.obj" + coff: { object: + { + machine: x64 + sections: + { + s1: { name: ".s1", permissions: (read, write), content: initialized_data, data: { text: "s1" } } + s2: { name: ".s2", permissions: (read, write), content: initialized_data, data: { text: "s2" } } + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "_DllMainCRTStartup", section: text, value: 0 } + s1: { kind: external, name: "s1", section: s1, value: 0 } + s2: { kind: external, name: "s2", section: s2, value: 0 } + } + directives: + { + directive: "/export:baf=s1" + directive: "/export:baz=s2" + } + } } + } + + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + + export_dll: { file_name: "export.dll", pe: {} } + baz_dll: { file_name: "baz.dll", pe: {} } + + loader_source: + { + file_name: "import_export_loader.c" + text: { data: { concat: + { + text: "#include " + hex: "0a" + text: "int main(void) {" + hex: "0a" + text: " if (!SetDllDirectoryA(" + hex: "22" + text: "." + hex: "22" + text: ")) return 1;" + hex: "0a" + text: " HMODULE export_dll = LoadLibraryA(" + hex: "22" + text: "export.dll" + hex: "22" + text: ");" + hex: "0a" + text: " if (!export_dll) return 2;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, " + hex: "22" + text: "bar" + hex: "22" + text: ")) return 3;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, " + hex: "22" + text: "foo" + hex: "22" + text: ")) return 4;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, " + hex: "22" + text: "ord" + hex: "22" + text: ")) return 5;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, " + hex: "22" + text: "ord2" + hex: "22" + text: ")) return 6;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, MAKEINTRESOURCEA(10))) return 7;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, MAKEINTRESOURCEA(12))) return 8;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, MAKEINTRESOURCEA(5))) return 9;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, MAKEINTRESOURCEA(6))) return 10;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, MAKEINTRESOURCEA(7))) return 11;" + hex: "0a" + text: " if (!GetProcAddress(export_dll, MAKEINTRESOURCEA(8))) return 12;" + hex: "0a" + text: " FreeLibrary(export_dll);" + hex: "0a" + text: " return 0;" + hex: "0a" + text: "}" + hex: "0a" + } } } + } + } + + // link dlls + build: + { + link: { args: "/dll /nodefaultlib /out:export.dll export.obj", artifact: export_dll } // export.dll + link: { args: "/dll /out:baz.dll /export:s1,@1,NONAME /export:qwe=s2 baz.obj", artifact: baz_dll } // baz.dll + compile_link: { tool: msvc, output: "import_export_loader.exe", args: "/nologo import_export_loader.c" } + } + + steps: + { + // validate export table in export.dll + // validate export table in export.dll + expect_pe: + { + artifact: export_dll + expected: + { + pe: + { + sections: { ".data": { virtual_offset: 12288 } } + exports: + { + // validate header + flags: 0 + timestamp: 4294967295 + major_version: 0 + minor_version: 0 + ordinal_base: 5 + count: 8 + entries: + { + // validate names + export_0: { name: "baf", forwarder: "BAZ.#1", ordinal: 9 } + export_1: { name: "bar", forwarder: "", virtual_offset: 12288, ordinal: 10 } + export_2: { name: "baz", forwarder: "BAZ.qwe", ordinal: 11 } + export_3: { name: "foo", forwarder: "", virtual_offset: 12288, ordinal: 12 } + export_4: { name: "ord", forwarder: "", virtual_offset: 12289, ordinal: 5 } + export_5: { name: "ord2", forwarder: "", virtual_offset: 12290, ordinal: 6 } + export_6: { name: "", forwarder: "", virtual_offset: 12297, ordinal: 7 } + export_7: { name: "", forwarder: "", virtual_offset: 12298, ordinal: 8 } + // validate forwarders + // validate voffs + // validate ordinals + } + } + } + } + } + + // validate export table in baz.dll + expect_pe: + { + artifact: baz_dll + expected: + { + pe: + { + exports: + { + // validate header + flags: 0 + timestamp: 4294967295 + major_version: 0 + minor_version: 0 + ordinal_base: 1 + count: 4 + entries: + { + // validate names + export_0: { name: "baf", forwarder: "", virtual_offset: 12288, ordinal: 2 } + export_1: { name: "baz", forwarder: "", virtual_offset: 16384, ordinal: 3 } + export_2: { name: "qwe", forwarder: "", virtual_offset: 16384, ordinal: 4 } + export_3: { name: "", forwarder: "", virtual_offset: 12288, ordinal: 1 } + // validate forwarders + // validate voffs + // validate ordinals + } + } + } + } + } + + // test query by function name + //T_Ok(GetProcAddress(export_dll, "baf")); + //T_Ok(GetProcAddress(export_dll, "baz")); + // test query by ordinal + //T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(9))); + //T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(11))); + run: { path: "import_export_loader.exe" } + + //T_Ok(t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /delayload:export.dll /export:entry kernel32.Lib delayimp.lib libcmt.lib export.lib import.obj entry.obj") == 0); + // TODO: check import table + } +} diff --git a/src/linker/tests/import_kernel32.tst b/src/linker/tests/import_kernel32.tst new file mode 100644 index 000000000..05850aeaf --- /dev/null +++ b/src/linker/tests/import_kernel32.tst @@ -0,0 +1,118 @@ +test: +{ + + artifacts: + { + import_obj: + { + file_name: "import.obj" + coff: { object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { hex: "7465737400" } + } + text: + { + name: ".text" + permissions: (read, execute) + content: code + // sub rsp,68h ; alloc space on stack + // mov dword ptr [rsp+48h],18h ; SECURITY_ATTRIBUTES.nLength + // mov qword ptr [rsp+50h],0 ; SECURITY_ATTRIBUTES.lpSecurityDescriptor + // mov dword ptr [rsp+58h],0 ; SECURITY_ATTRIBUTES.bInheritHandle + // mov qword ptr [rsp+30h],0 ; hTemplateFile + // mov dword ptr [rsp+28h],80h ; dwFlagsAndAttributes + // mov dword ptr [rsp+20h],2 ; dwCreationDisposition + // lea r9,[rsp+48h] ; lpSecurityAttributes + // xor r8d,r8d ; dwShareMode + // mov edx,40000000h ; dwDesiredAccess + // lea rcx,[test] ; lpFileName + // call qword ptr [__imp_CreateFileA] ; call CreateFileA + // mov rcx,rax ; hObject + // call qword ptr [__imp_CloseHandle] ; call CloseHandle + // xor eax,eax ; clear result + // add rsp,68h ; dealloc stack + // ret ; return + data: { hex: "4883ec68c74424481800000048c744245000000000c74424580000000048c744243000000000c744242880000000c7442420020000004c8d4c24484533c0ba00000040488d0d00000000ff15000000004889c1ff150000000033c04883c468c3" } + relocations: + { + test_ref: { type: Rel32, offset: 70, symbol: test } + create_file_ref: { type: Rel32, offset: 76, symbol: create_file } + close_handle_ref: { type: Rel32, offset: 85, symbol: close_handle } + } + } + dead: + { + name: ".text$dead" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + data: { zero: 4 } + relocations: + { + compare_string_ref: { type: Addr32Nb, offset: 0, symbol: compare_string } + } + } + } + symbols: + { + dead_definition: { kind: section_definition, section: dead, selection: Any } + test: { kind: external, name: "test", section: data, value: 0 } + entry: { kind: external, name: "entry", section: text, value: 0 } + dead: { kind: external_function, name: "dead", section: dead, value: 0 } + create_file: { kind: undefined, name: "__imp_CreateFileA" } + close_handle: { kind: undefined, name: "__imp_CloseHandle" } + compare_string: { kind: undefined, name: "__imp_CompareStringW" } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /fixed /opt:ref import.obj kernel32.lib" + artifact: image + } + } + + steps: + { + expect_pe: + { + artifact: image + // The two exact entries prove CreateFileA and CloseHandle are present and CompareStringW is absent. + expected: + { + pe: + { + imports: + { + count: 1 + dll_0: + { + count: 2 + entries: + { + import_0: { type: name, name: "CloseHandle" } + import_1: { type: name, name: "CreateFileA" } + } + } + } + } + } + } + run: { path: "a.exe" } + expect_file: { path: "test", nonempty: false } + } +} diff --git a/src/linker/tests/include.tst b/src/linker/tests/include.tst new file mode 100644 index 000000000..7c6a461cc --- /dev/null +++ b/src/linker/tests/include.tst @@ -0,0 +1,41 @@ +test: +{ + artifacts: + { + include_lib: + { + file_name: "include.lib" + coff: { library: { second_linker_member: true, members: { include_member: { + path: "include.obj" + object: { machine: x64, + sections: { data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "foo" } } } + symbols: { foo: { kind: external, name: "foo", section: data, value: 0 } } + } + } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + data: { hex: "48c7c000000000c3" } + // ret + relocations: { entry_ref: { type: Addr32Nb, offset: 0, symbol: entry } } + } }, symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + // simple include test + link: { args: "/subsystem:console /entry:entry /out:a.exe /include:foo entry.obj include.lib", artifact: image } + // test unresolved include + link: { args: "/subsystem:console /entry:entry /out:a.exe /include:ewq entry.obj", expect_exit: 47 } + } + steps: + { + // validate that linker pulled-in include.obj + expect_pe: { artifact: image, expected: { pe: { sections: { ".data": { data: 666f6f } } } } } + } +} diff --git a/src/linker/tests/infer_asan.tst b/src/linker/tests/infer_asan.tst new file mode 100644 index 000000000..51786f3c1 --- /dev/null +++ b/src/linker/tests/infer_asan.tst @@ -0,0 +1,52 @@ +test: +{ + artifacts: + { + source: + { + file_name: "main.c" + text: { data: { concat: + { + text: "#include " + hex: "0a" + text: " int main(void) {" + hex: "0a" + text: "int *foo = malloc(sizeof(*foo));" + hex: "0a" + text: "free(foo);" + hex: "0a" + text: "*foo = 1;" + hex: "0a" + text: "}" + hex: "0a" + } } } + } + } + + build: + { + // /MD + compile: { tool: cl, output: none, args: "/MD /fsanitize=address /Z7 /c /Fo:main_md.obj main.c" } + link: { output: none, args: "main_md.obj /debug:full" } + + // /MDd + compile: { tool: cl, output: none, args: "/MDd /fsanitize=address /Z7 /c /Fo:main_mdd.obj main.c" } + link: { output: none, args: "main_mdd.obj /debug:full" } + + // /MT + compile: { tool: cl, output: none, args: "/MT /fsanitize=address /Z7 /c /Fo:main_mt.obj main.c" } + link: { output: none, args: "main_mt.obj /debug:full" } + + // /MTd + compile: { tool: cl, output: none, args: "/MT /fsanitize=address /Z7 /c /Fo:main_mtd.obj main.c" } + link: { output: none, args: "main_mtd.obj /debug:full" } + } + + steps: + { + run: { path: "main_md.exe", expect_exit: nonzero, stderr_matches: "=================================================================*AddressSanitizer: heap-use-after-free on address*" } + run: { path: "main_mdd.exe", expect_exit: nonzero, stderr_matches: "=================================================================*AddressSanitizer: heap-use-after-free on address*" } + run: { path: "main_mt.exe", expect_exit: nonzero, stderr_matches: "=================================================================*AddressSanitizer: heap-use-after-free on address*" } + run: { path: "main_mtd.exe", expect_exit: nonzero, stderr_matches: "=================================================================*AddressSanitizer: heap-use-after-free on address*" } + } +} diff --git a/src/linker/tests/invalid_bss.tst b/src/linker/tests/invalid_bss.tst new file mode 100644 index 000000000..c8b84ccf1 --- /dev/null +++ b/src/linker/tests/invalid_bss.tst @@ -0,0 +1,49 @@ +test: +{ + artifacts: + { + bss_obj: + { + file_name: "bss.obj" + coff: { object: + { + machine: x64 + sections: { bss: { name: ".bss", permissions: (read), content: initialized_data, data: { text: "Hello, World" } } } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe bss.obj entry.obj", artifact: image } } + steps: + { + expect_pe: + { + artifact: image + expected: + { + pe: + { + sections: + { + ".bss": + { + virtual_size: 12 + raw_flags: 1073741888 + data: 48656c6c6f2c20576f726c64 + } + } + } + } + } + } +} diff --git a/src/linker/tests/lib_member_imp_and_regular_symbol_queued_once.tst b/src/linker/tests/lib_member_imp_and_regular_symbol_queued_once.tst new file mode 100644 index 000000000..ea60c799f --- /dev/null +++ b/src/linker/tests/lib_member_imp_and_regular_symbol_queued_once.tst @@ -0,0 +1,38 @@ +test: +{ + artifacts: + { + rust_style_lib: + { + file_name: "rust_style.rlib" + coff: { library: { second_linker_member: true, members: { rust_member: { + path: "core-9f9efb2036858c45.core.78298229696da45f-cgu.0.rcgu.o" + object: { machine: x64, sections: { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + idata: { name: ".idata", permissions: (read), content: initialized_data, alignment: 8, data: { zero: 8 } } + }, symbols: { + foo: { kind: external_function, name: "foo", section: text, value: 0 } + imp_foo: { kind: external, name: "__imp_foo", section: idata, value: 0 } + } } + } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, alignment: 1 + data: { hex: "48c7c00000000048c7c100000000c3" } + relocations: { + foo_ref: { type: Addr32Nb, offset: 3, symbol: foo } + imp_foo_ref: { type: Addr32Nb, offset: 10, symbol: imp_foo } + } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + foo: { kind: undefined, name: "foo" } + imp_foo: { kind: undefined, name: "__imp_foo" } + } } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj rust_style.rlib" } } + steps: {} +} diff --git a/src/linker/tests/lib_member_reloc_apply_off_out_of_bounds.tst b/src/linker/tests/lib_member_reloc_apply_off_out_of_bounds.tst new file mode 100644 index 000000000..22de23107 --- /dev/null +++ b/src/linker/tests/lib_member_reloc_apply_off_out_of_bounds.tst @@ -0,0 +1,55 @@ +test: +{ + artifacts: + { + bad_lib: + { + file_name: "bad.lib" + coff: { library: + { + second_linker_member: true + members: { malformed: + { + path: "bad_member.obj" + object: + { + machine: x64 + sections: { data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { hex: "00000000" } + relocations: { target: { type: Addr32, offset: 4294967295, symbol: target } } + } } + symbols: + { + target: { kind: absolute, name: "target", value: 0, storage: static } + bad: { kind: external, name: "bad", section: data, value: 0 } + } + } + } } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:archive_relocation_offset_out_of_bounds.exe /include:bad entry.obj bad.lib" + expect_exit: nonzero + } + } + steps: {} +} diff --git a/src/linker/tests/link_undef.tst b/src/linker/tests/link_undef.tst new file mode 100644 index 000000000..7363a9781 --- /dev/null +++ b/src/linker/tests/link_undef.tst @@ -0,0 +1,24 @@ +test: +{ + artifacts: + { + undef_obj: { file_name: "undef.obj", coff: { object: { machine: x64, symbols: { undef: { kind: undefined, name: "undef" } } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "48c7c000000000c3" } + relocations: { undef_ref: { type: Addr32Nb, offset: 0, symbol: undef } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + undef: { kind: undefined, name: "undef" } + } } } + } + } + build: + { + // try linking unresolved symbol and see if linker picks up on that + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj undef.obj", expect_exit: 47 } + } + steps: {} +} diff --git a/src/linker/tests/link_unref_undef.tst b/src/linker/tests/link_unref_undef.tst new file mode 100644 index 000000000..0ae636f23 --- /dev/null +++ b/src/linker/tests/link_unref_undef.tst @@ -0,0 +1,20 @@ +test: +{ + artifacts: + { + undef_obj: { file_name: "undef.obj", coff: { object: { machine: x64, symbols: { undef: { kind: undefined, name: "undef" } } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } + } }, symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } } } + } + } + build: + { + // try linking unreferenced unresolved symbol, this must link + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj undef.obj", expect_exit: nonzero } + } + steps: {} +} diff --git a/src/linker/tests/linker_tests.c b/src/linker/tests/linker_tests.c index c0b463317..f2fd59885 100644 --- a/src/linker/tests/linker_tests.c +++ b/src/linker/tests/linker_tests.c @@ -1,19 +1,6 @@ // Copyright (c) Epic Games Tools // Licensed under the MIT license (https://opensource.org/license/mit/) -// TODO: -// [x] defer_duplicate_imp_link -// [x] fold_two_funcs -// [x] same_but_different -// [x] fold_diamond -// [x] cyclic_icf -// [x] fold_with_largest_align -// [ ] opt_ref_comdat_undef_section -// [ ] opt_ref_weak_alias_comdat -// [ ] reloc_apply_off_out_of_bounds -// [ ] lib_member_reloc_apply_off_out_of_bounds -// [ ] relocate_undefined_section_symbol - //////////////////////////////// // Def -> COFF @@ -141,22 +128,6 @@ typedef struct T_COFF_DefLibMember *members; } T_COFF_DefLib; -typedef enum -{ - T_COFF_DefRootType_Null, - T_COFF_DefRootType_Obj, - T_COFF_DefRootType_Lib -} T_COFF_DefRootType; - -typedef struct -{ - T_COFF_DefRootType type; - union { - T_COFF_DefObj obj; - T_COFF_DefLib lib; - }; -} T_COFF_DefRoot; - internal COFF_ObjSection * t_coff_from_def_require_section(HashTable *section_ht, String8 id) { @@ -347,18 +318,6 @@ t_coff_from_def_lib(Arena *arena, T_COFF_DefLib lib) return result; } -internal String8 -t_coff_from_def_root(Arena *arena, T_COFF_DefRoot root) -{ - switch (root.type) { - case T_COFF_DefRootType_Obj: return t_coff_from_def_obj(arena, root.obj); - case T_COFF_DefRootType_Lib: return t_coff_from_def_lib(arena, root.lib); - case T_COFF_DefRootType_Null: break; - default: { NotImplemented; } break; - } - return str8_zero(); -} - internal B32 t_write_def_obj(char *path, T_COFF_DefObj obj) { @@ -368,43 +327,8 @@ t_write_def_obj(char *path, T_COFF_DefObj obj) return is_ok; } -internal B32 -t_write_def_lib(char *path, T_COFF_DefLib lib) -{ - Temp scratch = scratch_begin(0,0); - B32 is_ok = t_write_file(str8_cstring(path), t_coff_from_def_lib(scratch.arena, lib)); - scratch_end(scratch); - return is_ok; -} - //////////////////////////////// -internal String8 -t_make_sec_defn_obj(Arena *arena, String8 payload) -{ - return t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "mysect", ".mysect", payload, .flags = "r:data@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("mysect", COFF_ComdatSelect_Null), - {0} - } - }); -} - -internal String8 -t_make_obj_with_directive(Arena *arena, String8 directive) -{ - String8 directive_cstr = push_cstr(arena, directive); - return t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .directives = (char *[]){ (char *)directive_cstr.str, 0 } - }); -} - internal String8 t_make_entry_obj(Arena *arena) { @@ -431,9250 +355,21 @@ t_write_entry_obj(void) return is_ok; } -//////////////////////////////// - -TEST(machine_compat_check) -{ - // unknown.obj - T_Ok(t_write_def_obj("unknown.obj", (T_COFF_DefObj){ - .machine = &(COFF_MachineType){ COFF_MachineType_Unknown }, - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("unknown"), .flags = "rw:data" }, - {0} - } - })); - - // x64.obj - T_Ok(t_write_def_obj("x64.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("x64"), .flags = "rw:data" }, - {0} - } - })); - - // entry.obj - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - // arm64.obj - T_Ok(t_write_def_obj("arm64.obj", (T_COFF_DefObj){ - .machine = &(COFF_MachineType){ COFF_MachineType_Arm64 }, - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("arm64"), .flags = "rw:data" }, - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe entry.obj unknown.obj x64.obj"); - T_Ok(g_last_exit_code == 0); - - // test objs with conflicting machines - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe entry.obj unknown.obj x64.obj arm64.obj"); - T_Ok(g_last_exit_code != 0); - - // check /MACHINE switch - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe /machine:amd64 arm64.obj entry.obj"); - T_Ok(g_last_exit_code != 0); -} - -TEST(simple_link_test) -{ - U8 text_payload[] = { 0xC3 }; - - String8 main_obj; - { - main_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_array_fixed(text_payload), .flags = "rx:code" }, - { "data", ".data", str8_lit("qwe"), .flags = "rw:data" }, - { "zero", ".zero", str8(0, 5), .flags = "rw:bss" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - }); - } - - String8 main_obj_name = str8_lit("main.obj"); - T_Ok(t_write_file(main_obj_name, main_obj)); - - int file_align = 512; - int virt_align = 4096; - String8 out_name = str8_lit("a.exe"); - t_invoke_linkerf("/entry:my_entry /subsystem:console /fixed /filealign:%d /align:%d /out:%S %S", file_align, virt_align, out_name, main_obj_name); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, out_name); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - T_Ok(!pe.is_pe32); - T_Ok(pe.section_count == 3); - T_Ok(pe.arch == Arch_x64); - T_Ok(pe.subsystem == PE_WindowsSubsystem_WINDOWS_CUI); - T_Ok(pe.virt_section_align == virt_align); - T_Ok(pe.file_section_align == file_align); - T_Ok(pe.symbol_count == 0); - T_Ok(pe.data_dir_count == PE_DataDirectoryIndex_COUNT); - - // check section alignment - for EachIndex(sect_idx, pe.section_count) { - COFF_SectionHeader *sect_header = §ion_table[sect_idx]; - T_Ok(AlignPadPow2(sect_header->fsize, file_align) == 0); - T_Ok(AlignPadPow2(sect_header->voff, virt_align) == 0); - } - - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_section != 0); - T_Ok(text_section->foff == file_align); - T_Ok(pe.entry_point == text_section->voff); - - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_section != 0); - - COFF_SectionHeader *zero_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".zero")); - T_Ok(zero_section != 0); - - String8 text_data = str8_substr(exe, rng_1u64(text_section->foff, text_section->foff + text_section->vsize)); - T_Ok(str8_match(text_data, str8_array_fixed(text_payload), 0)); - - PE_OptionalHeader32Plus *opt = str8_deserial_get_raw_ptr(exe, pe.optional_header_off, sizeof(*opt)); - T_Ok(opt->sizeof_code == text_section->fsize); - T_Ok(opt->sizeof_inited_data == data_section->fsize); - T_Ok(opt->sizeof_uninited_data == 0x200); - T_Ok(opt->code_base == 0x1000); - T_Ok(opt->image_base == 0x140000000); - T_Ok(opt->major_os_ver == 6); - T_Ok(opt->minor_os_ver == 0); - T_Ok(opt->major_img_ver == 0); - T_Ok(opt->minor_img_ver == 0); - T_Ok(opt->major_subsystem_ver == 6); - T_Ok(opt->minor_subsystem_ver == 0); - T_Ok(opt->win32_version_value == 0); - T_Ok(opt->sizeof_image == 0x4000); - T_Ok(opt->sizeof_headers == 0x200); - T_Ok(opt->dll_characteristics == 0x8120); - T_Ok(opt->loader_flags == 0); -} - -TEST(map) -{ - T_Ok(t_write_def_obj("map.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("map_entry", "text", 0), - T_COFF_DefSymbol_Static("map_local", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/entry:map_entry /subsystem:console /out:mapped.exe /map:explicit.map map.obj"); - T_Ok(g_last_exit_code == 0); - - String8 map = t_read_file(arena, str8_lit("explicit.map")); - T_Ok(str8_find_needle(map, 0, str8_lit(" Timestamp is "), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit(" Preferred load address is "), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit(" Start Length Name Class"), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit(" Publics by Value"), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit("map_entry"), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit(" entry point at"), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit(" Static symbols"), 0) < map.size); - T_Ok(str8_find_needle(map, 0, str8_lit("map_local"), 0) < map.size); - - t_invoke_linkerf("/entry:map_entry /subsystem:console /out:default.exe /map map.obj"); - T_Ok(g_last_exit_code == 0); - T_Ok(t_read_file(arena, str8_lit("default.map")).size > 0); - - t_invoke_linkerf("/entry:map_entry /subsystem:console /out:collision.exe /map:collision.exe map.obj"); - T_Ok(g_last_exit_code != 0); -} - - -TEST(out_of_bounds_section_number) -{ - // bad.obj - { - String8 obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "foo", ".foo", str8_lit("foo"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("foo", "foo", 0), - {0} - } - }); - { - COFF_FileHeaderInfo header = coff_file_header_info_from_data(obj); - String8 string_table = str8_substr(obj, header.string_table_range); - String8 symbol_table = str8_substr(obj, header.symbol_table_range); - COFF_ParsedSymbol symbol = coff_parse_symbol(header, string_table, symbol_table, 0); - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = 123; - } - T_Ok(t_write_file(str8_lit("bad.obj"), obj)); - } - - // entry.obj - { - String8 obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00\xC3"), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "foo"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("foo"), - {0} - } - }); - T_Ok(t_write_file(str8_lit("entry.obj"), obj)); - } - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj bad.obj"); - T_Ok(g_last_exit_code != 0); -} - - -TEST(merge) -{ - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "test", ".test", str8_lit("hello, world"), .flags = "rw:data" }, - {0} - } - })); - - T_Ok(t_write_entry_obj()); - - // circular merge - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.test=.test entry.obj test.obj"); - T_Ok(g_last_exit_code != 0); - - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_CircularMerge); - } - - // circular merge with extra link - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.test=.data /merge:.data=.test entry.obj test.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_CircularMerge); - } - - // merge with non-defined section - { - g_last_exit_code; - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.test=.qwe entry.obj test.obj"); - T_Ok(g_last_exit_code == 0); - - // make sure linker created .qwe and merged .test into it - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".qwe")); - T_Ok(sect != 0); - T_Ok(sect->flags == PE_DATA_SECTION_FLAGS); - String8 qwe = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(qwe, str8_lit("hello, world"),0)); - } - - // illegal merge with .reloc - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.test=.reloc entry.obj test.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_IllegalSectionMerge); - } - - // illegal merge with .rsrc - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.test=.rsrc entry.obj test.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_IllegalSectionMerge); - } - - // merge non-defined section with defined section - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.qwe=.test entry.obj test.obj"); - T_Ok(g_last_exit_code == 0); - - // initialized data size includes the aligned virtual tail of merged BSS - { - T_Ok(t_write_def_obj("mixed.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit_comp("d"), .flags = "rw:data@1" }, - { "bss", ".bss", str8(0, 0x201), .flags = "rw:bss@1" }, - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.bss=.data entry.obj mixed.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - PE_OptionalHeader32Plus *opt = str8_deserial_get_raw_ptr(exe, pe.optional_header_off, sizeof(*opt)); - COFF_SectionHeader *data = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data != 0); - T_Ok(data->fsize == 0x200); - T_Ok(data->vsize == 0x202); - T_Ok(opt->sizeof_inited_data == 0x400); - } - - // merged contribution groups retain lexical order - { - T_Ok(t_write_def_obj("order.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a$m", str8_lit_comp("a"), .flags = "rw:data@1" }, - { "z", ".z$m", str8_lit_comp("z"), .flags = "rw:data@1" }, - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.a=.z entry.obj order.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".z")); - T_Ok(sect != 0); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + 2)); - T_Ok(str8_match(data, str8_lit("az"), 0)); - } - - // merge .test -> .qwe -> .data - { - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /merge:.test=.qwe /merge:.qwe=.data entry.obj test.obj"); - T_Ok(g_last_exit_code == 0); - - // make sure linker merged .test into .data - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".data")); - T_Ok(sect != 0); - T_Ok(sect->flags == PE_DATA_SECTION_FLAGS); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("hello, world"),0)); - } -} - -TEST(section_directive_read_only_grouped_section) -{ - T_Ok(t_write_entry_obj()); - - T_Ok(t_write_def_obj("prot.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "prot_a", "prot$a", str8_lit_comp("A"), .flags = "rw:data@1" }, - { "prot_mem", "prot$mem", str8_lit_comp("mem"), .flags = "rw:data@1" }, - { "prot_z", "prot$z", str8_lit_comp("Z"), .flags = "rw:data@1" }, - {0} - }, - .directives = (char *[]){ "/SECTION:prot,R", 0 }, - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj prot.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("prot")); - T_Ok(sect != 0); - T_Ok(sect->flags == (COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead)); -} - -TEST(section_directive_align_grouped_section) -{ - if (t_id_linker() != Linker_radlink) { return; } - - T_Ok(t_write_entry_obj()); - - T_Ok(t_write_def_obj("prot.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "prot_mem", "prot$mem", str8_lit_comp("mem"), .flags = "rw:data@1" }, - {0} - }, - .directives = (char *[]){ "/SECTION:prot,R,ALIGN=8192", 0 }, - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj prot.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("prot")); - T_Ok(sect != 0); - T_Ok(sect->flags == (COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_Align8192Bytes)); -} - -TEST(link_undef) -{ - T_Ok(t_write_def_obj("undef.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("undef"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00\xC3"), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "undef"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("undef"), - {0} - } - })); - - // try linking unresolved symbol and see if linker picks up on that - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj undef.obj"); - T_Ok(g_last_exit_code == LNK_Error_UnresolvedSymbol); -} - -TEST(link_unref_undef) -{ - T_Ok(t_write_def_obj("undef.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("undef"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - // try linking unreferenced unresolved symbol, this must link - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj undef.obj"); - T_Ok(g_last_exit_code != 0); -} - -TEST(weak_lib_vs_weak_lib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "q"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "e"), - {0} - } - })); - - // linker must pick weak symbol from a.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x111; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } - - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_lib_vs_weak_nolib) +internal String8 +data_from_pdb(Arena *arena, PDB_Context *pdb) { - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "q"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "e"), - {0} - } - })); - - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} + TP_Context *tp = tp_alloc(arena, 1, 1, str8_lit("foo")); + TP_Arena *tp_arena = tp_arena_alloc(tp); + pdb_build(tp, tp_arena, pdb, (CV_StringHashTable){0}, 1, 0, 0); -TEST(weak_lib_vs_weak_alias) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchAlias, "q"), - {0} - } - })); + AssertAlways(msf_build(pdb->msf) == MSF_Error_OK); + String8List raw_msf_list = msf_get_page_data_nodes(arena, pdb->msf); + AssertAlways(t_write_file_list(str8_lit("test.pdb"), raw_msf_list)); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "e"), - {0} - } - })); + String8 data = str8_list_join(arena, &raw_msf_list, 0); - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); -} + tp_arena_release(&tp_arena); + tp_release(tp); -TEST(weak_lib_vs_weak_antidep) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "q"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "e"), - {0} - } - })); - - // linker must pick weak symbol from a.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_alias_vs_weak_alias) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("qwe", 0x111), - T_COFF_DefSymbol_Weak("sym", COFF_WeakExt_SearchAlias, "qwe"), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("ewq", 0x222), - T_COFF_DefSymbol_Weak("sym", COFF_WeakExt_SearchAlias, "ewq"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "sym"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("sym"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); -} - -TEST(weak_alias_vs_weak_lib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "q"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" - "\xC3" - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchAlias, "e"), - {0} - } - })); - - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_alias_vs_weak_nolib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "q"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" - "\xC3" - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchAlias, "e"), - {0} - } - })); - - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_alias_vs_weak_antidep) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "q"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" - "\xC3" - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchAlias, "e"), - {0} - } - })); - - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_nolib_vs_weak_nolib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_nolib_vs_weak_lib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_nolib_vs_weak_alias) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchAlias, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); -} - -TEST(weak_nolib_vs_weak_antidep) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_antidep_vs_weak_antidep) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "e"), - {0} - } - })); - - // linker must pick weak symbol from a.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x111; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } - - // linker must pick weak symbol from entry.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_antidep_vs_weak_nolib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_NoLibrary, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_antidep_vs_weak_lib) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x222; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_antidep_vs_weak_alias) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("q", 0x111), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchAlias, "q"), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 3, "w"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_AbsExtern("e", 0x222), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_AntiDependency, "e"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - String8 imm = str8_substr(text_data, rng_1u64(3, 7)); - U32 expected = 0x111; - T_Ok(str8_match(imm, str8_struct(&expected), 0)); - } -} - -TEST(weak_vs_common) -{ - T_Ok(t_write_def_obj("weak.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "a", ".a", str8_lit("a"), .flags = "rw:data" }, {0} }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Static("_a", "a", 0), - T_COFF_DefSymbol_Weak("w", COFF_WeakExt_SearchLibrary, "_a"), - {0} - } - })); - T_Ok(t_write_def_obj("common.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Common("w", 2), {0} } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32Nb, 0, "w"), {0} } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("w"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe common.obj weak.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe weak.obj common.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *bss = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".bss")); - T_Ok(bss); - T_Ok(bss->fsize == 0); - T_Ok(bss->vsize == 2); -} - -TEST(abs_vs_weak) -{ - U32 abs_value = 0x123; - U8 text_code[] = { 0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC3 }; - - T_Ok(t_write_def_obj("abs.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("foo", abs_value), - {0}, - } - })); - - T_Ok(t_write_def_obj("text.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "mydata", ".mydata", str8_lit("mydata"), .flags = "rx:code@1" }, - { - "text", ".text", str8_array_fixed(text_code), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 2, "foo"), - {0}, - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("mydata", "mydata", 0), - T_COFF_DefSymbol_Weak("foo", COFF_WeakExt_NoLibrary, "mydata"), - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe abs.obj text.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe text.obj abs.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_section != 0); - - String8 text_data = str8_substr(exe, rng_1u64(text_section->foff, text_section->foff + text_section->fsize)); - String8 inst = str8_prefix(text_data, 2); - T_Ok(str8_match(inst, str8_array(text_code, 2), 0)); - - String8 imm = str8_prefix(str8_skip(text_data, 2), 8); - U64 expected_imm = abs_value; - T_Ok(str8_match(imm, str8_struct(&expected_imm), 0)); -} - -TEST(abs_vs_regular) -{ - String8 shared_symbol_name = str8_lit("foo"); - - U8 regular_payload[] = { 0xC0, 0xFF, 0xEE }; - String8 regular_obj_name = str8_lit("regular.obj"); - - T_Ok(t_write_def_obj("regular.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "data", ".data", str8_array_fixed(regular_payload), .flags = "rw:data" }, {0}, }, - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Extern("foo", "data", 0), {0}, } - })); - - String8 abs_obj_name = str8_lit("abs.obj"); - T_Ok(t_write_def_obj("abs.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_AbsExtern("foo", 0x1234), {0}, } - })); - - U8 entry_text[] = { - 0x48, 0xC7, 0xC0, 0x00, 0x00, 0x00, 0x00, // mov rax, $imm - 0xC3 // ret - }; - String8 entry_obj_name = str8_lit("entry.obj"); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "foo"), - {0}, - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - T_COFF_DefSymbol_Undef("foo"), - {0}, - } - })); - - // TODO: validate that linker issues multiply defined symbol error - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe abs.obj regular.obj entry.obj"); - // linker should complain about multiply defined symbol - T_Ok(g_last_exit_code != 0); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe regular.obj abs.obj entry.obj"); - // linker should complain even in case regular is before abs - T_Ok(g_last_exit_code != 0); -} - -TEST(abs_vs_common) -{ - String8 shared_symbol_name = str8_lit("foo"); - - String8 common_obj_name = str8_lit("common.obj"); - - T_Ok(t_write_def_obj("common.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Common("foo", 321), {0}, } - })); - - String8 abs_obj_name = str8_lit("abs.obj"); - T_Ok(t_write_def_obj("abs.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_AbsExtern("foo", 0x1234), {0}, } - })); - - U8 entry_text[] = { 0xC3 }; - String8 entry_obj_name = str8_lit("entry.obj"); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "text", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, {0}, }, - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Extern("my_entry", "text", 0), {0}, } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe abs.obj common.obj entry.obj"); - if (g_last_exit_code == 0) { - // TODO: validate that linker issues multiply defined symbol error - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe common.obj abs.obj entry.obj"); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); - } else { - T_Ok(g_last_exit_code != 0); - } - } -} - -TEST(abs_vs_abs) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_AbsExtern("foo", 'a'), {0}, }, - })); - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_AbsExtern("foo", 'b'), {0}, }, - })); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); -} - -TEST(undef_weak_lib) -{ - T_Ok(t_write_def_obj("weak.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("b", 0xc3000000), - T_COFF_DefSymbol_Weak("a", COFF_WeakExt_SearchLibrary, "b"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32Nb, 0, "a"), {0} } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("a"), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0}, - } - })); - - // undefined symbol must always replace weak symbol with search library - t_invoke_linkerf("/subsystem:console /out:a.exe /entry:entry entry.obj weak.obj"); - T_Ok(g_last_exit_code == LNK_Error_UnresolvedSymbol); -} - -TEST(undef_weak_search_alias) -{ - Temp scratch = scratch_begin(0,0); - - T_Ok(t_write_def_obj("weak.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "data", ".data", str8_lit_comp("\xde\xad\xbe\xef"), .flags = "rw:data" }, {0} }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("ptr"), - T_COFF_DefSymbol_Weak("foo", COFF_WeakExt_SearchAlias, "ptr"), - {0} - } - })); - - T_Ok(t_write_def_obj("ptr.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("entry"), - T_COFF_DefSymbol_Weak("ptr", COFF_WeakExt_SearchAlias, "entry"), - {0} - } - })); - - T_Ok(t_write_def_obj("undef.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rw:data", - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32Nb, 0, "foo"), {0} } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Undef("foo"), {0} } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, {0} }, - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Extern("entry", "text", 0), {0} } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe weak.obj entry.obj ptr.obj undef.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(weak_cycle) -{ - String8 ab_obj_name = str8_lit("ab.obj"); - - T_Ok(t_write_def_obj("ab.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("B"), - T_COFF_DefSymbol_Weak("A", COFF_WeakExt_SearchAlias, "B"), - {0}, - } - })); - - String8 ba_obj_name = str8_lit("ba.obj"); - T_Ok(t_write_def_obj("ba.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("A"), - T_COFF_DefSymbol_Weak("B", COFF_WeakExt_SearchAlias, "A"), - {0}, - } - })); - - String8 entry_obj_name = str8_lit("entry.obj"); - U8 entry_payload[] = { 0xC3 }; - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "text", ".text", str8_array_fixed(entry_payload), .flags = "rx:code@1" }, {0} }, - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Extern("my_entry", "text", 0), {0} } - })); - - U64 timeout = now_time_us() + 3 * 1000 * 1000; // give a generous 3 seconds - t_invoke_linker_timeoutf(timeout, "/subsystem:console /entry:my_entry %S %S %S", entry_obj_name, ab_obj_name, ba_obj_name); -} - -TEST(weak_tag) -{ - U32 weak_tag_expected_value = 0x12345678; - String8 weak_tag_obj_name = str8_lit("weak_tag.obj"); - - T_Ok(t_write_def_obj("weak_tag.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rw:data", - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32, 0, "strong_second"), {0} } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsStatic("abs", weak_tag_expected_value), - T_COFF_DefSymbol_Weak("strong_first", COFF_WeakExt_SearchAlias, "abs"), - T_COFF_DefSymbol_Weak("strong_second", COFF_WeakExt_SearchAlias, "strong_first"), - {0} - } - })); - - String8 entry_name = str8_lit("my_entry"); - U8 entry_text[] = { 0xC3 }; - String8 entry_obj_name = str8_lit("entry.obj"); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ { "text", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, {0} }, - .symbols = (T_COFF_DefSymbol[]){ T_COFF_DefSymbol_Extern((char *)entry_name.str, "text", 0), {0} } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe %S %S", weak_tag_obj_name, entry_obj_name); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - String8 data = str8_substr(exe, rng_1u64(data_section->foff, data_section->foff + data_section->vsize)); - T_Ok(data_section); - T_Ok(data_section->vsize == 4); - T_Ok(str8_match(data, str8_struct(&weak_tag_expected_value), 0)); -} - -TEST(undef_section) -{ - U8 payload[] = { 1, 2, 3 }; - String8 sec_defn_obj = t_make_sec_defn_obj(arena, str8_array_fixed(payload)); - - T_Ok(t_write_def_obj("main.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rw:data", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, ".mysect"), - {0} - } - }, - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_UndefSec(".mysect", COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead), - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - t_write_file(str8_lit("sec_defn.obj"), sec_defn_obj); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe main.obj sec_defn.obj"); - if (g_last_exit_code == 0) { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - COFF_SectionHeader *mysect_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".mysect")); - if (data_section && mysect_section) { - if (data_section->vsize == 4 && mysect_section->vsize == 3) { - String8 addr32nb = str8_substr(exe, rng_1u64(data_section->foff, data_section->foff + data_section->vsize)); - String8 expected_voff = str8_struct(&mysect_section->voff); - T_Ok(str8_match(addr32nb, expected_voff, 0)); - } - } - } -} - -TEST(sect_symbol) -{ - String8 sect_payload = str8_lit("hello, world"); - T_Ok(t_write_def_obj("sect.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "mysect1", ".mysect$1", sect_payload, .flags = "rw:data@1" }, - {0} - }, - .directives = (char *[]){ "/merge:.mysect=.data", 0 } - })); - - T_Ok(t_write_def_obj("main.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00\x00\x00\x00\x00"), .flags = "rw:data", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, ".mysect$2222"), - {0} - } - }, - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_UndefSec(".mysect$2222", t_coff_section_flags_from_cstr("rw:data")), - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe main.obj sect.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - - T_Ok(sect != 0); - - String8 sect_data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - - String8 addr_data = str8_substr(sect_data, rng_1u64(0, sizeof(U64))); - T_Ok(addr_data.size == sizeof(U64)); - - U64 addr = *(U64 *)addr_data.str; - T_Ok(addr - (pe.image_base + sect->voff) == 8); - - String8 payload_got = str8_substr(sect_data, rng_1u64(8, sect_data.size)); - T_Ok(str8_match(payload_got, sect_payload, 0)); -} - -TEST(undef_reloc_section) -{ - U8 payload[] = { 1, 2, 3 }; - String8 sec_defn_obj = t_make_sec_defn_obj(arena, str8_array_fixed(payload)); - - T_Ok(t_write_def_obj("main.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code" }, - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00\x00\x00\x00\x00"), .flags = "rw:data", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, ".reloc"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - T_COFF_DefSymbol_UndefSec(".reloc", PE_RELOC_SECTION_FLAGS), - {0} - } - })); - T_Ok(t_write_file(str8_lit("sec_defn.obj"), sec_defn_obj)); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe main.obj sec_defn.obj"); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_SectRefsDiscardedMemory); - } else { - T_Ok(g_last_exit_code != 0); - } -} - -TEST(find_merged_pdata) -{ - U8 foobar_payload[] = { - 0x40, 0x57, 0x48, 0x81, 0xEC, 0x00, 0x02, 0x00, 0x00, 0x48, 0x8B, 0x05, 0x00, 0x00, 0x00, 0x00, - 0x48, 0x33, 0xC4, 0x48, 0x89, 0x84, 0x24, 0xF0, 0x01, 0x00, 0x00, 0x48, 0x8D, 0x04, 0x24, 0x48, - 0x8B, 0xF8, 0x33, 0xC0, 0xB9, 0xEC, 0x01, 0x00, 0x00, 0xF3, 0xAA, 0xB8, 0x04, 0x00, 0x00, 0x00, - 0x48, 0x6B, 0xC0, 0x02, 0x8B, 0x04, 0x04, 0x48, 0x8B, 0x8C, 0x24, 0xF0, 0x01, 0x00, 0x00, 0x48, - 0x33, 0xCC, 0xE8, 0x00, 0x00, 0x00, 0x00, 0x48, 0x81, 0xC4, 0x00, 0x02, 0x00, 0x00, 0x5F, 0xC3, - 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, - 0x48, 0x83, 0xEC, 0x28, 0xE8, 0x00, 0x00, 0x00, 0x00, 0x48, 0x83, 0xC4, 0x28, 0xC3 - }; - U8 xdata_payload[] = { - 0x19, 0x1B, 0x03, 0x00, 0x09, 0x01, 0x40, 0x00, 0x02, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xF0, 0x01, 0x00, 0x00, 0x01, 0x04, 0x01, 0x00, 0x04, 0x42, 0x00, 0x00 - }; - PE_IntelPdata intel_pdata = {0}; - U8 text_payload[] = { 0xC3 }; - - T_Ok(t_write_def_obj("main.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "xdata", ".xdata", str8_array_fixed(xdata_payload), .flags = "r:data@4" }, - { - "pdata", ".pdata", str8_struct(&intel_pdata), .flags = "r:data@4", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_unwind_info), "$unwind$foobar"), - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_first), "foobar"), - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_one_past_last), "foobar"), - {0} - } - }, - { "foobar", ".foobar", str8_array_fixed(foobar_payload), .flags = "rx:code@1" }, - { "text", ".text", str8_array_fixed(text_payload), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Static("foobar", "foobar", 0), - T_COFF_DefSymbol_Secdef("xdata", COFF_ComdatSelect_Null), - T_COFF_DefSymbol_Static("$unwind$foobar", "xdata", 0), - T_COFF_DefSymbol_Secdef("pdata", COFF_ComdatSelect_Null), - T_COFF_DefSymbol_Static("$pdata$foobar", "pdata", 0), - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe main.obj /merge:.pdata=.rdata"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - T_Ok(dim_1u64(pe.data_dir_franges[PE_DataDirectoryIndex_EXCEPTIONS]) == 0xC); -} - -TEST(guard_cf_pulls_load_config) -{ - U8 load_config_data[0x40] = {0}; - U32 load_config_size = sizeof(load_config_data); - MemoryCopy(load_config_data, &load_config_size, sizeof(load_config_size)); - - T_Ok(t_write_entry_obj()); - T_Ok(t_write_def_lib("loadcfg.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("loadcfg.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "loadcfg", ".rdata", str8_array_fixed(load_config_data), .flags = "r:data@8" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("_load_config_used", "loadcfg", 0), - {0} - } - } - }, - {0} - } - })); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:a.exe /guard:cf entry.obj loadcfg.lib"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - T_Ok(dim_1u64(pe.data_dir_franges[PE_DataDirectoryIndex_LOAD_CONFIG]) == load_config_size); - - PE_OptionalHeader32Plus *opt = str8_deserial_get_raw_ptr(exe, pe.optional_header_off, sizeof(*opt)); - if (t_id_linker() == Linker_radlink) { - T_Ok(!(opt->dll_characteristics & PE_DllCharacteristic_GUARD_CF)); - } -} - -TEST(section_sort) -{ - COFF_SectionFlags data_flags = COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemRead|COFF_SectionFlag_Align1Bytes; - T_Ok(t_write_def_obj("data.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "idata_2", ".idata$2", str8_lit("last"), .flags = "rw:data@1" }, - { "idata_5", ".idata$5", str8_lit("first"), .flags = "rw:data@1" }, - { "rdata", ".rdata", str8_lit("middle"), .flags = "r:data@1" }, - { "data_z", ".data$z", str8_lit("five"), .raw_flags = data_flags }, - { "data_a", ".data$a", str8_lit("three"), .raw_flags = data_flags }, - { "data_bbbbb", ".data$bbbbb", str8_lit("four"), .raw_flags = data_flags }, - { "data_empty", ".data$", str8_lit("two"), .raw_flags = data_flags }, - { "data", ".data", str8_lit("one"), .raw_flags = data_flags }, - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe data.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_section); - - String8 data = str8_substr(exe, rng_1u64(data_section->foff, data_section->foff + data_section->vsize)); - String8 expected_data = str8_lit("onetwothreefourfive"); - T_Ok(str8_match(data, expected_data, 0)); - - COFF_SectionHeader *rdata_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - T_Ok(rdata_section); - - String8 rdata = str8_substr(exe, rng_1u64(rdata_section->foff, rdata_section->foff + 15)); - T_Ok(str8_match(rdata, str8_lit("firstmiddlelast"), 0)); -} - -TEST(flag_conf) -{ - COFF_SectionFlags my_sect0_flags = COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemExecute; - COFF_SectionFlags my_sect1_flags = COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemWrite; - T_Ok(t_write_def_obj("conf.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".mysect", str8_lit("one"), .raw_flags = my_sect0_flags }, - { "b", ".mysect", str8_lit("two"), .raw_flags = my_sect1_flags }, - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe conf.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeaderArray my_sects = coff_section_header_array_from_name(arena, string_table, section_table, pe.section_count, str8_lit(".mysect")); - T_Ok(my_sects.count == 2); - - COFF_SectionHeader *my_sect0 = &my_sects.v[0]; - COFF_SectionHeader *my_sect1 = &my_sects.v[1]; - T_Ok(my_sect0->flags == my_sect0_flags); - T_Ok(my_sect1->flags == my_sect1_flags); -} - -TEST(invalid_bss) -{ - COFF_SectionFlags bss_flags = COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead; - String8 bss_data = str8_lit("Hello, World"); - T_Ok(t_write_def_obj("bss.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "bss", ".bss", bss_data, .raw_flags = bss_flags }, - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe bss.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *bss_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".bss")); - T_Ok(bss_sect != 0); - T_Ok(bss_sect->vsize == 0xC); - T_Ok(bss_sect->flags == bss_flags); - String8 data = str8_substr(exe, rng_1u64(bss_sect->foff, bss_sect->foff + bss_sect->vsize)); - T_Ok(str8_match(data, bss_data, 0)); -} - -TEST(common_block) -{ - U8 a_data[6] = {0}; - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_array_fixed(a_data), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 0, "A"), - {0} - } - }, - { "bss", ".bss", str8(0, 1), .flags = "rw:bss" }, // shift common block's initial position - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Common("A", 3), - {0} - } - })); - - U8 b_data[9] = { 0 }; - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_array_fixed(b_data), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "B"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Common("B", 6), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe /fixed /largeaddressaware:no /merge:.bss=.comm a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - COFF_SectionHeader *comm_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".comm")); - COFF_SectionHeader *data_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(comm_sect != 0); - T_Ok(data_sect != 0); - - // blocks must be sorted in descending order to reduce alignment padding - T_Ok(comm_sect->vsize == 0x13); - - // ensure linker correctly patched addresses for symbols pointing into common block - String8 data = str8_substr(exe, rng_1u64(data_sect->foff, data_sect->foff + data_sect->fsize)); - U32 *a_addr = (U32 *)data.str; - U64 *b_addr = (U64 *)(data.str + sizeof(a_data)); - T_Ok(*a_addr == (pe.image_base + comm_sect->voff + 0x10)); - T_Ok(*b_addr == (pe.image_base + comm_sect->voff + 0x8)); -} - -TEST(base_relocs) -{ - // main.obj - String8 entry_name = str8_lit("my_entry"); - U64 mov_func_name64 = 2; - U64 mov_func_name32 = 16; - U8 main_text[] = { - 0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov rax, func_name - 0xff, 0xd0, // call rax - 0x48, 0x31, 0xc0, // xor rax, rax - 0xb8, 0x00, 0x00, 0x00, 0x00, // mov eax, func_name - 0xff, 0xd0, // call rax - 0xc3 // ret - }; - - // func.obj - String8 func_name = str8_lit("foo"); - U8 func_text[] = { 0xc3 }; - - // linker must not produce base relocations for absolute symbol - T_Ok(t_write_def_obj("main.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_array_fixed(main_text), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, mov_func_name64, "foo"), - T_COFF_DefReloc(X64_Addr32, mov_func_name32, "foo"), - {0} - } - }, - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rw:data", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, 0, "abs"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("foo"), - T_COFF_DefSymbol_AbsStatic("abs", 0x12345678), - T_COFF_DefSymbol_Extern((char *)entry_name.str, "text", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("func.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_array_fixed(func_text), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern((char *)func_name.str, "text", 0), - {0} - } - })); - - String8 out_name = str8_lit("a.exe"); - t_invoke_linkerf("/subsystem:console /entry:my_entry /dynamicbase /largeaddressaware:no /out:a.exe main.obj func.obj"); - T_Ok(g_last_exit_code == 0); - - // it is illegal to merge .reloc with other sections - t_invoke_linkerf("/subsystem:console /entry:my_entry /dynamicbase /largeaddressaware:no /out:a.exe /merge:.reloc=.rdata main.obj func.obj"); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_IllegalSectionMerge); - } else { - T_Ok(g_last_exit_code != 0); - } - - // the other way around is illegal too - t_invoke_linkerf("/subsystem:console /entry:my_entry /dynamicbase /largeaddressaware:no /out:a.exe /merge:.rdata=.reloc main.obj func.obj"); - if (t_id_linker() == Linker_radlink) { - T_Ok(g_last_exit_code == LNK_Error_IllegalSectionMerge); - } else { - T_Ok(g_last_exit_code != 0); - } -} - -TEST(simple_lib_test) -{ - String8 test_payload = str8_lit("The quick brown fox jumps over the lazy dog"); - - T_Ok(t_write_def_lib("test.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("test.obj"), - .machine = T_COFF_DefSetMachine(Unknown), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8(test_payload.str, test_payload.size+1), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("test", "data", 0), - {0} - } - } - }, - {0} - } - })); - - U8 entry_text[] = { - 0x48, 0xC7, 0xC0, 0x00, 0x00, 0x00, 0x00, - 0xC3 - }; - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "test"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("test"), - T_COFF_DefSymbol_Extern("my_entry", "text", 7), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe entry.obj test.lib"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - COFF_SectionHeader *data_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->fsize)); - String8 data_data = str8_substr(exe, rng_1u64(data_sect->foff, data_sect->foff + data_sect->fsize)); - - // was test payload linked? - String8 data_string = str8_cstring_capped(data_data.str, data_data.str + data_data.size); - T_Ok(str8_match(data_string, test_payload, 0)); - - // do we have enough bytes to read text? - T_Ok(text_data.size >= sizeof(entry_text)); - - // linker must pull-in test.obj and patch relocation for "test" symbol - U32 *data_addr32nb = (U32 *)(text_data.str+3); - T_Ok(*data_addr32nb == data_sect->voff); -} - -TEST(lib_member_imp_and_regular_symbol_queued_once) -{ - T_Ok(t_write_def_lib("rust_style.rlib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("core-9f9efb2036858c45.core.78298229696da45f-cgu.0.rcgu.o"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code@1" }, - { "idata", ".idata", str8_lit_comp("\x00\x00\x00\x00\x00\x00\x00\x00"), .flags = "r:data@8" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("foo", "text", 0), - T_COFF_DefSymbol_Extern("__imp_foo", "idata", 0), - {0} - } - } - }, - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" - "\x48\xC7\xC1\x00\x00\x00\x00" - "\xC3" - ), - .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "foo"), - T_COFF_DefReloc(X64_Addr32Nb, 10, "__imp_foo"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("foo"), - T_COFF_DefSymbol_Undef("__imp_foo"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj rust_style.rlib"); - T_Ok(g_last_exit_code == 0); -} - -#if OS_WINDOWS -TEST(import_export) -{ - // write objs - T_Ok(t_write_def_obj("import.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - .id = "data", - .name = ".data", - .flags = "rw:data", - .data = str8_array_fixed((U8[1024]){0}), - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0*4, "__imp_foo"), - T_COFF_DefReloc(X64_Addr32Nb, 1*4, "__imp_bar"), - T_COFF_DefReloc(X64_Addr32Nb, 2*4, "__imp_baz"), - T_COFF_DefReloc(X64_Addr32Nb, 3*4, "__imp_baf"), - T_COFF_DefReloc(X64_Addr32Nb, 4*4, "__imp_ord"), - T_COFF_DefReloc(X64_Addr32Nb, 5*4, "bar"), - T_COFF_DefReloc(X64_Addr32Nb, 6*4, "foo"), - T_COFF_DefReloc(X64_Addr32Nb, 7*4, "ord"), - {0}, - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("__imp_foo"), - T_COFF_DefSymbol_Undef("__imp_bar"), - T_COFF_DefSymbol_Undef("__imp_baz"), - T_COFF_DefSymbol_Undef("__imp_baf"), - T_COFF_DefSymbol_Undef("__imp_ord"), - T_COFF_DefSymbol_Undef("bar"), - T_COFF_DefSymbol_Undef("foo"), - T_COFF_DefSymbol_Undef("ord"), - //"baf", - //"baz", - //"__imp_ord2", - //"__imp_ord4", - {0}, - }, - })); - - T_Ok(t_write_def_obj("export.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("test"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("foo", "data", 0), - T_COFF_DefSymbol_Extern("ord", "data", 1), - T_COFF_DefSymbol_Extern("ord2", "data", 2), - T_COFF_DefSymbol_Extern("ord3", "data", 9), - T_COFF_DefSymbol_Extern("ord4", "data", 10), - {0} - }, - .directives = (char*[]){ - "/export:foo=foo", - "/export:bar=foo", - "/export:ord,@5", - "/export:ord2,@6,DATA", - "/export:ord3,@7,NONAME,PRIVATE", - "/export:ord4,@8,NONAME,DATA", - "/export:baz=BAZ.qwe", - "/export:baf=BAZ.#1", - 0, - } - })); - - T_Ok(t_write_def_obj("baz.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "s1", ".s1", str8_lit("s1"), .flags = "rw:data" }, - { "s2", ".s2", str8_lit("s2"), .flags = "rw:data" }, - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("_DllMainCRTStartup", "text", 0), - T_COFF_DefSymbol_Extern("s1", "s1", 0), - T_COFF_DefSymbol_Extern("s2", "s2", 0), - {0} - }, - .directives = (char*[]){ - "/export:baf=s1", - "/export:baz=s2", - 0, - } - })); - - T_Ok(t_write_entry_obj()); - - // link dlls - t_invoke_linkerf("/dll /out:export.dll libcmt.lib export.obj"); // export.dll - T_Ok(g_last_exit_code == 0); - t_invoke_linkerf("/dll /out:baz.dll /export:s1,@1,NONAME /export:qwe=s2 baz.obj"); // baz.dll - T_Ok(g_last_exit_code == 0); - - // validate export table in export.dll - if (t_id_linker() == Linker_radlink) { - // validate export table in export.dll - { - String8 dll = t_read_file(arena, str8_lit("export.dll")); - PE_BinInfo pe = pe_bin_info_from_data(arena, dll); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(dll, pe.section_table_range).str; - PE_ParsedExportTable export_table = pe_exports_from_data(arena, pe.section_count, section_table, dll, pe.data_dir_franges[PE_DataDirectoryIndex_EXPORT], pe.data_dir_vranges[PE_DataDirectoryIndex_EXPORT]); - COFF_SectionHeader *data_sect = coff_section_header_from_name(str8_zero(), section_table, pe.section_count, str8_lit(".data")); - - // validate header - T_Ok(export_table.flags == 0); - T_Ok(export_table.time_stamp == COFF_TimeStamp_Max); - T_Ok(export_table.major_ver == 0); - T_Ok(export_table.minor_ver == 0); - T_Ok(export_table.ordinal_base == 5); - T_Ok(export_table.export_count == 8); - - // validate names - T_Ok(str8_match(export_table.exports[0].name, str8_lit("baf"), 0)); - T_Ok(str8_match(export_table.exports[1].name, str8_lit("bar"), 0)); - T_Ok(str8_match(export_table.exports[2].name, str8_lit("baz"), 0)); - T_Ok(str8_match(export_table.exports[3].name, str8_lit("foo"), 0)); - T_Ok(str8_match(export_table.exports[4].name, str8_lit("ord"), 0)); - T_Ok(str8_match(export_table.exports[5].name, str8_lit("ord2"), 0)); - T_Ok(export_table.exports[6].name.size == 0); - T_Ok(export_table.exports[7].name.size == 0); - - // validate forwarders - T_Ok(str8_match(export_table.exports[0].forwarder, str8_lit("BAZ.#1"), 0)); - T_Ok(export_table.exports[1].forwarder.size == 0); - T_Ok(str8_match(export_table.exports[2].forwarder, str8_lit("BAZ.qwe"), 0)); - T_Ok(export_table.exports[3].forwarder.size == 0); - T_Ok(export_table.exports[4].forwarder.size == 0); - T_Ok(export_table.exports[5].forwarder.size == 0); - T_Ok(export_table.exports[6].forwarder.size == 0); - T_Ok(export_table.exports[7].forwarder.size == 0); - - // validate voffs - T_Ok(export_table.exports[1].voff == data_sect->voff + 0x0); - T_Ok(export_table.exports[3].voff == data_sect->voff + 0x0); - T_Ok(export_table.exports[4].voff == data_sect->voff + 0x1); - T_Ok(export_table.exports[5].voff == data_sect->voff + 0x2); - T_Ok(export_table.exports[6].voff == data_sect->voff + 0x9); - T_Ok(export_table.exports[7].voff == data_sect->voff + 0xa); - - // validate ordinals - T_Ok(export_table.exports[0].ordinal == 9); - T_Ok(export_table.exports[1].ordinal == 10); - T_Ok(export_table.exports[2].ordinal == 11); - T_Ok(export_table.exports[3].ordinal == 12); - T_Ok(export_table.exports[4].ordinal == 5); - T_Ok(export_table.exports[5].ordinal == 6); - T_Ok(export_table.exports[6].ordinal == 7); - T_Ok(export_table.exports[7].ordinal == 8); - } - - // validate export table in baz.dll - { - String8 dll = t_read_file(arena, str8_lit("baz.dll")); - PE_BinInfo pe = pe_bin_info_from_data(arena, dll); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(dll, pe.section_table_range).str; - PE_ParsedExportTable export_table = pe_exports_from_data(arena, pe.section_count, section_table, dll, pe.data_dir_franges[PE_DataDirectoryIndex_EXPORT], pe.data_dir_vranges[PE_DataDirectoryIndex_EXPORT]); - - // validate header - T_Ok(export_table.flags == 0); - T_Ok(export_table.time_stamp == COFF_TimeStamp_Max); - T_Ok(export_table.major_ver == 0); - T_Ok(export_table.minor_ver == 0); - T_Ok(export_table.ordinal_base == 1); - T_Ok(export_table.export_count == 4); - - // validate names - T_Ok(str8_match(export_table.exports[0].name, str8_lit("baf"), 0)); - T_Ok(str8_match(export_table.exports[1].name, str8_lit("baz"), 0)); - T_Ok(str8_match(export_table.exports[2].name, str8_lit("qwe"), 0)); - T_Ok(str8_match(export_table.exports[3].name, str8_zero(), 0)); - - // validate forwarders - T_Ok(str8_match(export_table.exports[0].forwarder, str8_zero(), 0)); - T_Ok(str8_match(export_table.exports[1].forwarder, str8_zero(), 0)); - T_Ok(str8_match(export_table.exports[2].forwarder, str8_zero(), 0)); - T_Ok(str8_match(export_table.exports[3].forwarder, str8_zero(), 0)); - - // validate voffs - T_Ok(export_table.exports[0].voff == 0x3000); - T_Ok(export_table.exports[1].voff == 0x4000); - T_Ok(export_table.exports[2].voff == 0x4000); - T_Ok(export_table.exports[3].voff == 0x3000); - - // validate ordinals - T_Ok(export_table.exports[0].ordinal == 2); - T_Ok(export_table.exports[1].ordinal == 3); - T_Ok(export_table.exports[2].ordinal == 4); - T_Ok(export_table.exports[3].ordinal == 1); - } - } - - { - T_Ok(SetDllDirectoryA((LPCSTR)g_wdir.str)); - HANDLE export_dll = LoadLibrary("export.dll"); - T_Ok(export_dll); - - // test query by function name - //T_Ok(GetProcAddress(export_dll, "baf")); - T_Ok(GetProcAddress(export_dll, "bar")); - //T_Ok(GetProcAddress(export_dll, "baz")); - T_Ok(GetProcAddress(export_dll, "foo")); - T_Ok(GetProcAddress(export_dll, "ord")); - T_Ok(GetProcAddress(export_dll, "ord2")); - - // test query by ordinal - //T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(9))); - T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(10))); - //T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(11))); - T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(12))); - T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(5))); - T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(6))); - T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(7))); - T_Ok(GetProcAddress(export_dll, MAKEINTRESOURCE(8))); - } - - //T_Ok(t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /delayload:export.dll /export:entry kernel32.Lib delayimp.lib libcmt.lib export.lib import.obj entry.obj") == 0); - // TODO: check import table -} - -TEST(def_file_full) -{ - if (t_id_linker() == Linker_lld) { return; } - - T_Ok(t_write_def_obj("def_full.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".rdata", str8_lit("test"), .flags = "rw:data" }, - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Extern("foo", "data", 0), - {0} - }, - })); - - T_Ok(t_write_file(str8_lit("full.def"), str8_lit( - "; leading comment\n" - "NAME \"def full.exe\" BASE=0x140020000\n" - "VERSION 7.8\n" - "HEAPSIZE 0x30000, 0x4000\n" - "STACKSIZE 0x50000,0x6000\n" - "SECTIONS .rdata READ\n" - "EXPORTS foo @ 2 DATA\n"))); - - t_invoke_linkerf("/subsystem:console /entry:entry /def:full.def def_full.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("def full.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - PE_OptionalHeader32Plus *opt = str8_deserial_get_raw_ptr(exe, pe.optional_header_off, sizeof(*opt)); - - T_Ok(opt->image_base == 0x140020000); - T_Ok(opt->major_img_ver == 7); - T_Ok(opt->minor_img_ver == 8); - T_Ok(opt->sizeof_heap_reserve == 0x30000); - T_Ok(opt->sizeof_heap_commit == 0x4000); - T_Ok(opt->sizeof_stack_reserve == 0x50000); - T_Ok(opt->sizeof_stack_commit == 0x6000); - - COFF_SectionHeader *rdata = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - T_Ok(rdata != 0); - T_Ok(rdata->flags == (COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead)); - - PE_ParsedExportTable export_table = pe_exports_from_data(arena, pe.section_count, section_table, exe, pe.data_dir_franges[PE_DataDirectoryIndex_EXPORT], pe.data_dir_vranges[PE_DataDirectoryIndex_EXPORT]); - T_Ok(export_table.export_count == 1); - T_Ok(str8_match(export_table.exports[0].name, str8_lit("foo"), 0)); - T_Ok(export_table.exports[0].ordinal == 2); - - T_Ok(t_write_file(str8_lit("bad_base_space.def"), str8_lit( - "NAME bad_base_space BASE = 0x140020000\n" - "EXPORTS foo @2 DATA\n"))); - T_Ok(t_write_file(str8_lit("bad_base_colon.def"), str8_lit( - "NAME bad_base_colon BASE:0x140020000\n" - "EXPORTS foo @2 DATA\n"))); - T_Ok(t_write_file(str8_lit("bad_section_align.def"), str8_lit( - "NAME bad_section_align\n" - "SECTIONS .rdata READ ALIGN=8192\n" - "EXPORTS foo @2 DATA\n"))); - - t_invoke_linkerf("/subsystem:console /entry:entry /def:bad_base_space.def /out:bad_base_space.exe def_full.obj"); - T_Ok(g_last_exit_code != 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /def:bad_base_colon.def /out:bad_base_colon.exe def_full.obj"); - T_Ok(g_last_exit_code != 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /def:bad_section_align.def /out:bad_section_align.exe def_full.obj"); - T_Ok(g_last_exit_code != 0); - - T_Ok(t_write_def_obj("def_full_dll.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("test"), .flags = "rw:data" }, - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("_DllMainCRTStartup", "text", 0), - T_COFF_DefSymbol_Extern("dll_foo", "data", 0), - {0} - }, - })); - - T_Ok(t_write_file(str8_lit("full_dll.def"), str8_lit( - "LIBRARY folded BASE=0x180020000\n" - "EXPORTS\n" - " dll_foo DATA\n"))); - - t_invoke_linkerf("/dll /subsystem:console /def:full_dll.def def_full_dll.obj"); - T_Ok(g_last_exit_code == 0); - - String8 dll = t_read_file(arena, str8_lit("folded.dll")); - PE_BinInfo dll_pe = pe_bin_info_from_data(arena, dll); - COFF_SectionHeader *dll_section_table = (COFF_SectionHeader *)str8_substr(dll, dll_pe.section_table_range).str; - PE_OptionalHeader32Plus *dll_opt = str8_deserial_get_raw_ptr(dll, dll_pe.optional_header_off, sizeof(*dll_opt)); - PE_ParsedExportTable dll_export_table = pe_exports_from_data(arena, dll_pe.section_count, dll_section_table, dll, dll_pe.data_dir_franges[PE_DataDirectoryIndex_EXPORT], dll_pe.data_dir_vranges[PE_DataDirectoryIndex_EXPORT]); - - T_Ok(dll_opt->image_base == 0x180020000); - T_Ok(dll_export_table.export_count == 1); - T_Ok(str8_match(dll_export_table.exports[0].name, str8_lit("dll_foo"), 0)); -} - -TEST(utf16_rsp) -{ - T_Ok(t_write_entry_obj()); - - String8 rsp_text = str8_lit("/subsystem:console /entry:entry /out:a.exe entry.obj\n"); - String16 rsp16 = str16_from_8(arena, rsp_text); - String8 rsp_file = str8_cat(arena, str8_lit("\xff\xfe"), str8_array(rsp16.str, rsp16.size)); - T_Ok(t_write_file(str8_lit("args.rsp"), rsp_file)); - - t_invoke_linkerf("@args.rsp"); - T_Ok(g_last_exit_code == 0); -} -#endif - -TEST(image_base) -{ - T_Ok(t_write_def_obj("image_base.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", - ".text", - str8_lit_comp( - "\x48\x8D\x0D\x00\x00\x00\x00" // lea rcx, [__ImageBase] - "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x00" // mov rax, __ImageBase - "\xB8\x00\x00\x00\x00" // mov eax, __ImageBase - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "__ImageBase"), - T_COFF_DefReloc(X64_Addr64, 9, "__ImageBase"), - T_COFF_DefReloc(X64_Addr32Nb, 18, "__ImageBase"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("__ImageBase"), - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /base:0x2000000140000000 /out:a.exe image_base.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_section); - - U8 expected_text[] = { - 0x48, 0x8D, 0x0D, 0xF9, 0xEF, 0xFF, 0xFF, - 0x48, 0xB8, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x20, - 0xB8, 0x00, 0x00, 0x00, 0x00, - 0xC3 - }; - String8 text_data = str8_substr(exe, rng_1u64(text_section->foff, text_section->foff + sizeof(expected_text))); - T_Ok(str8_match(text_data, str8_array_fixed(expected_text), 0)); -} - -TEST(comdat_any) -{ - T_Ok(t_write_def_obj("1.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "test", ".test$mn", str8_lit("1"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("test", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_ExternFunc("TEST", "test", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("2.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "test", ".test$mn", str8_lit("2"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("test", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("TEST", "test", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3"), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - { - t_invoke_linkerf("/subsystem:console /entry:entry /out:1.exe 1.obj 2.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - String8 exe = t_read_file(arena, str8_lit("1.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".test")); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("1"), 0)); - } - - { - t_invoke_linkerf("/subsystem:console /entry:entry /out:2.exe 2.obj 1.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - String8 exe = t_read_file(arena, str8_lit("2.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".test")); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("2"), 0)); - } -} - -// MSVC vftables use COMDAT sections whose public symbol can start past the -// section definition symbol; references to a replaced copy must target the winner. -TEST(comdat_external_symbol_at_nonzero_offset) -{ - U8 data[16] = {0}; - U8 ptr[8] = {0}; - U8 text[] = { - 0x48, 0x8D, 0x05, 0, 0, 0, 0, // lea rax, [rip + foo] - 0xC3 - }; - - T_Ok(t_write_def_obj("leader.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".rdata", str8_array_fixed(data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("foo", "data", 8), - {0} - } - })); - - T_Ok(t_write_def_obj("ref.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".rdata", str8_array_fixed(data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "ptr", ".data", str8_array_fixed(ptr), .flags = "rw:data@8", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "foo"), - {0} - }}, - { "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "foo"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("foo", "data", 8), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:a.exe leader.obj ref.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *rdata_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(rdata_section != 0); - T_Ok(data_section != 0); - T_Ok(text_section != 0); - - U64 actual_ptr = 0; - str8_deserial_read_struct(exe, data_section->foff, &actual_ptr); - U64 expected_ptr = pe.image_base + rdata_section->voff + 8; - T_Ok(actual_ptr == expected_ptr); - - S32 lea_disp = 0; - str8_deserial_read_struct(exe, text_section->foff + 3, &lea_disp); - U64 actual_lea_target = text_section->voff + 7 + lea_disp; - U64 expected_lea_target = rdata_section->voff + 8; - T_Ok(actual_lea_target == expected_lea_target); -} - -TEST(comdat_external_symbol_at_zero_offset) -{ - U8 data[8] = {0}; - U8 ptr[8] = {0}; - U8 text[] = { - 0x48, 0x8D, 0x05, 0, 0, 0, 0, // lea rax, [rip + foo] - 0xC3 - }; - - T_Ok(t_write_def_obj("leader.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".rdata", str8_array_fixed(data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("foo", "data", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("ref.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".rdata", str8_array_fixed(data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "ptr", ".data", str8_array_fixed(ptr), .flags = "rw:data@8", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "foo"), - {0} - }}, - { "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "foo"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("foo", "data", 0), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:a.exe leader.obj ref.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *rdata_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(rdata_section != 0); - T_Ok(data_section != 0); - T_Ok(text_section != 0); - - U64 actual_ptr = 0; - str8_deserial_read_struct(exe, data_section->foff, &actual_ptr); - U64 expected_ptr = pe.image_base + rdata_section->voff; - T_Ok(actual_ptr == expected_ptr); - - S32 lea_disp = 0; - str8_deserial_read_struct(exe, text_section->foff + 3, &lea_disp); - U64 actual_lea_target = text_section->voff + 7 + lea_disp; - U64 expected_lea_target = rdata_section->voff; - T_Ok(actual_lea_target == expected_lea_target); -} - -// Duplicate COMDAT sections can have identical bytes while their symbol tables -// disagree about where a same-named public symbol points inside the section. -// This mirrors MSVC vftable COMDATs: the selected copy may have leading RTTI data -// at offset 0 and the vftable symbol at offset 8, while a discarded copy's -// vftable symbol is at offset 0. Relocations against the discarded symbol must -// use the selected symbol's value, not just the selected section contribution -// plus the discarded symbol's original offset. -TEST(comdat_external_symbol_uses_leader_offset) -{ - U8 leader_data[16] = {0}; - U8 ref_data[16] = {0}; - U8 ptr[8] = {0}; - U8 text[] = { - 0x48, 0x8D, 0x05, 0, 0, 0, 0, // lea rax, [rip + ??_7X] - 0xC3 - }; - - T_Ok(t_write_def_obj("leader.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "vftable", ".rdata", str8_array_fixed(leader_data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("vftable", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("??_7X@@6B@", "vftable", 8), - {0} - } - })); - - T_Ok(t_write_def_obj("ref.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "vftable", ".rdata", str8_array_fixed(ref_data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "ptr", ".data", str8_array_fixed(ptr), .flags = "rw:data@8", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "??_7X@@6B@"), - {0} - }}, - { "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "??_7X@@6B@"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("vftable", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("??_7X@@6B@", "vftable", 0), - T_COFF_DefSymbol_ExternFunc("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:a.exe leader.obj ref.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *rdata_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(rdata_section != 0); - T_Ok(data_section != 0); - T_Ok(text_section != 0); - - U64 actual_ptr = 0; - str8_deserial_read_struct(exe, data_section->foff, &actual_ptr); - U64 expected_ptr = pe.image_base + rdata_section->voff + 8; - T_Ok(actual_ptr == expected_ptr); - - S32 lea_disp = 0; - str8_deserial_read_struct(exe, text_section->foff + 3, &lea_disp); - U64 actual_lea_target = text_section->voff + 7 + lea_disp; - U64 expected_lea_target = rdata_section->voff + 8; - T_Ok(actual_lea_target == expected_lea_target); -} - -// Chromium has duplicate vftable COMDATs where the referencing copy is an -// IMAGE_COMDAT_SELECT_ANY section with the public vftable at offset 0, while -// the selected IMAGE_COMDAT_SELECT_LARGEST copy has the same public symbol at -// offset 8. Relocations in the discarded object must resolve to the selected -// public symbol, not to the discarded section. -TEST(comdat_largest_external_symbol_uses_selected_offset) -{ - U8 discarded_data[16] = {0}; - U8 selected_data[24] = {0}; - U8 ptr[16] = {0}; - U8 selected_text[] = { 0xC3 }; - U8 text[] = { - 0x48, 0x8D, 0x05, 0, 0, 0, 0, // lea rax, [rip + ??_7X] - 0xC3 - }; - - T_Ok(t_write_def_obj("discarded.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "vftable", ".rdata", str8_array_fixed(discarded_data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "ptr", ".data", str8_array_fixed(ptr), .flags = "rw:data@8", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "??_7X@@6B@"), - T_COFF_DefReloc(X64_Addr64, 8, "force_selected"), - {0} - }}, - { "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "??_7X@@6B@"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("vftable", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("??_7X@@6B@", "vftable", 0), - T_COFF_DefSymbol_Undef("force_selected"), - T_COFF_DefSymbol_ExternFunc("entry", "text", 0), - {0} - } - })); - - T_Ok(t_write_def_lib("selected.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("selected.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "vftable", ".rdata", str8_array_fixed(selected_data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "force_text", ".text", str8_array_fixed(selected_text), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("vftable", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("??_7X@@6B@", "vftable", 8), - T_COFF_DefSymbol_ExternFunc("force_selected", "force_text", 0), - {0} - } - } - }, - {0} - } - })); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:a.exe /opt:ref,noicf discarded.obj selected.lib"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *rdata_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(rdata_section != 0); - T_Ok(data_section != 0); - T_Ok(text_section != 0); - - U64 actual_ptr = 0; - str8_deserial_read_struct(exe, data_section->foff, &actual_ptr); - U64 expected_ptr = pe.image_base + rdata_section->voff + 8; - T_Ok(actual_ptr == expected_ptr); - - S32 lea_disp = 0; - str8_deserial_read_struct(exe, text_section->foff + 3, &lea_disp); - U64 actual_lea_target = text_section->voff + 7 + lea_disp; - U64 expected_lea_target = rdata_section->voff + 8; - T_Ok(actual_lea_target == expected_lea_target); -} - -TEST(icf_vftable_external_symbol_at_nonzero_offset) -{ - U8 entry_text[] = { - 0x48, 0x8D, 0x05, 0, 0, 0, 0, // lea rax, [rip + ??_7B] - 0xC3, // ret - }; - U8 vftable_data[16] = {0}; - U8 addresses[16] = {0}; - - T_Ok(t_write_def_obj("vftable.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "??_7B@@6B@"), - {0} - }}, - { "vftable_a", ".rdata$vt", str8_array_fixed(vftable_data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "vftable_b", ".rdata$vt", str8_array_fixed(vftable_data), .flags = "r:data@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@8", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "??_7A@@6B@"), - T_COFF_DefReloc(X64_Addr64, 8, "??_7B@@6B@"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("vftable_a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("vftable_b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("??_R4A@@6B@", "vftable_a", 0), - T_COFF_DefSymbol_Extern("??_7A@@6B@", "vftable_a", 8), - T_COFF_DefSymbol_Extern("??_R4B@@6B@", "vftable_b", 0), - T_COFF_DefSymbol_Extern("??_7B@@6B@", "vftable_b", 8), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses vftable.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *rdata_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".rdata")); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(rdata_section != 0); - T_Ok(data_section != 0); - T_Ok(text_section != 0); - - U64 a_vftable_ptr = 0; - U64 b_vftable_ptr = 0; - str8_deserial_read_struct(exe, data_section->foff + 0, &a_vftable_ptr); - str8_deserial_read_struct(exe, data_section->foff + 8, &b_vftable_ptr); - T_Ok(a_vftable_ptr != 0); - T_Ok(b_vftable_ptr != 0); - T_Ok((a_vftable_ptr - pe.image_base - rdata_section->voff) % sizeof(vftable_data) == 8); - T_Ok((b_vftable_ptr - pe.image_base - rdata_section->voff) % sizeof(vftable_data) == 8); - - S32 lea_disp = 0; - str8_deserial_read_struct(exe, text_section->foff + 3, &lea_disp); - U64 actual_lea_target = text_section->voff + 7 + lea_disp; - T_Ok(actual_lea_target == b_vftable_ptr - pe.image_base); -} - -// A referenced zero-sized COMDAT symbol is meaningful enough for -// relocations, even though the COMDAT contributes no bytes to the image. -TEST(zero_length_comdat_referenced_by_reloc) -{ - U8 data[8] = {0}; - - T_Ok(t_write_entry_obj()); - T_Ok(t_write_def_obj("ref.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "pad", ".rdata$a", str8_lit("xy"), .flags = "r:data@1" }, - { "empty", ".rdata$b", str8_zero(), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "data", ".data", str8_array_fixed(data), .flags = "rw:data", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "EMPTY"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("empty", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("EMPTY", "empty", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref entry.obj ref.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - COFF_SectionHeader *rdata_section = coff_section_header_from_name(str8_zero(), section_table, pe.section_count, str8_lit(".rdata")); - COFF_SectionHeader *data_section = coff_section_header_from_name(str8_zero(), section_table, pe.section_count, str8_lit(".data")); - T_Ok(rdata_section != 0); - T_Ok(data_section != 0); - - U64 empty_va = 0; - str8_deserial_read_struct(exe, data_section->foff, &empty_va); - B32 empty_after_pad = empty_va >= pe.image_base + rdata_section->voff + 2; - T_Ok(empty_after_pad); - - U64 empty_off_in_rdata = empty_va - pe.image_base - rdata_section->voff; - T_Ok(rdata_section->foff + empty_off_in_rdata <= exe.size); - T_Ok(exe.str[rdata_section->foff + empty_off_in_rdata - 2] == 'x'); - T_Ok(exe.str[rdata_section->foff + empty_off_in_rdata - 1] == 'y'); -} - -TEST(zero_length_static_comdat_referenced_by_reloc) -{ - if (t_id_linker() != Linker_radlink) { return; } - - U8 text[] = { - 0x48, 0x8D, 0x05, 0, 0, 0, 0, // lea rax, [rip + EMPTY] - 0xC3 - }; - - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "empty", ".rdata", str8_zero(), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 3, "EMPTY"), - {0} - }}, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("empty", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Static("EMPTY", "empty", 0), - T_COFF_DefSymbol_Secdef("text", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,noicf test.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(comdat_no_duplicates) -{ - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "a"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("a"), - {0} - } - })); - - String8 test_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "test", ".test", str8_lit("a"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("test", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("a", "test", 0), - {0} - } - }); - T_Ok(t_write_file(str8_lit("a.obj"), test_obj)); - T_Ok(t_write_file(str8_lit("b.obj"), test_obj)); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); } - - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe a.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("b.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".test")); - T_Ok(sect); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("a"), 0)); -} - -TEST(comdat_same_size) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a", str8_lit("a"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_SameSize), - T_COFF_DefSymbol_Extern("TEST", "a", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "b", ".b", str8_lit("b"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_SameSize), - T_COFF_DefSymbol_Extern("TEST", "b", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("c.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "c", ".c", str8_lit("cc"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("c", COFF_ComdatSelect_SameSize), - T_COFF_DefSymbol_Extern("TEST", "c", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a")); - T_Ok(sect != 0); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("a"), 0)); - } - - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe a.obj b.obj c.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { T_Ok(g_last_exit_code == LNK_Error_MultiplyDefinedSymbol); } -} - -TEST(comdat_exact_match) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a", str8_lit("a"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_ExactMatch), - T_COFF_DefSymbol_Extern("TEST", "a", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("a2.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a2", ".a2", str8_lit("a"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a2", COFF_ComdatSelect_ExactMatch), - T_COFF_DefSymbol_Extern("TEST", "a2", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "b", ".b", str8_lit("b"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_ExactMatch), - T_COFF_DefSymbol_Extern("TEST", "b", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj b.obj"); - T_Ok(g_last_exit_code != 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe entry.obj a2.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("b.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a2")); - T_Ok(sect != 0); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("a"), 0)); - } -} - -TEST(comdat_largest) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a", str8_lit("a"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "a", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "b", ".b", str8_lit("bb"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "b", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("c.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "c", ".c", str8_lit("c"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("c", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "c", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /out:a.exe /entry:entry entry.obj a.obj b.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *discard_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a")); - T_Ok(discard_sect == 0); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".b")); - T_Ok(sect != 0); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("bb"), 0)); - } - - t_invoke_linkerf("/subsystem:console /out:b.exe /entry:entry entry.obj c.obj a.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("b.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".c")); - T_Ok(sect != 0); - String8 data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - T_Ok(str8_match(data, str8_lit("c"), 0)); - } -} - -TEST(comdat_associative) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", "a", str8_lit("a"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "aa", "aa", str8_lit("aa"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "a", 0), - T_COFF_DefSymbol_Associative("aa", "a"), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "bb", "bb", str8_lit("bb"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "b", "b", str8_lit("b"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "bbb", "bbb", str8_lit("bbb"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("bb", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Associative("b", "bb"), - T_COFF_DefSymbol_Associative("bbb", "bb"), - T_COFF_DefSymbol_Extern("TEST", "bb", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj b.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *a = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("a")); - COFF_SectionHeader *aa = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("aa")); - COFF_SectionHeader *b = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("b")); - COFF_SectionHeader *bb = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("bb")); - COFF_SectionHeader *bbb = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit("bbb")); - T_Ok(a == 0); - T_Ok(aa == 0); - T_Ok(b != 0); - T_Ok(bb != 0); - T_Ok(bbb != 0); - String8 b_data = str8_substr(exe, rng_1u64(b->foff, b->foff + b->vsize)); - String8 bb_data = str8_substr(exe, rng_1u64(bb->foff, bb->foff + bb->vsize)); - String8 bbb_data = str8_substr(exe, rng_1u64(bbb->foff, bbb->foff + bbb->vsize)); - T_Ok(str8_match(b_data, str8_lit("b"), 0)); - T_Ok(str8_match(bb_data, str8_lit("bb"), 0)); - T_Ok(str8_match(bbb_data, str8_lit("bbb"), 0)); -} - -TEST(comdat_associative_loop) -{ - T_Ok(t_write_def_obj("loop.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "aaaa", ".aaaa", str8_lit("aaaa"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "aa", ".aa", str8_lit("aa"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "a", ".a", str8_lit("a"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "aaa", ".aaa", str8_lit("aaa"), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Associative("aaa", "aa"), - T_COFF_DefSymbol_Associative("aaaa", "aaa"), - T_COFF_DefSymbol_Associative("a", "aa"), - T_COFF_DefSymbol_Associative("aa", "a"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe loop.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { T_Ok(g_last_exit_code == LNK_Error_AssociativeLoop); } -} - -TEST(comdat_associative_non_comdat) -{ - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a", str8_lit("a"), .flags = "rw:data" }, - { "b", ".b", str8_lit("b"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("TEST", "a", 0), - T_COFF_DefSymbol_Associative("b", "a"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj test.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *a = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a")); - COFF_SectionHeader *b = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".b")); - T_Ok(a != 0); - T_Ok(b != 0); - String8 a_data = str8_substr(exe, rng_1u64(a->foff, a->foff + a->vsize)); - String8 b_data = str8_substr(exe, rng_1u64(b->foff, b->foff + b->vsize)); - T_Ok(str8_match(a_data, str8_lit("a"), 0)); - T_Ok(str8_match(b_data, str8_lit("b"), 0)); -} - -TEST(comdat_associative_out_of_bounds) -{ - { - String8 obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a", str8_lit("a"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "aa", ".aa", str8_lit("aa"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("TEST", "a", 0), - T_COFF_DefSymbol_Associative("aa", "a"), - {0} - } - }); - { - COFF_FileHeaderInfo header = coff_file_header_info_from_data(obj); - String8 string_table = str8_substr(obj, header.string_table_range); - String8 symbol_table = str8_substr(obj, header.symbol_table_range); - COFF_ParsedSymbol symbol = coff_parse_symbol(header, string_table, symbol_table, 3); - AssertAlways(str8_match(symbol.name, str8_lit(".aa"), 0)); - AssertAlways(symbol.aux_symbol_count == 1); - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - COFF_SymbolSecDef *secdef = (COFF_SymbolSecDef *)(symbol16 + 1); - secdef->number_lo = 321; - } - T_Ok(t_write_file(str8_lit("bad.obj"), obj)); - } - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj bad.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { T_Ok(g_last_exit_code == LNK_Error_IllData); } -} - -TEST(comdat_with_offset) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "rdata", ".rdata", str8_lit_cstr("1Hello, World!"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("rdata", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "rdata", 1), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "rdata", ".rdata", str8_lit_cstr("Hello, World!"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("rdata", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "rdata", 1), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(reloc_against_removed_comdat) -{ - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "rdata", ".rdata", str8_lit_cstr("1Hello, World!"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("rdata", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "rdata", 1), - {0} - } - })); - - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "comdat", ".rdata", str8_lit_cstr("H"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "regular", ".rdata", str8_lit_comp("\x00\x00\x00\x00"), .flags = "r:data", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "STATIC"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("comdat", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "comdat", 1), - T_COFF_DefSymbol_Static("STATIC", "comdat", 2), - {0} - } - })); - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(sect_align) -{ - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "shift", ".a", str8_lit("q"), .flags = "rw:data" }, - { "none", ".a", str8_lit("abc"), .flags = "rw:data" }, - { "a1", ".a", str8_lit("wr"), .flags = "rw:data@1" }, - { "a2", ".a", str8_lit("e"), .flags = "rw:data@2" }, - { "a4", ".a", str8_lit("ttttt"), .flags = "rw:data@4" }, - { "a8", ".a", str8_lit("g"), .flags = "rw:data@8" }, - { "a16", ".a", str8_lit("o"), .flags = "rw:data@16" }, - { "a32", ".a", str8_lit("p"), .flags = "rw:data@32" }, - { "a64", ".a", str8_lit("f"), .flags = "rw:data@64" }, - { "a128", ".a", str8_lit("x"), .flags = "rw:data@128" }, - { "a256", ".a", str8_lit("c"), .flags = "rw:data@256" }, - { "a512", ".a", str8_lit("v"), .flags = "rw:data@512" }, - { "a1024", ".a", str8_lit("b"), .flags = "rw:data@1024" }, - { "a2048", ".a", str8_lit("n"), .flags = "rw:data@2048" }, - { "a4096", ".a", str8_lit("m"), .flags = "rw:data@4096" }, - { "a8192", ".a", str8_lit("z"), .flags = "rw:data@8192" }, - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("my_entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:my_entry /out:a.exe /align:8192 test.obj"); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".a")); - T_Ok(sect); - String8 sect_data = str8_substr(exe, rng_1u64(sect->foff, sect->foff + sect->vsize)); - - String8 shift = str8_substr(sect_data, rng_1u64(0, 1)); - T_Ok(str8_match(shift, str8_lit("q"), 0)); - String8 a_none = str8_substr(sect_data, rng_1u64(16, 16 + 3)); - T_Ok(str8_match(a_none, str8_lit("abc"), 0)); - String8 a_1 = str8_substr(sect_data, rng_1u64(19, 21)); - T_Ok(str8_match(a_1, str8_lit("wr"), 0)); - String8 a_2 = str8_substr(sect_data, rng_1u64(22, 23)); - T_Ok(str8_match(a_2, str8_lit("e"), 0)); - String8 a_4 = str8_substr(sect_data, rng_1u64(24, 29)); - T_Ok(str8_match(a_4, str8_lit("ttttt"), 0)); - String8 a_8 = str8_substr(sect_data, rng_1u64(32, 33)); - T_Ok(str8_match(a_8, str8_lit("g"), 0)); - String8 a_16 = str8_substr(sect_data, rng_1u64(48, 49)); - T_Ok(str8_match(a_16, str8_lit("o"), 0)); - String8 a_32 = str8_substr(sect_data, rng_1u64(64, 65)); - T_Ok(str8_match(a_32, str8_lit("p"), 0)); - String8 a_64 = str8_substr(sect_data, rng_1u64(128, 129)); - T_Ok(str8_match(a_64, str8_lit("f"), 0)); - String8 a_128 = str8_substr(sect_data, rng_1u64(256, 257)); - T_Ok(str8_match(a_128, str8_lit("x"), 0)); - String8 a_256 = str8_substr(sect_data, rng_1u64(512, 513)); - T_Ok(str8_match(a_256, str8_lit("c"), 0)); - String8 a_512 = str8_substr(sect_data, rng_1u64(1024, 1025)); - T_Ok(str8_match(a_512, str8_lit("v"), 0)); - String8 a_1024 = str8_substr(sect_data, rng_1u64(2048, 2049)); - T_Ok(str8_match(a_1024, str8_lit("b"), 0)); - String8 a_2048 = str8_substr(sect_data, rng_1u64(4096, 4097)); - T_Ok(str8_match(a_2048, str8_lit("n"), 0)); - String8 a_4096 = str8_substr(sect_data, rng_1u64(8192, 8193)); - T_Ok(str8_match(a_4096, str8_lit("m"), 0)); - String8 a_8192 = str8_substr(sect_data, rng_1u64(16384, 16385)); - T_Ok(str8_match(a_8192, str8_lit("z"), 0)); -} - -TEST(alt_name) -{ - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("test"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("test", "data", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("foo.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("foo"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("foo", "data", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "foo"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("foo"), - {0} - } - })); - - // basic alternate name test - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /alternatename:foo=test test.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - // linker should not chase alt name links - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe /alternatename:foo=bar /alternatename:bar=test test.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // alt name conflict - t_invoke_linkerf("/subsystem:console /entry:entry /out:c.exe /alternatename:foo=test /alternatename:foo=qwe test.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // syntax error - t_invoke_linkerf("/subsystem:console /entry:entry /out:d.exe /alternatename:foo foo.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // syntax error - t_invoke_linkerf("/subsystem:console /entry:entry /out:e.exe /alternatename:foo-oof foo.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // syntax error - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /alternatename:foo=test=bar foo.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // syntax error - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /alternatename:foo= foo.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // syntax error - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /alternatename:= foo.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // syntax error - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /alternatename: foo.obj entry.obj"); - T_Ok(g_last_exit_code != 0); - - // TODO: check that RAD Linker prints these warnings - - // warn about alt name to self alt name? - t_invoke_linkerf("/subsystem:console /entry:entry /out:f.exe /alternatename:foo=foo foo.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - // warn about alt name to unknown symbol? - t_invoke_linkerf("/subsystem:console /entry:entry /out:g.exe /alternatename:qwe=ewq foo.obj entry.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(include) -{ - T_Ok(t_write_def_lib("include.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("include.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("foo"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("foo", "data", 0), - {0} - } - } - }, - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "entry"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - // simple include test - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /include:foo entry.obj include.lib"); - T_Ok(g_last_exit_code == 0); - - // validate that linker pulled-in include.obj - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *foo_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(foo_sect != 0); - String8 foo_data = str8_substr(exe, rng_1u64(foo_sect->foff, foo_sect->foff + foo_sect->vsize)); - T_Ok(str8_match(foo_data, str8_lit("foo"), 0)); - } - - // test unresolved include - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /include:ewq entry.obj"); - T_Ok(g_last_exit_code != 0); - if (t_id_linker() == Linker_radlink) { T_Ok(g_last_exit_code == LNK_Error_UnresolvedSymbol); } -} - -TEST(communal_var_vs_regular) -{ - T_Ok(t_write_def_obj("communal.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Common("TEST", 1), - {0} - } - })); - - T_Ok(t_write_def_obj("defn.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("test"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("TEST", "data", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - // linker should replace communal TEST with .data TEST - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe communal.obj defn.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe defn.obj communal.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - char *exes[] = { "a.exe", "b.exe" }; - for EachElement(i, exes) { - String8 exe = t_read_file(arena, str8_cstring(exes[i])); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_sect); - String8 data = str8_substr(exe, rng_1u64(data_sect->foff, data_sect->foff + data_sect->vsize)); - T_Ok(str8_match(data, str8_lit("test"), 0)); - } -} - -TEST(communal_var_vs_regular_comdat) -{ - T_Ok(t_write_def_obj("communal.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Common("TEST", 1), - {0} - } - })); - - T_Ok(t_write_def_obj("large.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit("test"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("TEST", "data", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("TEST"), - {0} - } - })); - - // linker should replace communal TEST with .data TEST - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe communal.obj large.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe large.obj communal.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - char *exes[] = { "a.exe", "b.exe" }; - for EachElement(i, exes) { - String8 exe = t_read_file(arena, str8_cstring(exes[i])); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_sect); - String8 data = str8_substr(exe, rng_1u64(data_sect->foff, data_sect->foff + data_sect->vsize)); - T_Ok(str8_match(data, str8_lit("test"), 0)); - } -} - -#if OS_WINDOWS -TEST(import_kernel32) -{ - T_Ok(t_write_def_obj("import.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "data", ".data", str8_lit_cstr("test"), .flags = "rw:data" }, - { - "text", ".text", - str8_lit_comp( - "\x48\x83\xEC\x68" // sub rsp,68h ; alloc space on stack - "\xC7\x44\x24\x48\x18\x00\x00\x00" // mov dword ptr [rsp+48h],18h ; SECURITY_ATTRIBUTES.nLength - "\x48\xC7\x44\x24\x50\x00\x00\x00\x00" // mov qword ptr [rsp+50h],0 ; SECURITY_ATTRIBUTES.lpSecurityDescriptor - "\xC7\x44\x24\x58\x00\x00\x00\x00" // mov dword ptr [rsp+58h],0 ; SECURITY_ATTRIBUTES.bInheritHandle - "\x48\xC7\x44\x24\x30\x00\x00\x00\x00" // mov qword ptr [rsp+30h],0 ; hTemplateFile - "\xC7\x44\x24\x28\x80\x00\x00\x00" // mov dword ptr [rsp+28h],80h ; dwFlagsAndAttributes - "\xC7\x44\x24\x20\x02\x00\x00\x00" // mov dword ptr [rsp+20h],2 ; dwCreationDisposition - "\x4C\x8D\x4C\x24\x48" // lea r9,[rsp+48h] ; lpSecurityAttributes - "\x45\x33\xC0" // xor r8d,r8d ; dwShareMode - "\xBA\x00\x00\x00\x40" // mov edx,40000000h ; dwDesiredAccess - "\x48\x8D\x0D\x00\x00\x00\x00" // lea rcx,[test] ; lpFileName - "\xFF\x15\x00\x00\x00\x00" // call qword ptr [__imp_CreateFileA] ; call CreateFileA - "\x48\x89\xC1" // mov rcx,rax ; hObject - "\xFF\x15\x00\x00\x00\x00" // call qword ptr [__imp_CloseHandle] ; call CloseHandle - "\x33\xC0" // xor eax,eax ; clear result - "\x48\x83\xC4\x68" // add rsp,68h ; dealloc stack - "\xC3" // ret ; return - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 70, "test"), - T_COFF_DefReloc(X64_Rel32, 76, "__imp_CreateFileA"), - T_COFF_DefReloc(X64_Rel32, 85, "__imp_CloseHandle"), - {0}, - } - }, - { - "dead", ".text$dead", str8(0, 4), - .flags = "rx:code@1", - .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "__imp_CompareStringW"), - {0}, - } - }, - {0}, - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("dead", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("test", "data", 0), - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_ExternFunc("dead", "dead", 0), - T_COFF_DefSymbol_Undef("__imp_CreateFileA"), - T_COFF_DefSymbol_Undef("__imp_CloseHandle"), - T_COFF_DefSymbol_Undef("__imp_CompareStringW"), - {0}, - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /fixed /opt:ref import.obj kernel32.lib"); - T_Ok(g_last_exit_code == 0); - - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - PE_ParsedStaticImportTable imports = pe_static_imports_from_data(arena, pe.is_pe32, pe.section_count, section_table, exe, pe.data_dir_franges[PE_DataDirectoryIndex_IMPORT]); - B32 found_create = 0; - B32 found_close = 0; - B32 found_dead = 0; - for EachIndex(dll_idx, imports.count) { - PE_ParsedStaticDLLImport *dll = &imports.v[dll_idx]; - for EachIndex(import_idx, dll->import_count) { - PE_ParsedImport *import = &dll->imports[import_idx]; - if (import->type == PE_ParsedImport_Name) { - found_create |= str8_match(import->u.name.string, str8_lit("CreateFileA"), 0); - found_close |= str8_match(import->u.name.string, str8_lit("CloseHandle"), 0); - found_dead |= str8_match(import->u.name.string, str8_lit("CompareStringW"), 0); - } - } - } - T_Ok(found_create); - T_Ok(found_close); - T_Ok(!found_dead); - - String8 test_file_path = push_str8f(arena, "%S/test", g_wdir); - delete_file_at_path(test_file_path); - - ProcessLaunchParams launch_opts = {0}; - launch_opts.inherit_env = 0; - launch_opts.path = g_wdir; - str8_list_pushf(arena, &launch_opts.cmd_line, "%S/a.exe", g_wdir); - Process handle = process_launch(&launch_opts); - AssertAlways(!process_match(handle, process_zero())); - U64 exit_code = max_U64; - process_join(handle, max_U64, &exit_code); - T_Ok(exit_code == 0); - T_Ok(file_path_exists(test_file_path)); - } -} - -TEST(delay_import) -{ - { - U8 return_0[] = { - 0x48, 0xC7, 0xC0, 0x00, 0x00, 0x00, 0x00, // mov rax, 0 - 0xc3 // ret - }; - U8 return_1[] = { - 0x48, 0xC7, 0xC0, 0x01, 0x00, 0x00, 0x00, // mov rax, 1 - 0xc3 // ret - }; - U8 return_2[] = { - 0x48, 0xC7, 0xC0, 0x02, 0x00, 0x00, 0x00, // mov rax, 2 - 0xc3 // ret - }; - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "return_0", ".text", str8_array_fixed(return_0), .flags = "rx:code@1" }, - { "return_1", ".text", str8_array_fixed(return_1), .flags = "rx:code@1" }, - { "return_2", ".text", str8_array_fixed(return_2), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("return_1", "return_1", 0), - T_COFF_DefSymbol_ExternFunc("return_2", "return_2", 0), - {0} - } - })); - } - - { - U8 return_0[] = { - 0x48, 0xC7, 0xC0, 0x00, 0x00, 0x00, 0x00, // mov rax, 0 - 0xc3 // ret - }; - U8 return_123[] = { - 0x48, 0xC7, 0xC0, 0x7B, 0x00, 0x00, 0x00, // mov rax, 123 - 0xc3 // ret - }; - U8 return_321[] = { - 0x48, 0xC7, 0xC0, 0x41, 0x01, 0x00, 0x00, // mov rax, 321 - 0xc3 // ret - }; - T_Ok(t_write_def_obj("b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "return_0", ".text", str8_array_fixed(return_0), .flags = "rx:code@1" }, - { "return_123", ".text", str8_array_fixed(return_123), .flags = "rx:code@1" }, - { "return_321", ".text", str8_array_fixed(return_321), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("return_123", "return_123", 0), - T_COFF_DefSymbol_ExternFunc("return_321", "return_321", 0), - {0} - } - })); - } - - { - U8 text[] = { - 0x56, // push rsi - 0x57, // push rdi - 0x48, 0x83, 0xEC, 0x28, // sub rsp,28h - 0xE8, 0x00, 0x00, 0x00, 0x00, // call return_1 - 0x89, 0xC6, // mov esi,eax - 0xE8, 0x00, 0x00, 0x00, 0x00, // call return_2 - 0x89, 0xC7, // mov edi,eax - 0x01, 0xF7, // add edi,esi - 0xE8, 0x00, 0x00, 0x00, 0x00, // call return_123 - 0x89, 0xC6, // mov esi,eax - 0xE8, 0x00, 0x00, 0x00, 0x00, // call return_321 - 0x01, 0xF0, // add eax,esi - 0x01, 0xF8, // add eax,edi - 0x48, 0x83, 0xC4, 0x28, // add rsp,28h - 0x5F, // pop rdi - 0x5E, // pop rsi - 0xC3, // ret - }; - T_Ok(t_write_def_obj("main.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 7, "return_1"), - T_COFF_DefReloc(X64_Rel32, 14, "return_2"), - T_COFF_DefReloc(X64_Rel32, 23, "return_123"), - T_COFF_DefReloc(X64_Rel32, 30, "return_321"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("return_1"), - T_COFF_DefSymbol_Undef("return_2"), - T_COFF_DefSymbol_Undef("return_123"), - T_COFF_DefSymbol_Undef("return_321"), - {0} - } - })); - } - - t_invoke_linkerf("/dll /implib:a.lib /export:return_1 /export:return_2 /rad_time_stamp:0x69EB0E28 a.obj libcmt.lib"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/dll /implib:b.lib /export:return_123 /export:return_321 /rad_time_stamp:0x69EB0E28 b.obj libcmt.lib"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /fixed /debug:full /rad_time_stamp:0x69EB0E28 main.obj a.lib b.lib kernel32.lib delayimp.lib libcmt.lib /delayload:a.dll /delayload:b.dll"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - PE_ParsedDelayImportTable delay_import_table = pe_delay_imports_from_data(arena, pe.is_pe32, pe.section_count, section_table, exe, pe.data_dir_franges[PE_DataDirectoryIndex_DELAY_IMPORT]); - - PE_ParsedDelayDLLImport *a_import = &delay_import_table.v[0]; - T_Ok(a_import->attributes == 1); - T_Ok(str8_match(a_import->name, str8_lit("a.dll"), 0)); - T_Ok(a_import->module_handle_voff != 0); - T_Ok(a_import->name_table_voff != 0); - T_Ok(a_import->bound_table_voff != 0); - T_Ok(a_import->unload_table_voff != 0); - T_Ok(a_import->time_stamp == 0); - T_Ok(a_import->bound_table_count == 2); - T_Ok(a_import->unload_table_count == 2); - T_Ok(a_import->import_count == 2); - - PE_ParsedImport *return_1 = &a_import->imports[0]; - T_Ok(return_1->type == PE_ParsedImport_Name); - T_Ok(str8_match(return_1->u.name.string, str8_lit("return_1"), 0)); - T_Ok(return_1->u.name.hint == 0); - - PE_ParsedImport *return_2 = &a_import->imports[1]; - T_Ok(return_2->type == PE_ParsedImport_Name); - T_Ok(str8_match(return_2->u.name.string, str8_lit("return_2"), 0)); - T_Ok(return_2->u.name.hint == 1); - - PE_ParsedDelayDLLImport *b_import = &delay_import_table.v[1]; - T_Ok(b_import->attributes == 1); - T_Ok(str8_match(b_import->name, str8_lit("b.dll"), 0)); - T_Ok(b_import->module_handle_voff != 0); - T_Ok(b_import->name_table_voff != 0); - T_Ok(b_import->bound_table_voff != 0); - T_Ok(b_import->unload_table_voff != 0); - T_Ok(b_import->time_stamp == 0); - T_Ok(b_import->bound_table_count == 2); - T_Ok(b_import->unload_table_count == 2); - T_Ok(b_import->import_count == 2); - - PE_ParsedImport *return_123 = &b_import->imports[0]; - T_Ok(return_123->type == PE_ParsedImport_Name); - T_Ok(str8_match(return_123->u.name.string, str8_lit("return_123"), 0)); - T_Ok(return_123->u.name.hint == 0); - - PE_ParsedImport *return_321 = &b_import->imports[1]; - T_Ok(return_321->type == PE_ParsedImport_Name); - T_Ok(str8_match(return_321->u.name.string, str8_lit("return_321"), 0)); - T_Ok(return_321->u.name.hint == 1); -} - -TEST(delay_import_user32) -{ - { - U64 msg_off = 0; - U64 caption_off = 5; - String8 str_payload = str8_lit_comp("test\0foo\0"); - U8 text[] = { - 0x48, 0x83, 0xEC, 0x28, // sub rsp,28h - 0x45, 0x33, 0xC9, // xor r9d,r9d - 0x4C, 0x8D, 0x05, 0x00, 0x00, 0x00, 0x00, // lea r8,[msg] - 0x48, 0x8D, 0x15, 0x00, 0x00, 0x00, 0x00, // lea rdx,[caption] - 0x33, 0xC9, // xor ecx,ecx - 0xFF, 0x15, 0x00, 0x00, 0x00, 0x00, // call qword ptr [__imp_MessageBoxA] - 0x33, 0xC0, // xor eax,eax - 0x48, 0x83, 0xC4, 0x28, // add rsp,28h - 0xC3, // ret - }; - T_Ok(t_write_def_obj("delay_import.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "str", ".str", str_payload, .flags = "rw:data" }, - { - "text", ".text", str8_array_fixed(text), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 10, "msg"), - T_COFF_DefReloc(X64_Rel32, 17, "caption"), - T_COFF_DefReloc(X64_Rel32, 25, "__imp_MessageBoxA"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("msg", "str", msg_off), - T_COFF_DefSymbol_Extern("caption", "str", caption_off), - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("__imp_MessageBoxA"), - {0} - } - })); - } - - t_invoke_linkerf("/subsystem:console /out:a.exe /entry:entry /fixed /delayload:user32.dll kernel32.lib user32.lib libcmt.lib delayimp.lib delay_import.obj /debug:full"); - T_Ok(g_last_exit_code == 0); -} - -#endif - -TEST(empty_section) -{ - T_Ok(t_write_def_obj("empty_section.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "test", ".test", str8(0,0), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("TEST", "test", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("TEST"), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe empty_section.obj entry.obj"); - T_Ok(g_last_exit_code != 0); -} - -TEST(removed_section) -{ - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "test", ".test", str8_lit_comp("\xC3"), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkRemove }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("TEST", "test", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "TEST"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("TEST"), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe test.obj entry.obj"); - T_Ok(g_last_exit_code != 0); -} - -TEST(function_pad_min) -{ - T_Ok(t_write_def_obj("funcs.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@4" }, - { "b", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@4" }, - { "c", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0}, - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("A", "a", 0), - T_COFF_DefSymbol_ExternFunc("B", "b", 0), - T_COFF_DefSymbol_ExternFunc("C", "c", 0), - {0}, - } - })); - - t_invoke_linkerf("/subsystem:console /entry:A /functionpadmin:1 /out:a.exe funcs.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - T_Ok(text_sect != 0); - String8 text_data = str8_substr(exe, rng_1u64(text_sect->foff, text_sect->foff + text_sect->vsize)); - - U8 expected_text[] = { - 0xcc, 0xcc, 0xcc, 0xcc, 0xc3, - 0xcc, 0xcc, 0xcc, 0xc3, - 0xcc, 0xc3, - }; - T_Ok(str8_match(text_data, str8_array_fixed(expected_text), 0)); -} - -TEST(first_member_header) -{ - T_Ok(t_write_def_lib("test.lib", (T_COFF_DefLib){ - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("obj.obj"), - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("8", 0x8), - T_COFF_DefSymbol_AbsExtern("1", 0x1), - T_COFF_DefSymbol_AbsExtern("9", 0x9), - T_COFF_DefSymbol_AbsExtern("7", 0x7), - T_COFF_DefSymbol_AbsExtern("4", 0x4), - T_COFF_DefSymbol_AbsExtern("5", 0x5), - T_COFF_DefSymbol_AbsExtern("2", 0x2), - T_COFF_DefSymbol_AbsExtern("3", 0x3), - T_COFF_DefSymbol_AbsExtern("6", 0x6), - {0} - } - } - }, - {0} - } - })); - T_Ok(t_write_entry_obj()); - - String8 lib_before = t_read_file(arena, str8_lit("test.lib")); - String8 obj_before = t_read_file(arena, str8_lit("entry.obj")); - - t_invoke_linkerf("/RAD_MEMORY_MAP_FILES:READ_WRITE /subsystem:console /entry:entry /out:a.exe test.lib entry.obj /include:1 /include:2 /include:3 /include:4 /include:5 /include:6 /include:7 /include:8 /include:9"); - T_Ok(g_last_exit_code == 0); - T_Ok(str8_match(lib_before, t_read_file(arena, str8_lit("test.lib")), 0)); - T_Ok(str8_match(obj_before, t_read_file(arena, str8_lit("entry.obj")), 0)); -} - -TEST(second_member_header) -{ - T_Ok(t_write_def_lib("test.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("obj.obj"), - .machine = T_COFF_DefSetMachine(X64), - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsExtern("8", 0x8), - T_COFF_DefSymbol_AbsExtern("1", 0x1), - T_COFF_DefSymbol_AbsExtern("9", 0x9), - T_COFF_DefSymbol_AbsExtern("7", 0x7), - T_COFF_DefSymbol_AbsExtern("4", 0x4), - T_COFF_DefSymbol_AbsExtern("5", 0x5), - T_COFF_DefSymbol_AbsExtern("2", 0x2), - T_COFF_DefSymbol_AbsExtern("3", 0x3), - T_COFF_DefSymbol_AbsExtern("6", 0x6), - {0} - } - } - }, - {0} - } - })); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe test.lib entry.obj /include:1 /include:2 /include:3 /include:4 /include:5 /include:6 /include:7 /include:8 /include:9"); - T_Ok(g_last_exit_code == 0); -} - -TEST(defer_impl_link_to_second_search_pass) -{ - T_Ok(t_write_def_obj("imp_ref.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("entry", "text", 0), - T_COFF_DefSymbol_Undef("__imp_foo"), - T_COFF_DefSymbol_Undef("func"), - {0} - } - })); - - T_Ok(t_write_def_lib("impl_ref.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("impl_ref.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("func", "text", 0), - T_COFF_DefSymbol_Undef("foo"), - {0} - } - } - }, - {0} - } - })); - - T_Ok(t_write_def_lib("foo.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Import, - .import = { - "foo.dll", "foo", COFF_ImportBy_Name, COFF_ImportHeader_Code, - .hit_or_ordinal = 0, - .time_stamp = T_COFF_DefSetTimeStamp(~0u), - .machine = T_COFF_DefSetMachine(X64) - } - }, - {0} - } - })); - - T_Ok(t_write_def_lib("foo2.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Import, - .import = { - "foo.dll", "foo", COFF_ImportBy_Name, COFF_ImportHeader_Code, - .hit_or_ordinal = 0, - .time_stamp = T_COFF_DefSetTimeStamp(~0u), - .machine = T_COFF_DefSetMachine(X64) - } - }, - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe foo.lib foo2.lib imp_ref.obj impl_ref.lib"); - T_Ok(g_last_exit_code == 0); -} - -TEST(defer_duplicate_imp_link) -{ - T_COFF_DefLib bar_lib_any = { - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_DllImportStatic, - .dll_import = { .name = "bar.dll" } - }, - { - .type = T_COFF_DefLibMember_Import, - .import = { "bar.dll", "bar", COFF_ImportBy_Name, COFF_ImportHeader_Code, .hit_or_ordinal = 0 } - }, - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_lit_comp("\xff\x25\x00\x00\x00\x00"), .flags = "rx:code" - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("__imp_bar"), - T_COFF_DefSymbol_ExternFunc("qwe", "text", 0), - {0}, - } - } - }, - {0} - } - }; - - T_COFF_DefLib foo_lib_any = { - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_DllImportStatic, - .dll_import = { .name = "foo.dll" } - }, - { - .type = T_COFF_DefLibMember_Import, - .import = { "foo.dll", "bar", COFF_ImportBy_Name, COFF_ImportHeader_Code, .hit_or_ordinal = 0 } - }, - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp("\xff\x25\x00\x00\x00\x00"), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 2, "bar"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("bar"), - T_COFF_DefSymbol_Undef("qwe"), - T_COFF_DefSymbol_ExternFunc("thunk", "text", 0), - {0}, - } - } - }, - {0} - } - }; - - String8 bar_lib = t_coff_from_def_lib(arena, bar_lib_any); - String8 foo_lib = t_coff_from_def_lib(arena, foo_lib_any); - - T_Ok(t_write_file(str8_lit("bar.lib"), bar_lib)); - T_Ok(t_write_file(str8_lit("foo.lib"), foo_lib)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe bar.lib foo.lib entry.obj /include:thunk"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - PE_ParsedStaticImportTable static_imptab = pe_static_imports_from_data(arena, pe.is_pe32, pe.section_count, section_table, exe, pe.data_dir_franges[PE_DataDirectoryIndex_IMPORT]); - COFF_SectionHeader *data_sect = coff_section_header_from_name(str8_zero(), section_table, pe.section_count, str8_lit(".data")); - - T_Ok(static_imptab.count == 1); - - PE_ParsedStaticDLLImport *dll = &static_imptab.v[0]; - T_Ok(str8_match(dll->name, str8_lit("foo.dll"), StringMatchFlag_CaseInsensitive)); - - T_Ok(dll->import_count == 1); - PE_ParsedImport *imp = &dll->imports[0]; - T_Ok(imp->type == PE_ParsedImport_Name); - T_Ok(str8_match(imp->u.name.string, str8_lit("bar"), 0)); - - U64 iat_foff = pe_foff_from_voff(exe, &pe, dll->import_address_table_voff); - U64 ilt_foff = pe_foff_from_voff(exe, &pe, dll->import_name_table_voff); - - U64 bar_idx = 0; - U64 bar_iat_addr = 0; - U64 bar_ilt_addr = 0; - str8_deserial_read_struct(exe, iat_foff + bar_idx * sizeof(U64), &bar_iat_addr); - str8_deserial_read_struct(exe, ilt_foff + bar_idx * sizeof(U64), &bar_ilt_addr); - - T_Ok(bar_iat_addr != 0); - T_Ok(bar_ilt_addr != 0); - T_Ok(bar_iat_addr == bar_ilt_addr); - T_Ok(data_sect->voff <= bar_iat_addr && bar_iat_addr < data_sect->voff + data_sect->vsize); -} - -TEST(opt_ref_dangling_section) -{ - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" - ), // ret - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "f"), - {0} - } - }, - {0}, - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("f"), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_lit_cstr("A0000"), .flags = "rw:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 0, "q"), - {0} - } - }, - {0}, - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("q"), - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("f", "data", 0), - {0} - } - })); - - T_Ok(t_write_def_lib("b.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("b.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "q", ".q", str8_lit_comp("\x1\x2\x3\x4"), .flags = "rw:data" }, - { - "data", ".data", str8_lit_cstr("BBBBBBBBBBBBBBB"), - .flags = "rw:data", - .raw_flags = COFF_SectionFlag_LnkCOMDAT - }, - {0}, - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("q", "q", 0), - T_COFF_DefSymbol_Secdef("data", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_Extern("f", "data", 0), - {0}, - } - } - }, - {0}, - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe entry.obj a.obj b.lib"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".q")); - T_Ok(sect != 0); -} - -// TODO: relocations against undefined section symbols -#if 1 -TEST(relocate_undefined_section_symbol) -{ - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00\xC3"), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "caller"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("caller"), - {0} - } - })); - - U64 target_count = 25; - - String8 caller_data = str8( - push_array(arena, U8, target_count * 4), target_count * 4); - T_COFF_DefReloc *caller_relocs = push_array(arena, T_COFF_DefReloc, target_count + 1); - T_COFF_DefSymbol *caller_symbols = push_array(arena, T_COFF_DefSymbol, target_count + 3); - - caller_symbols[0] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef("caller", COFF_ComdatSelect_Any); - caller_symbols[1] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Extern("caller", "caller", 0); - - String8List targets = {0}; - - for EachIndex(i, target_count) { - char *obj_name = (char *)str8f(arena, "target%u.obj", i).str; - char *target_name = (char *)str8f(arena, "target%u", i).str; - char *sect_name = (char *)str8f(arena, ".target%u", i).str; - T_Ok(t_write_def_obj(obj_name, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { target_name, sect_name, str8_cstring(target_name), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef(target_name, COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern(target_name, target_name, 0), - {0} - } - })); - - caller_relocs[i] = (T_COFF_DefReloc)T_COFF_DefReloc(X64_Addr32Nb, i * 4, sect_name); - caller_symbols[i + 2] = (T_COFF_DefSymbol)T_COFF_DefSymbol_UndefSec(sect_name, COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead); - - str8_list_pushf(arena, &targets, obj_name); - } - - T_Ok(t_write_def_obj("caller.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "caller", ".caller", caller_data, .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = caller_relocs}, - {0} - }, - .symbols = caller_symbols, - })); - - String8 target_paths = str8_list_join(arena, &targets, &(StringJoin){.sep=str8_lit(" ")}); - T_Ok(t_invoke_linkerf("/subsystem:console /entry:entry /opt:ref /out:a.exe entry.obj caller.obj %S", target_paths) && g_last_exit_code == 0); - - String8 output = g_errors; - U64 match_count = 0; - while (output.size) { - if (t_match_line(&output, str8_lit("WARNING(*): *: relocation * in section *+* points to an undefined symbol *(*)"))) { - match_count += 1; - } - } - //T_Ok(match_count > 0); -} -#endif - -#if 1 -TEST(weak_alias_comdat_duplicate_fallback) -{ - U64 dummy_count = 64; - - for EachIndex(obj_idx, 2) { - char prefix = obj_idx == 0 ? 'a' : 'b'; - T_COFF_DefSection *sections = push_array(arena, T_COFF_DefSection, dummy_count + 5); - T_COFF_DefSymbol *symbols = push_array(arena, T_COFF_DefSymbol, dummy_count*2 + 11); - U64 section_idx = 0; - U64 symbol_idx = 0; - - for EachIndex(i, dummy_count) { - char *id = (char *)str8f(arena, "dummy_%c_%I64u", prefix, i).str; - char *name = (char *)str8f(arena, "?dummy_%c_%I64u@@YAXXZ", prefix, i).str; - sections[section_idx++] = (T_COFF_DefSection){ id, ".text", str8_lit_comp("\xC3"), .flags = "rx:code@16", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef(id, COFF_ComdatSelect_Any); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc(name, id, 0); - } - - sections[section_idx++] = (T_COFF_DefSection){ "text0", ".text", str8_lit_comp("\x33\xC0"), .flags = "rx:code@16", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; - sections[section_idx++] = (T_COFF_DefSection){ "text1", ".text", str8_lit_comp("\x33\xC0"), .flags = "rx:code@16", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; - sections[section_idx++] = (T_COFF_DefSection){ "xdata0", ".xdata", str8_lit_comp("\x01\x00\x00\x00"), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; - sections[section_idx++] = (T_COFF_DefSection){ "text2", ".text", str8_lit_comp("\x33\xC0\xC3"), .flags = "rx:code@16", .raw_flags = COFF_SectionFlag_LnkCOMDAT }; - - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef("text0", COFF_ComdatSelect_Any); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc("??_G__shared_count@__Cr@std@@MEAAPEAXI@Z", "text0", 0); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef("text1", COFF_ComdatSelect_Any); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc("??_G__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", "text1", 0); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Associative("xdata0", "text0"); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Secdef("text2", COFF_ComdatSelect_Any); - - if (obj_idx == 0) { - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc("??_G?$__shared_ptr_emplace@A@@UEAAPEAXI@Z", "text2", 0); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Weak("??_E?$__shared_ptr_emplace@A@@UEAAPEAXI@Z", COFF_WeakExt_SearchAlias, "??_G?$__shared_ptr_emplace@A@@UEAAPEAXI@Z"); - } else { - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_ExternFunc("??_G?$__shared_ptr_emplace@B@@UEAAPEAXI@Z", "text2", 0); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Weak("??_E?$__shared_ptr_emplace@B@@UEAAPEAXI@Z", COFF_WeakExt_SearchAlias, "??_G?$__shared_ptr_emplace@B@@UEAAPEAXI@Z"); - } - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Weak("??_E__shared_count@__Cr@std@@MEAAPEAXI@Z", COFF_WeakExt_SearchAlias, "??_G__shared_count@__Cr@std@@MEAAPEAXI@Z"); - symbols[symbol_idx++] = (T_COFF_DefSymbol)T_COFF_DefSymbol_Weak("??_E__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", COFF_WeakExt_SearchAlias, "??_G__shared_weak_count@__Cr@std@@MEAAPEAXI@Z"); - - T_Ok(t_write_def_obj(obj_idx == 0 ? "a.obj" : "b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = sections, - .symbols = symbols, - })); - } - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp( - "\x48\xC7\xC0\x00\x00\x00\x00" // mov rax, $imm - "\xC3" // ret - ), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "??_E__shared_count@__Cr@std@@MEAAPEAXI@Z"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("??_E__shared_count@__Cr@std@@MEAAPEAXI@Z"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe b.obj a.obj entry.obj"); - T_Ok(g_last_exit_code == 0); -} -#endif - -#if 1 -TEST(opt_ref_weak_alias_comdat) -{ - T_Ok(t_write_def_obj("weak.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "target", ".target", str8_lit("target"), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("target", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Extern("target", "target", 0), - T_COFF_DefSymbol_Weak("weak_target", COFF_WeakExt_SearchAlias, "target"), - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_lit_comp("\x48\xC7\xC0\x00\x00\x00\x00\xC3"), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, 3, "weak_target"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - T_COFF_DefSymbol_Undef("weak_target"), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /opt:ref /out:a.exe entry.obj weak.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *target_sect = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".target")); - T_Ok(target_sect != 0); -} -#endif - - -TEST(fail_if_mismatch) -{ - T_Ok(t_write_entry_obj()); - - // ------------------------------------------------------------ - // try linking two objs with mismatching directives - - String8 a1 = t_make_obj_with_directive(arena, str8_lit("/FAILIFMISMATCH:a=1")); - String8 a2 = t_make_obj_with_directive(arena, str8_lit("/FAILIFMISMATCH:a=2")); - T_Ok(t_write_file(str8_lit("a1.obj"), a1)); - T_Ok(t_write_file(str8_lit("a2.obj"), a2)); - - t_invoke_linkerf("entry.obj a1.obj a2.obj /entry:entry /subsystem:console /out:a2.exe"); - if (t_id_linker() == Linker_radlink) T_Ok(g_last_exit_code == LNK_Error_FailIfMismatch); - else T_Ok(g_last_exit_code != 0); - - // ------------------------------------------------------------ - // happy case - - T_Ok(t_write_file(str8_lit("a1_copy.obj"), a1)); - - t_invoke_linkerf("entry.obj a1.obj a1_copy.obj /entry:entry /subsystem:console /out:a1.exe"); - T_Ok(g_last_exit_code == 0); - - // ------------------------------------------------------------ - // test conflicting directives in obj - - String8 conf_dirs = t_make_obj_with_directive(arena, str8_lit("/FAILIFMISMATCH:a=1 /FAILIFMISMATCH:a=2")); - T_Ok(t_write_file(str8_lit("conf_dirs.obj"), conf_dirs)); - - t_invoke_linkerf("entry.obj conf_dirs.obj /entry:entry /subsystem:console /out:conf_dirs.exe"); - if (t_id_linker() == Linker_radlink) T_Ok(g_last_exit_code == LNK_Error_FailIfMismatch); - else T_Ok(g_last_exit_code != 0); - - // ------------------------------------------------------------ - // passing switch on command line - - t_invoke_linkerf("entry.obj a1.obj /FAILIFMISMATCH:a=2 /out:cmddir.exe"); - if (t_id_linker() == Linker_radlink) T_Ok(g_last_exit_code == LNK_Error_FailIfMismatch); - else T_Ok(g_last_exit_code != 0); -} - -TEST(guardsym_directive) -{ - T_Ok(t_write_entry_obj()); - - // MSVC link accepts GUARDSYM without treating the named symbol as /INCLUDE. - String8 guardsym = t_make_obj_with_directive(arena, str8_lit("/GUARDSYM:missing,S")); - T_Ok(t_write_file(str8_lit("guardsym.obj"), guardsym)); - - t_invoke_linkerf("entry.obj guardsym.obj /entry:entry /subsystem:console /out:guardsym.exe"); - T_Ok(g_last_exit_code == 0); -} - -TEST(long_section_name) -{ - Arch arch = Arch_x64; - - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code" }, - { "debug_info", ".debug_info", str8_lit("DEBUG_INFO"), .flags = "rw:data" }, - { "debug_abbrev", ".debug_abbrev", str8_lit("DEBUG_ABBREV"), .flags = "rw:data" }, - {0}, - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - // link test.obj - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe test.obj"); - T_Ok(g_last_exit_code == 0); - - // load linked exe - String8 exe = t_read_file(arena, str8_lit("a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - - COFF_SectionHeader *debug_info = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".debug_info")); - T_Ok(debug_info); - - COFF_SectionHeader *debug_abbrev = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".debug_abbrev")); - T_Ok(debug_abbrev); -} - -TEST(debug_p_sig_mismatch) -{ - String8 a_obj_name = str8_lit("a.obj"); - String8 b_obj_name = str8_lit("b.obj"); - U32 a_sig = 0xCAFEBABE; - U32 b_sig = 0xDEADBEEF; - - String8 a_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = a_sig; - String8 obj_name_string = a_obj_name; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - a_debug_s = str8_serial_end(arena, &srl); - } - String8 a_debug_p; - { - String8List srl; - str8_serial_begin(arena, &srl); - - // signature - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - // duplicate in a.obj - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(ptr)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // unique procedure type - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearPascal }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(proc)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // PCH ender - CV_LeafEndPreComp endprecomp = { .sig = a_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(endprecomp)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_ENDPRECOMP); - str8_serial_push_struct(arena, &srl, &endprecomp); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - a_debug_p = str8_serial_end(arena, &srl); - } - - String8 b_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = b_sig; - String8 obj_name_string = a_obj_name; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - b_debug_s = str8_serial_end(arena, &srl); - } - - String8 b_debug_t; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_LeafPreComp precomp = { .start_index = CV_MinComplexTypeIndex, .count = 2, sig = b_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(precomp) + a_obj_name.size + 1); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PRECOMP); - str8_serial_push_struct(arena, &srl, &precomp); - str8_serial_push_cstr(arena, &srl, a_obj_name); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafPointer)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearC }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafProcedure)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - b_debug_t = str8_serial_end(arena, &srl); - } - - String8 a_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_p", ".debug$P", a_debug_p, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", a_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 b_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", b_debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", b_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_file(str8_lit("a.obj"), a_obj)); - T_Ok(t_write_file(str8_lit("b.obj"), b_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - B32 found_error = 0; - String8 a_obj_path = t_make_file_path(arena, str8_lit("a.obj")); - String8 b_obj_path = t_make_file_path(arena, str8_lit("b.obj")); - String8 expected_line = str8f(arena, "Error(%03d): %S: PCH signature mismatch, expected 0x%x got 0x%x; PCH obj %S", LNK_Error_PrecompSigMismatch, b_obj_path, b_sig, a_sig, a_obj_path); - - String8 output = g_errors; - while (output.size) { - String8 line = t_chop_line(&output); - found_error = str8_match(line, expected_line, StringMatchFlag_CaseInsensitive|StringMatchFlag_SlashInsensitive); - if (found_error) { break; } - } - T_Ok(found_error); -} - -TEST(pch_sig_fallback) -{ - String8 a_obj_file_path = t_make_file_path(arena, str8_lit("a.obj")); - String8 b_obj_file_path = t_make_file_path(arena, str8_lit("b.obj")); - U32 a_sig = 0xCAFEBABE; - U32 b_sig = 0xCAFEBABE; - - String8 a_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = a_sig; - String8 obj_name_string = a_obj_file_path; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - a_debug_s = str8_serial_end(arena, &srl); - } - String8 a_debug_p; - { - String8List srl; - str8_serial_begin(arena, &srl); - - // signature - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - // duplicate in a.obj - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(ptr)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // unique procedure type - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearPascal }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(proc)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // PCH ender - CV_LeafEndPreComp endprecomp = { .sig = a_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(endprecomp)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_ENDPRECOMP); - str8_serial_push_struct(arena, &srl, &endprecomp); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - a_debug_p = str8_serial_end(arena, &srl); - } - - String8 b_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = b_sig; - String8 obj_name_string = a_obj_file_path; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - b_debug_s = str8_serial_end(arena, &srl); - } - - String8 b_debug_t; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - String8 corrupt_pch_path = str8_lit("corrupt-pch-file-path.obj"); - - CV_LeafPreComp precomp = { .start_index = CV_MinComplexTypeIndex, .count = 2, sig = b_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(precomp) + corrupt_pch_path.size + 1); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PRECOMP); - str8_serial_push_struct(arena, &srl, &precomp); - str8_serial_push_cstr(arena, &srl, corrupt_pch_path); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafPointer)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearC }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafProcedure)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - b_debug_t = str8_serial_end(arena, &srl); - } - - String8 a_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_p", ".debug$P", a_debug_p, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", a_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 b_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", b_debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", b_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(write_data_to_file_path(a_obj_file_path, a_obj)); - T_Ok(write_data_to_file_path(b_obj_file_path, b_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(gen_rrt) -{ - String8 a_obj_file_path = t_make_file_path(arena, str8_lit("a.obj")); - String8 b_obj_file_path = t_make_file_path(arena, str8_lit("b.obj")); - U32 a_sig = 0xCAFEBABE; - U32 b_sig = 0xCAFEBABE; - - String8 a_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = a_sig; - String8 obj_name_string = a_obj_file_path; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - a_debug_s = str8_serial_end(arena, &srl); - } - String8 a_debug_p; - { - String8List srl; - str8_serial_begin(arena, &srl); - - // signature - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - // duplicate in a.obj - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(ptr)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // unique procedure type - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearPascal }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(proc)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // PCH ender - CV_LeafEndPreComp endprecomp = { .sig = a_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(endprecomp)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_ENDPRECOMP); - str8_serial_push_struct(arena, &srl, &endprecomp); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - a_debug_p = str8_serial_end(arena, &srl); - } - - String8 b_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = b_sig; - String8 obj_name_string = a_obj_file_path; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - b_debug_s = str8_serial_end(arena, &srl); - } - - String8 b_debug_t; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - String8 corrupt_pch_path = str8_lit("corrupt-pch-file-path.obj"); - - CV_LeafPreComp precomp = { .start_index = CV_MinComplexTypeIndex, .count = 2, sig = b_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(precomp) + corrupt_pch_path.size + 1); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PRECOMP); - str8_serial_push_struct(arena, &srl, &precomp); - str8_serial_push_cstr(arena, &srl, corrupt_pch_path); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafPointer)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearC }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafProcedure)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - b_debug_t = str8_serial_end(arena, &srl); - } - - String8 a_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_p", ".debug$P", a_debug_p, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", a_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 b_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", b_debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", b_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(write_data_to_file_path(a_obj_file_path, a_obj)); - T_Ok(write_data_to_file_path(b_obj_file_path, b_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full /rad_type_server:foo.rrt a.obj b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash a.obj foo.rrt b.obj entry.obj"); - T_Ok(g_last_exit_code == 0); -} - -TEST(debug_p_and_debug_t_in_obj) -{ - U32 pch_sig = 0xCAFEBABE; - String8 pch_obj_name = str8_lit("pch.obj"); - - String8 pch_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = pch_sig; - String8 obj_name_string = pch_obj_name; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - pch_debug_s = str8_serial_end(arena, &srl); - } - - String8 pch_debug_p; - { - String8List srl; - str8_serial_begin(arena, &srl); - - // signature - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(ptr)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // PCH ender - CV_LeafEndPreComp endprecomp = { .sig = pch_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(endprecomp)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_ENDPRECOMP); - str8_serial_push_struct(arena, &srl, &endprecomp); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - pch_debug_p = str8_serial_end(arena, &srl); - } - - String8 pch_debug_t; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_LeafPreComp precomp = { .start_index = CV_MinComplexTypeIndex, .count = 1, sig = pch_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(precomp) + pch_obj_name.size + 1); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PRECOMP); - str8_serial_push_struct(arena, &srl, &precomp); - str8_serial_push_cstr(arena, &srl, pch_obj_name); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafPointer)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearC }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafProcedure)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - pch_debug_t = str8_serial_end(arena, &srl); - } - - T_Ok(t_write_def_obj("pch.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_p", ".debug$P", pch_debug_p, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_t", ".debug$T", pch_debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", pch_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - })); - T_Ok(t_write_entry_obj()); - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full pch.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - - String8 output = g_errors; - B32 found_warning = 0; - String8 pch_obj_path = t_make_file_path(arena, str8_lit("pch.obj")); - String8 expected_line = str8f(arena, "Warning(%03d): %S: multiple sections with debug types detected, obj must have either .debug$T or .debug$P; discarding both sections", LNK_Warning_MultipleDebugTAndDebugP, pch_obj_path); - while (output.size) { - String8 line = t_chop_line(&output); - found_warning = str8_match(line, expected_line, StringMatchFlag_CaseInsensitive|StringMatchFlag_SlashInsensitive); - if (found_warning) { break; } - } - T_Ok(found_warning); -} - -TEST(merge_duplicate_types) -{ - { - U32 pch_sig = 0xCAFEBABE; - String8 pch_obj_name = str8_lit("pch.obj"); - String8 a_obj_name = str8_lit("a.obj"); - String8 c_obj_name = str8_lit("c.obj"); - - String8 pch_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = pch_sig; - String8 obj_name_string = pch_obj_name; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - pch_debug_s = str8_serial_end(arena, &srl); - } - String8 debug_p; - { - String8List srl; - str8_serial_begin(arena, &srl); - - // signature - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - // duplicate in a.obj - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(ptr)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // unique procedure type - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearPascal }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(proc)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - // PCH ender - CV_LeafEndPreComp endprecomp = { .sig = pch_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(endprecomp)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_ENDPRECOMP); - str8_serial_push_struct(arena, &srl, &endprecomp); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - debug_p = str8_serial_end(arena, &srl); - } - - String8 a_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - CV_SymObjName obj_name = {0}; - obj_name.sig = pch_sig; - String8 obj_name_string = a_obj_name; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - a_debug_s = str8_serial_end(arena, &srl); - } - - String8 debug_t; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_LeafPreComp precomp = { .start_index = CV_MinComplexTypeIndex, .count = 2, sig = pch_sig }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(precomp) + pch_obj_name.size + 1); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PRECOMP); - str8_serial_push_struct(arena, &srl, &precomp); - str8_serial_push_cstr(arena, &srl, pch_obj_name); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafPointer ptr = { .itype = CV_BasicType_VOID }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafPointer)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearC }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafProcedure)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - debug_t = str8_serial_end(arena, &srl); - } - - String8 c_debug_t; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_LeafPointer ptr = { .itype = CV_BasicType_SHORT }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafPointer)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_POINTER); - str8_serial_push_struct(arena, &srl, &ptr); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - CV_LeafProcedure proc = { .ret_itype = 0x1000, .call_kind = CV_CallKind_NearC }; - str8_serial_push_u16(arena, &srl, sizeof(CV_LeafKind) + sizeof(CV_LeafProcedure)); - str8_serial_push_u16(arena, &srl, CV_LeafKind_PROCEDURE); - str8_serial_push_struct(arena, &srl, &proc); - str8_serial_push_align(arena, &srl, CV_LeafAlign); - - c_debug_t = str8_serial_end(arena, &srl); - } - - String8 c_debug_s; - { - String8List srl; - str8_serial_begin(arena, &srl); - - CV_Signature sig = CV_Signature_C13; - str8_serial_push_struct(arena, &srl, &sig); - - CV_C13SubSectionHeader *ss_header = str8_serial_push_size(arena, &srl, sizeof(*ss_header)); - U64 ss_start_off = srl.total_size; - - // S_OBJNAME - CV_SymObjName obj_name = {0}; - obj_name.sig = pch_sig; - String8 obj_name_string = c_obj_name; - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + sizeof(obj_name) + obj_name_string.size + 1); - str8_serial_push_u16(arena, &srl, CV_SymKind_OBJNAME); - str8_serial_push_struct(arena, &srl, &obj_name); - str8_serial_push_cstr(arena, &srl, obj_name_string); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - // S_COMPILE3 - String8 comp3_data = cv_make_comp3(arena, - 0, - CV_Language_C, - CV_Arch_X64, - /* ver_fe_major */ 0, - /* ver_fe_minor */ 0, - /* ver_fe_build */ 0, - /* ver_feqfe */ 0, - /* ver_major */ 14, - /* ver_minor */ 36, - /* ver_build */ 32537, - /* ver_qfe */ 0, - str8_lit(BUILD_TITLE)); - str8_serial_push_u16(arena, &srl, sizeof(CV_SymKind) + comp3_data.size); - str8_serial_push_u16(arena, &srl, CV_SymKind_COMPILE3); - str8_serial_push_string(arena, &srl, comp3_data); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - // S_LPROC32 - U16 *foo_size = str8_serial_push_size(arena, &srl, sizeof(*foo_size)); - U64 foo_off = srl.total_size; - str8_serial_push_u16(arena, &srl, CV_SymKind_LPROC32); - CV_SymProc32 *foo_proc = str8_serial_push_size(arena, &srl, sizeof(*foo_proc)); - foo_proc->itype = 0x1001; - foo_proc->sec = 1; - foo_proc->len = 1; - str8_serial_push_cstr(arena, &srl, str8_lit("foo")); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - *foo_size = srl.total_size - foo_off; - - // S_PROC_ID_END - str8_serial_push_u16(arena, &srl, 2); - str8_serial_push_u16(arena, &srl, CV_SymKind_END); - str8_serial_push_align(arena, &srl, CV_SymbolAlign); - - // $$Symbols header - ss_header->kind = CV_C13SubSectionKind_Symbols; - ss_header->size = srl.total_size - ss_start_off; - str8_serial_push_align(arena, &srl, CV_C13SubSectionAlign); - - c_debug_s = str8_serial_end(arena, &srl); - } - - String8 pch_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_p", ".debug$P", debug_p, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", pch_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 a_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", a_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 b_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 c_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp(""), .flags = "rx:code" }, - { "debug_t", ".debug$T", c_debug_t, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", c_debug_s, .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - String8 entry_obj = t_make_entry_obj(arena); - - T_Ok(t_write_file(str8_lit("entry.obj"), entry_obj)); - T_Ok(t_write_file(str8_lit("pch.obj"), pch_obj)); - T_Ok(t_write_file(str8_lit("a.obj"), a_obj)); - T_Ok(t_write_file(str8_lit("b.obj"), b_obj)); - T_Ok(t_write_file(str8_lit("c.obj"), c_obj)); - - t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /out:a.exe pch.obj a.obj b.obj c.obj entry.obj"); - T_Ok(g_last_exit_code == 0); - } - - // load msf - String8 pdb = t_read_file(arena, str8_lit("a.pdb")); - MSF_Parsed *msf = msf_parsed_from_data(arena, pdb); - // find named streams - String8 info_data = msf_data_from_stream(msf, PDB_FixedStream_Info); - PDB_Info *pdb_info = pdb_info_from_data(arena, info_data); - PDB_NamedStreamTable *named_streams = pdb_named_stream_table_from_info(arena, pdb_info); - // find string table - MSF_StreamNumber strtbl_sn = named_streams->sn[PDB_NamedStream_StringTable]; - String8 strtbl_data = msf_data_from_stream(msf, strtbl_sn); - PDB_Strtbl *strtbl = pdb_strtbl_from_data(arena, strtbl_data); - // find TPI - String8 tpi_data = msf_data_from_stream(msf, PDB_FixedStream_Tpi); - PDB_TpiParsed *tpi = pdb_tpi_from_data(arena, tpi_data); - - U64 type_count = tpi->itype_opl - tpi->itype_first; - T_Ok(type_count == 5); - - CV_DebugT debug_t = cv_debug_t_from_data(arena, pdb_leaf_data_from_tpi(tpi), 4); - T_Ok(debug_t.count == type_count); - - { - CV_Leaf ptr_leaf = cv_debug_t_get_leaf(&debug_t, 0); - T_Ok(ptr_leaf.kind == CV_LeafKind_POINTER); - T_Ok(ptr_leaf.data.size == sizeof(CV_LeafPointer)); - - CV_LeafPointer *ptr = (CV_LeafPointer *)ptr_leaf.data.str; - T_Ok(ptr->itype == CV_BasicType_VOID); - T_Ok(ptr->attribs == 0); - } - - { - CV_Leaf proc_leaf = cv_debug_t_get_leaf(&debug_t, 1); - T_Ok(proc_leaf.kind == CV_LeafKind_PROCEDURE); - T_Ok(proc_leaf.data.size == sizeof(CV_LeafProcedure)); - - CV_LeafProcedure *proc = (CV_LeafProcedure *)proc_leaf.data.str; - T_Ok(proc->ret_itype == 0x1000); - T_Ok(proc->call_kind == CV_CallKind_NearPascal); - } - - { - CV_Leaf proc_leaf = cv_debug_t_get_leaf(&debug_t, 2); - T_Ok(proc_leaf.kind == CV_LeafKind_PROCEDURE); - T_Ok(proc_leaf.data.size == sizeof(CV_LeafProcedure)); - - CV_LeafProcedure *proc = (CV_LeafProcedure *)proc_leaf.data.str; - T_Ok(proc->ret_itype == 0x1000); - T_Ok(proc->call_kind == CV_CallKind_NearC); - } - - { - CV_Leaf ptr_leaf = cv_debug_t_get_leaf(&debug_t, 3); - T_Ok(ptr_leaf.kind == CV_LeafKind_POINTER); - T_Ok(ptr_leaf.data.size == sizeof(CV_LeafPointer)); - - CV_LeafPointer *ptr = (CV_LeafPointer *)ptr_leaf.data.str; - T_Ok(ptr->itype == CV_BasicType_SHORT); - T_Ok(ptr->attribs == 0); - } - - { - CV_Leaf proc_leaf = cv_debug_t_get_leaf(&debug_t, 4); - T_Ok(proc_leaf.kind == CV_LeafKind_PROCEDURE); - T_Ok(proc_leaf.data.size == sizeof(CV_LeafProcedure)); - - CV_LeafProcedure *proc = (CV_LeafProcedure *)proc_leaf.data.str; - T_Ok(proc->ret_itype == 0x1003); - T_Ok(proc->call_kind == CV_CallKind_NearC); - } -} - -TEST(cyclic_type) -{ - String8List *debug_t = push_array(arena, String8List, 1); - str8_serial_begin(arena, debug_t); - str8_serial_push_u32(arena, debug_t, CV_Signature_C13); - str8_serial_push_string(arena, debug_t, cv_make_leaf(arena, CV_LeafKind_POINTER, str8_struct((&(CV_LeafPointer){ .itype = 0x1001 })), CV_LeafAlign)); - str8_serial_push_string(arena, debug_t, cv_make_leaf(arena, CV_LeafKind_POINTER, str8_struct((&(CV_LeafPointer){ .itype = 0x1000 })), CV_LeafAlign)); - - CV_DebugS debug_s = {0}; - str8_list_push(arena, &debug_s.data_list[CV_C13SubSectionIdxKind_Symbols], cv_make_symbol(arena, CV_SymKind_GPROC32, cv_make_proc32(arena, (CV_SymProc32){ .itype = 0x1001 }, str8_lit("foo")))); - String8List debug_s_string = cv_data_from_debug_s_c13(arena, &debug_s, 1); - - String8 raw_coff = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", str8_serial_end(arena, debug_t), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_s", ".debug$S", str8_list_join(arena, &debug_s_string, 0), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_entry_obj()); - T_Ok(t_write_file(str8_lit("cycle.obj"), raw_coff)); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full /rad_ignore:-%u cycle.obj entry.obj", LNK_Error_InvalidTypeIndex); - T_Ok(g_last_exit_code == 0); - - String8 output = g_errors; - B32 is_cycle_detected = 0; - while (output.size && !is_cycle_detected) { - String8 line = str8_chop_line(&output); - is_cycle_detected = str8_match_wildcard(line, str8_lit("Error(*): *: LF_POINTER(type_index: *) forward refs member type index * (leaf struct offset: *)"), StringMatchFlag_CaseInsensitive); - t_chop_line(&output); - } - T_Ok(is_cycle_detected); -} - -TEST(get_msf_stream_pages) -{ - MSF_Context *msf = msf_alloc(MSF_DEFAULT_PAGE_SIZE, MSF_DEFAULT_FPM); - - { - U64 stream_size = MB(150) + 1; - - MSF_StreamNumber sn = msf_stream_alloc_ex(msf, stream_size); - - U8 *test = push_array(arena, U8, stream_size); - MemorySet(test, 0xca, stream_size/2); - MemorySet(test + stream_size/2, 0xbe, stream_size/2); - - String8List stream_data = msf_data_from_sn(arena, msf, sn); - T_Ok(stream_data.total_size == stream_size); - T_Ok(stream_data.node_count == 12); - - String8Array a = str8_array_from_list(arena, &stream_data); - T_Ok(a.v[0].size == 0xffd000); - T_Ok(a.v[1].size == 0xffe000); - T_Ok(a.v[2].size == 0xffe000); - T_Ok(a.v[3].size == 0xffe000); - T_Ok(a.v[4].size == 0xffe000); - T_Ok(a.v[5].size == 0xffe000); - T_Ok(a.v[6].size == 0xffe000); - T_Ok(a.v[7].size == 0xffd000); - T_Ok(a.v[8].size == 0x1000); - T_Ok(a.v[9].size == 0xffe000); - T_Ok(a.v[10].size == 0x613000); - T_Ok(a.v[11].size == 1); - - String8Node buf = *stream_data.first; - U64 buf_pos = 0; - str8_buffer_write(&buf, &buf_pos, str8(test, stream_size)); - - String8 cmp = msf_stream_read_block(arena, msf, sn, stream_size); - T_Ok(cmp.size == stream_size); - T_Ok(MemoryCompare(cmp.str, test, stream_size) == 0); - } - - { - MSF_StreamNumber sn = msf_stream_alloc_ex(msf, 1); - String8List stream_data = msf_data_from_sn(arena, msf, sn); - T_Ok(stream_data.node_count == 1); - T_Ok(stream_data.total_size == 1); - T_Ok(stream_data.first->string.size == 1); - } - - msf_release(msf); -} - -internal String8 -data_from_pdb(Arena *arena, PDB_Context *pdb) -{ - TP_Context *tp = tp_alloc(arena, 1, 1, str8_lit("foo")); - TP_Arena *tp_arena = tp_arena_alloc(tp); - pdb_build(tp, tp_arena, pdb, (CV_StringHashTable){0}, 1, 0, 0); - - AssertAlways(msf_build(pdb->msf) == MSF_Error_OK); - String8List raw_msf_list = msf_get_page_data_nodes(arena, pdb->msf); - AssertAlways(t_write_file_list(str8_lit("test.pdb"), raw_msf_list)); - - String8 data = str8_list_join(arena, &raw_msf_list, 0); - - tp_arena_release(&tp_arena); - tp_release(tp); - - return data; -} - -TEST(validate_info_stream) -{ - COFF_TimeStamp time_stamp = 123; - U32 age = 1; - Guid guid = { .data1 = max_U32, .data2 = max_U16 - 1, .data3 = max_U16 - 2, .data4 = { 1, 2, 3, 4, 5, 6, 7, 8 } }; - PDB_Context *pdb = pdb_alloc(MSF_DEFAULT_PAGE_SIZE, COFF_MachineType_X64, time_stamp, age, guid); - - char *stream_names[] = { "one", "two", "three", "four", "five" }; - MSF_StreamNumber stream_numbers[ArrayCount(stream_names)] = {0}; - - for EachElement(i, stream_names) { - stream_numbers[i] = pdb_push_named_stream(&pdb->info->named_stream_ht, pdb->msf, str8_cstring(stream_names[i])); - T_Ok(stream_numbers[i] != MSF_INVALID_STREAM_NUMBER); - } - - String8 raw_msf = data_from_pdb(arena, pdb); - MSF_Parsed *msf_parsed = msf_parsed_from_data(arena, raw_msf); - String8 info_data = msf_data_from_stream(msf_parsed, PDB_FixedStream_Info); - -#if 0 - fprintf(stderr, "\n"); - for EachIndex(i, info_data.size) { - fprintf(stderr, "0x%02x, ", info_data.str[i]); - if (i % 19 == 18 && i > 0) { fprintf(stderr, "\n"); } - } -#endif - U8 expected_info_data[] = { - 0x94, 0x2e, 0x31, 0x01, 0x7b, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xfd, - 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x22, 0x00, 0x00, 0x00, 0x6f, 0x6e, 0x65, 0x00, 0x74, 0x77, - 0x6f, 0x00, 0x74, 0x68, 0x72, 0x65, 0x65, 0x00, 0x66, 0x6f, 0x75, 0x72, 0x00, 0x66, 0x69, 0x76, 0x65, 0x00, 0x2f, - 0x4c, 0x69, 0x6e, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x13, - 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x51, 0x33, 0x01, - }; - T_Ok(str8_match(info_data, str8_array_fixed(expected_info_data), 0)); - - pdb_release(pdb); -} - -TEST(validate_gsi) -{ - String8 raw_symbols[] = { - cv_make_symbol(arena, CV_SymKind_CONSTANT, cv_make_const(arena, (CV_SymConstant){ .itype = 0 }, 263, str8_lit("CV_SymKind_BLOCK16"))), - cv_make_symbol(arena, CV_SymKind_GDATA32, cv_make_data32(arena, (CV_SymData32) { .itype = 0, .off = 0x25440, .sec = 2 }, str8_lit("__newclmap"))), - cv_make_symbol(arena, CV_SymKind_GDATA32, cv_make_data32(arena, (CV_SymData32) { .itype = 0, .off = 123, .sec = 1 }, str8_lit("coffeebabe"))), - cv_make_symbol(arena, CV_SymKind_GDATA32, cv_make_data32(arena, (CV_SymData32) { .itype = 0, .off = 123, .sec = 1 }, str8_lit("deadbeef"))), - }; - - CV_Symbol symbols[ArrayCount(raw_symbols)] = {0}; - for EachElement(i, raw_symbols) { symbols[i] = cv_symbol_from_ptr(raw_symbols[i].str); } - - CV_DebugS debug_s = {0}; - for EachElement(i, symbols) { str8_list_push(arena, &debug_s.data_list[CV_C13SubSectionIdxKind_Symbols], cv_data_from_symbol(arena, &symbols[i], CV_SymbolAlign)); } - String8List raw_debug_s_list = cv_data_from_debug_s_c13(arena, &debug_s, 1); - String8 raw_debug_s = str8_list_join(arena, &raw_debug_s_list, 0); - - T_Ok(t_write_def_obj("debug.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_s", ".debug$S", raw_debug_s, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - })); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /out:a.exe entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - String8 raw_pdb = t_read_file(arena, str8_lit("a.pdb")); - MSF_Parsed *msf = msf_parsed_from_data(arena, raw_pdb); - String8 dbi_data = msf_data_from_stream(msf, PDB_FixedStream_Dbi); - PDB_DbiParsed *dbi = pdb_dbi_from_data(arena, dbi_data); - String8 gsi_data = msf_data_from_stream(msf, dbi->gsi_sn); - PDB_GsiParsed *gsi = pdb_gsi_from_data(arena, gsi_data); - String8 symbol_data = msf_data_from_stream(msf, dbi->sym_sn); - - for EachElement(i, symbols) { - CV_Symbol symbol = symbols[i]; - - String8 string = cv_name_from_symbol(symbols[i].kind, symbols[i].data); - U64 symbol_off = pdb_gsi_symbol_from_string(gsi, symbol_data, string); - T_Ok(symbol_off < symbol_data.size); - - CV_Symbol test_symbol = {0}; - U64 test_symbol_size = cv_read_symbol(str8_skip(symbol_data, symbol_off), 0, 1, &test_symbol); - T_Ok(test_symbol_size > 0); - - String8 test_symbol_name = cv_name_from_symbol(test_symbol.kind, test_symbol.data); - T_Ok(str8_match(test_symbol_name, string, 0)); - } -} - -TEST(validate_gsi_procs_and_typedefs) -{ - String8 raw_symbols[] = { - cv_make_symbol(arena, CV_SymKind_OBJNAME, cv_make_obj_name(arena, str8_lit("debug.obj"), 0x123)), - cv_make_symbol(arena, CV_SymKind_GPROC32_ID, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("global_proc"))), - cv_make_symbol(arena, CV_SymKind_PROC_ID_END, cv_make_end(arena)), - - cv_make_symbol(arena, CV_SymKind_UDT, cv_make_udt(arena, (CV_SymUDT){0}, str8_lit("global_typedef"))), - - cv_make_symbol(arena, CV_SymKind_OBJNAME, cv_make_obj_name(arena, str8_lit("debug.obj"), 0x123)), - cv_make_symbol(arena, CV_SymKind_LPROC32_ID, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("local_proc"))), - cv_make_symbol(arena, CV_SymKind_UDT, cv_make_udt(arena, (CV_SymUDT){0}, str8_lit("local_typedef"))), - cv_make_symbol(arena, CV_SymKind_PROC_ID_END, cv_make_end(arena)), - }; - - CV_Symbol symbols[ArrayCount(raw_symbols)] = {0}; - for EachElement(i, raw_symbols) { symbols[i] = cv_symbol_from_ptr(raw_symbols[i].str); } - - CV_DebugS debug_s = {0}; - for EachElement(i, symbols) { str8_list_push(arena, &debug_s.data_list[CV_C13SubSectionIdxKind_Symbols], cv_data_from_symbol(arena, &symbols[i], CV_SymbolAlign)); } - String8List raw_debug_s_list = cv_data_from_debug_s_c13(arena, &debug_s, 1); - String8 raw_debug_s = str8_list_join(arena, &raw_debug_s_list, 0); - - T_Ok(t_write_def_obj("debug.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_s", ".debug$S", raw_debug_s, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - })); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /out:a.exe entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - String8 raw_pdb = t_read_file(arena, str8_lit("a.pdb")); - MSF_Parsed *msf = msf_parsed_from_data(arena, raw_pdb); - String8 dbi_data = msf_data_from_stream(msf, PDB_FixedStream_Dbi); - PDB_DbiParsed *dbi = pdb_dbi_from_data(arena, dbi_data); - String8 gsi_data = msf_data_from_stream(msf, dbi->gsi_sn); - PDB_GsiParsed *gsi = pdb_gsi_from_data(arena, gsi_data); - String8 symbol_data = msf_data_from_stream(msf, dbi->sym_sn); - - struct { - char *name; - B32 is_global; - U64 offset; - U64 imod; - } procs[] = { - { "global_proc", 1, 0x18, 2, }, - { "local_proc", 0, 0x64, 2, }, - }; - for EachElement(i, procs) { - U64 symbol_off = pdb_gsi_symbol_from_string(gsi, symbol_data, str8_cstring(procs[i].name)); - T_Ok(symbol_off < symbol_data.size); - - CV_Symbol test_symbol = {0}; - U64 test_symbol_size = cv_read_symbol(str8_skip(symbol_data, symbol_off), 0, 1, &test_symbol); - String8 test_symbol_name = cv_name_from_symbol(test_symbol.kind, test_symbol.data); - T_Ok(test_symbol_size > 0); - T_Ok(str8_match(str8_cstring(procs[i].name), test_symbol_name, 0)); - T_Ok(test_symbol.kind == (procs[i].is_global ? CV_SymKind_PROCREF : CV_SymKind_LPROCREF)); - - CV_SymRef2 *proc_ref = str8_deserial_get_raw_ptr(test_symbol.data, 0, sizeof(*proc_ref)); - T_Ok(proc_ref->suc_name == 0); - T_Ok(proc_ref->sym_off == procs[i].offset); - T_Ok(proc_ref->imod == procs[i].imod); - } - - struct { - char *name; - B32 is_global; - } typedefs[] = { - { "global_typedef", 1 }, - { "local_typedef", 0 }, - }; - for EachElement(i, typedefs) { - U64 symbol_off = pdb_gsi_symbol_from_string(gsi, symbol_data, str8_cstring(typedefs[i].name)); - if (typedefs[i].is_global) { - T_Ok(symbol_off < symbol_data.size); - CV_Symbol test_symbol = {0}; - U64 test_symbol_size = cv_read_symbol(str8_skip(symbol_data, symbol_off), 0, 1, &test_symbol); - String8 test_symbol_name = cv_name_from_symbol(test_symbol.kind, test_symbol.data); - T_Ok(test_symbol_size > 0); - T_Ok(str8_match(str8_cstring(typedefs[i].name), test_symbol_name, 0)); - T_Ok(test_symbol.kind == CV_SymKind_UDT); - } else { - T_Ok(symbol_off >= symbol_data.size); - } - } -} - -TEST(validate_psi) -{ - T_Ok(t_write_def_obj("test.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit("FOOBAR"), .flags = "rx:code" }, - { "data", ".data", str8_lit("QWE"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("global_func", "text", 1), - T_COFF_DefSymbol_Extern("global_var", "data", 1), - T_COFF_DefSymbol_Static("static_var", "data", 1), - {0} - } - })); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full entry.obj test.obj"); - T_Ok(g_last_exit_code == 0); - - String8 raw_pdb = t_read_file(arena, str8_lit("a.pdb")); - MSF_Parsed *msf = msf_parsed_from_data(arena, raw_pdb); - String8 dbi_data = msf_data_from_stream(msf, PDB_FixedStream_Dbi); - PDB_DbiParsed *dbi = pdb_dbi_from_data(arena, dbi_data); - String8 psi_data = msf_data_from_stream(msf, dbi->psi_sn); - String8 psi_gsi_data = str8_range(psi_data.str + sizeof(PDB_PsiHeader), psi_data.str+psi_data.size); - PDB_GsiParsed *psi = pdb_gsi_from_data(arena, psi_gsi_data); - String8 symbol_data = msf_data_from_stream(msf, dbi->sym_sn); - - struct { - char *name; - B32 is_global; - CV_Pub32Flags flags; - } pubs[] = { - { "global_func", 1, CV_Pub32Flag_Function }, - { "global_var", 1, 0 }, - { "static_var", 0, 0 }, - }; - for EachElement(i, pubs) { - U64 symbol_off = pdb_gsi_symbol_from_string(psi, symbol_data, str8_cstring(pubs[i].name)); - if (pubs[i].is_global) { - T_Ok(symbol_off < symbol_data.size); - CV_Symbol test_symbol = {0}; - U64 test_symbol_size = cv_read_symbol(str8_skip(symbol_data, symbol_off), 0, 1, &test_symbol); - String8 test_symbol_name = cv_name_from_symbol(test_symbol.kind, test_symbol.data); - T_Ok(test_symbol_size > 0); - T_Ok(str8_match(str8_cstring(pubs[i].name), test_symbol_name, 0)); - T_Ok(test_symbol.kind == CV_SymKind_PUB32); - CV_SymPub32 *pub32 = str8_deserial_get_raw_ptr(test_symbol.data, 0, sizeof(*pub32)); - T_Ok(pub32->sec > 0); - T_Ok(pubs[i].flags == pub32->flags); - } else { - T_Ok(symbol_off >= symbol_data.size); - } - } - - U64 pub32_count = 0; - for EachElement(i, psi->buckets) { - PDB_GsiBucket bucket = psi->buckets[i]; - for EachIndex(k, bucket.count) { - CV_Symbol test_symbol = {0}; - U64 test_symbol_size = cv_read_symbol(str8_skip(symbol_data, bucket.offs[k]), 0, 1, &test_symbol); - T_Ok(test_symbol_size >= sizeof(CV_SymPub32)); - T_Ok(test_symbol.kind == CV_SymKind_PUB32); - pub32_count += 1; - } - } - T_Ok(pub32_count > 0); -} - -TEST(psi_addr_map_radix_sort) -{ - String8 names[] = { - str8_lit("alpha"), - str8_lit("bravo"), - str8_lit("charlie"), - str8_lit("delta"), - }; - U64 address_count = (1 << 15) + 1; - U64 record_count = address_count * ArrayCount(names); - - PDB_GsiSortRecord *records = push_array_no_zero(arena, PDB_GsiSortRecord, record_count); - PDB_GsiSortRecord *expected = push_array_no_zero(arena, PDB_GsiSortRecord, record_count); - for EachIndex(i, record_count) { - U64 address_idx = address_count - 1 - i / ArrayCount(names); - records[i].isect_off.isect = 1 + address_idx % 257; - records[i].isect_off.off = address_idx / 257; - records[i].name = names[ArrayCount(names) - 1 - i % ArrayCount(names)]; - records[i].offset = i * sizeof(U32); - } - MemoryCopyTyped(expected, records, record_count); - radsort(expected, record_count, psi_addr_map_compar_is_before); - - TP_Context *tp = tp_alloc(arena, 1, 1, str8_lit("psi addr map sort test")); - U32 *addr_map = psi_addr_map_from_gsi_records(tp, arena, records, record_count); - - for EachIndex(i, record_count) { - T_Ok(addr_map[i] == expected[i].offset); - } - tp_release(tp); -} - -TEST(u64_array_radix_sort_parallel) -{ - TP_Context *tp = tp_alloc(arena, 1, 1, str8_zero()); - - U64 small_values[] = { max_U64, 0, 7, 1, 7, 0x100000000ull, 2 }; - u64_array_sort_radix_parallel(tp, ArrayCount(small_values), small_values); - for (U64 i = 1; i < ArrayCount(small_values); i += 1) { - T_Ok(small_values[i-1] <= small_values[i]); - } - - U64 count = 200003; - U64 *values = push_array_no_zero(arena, U64, count); - U64 state = 0x9e3779b97f4a7c15ull; - U64 sum_before = 0; - U64 xor_before = 0; - for EachIndex(i, count) { - state ^= state >> 12; - state ^= state << 25; - state ^= state >> 27; - values[i] = state * 0x2545f4914f6cdd1dull; - if ((i % 97) == 0) { values[i] = 0; } - if ((i % 193) == 0) { values[i] = max_U64; } - if ((i % 389) == 0) { values[i] = 0x100000001ull; } - sum_before += values[i]; - xor_before ^= values[i]; - } - - u64_array_sort_radix_parallel(tp, count, values); - U64 sum_after = 0; - U64 xor_after = 0; - B32 is_sorted = 1; - for EachIndex(i, count) { - if (i > 0 && values[i-1] > values[i]) { is_sorted = 0; } - sum_after += values[i]; - xor_after ^= values[i]; - } - T_Ok(is_sorted); - T_Ok(sum_before == sum_after); - T_Ok(xor_before == xor_after); - tp_release(tp); -} - -TEST(pdbstripped) -{ - String8 debug_obj; - { - String8 raw_symbols[] = { - cv_make_symbol(arena, CV_SymKind_OBJNAME, cv_make_obj_name(arena, str8_lit("debug.obj"), 0x123)), - cv_make_symbol(arena, CV_SymKind_GPROC32_ID, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("global_proc"))), - cv_make_symbol(arena, CV_SymKind_PROC_ID_END, cv_make_end(arena)), - - cv_make_symbol(arena, CV_SymKind_UDT, cv_make_udt(arena, (CV_SymUDT){0}, str8_lit("global_typedef"))), - - cv_make_symbol(arena, CV_SymKind_LPROC32_ID, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("local_proc"))), - cv_make_symbol(arena, CV_SymKind_UDT, cv_make_udt(arena, (CV_SymUDT){0}, str8_lit("local_typedef"))), - cv_make_symbol(arena, CV_SymKind_PROC_ID_END, cv_make_end(arena)), - }; - - CV_Symbol symbols[ArrayCount(raw_symbols)] = {0}; - for EachElement(i, raw_symbols) { symbols[i] = cv_symbol_from_ptr(raw_symbols[i].str); } - - CV_DebugS debug_s = {0}; - for EachElement(i, symbols) { str8_list_push(arena, &debug_s.data_list[CV_C13SubSectionIdxKind_Symbols], cv_data_from_symbol(arena, &symbols[i], CV_SymbolAlign)); } - String8List raw_debug_s_list = cv_data_from_debug_s_c13(arena, &debug_s, 1); - String8 raw_debug_s = str8_list_join(arena, &raw_debug_s_list, 0); - - debug_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_s", ".debug$S", raw_debug_s, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - } - - String8 pub_obj; - { - pub_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit("FOOBAR"), .flags = "rx:code" }, - { "data", ".data", str8_lit("QWE"), .flags = "rw:data" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("global_func", "text", 1), - T_COFF_DefSymbol_Extern("global_var", "data", 1), - T_COFF_DefSymbol_Static("static_var", "data", 1), - {0} - } - }); - } - - T_Ok(t_write_file(str8_lit("debug.obj"), debug_obj)); - T_Ok(t_write_file(str8_lit("pub.obj"), pub_obj)); - T_Ok(t_write_entry_obj()); - t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /out:a.exe /pdbstripped:a.stripped.pdb entry.obj pub.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - String8 raw_pdb = t_read_file(arena, str8_lit("a.stripped.pdb")); - MSF_Parsed *msf = msf_parsed_from_data(arena, raw_pdb); - String8 dbi_data = msf_data_from_stream(msf, PDB_FixedStream_Dbi); - PDB_DbiParsed *dbi = pdb_dbi_from_data(arena, dbi_data); - - String8 mods_data = pdb_data_from_dbi_range(dbi, PDB_DbiRange_ModuleInfo); - PDB_CompUnitArray *mods = pdb_comp_unit_array_from_data(arena, mods_data); - T_Ok(mods->count > 0); - - // modules must contain only stubs for static procs - for EachIndex(i, mods->count) { - PDB_CompUnit *mod = mods->units[i]; - U64 sym_data_size = mod->range_off[PDB_DbiCompUnitRange_Symbols + 1] - mod->range_off[PDB_DbiCompUnitRange_Symbols]; - U64 c11_data_size = mod->range_off[PDB_DbiCompUnitRange_C11 + 1] - mod->range_off[PDB_DbiCompUnitRange_C11]; - U64 c13_data_size = mod->range_off[PDB_DbiCompUnitRange_C13 + 1] - mod->range_off[PDB_DbiCompUnitRange_C13]; - T_Ok(c11_data_size == 0); - T_Ok(c13_data_size == 0); - if (str8_match(str8_skip_last_slash(mod->obj_name), str8_lit("debug.obj"), 0)) { - T_Ok(sym_data_size > 0); - } else { - T_Ok(sym_data_size == 0); - } - - String8 mod_data = msf_data_from_stream(msf, mod->sn); - String8 sym_data = str8_substr(mod_data, r1u64(mod->range_off[PDB_DbiCompUnitRange_Symbols], mod->range_off[PDB_DbiCompUnitRange_Symbols + 1])); - for (U64 cursor = 0; cursor < sym_data_size; ) { - CV_Symbol symbol = {0}; - U64 read_size = cv_read_symbol(sym_data, cursor, PDB_SYMBOL_ALIGN, &symbol); - T_Ok(read_size > 0); - cursor += read_size; - T_Ok(symbol.kind == CV_SymKind_LPROC32 || symbol.kind == CV_SymKind_END); - } - } - - // global symbol stream must have public and references to the static stubs - String8 symbol_data = msf_data_from_stream(msf, dbi->sym_sn); - for (U64 cursor = 0; cursor < symbol_data.size; ) { - CV_Symbol symbol = {0}; - U64 read_size = cv_read_symbol(symbol_data, cursor, PDB_SYMBOL_ALIGN, &symbol); - T_Ok(read_size > 0); - cursor += read_size; - T_Ok(symbol.kind == CV_SymKind_PUB32 || - symbol.kind == CV_SymKind_LPROCREF); - } - - // types must be stripped - String8 tpi = msf_data_from_stream(msf, PDB_FixedStream_Tpi); - T_Ok(tpi.size == sizeof(PDB_TpiHeader)); - String8 ipi = msf_data_from_stream(msf, PDB_FixedStream_Ipi); - T_Ok(ipi.size == sizeof(PDB_TpiHeader)); -} - -TEST(ghash_check_corrupt) -{ - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - - String8List h = {0}; str8_serial_begin(arena, &h); - String8 debug_h = str8_serial_end(arena, &h); - - String8 debug_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_file(str8_lit("debug.obj"), debug_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - B32 is_warning_found = 0; - String8 debug_obj_path = t_make_file_path(arena, str8_lit("debug.obj")); - String8 expected_line = str8f(arena, "Warning(*): *: .debug$H section is too small to contain the header"); - for (String8 i = g_errors; i.size > 0 && !is_warning_found; ) { - String8 line = t_chop_line(&i); - is_warning_found = str8_match_wildcard(line, expected_line, StringMatchFlag_CaseInsensitive); - } - T_Ok(is_warning_found); -} - -TEST(ghash_check_magic) -{ - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - -#if 0 - U64 *hashes = push_array(arena, U64, t.node_count); - U64 i = 0; - for EachNode(n, String8Node, t.first->next) { - blake3(&hashes[i], sizeof(hashes[i]), n->string.str, n->string.size); - i += 1; - } -#endif - String8List h = {0}; str8_serial_begin(arena, &h); - str8_serial_push_struct(arena, &h, (&(LLVM_GHash){ .magic = 123, .hash_alg = LLVM_GHashAlg_BLAKE3, .version = LLVM_GHash_CurrentVersion })); - str8_serial_push_u64(arena, &h, 0x6ebacae08af4fda5ull); - str8_serial_push_u64(arena, &h, 0xc385876694f9769aull); - str8_serial_push_u64(arena, &h, 0x7ea8a529a89b2288ull); - String8 debug_h = str8_serial_end(arena, &h); - - String8 debug_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_file(str8_lit("debug.obj"), debug_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - B32 is_warning_found = 0; - String8 debug_obj_path = t_make_file_path(arena, str8_lit("debug.obj")); - String8 expected_line = str8f(arena, "Warning(*): *: .debug$H contains invalid magic: got 0x7b, expected 0x%x", LLVM_GHash_Magic); - for (String8 i = g_errors; i.size > 0 && !is_warning_found; ) { - String8 line = t_chop_line(&i); - is_warning_found = str8_match_wildcard(line, expected_line, StringMatchFlag_CaseInsensitive); - } - T_Ok(is_warning_found); -} - -TEST(ghash_check_version) -{ - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - -#if 0 - U64 *hashes = push_array(arena, U64, t.node_count); - U64 i = 0; - for EachNode(n, String8Node, t.first->next) { - blake3(&hashes[i], sizeof(hashes[i]), n->string.str, n->string.size); - i += 1; - } -#endif - String8List h = {0}; str8_serial_begin(arena, &h); - str8_serial_push_struct(arena, &h, (&(LLVM_GHash){ .magic = LLVM_GHash_Magic, .hash_alg = LLVM_GHashAlg_BLAKE3, .version = 0xbeef })); - str8_serial_push_u64(arena, &h, 0x6ebacae08af4fda5ull); - str8_serial_push_u64(arena, &h, 0xc385876694f9769aull); - str8_serial_push_u64(arena, &h, 0x7ea8a529a89b2288ull); - String8 debug_h = str8_serial_end(arena, &h); - - String8 debug_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_file(str8_lit("debug.obj"), debug_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - B32 is_warning_found = 0; - String8 debug_obj_path = t_make_file_path(arena, str8_lit("debug.obj")); - String8 expected_line = str8f(arena, "Warning(*): *: mismatched .debug$H version: got %u, expected %u", 0xbeef, LLVM_GHash_CurrentVersion); - for (String8 i = g_errors; i.size > 0 && !is_warning_found; ) { - String8 line = t_chop_line(&i); - is_warning_found = str8_match_wildcard(line, expected_line, StringMatchFlag_CaseInsensitive); - } - T_Ok(is_warning_found); -} - -TEST(ghash_check_hash_alg) -{ - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - - String8List h = {0}; str8_serial_begin(arena, &h); - str8_serial_push_struct(arena, &h, (&(LLVM_GHash){ .magic = LLVM_GHash_Magic, .hash_alg = LLVM_GHashAlg_SHA1_8, .version = LLVM_GHash_CurrentVersion })); - str8_serial_push_u64(arena, &h, 0x6ebacae08af4fda5ull); - str8_serial_push_u64(arena, &h, 0xc385876694f9769aull); - str8_serial_push_u64(arena, &h, 0x7ea8a529a89b2288ull); - String8 debug_h = str8_serial_end(arena, &h); - - String8 debug_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_file(str8_lit("debug.obj"), debug_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - B32 is_warning_found = 0; - String8 debug_obj_path = t_make_file_path(arena, str8_lit("debug.obj")); - String8 expected_line = str8f(arena, "Warning(*): *: mismatched .debug$H hash algorithm: got SHA1_8, expected *"); - for (String8 i = g_errors; i.size > 0 && !is_warning_found; ) { - String8 line = t_chop_line(&i); - is_warning_found = str8_match_wildcard(line, expected_line, StringMatchFlag_CaseInsensitive); - } - T_Ok(is_warning_found); -} - -TEST(ghash_match_debug_t) -{ - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - //str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - - String8List h = {0}; str8_serial_begin(arena, &h); - str8_serial_push_struct(arena, &h, (&(LLVM_GHash){ .magic = LLVM_GHash_Magic, .hash_alg = LLVM_GHashAlg_BLAKE3, .version = LLVM_GHash_CurrentVersion })); - str8_serial_push_u64(arena, &h, 0x6ebacae08af4fda5ull); - str8_serial_push_u64(arena, &h, 0xc385876694f9769aull); - str8_serial_push_u64(arena, &h, 0x7ea8a529a89b2288ull); - String8 debug_h = str8_serial_end(arena, &h); - - String8 debug_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - - T_Ok(t_write_file(str8_lit("debug.obj"), debug_obj)); - T_Ok(t_write_entry_obj()); - - String8 output = {0}; - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj debug.obj"); - T_Ok(g_last_exit_code == 0); - - B32 is_warning_found = 0; - String8 debug_obj_path = t_make_file_path(arena, str8_lit("debug.obj")); - String8 expected_line = str8f(arena, "Warning(*): *: mismatched .debug$H hash count and type count: got 3 hashes for 2 types"); - for (String8 i = g_errors; i.size > 0 && !is_warning_found; ) { - String8 line = t_chop_line(&i); - is_warning_found = str8_match_wildcard(line, expected_line, StringMatchFlag_CaseInsensitive); - } - T_Ok(is_warning_found); -} - -TEST(ghash_basic) -{ - String8 a_obj; - { - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - - String8List h = {0}; str8_serial_begin(arena, &h); - str8_serial_push_struct(arena, &h, (&(LLVM_GHash){ .magic = LLVM_GHash_Magic, .hash_alg = LLVM_GHashAlg_BLAKE3, .version = LLVM_GHash_CurrentVersion })); - str8_serial_push_u64(arena, &h, 1); - str8_serial_push_u64(arena, &h, 2); - str8_serial_push_u64(arena, &h, 3); - String8 debug_h = str8_serial_end(arena, &h); - - a_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - } - - String8 b_obj; - { - String8List t = {0}; str8_serial_begin(arena, &t); - str8_serial_push_u32(arena, &t, CV_Signature_C13); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_STRUCTURE, str8_struct(&(CV_LeafStruct){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_UNION, str8_struct(&(CV_LeafUnion){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - str8_serial_push_string(arena, &t, cv_make_leaf(arena, CV_LeafKind_ENUM, str8_struct(&(CV_LeafEnum){ .props = CV_TypeProp_FwdRef }), CV_LeafAlign)); - String8 debug_t = str8_serial_end(arena, &t); - - String8List h = {0}; str8_serial_begin(arena, &h); - str8_serial_push_struct(arena, &h, (&(LLVM_GHash){ .magic = LLVM_GHash_Magic, .hash_alg = LLVM_GHashAlg_BLAKE3, .version = LLVM_GHash_CurrentVersion })); - str8_serial_push_u64(arena, &h, 4); - str8_serial_push_u64(arena, &h, 5); - str8_serial_push_u64(arena, &h, 6); - String8 debug_h = str8_serial_end(arena, &h); - - b_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "debug_t", ".debug$T", debug_t, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - { "debug_h", ".debug$H", debug_h, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable }, - {0} - } - }); - } - - T_Ok(t_write_file(str8_lit("a.obj"), a_obj)); - T_Ok(t_write_file(str8_lit("b.obj"), b_obj)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:ghash entry.obj a.obj b.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 raw_pdb = t_read_file(arena, str8_lit("a.pdb")); - MSF_Parsed *msf_parsed = msf_parsed_from_data(arena, raw_pdb); - String8 raw_tpi = msf_data_from_stream(msf_parsed, PDB_FixedStream_Tpi); - PDB_TpiParsed *tpi = pdb_tpi_from_data(arena, raw_tpi); - U64 leaf_count = tpi->itype_opl - tpi->itype_first; - T_Ok(leaf_count == 6); - } - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /debug:full entry.obj a.obj b.obj"); - T_Ok(g_last_exit_code == 0); - - { - String8 raw_pdb = t_read_file(arena, str8_lit("a.pdb")); - MSF_Parsed *msf_parsed = msf_parsed_from_data(arena, raw_pdb); - String8 raw_tpi = msf_data_from_stream(msf_parsed, PDB_FixedStream_Tpi); - PDB_TpiParsed *tpi = pdb_tpi_from_data(arena, raw_tpi); - U64 leaf_count = tpi->itype_opl - tpi->itype_first; - T_Ok(leaf_count == 3); - } -} - -TEST(patch_cv_symbol_tree) -{ - String8List raw_symbols = {0}; - str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_OBJNAME, cv_make_obj_name(arena, str8_lit("foo.obj"), 123))); - str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_GPROC32, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("Proc")))); - str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_INLINESITE, cv_make_inline_site(arena, (CV_SymInlineSite){0}, str8_zero()))); - str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_INLINESITE_END, cv_make_inline_site_end(arena))); - str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_END, cv_make_end(arena))); - - U64 tree_size = cv_patch_symbol_tree_offsets(raw_symbols, sizeof(CV_Signature), 4); - T_Ok(tree_size == 84); - - { - String8Node buf = *raw_symbols.first; - U64 buf_pos = 0; - - CV_SymbolHeader obj_header; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(obj_header), &obj_header) == sizeof(obj_header)); - T_Ok(obj_header.kind == CV_SymKind_OBJNAME); - T_Ok(str8_buffer_skip(&buf, &buf_pos, obj_header.size - sizeof(CV_SymKind))); - - CV_SymbolHeader proc_header; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(proc_header), &proc_header) == sizeof(proc_header)); - T_Ok(proc_header.kind == CV_SymKind_GPROC32); - - CV_SymProc32 proc; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(proc), &proc) == sizeof(proc)); - T_Ok(proc.end == 0x54); - T_Ok(str8_buffer_skip(&buf, &buf_pos, proc_header.size - sizeof(CV_SymKind) - sizeof(proc))); - - CV_SymbolHeader inline_site_header; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(inline_site_header), &inline_site_header) == sizeof(inline_site_header)); - T_Ok(inline_site_header.kind == CV_SymKind_INLINESITE); - - CV_SymInlineSite inline_site; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(inline_site), &inline_site)); - T_Ok(inline_site.parent == 0x14); - T_Ok(inline_site.end == 0x50); - T_Ok(str8_buffer_skip(&buf, &buf_pos, inline_site_header.size - sizeof(CV_SymKind) - sizeof(inline_site))); - - CV_SymbolHeader inline_end_header; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(inline_end_header), &inline_end_header) == sizeof(inline_end_header)); - T_Ok(inline_end_header.kind == CV_SymKind_INLINESITE_END); - - CV_SymbolHeader proc_end_header; - T_Ok(str8_buffer_read(&buf, &buf_pos, sizeof(proc_end_header), &proc_end_header) == sizeof(proc_end_header)); - T_Ok(proc_end_header.kind == CV_SymKind_END); - - T_Ok(buf.string.size == 0); - T_Ok(buf.string.str == 0); - T_Ok(buf_pos == 0); - } -} - -TEST(whole_archive) -{ - T_Ok(t_write_entry_obj()); - T_Ok(t_write_def_lib("a.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("a.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".a", str8_lit("a"), .flags = "rw:data" }, - {0} - } - } - }, - {0} - } - })); - T_Ok(t_write_def_lib("b.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("b.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "b", ".b", str8_lit("b"), .flags = "rw:data" }, - {0} - } - } - }, - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:all_libs.exe entry.obj /wholearchive a.lib b.lib"); - T_Ok(g_last_exit_code == 0); - { - String8 exe = t_read_file(arena, str8_lit("all_libs.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *a_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a")); - COFF_SectionHeader *b_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".b")); - T_Ok(a_sect != 0); - T_Ok(b_sect != 0); - } - - t_invoke_linkerf("/subsystem:console /entry:entry /out:only_a.exe entry.obj /wholearchive:a.lib a.lib b.lib"); - T_Ok(g_last_exit_code == 0); - { - String8 exe = t_read_file(arena, str8_lit("only_a.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *a_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a")); - COFF_SectionHeader *b_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".b")); - T_Ok(a_sect != 0); - T_Ok(b_sect == 0); - } - - t_invoke_linkerf("/subsystem:console /entry:entry /out:only_b.exe /wholearchive:b.lib a.lib b.lib entry.obj"); - T_Ok(g_last_exit_code == 0); - { - String8 exe = t_read_file(arena, str8_lit("only_b.exe")); - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *a_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".a")); - COFF_SectionHeader *b_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".b")); - T_Ok(a_sect == 0); - T_Ok(b_sect != 0); - } -} -#if OS_WINDOWS - -internal B32 -t_radlink_validate_asan_out(String8 obj_name) -{ - Temp scratch = scratch_begin(0,0); - B32 is_ok = 0; - - t_invoke_linkerf("%S /debug:full", obj_name); - if (g_last_exit_code != 0) { goto exit; } - - String8 exe_path = t_make_file_path(scratch.arena, str8f(scratch.arena, "%S.exe", str8_chop_last_dot(obj_name))); - - char *old_path_cstr = getenv("PATH"); - String8List env = {0}; - str8_list_pushf(scratch.arena, &env, "PATH=%S;%S", str8_chop_last_slash(t_cl_path()), str8_cstring(old_path_cstr)); - t_invoke_env(exe_path, str8_zero(), env, max_U64); - - String8 s = g_errors; - - String8 header = t_chop_line(&s); - if ( ! str8_match(header, str8_lit("================================================================="), 0)) { - goto exit; - } - - String8 cause = t_chop_line(&s); - if ( str8_find_needle(cause, 0, str8_lit("AddressSanitizer: heap-use-after-free on address"), 0) >= cause.size) { - goto exit; - } - - is_ok = 1; - exit:; - scratch_end(scratch); - return is_ok; -} - -TEST(infer_asan) -{ - char *program = - "#include \n" - " int main(void) {\n" - "int *foo = malloc(sizeof(*foo));\n" - "free(foo);\n" - "*foo = 1;\n" - "}\n" - ; - - // /MD - { - T_Ok(t_write_file(str8_lit("main.c"), str8_cstring(program))); - t_invoke_cl("/MD /fsanitize=address /Z7 /c /Fo:main_md.obj main.c"); - T_Ok(g_last_exit_code == 0); - T_Ok(t_radlink_validate_asan_out(str8_lit("main_md.obj"))); - } - - // /MDd - { - T_Ok(t_write_file(str8_lit("main.c"), str8_cstring(program))); - t_invoke_cl("/MDd /fsanitize=address /Z7 /c /Fo:main_mdd.obj main.c"); - T_Ok(g_last_exit_code == 0); - T_Ok(t_radlink_validate_asan_out(str8_lit("main_mdd.obj"))); - } - - // /MT - { - T_Ok(t_write_file(str8_lit("main.c"), str8_cstring(program))); - t_invoke_cl("/MT /fsanitize=address /Z7 /c /Fo:main_mt.obj main.c"); - T_Ok(g_last_exit_code == 0); - T_Ok(t_radlink_validate_asan_out(str8_lit("main_mt.obj"))); - } - - // /MTd - { - T_Ok(t_write_file(str8_lit("main.c"), str8_cstring(program))); - t_invoke_cl("/MT /fsanitize=address /Z7 /c /Fo:main_mtd.obj main.c"); - T_Ok(g_last_exit_code == 0); - T_Ok(t_radlink_validate_asan_out(str8_lit("main_mtd.obj"))); - } -} - -#endif - -#if OS_WINDOWS -TEST(determ_test) -{ - // compile the test target (torture) - t_invoke_cl("/fsanitize=address /c /Z7 /Fo:test.obj -I%S /Zc:preprocessor %S/torture/torture_main.c", t_src_path(), t_src_path()); - T_Ok(g_last_exit_code == 0); - - U64 run_count = 25; - T_Ok(run_count > 1); - String8 test_path = t_make_file_path(arena, str8_lit("test.obj")); - - // single-threaded link - t_invoke_linkerf("%S /debug:full /rad_time_stamp:0 /rad_workers:1 /pdbaltpath:main.pdb /rad_log:-all /rad_ignore:74 /out:main.exe", test_path); - T_Ok(g_last_exit_code == 0); - - // read b - String8 main_exe = t_read_file(arena, str8_lit("main.exe")); - String8 main_pdb = t_read_file(arena, str8_lit("main.pdb")); - - // multi-threaded links - ProcessList linkers = {0}; - for EachIndex(i, run_count) { - String8 out_path = t_make_file_path(arena, str8f(arena, "%llu.exe", i)); - String8 cmdl = str8f(arena, "%S %S /debug:full /rad_time_stamp:0 /rad_imagealtpath:main.exe /pdbaltpath:main.pdb /rad_log:-all /rad_ignore:74 /out:%S", t_radlink_path(), test_path, out_path); - Process process_handle = launch_cmd_line(cmdl); - T_Ok(!process_match(process_zero(), process_handle)); - process_list_push(arena, &linkers, process_handle); - } - - // wait for linkers - for EachNode(n, ProcessNode, linkers.first) { process_join(n->v, max_U64, 0); } - - for EachIndex(i, run_count) { - Temp temp = temp_begin(arena); - String8 exe = t_read_file(temp.arena, str8f(temp.arena, "%llu.exe", i)); - String8 pdb = t_read_file(temp.arena, str8f(temp.arena, "%llu.pdb", i)); - T_Ok(exe.size); - T_Ok(pdb.size); - T_Ok(str8_match(main_exe, exe, 0)); - T_Ok(str8_match(main_pdb, pdb, 0)); - temp_end(temp); - } -} - -#endif - -internal B32 t_read_exe_data_vaddrs(Arena *arena, String8 exe_path, U64 *vaddrs, U64 count); - -#if OS_WINDOWS -TEST(ms_link_icfs_identical_comdats) -{ - U8 same_text[] = { - 0x48, 0x31, 0xc0, // xor rax, rax - 0xc3 // ret - }; - U8 entry_text[] = { - 0xc3, // ret - }; - U8 addresses[] = { - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - }; - - T_Ok(t_write_def_obj("ms_icf.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text$mn", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "a", ".text$mn", str8_array_fixed(same_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "b", ".text$mn", str8_array_fixed(same_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "a"), - T_COFF_DefReloc(X64_Addr64, 8, "b"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_ExternFunc("a", "a", 0), - T_COFF_DefSymbol_ExternFunc("b", "b", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - B32 is_invoke_ok = t_invoke(str8_lit("link.exe"), str8_lit("/nologo /nodefaultlib /subsystem:console /entry:entry /out:ms_icf.exe /opt:ref,icf /include:a /include:b /include:addresses ms_icf.obj"), max_U64); - T_Ok(is_invoke_ok); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("ms_icf.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_section != 0); - T_Ok(data_section->foff + sizeof(addresses) <= exe.size); - - String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + sizeof(addresses))); - U64 a_addr = 0; - U64 b_addr = 0; - str8_deserial_read_struct(data, 0, &a_addr); - str8_deserial_read_struct(data, 8, &b_addr); - T_Ok(a_addr != 0); - T_Ok(a_addr == b_addr); // COMDAT are folded -} - -#if 0 -TEST(ms_link_icf_section_flag_eligibility) -{ - U8 ret_text[] = { - 0xc3, // ret - }; - U8 data_bytes[] = { - 1, 2, 3, 4, 5, 6, 7, 8, - }; - U8 addresses[] = { - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - }; - - T_Ok(t_write_def_obj("ms_icf_flags.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1" }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "code_comdat_a"), - T_COFF_DefReloc(X64_Addr64, 8, "code_comdat_b"), - T_COFF_DefReloc(X64_Addr64, 16, "code_plain_a"), - T_COFF_DefReloc(X64_Addr64, 24, "code_plain_b"), - T_COFF_DefReloc(X64_Addr64, 32, "rdata_comdat_a"), - T_COFF_DefReloc(X64_Addr64, 40, "rdata_comdat_b"), - T_COFF_DefReloc(X64_Addr64, 48, "wdata_comdat_a"), - T_COFF_DefReloc(X64_Addr64, 56, "wdata_comdat_b"), - T_COFF_DefReloc(X64_Addr64, 64, "ro_code_comdat_a"), - T_COFF_DefReloc(X64_Addr64, 72, "ro_code_comdat_b"), - T_COFF_DefReloc(X64_Addr64, 80, "rw_code_comdat_a"), - T_COFF_DefReloc(X64_Addr64, 88, "rw_code_comdat_b"), - {0} - } - }, - { "code_comdat_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "code_comdat_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "code_plain_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1" }, - { "code_plain_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1" }, - { "ro_code_comdat_a", ".text$mn", str8_array_fixed(ret_text), .flags = "r:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "ro_code_comdat_b", ".text$mn", str8_array_fixed(ret_text), .flags = "r:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "rw_code_comdat_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rw:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "rw_code_comdat_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rw:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "rdata_comdat_a", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "rdata_comdat_b", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "wdata_comdat_a", ".data$mn", str8_array_fixed(data_bytes), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "wdata_comdat_b", ".data$mn", str8_array_fixed(data_bytes), .flags = "rw:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("code_comdat_a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("code_comdat_b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("rdata_comdat_a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("rdata_comdat_b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("wdata_comdat_a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("wdata_comdat_b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("ro_code_comdat_a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("ro_code_comdat_b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("rw_code_comdat_a", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_Secdef("rw_code_comdat_b", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_ExternFunc("code_comdat_a", "code_comdat_a", 0), - T_COFF_DefSymbol_ExternFunc("code_comdat_b", "code_comdat_b", 0), - T_COFF_DefSymbol_ExternFunc("code_plain_a", "code_plain_a", 0), - T_COFF_DefSymbol_ExternFunc("code_plain_b", "code_plain_b", 0), - T_COFF_DefSymbol_ExternFunc("ro_code_comdat_a", "ro_code_comdat_a", 0), - T_COFF_DefSymbol_ExternFunc("ro_code_comdat_b", "ro_code_comdat_b", 0), - T_COFF_DefSymbol_ExternFunc("rw_code_comdat_a", "rw_code_comdat_a", 0), - T_COFF_DefSymbol_ExternFunc("rw_code_comdat_b", "rw_code_comdat_b", 0), - T_COFF_DefSymbol_Extern("rdata_comdat_a", "rdata_comdat_a", 0), - T_COFF_DefSymbol_Extern("rdata_comdat_b", "rdata_comdat_b", 0), - T_COFF_DefSymbol_Extern("wdata_comdat_a", "wdata_comdat_a", 0), - T_COFF_DefSymbol_Extern("wdata_comdat_b", "wdata_comdat_b", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - B32 is_invoke_ok = t_invoke(str8_lit("link.exe"), str8_lit("/nologo /nodefaultlib /subsystem:console /entry:entry /out:ms_icf_flags.exe /opt:ref,icf /include:addresses ms_icf_flags.obj"), max_U64); - T_Ok(is_invoke_ok); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("ms_icf_flags.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_section != 0); - T_Ok(data_section->foff + sizeof(addresses) <= exe.size); - - String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + sizeof(addresses))); - U64 code_comdat_a_addr = 0; - U64 code_comdat_b_addr = 0; - U64 code_plain_a_addr = 0; - U64 code_plain_b_addr = 0; - U64 rdata_comdat_a_addr = 0; - U64 rdata_comdat_b_addr = 0; - U64 wdata_comdat_a_addr = 0; - U64 wdata_comdat_b_addr = 0; - U64 ro_code_comdat_a_addr = 0; - U64 ro_code_comdat_b_addr = 0; - U64 rw_code_comdat_a_addr = 0; - U64 rw_code_comdat_b_addr = 0; - str8_deserial_read_struct(data, 0, &code_comdat_a_addr); - str8_deserial_read_struct(data, 8, &code_comdat_b_addr); - str8_deserial_read_struct(data, 16, &code_plain_a_addr); - str8_deserial_read_struct(data, 24, &code_plain_b_addr); - str8_deserial_read_struct(data, 32, &rdata_comdat_a_addr); - str8_deserial_read_struct(data, 40, &rdata_comdat_b_addr); - str8_deserial_read_struct(data, 48, &wdata_comdat_a_addr); - str8_deserial_read_struct(data, 56, &wdata_comdat_b_addr); - str8_deserial_read_struct(data, 64, &ro_code_comdat_a_addr); - str8_deserial_read_struct(data, 72, &ro_code_comdat_b_addr); - str8_deserial_read_struct(data, 80, &rw_code_comdat_a_addr); - str8_deserial_read_struct(data, 88, &rw_code_comdat_b_addr); - - T_Ok(code_comdat_a_addr != 0); - T_Ok(code_comdat_a_addr == code_comdat_b_addr); // executable code COMDATs fold - T_Ok(ro_code_comdat_a_addr == ro_code_comdat_b_addr); // read-only code COMDATs fold - T_Ok(code_plain_a_addr != code_plain_b_addr); // non-COMDAT code does not fold - T_Ok(wdata_comdat_a_addr != wdata_comdat_b_addr); // writable data COMDATs do not fold - T_Ok(rw_code_comdat_a_addr != rw_code_comdat_b_addr); // writable code COMDATs do not - T_Ok(rdata_comdat_a_addr == rdata_comdat_b_addr); // read-only data COMDATs fold -} -#endif - -#endif - -#if 1 - -internal B32 -t_read_exe_data_vaddrs(Arena *arena, String8 exe_path, U64 *vaddrs, U64 count) -{ - B32 result = 0; - String8 exe = t_read_file(arena, exe_path); - if (exe.size) { - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - if (data_section != 0 && data_section->foff + count*sizeof(U64) <= exe.size) { - String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + count*sizeof(U64))); - result = str8_deserial_read_array(data, 0, vaddrs, count); - } - } - return result; -} - -TEST(icf_fold_two_funcs) -{ - U8 same_text[] = { - 0x48, 0x31, 0xc0, // xor rax, rax - 0xc3 // ret - }; - U8 entry_text[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, // call a - 0xe8, 0x00, 0x00, 0x00, 0x00, // call b - 0xc3, // ret - }; - T_Ok(t_write_def_obj("ident_funcs.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "a", ".text$mn", str8_array_fixed(same_text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "b", ".text$mb", str8_array_fixed(same_text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "a"), - T_COFF_DefReloc(X64_Rel32, 6, "b"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("a", "a", 0), - T_COFF_DefSymbol_ExternFunc("b", "b", 0), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:icf ident_funcs.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_sect = coff_section_header_from_name(exe, section_table, pe.section_count, str8_lit(".text")); - - // validate .text header - T_Ok(text_sect->voff == 0x1000); - T_Ok(text_sect->vsize >= 0x14); - T_Ok(text_sect->fsize == 0x200); - - T_Ok(text_sect->foff + text_sect->vsize <= exe.size); - String8 text_data = str8_substr(exe, r1u64(text_sect->foff, text_sect->foff + 0x14)); - - U8 expected_text[] = { - // entry - 0xe8, 0x0b, 0x00, 0x00, 0x00, - 0xe8, 0x06, 0x00, 0x00, 0x00, - 0xc3, - - // pad - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - - // a and b folded - 0x48, 0x31, 0xc0, - 0xc3, - }; - T_Ok(str8_match(text_data, str8_array_fixed(expected_text), 0)); -} - -TEST(icf_associative_child_prevents_fold) -{ - U8 ret_text[] = { 0xc3 }; - U8 handler_a[] = { 1, 2, 3, 4 }; - U8 handler_b[] = { 4, 3, 2, 1 }; - U8 addresses[2 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_associative_child.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(ret_text), .flags = "rx:code@1" }, - { "fn_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "fn_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "handler_a", ".xdata", str8_array_fixed(handler_a), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "handler_b", ".xdata", str8_array_fixed(handler_b), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "fn_a"), - T_COFF_DefReloc(X64_Addr64, sizeof(U64), "fn_b"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("fn_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Associative("handler_a", "fn_a"), - T_COFF_DefSymbol_Associative("handler_b", "fn_b"), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_ExternFunc("fn_a", "fn_a", 0), - T_COFF_DefSymbol_ExternFunc("fn_b", "fn_b", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_associative_child.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[1] != 0); - T_Ok(vaddrs[0] != vaddrs[1]); -} - -TEST(icf_comdat_reloc_targets_fold) -{ - U8 fn_text[] = { - 0x48, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, // mov rax, shared - 0xc3, // ret - }; - U8 shared_data[] = { 0 }; - U8 entry_text[] = { 0xc3 }; - U8 addresses[2 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_comdat_reloc_a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn_a", ".text$mn", str8_array_fixed(fn_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 2, "shared_local_a"), {0} } }, - { "shared", ".rdata", str8_array_fixed(shared_data), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("shared", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_ExternFunc("fn_a", "fn_a", 0), - T_COFF_DefSymbol_Extern("shared", "shared", 0), - T_COFF_DefSymbol_Static("shared_local_a", "shared", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_comdat_reloc_b.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn_b", ".text$mn", str8_array_fixed(fn_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 2, "shared_local_b"), {0} } }, - { "shared", ".rdata", str8_array_fixed(shared_data), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("shared", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_ExternFunc("fn_b", "fn_b", 0), - T_COFF_DefSymbol_Extern("shared", "shared", 0), - T_COFF_DefSymbol_Static("shared_local_b", "shared", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_comdat_reloc_entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "fn_a"), T_COFF_DefReloc(X64_Addr64, sizeof(U64), "fn_b"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_UndefFunc("fn_a"), - T_COFF_DefSymbol_UndefFunc("fn_b"), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_comdat_reloc_entry.obj icf_comdat_reloc_a.obj icf_comdat_reloc_b.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[0] == vaddrs[1]); -} - -TEST(icf_same_but_different) -{ - U8 text[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, // call $ - 0xc3 - }; - U8 return_1[] = { - 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, 0x00, // mov rax, 1 - 0xc3 // ret - }; - U8 return_2[] = { - 0x48, 0xc7, 0xc0, 0x02, 0x00, 0x00, 0x00, // mov rax, 2 - 0xc3 // ret - }; - U8 call_a_and_b[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xc3 - }; - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "entry", ".text", str8_array_fixed(call_a_and_b), .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "a"), - T_COFF_DefReloc(X64_Rel32, 6, "b"), - {0} - } - }, - { - "a", ".text", str8_array_fixed(text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "c"), - {0} - } - }, - { - "b", ".text", str8_array_fixed(text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "d"), - {0} - } - }, - { "c", ".text", str8_array_fixed(return_1), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "d", ".text", str8_array_fixed(return_2), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("c", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("d", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("entry", "entry", 0), - T_COFF_DefSymbol_Extern("a", "a", 0), - T_COFF_DefSymbol_Extern("b", "b", 0), - T_COFF_DefSymbol_Extern("c", "c", 0), - T_COFF_DefSymbol_Extern("d", "d", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:icf a.obj"); - T_Ok(g_last_exit_code == 0); - - // validate output - { - U8 expected_text[] = { - 0xe8, 0x0b, 0x00, 0x00, 0x00, // call a - 0xe8, 0x16, 0x00, 0x00, 0x00, // call b - 0xc3, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xe8, 0x1b, 0x00, 0x00, 0x00, // call c - 0xc3, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xe8, 0x1b, 0x00, 0x00, 0x00, // call d - 0xc3, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, 0x00, // mov rax, 1 - 0xc3, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0x48, 0xc7, 0xc0, 0x02, 0x00, 0x00, 0x00, // mov rax, 2 - 0xc3, - }; - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - - T_Ok(text_section); - T_Ok(text_section->foff + sizeof(expected_text) <= exe.size); - - String8 text = str8_substr(exe, r1u64(text_section->foff, text_section->foff + text_section->vsize)); - T_Ok(str8_match(text, str8_array_fixed(expected_text), 0)); - } -} - -TEST(icf_fold_diamond) -{ - U8 call_b_and_c[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xc3 - }; - U8 call_and_return[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xc3 - }; - U8 clear_and_return[] = { - 0x48, 0x31, 0xc0, - 0xc3 - }; - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "a", ".text", str8_array_fixed(call_b_and_c), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "b"), - T_COFF_DefReloc(X64_Rel32, 6, "c"), - {0} - } - }, - { - "b", ".text", str8_array_fixed(call_and_return), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "d"), - {0} - } - }, - { - "c", ".text", str8_array_fixed(call_and_return), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "d"), - {0} - } - }, - { "d", ".text", str8_array_fixed(clear_and_return), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("c", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("d", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("a", "a", 0), - T_COFF_DefSymbol_Extern("b", "b", 0), - T_COFF_DefSymbol_Extern("c", "c", 0), - T_COFF_DefSymbol_Extern("d", "d", 0), - {0} - } - })); - t_invoke_linkerf("/subsystem:console /entry:a /out:a.exe /opt:icf a.obj"); - T_Ok(g_last_exit_code == 0); - - // validate output - { - U8 expected_text[] = { - 0xe8, 0x0b, 0x00, 0x00, 0x00, - 0xe8, 0x06, 0x00, 0x00, 0x00, - 0xc3, - - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - - 0xe8, 0x0b, 0x00, 0x00, 0x00, - 0xc3, - - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - - 0x48, 0x31, 0xc0, - 0xc3 - }; - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - - T_Ok(text_section); - T_Ok(text_section->foff + sizeof(expected_text) <= exe.size); - - String8 text = str8_substr(exe, r1u64(text_section->foff, text_section->foff + text_section->vsize)); - T_Ok(str8_match(text, str8_array_fixed(expected_text), 0)); - } -} - -TEST(icf_cyclic_icf) -{ - U8 text[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xc3 - }; - T_Ok(t_write_def_obj("a.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "a", ".text", str8_array_fixed(text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "b"), - {0} - } - }, - { - "b", ".text", str8_array_fixed(text), .flags = "rx:code", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "a"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("a", "a", 0), - T_COFF_DefSymbol_Static("b", "b", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /out:a.exe /entry:a /opt:icf a.obj"); - T_Ok(g_last_exit_code == 0); - - // validate output - { - U8 expected_text[] = { - 0xe8, 0xfb, 0xff, 0xff, 0xff, // a and b folded into a self-call - 0xc3, - }; - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - - T_Ok(text_section); - T_Ok(text_section->foff + sizeof(expected_text) <= exe.size); - - String8 text = str8_substr(exe, r1u64(text_section->foff, text_section->foff + text_section->vsize)); - T_Ok(str8_match(text, str8_array_fixed(expected_text), 0)); - } -} - -// ICF must preserve identical sections with different symbol targets -TEST(icf_reloc_target_symbol_types_do_not_fold) -{ - U8 reloc_text[] = { - 0x48, 0xc7, 0xc0, 0x00, 0x00, 0x00, 0x00, // mov rax, target - 0xc3 // ret - }; - U8 ret_text[] = { - 0xc3, // ret - }; - U8 target_data[] = { - 0x00, - }; - U8 addresses[3 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_interp_entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(ret_text), .flags = "rx:code@1" }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "fn_regular"), - T_COFF_DefReloc(X64_Addr64, 8, "fn_common"), - T_COFF_DefReloc(X64_Addr64, 16, "fn_abs"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - T_COFF_DefSymbol_Undef("fn_regular"), - T_COFF_DefSymbol_Undef("fn_common"), - T_COFF_DefSymbol_Undef("fn_abs"), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_interp_regular.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_regular"), {0} } }, - { "target", ".rdata$mn", str8_array_fixed(target_data), .flags = "r:data@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("fn_regular", "fn", 0), - T_COFF_DefSymbol_Extern("target_regular", "target", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_interp_common.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_common"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("fn_common", "fn", 0), - T_COFF_DefSymbol_Common("target_common", 8), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_interp_abs.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_abs"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("fn_abs", "fn", 0), - T_COFF_DefSymbol_AbsExtern("target_abs", 0x1234), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf icf_interp_entry.obj icf_interp_regular.obj icf_interp_common.obj icf_interp_abs.obj"); - T_Ok(g_last_exit_code == 0); - - U64 fn_vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), fn_vaddrs, ArrayCount(fn_vaddrs))); - - for EachElement(i, fn_vaddrs) { - for (U64 j = i + 1; j < ArrayCount(fn_vaddrs); j += 1) { - T_Ok(fn_vaddrs[i] != fn_vaddrs[j]); - } - } -} - -// ICF must preserve sections with unresolved target symbols (with /FORCE) -TEST(icf_unresolved_reloc_targets_do_not_fold) -{ - U8 reloc_text[] = { - 0x48, 0xc7, 0xc0, 0x00, 0x00, 0x00, 0x00, // mov rax, target - 0xc3 // ret - }; - U8 ret_text[] = { - 0xc3, // ret - }; - U8 addresses[2 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_unresolved_entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(ret_text), .flags = "rx:code@1" }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "fn_undef"), - T_COFF_DefReloc(X64_Addr64, 8, "fn_weak"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - T_COFF_DefSymbol_Undef("fn_undef"), - T_COFF_DefSymbol_Undef("fn_weak"), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_unresolved_undef.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_undef"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("fn_undef", "fn", 0), - T_COFF_DefSymbol_Undef("target_undef"), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_unresolved_weak.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "fn", ".text$mn", str8_array_fixed(reloc_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 3, "target_weak"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("fn_weak", "fn", 0), - T_COFF_DefSymbol_AbsExtern("target_weak_fallback", 0), - T_COFF_DefSymbol_Weak("target_weak", COFF_WeakExt_NoLibrary, "target_weak_fallback"), - {0} - } - })); - - t_invoke_linkerf("/force /subsystem:console /entry:entry /out:a.exe /opt:ref,icf icf_unresolved_entry.obj icf_unresolved_undef.obj icf_unresolved_weak.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_section != 0); - T_Ok(data_section->foff + sizeof(addresses) <= exe.size); - - String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + sizeof(addresses))); - U64 undef_vaddr = 0; - U64 weak_vaddr = 0; - str8_deserial_read_struct(data, 0, &undef_vaddr); - str8_deserial_read_struct(data, 8, &weak_vaddr); - T_Ok(undef_vaddr != 0); - T_Ok(weak_vaddr != 0); - T_Ok(undef_vaddr != weak_vaddr); -} - -TEST(icf_fold_with_largest_align) -{ - U8 text[] = { - 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, 0x00, // mov rax, 1 - 0xc3 - }; - U8 call_a_and_b[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xc3 - }; - - String8 a_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "entry", ".text", str8_array_fixed(call_a_and_b), .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "a"), - T_COFF_DefReloc(X64_Rel32, 6, "b"), - {0} - } - }, - { "a", ".text", str8_array_fixed(text), .flags = "rx:code@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "b", ".text", str8_array_fixed(text), .flags = "rx:code@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("entry", "entry", 0), - T_COFF_DefSymbol_Static("a", "a", 0), - T_COFF_DefSymbol_Static("b", "b", 0), - {0} - } - }); - - // swap sections for a and b - String8 b_obj = t_coff_from_def_obj(arena, (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "entry", ".text", str8_array_fixed(call_a_and_b), .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 1, "a"), - T_COFF_DefReloc(X64_Rel32, 6, "b"), - {0} - } - }, - { "a", ".text", str8_array_fixed(text), .flags = "rx:code@8", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "b", ".text", str8_array_fixed(text), .flags = "rx:code@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Extern("entry", "entry", 0), - T_COFF_DefSymbol_Static("a", "a", 0), - T_COFF_DefSymbol_Static("b", "b", 0), - {0} - } - }); - - T_Ok(t_write_file(str8_lit("a.obj"), a_obj)); - T_Ok(t_write_file(str8_lit("b.obj"), b_obj)); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:icf a.obj"); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:b.exe /opt:icf b.obj"); - T_Ok(g_last_exit_code == 0); - - U8 expected_text[] = { - 0xe8, 0x0b, 0x00, 0x00, 0x00, - 0xe8, 0x06, 0x00, 0x00, 0x00, - 0xc3, - 0xcc, - 0xcc, - 0xcc, - 0xcc, - 0xcc, - 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, - 0x00, - 0xc3, - }; - - // validate output in a.exe - { - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - - T_Ok(text_section); - T_Ok(text_section->foff + sizeof(expected_text) <= exe.size); - - String8 text = str8_substr(exe, r1u64(text_section->foff, text_section->foff + text_section->vsize)); - T_Ok(str8_match(text, str8_array_fixed(expected_text), 0)); - } - - // validate output in b.exe - { - String8 exe = t_read_file(arena, str8_lit("b.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *text_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".text")); - - T_Ok(text_section); - T_Ok(text_section->foff + sizeof(expected_text) <= exe.size); - - String8 text = str8_substr(exe, r1u64(text_section->foff, text_section->foff + text_section->vsize)); - T_Ok(str8_match(text, str8_array_fixed(expected_text), 0)); - } -} - -TEST(icf_identical_bytes_different_color_spaces_do_not_fold) -{ - U8 same_bytes[] = { - 0xc3, - }; - U8 entry_text[] = { - 0xc3, - }; - U8 addresses[2 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_color_spaces.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "text", ".text$mn", str8_array_fixed(same_bytes), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "vftable", ".rdata$mn", str8_array_fixed(same_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "text"), - T_COFF_DefReloc(X64_Addr64, 8, "??_7type@@6B@"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("text", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("vftable", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_ExternFunc("text", "text", 0), - T_COFF_DefSymbol_Extern("??_7type@@6B@", "vftable", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_color_spaces.obj"); - T_Ok(g_last_exit_code == 0); - - String8 exe = t_read_file(arena, str8_lit("a.exe")); - T_Ok(exe.size); - - PE_BinInfo pe = pe_bin_info_from_data(arena, exe); - COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(exe, pe.section_table_range).str; - String8 string_table = str8_substr(exe, pe.string_table_range); - COFF_SectionHeader *data_section = coff_section_header_from_name(string_table, section_table, pe.section_count, str8_lit(".data")); - T_Ok(data_section != 0); - T_Ok(data_section->foff + sizeof(addresses) <= exe.size); - - String8 data = str8_substr(exe, r1u64(data_section->foff, data_section->foff + sizeof(addresses))); - U64 text_vaddr = 0; - U64 vftable_vaddr = 0; - str8_deserial_read_struct(data, 0, &text_vaddr); - str8_deserial_read_struct(data, 8, &vftable_vaddr); - T_Ok(text_vaddr != 0); - T_Ok(vftable_vaddr != 0); - T_Ok(text_vaddr != vftable_vaddr); -} - -TEST(icf_multihop_reloc_target_colors_do_not_fold) -{ - U8 call_text[] = { - 0xe8, 0x00, 0x00, 0x00, 0x00, - 0xc3, - }; - U8 return_1[] = { - 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, 0x00, - 0xc3, - }; - U8 return_2[] = { - 0x48, 0xc7, 0xc0, 0x02, 0x00, 0x00, 0x00, - 0xc3, - }; - U8 entry_text[] = { - 0xc3, - }; - U8 addresses[6 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_multihop.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { - "top_a", ".text$mn", str8_array_fixed(call_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "mid_a"), {0} } - }, - { - "top_b", ".text$mn", str8_array_fixed(call_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "mid_b"), {0} } - }, - { - "mid_a", ".text$mn", str8_array_fixed(call_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "leaf_a"), {0} } - }, - { - "mid_b", ".text$mn", str8_array_fixed(call_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Rel32, 1, "leaf_b"), {0} } - }, - { "leaf_a", ".text$mn", str8_array_fixed(return_1), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "leaf_b", ".text$mn", str8_array_fixed(return_2), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "top_a"), - T_COFF_DefReloc(X64_Addr64, 8, "top_b"), - T_COFF_DefReloc(X64_Addr64, 16, "mid_a"), - T_COFF_DefReloc(X64_Addr64, 24, "mid_b"), - T_COFF_DefReloc(X64_Addr64, 32, "leaf_a"), - T_COFF_DefReloc(X64_Addr64, 40, "leaf_b"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("top_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("top_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("mid_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("mid_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("leaf_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("leaf_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_ExternFunc("top_a", "top_a", 0), - T_COFF_DefSymbol_ExternFunc("top_b", "top_b", 0), - T_COFF_DefSymbol_ExternFunc("mid_a", "mid_a", 0), - T_COFF_DefSymbol_ExternFunc("mid_b", "mid_b", 0), - T_COFF_DefSymbol_ExternFunc("leaf_a", "leaf_a", 0), - T_COFF_DefSymbol_ExternFunc("leaf_b", "leaf_b", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_multihop.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - for EachIndex(i, ArrayCount(vaddrs)) { - T_Ok(vaddrs[i] != 0); - } - T_Ok(vaddrs[0] != vaddrs[1]); - T_Ok(vaddrs[2] != vaddrs[3]); - T_Ok(vaddrs[4] != vaddrs[5]); -} - -TEST(icf_comdat_symlink_chain) -{ - U8 ret_small[] = { 0xc3 }; - U8 ret_large[] = { 0xc3, 0x90 }; - U8 entry_text[] = { 0xc3 }; - U8 addresses[2 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_chain_leader.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "leader", ".text$mn", str8_array_fixed(ret_large), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("leader", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("leader", "leader", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_chain_duplicate.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "dup", ".text$mn", str8_array_fixed(ret_small), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "local_dup"), - T_COFF_DefReloc(X64_Addr64, 8, "leader"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("dup", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_ExternFunc("dup", "dup", 0), - T_COFF_DefSymbol_Static("local_dup", "dup", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - T_COFF_DefSymbol_UndefFunc("leader"), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_chain_selected.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "dup", ".text$mn", str8_array_fixed(ret_large), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("dup", COFF_ComdatSelect_Largest), - T_COFF_DefSymbol_ExternFunc("dup", "dup", 0), - {0} - } - })); - - T_Ok(t_write_def_obj("icf_chain_entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_chain_leader.obj icf_chain_duplicate.obj icf_chain_selected.obj icf_chain_entry.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[ArrayCount(addresses) / sizeof(U64)] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[0] == vaddrs[1]); -} - -TEST(icf_llvm_addrsig) -{ - char *main_c = "int foo() { return 123; }\n" - "int bar() { return 123; }\n" - "int main() {\n" - "int (*fn)() = &foo;\n" - "return fn != bar;\n" - "}\n"; - String8 main_path = t_make_file_path(arena, str8_lit("main.c")); - T_Ok(write_data_to_file_path(main_path, str8_cstring(main_c))); - - String8 main_obj_path = t_make_file_path(arena, str8_lit("main.obj"));; - t_invoke(t_clang_path(), str8f(arena, "%S -o %S -c -ffunction-sections -target x86_64-pc-windows-msvc", main_path, main_obj_path), max_U64); - T_Ok(g_last_exit_code == 0); - - String8 a_path = t_make_file_path(arena, str8_lit("a.exe")); - - t_invoke_linkerf("%S /opt:icf /out:a.exe libcmt.lib", main_obj_path); - T_Ok(g_last_exit_code == 0); - t_invoke(a_path, str8_zero(), max_U64); - T_Ok(g_last_exit_code == 1); - - t_invoke_linkerf("%S /opt:icf /out:a.exe libcmt.lib /llvm_addrsig:no", main_obj_path); - T_Ok(g_last_exit_code == 0); - t_invoke(a_path, str8_zero(), max_U64); - T_Ok(g_last_exit_code == 0); -} - -// .llvm_addrsig can name an undefined external whose definition is in another -// object; ICF must parse and mark the resolved symbol's object, not the referrer. -TEST(icf_llvm_addrsig_external_symbol) -{ - char *ref_c = "extern int ext_sig();\n" - "int (*ext_sig_addr)() = &ext_sig;\n" - "int entry() { return ext_sig_addr(); }\n"; - char *def_c = "int dummy0() { return 0; }\n" - "int dummy1() { return 1; }\n" - "int dummy2() { return 2; }\n" - "int dummy3() { return 3; }\n" - "int dummy4() { return 4; }\n" - "int ext_sig() { return 0; }\n"; - String8 ref_path = t_make_file_path(arena, str8_lit("ref.c")); - String8 def_path = t_make_file_path(arena, str8_lit("def.c")); - T_Ok(write_data_to_file_path(ref_path, str8_cstring(ref_c))); - T_Ok(write_data_to_file_path(def_path, str8_cstring(def_c))); - - String8 ref_obj_path = t_make_file_path(arena, str8_lit("ref.obj")); - String8 def_obj_path = t_make_file_path(arena, str8_lit("def.obj")); - t_invoke(t_clang_path(), str8f(arena, "%S -o %S -c -ffunction-sections -target x86_64-pc-windows-msvc", ref_path, ref_obj_path), max_U64); - T_Ok(g_last_exit_code == 0); - t_invoke(t_clang_path(), str8f(arena, "%S -o %S -c -ffunction-sections -target x86_64-pc-windows-msvc", def_path, def_obj_path), max_U64); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("%S %S /subsystem:console /entry:entry /opt:icf /out:addrsig_ext.exe libcmt.lib", ref_obj_path, def_obj_path); - T_Ok(g_last_exit_code == 0); -} - -TEST(icf_pdata_xdata_fold) -{ - U8 ret_text[] = { 0xc3 }; - U8 xdata[] = { 0x01, 0x00, 0x00, 0x00 }; - PE_IntelPdata pdata = {0}; - U8 entry_text[] = { 0xc3 }; - U8 addresses[4 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_pdata_xdata_fold.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "fn_a", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "fn_b", ".text$mn", str8_array_fixed(ret_text), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "xdata_a", ".xdata", str8_array_fixed(xdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "xdata_b", ".xdata", str8_array_fixed(xdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { - "pdata_a", ".pdata", str8_struct(&pdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_first), "fn_a"), - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_one_past_last), "fn_a"), - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_unwind_info), "$unwind$a"), - {0} - } - }, - { - "pdata_b", ".pdata", str8_struct(&pdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT, - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_first), "fn_b"), - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_one_past_last), "fn_b"), - T_COFF_DefReloc(X64_Addr32Nb, OffsetOf(PE_IntelPdata, voff_unwind_info), "$unwind$b"), - {0} - } - }, - { - "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "$pdata$a"), - T_COFF_DefReloc(X64_Addr64, 8, "$pdata$b"), - T_COFF_DefReloc(X64_Addr64, 16, "$unwind$a"), - T_COFF_DefReloc(X64_Addr64, 24, "$unwind$b"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("fn_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("xdata_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("xdata_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("pdata_a", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("pdata_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_ExternFunc("fn_a", "fn_a", 0), - T_COFF_DefSymbol_ExternFunc("fn_b", "fn_b", 0), - T_COFF_DefSymbol_Extern("$unwind$a", "xdata_a", 0), - T_COFF_DefSymbol_Extern("$unwind$b", "xdata_b", 0), - T_COFF_DefSymbol_Extern("$pdata$a", "pdata_a", 0), - T_COFF_DefSymbol_Extern("$pdata$b", "pdata_b", 0), - T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_pdata_xdata_fold.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[4] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[0] == vaddrs[1]); - T_Ok(vaddrs[2] != 0); - T_Ok(vaddrs[2] == vaddrs[3]); -} - -TEST(icf_pdata_differs_by_function_color) -{ - U8 ret_1[] = { 0xb8, 1, 0, 0, 0, 0xc3 }; - U8 ret_2[] = { 0xb8, 2, 0, 0, 0, 0xc3 }; - U8 xdata[] = { 0x01, 0x00, 0x00, 0x00 }; - PE_IntelPdata pdata = {0}; - U8 entry_text[] = { 0xc3 }; - U8 addresses[3 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_pdata_diff.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "fn_a", ".text$mn", str8_array_fixed(ret_1), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "fn_b", ".text$mn", str8_array_fixed(ret_2), .flags = "rx:code@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "xdata_a", ".xdata", str8_array_fixed(xdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "xdata_b", ".xdata", str8_array_fixed(xdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "pdata_a", ".pdata", str8_struct(&pdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32Nb, 0, "fn_a"), T_COFF_DefReloc(X64_Addr32Nb, 4, "fn_a"), T_COFF_DefReloc(X64_Addr32Nb, 8, "$unwind$a"), {0} } }, - { "pdata_b", ".pdata", str8_struct(&pdata), .flags = "r:data@4", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr32Nb, 0, "fn_b"), T_COFF_DefReloc(X64_Addr32Nb, 4, "fn_b"), T_COFF_DefReloc(X64_Addr32Nb, 8, "$unwind$b"), {0} } }, - { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "$pdata$a"), T_COFF_DefReloc(X64_Addr64, 8, "$pdata$b"), T_COFF_DefReloc(X64_Addr64, 16, "$unwind$a"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("fn_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("fn_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("xdata_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("xdata_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("pdata_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("pdata_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), T_COFF_DefSymbol_ExternFunc("fn_a", "fn_a", 0), T_COFF_DefSymbol_ExternFunc("fn_b", "fn_b", 0), - T_COFF_DefSymbol_Extern("$unwind$a", "xdata_a", 0), T_COFF_DefSymbol_Extern("$unwind$b", "xdata_b", 0), - T_COFF_DefSymbol_Extern("$pdata$a", "pdata_a", 0), T_COFF_DefSymbol_Extern("$pdata$b", "pdata_b", 0), T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_pdata_diff.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[3] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[1] != 0); - T_Ok(vaddrs[0] != vaddrs[1]); -} - -TEST(icf_vftable_and_vbtable_policy) -{ - U8 table_bytes[sizeof(U64)] = {0}; - U8 target_a[] = { 1 }; - U8 target_b[] = { 2 }; - U8 entry_text[] = { 0xc3 }; - U8 addresses[8 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_tables.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "vf_a", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "vf_b", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "vb_a", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "vb_b", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "rtti_a", ".rdata$mn", str8_array_fixed(target_a), .flags = "r:data@1" }, - { "rtti_b", ".rdata$mn", str8_array_fixed(target_b), .flags = "r:data@1" }, - { "vf_ref_a", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "rtti_a"), {0} } }, - { "vf_ref_b", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "rtti_b"), {0} } }, - { "vb_ref_a", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "rtti_a"), {0} } }, - { "vb_ref_b", ".rdata$mn", str8_array_fixed(table_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT, .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "rtti_b"), {0} } }, - { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr64, 0, "??_7a@@6B@"), T_COFF_DefReloc(X64_Addr64, 8, "??_7b@@6B@"), - T_COFF_DefReloc(X64_Addr64, 16, "??_8a@@7B@"), T_COFF_DefReloc(X64_Addr64, 24, "??_8b@@7B@"), - T_COFF_DefReloc(X64_Addr64, 32, "??_7ra@@6B@"), T_COFF_DefReloc(X64_Addr64, 40, "??_7rb@@6B@"), - T_COFF_DefReloc(X64_Addr64, 48, "??_8ra@@7B@"), T_COFF_DefReloc(X64_Addr64, 56, "??_8rb@@7B@"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("vf_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("vf_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("vb_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("vb_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("vf_ref_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("vf_ref_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("vb_ref_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("vb_ref_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), - T_COFF_DefSymbol_Extern("??_7a@@6B@", "vf_a", 0), T_COFF_DefSymbol_Extern("??_7b@@6B@", "vf_b", 0), - T_COFF_DefSymbol_Extern("??_8a@@7B@", "vb_a", 0), T_COFF_DefSymbol_Extern("??_8b@@7B@", "vb_b", 0), - T_COFF_DefSymbol_Extern("??_7ra@@6B@", "vf_ref_a", 0), T_COFF_DefSymbol_Extern("??_7rb@@6B@", "vf_ref_b", 0), - T_COFF_DefSymbol_Extern("??_8ra@@7B@", "vb_ref_a", 0), T_COFF_DefSymbol_Extern("??_8rb@@7B@", "vb_ref_b", 0), - T_COFF_DefSymbol_Extern("rtti_a", "rtti_a", 0), T_COFF_DefSymbol_Extern("rtti_b", "rtti_b", 0), T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_tables.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[8] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] == vaddrs[1]); - T_Ok(vaddrs[2] != vaddrs[3]); - T_Ok(vaddrs[4] != vaddrs[5]); - T_Ok(vaddrs[6] != vaddrs[7]); -} - -TEST(icf_readonly_non_vftable_data_policy) -{ - U8 data_bytes[] = { 1, 2, 3, 4 }; - U8 entry_text[] = { 0xc3 }; - U8 addresses[4 * sizeof(U64)] = {0}; - - T_Ok(t_write_def_obj("icf_rdata_policy.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "entry", ".text", str8_array_fixed(entry_text), .flags = "rx:code@1" }, - { "data_a", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "data_b", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "data_c", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "data_d", ".rdata$mn", str8_array_fixed(data_bytes), .flags = "r:data@1", .raw_flags = COFF_SectionFlag_LnkCOMDAT }, - { "addresses", ".data", str8_array_fixed(addresses), .flags = "rw:data@1", .relocs = (T_COFF_DefReloc[]){ T_COFF_DefReloc(X64_Addr64, 0, "data_a"), T_COFF_DefReloc(X64_Addr64, 8, "data_b"), T_COFF_DefReloc(X64_Addr64, 16, "data_c"), T_COFF_DefReloc(X64_Addr64, 24, "data_d"), {0} } }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Secdef("data_a", COFF_ComdatSelect_NoDuplicates), T_COFF_DefSymbol_Secdef("data_b", COFF_ComdatSelect_NoDuplicates), - T_COFF_DefSymbol_Secdef("data_c", COFF_ComdatSelect_Any), T_COFF_DefSymbol_Secdef("data_d", COFF_ComdatSelect_Any), - T_COFF_DefSymbol_ExternFunc("entry", "entry", 0), T_COFF_DefSymbol_Extern("data_a", "data_a", 0), T_COFF_DefSymbol_Extern("data_b", "data_b", 0), T_COFF_DefSymbol_Extern("data_c", "data_c", 0), T_COFF_DefSymbol_Extern("data_d", "data_d", 0), T_COFF_DefSymbol_Extern("addresses", "addresses", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /opt:ref,icf /include:addresses icf_rdata_policy.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[4] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("a.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[1] != 0); - T_Ok(vaddrs[0] != vaddrs[1]); - T_Ok(vaddrs[2] != 0); - T_Ok(vaddrs[2] == vaddrs[3]); -} - -TEST(icf_cpp_identical_functions_fold) -{ - char source[] = - "extern \"C\" __declspec(noinline) int a(void) { return 42; }\n" - "extern \"C\" __declspec(noinline) int b(void) { return 42; }\n" - "extern \"C\" int (* volatile pa)(void) = a;\n" - "extern \"C\" int (* volatile pb)(void) = b;\n" - "extern \"C\" int entry(void) { return pa == pb ? 0 : 1; }\n"; - - T_Ok(t_write_file(str8_lit("icf_cpp_fold.cpp"), str8_cstring(source))); - T_Ok(t_invoke_cl("/nologo /c /O2 /Gy /Zc:preprocessor /Fo:icf_cpp_fold.obj icf_cpp_fold.cpp")); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:icf_cpp_fold.exe /opt:ref,icf /include:pa /include:pb icf_cpp_fold.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[2] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("icf_cpp_fold.exe"), vaddrs, ArrayCount(vaddrs))); - T_Ok(vaddrs[0] != 0); - T_Ok(vaddrs[0] == vaddrs[1]); -} - -TEST(icf_cpp_multihop_functions_do_not_fold) -{ - char source[] = - "__declspec(noinline) int leaf_a(void) { return 1; }\n" - "__declspec(noinline) int leaf_b(void) { return 2; }\n" - "__declspec(noinline) int mid_a(void) { return leaf_a(); }\n" - "__declspec(noinline) int mid_b(void) { return leaf_b(); }\n" - "__declspec(noinline) int top_a(void) { return mid_a(); }\n" - "__declspec(noinline) int top_b(void) { return mid_b(); }\n" - "int (* volatile p_top_a)(void) = top_a;\n" - "int (* volatile p_top_b)(void) = top_b;\n" - "int (* volatile p_mid_a)(void) = mid_a;\n" - "int (* volatile p_mid_b)(void) = mid_b;\n" - "int (* volatile p_leaf_a)(void) = leaf_a;\n" - "int (* volatile p_leaf_b)(void) = leaf_b;\n" - "int entry(void) {\n" - " if (p_top_a == p_top_b) { return 1; }\n" - " if (p_mid_a == p_mid_b) { return 2; }\n" - " if (p_leaf_a == p_leaf_b) { return 3; }\n" - " return 0;\n" - "}\n"; - - T_Ok(t_write_file(str8_lit("icf_cpp_multihop.c"), str8_cstring(source))); - T_Ok(t_invoke_cl("/nologo /c /O2 /Gy /Zc:preprocessor /Fo:icf_cpp_multihop.obj icf_cpp_multihop.c")); - T_Ok(g_last_exit_code == 0); - - t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /out:icf_cpp_multihop.exe /opt:ref,icf /include:p_top_a /include:p_top_b /include:p_mid_a /include:p_mid_b /include:p_leaf_a /include:p_leaf_b icf_cpp_multihop.obj"); - T_Ok(g_last_exit_code == 0); - - U64 vaddrs[6] = {0}; - T_Ok(t_read_exe_data_vaddrs(arena, str8_lit("icf_cpp_multihop.exe"), vaddrs, ArrayCount(vaddrs))); - for EachIndex(i, ArrayCount(vaddrs)) { - T_Ok(vaddrs[i] != 0); - } - T_Ok(vaddrs[0] != vaddrs[1]); - T_Ok(vaddrs[2] != vaddrs[3]); - T_Ok(vaddrs[4] != vaddrs[5]); -} - -#endif - -#if 0 -TEST(defer_imp_link) -{ - T_COFF_DefLib bar_lib_any = { - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_DllImportStatic, - .dll_import = { .name = "bar.dll" } - }, - { - .type = T_COFF_DefLibMember_Import, - .import = { "bar.dll", "bar", COFF_ImportBy_Name, COFF_ImportHeader_Code, .hit_or_ordinal = 0 } - }, - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_lit_comp("\xff\x25\x00\x00\x00\x00"), .flags = "rx:code" - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("__imp_bar"), - T_COFF_DefSymbol_ExternFunc("qwe", "text", 0), - {0}, - } - } - }, - {0} - } - }; - - T_COFF_DefLib foo_lib_any = { - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_DllImportStatic, - .dll_import = { .name = "foo.dll" } - }, - { - .type = T_COFF_DefLibMember_Import, - .import = { "foo.dll", "bar", COFF_ImportBy_Name, COFF_ImportHeader_Code, .hit_or_ordinal = 0 } - }, - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", - str8_lit_comp("\xff\x25\x00\x00\x00\x00"), - .flags = "rx:code", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Rel32, 2, "bar"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Undef("bar"), - T_COFF_DefSymbol_Undef("qwe"), - T_COFF_DefSymbol_ExternFunc("thunk", "text", 0), - {0}, - } - } - }, - {0} - } - }; - - String8 bar_lib = t_coff_from_def_lib(arena, bar_lib_any); - String8 foo_lib = t_coff_from_def_lib(arena, foo_lib_any); - - T_Ok(t_write_file(str8_lit("bar.lib"), bar_lib)); - T_Ok(t_write_file(str8_lit("foo.lib"), foo_lib)); - T_Ok(t_write_entry_obj()); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe bar.lib foo.lib entry.obj /include:thunk"); - T_Ok(g_last_exit_code == 0); -} -#endif - -#if 0 -TEST(reloc_apply_off_out_of_bounds) -{ - T_Ok(t_write_def_obj("bad.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "text", ".text", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rx:code@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, max_U32, "target"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsStatic("target", 0), - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe bad.obj"); - T_Ok(g_last_exit_code != 0); -} -#endif - -#if 0 -TEST(lib_member_reloc_apply_off_out_of_bounds) -{ - T_Ok(t_write_def_lib("bad.lib", (T_COFF_DefLib){ - .emit_second_member = 1, - .members = (T_COFF_DefLibMember[]){ - { - .type = T_COFF_DefLibMember_Obj, - .obj = { - .path = str8_lit("bad_member.obj"), - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { - "data", ".data", str8_lit_comp("\x00\x00\x00\x00"), .flags = "rw:data@1", - .relocs = (T_COFF_DefReloc[]){ - T_COFF_DefReloc(X64_Addr32, max_U32, "target"), - {0} - } - }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_AbsStatic("target", 0), - T_COFF_DefSymbol_Extern("bad", "data", 0), - {0} - } - } - }, - {0} - } - })); - - T_Ok(t_write_def_obj("entry.obj", (T_COFF_DefObj){ - .machine = T_COFF_DefSetMachine(X64), - .sections = (T_COFF_DefSection[]){ - { "text", ".text", str8_lit_comp("\xC3"), .flags = "rx:code@1" }, - {0} - }, - .symbols = (T_COFF_DefSymbol[]){ - T_COFF_DefSymbol_Extern("entry", "text", 0), - {0} - } - })); - - t_invoke_linkerf("/subsystem:console /entry:entry /out:a.exe /include:bad entry.obj bad.lib"); - T_Ok(g_last_exit_code != 0); + return data; } -#endif diff --git a/src/linker/tests/long_section_name.tst b/src/linker/tests/long_section_name.tst new file mode 100644 index 000000000..6ed80d5a0 --- /dev/null +++ b/src/linker/tests/long_section_name.tst @@ -0,0 +1,33 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } + debug_info: { name: ".debug_info", permissions: (read, write), content: initialized_data, data: { text: "DEBUG_INFO" } } + debug_abbrev: { name: ".debug_abbrev", permissions: (read, write), content: initialized_data, data: { text: "DEBUG_ABBREV" } } + } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: + { + // link test.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe test.obj", artifact: image } + } + steps: + { + // load linked exe + expect_pe: { artifact: image, expected: { pe: { sections: { ".debug_info": {}, ".debug_abbrev": {} } } } } + } +} diff --git a/src/linker/tests/machine_compat_check.tst b/src/linker/tests/machine_compat_check.tst new file mode 100644 index 000000000..3e572fa88 --- /dev/null +++ b/src/linker/tests/machine_compat_check.tst @@ -0,0 +1,102 @@ +test: +{ + + artifacts: + { + unknown_obj: + { + file_name: "unknown.obj" + coff: + { + object: + { + machine: unknown + sections: + { + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "unknown" } } + } + } + } + } + x64_obj: + { + file_name: "x64.obj" + coff: + { + object: + { + machine: x64 + sections: + { + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "x64" } } + } + } + } + } + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } + } + symbols: + { + my_entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } + } + } + arm64_obj: + { + file_name: "arm64.obj" + coff: + { + object: + { + machine: arm64 + sections: + { + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "arm64" } } + } + } + } + } + image: { file_name: "a.exe", pe: {} } + } + + build: + { + link: + { + args: "/subsystem:console /entry:my_entry /out:a.exe entry.obj unknown.obj x64.obj" + artifact: image + } + // test objs with conflicting machines + link: + { + args: "/subsystem:console /entry:my_entry /out:a.exe entry.obj unknown.obj x64.obj arm64.obj" + expect_exit: nonzero + } + // check /MACHINE switch + link: + { + args: "/subsystem:console /entry:my_entry /out:a.exe /machine:amd64 arm64.obj entry.obj" + expect_exit: nonzero + } + } + + steps: + { + expect_pe: + { + artifact: image + expected: { pe: { arch: x64 } } + } + } +} diff --git a/src/linker/tests/map.tst b/src/linker/tests/map.tst new file mode 100644 index 000000000..c240ce390 --- /dev/null +++ b/src/linker/tests/map.tst @@ -0,0 +1,65 @@ +test: +{ + + artifacts: + { + map_input: + { + file_name: "map.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "c3" } + } + } + symbols: + { + entry: { kind: external, name: "map_entry", section: text, value: 0 } + local: { kind: static, name: "map_local", section: text, value: 0 } + } + } + } + } + } + + build: + { + link: + { + args: "/entry:map_entry /subsystem:console /out:map_test_explicit.exe /map:map_test_explicit.map map.obj" + } + + link: + { + args: "/entry:map_entry /subsystem:console /out:map_test_default.exe /map map.obj" + } + + link: + { + args: "/entry:map_entry /subsystem:console /out:map_test_collision.exe /map:map_test_collision.exe map.obj" + expect_exit: nonzero + } + } + + steps: + { + expect_file: { path: "map_test_explicit.map", contains: " Timestamp is " } + expect_file: { path: "map_test_explicit.map", contains: " Preferred load address is " } + expect_file: { path: "map_test_explicit.map", contains: " Start Length Name Class" } + expect_file: { path: "map_test_explicit.map", contains: " Publics by Value" } + expect_file: { path: "map_test_explicit.map", contains: "map_entry" } + expect_file: { path: "map_test_explicit.map", contains: " entry point at" } + expect_file: { path: "map_test_explicit.map", contains: " Static symbols" } + expect_file: { path: "map_test_explicit.map", contains: "map_local" } + expect_file: { path: "map_test_default.map", nonempty: true } + } +} diff --git a/src/linker/tests/merge.tst b/src/linker/tests/merge.tst new file mode 100644 index 000000000..f070fbc4d --- /dev/null +++ b/src/linker/tests/merge.tst @@ -0,0 +1,84 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: { machine: x64, sections: { test: { name: ".test", permissions: (read, write), content: initialized_data, data: { text: "hello, world" } } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + mixed_obj: + { + file_name: "mixed.obj" + coff: { object: { machine: x64, sections: + { + data: { name: ".data", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "d" } } + bss: { name: ".bss", permissions: (read, write), content: uninitialized_data, alignment: 1, data: { zero: 513 } } + } } } + } + order_obj: + { + file_name: "order.obj" + coff: { object: { machine: x64, sections: + { + a: { name: ".a$m", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "a" } } + z: { name: ".z$m", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "z" } } + } } } + } + created_image: { file_name: "merge_created.exe", pe: {} } + mixed_image: { file_name: "merge_mixed.exe", pe: {} } + order_image: { file_name: "merge_order.exe", pe: {} } + chained_image: { file_name: "merge_chained.exe", pe: {} } + } + + build: + { + // circular merge + link: { args: "/subsystem:console /entry:entry /out:a.exe /merge:.test=.test entry.obj test.obj", output: none, expect_exit: 28 } + + // circular merge with extra link + link: { args: "/subsystem:console /entry:entry /out:a.exe /merge:.test=.data /merge:.data=.test entry.obj test.obj", output: none, expect_exit: 28 } + + // merge with non-defined section + link: { args: "/subsystem:console /entry:entry /out:merge_created.exe /merge:.test=.qwe entry.obj test.obj", artifact: created_image } + // make sure linker created .qwe and merged .test into it + + // illegal merge with .reloc + link: { args: "/subsystem:console /entry:entry /out:a.exe /merge:.test=.reloc entry.obj test.obj", output: none, expect_exit: 26 } + + // illegal merge with .rsrc + link: { args: "/subsystem:console /entry:entry /out:a.exe /merge:.test=.rsrc entry.obj test.obj", output: none, expect_exit: 26 } + + // merge non-defined section with defined section + link: { args: "/subsystem:console /entry:entry /out:a.exe /merge:.qwe=.test entry.obj test.obj", output: none } + + // BSS and initialized data have incompatible flags and remain separate. + link: { args: "/subsystem:console /entry:entry /out:merge_mixed.exe /merge:.bss=.data entry.obj mixed.obj", artifact: mixed_image } + + // merged contribution groups retain lexical order + link: { args: "/subsystem:console /entry:entry /out:merge_order.exe /merge:.a=.z entry.obj order.obj", artifact: order_image } + + // merge .test -> .qwe -> .data + link: { args: "/subsystem:console /entry:entry /out:merge_chained.exe /merge:.test=.qwe /merge:.qwe=.data entry.obj test.obj", artifact: chained_image } + } + + steps: + { + expect_pe: { artifact: created_image, expected: { pe: { sections: { ".qwe": { @bits_set(3221225536) raw_flags: 3221225536, data: 68656c6c6f2c20776f726c64 } } } } } + expect_pe: { artifact: mixed_image, expected: { pe: { sections: { ".data": { file_size: 512, virtual_size: 1 }, ".bss": { file_size: 0, virtual_size: 513 } }, optional: { sizeof_initialized_data: 512, sizeof_uninitialized_data: 1024 } } } } + expect_pe: { artifact: order_image, expected: { pe: { sections: { ".z": { data: 617a } } } } } + + // make sure linker merged .test into .data + expect_pe: { artifact: chained_image, expected: { pe: { sections: { ".data": { @bits_set(3221225536) raw_flags: 3221225536, data: 68656c6c6f2c20776f726c64 } } } } } + } +} diff --git a/src/linker/tests/merge_duplicate_types.tst b/src/linker/tests/merge_duplicate_types.tst new file mode 100644 index 000000000..bb23cd465 --- /dev/null +++ b/src/linker/tests/merge_duplicate_types.tst @@ -0,0 +1,285 @@ +test: +{ + + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + + pch_obj: + { + file_name: "pch.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_p: + { + name: ".debug$P" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + // signature + hex: "04000000" + // duplicate in a.obj + hex: "0a0002100300000000000000" + // unique procedure type + hex: "0e000810001000000200000000000000" + // PCH ender + hex: "06001400bebafeca" + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + hex: "04000000f1000000320000000e000111bebafeca7063682e6f626a00" + // ver_fe_major + // ver_fe_minor + // ver_fe_build + // ver_feqfe + // ver_major + // ver_minor + // ver_build + // ver_qfe + hex: "20003c1100000000d00000000000000000000e002400197f0000544f5254555245000000" + } + } + } + } + } + } + } + + a_obj: + { + file_name: "a.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + hex: "04000000160009150010000002000000bebafeca7063682e6f626a00" + hex: "0a0002100300000000000000" + hex: "0e000810001000000000000000000000" + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + hex: "04000000f1000000300000000c000111bebafeca612e6f626a00" + // ver_fe_major + // ver_fe_minor + // ver_fe_build + // ver_feqfe + // ver_major + // ver_minor + // ver_build + // ver_qfe + hex: "20003c1100000000d00000000000000000000e002400197f0000544f525455524500" + } + } + } + } + } + } + } + + b_obj: + { + file_name: "b.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + hex: "04000000160009150010000002000000bebafeca7063682e6f626a00" + hex: "0a0002100300000000000000" + hex: "0e000810001000000000000000000000" + } + } + } + } + } + } + } + + c_obj: + { + file_name: "c.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { zero: 0 } + } + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + hex: "04000000" + hex: "0a0002101100000000000000" + hex: "0e000810001000000000000000000000" + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 33554432 + data: + { + concat: + { + hex: "04000000f10000005f000000" + // S_OBJNAME + hex: "0c000111bebafeca632e6f626a00" + // S_COMPILE3 + // ver_fe_major + // ver_fe_minor + // ver_fe_build + // ver_feqfe + // ver_major + // ver_minor + // ver_build + // ver_qfe + hex: "20003c1100000000d00000000000000000000e002400197f0000544f525455524500" + // S_LPROC32 + hex: "29000f110000000000000000000000000100000000000000000000000110000000000000010000666f6f00" + // S_PROC_ID_END + hex: "02000600" + // $$Symbols header + hex: "00" + } + } + } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /debug:full /out:a.exe pch.obj a.obj b.obj c.obj entry.obj" + } + } + + steps: + { + // load msf + // find named streams + // find string table + // find TPI + expect_pdb: + { + path: "a.pdb" + expected: + { + pdb: + { + tpi: + { + leaf_count: 5 + @count(5) leaves: + { + leaf_0: { kind: LF_POINTER, data_size: 8, type: 3, attributes: 0 } + leaf_1: { kind: LF_PROCEDURE, data_size: 12, return_type: 4096, call_kind: 2 } + leaf_2: { kind: LF_PROCEDURE, data_size: 12, return_type: 4096, call_kind: 0 } + leaf_3: { kind: LF_POINTER, data_size: 8, type: 17, attributes: 0 } + leaf_4: { kind: LF_PROCEDURE, data_size: 12, return_type: 4099, call_kind: 0 } + } + } + } + } + } + } +} diff --git a/src/linker/tests/ms_link_icf_section_flag_eligibility.tst b/src/linker/tests/ms_link_icf_section_flag_eligibility.tst new file mode 100644 index 000000000..953833887 --- /dev/null +++ b/src/linker/tests/ms_link_icf_section_flag_eligibility.tst @@ -0,0 +1,110 @@ +test: +{ + // ret + artifacts: + { + object: + { + file_name: "ms_icf_flags.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "c3" } + } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { zero: 96 } + relocations: + { + code_comdat_a_ref: { type: Addr64, offset: 0, symbol: code_comdat_a_symbol } + code_comdat_b_ref: { type: Addr64, offset: 8, symbol: code_comdat_b_symbol } + code_plain_a_ref: { type: Addr64, offset: 16, symbol: code_plain_a_symbol } + code_plain_b_ref: { type: Addr64, offset: 24, symbol: code_plain_b_symbol } + rdata_comdat_a_ref: { type: Addr64, offset: 32, symbol: rdata_comdat_a_symbol } + rdata_comdat_b_ref: { type: Addr64, offset: 40, symbol: rdata_comdat_b_symbol } + wdata_comdat_a_ref: { type: Addr64, offset: 48, symbol: wdata_comdat_a_symbol } + wdata_comdat_b_ref: { type: Addr64, offset: 56, symbol: wdata_comdat_b_symbol } + ro_code_comdat_a_ref: { type: Addr64, offset: 64, symbol: ro_code_comdat_a_symbol } + ro_code_comdat_b_ref: { type: Addr64, offset: 72, symbol: ro_code_comdat_b_symbol } + rw_code_comdat_a_ref: { type: Addr64, offset: 80, symbol: rw_code_comdat_a_symbol } + rw_code_comdat_b_ref: { type: Addr64, offset: 88, symbol: rw_code_comdat_b_symbol } + } + } + code_comdat_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + code_comdat_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + code_plain_a: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + code_plain_b: { name: ".text$mn", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + ro_code_comdat_a: { name: ".text$mn", permissions: (read), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + ro_code_comdat_b: { name: ".text$mn", permissions: (read), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + rw_code_comdat_a: { name: ".text$mn", permissions: (read, write), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + rw_code_comdat_b: { name: ".text$mn", permissions: (read, write), content: code, alignment: 1, flags: (link_comdat), data: { hex: "c3" } } + rdata_comdat_a: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0102030405060708" } } + rdata_comdat_b: { name: ".rdata$mn", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0102030405060708" } } + wdata_comdat_a: { name: ".data$mn", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0102030405060708" } } + wdata_comdat_b: { name: ".data$mn", permissions: (read, write), content: initialized_data, alignment: 1, flags: (link_comdat), data: { hex: "0102030405060708" } } + } + symbols: + { + code_comdat_a_definition: { kind: section_definition, section: code_comdat_a, selection: Any } + code_comdat_b_definition: { kind: section_definition, section: code_comdat_b, selection: Any } + rdata_comdat_a_definition: { kind: section_definition, section: rdata_comdat_a, selection: Any } + rdata_comdat_b_definition: { kind: section_definition, section: rdata_comdat_b, selection: Any } + wdata_comdat_a_definition: { kind: section_definition, section: wdata_comdat_a, selection: Any } + wdata_comdat_b_definition: { kind: section_definition, section: wdata_comdat_b, selection: Any } + ro_code_comdat_a_definition: { kind: section_definition, section: ro_code_comdat_a, selection: Any } + ro_code_comdat_b_definition: { kind: section_definition, section: ro_code_comdat_b, selection: Any } + rw_code_comdat_a_definition: { kind: section_definition, section: rw_code_comdat_a, selection: Any } + rw_code_comdat_b_definition: { kind: section_definition, section: rw_code_comdat_b, selection: Any } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + code_comdat_a_symbol: { kind: external_function, name: "code_comdat_a", section: code_comdat_a, value: 0 } + code_comdat_b_symbol: { kind: external_function, name: "code_comdat_b", section: code_comdat_b, value: 0 } + code_plain_a_symbol: { kind: external_function, name: "code_plain_a", section: code_plain_a, value: 0 } + code_plain_b_symbol: { kind: external_function, name: "code_plain_b", section: code_plain_b, value: 0 } + ro_code_comdat_a_symbol: { kind: external_function, name: "ro_code_comdat_a", section: ro_code_comdat_a, value: 0 } + ro_code_comdat_b_symbol: { kind: external_function, name: "ro_code_comdat_b", section: ro_code_comdat_b, value: 0 } + rw_code_comdat_a_symbol: { kind: external_function, name: "rw_code_comdat_a", section: rw_code_comdat_a, value: 0 } + rw_code_comdat_b_symbol: { kind: external_function, name: "rw_code_comdat_b", section: rw_code_comdat_b, value: 0 } + rdata_comdat_a_symbol: { kind: external, name: "rdata_comdat_a", section: rdata_comdat_a, value: 0 } + rdata_comdat_b_symbol: { kind: external, name: "rdata_comdat_b", section: rdata_comdat_b, value: 0 } + wdata_comdat_a_symbol: { kind: external, name: "wdata_comdat_a", section: wdata_comdat_a, value: 0 } + wdata_comdat_b_symbol: { kind: external, name: "wdata_comdat_b", section: wdata_comdat_b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "ms_icf_flags.exe", pe: {} } + } + + build: + { + compile_link: + { + tool: msvc + artifact: image + args: "ms_icf_flags.obj /link /nologo /nodefaultlib /subsystem:console /entry:entry /out:ms_icf_flags.exe /opt:ref,icf /include:addresses" + } + } + + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, other_section: ".data", other_offset: 8, relation: equal } // executable code COMDATs fold + expect_pe_word: { artifact: image, section: ".data", offset: 64, type: u64, other_section: ".data", other_offset: 72, relation: equal } // read-only code COMDATs fold + expect_pe_word: { artifact: image, section: ".data", offset: 16, type: u64, other_section: ".data", other_offset: 24, relation: not_equal } // non-COMDAT code does not fold + expect_pe_word: { artifact: image, section: ".data", offset: 48, type: u64, other_section: ".data", other_offset: 56, relation: not_equal } // writable data COMDATs do not fold + expect_pe_word: { artifact: image, section: ".data", offset: 80, type: u64, other_section: ".data", other_offset: 88, relation: not_equal } // writable code COMDATs do not + expect_pe_word: { artifact: image, section: ".data", offset: 32, type: u64, other_section: ".data", other_offset: 40, relation: equal } // read-only data COMDATs fold + } +} diff --git a/src/linker/tests/ms_link_icfs_identical_comdats.tst b/src/linker/tests/ms_link_icfs_identical_comdats.tst new file mode 100644 index 000000000..231e6abda --- /dev/null +++ b/src/linker/tests/ms_link_icfs_identical_comdats.tst @@ -0,0 +1,85 @@ +test: +{ + // xor rax, rax + // ret + // ret + artifacts: + { + object: + { + file_name: "ms_icf.obj" + coff: { object: + { + machine: x64 + sections: + { + entry: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "c3" } + } + a: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + data: { hex: "4831c0c3" } + } + b: + { + name: ".text$mn" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + data: { hex: "4831c0c3" } + } + addresses: + { + name: ".data" + permissions: (read, write) + content: initialized_data + alignment: 1 + data: { zero: 16 } + relocations: + { + a_ref: { type: Addr64, offset: 0, symbol: a_symbol } + b_ref: { type: Addr64, offset: 8, symbol: b_symbol } + } + } + } + symbols: + { + a_definition: { kind: section_definition, section: a, selection: Any } + b_definition: { kind: section_definition, section: b, selection: Any } + entry_symbol: { kind: external_function, name: "entry", section: entry, value: 0 } + a_symbol: { kind: external_function, name: "a", section: a, value: 0 } + b_symbol: { kind: external_function, name: "b", section: b, value: 0 } + addresses_symbol: { kind: external, name: "addresses", section: addresses, value: 0 } + } + } } + } + image: { file_name: "ms_icf.exe", pe: {} } + } + + build: + { + compile_link: + { + tool: msvc + artifact: image + args: "ms_icf.obj /link /nologo /nodefaultlib /subsystem:console /entry:entry /out:ms_icf.exe /opt:ref,icf /include:a /include:b /include:addresses" + } + } + + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, nonzero: true } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, other_section: ".data", other_offset: 8, relation: equal } // COMDAT are folded + } +} diff --git a/src/linker/tests/opt_ref_dangling_section.tst b/src/linker/tests/opt_ref_dangling_section.tst new file mode 100644 index 000000000..7f2de104b --- /dev/null +++ b/src/linker/tests/opt_ref_dangling_section.tst @@ -0,0 +1,54 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + data: { hex: "48c7c000000000c3" } + // ret + relocations: { f_ref: { type: Addr32Nb, offset: 0, symbol: f } } + } }, symbols: { + f: { kind: undefined, name: "f" } + entry: { kind: external, name: "entry", section: text, value: 0 } + } } } + } + a_obj: + { + file_name: "a.obj" + coff: { object: { machine: x64, sections: { data: { + name: ".data", permissions: (read, write), content: initialized_data, flags: (link_comdat) + data: { concat: { text: "A0000", zero: 1 } } + relocations: { q_ref: { type: Addr32Nb, offset: 0, symbol: q } } + } }, symbols: { + q: { kind: undefined, name: "q" } + data_def: { kind: section_definition, section: data, selection: Largest } + f: { kind: external, name: "f", section: data, value: 0 } + } } } + } + b_lib: + { + file_name: "b.lib" + coff: { library: { second_linker_member: true, members: { b_member: { + path: "b.obj" + object: { machine: x64, sections: { + q: { name: ".q", permissions: (read, write), content: initialized_data, data: { hex: "01020304" } } + data: { + name: ".data", permissions: (read, write), content: initialized_data, flags: (link_comdat) + data: { concat: { text: "BBBBBBBBBBBBBBB", zero: 1 } } + } + }, symbols: { + q: { kind: external, name: "q", section: q, value: 0 } + data_def: { kind: section_definition, section: data, selection: Largest } + f: { kind: external, name: "f", section: data, value: 0 } + } } + } } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj b.lib", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".q": {} } } } } } +} diff --git a/src/linker/tests/opt_ref_weak_alias_comdat.tst b/src/linker/tests/opt_ref_weak_alias_comdat.tst new file mode 100644 index 000000000..81cedb705 --- /dev/null +++ b/src/linker/tests/opt_ref_weak_alias_comdat.tst @@ -0,0 +1,32 @@ +test: +{ + artifacts: + { + weak_obj: + { + file_name: "weak.obj" + coff: { object: { machine: x64, + sections: { target: { name: ".target", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target" } } } + symbols: { + target_def: { kind: section_definition, section: target, selection: Any } + target: { kind: external, name: "target", section: target, value: 0 } + weak_target: { kind: weak, name: "weak_target", fallback: target, search: alias } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "48c7c000000000c3" } + relocations: { weak_ref: { type: Addr32Nb, offset: 3, symbol: weak_target } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + weak_target: { kind: undefined, name: "weak_target" } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /opt:ref /out:a.exe entry.obj weak.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".target": {} } } } } } +} diff --git a/src/linker/tests/out_of_bounds_section_number.tst b/src/linker/tests/out_of_bounds_section_number.tst new file mode 100644 index 000000000..64a25a847 --- /dev/null +++ b/src/linker/tests/out_of_bounds_section_number.tst @@ -0,0 +1,45 @@ +test: +{ + artifacts: + { + // bad.obj + bad_obj: + { + file_name: "bad.obj" + bytes: { data: { concat: + { + hex: "64860100000000003f0000000100000000000000" + hex: "2e666f6f000000000000000000000000030000003c000000000000000000000000000000400000c0" + text: "foo" + hex: "666f6f0000000000000000007b0000000200" + hex: "04000000" + } } } + } + + // entry.obj + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { hex: "48c7c000000000c3" } + relocations: { foo: { type: Addr32Nb, offset: 0, symbol: foo } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + foo: { kind: undefined, name: "foo" } + } + } } + } + } + + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj bad.obj", output: none, expect_exit: nonzero } } + steps: {} +} diff --git a/src/linker/tests/patch_cv_symbol_tree.tst b/src/linker/tests/patch_cv_symbol_tree.tst new file mode 100644 index 000000000..d8a15d714 --- /dev/null +++ b/src/linker/tests/patch_cv_symbol_tree.tst @@ -0,0 +1,4 @@ +test: +{ + steps: { patch_cv_symbol_tree: {} } +} diff --git a/src/linker/tests/pch_sig_fallback.tst b/src/linker/tests/pch_sig_fallback.tst new file mode 100644 index 000000000..36d9d3ec6 --- /dev/null +++ b/src/linker/tests/pch_sig_fallback.tst @@ -0,0 +1,252 @@ +test: +{ + + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_p: + { + name: ".debug$P" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + concat: + { + // signature + u32le: 4 + + // duplicate in a.obj + u16le: 10 + u16le: 0x1002 + u32le: 3 + u32le: 0 + + // unique procedure type + u16le: 14 + u16le: 0x1008 + u32le: 0x1000 + hex: "02 00" + u16le: 0 + u32le: 0 + + // PCH ender + u16le: 6 + u16le: 0x0014 + u32le: 0xcafebabe + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + concat: + { + u32le: 4 + align4: + { + concat: + { + u32le: 0x000000f1 + size32le: + { + concat: + { + size16le: + { + concat: + { + u16le: 0x1101 + u32le: 0xcafebabe + work_path: "a.obj" + zero: 1 + } + } + + size16le: + { + concat: + { + u16le: 0x113c + u32le: 0 + u16le: 0x00d0 + u16le: 0 + u16le: 0 + u16le: 0 + u16le: 0 + u16le: 14 + u16le: 36 + u16le: 32537 + u16le: 0 + text: "TORTURE" + zero: 1 + } + } + } + } + } + } + } + } + } + } + } + } + } + + b_obj: + { + file_name: "b.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_t: + { + name: ".debug$T" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + concat: + { + u32le: 4 + + u16le: 40 + u16le: 0x1509 + u32le: 0x1000 + u32le: 2 + u32le: 0xcafebabe + text: "corrupt-pch-file-path.obj" + zero: 1 + + u16le: 10 + u16le: 0x1002 + u32le: 3 + u32le: 0 + + u16le: 14 + u16le: 0x1008 + u32le: 0x1000 + hex: "00 00" + u16le: 0 + u32le: 0 + } + } + } + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + alignment: 1 + raw_flags: 0x02000000 + data: + { + concat: + { + u32le: 4 + align4: + { + concat: + { + u32le: 0x000000f1 + size32le: + { + concat: + { + size16le: + { + concat: + { + u16le: 0x1101 + u32le: 0xcafebabe + work_path: "a.obj" + zero: 1 + } + } + + size16le: + { + concat: + { + u16le: 0x113c + u32le: 0 + u16le: 0x00d0 + u16le: 0 + u16le: 0 + u16le: 0 + u16le: 0 + u16le: 14 + u16le: 36 + u16le: 32537 + u16le: 0 + text: "TORTURE" + zero: 1 + } + } + } + } + } + } + } + } + } + } + } + } + } + + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:full a.obj b.obj entry.obj" + } + } + + steps: {} +} diff --git a/src/linker/tests/pdbstripped.tst b/src/linker/tests/pdbstripped.tst new file mode 100644 index 000000000..4ba16bbbd --- /dev/null +++ b/src/linker/tests/pdbstripped.tst @@ -0,0 +1,161 @@ +test: +{ + + artifacts: + { + debug_obj: + { + file_name: "debug.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + raw_flags: 0x02000000 + data: + { + concat: + { + u32le: 4 + u32le: 0x000000f1 + u32le: 172 + + u16le: 16 + u16le: 0x1101 + u32le: 0x123 + text: "debug.obj" + zero: 1 + + u16le: 49 + u16le: 0x1147 + zero: 35 + text: "global_proc" + zero: 1 + + u16le: 2 + u16le: 0x114f + + u16le: 21 + u16le: 0x1108 + u32le: 0 + text: "global_typedef" + zero: 1 + + u16le: 48 + u16le: 0x1146 + zero: 35 + text: "local_proc" + zero: 1 + + u16le: 20 + u16le: 0x1108 + u32le: 0 + text: "local_typedef" + zero: 1 + + u16le: 2 + u16le: 0x114f + } + } + } + } + } + } + } + + pub_obj: + { + file_name: "pub.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, data: { text: "FOOBAR" } } + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "QWE" } } + } + symbols: + { + global_func: { kind: external_function, name: "global_func", section: text, value: 1 } + global_var: { kind: external, name: "global_var", section: data, value: 1 } + static_var: { kind: static, name: "static_var", section: data, value: 1 } + } + } + } + } + + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /debug:full /out:a.exe /pdbstripped:a.stripped.pdb entry.obj pub.obj debug.obj" + } + } + + steps: + { + expect_pdb: + { + path: "a.stripped.pdb" + expected: + { + pdb: + { + dbi: + { + @range(1, 0xffffffffffffffff) module_count + + // modules must contain only stubs for static procs + @all("*.c11_size", 0) + @all("*.c13_size", 0) + @all("*.module_symbols.symbols.*.kind", S_LPROC32, S_END) + modules: + { + module_0: { object_file_name: "entry.obj", symbol_size: 0 } + module_1: { object_file_name: "pub.obj", symbol_size: 0 } + module_2: { object_file_name: "debug.obj", @range(1, 0xffffffffffffffff) symbol_size } + } + } + + // global symbol stream must have public and references to the static stubs + @all("symbols.*.kind", S_PUB32, S_LPROCREF) + global_symbols: {} + + // types must be stripped + tpi: { header_only: true } + ipi: { header_only: true } + } + } + } + } +} diff --git a/src/linker/tests/psi_addr_map_radix_sort.tst b/src/linker/tests/psi_addr_map_radix_sort.tst new file mode 100644 index 000000000..81e26853e --- /dev/null +++ b/src/linker/tests/psi_addr_map_radix_sort.tst @@ -0,0 +1,4 @@ +test: +{ + steps: { psi_addr_map_radix_sort: {} } +} diff --git a/src/linker/tests/reloc_against_removed_comdat.tst b/src/linker/tests/reloc_against_removed_comdat.tst new file mode 100644 index 000000000..0616997bd --- /dev/null +++ b/src/linker/tests/reloc_against_removed_comdat.tst @@ -0,0 +1,71 @@ +test: +{ + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: { rdata: { name: ".rdata", permissions: (read), content: initialized_data, flags: (link_comdat), data: { concat: { text: "1Hello, World!", zero: 1 } } } } + symbols: + { + rdata_def: { kind: section_definition, section: rdata, selection: Largest } + test: { kind: external, name: "TEST", section: rdata, value: 1 } + } + } } + } + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: + { + comdat: { name: ".rdata", permissions: (read), content: initialized_data, flags: (link_comdat), data: { concat: { text: "H", zero: 1 } } } + regular: + { + name: ".rdata" + permissions: (read) + content: initialized_data + data: { zero: 4 } + relocations: { static_ref: { type: Addr32Nb, offset: 0, symbol: static_symbol } } + } + } + symbols: + { + comdat_def: { kind: section_definition, section: comdat, selection: Largest } + test: { kind: external, name: "TEST", section: comdat, value: 1 } + static_symbol: { kind: static, name: "STATIC", section: comdat, value: 2 } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + data: { concat: { hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } } + relocations: { test_ref: { type: Addr32Nb, offset: 3, symbol: test } } + } } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + test: { kind: undefined, name: "TEST" } + } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj" } } + steps: {} +} diff --git a/src/linker/tests/reloc_apply_off_out_of_bounds.tst b/src/linker/tests/reloc_apply_off_out_of_bounds.tst new file mode 100644 index 000000000..e4ffe3536 --- /dev/null +++ b/src/linker/tests/reloc_apply_off_out_of_bounds.tst @@ -0,0 +1,30 @@ +test: +{ + artifacts: + { + bad_obj: + { + file_name: "bad.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "00000000" } + relocations: { target: { type: Addr32, offset: 4294967295, symbol: target } } + } } + symbols: + { + target: { kind: absolute, name: "target", value: 0, storage: static } + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:relocation_offset_out_of_bounds.exe bad.obj", expect_exit: nonzero } } + steps: {} +} diff --git a/src/linker/tests/relocate_undefined_section_symbol.tst b/src/linker/tests/relocate_undefined_section_symbol.tst new file mode 100644 index 000000000..8459abb6c --- /dev/null +++ b/src/linker/tests/relocate_undefined_section_symbol.tst @@ -0,0 +1,150 @@ +test: +{ + + // TODO: relocations against undefined section symbols + artifacts: + { + entry_obj: + { + file_name: "relocate_undefined_section_entry.obj" + coff: { object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "48c7c000000000c3" } + relocations: { caller_ref: { type: Addr32Nb, offset: 3, symbol: caller } } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + caller: { kind: undefined, name: "caller" } + } + } } + } + caller_obj: + { + file_name: "relocate_undefined_section_caller.obj" + coff: { object: + { + machine: x64 + sections: + { + caller: + { + name: ".caller" + permissions: (read, write) + content: initialized_data + alignment: 1 + flags: (link_comdat) + data: { zero: 100 } + relocations: + { + target0_ref: { type: Addr32Nb, offset: 0, symbol: target0_section } + target1_ref: { type: Addr32Nb, offset: 4, symbol: target1_section } + target2_ref: { type: Addr32Nb, offset: 8, symbol: target2_section } + target3_ref: { type: Addr32Nb, offset: 12, symbol: target3_section } + target4_ref: { type: Addr32Nb, offset: 16, symbol: target4_section } + target5_ref: { type: Addr32Nb, offset: 20, symbol: target5_section } + target6_ref: { type: Addr32Nb, offset: 24, symbol: target6_section } + target7_ref: { type: Addr32Nb, offset: 28, symbol: target7_section } + target8_ref: { type: Addr32Nb, offset: 32, symbol: target8_section } + target9_ref: { type: Addr32Nb, offset: 36, symbol: target9_section } + target10_ref: { type: Addr32Nb, offset: 40, symbol: target10_section } + target11_ref: { type: Addr32Nb, offset: 44, symbol: target11_section } + target12_ref: { type: Addr32Nb, offset: 48, symbol: target12_section } + target13_ref: { type: Addr32Nb, offset: 52, symbol: target13_section } + target14_ref: { type: Addr32Nb, offset: 56, symbol: target14_section } + target15_ref: { type: Addr32Nb, offset: 60, symbol: target15_section } + target16_ref: { type: Addr32Nb, offset: 64, symbol: target16_section } + target17_ref: { type: Addr32Nb, offset: 68, symbol: target17_section } + target18_ref: { type: Addr32Nb, offset: 72, symbol: target18_section } + target19_ref: { type: Addr32Nb, offset: 76, symbol: target19_section } + target20_ref: { type: Addr32Nb, offset: 80, symbol: target20_section } + target21_ref: { type: Addr32Nb, offset: 84, symbol: target21_section } + target22_ref: { type: Addr32Nb, offset: 88, symbol: target22_section } + target23_ref: { type: Addr32Nb, offset: 92, symbol: target23_section } + target24_ref: { type: Addr32Nb, offset: 96, symbol: target24_section } + } + } + } + symbols: + { + caller_def: { kind: section_definition, section: caller, selection: Any } + caller: { kind: external, name: "caller", section: caller, value: 0 } + target0_section: { kind: undefined_section, name: ".target0", value: 1073741888 } + target1_section: { kind: undefined_section, name: ".target1", value: 1073741888 } + target2_section: { kind: undefined_section, name: ".target2", value: 1073741888 } + target3_section: { kind: undefined_section, name: ".target3", value: 1073741888 } + target4_section: { kind: undefined_section, name: ".target4", value: 1073741888 } + target5_section: { kind: undefined_section, name: ".target5", value: 1073741888 } + target6_section: { kind: undefined_section, name: ".target6", value: 1073741888 } + target7_section: { kind: undefined_section, name: ".target7", value: 1073741888 } + target8_section: { kind: undefined_section, name: ".target8", value: 1073741888 } + target9_section: { kind: undefined_section, name: ".target9", value: 1073741888 } + target10_section: { kind: undefined_section, name: ".target10", value: 1073741888 } + target11_section: { kind: undefined_section, name: ".target11", value: 1073741888 } + target12_section: { kind: undefined_section, name: ".target12", value: 1073741888 } + target13_section: { kind: undefined_section, name: ".target13", value: 1073741888 } + target14_section: { kind: undefined_section, name: ".target14", value: 1073741888 } + target15_section: { kind: undefined_section, name: ".target15", value: 1073741888 } + target16_section: { kind: undefined_section, name: ".target16", value: 1073741888 } + target17_section: { kind: undefined_section, name: ".target17", value: 1073741888 } + target18_section: { kind: undefined_section, name: ".target18", value: 1073741888 } + target19_section: { kind: undefined_section, name: ".target19", value: 1073741888 } + target20_section: { kind: undefined_section, name: ".target20", value: 1073741888 } + target21_section: { kind: undefined_section, name: ".target21", value: 1073741888 } + target22_section: { kind: undefined_section, name: ".target22", value: 1073741888 } + target23_section: { kind: undefined_section, name: ".target23", value: 1073741888 } + target24_section: { kind: undefined_section, name: ".target24", value: 1073741888 } + } + } } + } + + target0: { file_name: "relocate_undefined_section_target0.obj", coff: { object: { machine: x64, sections: { target: { name: ".target0", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target0" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target0", section: target, value: 0 } } } } } + target1: { file_name: "relocate_undefined_section_target1.obj", coff: { object: { machine: x64, sections: { target: { name: ".target1", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target1" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target1", section: target, value: 0 } } } } } + target2: { file_name: "relocate_undefined_section_target2.obj", coff: { object: { machine: x64, sections: { target: { name: ".target2", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target2" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target2", section: target, value: 0 } } } } } + target3: { file_name: "relocate_undefined_section_target3.obj", coff: { object: { machine: x64, sections: { target: { name: ".target3", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target3" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target3", section: target, value: 0 } } } } } + target4: { file_name: "relocate_undefined_section_target4.obj", coff: { object: { machine: x64, sections: { target: { name: ".target4", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target4" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target4", section: target, value: 0 } } } } } + target5: { file_name: "relocate_undefined_section_target5.obj", coff: { object: { machine: x64, sections: { target: { name: ".target5", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target5" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target5", section: target, value: 0 } } } } } + target6: { file_name: "relocate_undefined_section_target6.obj", coff: { object: { machine: x64, sections: { target: { name: ".target6", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target6" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target6", section: target, value: 0 } } } } } + target7: { file_name: "relocate_undefined_section_target7.obj", coff: { object: { machine: x64, sections: { target: { name: ".target7", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target7" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target7", section: target, value: 0 } } } } } + target8: { file_name: "relocate_undefined_section_target8.obj", coff: { object: { machine: x64, sections: { target: { name: ".target8", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target8" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target8", section: target, value: 0 } } } } } + target9: { file_name: "relocate_undefined_section_target9.obj", coff: { object: { machine: x64, sections: { target: { name: ".target9", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target9" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target9", section: target, value: 0 } } } } } + target10: { file_name: "relocate_undefined_section_target10.obj", coff: { object: { machine: x64, sections: { target: { name: ".target10", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target10" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target10", section: target, value: 0 } } } } } + target11: { file_name: "relocate_undefined_section_target11.obj", coff: { object: { machine: x64, sections: { target: { name: ".target11", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target11" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target11", section: target, value: 0 } } } } } + target12: { file_name: "relocate_undefined_section_target12.obj", coff: { object: { machine: x64, sections: { target: { name: ".target12", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target12" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target12", section: target, value: 0 } } } } } + target13: { file_name: "relocate_undefined_section_target13.obj", coff: { object: { machine: x64, sections: { target: { name: ".target13", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target13" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target13", section: target, value: 0 } } } } } + target14: { file_name: "relocate_undefined_section_target14.obj", coff: { object: { machine: x64, sections: { target: { name: ".target14", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target14" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target14", section: target, value: 0 } } } } } + target15: { file_name: "relocate_undefined_section_target15.obj", coff: { object: { machine: x64, sections: { target: { name: ".target15", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target15" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target15", section: target, value: 0 } } } } } + target16: { file_name: "relocate_undefined_section_target16.obj", coff: { object: { machine: x64, sections: { target: { name: ".target16", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target16" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target16", section: target, value: 0 } } } } } + target17: { file_name: "relocate_undefined_section_target17.obj", coff: { object: { machine: x64, sections: { target: { name: ".target17", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target17" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target17", section: target, value: 0 } } } } } + target18: { file_name: "relocate_undefined_section_target18.obj", coff: { object: { machine: x64, sections: { target: { name: ".target18", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target18" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target18", section: target, value: 0 } } } } } + target19: { file_name: "relocate_undefined_section_target19.obj", coff: { object: { machine: x64, sections: { target: { name: ".target19", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target19" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target19", section: target, value: 0 } } } } } + target20: { file_name: "relocate_undefined_section_target20.obj", coff: { object: { machine: x64, sections: { target: { name: ".target20", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target20" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target20", section: target, value: 0 } } } } } + target21: { file_name: "relocate_undefined_section_target21.obj", coff: { object: { machine: x64, sections: { target: { name: ".target21", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target21" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target21", section: target, value: 0 } } } } } + target22: { file_name: "relocate_undefined_section_target22.obj", coff: { object: { machine: x64, sections: { target: { name: ".target22", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target22" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target22", section: target, value: 0 } } } } } + target23: { file_name: "relocate_undefined_section_target23.obj", coff: { object: { machine: x64, sections: { target: { name: ".target23", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target23" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target23", section: target, value: 0 } } } } } + target24: { file_name: "relocate_undefined_section_target24.obj", coff: { object: { machine: x64, sections: { target: { name: ".target24", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { text: "target24" } } }, symbols: { target_def: { kind: section_definition, section: target, selection: Any }, target: { kind: external, name: "target24", section: target, value: 0 } } } } } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /opt:ref /out:relocate_undefined_section_symbol.exe relocate_undefined_section_entry.obj relocate_undefined_section_caller.obj relocate_undefined_section_target0.obj relocate_undefined_section_target1.obj relocate_undefined_section_target2.obj relocate_undefined_section_target3.obj relocate_undefined_section_target4.obj relocate_undefined_section_target5.obj relocate_undefined_section_target6.obj relocate_undefined_section_target7.obj relocate_undefined_section_target8.obj relocate_undefined_section_target9.obj relocate_undefined_section_target10.obj relocate_undefined_section_target11.obj relocate_undefined_section_target12.obj relocate_undefined_section_target13.obj relocate_undefined_section_target14.obj relocate_undefined_section_target15.obj relocate_undefined_section_target16.obj relocate_undefined_section_target17.obj relocate_undefined_section_target18.obj relocate_undefined_section_target19.obj relocate_undefined_section_target20.obj relocate_undefined_section_target21.obj relocate_undefined_section_target22.obj relocate_undefined_section_target23.obj relocate_undefined_section_target24.obj" + } + } + + steps: + { + // T_Ok(match_count > 0); + } +} diff --git a/src/linker/tests/removed_section.tst b/src/linker/tests/removed_section.tst new file mode 100644 index 000000000..02cea364e --- /dev/null +++ b/src/linker/tests/removed_section.tst @@ -0,0 +1,45 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: + { + machine: x64 + // TEST is defined in a section explicitly removed by the object flags. + sections: { test: { name: ".test", permissions: (read, execute), content: code, flags: (link_remove), data: { hex: "c3" } } } + symbols: { test: { kind: external, name: "TEST", section: test, value: 0 } } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + data: { hex: "48c7c000000000c3" } + relocations: { test: { type: Addr32Nb, offset: 3, symbol: test } } + } } + symbols: + { + test: { kind: undefined, name: "TEST" } + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + } + build: + { + // A relocation cannot resolve to a definition in a removed section. + link: { args: "/subsystem:console /entry:entry /out:removed_section.exe test.obj entry.obj", expect_exit: nonzero } + } + steps: {} +} diff --git a/src/linker/tests/second_member_header.tst b/src/linker/tests/second_member_header.tst new file mode 100644 index 000000000..0579139d0 --- /dev/null +++ b/src/linker/tests/second_member_header.tst @@ -0,0 +1,34 @@ +test: +{ + artifacts: + { + test_lib: + { + file_name: "test.lib" + coff: { library: { second_linker_member: true, members: { symbols: { + path: "obj.obj" + object: { machine: x64, symbols: { + eight: { kind: absolute, name: "8", value: 8, storage: external } + one: { kind: absolute, name: "1", value: 1, storage: external } + nine: { kind: absolute, name: "9", value: 9, storage: external } + seven: { kind: absolute, name: "7", value: 7, storage: external } + four: { kind: absolute, name: "4", value: 4, storage: external } + five: { kind: absolute, name: "5", value: 5, storage: external } + two: { kind: absolute, name: "2", value: 2, storage: external } + three: { kind: absolute, name: "3", value: 3, storage: external } + six: { kind: absolute, name: "6", value: 6, storage: external } + } } + } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe test.lib entry.obj /include:1 /include:2 /include:3 /include:4 /include:5 /include:6 /include:7 /include:8 /include:9" } } + steps: {} +} diff --git a/src/linker/tests/sect_align.tst b/src/linker/tests/sect_align.tst new file mode 100644 index 000000000..fd8d82653 --- /dev/null +++ b/src/linker/tests/sect_align.tst @@ -0,0 +1,56 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: + { + machine: x64 + sections: + { + shift: { name: ".a", permissions: (read, write), content: initialized_data, data: { text: "q" } } + none: { name: ".a", permissions: (read, write), content: initialized_data, data: { text: "abc" } } + a1: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "wr" } } + a2: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 2, data: { text: "e" } } + a4: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 4, data: { text: "ttttt" } } + a8: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 8, data: { text: "g" } } + a16: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 16, data: { text: "o" } } + a32: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 32, data: { text: "p" } } + a64: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 64, data: { text: "f" } } + a128: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 128, data: { text: "x" } } + a256: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 256, data: { text: "c" } } + a512: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 512, data: { text: "v" } } + a1024: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 1024, data: { text: "b" } } + a2048: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 2048, data: { text: "n" } } + a4096: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 4096, data: { text: "m" } } + a8192: { name: ".a", permissions: (read, write), content: initialized_data, alignment: 8192, data: { text: "z" } } + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe /align:8192 test.obj", artifact: image } } + steps: + { + expect_pe_bytes: { artifact: image, section: ".a", offset: 0, hex: "71" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 16, hex: "616263" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 19, hex: "7772" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 22, hex: "65" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 24, hex: "7474747474" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 32, hex: "67" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 48, hex: "6f" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 64, hex: "70" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 128, hex: "66" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 256, hex: "78" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 512, hex: "63" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 1024, hex: "76" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 2048, hex: "62" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 4096, hex: "6e" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 8192, hex: "6d" } + expect_pe_bytes: { artifact: image, section: ".a", offset: 16384, hex: "7a" } + } +} diff --git a/src/linker/tests/sect_symbol.tst b/src/linker/tests/sect_symbol.tst new file mode 100644 index 000000000..f81fcecc6 --- /dev/null +++ b/src/linker/tests/sect_symbol.tst @@ -0,0 +1,49 @@ +test: +{ + artifacts: + { + sect_obj: + { + file_name: "sect.obj" + coff: { object: + { + machine: x64 + sections: { mysect1: { name: ".mysect$1", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "hello, world" } } } + directives: { directive: "/merge:.mysect=.data" } + } } + } + main_obj: + { + file_name: "main.obj" + coff: { object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { zero: 8 } + relocations: { mysect_address: { type: Addr64, offset: 0, symbol: mysect_group } } + } + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + mysect_group: { kind: undefined_section, name: ".mysect$2222", value: 3221225536 } + entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe main.obj sect.obj", artifact: image } } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, target_section: ".data", target_offset: 8 } + expect_pe_bytes: { artifact: image, section: ".data", offset: 8, hex: "68656c6c6f2c20776f726c64" } + } +} diff --git a/src/linker/tests/section_directive_align_grouped_section.tst b/src/linker/tests/section_directive_align_grouped_section.tst new file mode 100644 index 000000000..42b6d8b9c --- /dev/null +++ b/src/linker/tests/section_directive_align_grouped_section.tst @@ -0,0 +1,36 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + prot_obj: + { + file_name: "prot.obj" + coff: { object: + { + machine: x64 + sections: { prot_mem: { name: "prot$mem", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "mem" } } } + directives: { directive: "/SECTION:prot,R,ALIGN=8192" } + } } + } + image: { file_name: "grouped_section_align.exe", pe: {} } + } + build: + { + // Apply read-only section flags and 8192-byte alignment to the grouped prot output section. + link: { args: "/subsystem:console /entry:entry /out:grouped_section_align.exe entry.obj prot.obj", artifact: image } + } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { "prot": { alignment: 8192, @bits_set(1088421952) raw_flags: 1088421952 } } } } } + } +} diff --git a/src/linker/tests/section_directive_read_only_grouped_section.tst b/src/linker/tests/section_directive_read_only_grouped_section.tst new file mode 100644 index 000000000..dad4167f2 --- /dev/null +++ b/src/linker/tests/section_directive_read_only_grouped_section.tst @@ -0,0 +1,35 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + prot_obj: + { + file_name: "prot.obj" + coff: { object: + { + machine: x64 + sections: + { + prot_a: { name: "prot$a", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "A" } } + prot_mem: { name: "prot$mem", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "mem" } } + prot_z: { name: "prot$z", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "Z" } } + } + directives: { directive: "/SECTION:prot,R" } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj prot.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { "prot": { @bits_set(1073741888) @bits_clear(2684354560) raw_flags: 1073741888 } } } } } } +} diff --git a/src/linker/tests/section_sort.tst b/src/linker/tests/section_sort.tst new file mode 100644 index 000000000..12f0f63cc --- /dev/null +++ b/src/linker/tests/section_sort.tst @@ -0,0 +1,43 @@ +test: +{ + artifacts: + { + data_obj: + { + file_name: "data.obj" + coff: { object: + { + machine: x64 + sections: + { + idata_2: { name: ".idata$2", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "last" } } + idata_5: { name: ".idata$5", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "first" } } + rdata: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 1, data: { text: "middle" } } + data_z: { name: ".data$z", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "five" } } + data_a: { name: ".data$a", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "three" } } + data_bbbbb: { name: ".data$bbbbb", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "four" } } + data_empty: { name: ".data$", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "two" } } + data: { name: ".data", permissions: (read, write), content: initialized_data, alignment: 1, data: { text: "one" } } + } + } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe data.obj entry.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: + { + ".data": { data: 66697273746f6e6574776f7468726565666f7572666976656c617374 } + ".rdata": { data: 6d6964646c65 } + } } } } } +} diff --git a/src/linker/tests/simple_lib_test.tst b/src/linker/tests/simple_lib_test.tst new file mode 100644 index 000000000..4bc5f255c --- /dev/null +++ b/src/linker/tests/simple_lib_test.tst @@ -0,0 +1,43 @@ +test: +{ + artifacts: + { + test_lib: + { + file_name: "test.lib" + coff: { library: { second_linker_member: true, members: { test_member: { + path: "test.obj" + object: { machine: unknown, + sections: { data: { + name: ".data", permissions: (read, write), content: initialized_data + data: { concat: { text: "The quick brown fox jumps over the lazy dog", zero: 1 } } + } } + symbols: { test: { kind: external, name: "test", section: data, value: 0 } } + } + } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, alignment: 1 + data: { hex: "48c7c000000000c3" } + relocations: { test_ref: { type: Addr32Nb, offset: 3, symbol: test } } + } }, symbols: { + test: { kind: undefined, name: "test" } + entry: { kind: external, name: "my_entry", section: text, value: 7 } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe entry.obj test.lib", artifact: image } } + steps: + { + // was test payload linked? + expect_pe: { artifact: image, expected: { pe: { sections: { ".data": { data: 54686520717569636b2062726f776e20666f78206a756d7073206f76657220746865206c617a7920646f6700 } } } } } + // do we have enough bytes to read text? + expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { virtual_size: 8 } } } } } + // linker must pull-in test.obj and patch relocation for "test" symbol + expect_pe_word: { artifact: image, section: ".text", offset: 3, type: u32, target_section: ".data", target_offset: 0, target_address: rva } + } +} diff --git a/src/linker/tests/simple_link_test.tst b/src/linker/tests/simple_link_test.tst new file mode 100644 index 000000000..f1513af69 --- /dev/null +++ b/src/linker/tests/simple_link_test.tst @@ -0,0 +1,94 @@ +test: +{ + artifacts: + { + main_obj: + { + file_name: "main.obj" + coff: { object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "qwe" } } + zero: { name: ".zero", permissions: (read, write), content: uninitialized_data, data: { zero: 5 } } + } + symbols: { my_entry: { kind: external, name: "my_entry", section: text, value: 0 } } + } } + } + image: { file_name: "a.exe", pe: {} } + } + + build: + { + link: + { + args: "/entry:my_entry /subsystem:console /fixed /filealign:512 /align:4096 /out:a.exe main.obj" + artifact: image + } + } + + steps: + { + expect_coff: + { + artifact: main_obj + expected: { coff: { object: + { + machine: Amd64 + section_count: 3 + sections: + { + section_1: { name: ".text", data: c3 } + section_2: { name: ".data", data: 717765 } + section_3: { name: ".zero", data: "" } + } + symbol_count: 1 + symbols: { symbol_0: { name: "my_entry", value: 0, section_number: 1 } } + } } } + } + expect_pe: + { + artifact: image + expected: { pe: + { + is_pe32: false + arch: x64 + subsystem: console + section_count: 3 + section_alignment: 4096 + file_alignment: 512 + symbol_count: 0 + data_directory_count: 16 + entry_point: 4096 + // check section alignment + @count(3) sections: + { + ".text": { virtual_size: 1, virtual_offset: 4096, @aligned(512) file_size, file_offset: 512, data: c3 } + ".data": { virtual_size: 3, @aligned(4096) virtual_offset, @aligned(512) file_size, data: 717765 } + ".zero": { virtual_size: 5, @aligned(4096) virtual_offset, @aligned(512) file_size } + } + optional: + { + sizeof_code: 512 + sizeof_initialized_data: 512 + sizeof_uninitialized_data: 512 + code_base: 4096 + image_base: 5368709120 + major_os_version: 6 + minor_os_version: 0 + major_image_version: 0 + minor_image_version: 0 + major_subsystem_version: 6 + minor_subsystem_version: 0 + win32_version: 0 + sizeof_image: 16384 + sizeof_headers: 512 + dll_characteristics: 33056 + loader_flags: 0 + } + } } + } + } +} diff --git a/src/linker/tests/u64_array_radix_sort_parallel.tst b/src/linker/tests/u64_array_radix_sort_parallel.tst new file mode 100644 index 000000000..0e3a86cf4 --- /dev/null +++ b/src/linker/tests/u64_array_radix_sort_parallel.tst @@ -0,0 +1,4 @@ +test: +{ + steps: { u64_array_radix_sort_parallel: {} } +} diff --git a/src/linker/tests/undef_reloc_section.tst b/src/linker/tests/undef_reloc_section.tst new file mode 100644 index 000000000..ed89b9ca0 --- /dev/null +++ b/src/linker/tests/undef_reloc_section.tst @@ -0,0 +1,29 @@ +test: +{ + artifacts: + { + main_obj: + { + file_name: "main.obj" + coff: { object: { machine: x64, sections: { + text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } + data: { + name: ".data", permissions: (read, write), content: initialized_data, data: { hex: "0000000000000000" } + relocations: { reloc_ref: { type: Addr64, offset: 0, symbol: reloc } } + } + }, symbols: { + entry: { kind: external, name: "my_entry", section: text, value: 0 } + reloc: { kind: undefined_section, name: ".reloc", value: 1107296320 } + } } } + } + sec_defn_obj: + { + file_name: "sec_defn.obj" + coff: { object: { machine: x64, sections: { mysect: { + name: ".mysect", permissions: (read), content: initialized_data, alignment: 1, data: { hex: "010203" } + } }, symbols: { mysect_definition: { kind: section_definition, section: mysect, selection: Null } } } } + } + } + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe main.obj sec_defn.obj", expect_exit: 25 } } + steps: {} +} diff --git a/src/linker/tests/undef_section.tst b/src/linker/tests/undef_section.tst new file mode 100644 index 000000000..e76c4e95f --- /dev/null +++ b/src/linker/tests/undef_section.tst @@ -0,0 +1,52 @@ +test: +{ + artifacts: + { + sec_defn_obj: + { + file_name: "sec_defn.obj" + coff: { object: + { + machine: x64 + sections: { mysect: { name: ".mysect", permissions: (read), content: initialized_data, alignment: 1, data: { hex: "010203" } } } + symbols: { mysect_definition: { kind: section_definition, section: mysect, selection: Null } } + } } + } + main_obj: + { + file_name: "main.obj" + coff: { object: + { + machine: x64 + sections: + { + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { zero: 4 } + relocations: { mysect_voff: { type: Addr32Nb, offset: 0, symbol: mysect } } + } + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + mysect: { kind: undefined_section, name: ".mysect", value: 1073741888 } + entry: { kind: external, name: "my_entry", section: text, value: 0 } + } + } } + } + image: { file_name: "undef_section.exe", pe: {} } + } + build: + { + // The section symbol's expected flags select .mysect and Addr32Nb stores its image-relative offset. + link: { args: "/subsystem:console /entry:my_entry /out:undef_section.exe main.obj sec_defn.obj", artifact: image } + } + steps: + { + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u32, target_section: ".mysect", target_offset: 0, target_address: rva } + expect_pe_bytes: { artifact: image, section: ".mysect", offset: 0, hex: "010203" } + } +} diff --git a/src/linker/tests/undef_weak_lib.tst b/src/linker/tests/undef_weak_lib.tst new file mode 100644 index 000000000..5310f4456 --- /dev/null +++ b/src/linker/tests/undef_weak_lib.tst @@ -0,0 +1,27 @@ +test: +{ + artifacts: + { + weak_obj: { file_name: "weak.obj", coff: { object: { machine: x64, symbols: { + b: { kind: absolute, name: "b", value: 3271557120, storage: external } + a: { kind: weak, name: "a", fallback: b, search: search_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "00000000" } + relocations: { a_ref: { type: Addr32Nb, offset: 0, symbol: a } } + } }, symbols: { + a: { kind: undefined, name: "a" } + entry: { kind: external, name: "entry", section: text, value: 0 } + } } } + } + } + build: + { + // undefined symbol must always replace weak symbol with search library + link: { args: "/subsystem:console /out:a.exe /entry:entry entry.obj weak.obj", expect_exit: 47 } + } + steps: {} +} diff --git a/src/linker/tests/undef_weak_search_alias.tst b/src/linker/tests/undef_weak_search_alias.tst new file mode 100644 index 000000000..4dbfd8e39 --- /dev/null +++ b/src/linker/tests/undef_weak_search_alias.tst @@ -0,0 +1,37 @@ +test: +{ + artifacts: + { + weak_obj: + { + file_name: "weak.obj" + coff: { object: { machine: x64, sections: { data: { + name: ".data", permissions: (read, write), content: initialized_data, data: { hex: "deadbeef" } + } }, symbols: { + ptr: { kind: undefined, name: "ptr" } + foo: { kind: weak, name: "foo", fallback: ptr, search: alias } + } } } + } + ptr_obj: { file_name: "ptr.obj", coff: { object: { machine: x64, symbols: { + entry: { kind: undefined, name: "entry" } + ptr: { kind: weak, name: "ptr", fallback: entry, search: alias } + } } } } + undef_obj: + { + file_name: "undef.obj" + coff: { object: { machine: x64, sections: { data: { + name: ".data", permissions: (read, write), content: initialized_data, data: { hex: "00000000" } + relocations: { foo_ref: { type: Addr32Nb, offset: 0, symbol: foo } } + } }, symbols: { foo: { kind: undefined, name: "foo" } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } + } }, symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe weak.obj entry.obj ptr.obj undef.obj" } } + steps: {} +} diff --git a/src/linker/tests/utf16_rsp.tst b/src/linker/tests/utf16_rsp.tst new file mode 100644 index 000000000..8e64d9178 --- /dev/null +++ b/src/linker/tests/utf16_rsp.tst @@ -0,0 +1,24 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + rsp_file: + { + file_name: "args.rsp" + bytes: { data: { hex: "fffe2f00730075006200730079007300740065006d003a0063006f006e0073006f006c00650020002f0065006e007400720079003a0065006e0074007200790020002f006f00750074003a0061002e00650078006500200065006e007400720079002e006f0062006a000a00" } } + } + } + + build: { link: { args: "@args.rsp", output: none } } + steps: {} +} diff --git a/src/linker/tests/validate_gsi.tst b/src/linker/tests/validate_gsi.tst new file mode 100644 index 000000000..4d8760635 --- /dev/null +++ b/src/linker/tests/validate_gsi.tst @@ -0,0 +1,157 @@ +test: +{ + + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + + debug_obj: + { + file_name: "debug.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + raw_flags: 0x02000000 + data: + { + concat: + { + // C13 signature and symbols subsection header. + u32le: 4 + u32le: 0xf1 + u32le: 102 + + // S_CONSTANT: type 0, immediate value 263, "CV_SymKind_BLOCK16". + u16le: 27 + u16le: 0x1107 + u32le: 0 + u16le: 263 + text: "CV_SymKind_BLOCK16" + zero: 1 + + // S_GDATA32: type 0, section 2 + 0x25440, "__newclmap". + u16le: 23 + u16le: 0x110d + u32le: 0 + u32le: 0x25440 + u16le: 2 + text: "__newclmap" + zero: 1 + + // S_GDATA32: type 0, section 1 + 123, "coffeebabe". + u16le: 23 + u16le: 0x110d + u32le: 0 + u32le: 123 + u16le: 1 + text: "coffeebabe" + zero: 1 + + // S_GDATA32: type 0, section 1 + 123, "deadbeef". + u16le: 21 + u16le: 0x110d + u32le: 0 + u32le: 123 + u16le: 1 + text: "deadbeef" + zero: 1 + + // Align the complete C13 subsection to four bytes. + zero: 2 + } + } + } + } + } + } + } + + // Raw helper output for byte-for-byte parity with the typed CodeView records above. + raw_debug_obj: + { + coff: + { + object: + { + machine: x64 + sections: + { + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + raw_flags: 0x02000000 + data: + { + hex: "04000000 f1000000 66000000 1b000711 00000000 0701 43565f53796d4b696e645f424c4f434b3136 00 17000d11 00000000 40540200 0200 5f5f6e6577636c6d6170 00 17000d11 00000000 7b000000 0100 636f6666656562616265 00 15000d11 00000000 7b000000 0100 6465616462656566 00 0000" + } + } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /debug:full /out:a.exe entry.obj debug.obj" + } + } + + steps: + { + compare: { left: debug_obj, right: raw_debug_obj } + expect_pdb: + { + path: "a.pdb" + expected: + { + pdb: + { + gsi: + { + @count(4) symbols: + { + "CV_SymKind_BLOCK16": { kind: S_CONSTANT } + "__newclmap": { kind: S_GDATA32 } + coffeebabe: { kind: S_GDATA32 } + deadbeef: { kind: S_GDATA32 } + } + indexed_symbol_count: 4 + kind_counts: { S_CONSTANT: 1, S_GDATA32: 3 } + } + } + } + } + } +} diff --git a/src/linker/tests/validate_gsi_procs_and_typedefs.tst b/src/linker/tests/validate_gsi_procs_and_typedefs.tst new file mode 100644 index 000000000..518e6745d --- /dev/null +++ b/src/linker/tests/validate_gsi_procs_and_typedefs.tst @@ -0,0 +1,176 @@ +test: +{ + + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + + debug_obj: + { + file_name: "debug.obj" + coff: + { + object: + { + machine: x64 + sections: + { + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + raw_flags: 0x02000000 + data: + { + concat: + { + // C13 signature and symbols subsection header. + u32le: 4 + u32le: 0xf1 + u32le: 190 + + // S_OBJNAME: signature 0x123, "debug.obj". + u16le: 16 + u16le: 0x1101 + u32le: 0x123 + text: "debug.obj" + zero: 1 + + // S_GPROC32_ID: zeroed procedure fields, "global_proc". + u16le: 49 + u16le: 0x1147 + zero: 32 + u16le: 0 + hex: "00" + text: "global_proc" + zero: 1 + + // S_PROC_ID_END. + u16le: 2 + u16le: 0x114f + + // S_UDT: type 0, "global_typedef". + u16le: 21 + u16le: 0x1108 + u32le: 0 + text: "global_typedef" + zero: 1 + + // S_OBJNAME: signature 0x123, "debug.obj". + u16le: 16 + u16le: 0x1101 + u32le: 0x123 + text: "debug.obj" + zero: 1 + + // S_LPROC32_ID: zeroed procedure fields, "local_proc". + u16le: 48 + u16le: 0x1146 + zero: 32 + u16le: 0 + hex: "00" + text: "local_proc" + zero: 1 + + // S_UDT: type 0, "local_typedef". + u16le: 20 + u16le: 0x1108 + u32le: 0 + text: "local_typedef" + zero: 1 + + // S_PROC_ID_END. + u16le: 2 + u16le: 0x114f + + // Align the complete C13 subsection to four bytes. + zero: 2 + } + } + } + } + } + } + } + + // Raw helper output for byte-for-byte parity with the typed CodeView records above. + raw_debug_obj: + { + coff: + { + object: + { + machine: x64 + sections: + { + debug_s: + { + name: ".debug$S" + permissions: (read) + content: initialized_data + raw_flags: 0x02000000 + data: + { + hex: "04000000 f1000000 be000000 10000111 23010000 64656275672e6f626a 00 31004711 0000000000000000000000000000000000000000000000000000000000000000 0000 00 676c6f62616c5f70726f63 00 02004f11 15000811 00000000 676c6f62616c5f74797065646566 00 10000111 23010000 64656275672e6f626a 00 30004611 0000000000000000000000000000000000000000000000000000000000000000 0000 00 6c6f63616c5f70726f63 00 14000811 00000000 6c6f63616c5f74797065646566 00 02004f11 0000" + } + } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /debug:full /out:a.exe entry.obj debug.obj" + } + } + + steps: + { + compare: { left: debug_obj, right: raw_debug_obj } + expect_pdb: + { + path: "a.pdb" + expected: + { + pdb: + { + gsi: + { + @count(3) symbols: + { + global_proc: { kind: S_PROCREF, suc_name: 0, sym_off: 24, imod: 2 } + local_proc: { kind: S_LPROCREF, suc_name: 0, sym_off: 100, imod: 2 } + global_typedef: { kind: S_UDT } + @absent local_typedef + } + indexed_symbol_count: 3 + kind_counts: { S_PROCREF: 1, S_LPROCREF: 1, S_UDT: 1 } + } + } + } + } + } +} diff --git a/src/linker/tests/validate_info_stream.tst b/src/linker/tests/validate_info_stream.tst new file mode 100644 index 000000000..c5d408789 --- /dev/null +++ b/src/linker/tests/validate_info_stream.tst @@ -0,0 +1,4 @@ +test: +{ + steps: { validate_info_stream: {} } +} diff --git a/src/linker/tests/validate_psi.tst b/src/linker/tests/validate_psi.tst new file mode 100644 index 000000000..6ca523d5d --- /dev/null +++ b/src/linker/tests/validate_psi.tst @@ -0,0 +1,85 @@ +test: +{ + + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: + { + object: + { + machine: x64 + sections: + { + text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } + } + } + + test_obj: + { + file_name: "test.obj" + coff: + { + object: + { + machine: x64 + sections: + { + // Executable bytes containing the externally visible function. + text: { name: ".text", permissions: (read, execute), content: code, data: { text: "FOOBAR" } } + // Writable bytes containing the global and static variables. + data: { name: ".data", permissions: (read, write), content: initialized_data, data: { text: "QWE" } } + } + symbols: + { + global_func: { kind: external_function, name: "global_func", section: text, value: 1 } + global_var: { kind: external, name: "global_var", section: data, value: 1 } + static_var: { kind: static, name: "static_var", section: data, value: 1 } + } + } + } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe /debug:full entry.obj test.obj" + } + } + + steps: + { + expect_pdb: + { + path: "a.pdb" + expected: + { + pdb: + { + psi: + { + symbols: + { + global_func: { kind: S_PUB32, flags: 2, @range(1, 65535) section } + global_var: { kind: S_PUB32, flags: 0, @range(1, 65535) section } + @absent static_var + } + // Every indexed PSI record must be PUB32, and at least one must exist. + @count(1) kind_counts: { @range(1, 4294967295) S_PUB32 } + @range(1, 4294967295) indexed_symbol_count + } + } + } + } + } +} diff --git a/src/linker/tests/weak_alias_comdat_duplicate_fallback.tst b/src/linker/tests/weak_alias_comdat_duplicate_fallback.tst new file mode 100644 index 000000000..b9cb50288 --- /dev/null +++ b/src/linker/tests/weak_alias_comdat_duplicate_fallback.tst @@ -0,0 +1,493 @@ +test: +{ + + artifacts: + { + a_obj: + { + file_name: "a.obj" + coff: { object: + { + machine: x64 + sections: + { + dummy_0: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_1: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_2: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_3: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_4: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_5: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_6: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_7: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_8: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_9: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_10: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_11: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_12: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_13: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_14: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_15: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_16: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_17: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_18: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_19: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_20: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_21: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_22: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_23: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_24: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_25: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_26: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_27: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_28: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_29: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_30: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_31: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_32: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_33: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_34: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_35: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_36: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_37: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_38: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_39: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_40: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_41: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_42: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_43: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_44: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_45: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_46: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_47: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_48: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_49: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_50: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_51: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_52: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_53: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_54: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_55: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_56: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_57: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_58: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_59: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_60: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_61: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_62: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_63: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + text0: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "33c0" } } + text1: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "33c0" } } + xdata0: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01000000" } } + text2: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "33c0c3" } } + } + symbols: + { + dummy_0_def: { kind: section_definition, section: dummy_0, selection: Any } + dummy_0_symbol: { kind: external_function, name: "?dummy_a_0@@YAXXZ", section: dummy_0, value: 0 } + dummy_1_def: { kind: section_definition, section: dummy_1, selection: Any } + dummy_1_symbol: { kind: external_function, name: "?dummy_a_1@@YAXXZ", section: dummy_1, value: 0 } + dummy_2_def: { kind: section_definition, section: dummy_2, selection: Any } + dummy_2_symbol: { kind: external_function, name: "?dummy_a_2@@YAXXZ", section: dummy_2, value: 0 } + dummy_3_def: { kind: section_definition, section: dummy_3, selection: Any } + dummy_3_symbol: { kind: external_function, name: "?dummy_a_3@@YAXXZ", section: dummy_3, value: 0 } + dummy_4_def: { kind: section_definition, section: dummy_4, selection: Any } + dummy_4_symbol: { kind: external_function, name: "?dummy_a_4@@YAXXZ", section: dummy_4, value: 0 } + dummy_5_def: { kind: section_definition, section: dummy_5, selection: Any } + dummy_5_symbol: { kind: external_function, name: "?dummy_a_5@@YAXXZ", section: dummy_5, value: 0 } + dummy_6_def: { kind: section_definition, section: dummy_6, selection: Any } + dummy_6_symbol: { kind: external_function, name: "?dummy_a_6@@YAXXZ", section: dummy_6, value: 0 } + dummy_7_def: { kind: section_definition, section: dummy_7, selection: Any } + dummy_7_symbol: { kind: external_function, name: "?dummy_a_7@@YAXXZ", section: dummy_7, value: 0 } + dummy_8_def: { kind: section_definition, section: dummy_8, selection: Any } + dummy_8_symbol: { kind: external_function, name: "?dummy_a_8@@YAXXZ", section: dummy_8, value: 0 } + dummy_9_def: { kind: section_definition, section: dummy_9, selection: Any } + dummy_9_symbol: { kind: external_function, name: "?dummy_a_9@@YAXXZ", section: dummy_9, value: 0 } + dummy_10_def: { kind: section_definition, section: dummy_10, selection: Any } + dummy_10_symbol: { kind: external_function, name: "?dummy_a_10@@YAXXZ", section: dummy_10, value: 0 } + dummy_11_def: { kind: section_definition, section: dummy_11, selection: Any } + dummy_11_symbol: { kind: external_function, name: "?dummy_a_11@@YAXXZ", section: dummy_11, value: 0 } + dummy_12_def: { kind: section_definition, section: dummy_12, selection: Any } + dummy_12_symbol: { kind: external_function, name: "?dummy_a_12@@YAXXZ", section: dummy_12, value: 0 } + dummy_13_def: { kind: section_definition, section: dummy_13, selection: Any } + dummy_13_symbol: { kind: external_function, name: "?dummy_a_13@@YAXXZ", section: dummy_13, value: 0 } + dummy_14_def: { kind: section_definition, section: dummy_14, selection: Any } + dummy_14_symbol: { kind: external_function, name: "?dummy_a_14@@YAXXZ", section: dummy_14, value: 0 } + dummy_15_def: { kind: section_definition, section: dummy_15, selection: Any } + dummy_15_symbol: { kind: external_function, name: "?dummy_a_15@@YAXXZ", section: dummy_15, value: 0 } + dummy_16_def: { kind: section_definition, section: dummy_16, selection: Any } + dummy_16_symbol: { kind: external_function, name: "?dummy_a_16@@YAXXZ", section: dummy_16, value: 0 } + dummy_17_def: { kind: section_definition, section: dummy_17, selection: Any } + dummy_17_symbol: { kind: external_function, name: "?dummy_a_17@@YAXXZ", section: dummy_17, value: 0 } + dummy_18_def: { kind: section_definition, section: dummy_18, selection: Any } + dummy_18_symbol: { kind: external_function, name: "?dummy_a_18@@YAXXZ", section: dummy_18, value: 0 } + dummy_19_def: { kind: section_definition, section: dummy_19, selection: Any } + dummy_19_symbol: { kind: external_function, name: "?dummy_a_19@@YAXXZ", section: dummy_19, value: 0 } + dummy_20_def: { kind: section_definition, section: dummy_20, selection: Any } + dummy_20_symbol: { kind: external_function, name: "?dummy_a_20@@YAXXZ", section: dummy_20, value: 0 } + dummy_21_def: { kind: section_definition, section: dummy_21, selection: Any } + dummy_21_symbol: { kind: external_function, name: "?dummy_a_21@@YAXXZ", section: dummy_21, value: 0 } + dummy_22_def: { kind: section_definition, section: dummy_22, selection: Any } + dummy_22_symbol: { kind: external_function, name: "?dummy_a_22@@YAXXZ", section: dummy_22, value: 0 } + dummy_23_def: { kind: section_definition, section: dummy_23, selection: Any } + dummy_23_symbol: { kind: external_function, name: "?dummy_a_23@@YAXXZ", section: dummy_23, value: 0 } + dummy_24_def: { kind: section_definition, section: dummy_24, selection: Any } + dummy_24_symbol: { kind: external_function, name: "?dummy_a_24@@YAXXZ", section: dummy_24, value: 0 } + dummy_25_def: { kind: section_definition, section: dummy_25, selection: Any } + dummy_25_symbol: { kind: external_function, name: "?dummy_a_25@@YAXXZ", section: dummy_25, value: 0 } + dummy_26_def: { kind: section_definition, section: dummy_26, selection: Any } + dummy_26_symbol: { kind: external_function, name: "?dummy_a_26@@YAXXZ", section: dummy_26, value: 0 } + dummy_27_def: { kind: section_definition, section: dummy_27, selection: Any } + dummy_27_symbol: { kind: external_function, name: "?dummy_a_27@@YAXXZ", section: dummy_27, value: 0 } + dummy_28_def: { kind: section_definition, section: dummy_28, selection: Any } + dummy_28_symbol: { kind: external_function, name: "?dummy_a_28@@YAXXZ", section: dummy_28, value: 0 } + dummy_29_def: { kind: section_definition, section: dummy_29, selection: Any } + dummy_29_symbol: { kind: external_function, name: "?dummy_a_29@@YAXXZ", section: dummy_29, value: 0 } + dummy_30_def: { kind: section_definition, section: dummy_30, selection: Any } + dummy_30_symbol: { kind: external_function, name: "?dummy_a_30@@YAXXZ", section: dummy_30, value: 0 } + dummy_31_def: { kind: section_definition, section: dummy_31, selection: Any } + dummy_31_symbol: { kind: external_function, name: "?dummy_a_31@@YAXXZ", section: dummy_31, value: 0 } + dummy_32_def: { kind: section_definition, section: dummy_32, selection: Any } + dummy_32_symbol: { kind: external_function, name: "?dummy_a_32@@YAXXZ", section: dummy_32, value: 0 } + dummy_33_def: { kind: section_definition, section: dummy_33, selection: Any } + dummy_33_symbol: { kind: external_function, name: "?dummy_a_33@@YAXXZ", section: dummy_33, value: 0 } + dummy_34_def: { kind: section_definition, section: dummy_34, selection: Any } + dummy_34_symbol: { kind: external_function, name: "?dummy_a_34@@YAXXZ", section: dummy_34, value: 0 } + dummy_35_def: { kind: section_definition, section: dummy_35, selection: Any } + dummy_35_symbol: { kind: external_function, name: "?dummy_a_35@@YAXXZ", section: dummy_35, value: 0 } + dummy_36_def: { kind: section_definition, section: dummy_36, selection: Any } + dummy_36_symbol: { kind: external_function, name: "?dummy_a_36@@YAXXZ", section: dummy_36, value: 0 } + dummy_37_def: { kind: section_definition, section: dummy_37, selection: Any } + dummy_37_symbol: { kind: external_function, name: "?dummy_a_37@@YAXXZ", section: dummy_37, value: 0 } + dummy_38_def: { kind: section_definition, section: dummy_38, selection: Any } + dummy_38_symbol: { kind: external_function, name: "?dummy_a_38@@YAXXZ", section: dummy_38, value: 0 } + dummy_39_def: { kind: section_definition, section: dummy_39, selection: Any } + dummy_39_symbol: { kind: external_function, name: "?dummy_a_39@@YAXXZ", section: dummy_39, value: 0 } + dummy_40_def: { kind: section_definition, section: dummy_40, selection: Any } + dummy_40_symbol: { kind: external_function, name: "?dummy_a_40@@YAXXZ", section: dummy_40, value: 0 } + dummy_41_def: { kind: section_definition, section: dummy_41, selection: Any } + dummy_41_symbol: { kind: external_function, name: "?dummy_a_41@@YAXXZ", section: dummy_41, value: 0 } + dummy_42_def: { kind: section_definition, section: dummy_42, selection: Any } + dummy_42_symbol: { kind: external_function, name: "?dummy_a_42@@YAXXZ", section: dummy_42, value: 0 } + dummy_43_def: { kind: section_definition, section: dummy_43, selection: Any } + dummy_43_symbol: { kind: external_function, name: "?dummy_a_43@@YAXXZ", section: dummy_43, value: 0 } + dummy_44_def: { kind: section_definition, section: dummy_44, selection: Any } + dummy_44_symbol: { kind: external_function, name: "?dummy_a_44@@YAXXZ", section: dummy_44, value: 0 } + dummy_45_def: { kind: section_definition, section: dummy_45, selection: Any } + dummy_45_symbol: { kind: external_function, name: "?dummy_a_45@@YAXXZ", section: dummy_45, value: 0 } + dummy_46_def: { kind: section_definition, section: dummy_46, selection: Any } + dummy_46_symbol: { kind: external_function, name: "?dummy_a_46@@YAXXZ", section: dummy_46, value: 0 } + dummy_47_def: { kind: section_definition, section: dummy_47, selection: Any } + dummy_47_symbol: { kind: external_function, name: "?dummy_a_47@@YAXXZ", section: dummy_47, value: 0 } + dummy_48_def: { kind: section_definition, section: dummy_48, selection: Any } + dummy_48_symbol: { kind: external_function, name: "?dummy_a_48@@YAXXZ", section: dummy_48, value: 0 } + dummy_49_def: { kind: section_definition, section: dummy_49, selection: Any } + dummy_49_symbol: { kind: external_function, name: "?dummy_a_49@@YAXXZ", section: dummy_49, value: 0 } + dummy_50_def: { kind: section_definition, section: dummy_50, selection: Any } + dummy_50_symbol: { kind: external_function, name: "?dummy_a_50@@YAXXZ", section: dummy_50, value: 0 } + dummy_51_def: { kind: section_definition, section: dummy_51, selection: Any } + dummy_51_symbol: { kind: external_function, name: "?dummy_a_51@@YAXXZ", section: dummy_51, value: 0 } + dummy_52_def: { kind: section_definition, section: dummy_52, selection: Any } + dummy_52_symbol: { kind: external_function, name: "?dummy_a_52@@YAXXZ", section: dummy_52, value: 0 } + dummy_53_def: { kind: section_definition, section: dummy_53, selection: Any } + dummy_53_symbol: { kind: external_function, name: "?dummy_a_53@@YAXXZ", section: dummy_53, value: 0 } + dummy_54_def: { kind: section_definition, section: dummy_54, selection: Any } + dummy_54_symbol: { kind: external_function, name: "?dummy_a_54@@YAXXZ", section: dummy_54, value: 0 } + dummy_55_def: { kind: section_definition, section: dummy_55, selection: Any } + dummy_55_symbol: { kind: external_function, name: "?dummy_a_55@@YAXXZ", section: dummy_55, value: 0 } + dummy_56_def: { kind: section_definition, section: dummy_56, selection: Any } + dummy_56_symbol: { kind: external_function, name: "?dummy_a_56@@YAXXZ", section: dummy_56, value: 0 } + dummy_57_def: { kind: section_definition, section: dummy_57, selection: Any } + dummy_57_symbol: { kind: external_function, name: "?dummy_a_57@@YAXXZ", section: dummy_57, value: 0 } + dummy_58_def: { kind: section_definition, section: dummy_58, selection: Any } + dummy_58_symbol: { kind: external_function, name: "?dummy_a_58@@YAXXZ", section: dummy_58, value: 0 } + dummy_59_def: { kind: section_definition, section: dummy_59, selection: Any } + dummy_59_symbol: { kind: external_function, name: "?dummy_a_59@@YAXXZ", section: dummy_59, value: 0 } + dummy_60_def: { kind: section_definition, section: dummy_60, selection: Any } + dummy_60_symbol: { kind: external_function, name: "?dummy_a_60@@YAXXZ", section: dummy_60, value: 0 } + dummy_61_def: { kind: section_definition, section: dummy_61, selection: Any } + dummy_61_symbol: { kind: external_function, name: "?dummy_a_61@@YAXXZ", section: dummy_61, value: 0 } + dummy_62_def: { kind: section_definition, section: dummy_62, selection: Any } + dummy_62_symbol: { kind: external_function, name: "?dummy_a_62@@YAXXZ", section: dummy_62, value: 0 } + dummy_63_def: { kind: section_definition, section: dummy_63, selection: Any } + dummy_63_symbol: { kind: external_function, name: "?dummy_a_63@@YAXXZ", section: dummy_63, value: 0 } + text0_def: { kind: section_definition, section: text0, selection: Any } + shared_count_delete: { kind: external_function, name: "??_G__shared_count@__Cr@std@@MEAAPEAXI@Z", section: text0, value: 0 } + text1_def: { kind: section_definition, section: text1, selection: Any } + shared_weak_count_delete: { kind: external_function, name: "??_G__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", section: text1, value: 0 } + xdata0_def: { kind: section_definition, section: xdata0, selection: Associative, associate: text0 } + text2_def: { kind: section_definition, section: text2, selection: Any } + ptr_delete: { kind: external_function, name: "??_G?$__shared_ptr_emplace@A@@UEAAPEAXI@Z", section: text2, value: 0 } + ptr_destroy: { kind: weak, name: "??_E?$__shared_ptr_emplace@A@@UEAAPEAXI@Z", fallback: ptr_delete, search: alias } + shared_count_destroy: { kind: weak, name: "??_E__shared_count@__Cr@std@@MEAAPEAXI@Z", fallback: shared_count_delete, search: alias } + shared_weak_count_destroy: { kind: weak, name: "??_E__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", fallback: shared_weak_count_delete, search: alias } + } + } } + } + + b_obj: + { + file_name: "b.obj" + coff: { object: + { + machine: x64 + sections: + { + dummy_0: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_1: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_2: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_3: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_4: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_5: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_6: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_7: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_8: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_9: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_10: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_11: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_12: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_13: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_14: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_15: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_16: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_17: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_18: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_19: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_20: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_21: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_22: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_23: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_24: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_25: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_26: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_27: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_28: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_29: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_30: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_31: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_32: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_33: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_34: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_35: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_36: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_37: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_38: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_39: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_40: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_41: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_42: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_43: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_44: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_45: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_46: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_47: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_48: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_49: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_50: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_51: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_52: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_53: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_54: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_55: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_56: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_57: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_58: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_59: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_60: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_61: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_62: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + dummy_63: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "c3" } } + text0: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "33c0" } } + text1: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "33c0" } } + xdata0: { name: ".xdata", permissions: (read), content: initialized_data, alignment: 4, flags: (link_comdat), data: { hex: "01000000" } } + text2: { name: ".text", permissions: (read, execute), content: code, alignment: 16, flags: (link_comdat), data: { hex: "33c0c3" } } + } + symbols: + { + dummy_0_def: { kind: section_definition, section: dummy_0, selection: Any } + dummy_0_symbol: { kind: external_function, name: "?dummy_b_0@@YAXXZ", section: dummy_0, value: 0 } + dummy_1_def: { kind: section_definition, section: dummy_1, selection: Any } + dummy_1_symbol: { kind: external_function, name: "?dummy_b_1@@YAXXZ", section: dummy_1, value: 0 } + dummy_2_def: { kind: section_definition, section: dummy_2, selection: Any } + dummy_2_symbol: { kind: external_function, name: "?dummy_b_2@@YAXXZ", section: dummy_2, value: 0 } + dummy_3_def: { kind: section_definition, section: dummy_3, selection: Any } + dummy_3_symbol: { kind: external_function, name: "?dummy_b_3@@YAXXZ", section: dummy_3, value: 0 } + dummy_4_def: { kind: section_definition, section: dummy_4, selection: Any } + dummy_4_symbol: { kind: external_function, name: "?dummy_b_4@@YAXXZ", section: dummy_4, value: 0 } + dummy_5_def: { kind: section_definition, section: dummy_5, selection: Any } + dummy_5_symbol: { kind: external_function, name: "?dummy_b_5@@YAXXZ", section: dummy_5, value: 0 } + dummy_6_def: { kind: section_definition, section: dummy_6, selection: Any } + dummy_6_symbol: { kind: external_function, name: "?dummy_b_6@@YAXXZ", section: dummy_6, value: 0 } + dummy_7_def: { kind: section_definition, section: dummy_7, selection: Any } + dummy_7_symbol: { kind: external_function, name: "?dummy_b_7@@YAXXZ", section: dummy_7, value: 0 } + dummy_8_def: { kind: section_definition, section: dummy_8, selection: Any } + dummy_8_symbol: { kind: external_function, name: "?dummy_b_8@@YAXXZ", section: dummy_8, value: 0 } + dummy_9_def: { kind: section_definition, section: dummy_9, selection: Any } + dummy_9_symbol: { kind: external_function, name: "?dummy_b_9@@YAXXZ", section: dummy_9, value: 0 } + dummy_10_def: { kind: section_definition, section: dummy_10, selection: Any } + dummy_10_symbol: { kind: external_function, name: "?dummy_b_10@@YAXXZ", section: dummy_10, value: 0 } + dummy_11_def: { kind: section_definition, section: dummy_11, selection: Any } + dummy_11_symbol: { kind: external_function, name: "?dummy_b_11@@YAXXZ", section: dummy_11, value: 0 } + dummy_12_def: { kind: section_definition, section: dummy_12, selection: Any } + dummy_12_symbol: { kind: external_function, name: "?dummy_b_12@@YAXXZ", section: dummy_12, value: 0 } + dummy_13_def: { kind: section_definition, section: dummy_13, selection: Any } + dummy_13_symbol: { kind: external_function, name: "?dummy_b_13@@YAXXZ", section: dummy_13, value: 0 } + dummy_14_def: { kind: section_definition, section: dummy_14, selection: Any } + dummy_14_symbol: { kind: external_function, name: "?dummy_b_14@@YAXXZ", section: dummy_14, value: 0 } + dummy_15_def: { kind: section_definition, section: dummy_15, selection: Any } + dummy_15_symbol: { kind: external_function, name: "?dummy_b_15@@YAXXZ", section: dummy_15, value: 0 } + dummy_16_def: { kind: section_definition, section: dummy_16, selection: Any } + dummy_16_symbol: { kind: external_function, name: "?dummy_b_16@@YAXXZ", section: dummy_16, value: 0 } + dummy_17_def: { kind: section_definition, section: dummy_17, selection: Any } + dummy_17_symbol: { kind: external_function, name: "?dummy_b_17@@YAXXZ", section: dummy_17, value: 0 } + dummy_18_def: { kind: section_definition, section: dummy_18, selection: Any } + dummy_18_symbol: { kind: external_function, name: "?dummy_b_18@@YAXXZ", section: dummy_18, value: 0 } + dummy_19_def: { kind: section_definition, section: dummy_19, selection: Any } + dummy_19_symbol: { kind: external_function, name: "?dummy_b_19@@YAXXZ", section: dummy_19, value: 0 } + dummy_20_def: { kind: section_definition, section: dummy_20, selection: Any } + dummy_20_symbol: { kind: external_function, name: "?dummy_b_20@@YAXXZ", section: dummy_20, value: 0 } + dummy_21_def: { kind: section_definition, section: dummy_21, selection: Any } + dummy_21_symbol: { kind: external_function, name: "?dummy_b_21@@YAXXZ", section: dummy_21, value: 0 } + dummy_22_def: { kind: section_definition, section: dummy_22, selection: Any } + dummy_22_symbol: { kind: external_function, name: "?dummy_b_22@@YAXXZ", section: dummy_22, value: 0 } + dummy_23_def: { kind: section_definition, section: dummy_23, selection: Any } + dummy_23_symbol: { kind: external_function, name: "?dummy_b_23@@YAXXZ", section: dummy_23, value: 0 } + dummy_24_def: { kind: section_definition, section: dummy_24, selection: Any } + dummy_24_symbol: { kind: external_function, name: "?dummy_b_24@@YAXXZ", section: dummy_24, value: 0 } + dummy_25_def: { kind: section_definition, section: dummy_25, selection: Any } + dummy_25_symbol: { kind: external_function, name: "?dummy_b_25@@YAXXZ", section: dummy_25, value: 0 } + dummy_26_def: { kind: section_definition, section: dummy_26, selection: Any } + dummy_26_symbol: { kind: external_function, name: "?dummy_b_26@@YAXXZ", section: dummy_26, value: 0 } + dummy_27_def: { kind: section_definition, section: dummy_27, selection: Any } + dummy_27_symbol: { kind: external_function, name: "?dummy_b_27@@YAXXZ", section: dummy_27, value: 0 } + dummy_28_def: { kind: section_definition, section: dummy_28, selection: Any } + dummy_28_symbol: { kind: external_function, name: "?dummy_b_28@@YAXXZ", section: dummy_28, value: 0 } + dummy_29_def: { kind: section_definition, section: dummy_29, selection: Any } + dummy_29_symbol: { kind: external_function, name: "?dummy_b_29@@YAXXZ", section: dummy_29, value: 0 } + dummy_30_def: { kind: section_definition, section: dummy_30, selection: Any } + dummy_30_symbol: { kind: external_function, name: "?dummy_b_30@@YAXXZ", section: dummy_30, value: 0 } + dummy_31_def: { kind: section_definition, section: dummy_31, selection: Any } + dummy_31_symbol: { kind: external_function, name: "?dummy_b_31@@YAXXZ", section: dummy_31, value: 0 } + dummy_32_def: { kind: section_definition, section: dummy_32, selection: Any } + dummy_32_symbol: { kind: external_function, name: "?dummy_b_32@@YAXXZ", section: dummy_32, value: 0 } + dummy_33_def: { kind: section_definition, section: dummy_33, selection: Any } + dummy_33_symbol: { kind: external_function, name: "?dummy_b_33@@YAXXZ", section: dummy_33, value: 0 } + dummy_34_def: { kind: section_definition, section: dummy_34, selection: Any } + dummy_34_symbol: { kind: external_function, name: "?dummy_b_34@@YAXXZ", section: dummy_34, value: 0 } + dummy_35_def: { kind: section_definition, section: dummy_35, selection: Any } + dummy_35_symbol: { kind: external_function, name: "?dummy_b_35@@YAXXZ", section: dummy_35, value: 0 } + dummy_36_def: { kind: section_definition, section: dummy_36, selection: Any } + dummy_36_symbol: { kind: external_function, name: "?dummy_b_36@@YAXXZ", section: dummy_36, value: 0 } + dummy_37_def: { kind: section_definition, section: dummy_37, selection: Any } + dummy_37_symbol: { kind: external_function, name: "?dummy_b_37@@YAXXZ", section: dummy_37, value: 0 } + dummy_38_def: { kind: section_definition, section: dummy_38, selection: Any } + dummy_38_symbol: { kind: external_function, name: "?dummy_b_38@@YAXXZ", section: dummy_38, value: 0 } + dummy_39_def: { kind: section_definition, section: dummy_39, selection: Any } + dummy_39_symbol: { kind: external_function, name: "?dummy_b_39@@YAXXZ", section: dummy_39, value: 0 } + dummy_40_def: { kind: section_definition, section: dummy_40, selection: Any } + dummy_40_symbol: { kind: external_function, name: "?dummy_b_40@@YAXXZ", section: dummy_40, value: 0 } + dummy_41_def: { kind: section_definition, section: dummy_41, selection: Any } + dummy_41_symbol: { kind: external_function, name: "?dummy_b_41@@YAXXZ", section: dummy_41, value: 0 } + dummy_42_def: { kind: section_definition, section: dummy_42, selection: Any } + dummy_42_symbol: { kind: external_function, name: "?dummy_b_42@@YAXXZ", section: dummy_42, value: 0 } + dummy_43_def: { kind: section_definition, section: dummy_43, selection: Any } + dummy_43_symbol: { kind: external_function, name: "?dummy_b_43@@YAXXZ", section: dummy_43, value: 0 } + dummy_44_def: { kind: section_definition, section: dummy_44, selection: Any } + dummy_44_symbol: { kind: external_function, name: "?dummy_b_44@@YAXXZ", section: dummy_44, value: 0 } + dummy_45_def: { kind: section_definition, section: dummy_45, selection: Any } + dummy_45_symbol: { kind: external_function, name: "?dummy_b_45@@YAXXZ", section: dummy_45, value: 0 } + dummy_46_def: { kind: section_definition, section: dummy_46, selection: Any } + dummy_46_symbol: { kind: external_function, name: "?dummy_b_46@@YAXXZ", section: dummy_46, value: 0 } + dummy_47_def: { kind: section_definition, section: dummy_47, selection: Any } + dummy_47_symbol: { kind: external_function, name: "?dummy_b_47@@YAXXZ", section: dummy_47, value: 0 } + dummy_48_def: { kind: section_definition, section: dummy_48, selection: Any } + dummy_48_symbol: { kind: external_function, name: "?dummy_b_48@@YAXXZ", section: dummy_48, value: 0 } + dummy_49_def: { kind: section_definition, section: dummy_49, selection: Any } + dummy_49_symbol: { kind: external_function, name: "?dummy_b_49@@YAXXZ", section: dummy_49, value: 0 } + dummy_50_def: { kind: section_definition, section: dummy_50, selection: Any } + dummy_50_symbol: { kind: external_function, name: "?dummy_b_50@@YAXXZ", section: dummy_50, value: 0 } + dummy_51_def: { kind: section_definition, section: dummy_51, selection: Any } + dummy_51_symbol: { kind: external_function, name: "?dummy_b_51@@YAXXZ", section: dummy_51, value: 0 } + dummy_52_def: { kind: section_definition, section: dummy_52, selection: Any } + dummy_52_symbol: { kind: external_function, name: "?dummy_b_52@@YAXXZ", section: dummy_52, value: 0 } + dummy_53_def: { kind: section_definition, section: dummy_53, selection: Any } + dummy_53_symbol: { kind: external_function, name: "?dummy_b_53@@YAXXZ", section: dummy_53, value: 0 } + dummy_54_def: { kind: section_definition, section: dummy_54, selection: Any } + dummy_54_symbol: { kind: external_function, name: "?dummy_b_54@@YAXXZ", section: dummy_54, value: 0 } + dummy_55_def: { kind: section_definition, section: dummy_55, selection: Any } + dummy_55_symbol: { kind: external_function, name: "?dummy_b_55@@YAXXZ", section: dummy_55, value: 0 } + dummy_56_def: { kind: section_definition, section: dummy_56, selection: Any } + dummy_56_symbol: { kind: external_function, name: "?dummy_b_56@@YAXXZ", section: dummy_56, value: 0 } + dummy_57_def: { kind: section_definition, section: dummy_57, selection: Any } + dummy_57_symbol: { kind: external_function, name: "?dummy_b_57@@YAXXZ", section: dummy_57, value: 0 } + dummy_58_def: { kind: section_definition, section: dummy_58, selection: Any } + dummy_58_symbol: { kind: external_function, name: "?dummy_b_58@@YAXXZ", section: dummy_58, value: 0 } + dummy_59_def: { kind: section_definition, section: dummy_59, selection: Any } + dummy_59_symbol: { kind: external_function, name: "?dummy_b_59@@YAXXZ", section: dummy_59, value: 0 } + dummy_60_def: { kind: section_definition, section: dummy_60, selection: Any } + dummy_60_symbol: { kind: external_function, name: "?dummy_b_60@@YAXXZ", section: dummy_60, value: 0 } + dummy_61_def: { kind: section_definition, section: dummy_61, selection: Any } + dummy_61_symbol: { kind: external_function, name: "?dummy_b_61@@YAXXZ", section: dummy_61, value: 0 } + dummy_62_def: { kind: section_definition, section: dummy_62, selection: Any } + dummy_62_symbol: { kind: external_function, name: "?dummy_b_62@@YAXXZ", section: dummy_62, value: 0 } + dummy_63_def: { kind: section_definition, section: dummy_63, selection: Any } + dummy_63_symbol: { kind: external_function, name: "?dummy_b_63@@YAXXZ", section: dummy_63, value: 0 } + text0_def: { kind: section_definition, section: text0, selection: Any } + shared_count_delete: { kind: external_function, name: "??_G__shared_count@__Cr@std@@MEAAPEAXI@Z", section: text0, value: 0 } + text1_def: { kind: section_definition, section: text1, selection: Any } + shared_weak_count_delete: { kind: external_function, name: "??_G__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", section: text1, value: 0 } + xdata0_def: { kind: section_definition, section: xdata0, selection: Associative, associate: text0 } + text2_def: { kind: section_definition, section: text2, selection: Any } + ptr_delete: { kind: external_function, name: "??_G?$__shared_ptr_emplace@B@@UEAAPEAXI@Z", section: text2, value: 0 } + ptr_destroy: { kind: weak, name: "??_E?$__shared_ptr_emplace@B@@UEAAPEAXI@Z", fallback: ptr_delete, search: alias } + shared_count_destroy: { kind: weak, name: "??_E__shared_count@__Cr@std@@MEAAPEAXI@Z", fallback: shared_count_delete, search: alias } + shared_weak_count_destroy: { kind: weak, name: "??_E__shared_weak_count@__Cr@std@@MEAAPEAXI@Z", fallback: shared_weak_count_delete, search: alias } + } + } } + } + + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: + { + text: + { + name: ".text" + permissions: (read, execute) + content: code + data: + { + concat: + { + hex: "48c7c000000000" // mov rax, $imm + hex: "c3" // ret + } + } + relocations: { shared_count_ref: { type: Addr32Nb, offset: 3, symbol: shared_count_destroy } } + } + } + symbols: + { + entry: { kind: external, name: "entry", section: text, value: 0 } + shared_count_destroy: { kind: undefined, name: "??_E__shared_count@__Cr@std@@MEAAPEAXI@Z" } + } + } } + } + } + + build: + { + link: + { + args: "/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj" + } + link: + { + args: "/subsystem:console /entry:entry /out:a.exe b.obj a.obj entry.obj" + } + } + steps: {} +} diff --git a/src/linker/tests/weak_alias_vs_weak_alias.tst b/src/linker/tests/weak_alias_vs_weak_alias.tst new file mode 100644 index 000000000..4efffde90 --- /dev/null +++ b/src/linker/tests/weak_alias_vs_weak_alias.tst @@ -0,0 +1,30 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + qwe: { kind: absolute, name: "qwe", value: 273, storage: external } + sym: { kind: weak, name: "sym", fallback: qwe, search: alias } + } } } } + b_obj: { file_name: "b.obj", coff: { object: { machine: x64, symbols: { + ewq: { kind: absolute, name: "ewq", value: 546, storage: external } + sym: { kind: weak, name: "sym", fallback: ewq, search: alias } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { sym_ref: { type: Addr32, offset: 3, symbol: sym } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + sym: { kind: undefined, name: "sym" } + } } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe a.obj b.obj entry.obj", expect_exit: 24 } } + steps: {} +} diff --git a/src/linker/tests/weak_alias_vs_weak_antidep.tst b/src/linker/tests/weak_alias_vs_weak_antidep.tst new file mode 100644 index 000000000..99bfa6bed --- /dev/null +++ b/src/linker/tests/weak_alias_vs_weak_antidep.tst @@ -0,0 +1,29 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: anti_dependency } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: alias } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_alias_vs_weak_lib.tst b/src/linker/tests/weak_alias_vs_weak_lib.tst new file mode 100644 index 000000000..99bfa6bed --- /dev/null +++ b/src/linker/tests/weak_alias_vs_weak_lib.tst @@ -0,0 +1,29 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: anti_dependency } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: alias } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_alias_vs_weak_nolib.tst b/src/linker/tests/weak_alias_vs_weak_nolib.tst new file mode 100644 index 000000000..256f89205 --- /dev/null +++ b/src/linker/tests/weak_alias_vs_weak_nolib.tst @@ -0,0 +1,29 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: no_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: alias } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_antidep_vs_weak_alias.tst b/src/linker/tests/weak_antidep_vs_weak_alias.tst new file mode 100644 index 000000000..e8eb9d413 --- /dev/null +++ b/src/linker/tests/weak_antidep_vs_weak_alias.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: alias } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: anti_dependency } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c011010000c3 } } } } } } +} diff --git a/src/linker/tests/weak_antidep_vs_weak_antidep.tst b/src/linker/tests/weak_antidep_vs_weak_antidep.tst new file mode 100644 index 000000000..0170df800 --- /dev/null +++ b/src/linker/tests/weak_antidep_vs_weak_antidep.tst @@ -0,0 +1,39 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: anti_dependency } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: anti_dependency } + } } } + } + a_first_image: { file_name: "a_first.exe", pe: {} } + entry_first_image: { file_name: "entry_first.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from a.obj + link: { args: "/subsystem:console /entry:entry /out:a_first.exe a.obj entry.obj", artifact: a_first_image } + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:entry_first.exe entry.obj a.obj", artifact: entry_first_image } + } + steps: + { + expect_pe: { artifact: a_first_image, expected: { pe: { sections: { ".text": { data: 48c7c011010000c3 } } } } } + expect_pe: { artifact: entry_first_image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } + } +} diff --git a/src/linker/tests/weak_antidep_vs_weak_lib.tst b/src/linker/tests/weak_antidep_vs_weak_lib.tst new file mode 100644 index 000000000..e3aaac390 --- /dev/null +++ b/src/linker/tests/weak_antidep_vs_weak_lib.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: search_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: anti_dependency } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_antidep_vs_weak_nolib.tst b/src/linker/tests/weak_antidep_vs_weak_nolib.tst new file mode 100644 index 000000000..0ea8dc8ff --- /dev/null +++ b/src/linker/tests/weak_antidep_vs_weak_nolib.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: no_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: anti_dependency } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_cycle.tst b/src/linker/tests/weak_cycle.tst new file mode 100644 index 000000000..b9085c3bb --- /dev/null +++ b/src/linker/tests/weak_cycle.tst @@ -0,0 +1,44 @@ +test: +{ + + artifacts: + { + ab: + { + file_name: "ab.obj" + coff: { object: { machine: x64, symbols: { + B: { kind: undefined, name: "B" } + A: { kind: weak, name: "A", fallback: B, search: alias } + } } } + } + ba: + { + file_name: "ba.obj" + coff: { object: { machine: x64, symbols: { + A: { kind: undefined, name: "A" } + B: { kind: weak, name: "B", fallback: A, search: alias } + } } } + } + entry: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } + } }, symbols: { + my_entry: { kind: external, name: "my_entry", section: text, value: 0 } + } } } + } + } + + build: + { + // give a generous 3 seconds + link: + { + args: "/subsystem:console /entry:my_entry entry.obj ab.obj ba.obj" + timeout_ms: 3000 + expect_exit: any + } + } + steps: {} +} diff --git a/src/linker/tests/weak_lib_vs_weak_alias.tst b/src/linker/tests/weak_lib_vs_weak_alias.tst new file mode 100644 index 000000000..d52e1ad9c --- /dev/null +++ b/src/linker/tests/weak_lib_vs_weak_alias.tst @@ -0,0 +1,31 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: alias } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: search_library } + } } } + } + } + build: + { + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", expect_exit: 24 } + } + steps: {} +} diff --git a/src/linker/tests/weak_lib_vs_weak_antidep.tst b/src/linker/tests/weak_lib_vs_weak_antidep.tst new file mode 100644 index 000000000..d15e5ac3d --- /dev/null +++ b/src/linker/tests/weak_lib_vs_weak_antidep.tst @@ -0,0 +1,32 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: anti_dependency } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: search_library } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from a.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_lib_vs_weak_lib.tst b/src/linker/tests/weak_lib_vs_weak_lib.tst new file mode 100644 index 000000000..c6bd136e4 --- /dev/null +++ b/src/linker/tests/weak_lib_vs_weak_lib.tst @@ -0,0 +1,39 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: search_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: search_library } + } } } + } + a_first_image: { file_name: "a_first.exe", pe: {} } + entry_first_image: { file_name: "entry_first.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from a.obj + link: { args: "/subsystem:console /entry:entry /out:a_first.exe a.obj entry.obj", artifact: a_first_image } + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:entry_first.exe entry.obj a.obj", artifact: entry_first_image } + } + steps: + { + expect_pe: { artifact: a_first_image, expected: { pe: { sections: { ".text": { data: 48c7c011010000c3 } } } } } + expect_pe: { artifact: entry_first_image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } + } +} diff --git a/src/linker/tests/weak_lib_vs_weak_nolib.tst b/src/linker/tests/weak_lib_vs_weak_nolib.tst new file mode 100644 index 000000000..046ad1ff1 --- /dev/null +++ b/src/linker/tests/weak_lib_vs_weak_nolib.tst @@ -0,0 +1,32 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: no_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: search_library } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + // linker must pick weak symbol from entry.obj + link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_nolib_vs_weak_alias.tst b/src/linker/tests/weak_nolib_vs_weak_alias.tst new file mode 100644 index 000000000..a8518f0f9 --- /dev/null +++ b/src/linker/tests/weak_nolib_vs_weak_alias.tst @@ -0,0 +1,27 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: alias } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: no_library } + } } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", expect_exit: 24 } } + steps: {} +} diff --git a/src/linker/tests/weak_nolib_vs_weak_antidep.tst b/src/linker/tests/weak_nolib_vs_weak_antidep.tst new file mode 100644 index 000000000..2b8ae25e9 --- /dev/null +++ b/src/linker/tests/weak_nolib_vs_weak_antidep.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: anti_dependency } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: no_library } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_nolib_vs_weak_lib.tst b/src/linker/tests/weak_nolib_vs_weak_lib.tst new file mode 100644 index 000000000..e5446306a --- /dev/null +++ b/src/linker/tests/weak_nolib_vs_weak_lib.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: search_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: no_library } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_nolib_vs_weak_nolib.tst b/src/linker/tests/weak_nolib_vs_weak_nolib.tst new file mode 100644 index 000000000..79a1e5579 --- /dev/null +++ b/src/linker/tests/weak_nolib_vs_weak_nolib.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + a_obj: { file_name: "a.obj", coff: { object: { machine: x64, symbols: { + q: { kind: absolute, name: "q", value: 273, storage: external } + w: { kind: weak, name: "w", fallback: q, search: no_library } + } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + // ret + data: { hex: "48c7c000000000c3" } + relocations: { w_ref: { type: Addr32, offset: 3, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + e: { kind: absolute, name: "e", value: 546, storage: external } + w: { kind: weak, name: "w", fallback: e, search: no_library } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj a.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".text": { data: 48c7c022020000c3 } } } } } } +} diff --git a/src/linker/tests/weak_tag.tst b/src/linker/tests/weak_tag.tst new file mode 100644 index 000000000..84ed22dcb --- /dev/null +++ b/src/linker/tests/weak_tag.tst @@ -0,0 +1,28 @@ +test: +{ + artifacts: + { + weak_tag_obj: + { + file_name: "weak_tag.obj" + coff: { object: { machine: x64, sections: { data: { + name: ".data", permissions: (read, write), content: initialized_data, data: { hex: "00000000" } + relocations: { strong_second_ref: { type: Addr32, offset: 0, symbol: strong_second } } + } }, symbols: { + abs: { kind: absolute, name: "abs", value: 305419896, storage: static } + strong_first: { kind: weak, name: "strong_first", fallback: abs, search: alias } + strong_second: { kind: weak, name: "strong_second", fallback: strong_first, search: alias } + } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } + } }, symbols: { entry: { kind: external, name: "my_entry", section: text, value: 0 } } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:my_entry /out:a.exe weak_tag.obj entry.obj", artifact: image } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".data": { virtual_size: 4, data: 78563412 } } } } } } +} diff --git a/src/linker/tests/weak_vs_common.tst b/src/linker/tests/weak_vs_common.tst new file mode 100644 index 000000000..697f1b037 --- /dev/null +++ b/src/linker/tests/weak_vs_common.tst @@ -0,0 +1,39 @@ +test: +{ + artifacts: + { + weak_obj: + { + file_name: "weak.obj" + coff: { object: { machine: x64, + sections: { a: { name: ".a", permissions: (read, write), content: initialized_data, data: { text: "a" } } } + symbols: { + a: { kind: static, name: "_a", section: a, value: 0 } + w: { kind: weak, name: "w", fallback: a, search: search_library } + } + } } + } + common_obj: { file_name: "common.obj", coff: { object: { machine: x64, symbols: { w: { kind: common, name: "w", size: 2 } } } } } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, sections: { text: { + name: ".text", permissions: (read, execute), content: code + // mov rax, $imm + data: { hex: "48c7c000000000c3" } + // ret + relocations: { w_ref: { type: Addr32Nb, offset: 0, symbol: w } } + } }, symbols: { + entry: { kind: external, name: "entry", section: text, value: 0 } + w: { kind: undefined, name: "w" } + } } } + } + image: { file_name: "a.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:a.exe common.obj weak.obj entry.obj" } + link: { args: "/subsystem:console /entry:entry /out:a.exe weak.obj common.obj entry.obj", artifact: image } + } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { ".bss": { virtual_size: 2, file_size: 0 } } } } } } +} diff --git a/src/linker/tests/whole_archive.tst b/src/linker/tests/whole_archive.tst new file mode 100644 index 000000000..9b64fb54d --- /dev/null +++ b/src/linker/tests/whole_archive.tst @@ -0,0 +1,45 @@ +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + a_lib: + { + file_name: "a.lib" + coff: { library: { second_linker_member: true, members: { a_member: { + path: "a.obj" + object: { machine: x64, sections: { a: { name: ".a", permissions: (read, write), content: initialized_data, data: { text: "a" } } } } + } } } } + } + b_lib: + { + file_name: "b.lib" + coff: { library: { second_linker_member: true, members: { b_member: { + path: "b.obj" + object: { machine: x64, sections: { b: { name: ".b", permissions: (read, write), content: initialized_data, data: { text: "b" } } } } + } } } } + } + all_image: { file_name: "all_libs.exe", pe: {} } + a_image: { file_name: "only_a.exe", pe: {} } + b_image: { file_name: "only_b.exe", pe: {} } + } + build: + { + link: { args: "/subsystem:console /entry:entry /out:all_libs.exe entry.obj /wholearchive a.lib b.lib", artifact: all_image } + link: { args: "/subsystem:console /entry:entry /out:only_a.exe entry.obj /wholearchive:a.lib a.lib b.lib", artifact: a_image } + link: { args: "/subsystem:console /entry:entry /out:only_b.exe /wholearchive:b.lib a.lib b.lib entry.obj", artifact: b_image } + } + steps: + { + expect_pe: { artifact: all_image, expected: { pe: { sections: { ".a": {}, ".b": {} } } } } + expect_pe: { artifact: a_image, expected: { pe: { sections: { ".a": {}, @absent ".b" } } } } + expect_pe: { artifact: b_image, expected: { pe: { sections: { @absent ".a", ".b": {} } } } } + } +} diff --git a/src/linker/tests/zero_length_comdat_referenced_by_reloc.tst b/src/linker/tests/zero_length_comdat_referenced_by_reloc.tst new file mode 100644 index 000000000..ba5c2a3b3 --- /dev/null +++ b/src/linker/tests/zero_length_comdat_referenced_by_reloc.tst @@ -0,0 +1,52 @@ +// A referenced zero-sized COMDAT symbol is meaningful enough for +// relocations, even though the COMDAT contributes no bytes to the image. +test: +{ + artifacts: + { + entry_obj: + { + file_name: "entry.obj" + coff: { object: + { + machine: x64 + sections: { text: { name: ".text", permissions: (read, execute), content: code, alignment: 1, data: { hex: "c3" } } } + symbols: { entry: { kind: external, name: "entry", section: text, value: 0 } } + } } + } + ref_obj: + { + file_name: "ref.obj" + coff: { object: + { + machine: x64 + sections: + { + pad: { name: ".rdata$a", permissions: (read), content: initialized_data, alignment: 1, data: { text: "xy" } } + empty: { name: ".rdata$b", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { zero: 0 } } + data: + { + name: ".data" + permissions: (read, write) + content: initialized_data + data: { zero: 8 } + relocations: { empty_ref: { type: Addr64, offset: 0, symbol: empty } } + } + } + symbols: + { + empty_def: { kind: section_definition, section: empty, selection: Any } + empty: { kind: external, name: "EMPTY", section: empty, value: 0 } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref entry.obj ref.obj", artifact: image } } + steps: + { + expect_pe: { artifact: image, expected: { pe: { sections: { ".rdata": {}, ".data": {} } } } } + expect_pe_word: { artifact: image, section: ".data", offset: 0, type: u64, target_section: ".rdata", target_offset: 2 } + expect_pe_bytes: { artifact: image, section: ".rdata", offset: 0, hex: "7879" } + } +} diff --git a/src/linker/tests/zero_length_static_comdat_referenced_by_reloc.tst b/src/linker/tests/zero_length_static_comdat_referenced_by_reloc.tst new file mode 100644 index 000000000..7c329260a --- /dev/null +++ b/src/linker/tests/zero_length_static_comdat_referenced_by_reloc.tst @@ -0,0 +1,44 @@ +test: +{ + artifacts: + { + test_obj: + { + file_name: "test.obj" + coff: { object: + { + machine: x64 + sections: + { + empty: { name: ".rdata", permissions: (read), content: initialized_data, alignment: 1, flags: (link_comdat), data: { zero: 0 } } + text: + { + name: ".text" + permissions: (read, execute) + content: code + alignment: 1 + flags: (link_comdat) + data: + { + concat: + { + hex: "488d0500000000" // lea rax, [rip + EMPTY] + hex: "c3" + } + } + relocations: { empty_ref: { type: Rel32, offset: 3, symbol: empty } } + } + } + symbols: + { + empty_def: { kind: section_definition, section: empty, selection: NoDuplicates } + empty: { kind: static, name: "EMPTY", section: empty, value: 0 } + text_def: { kind: section_definition, section: text, selection: NoDuplicates } + entry: { kind: external, name: "entry", section: text, value: 0 } + } + } } + } + } + build: { link: { args: "/subsystem:console /entry:entry /out:a.exe /opt:ref,noicf test.obj" } } + steps: {} +} diff --git a/src/mdesk/mdesk.h b/src/mdesk/mdesk.h index d6c6e5fec..4c0cc2617 100644 --- a/src/mdesk/mdesk.h +++ b/src/mdesk/mdesk.h @@ -276,6 +276,7 @@ internal MD_NodeFlags md_node_flags_from_token_flags(MD_TokenFlags flags); //- rjf: nil internal B32 md_node_is_nil(MD_Node *node); +internal B32 t_md_node_is_valid(MD_Node *node); //- rjf: iteration #define MD_EachNode(it, first) (MD_Node *it = first; !md_node_is_nil(it); it = it->next) diff --git a/src/torture/tests/torture_coff_tests.c b/src/torture/tests/torture_coff_tests.c new file mode 100644 index 000000000..c40504944 --- /dev/null +++ b/src/torture/tests/torture_coff_tests.c @@ -0,0 +1,464 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +internal String8 +t_coff_test_encode(Arena *arena, TestCtx *ctx, String8 file_name, String8 source, T_Result *result_out) +{ + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, file_name, source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + T_Artifact *value = t_artifact_from_name(&script, str8_lit("value")); + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + *result_out = result; + return value != 0 ? value->data : str8_zero(); +} + +TEST(coff_codec_object_parity) +{ + String8 source = str8_lit( + "test: { artifacts: { value: { coff: { object: { machine: x64, sections: { text: { name: \".text\", permissions: (read, execute), content: code, " + "data: { hex: \"00000000c3\" }, relocations: { ref: { type: Addr32Nb, offset: 0, symbol: undef } } }, head: { name: \".head\", permissions: (read), content: " + "initialized_data, " + "flags: (link_comdat), data: { text: \"head\" } }, assoc: { name: \".assoc\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"assoc\" " + "} } }, symbols: { " + "fallback: { kind: absolute, name: \"fallback\", value: 17, storage: external }, entry: { kind: external, name: \"entry\", section: text, value: 0 }, func: { kind: " + "external_function, " + "name: \"func\", section: text, value: 1 }, local: { kind: static, name: \"local\", section: text, value: 2 }, head_def: { kind: section_definition, section: head, " + "selection: Any }, " + "assoc_def: { kind: section_definition, section: assoc, selection: Associative, associate: head }, weak: { kind: weak, name: \"weak\", fallback: fallback, search: alias }, " + "abs_static: { kind: absolute, " + "name: \"abs_static\", value: 34, storage: static }, undef: { kind: undefined, name: \"undef\" }, undef_func: { kind: undefined_function, name: \"undef_func\" }, " + "undef_section: { kind: undefined_section, " + "name: \"undef_section\", value: 51 }, section_symbol: { kind: section, name: \"section_symbol\", section: text }, common: { kind: common, name: \"common\", size: 64 } }, " + "directives: { directive: \"/export:entry\" } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("object_parity.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + T_Artifact *value = t_artifact_from_name(&script, str8_lit("value")); + String8 expected = + t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("\0\0\0\0\xc3"), .flags = "rx:code", + .relocs = (T_COFF_DefReloc[]){T_COFF_DefReloc(X64_Addr32Nb, 0, "undef"), {0}}}, + {"head", ".head", str8_lit_comp("head"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"assoc", ".assoc", str8_lit_comp("assoc"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_AbsExtern("fallback", 17), + T_COFF_DefSymbol_Extern("entry", "text", 0), + T_COFF_DefSymbol_ExternFunc("func", "text", 1), + T_COFF_DefSymbol_Static("local", "text", 2), + T_COFF_DefSymbol_Secdef("head", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Associative("assoc", "head"), + T_COFF_DefSymbol_Weak("weak", COFF_WeakExt_SearchAlias, "fallback"), + T_COFF_DefSymbol_AbsStatic("abs_static", 34), + T_COFF_DefSymbol_Undef("undef"), + T_COFF_DefSymbol_UndefFunc("undef_func"), + T_COFF_DefSymbol_UndefSec("undef_section", 51), + T_COFF_DefSymbol_Sect("section_symbol", "text"), + T_COFF_DefSymbol_Common("common", 64), + {0}}, + .directives = (char *[]){"/export:entry", 0}, + }); + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + T_Ok(value != 0 && str8_match(value->data, expected, 0)); + T_Ok(make_directory(t_make_file_path(arena, str8_lit("legacy")))); + T_Ok(make_directory(t_make_file_path(arena, str8_lit("script")))); + T_Ok(t_write_file(str8_lit("legacy/value.obj"), expected)); + T_Ok(t_write_file(str8_lit("script/value.obj"), value->data)); + T_Ok(t_match_folders(t_make_file_path(arena, str8_lit("legacy")), t_make_file_path(arena, str8_lit("script")))); + MD_Node *semantic = 0; + result = value->codec->decode(&script, value, &semantic); + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + if (semantic != 0) { + MD_Node *object = t_child_from_string(semantic, "object"); + MD_Node *sections = t_child_from_string(object, "sections"); + MD_Node *symbols = t_child_from_string(object, "symbols"); + T_Ok(str8_match(t_scalar_string_from_node(t_child_from_string(object, "machine")), str8_lit("Amd64"), 0)); + T_Ok(str8_match(t_scalar_string_from_node(t_child_from_string(t_child_from_string(sections, "section_1"), "data")), str8_lit("00000000c3"), 0)); + T_Ok(str8_match(t_scalar_string_from_node(t_child_from_string(t_child_from_string(symbols, "symbol_0"), "name")), str8_lit("fallback"), 0)); + } +} + +TEST(coff_codec_library_parity) +{ + String8 source = str8_lit("test: { artifacts: { value: { coff: { library: { timestamp: 0, mode: 0, second_linker_member: true, members: { implementation: " + "{ path: \"member.obj\", object: { machine: x64, " + "sections: { text: { name: \".text\", permissions: (read, execute), content: code, data: { hex: \"c3\" } } }, symbols: { func: { kind: " + "external_function, name: \"func\", section: text, value: 0 } } } }, " + "imported: { import: { dll: \"foo.dll\", name: \"foo\", machine: x64, timestamp: 4294967295, type: code, lookup: name, hint: 0 } }, scaffold: { " + "dll_import: { name: \"bar.dll\", machine: x64, timestamp: 0 } } } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("library_parity.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + T_Artifact *value = t_artifact_from_name(&script, str8_lit("value")); + String8 expected = t_coff_from_def_lib( + arena, (T_COFF_DefLib){ + .emit_second_member = 1, + .members = (T_COFF_DefLibMember[]){{.type = T_COFF_DefLibMember_Obj, + .obj = {.machine = T_COFF_DefSetMachine(X64), + .path = str8_lit_comp("member.obj"), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code"}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_ExternFunc("func", "text", 0), {0}}}}, + {.type = T_COFF_DefLibMember_Import, + .import = {.dll = "foo.dll", + .name = "foo", + .import_by = COFF_ImportBy_Name, + .type = COFF_ImportHeader_Code, + .hit_or_ordinal = 0, + .time_stamp = T_COFF_DefSetTimeStamp(~0u), + .machine = T_COFF_DefSetMachine(X64)}}, + {.type = T_COFF_DefLibMember_DllImportStatic, .dll_import = {.name = "bar.dll", .machine = T_COFF_DefSetMachine(X64)}}, + {0}}, + }); + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + T_Ok(value != 0 && str8_match(value->data, expected, 0)); + T_Ok(make_directory(t_make_file_path(arena, str8_lit("legacy")))); + T_Ok(make_directory(t_make_file_path(arena, str8_lit("script")))); + T_Ok(t_write_file(str8_lit("legacy/value.lib"), expected)); + T_Ok(t_write_file(str8_lit("script/value.lib"), value->data)); + T_Ok(t_match_folders(t_make_file_path(arena, str8_lit("legacy")), t_make_file_path(arena, str8_lit("script")))); + MD_Node *semantic = 0; + result = value->codec->decode(&script, value, &semantic); + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + if (semantic != 0) { + MD_Node *library = t_child_from_string(semantic, "library"); + T_Ok(str8_match(t_scalar_string_from_node(t_child_from_string(library, "member_count")), str8_lit("5"), 0)); + T_Ok(!md_node_is_nil(t_child_from_string(t_child_from_string(t_child_from_string(library, "members"), "member_1"), "import"))); + } +} + +TEST(coff_codec_object_layout_parity) +{ + String8 source = str8_lit( + "test: { artifacts: { value: { coff: { object: { machine: x64, timestamp: 305419896, sections: { " + "write: { name: \".write\", permissions: (read, write), content: initialized_data, alignment: 32, data: { text: \"write\" } }, " + "bss: { name: \".bss\", permissions: (read, write), content: uninitialized_data, data: { zero: 4 } }, " + "long: { name: \".very_long_section_name\", permissions: (read), content: initialized_data, raw_flags: 8, data: { text: \"long\" } }, " + "empty: { name: \".empty\", permissions: (read), content: initialized_data, data: { zero: 0 } } }, " + "directives: { directive: \"/defaultlib:first\", directive: \"/include:middle\", directive: \"/export:last\" } } } } } }"); + T_Result result = {0}; + String8 actual = t_coff_test_encode(arena, ctx, str8_lit("object_layout_parity.tst"), source, &result); + String8 expected = t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .time_stamp = T_COFF_DefSetTimeStamp(305419896), + .sections = (T_COFF_DefSection[]){ + {"write", ".write", str8_lit_comp("write"), .flags = "rw:data@32"}, + {"bss", ".bss", str8_lit_comp("\0\0\0\0"), .flags = "rw:bss"}, + {"long", ".very_long_section_name", str8_lit_comp("long"), .flags = "r:data", .raw_flags = COFF_SectionFlag_TypeNoPad}, + {"empty", ".empty", str8_zero(), .flags = "r:data"}, + {0}}, + .directives = (char *[]){"/defaultlib:first", "/include:middle", "/export:last", 0}, + }); + T_Ok(t_result_is_ok(result)); + T_Ok(str8_match(actual, expected, 0)); +} + +TEST(coff_codec_comdat_weak_aux_parity) +{ + String8 source = str8_lit( + "test: { artifacts: { value: { coff: { object: { machine: x64, sections: { " + "null: { name: \".null\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"0\" } }, " + "nodup: { name: \".nodup\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"1\" } }, " + "any: { name: \".any\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"2\" } }, " + "same: { name: \".same\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"3\" } }, " + "exact: { name: \".exact\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"4\" } }, " + "assoc: { name: \".assoc\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"5\" } }, " + "largest: { name: \".largest\", permissions: (read), content: initialized_data, flags: (link_comdat), data: { text: \"6\" } }, " + "relocs: { name: \".relocs\", permissions: (read), content: initialized_data, data: { zero: 4 }, relocations: { " + "late: { type: Addr32Nb, offset: 0, symbol: late } } } }, symbols: { " + "fallback: { kind: absolute, name: \"fallback\", value: 1, storage: external }, " + "null_def: { kind: section_definition, section: null, selection: Null }, " + "nodup_def: { kind: section_definition, section: nodup, selection: NoDuplicates }, " + "any_def: { kind: section_definition, section: any, selection: Any }, " + "same_def: { kind: section_definition, section: same, selection: SameSize }, " + "exact_def: { kind: section_definition, section: exact, selection: ExactMatch }, " + "assoc_def: { kind: section_definition, section: assoc, selection: Associative, associate: any }, " + "largest_def: { kind: section_definition, section: largest, selection: Largest }, " + "weak_no_library: { kind: weak, name: \"weak_no_library\", fallback: fallback, search: no_library }, " + "weak_search_library: { kind: weak, name: \"weak_search_library\", fallback: fallback, search: search_library }, " + "weak_alias: { kind: weak, name: \"weak_alias\", fallback: fallback, search: alias }, " + "weak_anti_dependency: { kind: weak, name: \"weak_anti_dependency\", fallback: fallback, search: anti_dependency }, " + "late: { kind: undefined, name: \"relocation_target_after_aux_records\" } } } } } } }"); + T_Result result = {0}; + String8 actual = t_coff_test_encode(arena, ctx, str8_lit("comdat_weak_aux_parity.tst"), source, &result); + String8 expected = t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){ + {"null", ".null", str8_lit_comp("0"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"nodup", ".nodup", str8_lit_comp("1"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"any", ".any", str8_lit_comp("2"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"same", ".same", str8_lit_comp("3"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"exact", ".exact", str8_lit_comp("4"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"assoc", ".assoc", str8_lit_comp("5"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"largest", ".largest", str8_lit_comp("6"), .flags = "r:data", .raw_flags = COFF_SectionFlag_LnkCOMDAT}, + {"relocs", ".relocs", str8_lit_comp("\0\0\0\0"), .flags = "r:data", + .relocs = (T_COFF_DefReloc[]){T_COFF_DefReloc(X64_Addr32Nb, 0, "relocation_target_after_aux_records"), {0}}}, + {0}}, + .symbols = (T_COFF_DefSymbol[]){ + T_COFF_DefSymbol_AbsExtern("fallback", 1), + T_COFF_DefSymbol_Secdef("null", COFF_ComdatSelect_Null), + T_COFF_DefSymbol_Secdef("nodup", COFF_ComdatSelect_NoDuplicates), + T_COFF_DefSymbol_Secdef("any", COFF_ComdatSelect_Any), + T_COFF_DefSymbol_Secdef("same", COFF_ComdatSelect_SameSize), + T_COFF_DefSymbol_Secdef("exact", COFF_ComdatSelect_ExactMatch), + T_COFF_DefSymbol_Associative("assoc", "any"), + T_COFF_DefSymbol_Secdef("largest", COFF_ComdatSelect_Largest), + T_COFF_DefSymbol_Weak("weak_no_library", COFF_WeakExt_NoLibrary, "fallback"), + T_COFF_DefSymbol_Weak("weak_search_library", COFF_WeakExt_SearchLibrary, "fallback"), + T_COFF_DefSymbol_Weak("weak_alias", COFF_WeakExt_SearchAlias, "fallback"), + T_COFF_DefSymbol_Weak("weak_anti_dependency", COFF_WeakExt_AntiDependency, "fallback"), + T_COFF_DefSymbol_Undef("relocation_target_after_aux_records"), + {0}}, + }); + T_Ok(t_result_is_ok(result)); + T_Ok(str8_match(actual, expected, 0)); +} + +TEST(coff_codec_relocation_variants_parity) +{ + struct RelocationCase + { + String8 machine_name; + COFF_MachineType machine; + } cases[] = { + {str8_lit_comp("x64"), COFF_MachineType_X64}, + {str8_lit_comp("x86"), COFF_MachineType_X86}, + {str8_lit_comp("arm"), COFF_MachineType_Arm}, + {str8_lit_comp("arm64"), COFF_MachineType_Arm64}, + }; + for EachElement(case_idx, cases) + { + U64 relocation_count = 0; + for EachIndex(type, 256) { relocation_count += coff_string_from_reloc(cases[case_idx].machine, (COFF_RelocType)type).size != 0; } + COFF_RelocType *types = push_array(arena, COFF_RelocType, relocation_count); + T_COFF_DefReloc *relocations = push_array(arena, T_COFF_DefReloc, relocation_count + 1); + String8List source = {0}; + str8_list_pushf(arena, &source, + "test: { artifacts: { value: { coff: { object: { machine: %S, sections: { data: { name: \".data\", " + "permissions: (read), content: initialized_data, data: { zero: %llu }, relocations: {", + cases[case_idx].machine_name, relocation_count); + U64 relocation_idx = 0; + for EachIndex(type, 256) + { + String8 type_name = coff_string_from_reloc(cases[case_idx].machine, (COFF_RelocType)type); + if (type_name.size != 0) + { + types[relocation_idx] = (COFF_RelocType)type; + relocations[relocation_idx].type = &types[relocation_idx]; + relocations[relocation_idx].apply_off = relocation_idx; + relocations[relocation_idx].symbol = "target"; + str8_list_pushf(arena, &source, " r%llu: { type: %S, offset: %llu, symbol: target }", relocation_idx, type_name, relocation_idx); + relocation_idx += 1; + } + } + str8_list_pushf(arena, &source, " } } }, symbols: { target: { kind: undefined, name: \"target\" } } } } } } }"); + String8 file_name = push_str8f(arena, "relocation_variants_%S.tst", cases[case_idx].machine_name); + T_Result result = {0}; + String8 actual = t_coff_test_encode(arena, ctx, file_name, str8_list_join(arena, &source, 0), &result); + String8 expected = t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = &cases[case_idx].machine, + .sections = (T_COFF_DefSection[]){{"data", ".data", str8(push_array(arena, U8, relocation_count), relocation_count), .flags = "r:data", .relocs = relocations}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_Undef("target"), {0}}, + }); + T_Ok(t_result_is_ok(result)); + T_Ok(str8_match(actual, expected, 0)); + } +} + +TEST(coff_codec_library_object_paths_parity) +{ + String8 source = str8_lit( + "test: { artifacts: { value: { coff: { library: { timestamp: 0, mode: 420, second_linker_member: false, members: { " + "short: { path: \"a.obj\", object: { machine: x64, sections: { text: { name: \".text\", permissions: (read, execute), content: code, data: { hex: \"c3\" } } }, " + "symbols: { a: { kind: external_function, name: \"a\", section: text, value: 0 } } } }, " + "middle: { path: \"middle.obj\", object: { machine: x64, sections: { data: { name: \".data\", permissions: (read, write), content: initialized_data, data: { text: \"middle\" } } }, " + "symbols: { middle: { kind: external, name: \"middle\", section: data, value: 0 } } } }, " + "long: { path: \"objects/this_is_a_long_member_name.obj\", object: { machine: x64, sections: { text: { name: \".text\", permissions: (read, execute), content: code, data: { hex: \"c3\" } } }, " + "symbols: { z: { kind: external_function, name: \"z\", section: text, value: 0 } } } } } } } } } }"); + T_Result result = {0}; + String8 actual = t_coff_test_encode(arena, ctx, str8_lit("library_object_paths_parity.tst"), source, &result); + String8 expected = t_coff_from_def_lib(arena, (T_COFF_DefLib){ + .mode = 420, + .emit_second_member = 0, + .members = (T_COFF_DefLibMember[]){ + {.type = T_COFF_DefLibMember_Obj, + .obj = {.machine = T_COFF_DefSetMachine(X64), .path = str8_lit_comp("a.obj"), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code"}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_ExternFunc("a", "text", 0), {0}}}}, + {.type = T_COFF_DefLibMember_Obj, + .obj = {.machine = T_COFF_DefSetMachine(X64), .path = str8_lit_comp("middle.obj"), + .sections = (T_COFF_DefSection[]){{"data", ".data", str8_lit_comp("middle"), .flags = "rw:data"}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_Extern("middle", "data", 0), {0}}}}, + {.type = T_COFF_DefLibMember_Obj, + .obj = {.machine = T_COFF_DefSetMachine(X64), .path = str8_lit_comp("objects/this_is_a_long_member_name.obj"), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code"}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_ExternFunc("z", "text", 0), {0}}}}, + {0}}, + }); + T_Ok(t_result_is_ok(result)); + T_Ok(str8_match(actual, expected, 0)); +} + +TEST(coff_codec_import_variants_parity) +{ + struct ImportTypeCase + { + String8 name; + COFF_ImportType type; + } type_cases[] = { + {str8_lit_comp("code"), COFF_ImportHeader_Code}, + {str8_lit_comp("data"), COFF_ImportHeader_Data}, + {str8_lit_comp("const"), COFF_ImportHeader_Const}, + }; + struct ImportByCase + { + String8 name; + COFF_ImportByType type; + } import_by_cases[] = { + {str8_lit_comp("ordinal"), COFF_ImportBy_Ordinal}, + {str8_lit_comp("name"), COFF_ImportBy_Name}, + {str8_lit_comp("name_no_prefix"), COFF_ImportBy_NameNoPrefix}, + {str8_lit_comp("undecorate"), COFF_ImportBy_Undecorate}, + }; + struct ImportMachineCase + { + String8 name; + COFF_MachineType type; + } machine_cases[] = { + {str8_lit_comp("x86"), COFF_MachineType_X86}, + {str8_lit_comp("x64"), COFF_MachineType_X64}, + }; + U64 member_count = ArrayCount(type_cases) * ArrayCount(import_by_cases) * ArrayCount(machine_cases); + T_COFF_DefLibMember *members = push_array(arena, T_COFF_DefLibMember, member_count + 1); + COFF_MachineType *machines = push_array(arena, COFF_MachineType, member_count); + COFF_TimeStamp *timestamps = push_array(arena, COFF_TimeStamp, member_count); + String8List source = {0}; + str8_list_pushf(arena, &source, + "test: { artifacts: { value: { coff: { library: { timestamp: 0, mode: 0, second_linker_member: false, members: {"); + U64 member_idx = 0; + for EachElement(machine_idx, machine_cases) + { + for EachElement(type_idx, type_cases) + { + for EachElement(import_by_idx, import_by_cases) + { + U16 hint_or_ordinal = (U16)(100 + member_idx); + COFF_TimeStamp timestamp = (COFF_TimeStamp)(1000 + member_idx); + String8 name = machine_cases[machine_idx].type == COFF_MachineType_X86 ? push_str8f(arena, "_codec_import_%llu@4", member_idx) + : push_str8f(arena, "codec_import_%llu", member_idx); + char *number_field = import_by_cases[import_by_idx].type == COFF_ImportBy_Ordinal ? "ordinal" : "hint"; + str8_list_pushf(arena, &source, + " m%llu: { import: { dll: \"imports.dll\", name: \"%S\", machine: %S, timestamp: %u, type: %S, lookup: %S, %s: %u } }", + member_idx, name, machine_cases[machine_idx].name, timestamp, type_cases[type_idx].name, import_by_cases[import_by_idx].name, number_field, + hint_or_ordinal); + machines[member_idx] = machine_cases[machine_idx].type; + timestamps[member_idx] = timestamp; + members[member_idx].type = T_COFF_DefLibMember_Import; + members[member_idx].import.dll = "imports.dll"; + members[member_idx].import.name = (char *)name.str; + members[member_idx].import.import_by = import_by_cases[import_by_idx].type; + members[member_idx].import.type = type_cases[type_idx].type; + members[member_idx].import.hit_or_ordinal = hint_or_ordinal; + members[member_idx].import.time_stamp = ×tamps[member_idx]; + members[member_idx].import.machine = &machines[member_idx]; + member_idx += 1; + } + } + } + str8_list_pushf(arena, &source, " } } } } } }"); + T_Result result = {0}; + String8 actual = t_coff_test_encode(arena, ctx, str8_lit("import_variants_parity.tst"), str8_list_join(arena, &source, 0), &result); + String8 expected = t_coff_from_def_lib(arena, (T_COFF_DefLib){.emit_second_member = 0, .members = members}); + T_Ok(t_result_is_ok(result)); + T_Ok(str8_match(actual, expected, 0)); +} + +TEST(coff_codec_validation_precedes_encoding) +{ + String8 source = str8_lit("test: { artifacts: { value: { coff: { object: { machine: x64, sections: { text: { name: \".text\", permissionz: (read), content: " + "code, data: { hex: \"c3\" } } } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("invalid_coff.tst"), source, &script); + T_Artifact *value = t_artifact_from_name(&script, str8_lit("value")); + T_Ok(result.code == T_ResultCode_ValidationError); + T_Ok(value != 0 && value->data.size == 0 && value->state == T_ArtifactState_Failed); +} + +TEST(coff_codec_const_import) +{ + String8 source = str8_lit("test: { artifacts: { value: { coff: { library: { members: { imported: { import: { dll: \"foo.dll\", name: \"foo\", machine: x64, " + "type: const, lookup: undecorate } } } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("const_import.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + T_Artifact *value = t_artifact_from_name(&script, str8_lit("value")); + MD_Node *semantic = 0; + if (t_result_is_ok(result)) { result = value->codec->decode(&script, value, &semantic); } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + if (semantic != 0) { + MD_Node *members = t_child_from_string(t_child_from_string(semantic, "library"), "members"); + MD_Node *import = t_child_from_string(t_child_from_string(members, "member_0"), "import"); + T_Ok(str8_match(t_scalar_string_from_node(t_child_from_string(import, "type")), str8_lit("Const"), 0)); + T_Ok(str8_match(t_scalar_string_from_node(t_child_from_string(import, "lookup")), str8_lit("3"), 0)); + } +} + +TEST(coff_codec_rejects_variant_fields) +{ + String8 source = + str8_lit("test: { artifacts: { value: { coff: { object: { machine: x64, sections: { text: { name: \".text\", permissions: (read), content: code, " + "data: { hex: \"c3\" } } }, " + "symbols: { bad_external: { kind: external, name: \"bad\", section: text, size: 4 }, bad_absolute: { kind: absolute, name: \"abs\", value: 1 } } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("invalid_variant.tst"), source, &script); + T_Artifact *value = t_artifact_from_name(&script, str8_lit("value")); + T_Ok(result.code == T_ResultCode_ValidationError); + T_Ok(result.diagnostics.count >= 2); + T_Ok(value != 0 && value->data.size == 0 && value->state == T_ArtifactState_Failed); +} + +TEST(coff_codec_rejects_invalid_section_name_offset) +{ + String8 data = t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("\xc3"), .flags = "rx:code"}, {0}}, + }); + COFF_FileHeaderInfo header = coff_file_header_info_from_data(data); + COFF_SectionHeader *section = (COFF_SectionHeader *)(data.str + header.section_table_range.min); + MemoryZeroArray(section->name); + MemoryCopy(section->name, "/999999", 7); + T_Context decode = {.arena = arena}; + T_Artifact artifact = {.data = data, .definition = &md_nil_node}; + MD_Node *semantic = 0; + T_Result result = t_coff_decode(&decode, &artifact, &semantic); + T_Ok(result.code == T_ResultCode_ValidationError); + T_Ok(semantic == 0); +} + +TEST(coff_codec_rejects_truncated_import) +{ + COFF_LibWriter *writer = coff_lib_writer_alloc(); + coff_lib_writer_push_import(writer, COFF_MachineType_X64, 0, str8_lit("foo.dll"), COFF_ImportBy_Name, str8_lit("foo"), 0, COFF_ImportHeader_Code); + String8 data = coff_lib_writer_serialize(arena, writer, 0, 0, 1); + coff_lib_writer_release(&writer); + U64 offset = coff_regular_archive_member_iter_init(data); + COFF_ArchiveMember member = {0}; + while (coff_regular_archive_member_iter_next(data, &offset, &member)) { + if (coff_is_import(member.data)) { + COFF_ImportHeader *header = (COFF_ImportHeader *)member.data.str; + header->data_size += 1; + break; + } + } + T_Context decode = {.arena = arena}; + T_Artifact artifact = {.data = data, .definition = &md_nil_node}; + MD_Node *semantic = 0; + T_Result result = t_coff_decode(&decode, &artifact, &semantic); + T_Ok(result.code == T_ResultCode_ValidationError); + T_Ok(semantic == 0); +} diff --git a/src/torture/tests/torture_linker_tests.c b/src/torture/tests/torture_linker_tests.c new file mode 100644 index 000000000..fac1f1eee --- /dev/null +++ b/src/torture/tests/torture_linker_tests.c @@ -0,0 +1,279 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +internal String8 +t_codec_test_pdb_data(Arena *arena) +{ + PDB_Context *pdb = pdb_alloc(MSF_DEFAULT_PAGE_SIZE, COFF_MachineType_X64, 123, 1, (Guid){0}); + String8 data = data_from_pdb(arena, pdb); + pdb_release(pdb); + return data; +} + +TEST(generic_run_operation) +{ +#if OS_WINDOWS + T_Ok(copy_file_path(t_make_file_path(arena, str8_lit("run_fixture.exe")), get_process_info()->binary_file_path)); + + String8 source = str8_lit( + "test:\n" + "{\n" + " steps:\n" + " {\n" + " run: { path: \"run_fixture.exe\", args: \"-run_fixture_ok\" }\n" + " run: { path: \"run_fixture.exe\", args: \"-run_fixture_nonzero\", expect_exit: 7 }\n" + " run: { path: \"run_fixture.exe\", args: \"-run_fixture_nonzero\", expect_exit: nonzero }\n" + " run: { path: \"run_fixture.exe\", args: \"-run_fixture_nonzero\", expect_exit: any }\n" + " run: { path: \"run_fixture.exe\", args: \"-run_fixture_ok\",\n" + " stdout_matches: \"*genericrunstdout*\", stderr_matches: \"*genericrunstderr*\" }\n" + " }\n" + "}\n"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("generic_run_operation.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + + String8 timeout_source = str8_lit( + "test: { steps: { run: { path: \"run_fixture.exe\", args: \"-run_fixture_sleep\", timeout_ms: 10 } } }"); + T_Context timeout_script = {0}; + result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("generic_run_timeout.tst"), timeout_source, &timeout_script); + U64 begin_us = now_time_us(); + if (t_result_is_ok(result)) { result = t_script_execute(&timeout_script); } + U64 elapsed_us = now_time_us() - begin_us; + T_Ok(result.code == T_ResultCode_Mismatch); + T_Ok(g_last_exit_code == 999); + T_Ok(elapsed_us < TIMEOUT_SEC(5)); +#else + TestSkip(); +#endif +} + +TEST(generic_run_rejects_unsafe_path) +{ + String8 source = str8_lit( + "test:\n" + "{\n" + " steps:\n" + " {\n" + " run: { path: \"../cmd.exe\" }\n" + " run: { path: \"..\\cmd.exe\" }\n" + " run: { path: \"C:\\Windows\\System32\\cmd.exe\" }\n" + " }\n" + "}\n"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("generic_run_unsafe_path.tst"), source, &script); + T_Ok(result.code == T_ResultCode_ValidationError); + T_Ok(result.diagnostics.count == 3); +} + +TEST(generic_build_default_output) +{ +#if OS_WINDOWS + String8 source = str8_lit( + "test:\n" + "{\n" + " artifacts:\n" + " {\n" + " source: { file_name: \"main.c\", text: { data: \"int main(void) { return 0; }\" } }\n" + " }\n" + " build:\n" + " {\n" + " windows: { compile_link: \"/nologo main.c\" }\n" + " linux: { compile_link: \"main.c\" }\n" + " }\n" + "}\n"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("generic_build.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + T_Ok(file_path_exists(t_make_file_path(arena, str8_lit("main.exe")))); +#else + TestSkip(); +#endif +} + +TEST(generic_build_output_with_spaces) +{ +#if OS_WINDOWS + String8 source = str8_lit( + "test:\n" + "{\n" + " artifacts:\n" + " {\n" + " source: { file_name: \"main.c\", text: { data: \"int main(void) { return 0; }\" } }\n" + " }\n" + " build:\n" + " {\n" + " compile_link: { args: \"/nologo main.c\", output: \"main image.exe\" }\n" + " }\n" + "}\n"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("generic_build_spaces.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + T_Ok(file_path_exists(t_make_file_path(arena, str8_lit("main image.exe")))); +#else + TestSkip(); +#endif +} + +TEST(generic_build_previous_exit_condition) +{ +#if OS_WINDOWS + String8 source = str8_lit( + "test:\n" + "{\n" + " build:\n" + " {\n" + " compile: { args: \"missing.c\", output: none, expect_exit: any }\n" + " compile: { args: \"also_missing.c\", output: none, when_previous_exit: 0 }\n" + " }\n" + "}\n"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("generic_build_previous_exit.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); +#else + TestSkip(); +#endif +} + +TEST(expect_pdb_operation) +{ + String8 source = str8_lit( + "test: { steps: { expect_pdb: { path: \"generated.pdb\", expected: { pdb: { fixed_streams: { info: { present: true }, tpi: { present: true }, " + "dbi: { present: true }, ipi: { present: true } }, tpi: { leaf_count: 0, header_only: true }, ipi: { leaf_count: 0, header_only: true } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("expect_pdb_operation.tst"), source, &script); + T_Ok(t_result_is_ok(result)); + + // The operation must not snapshot the side file during script parsing. + if (t_result_is_ok(result)) { + String8 pdb = t_codec_test_pdb_data(arena); + T_Ok(t_write_file(str8_lit("generated.pdb"), pdb)); + result = t_script_execute(&script); + } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); +} + +TEST(expect_pdb_rejects_unsafe_path) +{ + String8 source = str8_lit("test: { steps: { expect_pdb: { path: \"generated.pdb\", expected: { pdb: { stream_count: 0 } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("unsafe_pdb.tst"), source, &script); + T_Ok(t_result_is_ok(result) && script.command_count == 1); + if (t_result_is_ok(result) && script.command_count == 1) { + t_codec_child(script.commands[0].arguments, "path")->first->string = str8_lit(".."); + script.result = (T_Result){0}; + T_ParseContext parse = {arena, &script, &t_codec_script_suite, script.file_path, script.source, str8_lit("expect_pdb")}; + result = t_codec_expect_pdb_validate(&parse, script.commands[0].arguments); + } + T_Ok(result.code == T_ResultCode_ValidationError); +} + +TEST(pdb_codec_rejects_malformed_tpi) +{ + String8 data = t_codec_test_pdb_data(arena); + MSF_RawStreamTable *streams = msf_raw_stream_table_from_data(arena, data); + T_Ok(streams != 0 && streams->stream_count > PDB_FixedStream_Tpi && streams->streams[PDB_FixedStream_Tpi].page_count > 0); + if (streams != 0 && streams->stream_count > PDB_FixedStream_Tpi && streams->streams[PDB_FixedStream_Tpi].page_count > 0) { + U32 page = streams->streams[PDB_FixedStream_Tpi].u.page_indices_u32[0]; + PDB_TpiHeader *header = (PDB_TpiHeader *)(data.str + (U64)page * streams->page_size); + header->leaf_data_size = max_U32; + } + T_Context decode = {.arena = arena}; + MD_Node *semantic = 0; + T_Result result = t_codec_pdb_decode(&decode, data, &semantic); + T_Ok(result.code == T_ResultCode_ValidationError); + T_Ok(semantic == 0); +} + +TEST(expect_pdb_indexes_linker_output) +{ + String8 raw_symbol = cv_make_symbol(arena, CV_SymKind_GDATA32, + cv_make_data32(arena, (CV_SymData32){.off = 1, .sec = 1}, str8_lit("indexed_data"))); + CV_Symbol symbol = cv_symbol_from_ptr(raw_symbol.str); + CV_DebugS debug_s = {0}; + str8_list_push(arena, &debug_s.data_list[CV_C13SubSectionIdxKind_Symbols], cv_data_from_symbol(arena, &symbol, CV_SymbolAlign)); + String8List debug_s_data = cv_data_from_debug_s_c13(arena, &debug_s, 1); + T_Ok(t_write_def_obj("indexed.obj", (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("x"), .flags = "rx:code"}, + {"debug", ".debug$S", str8_list_join(arena, &debug_s_data, 0), .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_ExternFunc("public_func", "text", 0), {0}}, + })); + T_Ok(t_write_entry_obj()); + t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /out:indexed.exe /pdbstripped:indexed.stripped.pdb entry.obj indexed.obj"); + T_Ok(g_last_exit_code == 0); + + String8 source = str8_lit( + "test: { steps: { expect_pdb: { path: \"indexed.pdb\", expected: { pdb: { " + "gsi: { symbols: { indexed_data: { kind: S_GDATA32 } } }, psi: { symbols: { public_func: { kind: S_PUB32, flags: 2 } } }, " + "global_symbols: { kind_counts: { S_GDATA32: 1 } }, dbi: { modules: { module_1: { object_file_name: \"indexed.obj\" } } } } } }, " + "expect_pdb: { path: \"indexed.stripped.pdb\", expected: { pdb: { tpi: { header_only: true }, ipi: { header_only: true }, " + "global_symbols: { non_public_or_proc_ref_symbol_count: 0 } } } } } }"); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, str8_lit("indexed_pdb.tst"), source, &script); + if (t_result_is_ok(result)) { result = t_script_execute(&script); } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); +} + +TEST(pdbstripped_coff_artifact_parity) +{ + String8 path = str8f(arena, "%S/linker/tests/pdbstripped.tst", t_src_path()); + String8 source = data_from_file_path(arena, path); + T_Context script = {0}; + T_Result result = t_script_parse(arena, ctx, &t_codec_script_suite, path, source, &script); + for (T_Artifact *artifact = script.first_artifact; t_result_is_ok(result) && artifact != 0; artifact = artifact->next) { + if (artifact->codec->encode != 0) { result = artifact->codec->encode(&script, artifact); } + } + if (!t_result_is_ok(result)) { t_script_test_log_result(arena, ctx, result); } + T_Ok(t_result_is_ok(result)); + + String8 raw_symbols[] = { + cv_make_symbol(arena, CV_SymKind_OBJNAME, cv_make_obj_name(arena, str8_lit("debug.obj"), 0x123)), + cv_make_symbol(arena, CV_SymKind_GPROC32_ID, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("global_proc"))), + cv_make_symbol(arena, CV_SymKind_PROC_ID_END, cv_make_end(arena)), + + cv_make_symbol(arena, CV_SymKind_UDT, cv_make_udt(arena, (CV_SymUDT){0}, str8_lit("global_typedef"))), + + cv_make_symbol(arena, CV_SymKind_LPROC32_ID, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("local_proc"))), + cv_make_symbol(arena, CV_SymKind_UDT, cv_make_udt(arena, (CV_SymUDT){0}, str8_lit("local_typedef"))), + cv_make_symbol(arena, CV_SymKind_PROC_ID_END, cv_make_end(arena)), + }; + CV_DebugS debug_s = {0}; + for EachElement(i, raw_symbols) { + CV_Symbol symbol = cv_symbol_from_ptr(raw_symbols[i].str); + str8_list_push(arena, &debug_s.data_list[CV_C13SubSectionIdxKind_Symbols], cv_data_from_symbol(arena, &symbol, CV_SymbolAlign)); + } + String8List raw_debug_s_list = cv_data_from_debug_s_c13(arena, &debug_s, 1); + String8 raw_debug_s = str8_list_join(arena, &raw_debug_s_list, 0); + String8 expected_debug = t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){{"debug_s", ".debug$S", raw_debug_s, .flags = "r:data", .raw_flags = COFF_SectionFlag_MemDiscardable}, {0}}, + }); + String8 expected_pub = t_coff_from_def_obj(arena, (T_COFF_DefObj){ + .machine = T_COFF_DefSetMachine(X64), + .sections = (T_COFF_DefSection[]){{"text", ".text", str8_lit_comp("FOOBAR"), .flags = "rx:code"}, {"data", ".data", str8_lit_comp("QWE"), .flags = "rw:data"}, {0}}, + .symbols = (T_COFF_DefSymbol[]){T_COFF_DefSymbol_ExternFunc("global_func", "text", 1), T_COFF_DefSymbol_Extern("global_var", "data", 1), + T_COFF_DefSymbol_Static("static_var", "data", 1), {0}}, + }); + + T_Artifact *debug = t_artifact_from_name(&script, str8_lit("debug_obj")); + T_Artifact *pub = t_artifact_from_name(&script, str8_lit("pub_obj")); + T_Artifact *entry = t_artifact_from_name(&script, str8_lit("entry_obj")); + if (debug != 0 && !str8_match(debug->data, expected_debug, 0)) { + U64 mismatch = 0; + while (mismatch < Min(debug->data.size, expected_debug.size) && debug->data.str[mismatch] == expected_debug.str[mismatch]) { mismatch += 1; } + test_outf("debug.obj differs at byte %llu (script size %llu, helper size %llu, helper section size %llu)\n", mismatch, debug->data.size, expected_debug.size, raw_debug_s.size); + } + T_Ok(debug != 0 && str8_match(debug->data, expected_debug, 0)); + T_Ok(pub != 0 && str8_match(pub->data, expected_pub, 0)); + T_Ok(entry != 0 && str8_match(entry->data, t_make_entry_obj(arena), 0)); +} diff --git a/src/torture/tests/torture_script_tests.c b/src/torture/tests/torture_script_tests.c new file mode 100644 index 000000000..32a85c2a1 --- /dev/null +++ b/src/torture/tests/torture_script_tests.c @@ -0,0 +1,18 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +internal void +t_script_test_log_result(Arena *arena, TestCtx *ctx, T_Result result) +{ + for EachNode(diagnostic, T_Diagnostic, result.diagnostics.first) { + String8 operation = diagnostic->operation.size == 0 ? str8_zero() : str8f(arena, " [%S]", diagnostic->operation); + test_outf("%S:%lld:%lld%S: %S\n", diagnostic->file_path, diagnostic->location.line, diagnostic->location.column, operation, diagnostic->message); + } +} + +TEST(script_path_identity) +{ + T_Ok(str8_match(test_layer_from_file_path(str8_lit("C:/repo/src/linker/tests/foo.tst")), str8_lit("linker"), 0)); + T_Ok(str8_match(test_layer_from_file_path(str8_lit("C:\\repo\\src\\torture\\tests\\foo.c")), str8_lit("torture"), 0)); + T_Ok(test_layer_from_file_path(str8_lit("C:/repo/tests/foo.tst")).size == 0); +} diff --git a/src/torture/tests/torture_subprocess_tests.c b/src/torture/tests/torture_subprocess_tests.c new file mode 100644 index 000000000..329a3eaa3 --- /dev/null +++ b/src/torture/tests/torture_subprocess_tests.c @@ -0,0 +1,147 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#include + +internal void +t_controller_fixture(void) +{ + fprintf(stdout, "stdout:ready\n"); + fprintf(stderr, "stderr:ready\n"); + fflush(stdout); + fflush(stderr); + + char line[4096]; + while (fgets(line, sizeof(line), stdin) != 0) { + U64 size = strlen(line); + while (size != 0 && (line[size - 1] == '\r' || line[size - 1] == '\n')) { line[--size] = 0; } + if (strcmp(line, "exit") == 0) { break; } + fprintf(stdout, "stdout:%s\n", line); + fprintf(stderr, "stderr:%s\n", line); + fflush(stdout); + fflush(stderr); + } +} + +global volatile sig_atomic_t t_controller_fixture_interrupted; + +internal void +t_controller_fixture_signal_handler(int signal) +{ + t_controller_fixture_interrupted = signal == SIGINT; +} + +internal void +t_controller_interrupt_fixture(void) +{ + signal(SIGINT, t_controller_fixture_signal_handler); + fprintf(stdout, "interrupt:ready\n"); + fflush(stdout); + while (!t_controller_fixture_interrupted) { sleep_ms(1); } + fprintf(stdout, "interrupt:received\n"); + fflush(stdout); +} + +internal void +t_run_operation_fixture(B32 sleep, U64 exit_code) +{ + fprintf(stdout, "GenericRunStdout\n"); + fprintf(stderr, "GenericRunStderr\n"); + fflush(stdout); + fflush(stderr); + if (sleep) { sleep_ms(30000); } + exit((int)exit_code); +} + +internal T_ControllerResult +t_controller_test_launch(Arena *arena, T_Controller *controller, B32 merge_outputs) +{ + String8List cmd_line = {0}; + str8_list_push(arena, &cmd_line, get_process_info()->binary_file_path); + str8_list_push(arena, &cmd_line, str8_lit("-controller_fixture")); + T_ControllerLaunchParams params = { + .cmd_line = cmd_line, + .path = get_process_info()->binary_path, + .inherit_env = 1, + .consoleless = 1, + .merge_outputs = merge_outputs, + }; + return t_controller_launch(arena, controller, ¶ms); +} + +TEST(controller_interactive_separate_output) +{ + T_Controller controller = {0}; + B32 is_ok = 0; + + if (t_controller_test_launch(arena, &controller, 0).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("stdout:ready"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stderr, str8_lit("stderr:ready"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_wait_until_quiet(&controller, TIMEOUT_MS(10), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_send_line(&controller, str8_lit("hello")).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("stdout:hello"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stderr, str8_lit("stderr:hello"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("missing"), TIMEOUT_MS(10)).code != T_ControllerResultCode_Timeout) { goto exit; } + if (t_controller_send_line(&controller, str8_lit("exit")).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_wait(&controller, TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok || controller.exit_code != 0) { goto exit; } + is_ok = 1; + +exit:; + t_controller_close(&controller); + if (!is_ok) { + test_outf("stdout:\n%S\n", t_controller_stdout(arena, &controller)); + test_outf("stderr:\n%S\n", t_controller_stderr(arena, &controller)); + } + T_Ok(is_ok); +} + +TEST(controller_interactive_merged_output) +{ + T_Controller controller = {0}; + B32 is_ok = 0; + + if (t_controller_test_launch(arena, &controller, 1).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("stdout:ready"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stderr, str8_lit("stderr:ready"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_send_line(&controller, str8_lit("merged")).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("stdout:merged"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_send_line(&controller, str8_lit("exit")).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_wait(&controller, TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok || controller.exit_code != 0) { goto exit; } + { + String8 output = t_controller_stdout(arena, &controller); + is_ok = str8_find_needle(output, 0, str8_lit("stderr:ready"), 0) < output.size && str8_find_needle(output, 0, str8_lit("stderr:merged"), 0) < output.size; + } + +exit:; + t_controller_close(&controller); + if (!is_ok) { test_outf("merged output:\n%S\n", t_controller_stdout(arena, &controller)); } + T_Ok(is_ok); +} + +TEST(controller_interrupt) +{ + T_Controller controller = {0}; + B32 is_ok = 0; + String8List cmd_line = {0}; + str8_list_push(arena, &cmd_line, get_process_info()->binary_file_path); + str8_list_push(arena, &cmd_line, str8_lit("-controller_interrupt_fixture")); + T_ControllerLaunchParams params = { + .cmd_line = cmd_line, + .path = get_process_info()->binary_path, + .inherit_env = 1, + .new_console = 1, + .merge_outputs = 1, + }; + + if (t_controller_launch(arena, &controller, ¶ms).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("interrupt:ready"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_interrupt(&controller).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_expect(&controller, T_ControllerStream_Stdout, str8_lit("interrupt:received"), TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok) { goto exit; } + if (t_controller_wait(&controller, TIMEOUT_SEC(5)).code != T_ControllerResultCode_Ok || controller.exit_code != 0) { goto exit; } + is_ok = 1; + +exit:; + t_controller_close(&controller); + if (!is_ok) { test_outf("interrupt output:\n%S\n", t_controller_stdout(arena, &controller)); } + T_Ok(is_ok); +} diff --git a/src/torture/torture.c b/src/torture/torture.c index b85f26601..9a26ab0fa 100644 --- a/src/torture/torture.c +++ b/src/torture/torture.c @@ -24,7 +24,6 @@ global String8 g_errors; // tools global B32 g_gui; -global String8 g_radbin_path; global String8 g_cl_path; global String8 g_clang_path; global String8 g_gcc_path; @@ -54,18 +53,6 @@ t_test_layer_from_name(Arena *arena, String8 pattern) return matches; } -//////////////////////////////// - -internal Linker -t_id_linker(void) -{ - String8 name = str8_chop_last_dot(str8_skip_last_slash(g_linker_path)); - if (str8_match(name, str8_lit("radlink"), StringMatchFlag_CaseInsensitive)) { return Linker_radlink; } - if (str8_match(name, str8_lit("link"), StringMatchFlag_CaseInsensitive)) { return Linker_msvc; } - if (str8_match(name, str8_lit("lld-link"), StringMatchFlag_CaseInsensitive)) { return Linker_lld; } - return Linker_Null; -} - internal B32 t_write_file_list(String8 name, String8List data) { @@ -162,69 +149,68 @@ t_run_caller(void *raw_ctx) g_is_first_print = 1; - T_RunCtx *ctx = raw_ctx; - ctx->result.status = TestStatus_Pass; + T_RunCtx *run_ctx = raw_ctx; + run_ctx->result.status = TestStatus_Pass; String8List test_out = {0}; - if (ctx->test->skip) { - ctx->result.status = TestStatus_Skip; + if (run_ctx->test->skip) { + run_ctx->result.status = TestStatus_Skip; } else { - TestCtx test_ctx = - { - .cmdline = ctx->cmdline, - .exemplars_path = g_exemplar_dir, - .artifacts_path = g_wdir, + + // run test + TestCtx test_ctx = { + .cmdline = run_ctx->cmdline, + .exemplars_path = g_exemplar_dir, + .artifacts_path = g_wdir, .input_data_path = g_input_data_dir, - .result_out = &ctx->result, - .test_out = &test_out, + .user_data = run_ctx->user_data, + .result_out = &run_ctx->result, + .test_out = &test_out, }; log_scope_begin(); - ctx->test->test_fn(scratch.arena, &test_ctx); + run_ctx->test->test_fn(scratch.arena, &test_ctx); LogScopeResult log_scope_result = log_scope_end(scratch.arena); - if(log_scope_result.strings[LogMsgKind_Info].size != 0) - { - String8 current_path = str8f(scratch.arena, "%S/current", g_wdir); - String8 exemplar_path = str8f(scratch.arena, "%S/exemplar_%S_%S", g_exemplar_dir, lower_from_str8(scratch.arena, string_from_operating_system(OperatingSystem_CURRENT)), lower_from_str8(scratch.arena, string_from_arch(Arch_CURRENT))); - String8 current = log_scope_result.strings[LogMsgKind_Info]; - String8 exemplar = data_from_file_path(scratch.arena, exemplar_path); + + // epilog for the diff runs + if (log_scope_result.strings[LogMsgKind_Info].size) { + String8 current_path = str8f(scratch.arena, "%S/current", g_wdir); + String8 exemplar_path = str8f(scratch.arena, "%S/exemplar_%S_%S", + g_exemplar_dir, + lower_from_str8(scratch.arena, string_from_operating_system(OperatingSystem_CURRENT)), + lower_from_str8(scratch.arena, string_from_arch(Arch_CURRENT))); + String8 current = log_scope_result.strings[LogMsgKind_Info]; + String8 exemplar = data_from_file_path(scratch.arena, exemplar_path); write_data_to_file_path(current_path, current); - if(exemplar.size == 0) - { + + if (exemplar.size == 0) { make_directory(g_exemplar_dir); copy_file_path(exemplar_path, current_path); - } - else - { - // TODO(rjf): @hack, see below - TestCtx *ctx = &test_ctx; - String8List exemplar_lines = str8_split(scratch.arena, exemplar, (U8 *)"\n", 1, StringSplitFlag_KeepEmpties); + } else { + String8List exemplar_lines = str8_split(scratch.arena, exemplar, (U8 *)"\n", 1, StringSplitFlag_KeepEmpties); String8List exemplar_lines_sanitized = {0}; - for EachNode(n, String8Node, exemplar_lines.first) - { + for EachNode(n, String8Node, exemplar_lines.first) { String8 line_trimmed = n->string; - if(line_trimmed.size != 0 && line_trimmed.str[line_trimmed.size-1] == '\r') - { + if(line_trimmed.size != 0 && line_trimmed.str[line_trimmed.size-1] == '\r') { line_trimmed.size -= 1; } str8_list_push(scratch.arena, &exemplar_lines_sanitized, line_trimmed); } - StringJoin join = {.sep = s("\n"), .post = s("\n")}; - String8 exemplar_sanitized = str8_list_join(scratch.arena, &exemplar_lines_sanitized, &join); - B32 current_matches_exemplar = str8_match(exemplar_sanitized, current, 0); - if(!current_matches_exemplar) - { + + String8 exemplar_sanitized = str8_list_join(scratch.arena, &exemplar_lines_sanitized, &(StringJoin){.sep = s("\n"), .post = s("\n")}); + B32 current_matches_exemplar = str8_match(exemplar_sanitized, current, 0); + if(!current_matches_exemplar) { // TODO(rjf): @hack, because we need to do test things in the outer scope, but this is // not a test function - all test helpers should wrap a 100% parameterized layer - Arena *arena = scratch.arena; String8 diff_cmd = str8f(scratch.arena, "diff %S %S", path_normalized_from_string(scratch.arena, exemplar_path), path_normalized_from_string(scratch.arena, current_path)); - test_outf("Current log does not match exemplar; run `%S`\n", diff_cmd); + str8_list_pushf(scratch.arena, &test_out, "Current log does not match exemplar; run `%S`\n", diff_cmd); + // TODO(rjf): @hack need to hack this in, because TestCheck assumes `return` - ctx->result_out[0] = (TestResult){.fail_file = __FILE__, .fail_line = __LINE__, .fail_cond = "current_matches_exemplar"}; - if(debugger_is_attached()) - { + test_ctx.result_out[0] = (TestResult){.fail_file = __FILE__, .fail_line = __LINE__, .fail_cond = "current_matches_exemplar"}; + + if(debugger_is_attached()) { Trap(); } } @@ -232,16 +218,10 @@ t_run_caller(void *raw_ctx) } } - if (ctx->result.status == TestStatus_Fail || ctx->result.status == TestStatus_Crash) { - for EachNode(n, String8Node, test_out.first) { - t_errorf("%S", n->string); - } - if (g_errors.size) { - t_errorf("%S\n", g_errors); - } - if (g_output.size) { - t_errorf("%S\n", g_output); - } + if (run_ctx->result.status == TestStatus_Fail || run_ctx->result.status == TestStatus_Crash) { + for EachNode(n, String8Node, test_out.first) { t_errorf("%S", n->string); } + if (g_errors.size) { t_errorf("%S\n", g_errors); } + if (g_output.size) { t_errorf("%S\n", g_output); } } scratch_end(scratch); @@ -267,22 +247,6 @@ t_run(CmdLine *cmdline, TestInfo *test, String8 user_data) return ctx.result; } -internal String8 -t_radbin_path(void) -{ - if (g_radbin_path.size == 0) { - local_persist U8 buffer[4096]; - Arena *arena = arena_alloc_(&(ArenaParams){ .reserve_size = sizeof(buffer), .commit_size = sizeof(buffer), .optional_backing_buffer = buffer }); -#if OS_WINDOWS - g_radbin_path = full_path_from_path(arena, str8_lit("radbin.exe")); -#else - g_radbin_path = full_path_from_path(arena, str8_lit("radbin")); -#endif - } - AssertAlways(g_radbin_path.size); - return g_radbin_path; -} - internal String8 t_cl_path(void) { @@ -885,80 +849,19 @@ t_invoke_linkerf(char *fmt, ...) return is_ok; } -internal B32 -t_invoke_radbin(char *fmt, ...) -{ - Temp scratch = scratch_begin(0,0); - va_list args; - va_start(args, fmt); - String8 cmdl = push_str8fv(scratch.arena, fmt, args); - va_end(args); - B32 is_ok = t_invoke(t_radbin_path(), cmdl, max_U64); - scratch_end(scratch); - return is_ok; -} - -internal void -t_kill_all(String8 pattern) -{ - Temp scratch = scratch_begin(0,0); - DMN_ProcessIter it = {0}; - dmn_process_iter_begin(&it); - DMN_ProcessInfo info = {0}; - while (dmn_process_iter_next(scratch.arena, &it, &info)) { - if (str8_match_wildcard(info.name, pattern, StringMatchFlag_CaseInsensitive|StringMatchFlag_SlashInsensitive)) { -#if OS_WINDOWS - if (!t_invoke(str8_lit("taskkill"), str8f(scratch.arena, "/PID %u /F", info.pid), max_U64)) { fprintf(stderr, "ERROR: failed to invoke taskkill\n"); } -#elif OS_LINUX - NotImplemented; // TODO: test - if (!t_invoke(str8_lit("kill"), str8f(scratch.arena, " -9 %u", info.pid), max_U64)) { fprintf(stderr, "ERROR: failed to invoke kill\n"); } -#else -# error NotImplemented -#endif - if (g_last_exit_code != 0) { fprintf(stderr, "ERROR: failed to kill %u\n", info.pid); } - } - } - dmn_process_iter_end(&it); - scratch_end(scratch); -} - -// TODO: obsolete -internal String8 -t_chop_line(String8 *string) -{ - return str8_chop_line(string); -} -// TODO: obsolete -internal B32 -t_match_line(String8 *output, String8 expected_line) -{ - return str8_match_wildcard(t_chop_line(output), expected_line, 0); -} - -internal B32 -t_match_linef(String8 *output, char *fmt, ...) -{ - Temp scratch = scratch_begin(0, 0); - va_list args; - va_start(args, fmt); - String8 expected_line = push_str8fv(scratch.arena, fmt, args); - B32 is_match = t_match_line(output, expected_line); - va_end(args); - scratch_end(scratch); - return is_match; -} - -force_inline int +internal inline int t_test_info_is_before(TestInfo **a, TestInfo **b) { String8 layer_a = a[0]->layer; String8 layer_b = b[0]->layer; int cmp = str8_compar(layer_a, layer_b, 0); - if(cmp == 0) - { - cmp = u64_compar(&a[0]->decl_line, &b[0]->decl_line); + if (cmp == 0) { + cmp = str8_compar(a[0]->label, b[0]->label, 0); + if (cmp == 0) { + cmp = u64_compar(&a[0]->decl_line, &b[0]->decl_line); + } } - return cmp; + return cmp < 0; } internal String8List @@ -1073,6 +976,85 @@ t_errorf(char *fmt, ...) scratch_end(scratch); } +//////////////////////////////// + +internal void +t_run_script_test(Arena *arena, TestCtx *ctx) +{ + T_Result run_status = {0}; + + // read script file + String8 path = ctx->user_data; + String8 source = data_from_file_path(arena, path); + if (source.size == 0) { test_outf("failed to read script: '%S'\n", path); } + T_Ok(source.size != 0); + + // parse the script + T_Context script = {0}; + run_status = t_script_parse(arena, ctx, &t_codec_script_suite, path, source, &script); + if (!t_result_is_ok(run_status)) { goto exit; } + + // execute the script if parsed + run_status = t_script_execute(&script); + if (!t_result_is_ok(run_status)) { goto exit; } + + // report run diagnostics + exit:; + for EachNode(diagnostic, T_Diagnostic, run_status.diagnostics.first) { + test_outf("%S:%lld:%lld [%S]: %S\n", + diagnostic->file_path, + diagnostic->location.line, + diagnostic->location.column, + diagnostic->operation, + diagnostic->message); + } + + T_Ok(t_result_is_ok(run_status)); +} + +internal void +t_register_scripts(void) +{ + local_persist B32 registered; + if (registered) { return; } + registered = 1; + + Temp scratch = scratch_begin(0,0); + Arena *arena = arena_alloc(.name="Torture Scripts Arena"); + + FileIter *iter = file_iter_begin(scratch.arena, t_src_path(), FileIterFlag_SkipFiles | FileIterFlag_SkipHiddenFiles); + for (FileInfo file = {0}; file_iter_next(scratch.arena, iter, &file); ) { + // skip third party folder because other projects can freely reserve the test extension + if (str8_matchi(file.name, str8_lit("third_party"))) { continue; } + + String8 scripts_dir = str8f(arena, "%S/%S/tests", t_src_path(), file.name); + String8List files = t_file_paths_from_dir(arena, scripts_dir); + String8Array paths = str8_array_from_list(arena, &files); + t_sort_str8_array(paths); + + for EachIndex(i, paths.count) { + if ( ! str8_matchi(str8_skip_last_dot(paths.v[i]), str8_lit("tst"))) { continue; } + + String8 layer = test_layer_from_file_path(paths.v[i]); + if (layer.size == 0) { + layer = str8_lit(""); + } + + TestInfo *test = &test_infos[test_infos_count++]; + test->layer = layer; + test->label = str8_chop_last_dot(str8_skip_last_slash(paths.v[i])); + test->decl_line = (S64)i; + test->test_fn = t_run_script_test; + test->user_data = paths.v[i]; + } + } + file_iter_end(iter); + + scratch_end(scratch); +} + +//////////////////////////////// + internal void t_help(void) { @@ -1104,8 +1086,10 @@ internal void t_entry_point(CmdLine *cmdline) { Temp scratch = scratch_begin(0,0); + Log *log = log_alloc(); log_select(log); + U64 exit_code = max_U64; U64 dashes_size = 9999; @@ -1125,12 +1109,15 @@ t_entry_point(CmdLine *cmdline) // { B32 print_help = cmd_line_has_flag(cmdline, str8_lit("help")) || - cmd_line_has_flag(cmdline, str8_lit("h")); + cmd_line_has_flag(cmdline, str8_lit("h")); if (print_help) { t_help(); goto exit; } } + + // Register Test Scripts + t_register_scripts(); // Gather tests { @@ -1282,14 +1269,28 @@ t_entry_point(CmdLine *cmdline) // // Run tests // - { - U64Array target_indices = u64_array_from_list(scratch.arena, &targets); + { + // sort selected tests + U64Array target_indices = {0}; + { + U64 count = 0; + TestInfo **v = push_array(scratch.arena, TestInfo*, targets.count); + for EachNode(n, U64Node, targets.first) { + v[count++] = g_sorted_test_infos[n->data]; + } + radsort(v, count, t_test_info_is_before); + + target_indices.v = push_array(scratch.arena, U64, count); + for EachIndex(i, count) { + target_indices.v[target_indices.count++] = IntFromPtr(v[i] - &test_infos[0]); + } + } U64 max_label_size = 0; U64 max_layer_size = 0; for EachIndex(i, target_indices.count) { - U64 test_idx = target_indices.v[i]; - TestInfo *test_info = g_sorted_test_infos[test_idx]; + U64 test_idx = target_indices.v[i]; + TestInfo *test_info = &test_infos[test_idx]; max_label_size = Max(max_label_size, test_info->label.size); max_layer_size = Max(max_layer_size, test_info->layer.size); } @@ -1308,7 +1309,7 @@ t_entry_point(CmdLine *cmdline) if (i == 0) { PrintHeader("Tests"); } U64 target_idx = target_indices.v[i]; - TestInfo *test = g_sorted_test_infos[target_idx]; + TestInfo *test = &test_infos[target_idx]; // print run progress U64 dots_min = 10; @@ -1351,7 +1352,7 @@ t_entry_point(CmdLine *cmdline) // run test U64 run_start_time = now_time_us(); - TestResult result = t_run(cmdline, test, str8_zero()); + TestResult result = t_run(cmdline, test, test->user_data); U64 run_end_time = now_time_us(); // update @@ -1436,8 +1437,8 @@ t_entry_point(CmdLine *cmdline) U64 layer_max = 0; for EachIndex(i, slow_count) { Slowest s = slowest[i]; - label_max = Max(g_sorted_test_infos[s.target_idx]->label.size, label_max); - layer_max = Max(g_sorted_test_infos[s.target_idx]->layer.size, layer_max); + label_max = Max(test_infos[s.target_idx].label.size, label_max); + layer_max = Max(test_infos[s.target_idx].layer.size, layer_max); } fprintf(stderr, " \nSlow Tests\n"); @@ -1445,7 +1446,7 @@ t_entry_point(CmdLine *cmdline) Slowest s = slowest[i]; if (s.target_idx >= test_infos_count) { break; } - TestInfo *test_info = g_sorted_test_infos[s.target_idx]; + TestInfo *test_info = &test_infos[s.target_idx]; String8 elapsed_time = string_from_elapsed_time(scratch.arena, date_time_from_micro_seconds(s.d)); fprintf(stderr, " %.*s %.*s/ %.*s %.*s %.*s\n", diff --git a/src/torture/torture.h b/src/torture/torture.h index 56578809d..eb5685061 100644 --- a/src/torture/torture.h +++ b/src/torture/torture.h @@ -46,7 +46,6 @@ internal void t_run_fail_handler(void *raw_ctx); internal TestResult t_run(CmdLine *cmdline, TestInfo *test, String8 user_data); // tools -internal String8 t_radbin_path(void); internal String8 t_cl_path(void); internal String8 t_clang_path(void); internal String8 t_gcc_path(void); @@ -58,17 +57,10 @@ internal String8 t_src_path(void); internal B32 t_invoke(String8 exe, String8 cmdline, U64 timeout); internal B32 t_invoke_cl(char *fmt, ...); internal B32 t_invoke_linkerf(char *fmt, ...); -internal B32 t_invoke_radbin(char *fmt, ...); -internal void t_kill_all(String8 pattern); #define t_invoke_linker_timeout(c, t) T_Ok(t_invoke(t_radlink_path(), c, t)) #define t_invoke_linker_timeoutf(t, f, ...) t_invoke_linker_timeout(push_str8f(arena, f, ##__VA_ARGS__), t) #define t_invoke_linker(c) t_invoke_linker_timeout(c, max_U64) -internal String8 t_chop_line(String8 *string); -internal B32 t_match_line(String8 *output, String8 expected_line); -internal B32 t_match_linef(String8 *output, char *fmt, ...); -#define T_MatchLinef(out, ...) T_Ok(t_match_linef(out, __VA_ARGS__)) - // files helper internal String8List t_file_paths_from_dir(Arena *arena, String8 dir); diff --git a/src/torture/torture_base.c b/src/torture/torture_base.c new file mode 100644 index 000000000..f848d9e07 --- /dev/null +++ b/src/torture/torture_base.c @@ -0,0 +1,107 @@ +//////////////////////////////// +// Generic Parser Helpers + +internal B32 +t_md_node_is_valid(MD_Node *node) +{ + return !md_node_is_nil(node); +} + +internal B32 +t_bool_is_string_true(String8 string) +{ + return str8_matchi(string, str8_lit("true")) || str8_matchi(string, str8_lit("1")); +} + +internal B32 +t_bool_is_string_false(String8 string) +{ + return str8_matchi(string, str8_lit("false")) || str8_matchi(string, str8_lit("0")); +} + +internal B32 +t_bool_from_string(String8 string) +{ + if (str8_matchi(string, str8_lit("false"))) { + return 0; + } + return (B32)u32_from_str8(string, 10); +} + +internal String8 +t_hex_from_data(Arena *arena, String8 data) +{ + U64 hex_size = data.size * 2; + U8 *hex = push_array_no_zero(arena, U8, hex_size); + for EachIndex(i, data.size) { + local_persist U8 digits[] = "0123456789abcdef"; + hex[i * 2 + 0] = digits[data.str[i] >> 4]; + hex[i * 2 + 1] = digits[data.str[i] & 15]; + } + return str8(hex, hex_size); +} + +//////////////////////////////// +// Codec Helpers + +internal MD_Node * +t_codec_child(MD_Node *node, char *name) +{ + return md_child_from_string(node, str8_cstring(name), StringMatchFlag_CaseInsensitive); +} + +internal String8 +t_codec_scalar(MD_Node *node) +{ + return md_node_is_nil(node) || md_node_is_nil(node->first) ? str8_zero() : node->first->string; +} + +internal MD_Node * +t_codec_push_node(Arena *arena, MD_Node *parent, String8 string) +{ + MD_Node *node = md_push_node(arena, MD_NodeKind_Main, MD_NodeFlag_Identifier, string, string, 0); + if (parent != 0) { md_node_push_child(parent, node); } + return node; +} + +internal MD_Node * +t_codec_push_field(T_Context *ctx, MD_Node *parent, char *name, String8 value) +{ + MD_Node *field = t_codec_push_node(ctx->arena, parent, str8_cstring(name)); + t_codec_push_node(ctx->arena, field, str8_copy(ctx->arena, value)); + return field; +} + +internal MD_Node * +t_codec_push_u64(T_Context *ctx, MD_Node *parent, char *name, U64 value) +{ + return t_codec_push_field(ctx, parent, name, str8f(ctx->arena, "%llu", value)); +} + +internal MD_Node * +t_codec_push_u64_s8(T_Context *ctx, MD_Node *parent, String8 name, U64 value) +{ + MD_Node *field = t_codec_push_node(ctx->arena, parent, name); + t_codec_push_node(ctx->arena, field, str8f(ctx->arena, "%llu", value)); + return field; +} + +internal B32 +t_bool_from_scalar(MD_Node *node, B32 *value_out) +{ + String8 value = t_codec_scalar(node); + + if (t_bool_is_string_true(value)) { + *value_out = 1; + return 1; + } + + if (t_bool_is_string_false(value)) { + *value_out = 0; + return 1; + } + + return 0; +} + + diff --git a/src/torture/torture_build.c b/src/torture/torture_build.c new file mode 100644 index 000000000..674fd3a60 --- /dev/null +++ b/src/torture/torture_build.c @@ -0,0 +1,757 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +global T_OpSpec t_op_repeat = {str8_lit_comp("repeat"), t_op_repeat_validate, t_op_repeat_execute}; +global T_OpSpec t_op_compare_file = {str8_lit_comp("compare_file"), t_op_compare_file_validate, t_op_compare_file_execute}; + +internal B32 +t_build_string_is_bool(String8 string) +{ + return t_bool_from_string(string) || str8_matchi(string, str8_lit("false")) || str8_match(string, str8_lit("0"), 0); +} + +internal B32 +t_build_safe_relative_path(String8 path) +{ + if (path.size == 0 || path.str[0] == '/' || path.str[0] == '\\' || str8_find_needle(path, 0, str8_lit(":"), 0) < path.size) { return 0; } + for (U64 off = 0; off < path.size;) { + U64 end = off; + while (end < path.size && path.str[end] != '/' && path.str[end] != '\\') { end += 1; } + String8 part = str8_substr(path, rng_1u64(off, end)); + if (str8_match(part, str8_lit(".."), 0)) { return 0; } + off = end + 1; + } + return 1; +} + +internal T_BuildCommandKind +t_build_command_kind_from_string(String8 string) +{ + if (str8_matchi(string, str8_lit("compile"))) { return T_BuildCommandKind_Compile; } + if (str8_matchi(string, str8_lit("compile_link"))) { return T_BuildCommandKind_CompileLink; } + if (str8_matchi(string, str8_lit("link"))) { return T_BuildCommandKind_Link; } + if (str8_matchi(string, str8_lit("resource"))) { return T_BuildCommandKind_Resource; } + if (str8_matchi(string, str8_lit("run"))) { return T_BuildCommandKind_Run; } + if (str8_matchi(string, str8_lit("copy"))) { return T_BuildCommandKind_Copy; } + return T_BuildCommandKind_Null; +} + +internal String8 +t_build_default_output(Arena *arena, String8 target_name, String8 target_kind, OperatingSystem os) +{ + if (target_name.size == 0) { target_name = str8_lit("main"); } + if (str8_matchi(target_kind, str8_lit("shared_library"))) { + return os == OperatingSystem_Windows ? str8f(arena, "%S.dll", target_name) : str8f(arena, "lib%S.so", target_name); + } + return os == OperatingSystem_Windows ? str8f(arena, "%S.exe", target_name) : str8_copy(arena, target_name); +} + +internal void +t_build_declaration_list_parse(Arena *arena, MD_Node *container, T_BuildDeclarationList *list) +{ + for MD_EachNode(node, container->first) { + T_BuildDeclaration *declaration = push_array(arena, T_BuildDeclaration, 1); + declaration->name = node->string; + declaration->definition = node; + SLLQueuePush(list->first, list->last, declaration); + list->count += 1; + } +} + +internal T_BuildOutput * +t_build_output_push(Arena *arena, T_BuildVariant *variant, String8 name, String8 path, MD_Node *definition) +{ + T_BuildOutput *output = push_array(arena, T_BuildOutput, 1); + output->name = str8_copy(arena, name); + output->path = str8_copy(arena, path); + output->definition = definition; + SLLQueuePush(variant->first_output, variant->last_output, output); + variant->output_count += 1; + return output; +} + +internal T_BuildOutput * +t_build_primary_output(T_BuildVariant *variant) +{ + for (T_BuildOutput *output = variant->first_output; output != 0; output = output->next) { + if (str8_matchi(output->name, str8_lit("executable"))) { return output; } + } + return variant->first_output; +} + +internal String8 +t_build_command_scalar(MD_Node *command) +{ + if (md_node_is_nil(command->first) || !md_node_is_nil(command->first->first)) { return str8_zero(); } + return command->first->string; +} + +internal T_BuildCommand * +t_build_command_parse(T_ParseContext *ctx, MD_Node *node) +{ + T_BuildCommandKind kind = t_build_command_kind_from_string(node->string); + if (kind == T_BuildCommandKind_Null) { return 0; } + + T_BuildCommand *command = push_array(ctx->arena, T_BuildCommand, 1); + command->kind = kind; + command->definition = node; + command->repeat_count = 1; + command->timeout_ms = max_U64; + command->expected_exit = str8_lit("0"); + command->output_mode = T_BuildOutputMode_Default; + + String8 scalar = t_build_command_scalar(node); + if (scalar.size != 0) { + command->arguments = scalar; + command->inject_output = 1; + return command; + } + + char *allowed[] = {"args", "tool", "output", "artifact", "expect_exit", "when_previous_exit", "timeout_ms", "stdout_matches", "stderr_matches", "index", "repeat", "parallel", "produces", 0}; + for MD_EachNode(field, node->first) { + B32 is_allowed = 0; + for (U64 i = 0; allowed[i] != 0; i += 1) { is_allowed |= str8_matchi(field->string, str8_cstring(allowed[i])); } + if (!is_allowed) { t_parse_errorf(ctx, T_ResultCode_ValidationError, field, "unknown build command field '%S'", field->string); } + for (MD_Node *other = field->next; !md_node_is_nil(other); other = other->next) { + if (str8_matchi(field->string, other->string)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, other, "duplicate build command field '%S'", other->string); + break; + } + } + } + + MD_Node *args = t_child_from_string(node, "args"); + command->arguments = t_scalar_string_from_node(args); + command->tool = t_scalar_string_from_node(t_child_from_string(node, "tool")); + MD_Node *output = t_child_from_string(node, "output"); + if (!md_node_is_nil(output)) { + command->output = t_scalar_string_from_node(output); + command->output_mode = str8_matchi(command->output, str8_lit("none")) ? T_BuildOutputMode_None : T_BuildOutputMode_Explicit; + command->inject_output = command->output_mode != T_BuildOutputMode_None; + } + command->artifact = t_scalar_string_from_node(t_child_from_string(node, "artifact")); + if (command->artifact.size != 0 && command->output_mode != T_BuildOutputMode_Default) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "build command cannot contain both artifact and output"); + } + command->expected_exit = t_scalar_string_from_node(t_child_from_string(node, "expect_exit")); + if (command->expected_exit.size == 0) { command->expected_exit = str8_lit("0"); } + command->when_previous_exit = t_scalar_string_from_node(t_child_from_string(node, "when_previous_exit")); + command->stdout_pattern = t_scalar_string_from_node(t_child_from_string(node, "stdout_matches")); + command->stderr_pattern = t_scalar_string_from_node(t_child_from_string(node, "stderr_matches")); + command->index_name = t_scalar_string_from_node(t_child_from_string(node, "index")); + MD_Node *parallel = t_child_from_string(node, "parallel"); + String8 parallel_string = t_scalar_string_from_node(parallel); + if (!md_node_is_nil(parallel) && !t_build_string_is_bool(parallel_string)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, parallel, "build command parallel must be true or false"); + } + command->parallel = t_bool_from_string(parallel_string); + command->produces = t_child_from_string(node, "produces"); + + MD_Node *repeat = t_child_from_string(node, "repeat"); + if (!md_node_is_nil(repeat) && !try_u64_from_str8_c_rules(t_scalar_string_from_node(repeat), &command->repeat_count)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, repeat, "build command repeat must be an integer"); + } + MD_Node *timeout = t_child_from_string(node, "timeout_ms"); + if (!md_node_is_nil(timeout) && !try_u64_from_str8_c_rules(t_scalar_string_from_node(timeout), &command->timeout_ms)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, timeout, "build command timeout_ms must be an integer"); + } + if (command->arguments.size == 0 && kind != T_BuildCommandKind_Copy) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "build command '%S' requires args", node->string); + } + U64 expected_exit_code = 0; + if (!str8_matchi(command->expected_exit, str8_lit("any")) && !str8_matchi(command->expected_exit, str8_lit("nonzero")) && + !try_u64_from_str8_c_rules(command->expected_exit, &expected_exit_code)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_child_from_string(node, "expect_exit"), "invalid expected build command exit code"); + } + if (command->when_previous_exit.size != 0 && !str8_matchi(command->when_previous_exit, str8_lit("any")) && + !str8_matchi(command->when_previous_exit, str8_lit("nonzero")) && !try_u64_from_str8_c_rules(command->when_previous_exit, &expected_exit_code)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_child_from_string(node, "when_previous_exit"), "invalid previous build command exit code"); + } + if (command->tool.size != 0 && !str8_matchi(command->tool, str8_lit("cl")) && !str8_matchi(command->tool, str8_lit("msvc")) && + !str8_matchi(command->tool, str8_lit("clang")) && !str8_matchi(command->tool, str8_lit("gcc")) && !str8_matchi(command->tool, str8_lit("radlink"))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_child_from_string(node, "tool"), "unknown build command tool '%S'", command->tool); + } + if (command->repeat_count == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, repeat, "build command repeat must be greater than zero"); } + if (command->repeat_count > 1 && command->index_name.size == 0) { command->index_name = str8_lit("index"); } + return command; +} + +internal T_BuildVariant * +t_build_variant_parse(T_ParseContext *ctx, MD_Node *body, OperatingSystem os, String8 target_name, String8 target_kind) +{ + T_BuildVariant *variant = push_array(ctx->arena, T_BuildVariant, 1); + variant->os = os; + variant->definition = body; + + MD_Node *output = t_child_from_string(body, "output"); + MD_Node *outputs = t_child_from_string(body, "outputs"); + if (!md_node_is_nil(output) && !md_node_is_nil(outputs)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, output, "build variant cannot contain both output and outputs"); + } else if (!md_node_is_nil(output)) { + String8 path = t_scalar_string_from_node(output); + if (str8_matchi(path, str8_lit("none"))) { variant->output_none = 1; } + else if (path.size == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, output, "build output requires one path"); } + else { t_build_output_push(ctx->arena, variant, str8_lit("executable"), path, output); } + } else if (!md_node_is_nil(outputs)) { + for MD_EachNode(node, outputs->first) { + String8 path = t_scalar_string_from_node(node); + if (path.size == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "build output '%S' requires one path", node->string); } + else { t_build_output_push(ctx->arena, variant, node->string, path, node); } + } + } else { + OperatingSystem output_os = os == OperatingSystem_Null ? OperatingSystem_CURRENT : os; + t_build_output_push(ctx->arena, variant, str8_lit("executable"), t_build_default_output(ctx->arena, target_name, target_kind, output_os), body); + } + + MD_Node *commands = t_child_from_string(body, "commands"); + MD_Node *first = md_node_is_nil(commands) ? body->first : commands->first; + for MD_EachNode(node, first) { + T_BuildCommand *command = t_build_command_parse(ctx, node); + if (command != 0) { + SLLQueuePush(variant->first_command, variant->last_command, command); + variant->command_count += 1; + } else if (!md_node_is_nil(commands)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "unknown build command '%S'", node->string); + } else { + char *metadata[] = {"output", "outputs", "kind", "source", "sources", "inputs", "depends", "generator", "resources", "packages", "defines", "link_args", "platforms", 0}; + B32 is_metadata = 0; + for (U64 i = 0; metadata[i] != 0; i += 1) { is_metadata |= str8_matchi(node->string, str8_cstring(metadata[i])); } + if (!is_metadata) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "unknown build declaration '%S'", node->string); } + } + } + return variant; +} + +internal void +t_build_target_parse_variants(T_ParseContext *ctx, T_BuildTarget *target, MD_Node *body) +{ + B32 has_platform = 0; + for MD_EachNode(node, body->first) { + OperatingSystem os = operating_system_from_string(node->string); + has_platform |= os == OperatingSystem_Windows || os == OperatingSystem_Linux; + } + + if (has_platform) { + B32 has_windows = 0; + B32 has_linux = 0; + for MD_EachNode(node, body->first) { + OperatingSystem os = operating_system_from_string(node->string); + if (os == OperatingSystem_Windows || os == OperatingSystem_Linux) { + B32 *seen = os == OperatingSystem_Windows ? &has_windows : &has_linux; + if (*seen) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "duplicate build platform '%S'", node->string); } + *seen = 1; + T_BuildVariant *variant = t_build_variant_parse(ctx, node, os, target->name, target->kind); + SLLQueuePush(target->first_variant, target->last_variant, variant); + target->variant_count += 1; + } else { + char *metadata[] = {"kind", "source", "sources", "inputs", "depends", "generator", "resources", "packages", "defines", "link_args", 0}; + B32 is_metadata = 0; + for (U64 i = 0; metadata[i] != 0; i += 1) { is_metadata |= str8_matchi(node->string, str8_cstring(metadata[i])); } + if (!is_metadata) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "unknown build declaration '%S'", node->string); } + } + } + } else { + T_BuildVariant *variant = t_build_variant_parse(ctx, body, OperatingSystem_Null, target->name, target->kind); + SLLQueuePush(target->first_variant, target->last_variant, variant); + target->variant_count += 1; + } +} + +internal T_BuildTarget * +t_build_target_parse(T_ParseContext *ctx, MD_Node *node) +{ + T_BuildTarget *target = push_array(ctx->arena, T_BuildTarget, 1); + target->name = node->string; + target->definition = node; + target->kind = t_scalar_string_from_node(t_child_from_string(node, "kind")); + if (target->kind.size == 0) { target->kind = str8_lit("executable"); } + t_build_target_parse_variants(ctx, target, node); + return target; +} + +internal T_Result +t_build_parse(T_ParseContext *ctx, MD_Node *build, T_BuildPlan **plan_out) +{ + T_BuildPlan *plan = push_array(ctx->arena, T_BuildPlan, 1); + plan->definition = build; + *plan_out = plan; + + MD_Node *targets = t_child_from_string(build, "targets"); + if (md_node_is_nil(targets)) { + + T_BuildTarget *target = push_array(ctx->arena, T_BuildTarget, 1); + target->name = str8_lit("main"); + target->kind = str8_lit("executable"); + target->definition = build; + + t_build_target_parse_variants(ctx, target, build); + + plan->is_single_target = 1; + SLLQueuePush(plan->first_target, plan->last_target, target); + plan->target_count = 1; + + return ctx->run->result; + } + + char *allowed[] = {"defaults", "values", "configurations", "features", "toolchains", "packages", "resources", "generators", "targets", 0}; + for MD_EachNode(node, build->first) { + B32 is_allowed = 0; + for (U64 i = 0; allowed[i] != 0; i += 1) { is_allowed |= str8_matchi(node->string, str8_cstring(allowed[i])); } + if (!is_allowed) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "unknown project build declaration '%S'", node->string); } + } + + plan->defaults = t_child_from_string(build, "defaults"); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "values"), &plan->values); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "configurations"), &plan->configurations); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "features"), &plan->features); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "toolchains"), &plan->toolchains); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "packages"), &plan->packages); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "resources"), &plan->resources); + t_build_declaration_list_parse(ctx->arena, t_child_from_string(build, "generators"), &plan->generators); + for MD_EachNode(node, targets->first) { + T_BuildTarget *target = t_build_target_parse(ctx, node); + SLLQueuePush(plan->first_target, plan->last_target, target); + plan->target_count += 1; + } + if (plan->target_count == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, targets, "project build requires at least one target"); } + return ctx->run->result; +} + +internal T_BuildVariant * +t_build_current_variant(T_BuildTarget *target) +{ + T_BuildVariant *fallback = 0; + for (T_BuildVariant *variant = target->first_variant; variant != 0; variant = variant->next) { + if (variant->os == OperatingSystem_CURRENT) { return variant; } + if (variant->os == OperatingSystem_Null) { fallback = variant; } + } + return fallback; +} + +internal String8 +t_script_expand(T_Context *ctx, MD_Node *node, String8 string, T_BuildVariant *variant, T_ScriptBinding *bindings) +{ + (void)node; + if (variant == 0 && ctx->build != 0 && ctx->build->first_target != 0) { variant = t_build_current_variant(ctx->build->first_target); } + Temp scratch = scratch_begin(&ctx->arena, 1); + HashMap variables = {0}; + for (T_ScriptBinding *binding = bindings != 0 ? bindings : ctx->bindings; binding != 0; binding = binding->next) { + lnk_env_var_push(scratch.arena, &variables, binding->name, binding->value, LNK_EnvVarRule_Current); + } + lnk_env_var_push(scratch.arena, &variables, str8_lit("source_dir"), t_src_path(), LNK_EnvVarRule_Current); + lnk_env_var_push(scratch.arena, &variables, str8_lit("work_dir"), g_wdir, LNK_EnvVarRule_Current); + lnk_env_var_push(scratch.arena, &variables, str8_lit("platform"), lower_from_str8(scratch.arena, string_from_operating_system(OperatingSystem_CURRENT)), LNK_EnvVarRule_Current); + if (variant != 0) { + T_BuildOutput *primary = t_build_primary_output(variant); + if (primary != 0) { + lnk_env_var_push(scratch.arena, &variables, str8_lit("output"), primary->path, LNK_EnvVarRule_Current); + lnk_env_var_push(scratch.arena, &variables, str8_lit("build.output"), primary->path, LNK_EnvVarRule_Current); + } + for (T_BuildOutput *output = variant->first_output; output != 0; output = output->next) { + lnk_env_var_push(scratch.arena, &variables, str8f(scratch.arena, "outputs.%S", output->name), output->path, LNK_EnvVarRule_Current); + lnk_env_var_push(scratch.arena, &variables, str8f(scratch.arena, "build.outputs.%S", output->name), output->path, LNK_EnvVarRule_Current); + } + } + if (ctx->build != 0) { + for (T_BuildDeclaration *value = ctx->build->values.first; value != 0; value = value->next) { + lnk_env_var_push(scratch.arena, &variables, str8f(scratch.arena, "values.%S", value->name), t_scalar_string_from_node(value->definition), LNK_EnvVarRule_Current); + } + } + String8 result = lnk_expand_env_vars_windows(ctx->arena, &variables, string); + scratch_end(scratch); + return result; +} + +internal String8 +t_build_command_output(T_Context *ctx, T_BuildVariant *variant, T_BuildCommand *command, T_ScriptBinding *binding) +{ + if (command->output_mode == T_BuildOutputMode_None) { return str8_zero(); } + if (command->output_mode == T_BuildOutputMode_Explicit) { + return t_script_expand(ctx, command->definition, command->output, variant, binding); + } + if (command->artifact.size != 0) { + T_Artifact *artifact = t_artifact_from_name(ctx, command->artifact); + if (artifact == 0 || artifact->file_name.size == 0) { + t_context_errorf(ctx, T_ResultCode_ValidationError, command->definition, str8_lit("build"), "unknown output artifact '%S'", command->artifact); + return str8_zero(); + } + return artifact->file_name; + } + if (!command->inject_output) { return str8_zero(); } + T_BuildOutput *output = t_build_primary_output(variant); + return output == 0 ? str8_zero() : t_script_expand(ctx, output->definition, output->path, variant, binding); +} + +internal String8 +t_build_command_arguments(T_Context *ctx, T_BuildVariant *variant, T_BuildCommand *command, T_ScriptBinding *binding) +{ + String8 args = t_script_expand(ctx, command->definition, command->arguments, variant, binding); + String8 output = t_build_command_output(ctx, variant, command, binding); + if (output.size == 0 || !command->inject_output) { return args; } + OperatingSystem os = variant->os == OperatingSystem_Null ? OperatingSystem_CURRENT : variant->os; + B32 msvc_style = os == OperatingSystem_Windows && (command->tool.size == 0 || str8_matchi(command->tool, str8_lit("cl")) || str8_matchi(command->tool, str8_lit("msvc"))); + switch (command->kind) { + case T_BuildCommandKind_Compile: + return msvc_style ? str8f(ctx->arena, "%S /c /Fo:\"%S\"", args, output) : str8f(ctx->arena, "%S -c -o \"%S\"", args, output); + case T_BuildCommandKind_CompileLink: + return msvc_style ? str8f(ctx->arena, "%S /Fe:\"%S\"", args, output) : str8f(ctx->arena, "%S -o \"%S\"", args, output); + case T_BuildCommandKind_Link: + return str8f(ctx->arena, "%S /out:\"%S\"", args, output); + default: break; + } + return args; +} + +internal String8 +t_build_command_tool(T_BuildVariant *variant, T_BuildCommand *command) +{ + if (command->tool.size != 0) { + if (str8_matchi(command->tool, str8_lit("cl")) || str8_matchi(command->tool, str8_lit("msvc"))) { return t_cl_path(); } + if (str8_matchi(command->tool, str8_lit("clang"))) { return t_clang_path(); } + if (str8_matchi(command->tool, str8_lit("gcc"))) { return t_gcc_path(); } + if (str8_matchi(command->tool, str8_lit("radlink"))) { return t_radlink_path(); } + return str8_zero(); + } + switch (command->kind) { + case T_BuildCommandKind_Compile: + case T_BuildCommandKind_CompileLink: return OperatingSystem_CURRENT == OperatingSystem_Windows ? t_cl_path() : t_clang_path(); + case T_BuildCommandKind_Link: return t_radlink_path(); + default: break; + } + return str8_zero(); +} + +internal B32 +t_build_exit_matches(String8 expected, U64 actual) +{ + if (str8_matchi(expected, str8_lit("any"))) { return 1; } + if (str8_matchi(expected, str8_lit("nonzero"))) { return actual != 0; } + U64 value = 0; + return try_u64_from_str8_c_rules(expected, &value) && value == actual; +} + +internal T_Result +t_build_load_artifact_output(T_Context *ctx, String8 output_path) +{ + for (T_Artifact *artifact = ctx->first_artifact; artifact != 0; artifact = artifact->next) { + if (artifact->file_name.size != 0 && str8_match(artifact->file_name, output_path, StringMatchFlag_CaseInsensitive | StringMatchFlag_SlashInsensitive)) { + artifact->data = t_read_file(ctx->arena, output_path); + if (artifact->data.size == 0) { + artifact->state = T_ArtifactState_Failed; + return t_context_errorf(ctx, T_ResultCode_IoError, artifact->definition, str8_lit("build"), "unable to read output artifact '%S'", output_path); + } + artifact->state = T_ArtifactState_Materialized; + } + } + return ctx->result; +} + +internal T_Result +t_build_validate_command_result(T_Context *ctx, T_BuildVariant *variant, T_BuildCommand *command, T_ScriptBinding *binding, U64 exit_code, String8 stdout_data, String8 stderr_data) +{ + if (!t_build_exit_matches(command->expected_exit, exit_code)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, command->definition, str8_lit("build"), "build command exited with %llu, expected %S\n%S", exit_code, + command->expected_exit, stderr_data); + } + StringMatchFlags flags = StringMatchFlag_CaseInsensitive | StringMatchFlag_SlashInsensitive; + if (command->stdout_pattern.size != 0 && !str8_match_wildcard(stdout_data, command->stdout_pattern, flags)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, command->definition, str8_lit("build"), "stdout does not match '%S'\n%S", command->stdout_pattern, stdout_data); + } + if (command->stderr_pattern.size != 0 && !str8_match_wildcard(stderr_data, command->stderr_pattern, flags)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, command->definition, str8_lit("build"), "stderr does not match '%S'\n%S", command->stderr_pattern, stderr_data); + } + String8 output = t_build_command_output(ctx, variant, command, binding); + if (output.size != 0 && exit_code == 0) { + String8 full_path = t_make_file_path(ctx->arena, output); + if (!file_path_exists(full_path)) { return t_context_errorf(ctx, T_ResultCode_IoError, command->definition, str8_lit("build"), "expected output '%S' was not produced", output); } + T_Result result = t_build_load_artifact_output(ctx, output); + if (!t_result_is_ok(result)) { return result; } + } + MD_Node *first_produced = command->produces == 0 ? &md_nil_node : command->produces->first; + for MD_EachNode(produced, first_produced) { + String8 path = t_script_expand(ctx, produced, produced->string, variant, binding); + if (!file_path_exists(t_make_file_path(ctx->arena, path))) { + return t_context_errorf(ctx, T_ResultCode_IoError, produced, str8_lit("build"), "expected output '%S' was not produced", path); + } + } + return ctx->result; +} + +internal void +t_build_remove_command_outputs(T_Context *ctx, T_BuildVariant *variant, T_BuildCommand *command, T_ScriptBinding *binding) +{ + String8 output = t_build_command_output(ctx, variant, command, binding); + if (output.size != 0) { + if (!t_build_safe_relative_path(output)) { + t_context_errorf(ctx, T_ResultCode_ValidationError, command->definition, str8_lit("build"), "output path '%S' must be work-relative", output); + return; + } + t_delete_file(output); + } + MD_Node *first_produced = command->produces == 0 ? &md_nil_node : command->produces->first; + for MD_EachNode(produced, first_produced) { + String8 path = t_script_expand(ctx, produced, produced->string, variant, binding); + if (path.size != 0) { + if (!t_build_safe_relative_path(path)) { + t_context_errorf(ctx, T_ResultCode_ValidationError, produced, str8_lit("build"), "produced path '%S' must be work-relative", path); + return; + } + t_delete_file(path); + } + } +} + +internal T_Result +t_build_execute_command(T_Context *ctx, T_BuildVariant *variant, T_BuildCommand *command, U64 index, U64 *exit_code_out) +{ + T_ScriptBinding binding = {0}; + if (command->repeat_count > 1) { + binding.name = command->index_name; + binding.value = str8f(ctx->arena, "%llu", index); + } + t_build_remove_command_outputs(ctx, variant, command, &binding); + if (!t_result_is_ok(ctx->result)) { return ctx->result; } + + String8 tool = t_build_command_tool(variant, command); + if (tool.size == 0) { return t_context_errorf(ctx, T_ResultCode_ValidationError, command->definition, str8_lit("build"), "build command is not executable yet"); } + String8 args = t_build_command_arguments(ctx, variant, command, &binding); + U64 timeout_us = command->timeout_ms == max_U64 || command->timeout_ms > max_U64 / 1000 ? max_U64 : command->timeout_ms * 1000; + if (!t_invoke(tool, args, timeout_us)) { return t_context_errorf(ctx, T_ResultCode_IoError, command->definition, str8_lit("build"), "unable to launch '%S'", tool); } + *exit_code_out = g_last_exit_code; + if (g_last_exit_code == max_U64) { return t_context_errorf(ctx, T_ResultCode_Mismatch, command->definition, str8_lit("build"), "build command did not exit before timeout"); } + return t_build_validate_command_result(ctx, variant, command, &binding, g_last_exit_code, g_output, g_errors); +} + +internal T_Result +t_build_execute_parallel(T_Context *ctx, T_BuildVariant *variant, T_BuildCommand *command, U64 *exit_code_out) +{ + U64 count = command->repeat_count; + T_Controller *controllers = push_array(ctx->arena, T_Controller, count); + T_ScriptBinding *bindings = push_array(ctx->arena, T_ScriptBinding, count); + + B32 launched_all = 1; + for EachIndex(index, count) { + bindings[index].name = command->index_name; + bindings[index].value = str8f(ctx->arena, "%llu", index); + t_build_remove_command_outputs(ctx, variant, command, &bindings[index]); + } + + if (!t_result_is_ok(ctx->result)) { return ctx->result; } + + for EachIndex(index, count) { + String8 tool = t_build_command_tool(variant, command); + String8 args = t_build_command_arguments(ctx, variant, command, &bindings[index]); + String8List cmd_line = lnk_arg_list_parse_windows_rules(ctx->arena, args); + str8_list_push_front(ctx->arena, &cmd_line, tool); + + T_ControllerLaunchParams params = { + .cmd_line = cmd_line, + .path = g_wdir, + .inherit_env = 1, + .consoleless = 1, + }; + + if (t_controller_launch(ctx->arena, &controllers[index], ¶ms).code != T_ControllerResultCode_Ok) { + launched_all = 0; + break; + } + } + + if (!launched_all) { + for EachIndex(index, count) { t_controller_close(&controllers[index]); } + return t_context_errorf(ctx, T_ResultCode_IoError, command->definition, str8_lit("build"), "unable to launch parallel build command"); + } + + U64 timeout_us = command->timeout_ms == max_U64 || command->timeout_ms > max_U64 / 1000 ? max_U64 : command->timeout_ms * 1000; + U64 endt_us = timeout_us == max_U64 ? max_U64 : now_time_us() + timeout_us; + B32 timed_out = 0; + + for (;;) { + + U64 exited_count = 0; + + for EachIndex(index, count) { + T_ControllerResult result = t_controller_pump(&controllers[index]); + if (result.code != T_ControllerResultCode_Ok) { + for EachIndex(close_index, count) { + t_controller_close(&controllers[close_index]); + } + + return t_context_errorf(ctx, + T_ResultCode_IoError, + command->definition, + str8_lit("build"), + "unable to capture parallel build command output"); + } + exited_count += controllers[index].exited; + } + + if (exited_count == count) { break; } + + // TODO: comprehensive solution to the waiting problem + if (endt_us != max_U64 && now_time_us() >= endt_us) { + timed_out = 1; + break; + } + sleep_ms(1); + } + + for EachIndex(index, count) { t_controller_close(&controllers[index]); } + + if (timed_out) { + return t_context_errorf(ctx, + T_ResultCode_Mismatch, + command->definition, + str8_lit("build"), + "parallel build command did not exit before timeout"); + } + + *exit_code_out = 0; + for EachIndex(index, count) { + if (controllers[index].exit_code != 0) { *exit_code_out = controllers[index].exit_code; } + T_Result result = t_build_validate_command_result(ctx, + variant, + command, + &bindings[index], + controllers[index].exit_code, + t_controller_stdout(ctx->arena, &controllers[index]), + t_controller_stderr(ctx->arena, &controllers[index])); + if (!t_result_is_ok(result)) { return result; } + } + + return ctx->result; +} + +internal T_Result +t_build_execute(T_Context *ctx) +{ + if (ctx->build == 0 || ctx->build->first_target == 0) { return ctx->result; } + + if (!ctx->build->is_single_target) { + return t_context_errorf(ctx, + T_ResultCode_ValidationError, + ctx->build->definition, + str8_lit("build"), + "project build declarations require target selection or a build-script codec"); + } + + T_BuildTarget *target = ctx->build->first_target; + T_BuildVariant *variant = t_build_current_variant(target); + + if (variant == 0) { + return t_context_errorf(ctx, + T_ResultCode_ValidationError, + target->definition, + str8_lit("build"), + "build target has no variant for the current platform"); + } + + U64 previous_exit = max_U64; + B32 has_previous_exit = 0; + for EachNode(command, T_BuildCommand, variant->first_command) { + if (command->when_previous_exit.size != 0) { + if (!has_previous_exit) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, command->definition, str8_lit("build"), "conditional build command has no previous command"); + } + if (!t_build_exit_matches(command->when_previous_exit, previous_exit)) { continue; } + } + if (command->parallel && command->repeat_count > 1) { + T_Result result = t_build_execute_parallel(ctx, variant, command, &previous_exit); + if ( ! t_result_is_ok(result)) { return result; } + has_previous_exit = 1; + continue; + } + + for EachIndex(index, command->repeat_count) { + T_Result result = t_build_execute_command(ctx, variant, command, index, &previous_exit); + if ( ! t_result_is_ok(result)) { return result; } + has_previous_exit = 1; + } + } + + return ctx->result; +} + +internal T_Result +t_op_repeat_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + MD_Node *count = t_child_from_string(arguments, "count"); + MD_Node *steps = t_child_from_string(arguments, "steps"); + U64 value = 0; + if (md_node_is_nil(count) || !try_u64_from_str8_c_rules(t_scalar_string_from_node(count), &value) || value == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, count, "repeat requires a positive integer count"); + } + if (md_node_is_nil(steps)) { t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments, "repeat requires steps"); } + MD_Node *index = t_child_from_string(arguments, "index"); + if (!md_node_is_nil(index) && t_scalar_string_from_node(index).size == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, index, "repeat index requires a name"); } + for MD_EachNode(node, steps->first) { + T_OpSpec *spec = t_op_spec_from_name(ctx->suite, node->string); + if (spec == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "unknown repeated operation '%S'", node->string); } + else if (spec->validate != 0) { + String8 saved = ctx->operation; + ctx->operation = spec->name; + T_Result result = spec->validate(ctx, node); + t_context_absorb_result(ctx->run, result); + ctx->operation = saved; + } + } + return ctx->run->result; +} + +internal T_Result +t_op_repeat_execute(T_Context *ctx, MD_Node *arguments) +{ + U64 count = 0; + try_u64_from_str8_c_rules(t_scalar_string_from_node(t_child_from_string(arguments, "count")), &count); + String8 index_name = t_scalar_string_from_node(t_child_from_string(arguments, "index")); + if (index_name.size == 0) { index_name = str8_lit("index"); } + MD_Node *steps = t_child_from_string(arguments, "steps"); + for (U64 index = 0; index < count; index += 1) { + T_ScriptBinding binding = {.next = ctx->bindings, .name = index_name, .value = str8f(ctx->arena, "%llu", index)}; + T_ScriptBinding *saved = ctx->bindings; + ctx->bindings = &binding; + for MD_EachNode(node, steps->first) { + T_OpSpec *spec = t_op_spec_from_name(ctx->suite, node->string); + T_Result result = spec->execute(ctx, node); + t_context_absorb_result(ctx, result); + if (!t_result_is_ok(ctx->result)) { break; } + } + ctx->bindings = saved; + if (!t_result_is_ok(ctx->result)) { break; } + } + return ctx->result; +} + +internal T_Result +t_op_compare_file_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + String8 left = t_scalar_string_from_node(t_child_from_string(arguments, "left")); + String8 right = t_scalar_string_from_node(t_child_from_string(arguments, "right")); + if (left.size == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments, "compare_file requires left"); } + if (right.size == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments, "compare_file requires right"); } + return ctx->run->result; +} + +internal T_Result +t_op_compare_file_execute(T_Context *ctx, MD_Node *arguments) +{ + String8 left = t_script_expand(ctx, arguments, t_scalar_string_from_node(t_child_from_string(arguments, "left")), 0, 0); + String8 right = t_script_expand(ctx, arguments, t_scalar_string_from_node(t_child_from_string(arguments, "right")), 0, 0); + if (!t_build_safe_relative_path(left) || !t_build_safe_relative_path(right)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, arguments, str8_lit("compare_file"), "paths must be work-relative"); + } + String8 left_path = t_make_file_path(ctx->arena, left); + String8 right_path = t_make_file_path(ctx->arena, right); + if (!file_path_exists(left_path)) { return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("compare_file"), "file '%S' does not exist", left); } + if (!file_path_exists(right_path)) { return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("compare_file"), "file '%S' does not exist", right); } + String8 left_data = t_read_file(ctx->arena, left); + String8 right_data = t_read_file(ctx->arena, right); + if (!str8_match(left_data, right_data, 0)) { + U64 mismatch = 0; + U64 common_size = Min(left_data.size, right_data.size); + while (mismatch < common_size && left_data.str[mismatch] == right_data.str[mismatch]) { mismatch += 1; } + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("compare_file"), "files '%S' and '%S' differ at byte %llu (sizes %llu and %llu)", left, right, + mismatch, left_data.size, right_data.size); + } + return ctx->result; +} diff --git a/src/torture/torture_build.h b/src/torture/torture_build.h new file mode 100644 index 000000000..916e3c149 --- /dev/null +++ b/src/torture/torture_build.h @@ -0,0 +1,134 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +typedef enum T_BuildCommandKind +{ + T_BuildCommandKind_Null, + T_BuildCommandKind_Compile, + T_BuildCommandKind_CompileLink, + T_BuildCommandKind_Link, + T_BuildCommandKind_Resource, + T_BuildCommandKind_Run, + T_BuildCommandKind_Copy, +} T_BuildCommandKind; + +typedef enum T_BuildOutputMode +{ + T_BuildOutputMode_Default, + T_BuildOutputMode_Explicit, + T_BuildOutputMode_None, +} T_BuildOutputMode; + +typedef struct T_BuildDeclaration T_BuildDeclaration; +struct T_BuildDeclaration +{ + T_BuildDeclaration *next; + String8 name; + MD_Node *definition; +}; + +typedef struct T_BuildDeclarationList T_BuildDeclarationList; +struct T_BuildDeclarationList +{ + T_BuildDeclaration *first; + T_BuildDeclaration *last; + U64 count; +}; + +typedef struct T_BuildOutput T_BuildOutput; +struct T_BuildOutput +{ + T_BuildOutput *next; + String8 name; + String8 path; + MD_Node *definition; +}; + +typedef struct T_BuildCommand T_BuildCommand; +struct T_BuildCommand +{ + T_BuildCommand *next; + T_BuildCommandKind kind; + String8 tool; + String8 arguments; + String8 artifact; + String8 output; + T_BuildOutputMode output_mode; + String8 index_name; + U64 repeat_count; + U64 timeout_ms; + String8 expected_exit; + String8 when_previous_exit; + String8 stdout_pattern; + String8 stderr_pattern; + MD_Node *produces; + MD_Node *definition; + B32 parallel; + B32 inject_output; +}; + +typedef struct T_BuildVariant T_BuildVariant; +struct T_BuildVariant +{ + T_BuildVariant *next; + OperatingSystem os; + T_BuildOutput *first_output; + T_BuildOutput *last_output; + U64 output_count; + T_BuildCommand *first_command; + T_BuildCommand *last_command; + U64 command_count; + MD_Node *definition; + B32 output_none; +}; + +typedef struct T_BuildTarget T_BuildTarget; +struct T_BuildTarget +{ + T_BuildTarget *next; + String8 name; + String8 kind; + T_BuildVariant *first_variant; + T_BuildVariant *last_variant; + U64 variant_count; + MD_Node *definition; +}; + +struct T_BuildPlan +{ + B32 is_single_target; + MD_Node *definition; + MD_Node *defaults; + T_BuildDeclarationList values; + T_BuildDeclarationList configurations; + T_BuildDeclarationList features; + T_BuildDeclarationList toolchains; + T_BuildDeclarationList packages; + T_BuildDeclarationList resources; + T_BuildDeclarationList generators; + T_BuildTarget *first_target; + T_BuildTarget *last_target; + U64 target_count; +}; + +typedef struct T_ScriptBinding T_ScriptBinding; +struct T_ScriptBinding +{ + T_ScriptBinding *next; + String8 name; + String8 value; +}; + +internal T_Result t_build_parse(T_ParseContext *ctx, MD_Node *build, T_BuildPlan **plan_out); +internal T_Result t_build_execute(T_Context *ctx); +internal String8 t_script_expand(T_Context *ctx, MD_Node *node, String8 string, T_BuildVariant *variant, T_ScriptBinding *bindings); + +internal T_Result t_op_repeat_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_op_repeat_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_op_compare_file_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_op_compare_file_execute(T_Context *ctx, MD_Node *arguments); + +global T_OpSpec t_op_repeat; +global T_OpSpec t_op_compare_file; diff --git a/src/torture/torture_coff.c b/src/torture/torture_coff.c new file mode 100644 index 000000000..4b326ee76 --- /dev/null +++ b/src/torture/torture_coff.c @@ -0,0 +1,931 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +internal MD_Node * +t_coff_field(T_ParseContext *ctx, MD_Node *parent, char *name) +{ + MD_Node *result = &md_nil_node; + U64 count = 0; + for + MD_EachNode(child, parent->first) + { + if (str8_match(child->string, str8_cstring(name), StringMatchFlag_CaseInsensitive)) { + if (count == 0) { result = child; } + count += 1; + } + } + if (count > 1) { t_parse_errorf(ctx, T_ResultCode_ValidationError, result, "field '%s' may only appear once", name); } + return result; +} + +internal B32 +t_coff_name_is_allowed(String8 name, char **allowed) +{ + for (char **item = allowed; *item != 0; item += 1) { + if (str8_match(name, str8_cstring(*item), StringMatchFlag_CaseInsensitive)) { return 1; } + } + return 0; +} + +internal void +t_coff_reject_unknown(T_ParseContext *ctx, MD_Node *node, char **allowed) +{ + for + MD_EachNode(child, node->first) + { + if (!t_coff_name_is_allowed(child->string, allowed)) { t_parse_errorf(ctx, T_ResultCode_ValidationError, child, "unknown field '%S'", child->string); } + } +} + +internal B32 +t_coff_scalar(T_ParseContext *ctx, MD_Node *parent, char *name, B32 required, String8 *value_out) +{ + MD_Node *field = t_coff_field(ctx, parent, name); + if (md_node_is_nil(field)) { + if (required) { t_parse_errorf(ctx, T_ResultCode_ValidationError, parent, "missing field '%s'", name); } + return !required; + } + if (md_node_is_nil(field->first) || !md_node_is_nil(field->first->next)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, field, "field '%s' requires exactly one value", name); + return 0; + } + *value_out = field->first->string; + return 1; +} + +internal B32 +t_coff_u64(T_ParseContext *ctx, MD_Node *parent, char *name, B32 required, U64 default_value, U64 max_value, U64 *value_out) +{ + if (md_node_is_nil(t_coff_field(ctx, parent, name))) { + *value_out = default_value; + if (required) { t_parse_errorf(ctx, T_ResultCode_ValidationError, parent, "missing field '%s'", name); } + return !required; + } + String8 string = {0}; + if (!t_coff_scalar(ctx, parent, name, required, &string)) { + *value_out = default_value; + return !required; + } + U64 value = 0; + if (!try_u64_from_str8_c_rules(string, &value) || value > max_value) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, parent, name), "field '%s' must be an integer no greater than %llu", name, max_value); + return 0; + } + *value_out = value; + return 1; +} + +internal B32 +t_coff_bool(T_ParseContext *ctx, MD_Node *parent, char *name, B32 default_value, B32 *value_out) +{ + if (md_node_is_nil(t_coff_field(ctx, parent, name))) { + *value_out = default_value; + return 1; + } + String8 string = {0}; + if (!t_coff_scalar(ctx, parent, name, 0, &string)) { + *value_out = default_value; + return 1; + } + if (str8_match(string, str8_lit("true"), StringMatchFlag_CaseInsensitive)) { + *value_out = 1; + } else if (str8_match(string, str8_lit("false"), StringMatchFlag_CaseInsensitive)) { + *value_out = 0; + } else { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, parent, name), "field '%s' must be true or false", name); + return 0; + } + return 1; +} + +internal B32 +t_coff_machine(T_ParseContext *ctx, MD_Node *parent, COFF_MachineType *machine_out) +{ + String8 string = {0}; + if (!t_coff_scalar(ctx, parent, "machine", 1, &string)) { return 0; } + COFF_MachineType machine = coff_machine_from_string(string); + if (machine == COFF_MachineType_Unknown && !str8_match(string, str8_lit("unknown"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, parent, "machine"), "unknown COFF machine '%S'", string); + return 0; + } + *machine_out = machine; + return 1; +} + +internal T_COFF_Section * +t_coff_section_from_id(T_COFF_Object *object, String8 id) +{ + for (T_COFF_Section *section = object->first_section; section != 0; section = section->next) { + if (str8_match(section->id, id, 0)) { return section; } + } + return 0; +} + +internal T_COFF_Symbol * +t_coff_symbol_from_id(T_COFF_Object *object, String8 id) +{ + for + EachNode(symbol, T_COFF_Symbol, object->first_symbol) + { + if (str8_match(symbol->id, id, 0)) { return symbol; } + } + return 0; +} + +internal COFF_SectionFlags +t_coff_permission_flag(String8 string) +{ + if (str8_match(string, str8_lit("read"), StringMatchFlag_CaseInsensitive)) { return COFF_SectionFlag_MemRead; } + if (str8_match(string, str8_lit("write"), StringMatchFlag_CaseInsensitive)) { return COFF_SectionFlag_MemWrite; } + if (str8_match(string, str8_lit("execute"), StringMatchFlag_CaseInsensitive)) { return COFF_SectionFlag_MemExecute; } + return 0; +} + +internal COFF_SectionFlags +t_coff_section_flag(String8 string) +{ + struct Map + { + char *name; + COFF_SectionFlags flag; + }; + local_persist struct Map map[] = { + {"discardable", COFF_SectionFlag_MemDiscardable}, {"not_cached", COFF_SectionFlag_MemNotCached}, {"not_paged", COFF_SectionFlag_MemNotPaged}, + {"shared", COFF_SectionFlag_MemShared}, {"link_comdat", COFF_SectionFlag_LnkCOMDAT}, {"link_info", COFF_SectionFlag_LnkInfo}, + {"link_remove", COFF_SectionFlag_LnkRemove}, {"link_other", COFF_SectionFlag_LnkOther}, {"link_nreloc_overflow", COFF_SectionFlag_LnkNRelocOvfl}, + {"gp_relative", COFF_SectionFlag_GpRel}, {"type_no_pad", COFF_SectionFlag_TypeNoPad}, {"16bit", COFF_SectionFlag_Mem16Bit}, + {"locked", COFF_SectionFlag_MemLocked}, {"preload", COFF_SectionFlag_MemPreload}, + }; + for + EachElement(i, map) + { + if (str8_match(string, str8_cstring(map[i].name), StringMatchFlag_CaseInsensitive)) { return map[i].flag; } + } + return 0; +} + +internal void +t_coff_parse_named_flags(T_ParseContext *ctx, MD_Node *field, B32 permissions, COFF_SectionFlags *flags_io) +{ + if (md_node_is_nil(field)) { return; } + COFF_SectionFlags seen = 0; + for + MD_EachNode(value, field->first) + { + COFF_SectionFlags flag = permissions ? t_coff_permission_flag(value->string) : t_coff_section_flag(value->string); + if (flag == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, value, "unknown COFF section %s '%S'", permissions ? "permission" : "flag", value->string); + } else if (seen & flag) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, value, "duplicate COFF section %s '%S'", permissions ? "permission" : "flag", value->string); + } else { + seen |= flag; + } + } + *flags_io |= seen; +} + +internal B32 +t_coff_reloc_type_from_string(COFF_MachineType machine, String8 string, COFF_RelocType *type_out) +{ + for + EachIndex(value, 256) + { + String8 candidate = coff_string_from_reloc(machine, (COFF_RelocType)value); + if (candidate.size != 0 && str8_match(candidate, string, StringMatchFlag_CaseInsensitive)) { + *type_out = (COFF_RelocType)value; + return 1; + } + } + return 0; +} + +internal T_COFF_SymbolKind +t_coff_symbol_kind_from_string(String8 string) +{ + struct Map + { + char *name; + T_COFF_SymbolKind kind; + }; + local_persist struct Map map[] = { + {"external", T_COFF_SymbolKind_External}, + {"external_function", T_COFF_SymbolKind_ExternalFunction}, + {"static", T_COFF_SymbolKind_Static}, + {"section_definition", T_COFF_SymbolKind_SectionDefinition}, + {"weak", T_COFF_SymbolKind_Weak}, + {"absolute", T_COFF_SymbolKind_Absolute}, + {"undefined", T_COFF_SymbolKind_Undefined}, + {"undefined_function", T_COFF_SymbolKind_UndefinedFunction}, + {"undefined_section", T_COFF_SymbolKind_UndefinedSection}, + {"section", T_COFF_SymbolKind_Section}, + {"common", T_COFF_SymbolKind_Common}, + }; + for + EachElement(i, map) + { + if (str8_match(string, str8_cstring(map[i].name), StringMatchFlag_CaseInsensitive)) { return map[i].kind; } + } + return T_COFF_SymbolKind_Null; +} + +internal COFF_ComdatSelectType +t_coff_selection_from_string(String8 string) +{ + for (U64 value = COFF_ComdatSelect_Null; value <= COFF_ComdatSelect_Largest; value += 1) { + if (str8_match(coff_string_from_comdat_select_type((COFF_ComdatSelectType)value), string, StringMatchFlag_CaseInsensitive)) { return (COFF_ComdatSelectType)value; } + } + return (COFF_ComdatSelectType)max_U32; +} + +internal COFF_WeakExtType +t_coff_weak_search_from_string(String8 string) +{ + if (str8_match(string, str8_lit("no_library"), StringMatchFlag_CaseInsensitive)) { return COFF_WeakExt_NoLibrary; } + if (str8_match(string, str8_lit("search_library"), StringMatchFlag_CaseInsensitive)) { return COFF_WeakExt_SearchLibrary; } + if (str8_match(string, str8_lit("alias"), StringMatchFlag_CaseInsensitive)) { return COFF_WeakExt_SearchAlias; } + if (str8_match(string, str8_lit("anti_dependency"), StringMatchFlag_CaseInsensitive)) { return COFF_WeakExt_AntiDependency; } + return COFF_WeakExt_Null; +} + +internal void +t_coff_parse_relocations(T_ParseContext *ctx, T_COFF_Object *object, T_COFF_Section *section, MD_Node *list) +{ + if (md_node_is_nil(list)) { return; } + char *allowed[] = {"type", "offset", "symbol", 0}; + for + MD_EachNode(node, list->first) + { + if (section->relocation_count >= max_U16) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "section '%S' exceeds the COFF relocation limit", section->id); + continue; + } + t_coff_reject_unknown(ctx, node, allowed); + T_COFF_Relocation *relocation = push_array(ctx->arena, T_COFF_Relocation, 1); + relocation->node = node; + relocation->id = node->string; + String8 type = {0}; + U64 offset = 0; + t_coff_scalar(ctx, node, "type", 1, &type); + t_coff_u64(ctx, node, "offset", 1, 0, max_U32, &offset); + t_coff_scalar(ctx, node, "symbol", 1, &relocation->symbol_id); + relocation->offset = (U32)offset; + if (!t_coff_reloc_type_from_string(object->machine, type, &relocation->type)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "type"), "relocation '%S' is invalid for machine %S", type, + coff_string_from_machine_type(object->machine)); + } + SLLQueuePush(section->first_relocation, section->last_relocation, relocation); + section->relocation_count += 1; + } +} + +internal void +t_coff_parse_sections(T_ParseContext *ctx, T_COFF_Object *object, MD_Node *list) +{ + if (md_node_is_nil(list)) { return; } + char *allowed[] = {"name", "permissions", "content", "alignment", "flags", "raw_flags", "data", "relocations", 0}; + for + MD_EachNode(node, list->first) + { + if (object->section_count >= max_U16) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "object exceeds the COFF section limit"); + continue; + } + t_coff_reject_unknown(ctx, node, allowed); + if (node->string.size == 0 || t_coff_section_from_id(object, node->string) != 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "section IDs must be non-empty and unique"); + continue; + } + T_COFF_Section *section = push_array(ctx->arena, T_COFF_Section, 1); + section->node = node; + section->id = node->string; + t_coff_scalar(ctx, node, "name", 1, §ion->name); + t_coff_parse_named_flags(ctx, t_coff_field(ctx, node, "permissions"), 1, §ion->flags); + String8 content = {0}; + if (t_coff_scalar(ctx, node, "content", 1, &content)) { + if (str8_match(content, str8_lit("code"), StringMatchFlag_CaseInsensitive)) { + section->flags |= COFF_SectionFlag_CntCode; + } else if (str8_match(content, str8_lit("initialized_data"), StringMatchFlag_CaseInsensitive)) { + section->flags |= COFF_SectionFlag_CntInitializedData; + } else if (str8_match(content, str8_lit("uninitialized_data"), StringMatchFlag_CaseInsensitive)) { + section->flags |= COFF_SectionFlag_CntUninitializedData; + } else { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "content"), "unknown COFF section content kind '%S'", content); + } + } + U64 alignment = 0; + if (!md_node_is_nil(t_coff_field(ctx, node, "alignment"))) { + t_coff_u64(ctx, node, "alignment", 0, 0, 8192, &alignment); + COFF_SectionFlags align_flags = coff_section_flag_from_align_size(alignment); + if (align_flags == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "alignment"), "unsupported COFF section alignment %llu", alignment); + } else { + section->flags |= align_flags; + } + } + t_coff_parse_named_flags(ctx, t_coff_field(ctx, node, "flags"), 0, §ion->flags); + U64 raw_flags = 0; + if (!md_node_is_nil(t_coff_field(ctx, node, "raw_flags"))) { + t_coff_u64(ctx, node, "raw_flags", 0, 0, max_U32, &raw_flags); + COFF_SectionFlags align_mask = COFF_SectionFlag_AlignMask << COFF_SectionFlag_AlignShift; + COFF_SectionFlags content_mask = COFF_SectionFlag_CntCode | COFF_SectionFlag_CntInitializedData | COFF_SectionFlag_CntUninitializedData; + if ((section->flags & raw_flags) || ((section->flags & align_mask) && (raw_flags & align_mask)) || ((section->flags & content_mask) && (raw_flags & content_mask))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "raw_flags"), "raw_flags conflicts with typed section flags"); + } + COFF_SectionFlags raw_content = (COFF_SectionFlags)raw_flags & content_mask; + if ((raw_content & (raw_content - 1)) != 0 || COFF_SectionFlags_ExtractAlign(raw_flags) == COFF_SectionFlag_AlignMask) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "raw_flags"), "raw_flags contains an invalid content kind or alignment"); + } + section->flags |= (COFF_SectionFlags)raw_flags; + } + if (section->flags & COFF_SectionFlag_LnkNRelocOvfl) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "link_nreloc_overflow is writer-owned and cannot be specified"); + } + MD_Node *data = t_coff_field(ctx, node, "data"); + if (md_node_is_nil(data)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "section requires data"); + } else { + t_bytes_from_producer(ctx->run, data, §ion->data); + } + SLLQueuePush(object->first_section, object->last_section, section); + object->section_count += 1; + t_coff_parse_relocations(ctx, object, section, t_coff_field(ctx, node, "relocations")); + } +} + +internal void +t_coff_parse_symbols(T_ParseContext *ctx, T_COFF_Object *object, MD_Node *list) +{ + if (md_node_is_nil(list)) { return; } + for + MD_EachNode(node, list->first) + { + if (node->string.size == 0 || t_coff_symbol_from_id(object, node->string) != 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "symbol IDs must be non-empty and unique"); + continue; + } + T_COFF_Symbol *symbol = push_array(ctx->arena, T_COFF_Symbol, 1); + symbol->node = node; + symbol->id = node->string; + String8 kind = {0}; + t_coff_scalar(ctx, node, "kind", 1, &kind); + symbol->kind = t_coff_symbol_kind_from_string(kind); + if (symbol->kind == T_COFF_SymbolKind_Null) { t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "kind"), "unknown COFF symbol kind '%S'", kind); } + char *known_fields[] = {"kind", "name", "section", "value", "size", "selection", "associate", "fallback", "search", "storage", 0}; + char *regular_fields[] = {"kind", "name", "section", "value", 0}; + char *section_definition_fields[] = {"kind", "section", "selection", "associate", 0}; + char *weak_fields[] = {"kind", "name", "fallback", "search", 0}; + char *absolute_fields[] = {"kind", "name", "value", "storage", 0}; + char *undefined_fields[] = {"kind", "name", 0}; + char *undefined_section_fields[] = {"kind", "name", "value", 0}; + char *section_fields[] = {"kind", "name", "section", 0}; + char *common_fields[] = {"kind", "name", "size", 0}; + char **allowed = known_fields; + switch (symbol->kind) { + case T_COFF_SymbolKind_External: + case T_COFF_SymbolKind_ExternalFunction: + case T_COFF_SymbolKind_Static: + allowed = regular_fields; + break; + case T_COFF_SymbolKind_SectionDefinition: + allowed = section_definition_fields; + break; + case T_COFF_SymbolKind_Weak: + allowed = weak_fields; + break; + case T_COFF_SymbolKind_Absolute: + allowed = absolute_fields; + break; + case T_COFF_SymbolKind_Undefined: + case T_COFF_SymbolKind_UndefinedFunction: + allowed = undefined_fields; + break; + case T_COFF_SymbolKind_UndefinedSection: + allowed = undefined_section_fields; + break; + case T_COFF_SymbolKind_Section: + allowed = section_fields; + break; + case T_COFF_SymbolKind_Common: + allowed = common_fields; + break; + default: + break; + } + t_coff_reject_unknown(ctx, node, allowed); + t_coff_scalar(ctx, node, "name", 0, &symbol->name); + t_coff_scalar(ctx, node, "section", 0, &symbol->section_id); + t_coff_scalar(ctx, node, "associate", 0, &symbol->associate_id); + t_coff_scalar(ctx, node, "fallback", 0, &symbol->fallback_id); + U64 value = 0, size = 0; + t_coff_u64(ctx, node, "value", 0, 0, max_U32, &value); + t_coff_u64(ctx, node, "size", 0, 0, max_U32, &size); + symbol->value = (U32)value; + symbol->size = (U32)size; + String8 selection = {0}; + if (!md_node_is_nil(t_coff_field(ctx, node, "selection")) && t_coff_scalar(ctx, node, "selection", 0, &selection)) { + symbol->selection = t_coff_selection_from_string(selection); + if (symbol->selection == (COFF_ComdatSelectType)max_U32) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "selection"), "unknown COMDAT selection '%S'", selection); + } + } + String8 search = {0}; + if (!md_node_is_nil(t_coff_field(ctx, node, "search")) && t_coff_scalar(ctx, node, "search", 0, &search)) { + symbol->weak_search = t_coff_weak_search_from_string(search); + if (symbol->weak_search == COFF_WeakExt_Null) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "search"), "unknown weak search policy '%S'", search); + } + } + String8 storage = {0}; + if (!md_node_is_nil(t_coff_field(ctx, node, "storage")) && t_coff_scalar(ctx, node, "storage", 0, &storage)) { + if (str8_match(storage, str8_lit("external"), StringMatchFlag_CaseInsensitive)) { + symbol->storage_class = COFF_SymStorageClass_External; + } else if (str8_match(storage, str8_lit("static"), StringMatchFlag_CaseInsensitive)) { + symbol->storage_class = COFF_SymStorageClass_Static; + } else { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, node, "storage"), "absolute symbol storage must be external or static"); + } + } + SLLQueuePush(object->first_symbol, object->last_symbol, symbol); + object->symbol_count += 1; + } + + for (T_COFF_Symbol *symbol = object->first_symbol; symbol != 0; symbol = symbol->next) { + B32 needs_name = symbol->kind != T_COFF_SymbolKind_SectionDefinition; + B32 needs_section = symbol->kind == T_COFF_SymbolKind_External || symbol->kind == T_COFF_SymbolKind_ExternalFunction || symbol->kind == T_COFF_SymbolKind_Static || + symbol->kind == T_COFF_SymbolKind_SectionDefinition || symbol->kind == T_COFF_SymbolKind_Section; + if (needs_name && symbol->name.size == 0) { t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "symbol '%S' requires name", symbol->id); } + if (needs_section && t_coff_section_from_id(object, symbol->section_id) == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "symbol '%S' references unknown section '%S'", symbol->id, symbol->section_id); + } + if (symbol->kind == T_COFF_SymbolKind_SectionDefinition) { + if (md_node_is_nil(t_coff_field(ctx, symbol->node, "selection"))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "section definition requires selection"); + } + if (symbol->selection == (COFF_ComdatSelectType)max_U32) { continue; } + if (symbol->selection == COFF_ComdatSelect_Associative && t_coff_section_from_id(object, symbol->associate_id) == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "associative section definition requires a valid associate section"); + } + } + if (symbol->kind == T_COFF_SymbolKind_Weak) { + T_COFF_Symbol *fallback = t_coff_symbol_from_id(object, symbol->fallback_id); + if (fallback == 0 || fallback == symbol || fallback->encoded != (COFF_ObjSymbol *)1) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "weak symbol fallback '%S' must refer to an earlier symbol", symbol->fallback_id); + } + if (symbol->weak_search == COFF_WeakExt_Null) { t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "weak symbol requires search policy"); } + } + if (symbol->kind == T_COFF_SymbolKind_Absolute && md_node_is_nil(t_coff_field(ctx, symbol->node, "storage"))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "absolute symbol requires storage"); + } + if (symbol->kind == T_COFF_SymbolKind_UndefinedSection && md_node_is_nil(t_coff_field(ctx, symbol->node, "value"))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "undefined section symbol requires value"); + } + if (symbol->kind == T_COFF_SymbolKind_Common && md_node_is_nil(t_coff_field(ctx, symbol->node, "size"))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, symbol->node, "common symbol requires size"); + } + symbol->encoded = (COFF_ObjSymbol *)1; + } + for (T_COFF_Symbol *symbol = object->first_symbol; symbol != 0; symbol = symbol->next) { symbol->encoded = 0; } + + for + EachNode(section, T_COFF_Section, object->first_section) + { + for + EachNode(relocation, T_COFF_Relocation, section->first_relocation) + { + if (t_coff_symbol_from_id(object, relocation->symbol_id) == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, relocation->node, "relocation references unknown symbol '%S'", relocation->symbol_id); + } + } + } +} + +internal T_COFF_Object * +t_coff_parse_object(T_ParseContext *ctx, MD_Node *node) +{ + char *allowed[] = {"machine", "timestamp", "sections", "symbols", "directives", 0}; + t_coff_reject_unknown(ctx, node, allowed); + T_COFF_Object *object = push_array(ctx->arena, T_COFF_Object, 1); + object->node = node; + t_coff_machine(ctx, node, &object->machine); + U64 timestamp = 0; + t_coff_u64(ctx, node, "timestamp", 0, 0, max_U32, ×tamp); + object->timestamp = (COFF_TimeStamp)timestamp; + t_coff_parse_sections(ctx, object, t_coff_field(ctx, node, "sections")); + t_coff_parse_symbols(ctx, object, t_coff_field(ctx, node, "symbols")); + MD_Node *directives = t_coff_field(ctx, node, "directives"); + for + MD_EachNode(directive_node, directives->first) + { + if (!str8_match(directive_node->string, str8_lit("directive"), StringMatchFlag_CaseInsensitive) || md_node_is_nil(directive_node->first) || + !md_node_is_nil(directive_node->first->next)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, directive_node, "directives must contain 'directive: value' entries"); + continue; + } + T_COFF_Directive *directive = push_array(ctx->arena, T_COFF_Directive, 1); + directive->node = directive_node; + directive->string = directive_node->first->string; + SLLQueuePush(object->first_directive, object->last_directive, directive); + object->directive_count += 1; + } + return object; +} + +internal COFF_ImportByType +t_coff_import_by_from_string(String8 string) +{ + if (str8_match(string, str8_lit("ordinal"), StringMatchFlag_CaseInsensitive)) { return COFF_ImportBy_Ordinal; } + if (str8_match(string, str8_lit("name"), StringMatchFlag_CaseInsensitive)) { return COFF_ImportBy_Name; } + if (str8_match(string, str8_lit("name_no_prefix"), StringMatchFlag_CaseInsensitive)) { return COFF_ImportBy_NameNoPrefix; } + if (str8_match(string, str8_lit("undecorate"), StringMatchFlag_CaseInsensitive)) { return COFF_ImportBy_Undecorate; } + return (COFF_ImportByType)max_U32; +} + +internal void +t_coff_parse_library_member(T_ParseContext *ctx, T_COFF_Library *library, MD_Node *node) +{ + char *allowed[] = {"path", "object", "import", "dll_import", 0}; + t_coff_reject_unknown(ctx, node, allowed); + T_COFF_LibraryMember *member = push_array(ctx->arena, T_COFF_LibraryMember, 1); + member->node = node; + member->id = node->string; + MD_Node *object_node = t_coff_field(ctx, node, "object"); + MD_Node *import_node = t_coff_field(ctx, node, "import"); + MD_Node *dll_node = t_coff_field(ctx, node, "dll_import"); + U64 kind_count = !md_node_is_nil(object_node) + !md_node_is_nil(import_node) + !md_node_is_nil(dll_node); + if (kind_count != 1) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "library member requires exactly one object, import, or dll_import definition"); + } else if (!md_node_is_nil(object_node)) { + char *object_allowed[] = {"path", "object", 0}; + t_coff_reject_unknown(ctx, node, object_allowed); + member->kind = T_COFF_LibraryMemberKind_Object; + t_coff_scalar(ctx, node, "path", 0, &member->path); + if (member->path.size == 0) { member->path = str8f(ctx->arena, "%S.obj", member->id); } + member->object = t_coff_parse_object(ctx, object_node); + } else if (!md_node_is_nil(import_node)) { + char *member_allowed[] = {"import", 0}; + t_coff_reject_unknown(ctx, node, member_allowed); + member->kind = T_COFF_LibraryMemberKind_Import; + char *import_allowed[] = {"dll", "name", "machine", "timestamp", "type", "lookup", "hint", "ordinal", 0}; + t_coff_reject_unknown(ctx, import_node, import_allowed); + t_coff_scalar(ctx, import_node, "dll", 1, &member->dll); + t_coff_scalar(ctx, import_node, "name", 1, &member->name); + t_coff_machine(ctx, import_node, &member->machine); + U64 timestamp = 0; + t_coff_u64(ctx, import_node, "timestamp", 0, 0, max_U32, ×tamp); + member->timestamp = (COFF_TimeStamp)timestamp; + String8 type = {0}; + t_coff_scalar(ctx, import_node, "type", 1, &type); + member->import_type = coff_import_header_type_from_string(type); + if (member->import_type == COFF_ImportType_Invalid) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, import_node, "type"), "unsupported import type '%S'", type); + } + String8 lookup = {0}; + t_coff_scalar(ctx, import_node, "lookup", 1, &lookup); + member->import_by = t_coff_import_by_from_string(lookup); + if (member->import_by == (COFF_ImportByType)max_U32) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, import_node, "lookup"), "unknown import lookup '%S'", lookup); + } + U64 hint_or_ordinal = 0; + char *number_field = member->import_by == COFF_ImportBy_Ordinal ? "ordinal" : "hint"; + char *name_import_allowed[] = {"dll", "name", "machine", "timestamp", "type", "lookup", "hint", 0}; + char *ordinal_import_allowed[] = {"dll", "name", "machine", "timestamp", "type", "lookup", "ordinal", 0}; + t_coff_reject_unknown(ctx, import_node, member->import_by == COFF_ImportBy_Ordinal ? ordinal_import_allowed : name_import_allowed); + t_coff_u64(ctx, import_node, number_field, 0, 0, max_U16, &hint_or_ordinal); + member->hint_or_ordinal = (U16)hint_or_ordinal; + } else if (!md_node_is_nil(dll_node)) { + char *member_allowed[] = {"dll_import", 0}; + t_coff_reject_unknown(ctx, node, member_allowed); + member->kind = T_COFF_LibraryMemberKind_DllImport; + char *dll_allowed[] = {"name", "machine", "timestamp", 0}; + t_coff_reject_unknown(ctx, dll_node, dll_allowed); + t_coff_scalar(ctx, dll_node, "name", 1, &member->name); + t_coff_machine(ctx, dll_node, &member->machine); + if (member->machine != COFF_MachineType_X64) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_coff_field(ctx, dll_node, "machine"), "dll_import currently supports only x64"); + } + U64 timestamp = 0; + t_coff_u64(ctx, dll_node, "timestamp", 0, 0, max_U32, ×tamp); + member->timestamp = (COFF_TimeStamp)timestamp; + } + SLLQueuePush(library->first_member, library->last_member, member); + library->member_count += 1; +} + +internal T_COFF_Library * +t_coff_parse_library(T_ParseContext *ctx, MD_Node *node) +{ + char *allowed[] = {"timestamp", "mode", "second_linker_member", "members", 0}; + t_coff_reject_unknown(ctx, node, allowed); + T_COFF_Library *library = push_array(ctx->arena, T_COFF_Library, 1); + library->node = node; + U64 timestamp = 0, mode = 0; + t_coff_u64(ctx, node, "timestamp", 0, 0, max_U32, ×tamp); + t_coff_u64(ctx, node, "mode", 0, 0, max_U16, &mode); + t_coff_bool(ctx, node, "second_linker_member", 0, &library->second_linker_member); + library->timestamp = (COFF_TimeStamp)timestamp; + library->mode = (U16)mode; + MD_Node *members = t_coff_field(ctx, node, "members"); + for + MD_EachNode(member, members->first) { t_coff_parse_library_member(ctx, library, member); } + if (library->second_linker_member) { + U64 emitted_member_count = 0; + for (T_COFF_LibraryMember *member = library->first_member; member != 0; member = member->next) { + emitted_member_count += member->kind == T_COFF_LibraryMemberKind_DllImport ? 3 : 1; + } + if (emitted_member_count > max_U16) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "library exceeds the second linker member limit"); } + } + return library; +} + +internal T_Result +t_coff_validate(T_ParseContext *ctx, T_Artifact *artifact) +{ + ctx->operation = str8_lit("coff"); + MD_Node *definition = artifact->definition; + char *allowed[] = {"object", "library", 0}; + t_coff_reject_unknown(ctx, definition, allowed); + MD_Node *object = t_coff_field(ctx, definition, "object"); + MD_Node *library = t_coff_field(ctx, definition, "library"); + if (md_node_is_nil(object) == md_node_is_nil(library)) { + return t_parse_errorf(ctx, T_ResultCode_ValidationError, definition, "coff requires exactly one object or library definition"); + } + T_COFF_Model *model = push_array(ctx->arena, T_COFF_Model, 1); + if (!md_node_is_nil(object)) { + model->kind = T_COFF_ModelKind_Object; + model->object = t_coff_parse_object(ctx, object); + } else { + model->kind = T_COFF_ModelKind_Library; + model->library = t_coff_parse_library(ctx, library); + } + artifact->codec_data = model; + return ctx->run->result; +} + +internal String8 +t_coff_encode_object(T_Context *ctx, T_COFF_Object *object) +{ + COFF_ObjWriter *writer = coff_obj_writer_alloc(object->timestamp, object->machine); + for (T_COFF_Section *section = object->first_section; section != 0; section = section->next) { + section->encoded = coff_obj_writer_push_section(writer, section->name, section->flags, section->data); + } + for (T_COFF_Directive *directive = object->first_directive; directive != 0; directive = directive->next) { coff_obj_writer_push_directive(writer, directive->string); } + for (T_COFF_Symbol *symbol = object->first_symbol; symbol != 0; symbol = symbol->next) { + T_COFF_Section *section = t_coff_section_from_id(object, symbol->section_id); + switch (symbol->kind) { + case T_COFF_SymbolKind_External: + symbol->encoded = coff_obj_writer_push_symbol_extern(writer, symbol->name, symbol->value, section->encoded); + break; + case T_COFF_SymbolKind_ExternalFunction: + symbol->encoded = coff_obj_writer_push_symbol_extern_func(writer, symbol->name, symbol->value, section->encoded); + break; + case T_COFF_SymbolKind_Static: + symbol->encoded = coff_obj_writer_push_symbol_static(writer, symbol->name, symbol->value, section->encoded); + break; + case T_COFF_SymbolKind_SectionDefinition: { + if (symbol->selection == COFF_ComdatSelect_Associative) { + T_COFF_Section *associate = t_coff_section_from_id(object, symbol->associate_id); + symbol->encoded = coff_obj_writer_push_symbol_associative(writer, section->encoded, associate->encoded); + } else { + symbol->encoded = coff_obj_writer_push_symbol_secdef(writer, section->encoded, symbol->selection); + } + } break; + case T_COFF_SymbolKind_Weak: { + T_COFF_Symbol *fallback = t_coff_symbol_from_id(object, symbol->fallback_id); + symbol->encoded = coff_obj_writer_push_symbol_weak(writer, symbol->name, symbol->weak_search, fallback->encoded); + } break; + case T_COFF_SymbolKind_Absolute: + symbol->encoded = coff_obj_writer_push_symbol_abs(writer, symbol->name, symbol->value, symbol->storage_class); + break; + case T_COFF_SymbolKind_Undefined: + symbol->encoded = coff_obj_writer_push_symbol_undef(writer, symbol->name); + break; + case T_COFF_SymbolKind_UndefinedFunction: + symbol->encoded = coff_obj_writer_push_symbol_undef_func(writer, symbol->name); + break; + case T_COFF_SymbolKind_UndefinedSection: + symbol->encoded = coff_obj_writer_push_symbol_undef_sect(writer, symbol->name, symbol->value); + break; + case T_COFF_SymbolKind_Section: + symbol->encoded = coff_obj_writer_push_symbol_sect(writer, symbol->name, section->encoded); + break; + case T_COFF_SymbolKind_Common: + symbol->encoded = coff_obj_writer_push_symbol_common(writer, symbol->name, symbol->size); + break; + default: + break; + } + } + for (T_COFF_Section *section = object->first_section; section != 0; section = section->next) { + for (T_COFF_Relocation *relocation = section->first_relocation; relocation != 0; relocation = relocation->next) { + T_COFF_Symbol *symbol = t_coff_symbol_from_id(object, relocation->symbol_id); + coff_obj_writer_section_push_reloc(writer, section->encoded, relocation->offset, symbol->encoded, relocation->type); + } + } + String8 result = coff_obj_writer_serialize(ctx->arena, writer); + coff_obj_writer_release(&writer); + return result; +} + +internal String8 +t_coff_encode_library(T_Context *ctx, T_COFF_Library *library) +{ + COFF_LibWriter *writer = coff_lib_writer_alloc(); + for (T_COFF_LibraryMember *member = library->first_member; member != 0; member = member->next) { + switch (member->kind) { + case T_COFF_LibraryMemberKind_Object: + coff_lib_writer_push_obj(writer, member->path, t_coff_encode_object(ctx, member->object)); + break; + case T_COFF_LibraryMemberKind_Import: + coff_lib_writer_push_import(writer, member->machine, member->timestamp, member->dll, member->import_by, member->name, member->hint_or_ordinal, member->import_type); + break; + case T_COFF_LibraryMemberKind_DllImport: { + String8 dll_name = str8_chop_last_dot(member->name); + String8 debug = lnk_make_linker_debug_symbols(writer->arena, member->machine); + coff_lib_writer_push_obj(writer, dll_name, pe_make_import_entry_obj(writer->arena, dll_name, member->timestamp, member->machine, debug)); + coff_lib_writer_push_obj(writer, dll_name, pe_make_null_import_descriptor_obj(writer->arena, member->timestamp, member->machine, debug)); + coff_lib_writer_push_obj(writer, dll_name, pe_make_null_thunk_data_obj(writer->arena, dll_name, member->timestamp, member->machine, debug)); + } break; + default: + break; + } + } + String8 result = coff_lib_writer_serialize(ctx->arena, writer, library->timestamp, library->mode, library->second_linker_member); + coff_lib_writer_release(&writer); + return result; +} + +internal T_Result +t_coff_encode(T_Context *ctx, T_Artifact *artifact) +{ + T_COFF_Model *model = artifact->codec_data; + if (model->kind == T_COFF_ModelKind_Object) { + artifact->data = t_coff_encode_object(ctx, model->object); + } else if (model->kind == T_COFF_ModelKind_Library) { + artifact->data = t_coff_encode_library(ctx, model->library); + } + return ctx->result; +} + +internal T_Result +t_coff_decode_object(T_Context *ctx, T_Artifact *artifact, String8 data, MD_Node *object) +{ + COFF_FileHeaderInfo header = coff_file_header_info_from_data(data); + if (header.section_table_range.max > data.size || header.symbol_table_range.max > data.size || header.string_table_range.max > data.size || + dim_1u64(header.section_table_range) != header.section_count_no_null * sizeof(COFF_SectionHeader) || + dim_1u64(header.symbol_table_range) != header.symbol_count * header.symbol_size) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "COFF object has invalid table ranges"); + } + + String8 string_table = str8_substr(data, header.string_table_range); + String8 symbol_table = str8_substr(data, header.symbol_table_range); + COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(data, header.section_table_range).str; + String8 machine = coff_string_from_machine_type(header.machine); + if (machine.size == 0 && header.machine == COFF_MachineType_Unknown) { machine = str8_lit("Unknown"); } + t_codec_push_field(ctx, object, "machine", machine); + t_codec_push_field(ctx, object, "big_object", header.is_big_obj ? str8_lit("true") : str8_lit("false")); + t_codec_push_u64(ctx, object, "section_count", header.section_count_no_null); + t_codec_push_u64(ctx, object, "symbol_record_count", header.symbol_count); + + MD_Node *sections = t_codec_push_node(ctx->arena, object, str8_lit("sections")); + for + EachIndex(section_idx, header.section_count_no_null) + { + COFF_SectionHeader *section_header = §ion_table[section_idx]; + MD_Node *section = t_codec_push_node(ctx->arena, sections, str8f(ctx->arena, "section_%llu", section_idx + 1)); + String8 section_name = {0}; + if (!t_codec_coff_section_name(string_table, section_header, §ion_name)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "COFF section %llu has an invalid name", section_idx + 1); + } + t_codec_push_field(ctx, section, "name", section_name); + t_codec_push_u64(ctx, section, "virtual_size", section_header->vsize); + t_codec_push_u64(ctx, section, "virtual_offset", section_header->voff); + t_codec_push_u64(ctx, section, "file_size", section_header->fsize); + t_codec_push_u64(ctx, section, "file_offset", section_header->foff); + t_codec_push_u64(ctx, section, "alignment", coff_align_size_from_section_flags(section_header->flags)); + t_codec_push_u64(ctx, section, "raw_flags", section_header->flags); + + String8 section_data = str8_zero(); + if (!(section_header->flags & COFF_SectionFlag_CntUninitializedData) && section_header->fsize != 0) { + if (section_header->foff > data.size || section_header->fsize > data.size - section_header->foff) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "COFF section %llu has an invalid data range", section_idx + 1); + } + section_data = str8_substr(data, rng_1u64(section_header->foff, section_header->foff + section_header->fsize)); + } + t_codec_push_field(ctx, section, "data", t_hex_from_data(ctx->arena, section_data)); + + COFF_RelocInfo reloc_info = coff_reloc_info_from_section_header(data, section_header); + if (reloc_info.array_off > data.size || reloc_info.count > (data.size - reloc_info.array_off) / sizeof(COFF_Reloc)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "COFF section %llu has an invalid relocation range", section_idx + 1); + } + MD_Node *relocations = t_codec_push_node(ctx->arena, section, str8_lit("relocations")); + COFF_Reloc *reloc_array = (COFF_Reloc *)(data.str + reloc_info.array_off); + for + EachIndex(reloc_idx, reloc_info.count) + { + COFF_Reloc *reloc = &reloc_array[reloc_idx]; + MD_Node *relocation = t_codec_push_node(ctx->arena, relocations, str8f(ctx->arena, "relocation_%llu", reloc_idx)); + t_codec_push_u64(ctx, relocation, "offset", reloc->apply_off); + t_codec_push_u64(ctx, relocation, "symbol_index", reloc->isymbol); + String8 type = coff_string_from_reloc(header.machine, reloc->type); + if (type.size != 0) { t_codec_push_field(ctx, relocation, "type", type); } + t_codec_push_u64(ctx, relocation, "raw_type", reloc->type); + } + } + + MD_Node *symbols = t_codec_push_node(ctx->arena, object, str8_lit("symbols")); + U64 symbol_idx = 0; + U64 primary_symbol_count = 0; + while (symbol_idx < header.symbol_count) { + COFF_ParsedSymbol symbol = coff_parse_symbol(header, string_table, symbol_table, (U32)symbol_idx); + if (symbol.aux_symbol_count > header.symbol_count - symbol_idx - 1) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "COFF symbol %llu has an invalid auxiliary record count", symbol_idx); + } + MD_Node *symbol_node = t_codec_push_node(ctx->arena, symbols, str8f(ctx->arena, "symbol_%llu", primary_symbol_count)); + t_codec_push_u64(ctx, symbol_node, "record_index", symbol_idx); + t_codec_push_field(ctx, symbol_node, "name", symbol.name); + t_codec_push_u64(ctx, symbol_node, "value", symbol.value); + t_codec_push_u64(ctx, symbol_node, "section_number", symbol.section_number); + t_codec_push_u64(ctx, symbol_node, "raw_type", symbol.type.v); + t_codec_push_field(ctx, symbol_node, "storage_class", coff_string_from_sym_storage_class(symbol.storage_class)); + t_codec_push_u64(ctx, symbol_node, "raw_storage_class", symbol.storage_class); + t_codec_push_u64(ctx, symbol_node, "auxiliary_record_count", symbol.aux_symbol_count); + if (symbol.aux_symbol_count != 0) { + U64 aux_off = (symbol_idx + 1) * header.symbol_size; + String8 aux_data = str8_substr(symbol_table, rng_1u64(aux_off, aux_off + symbol.aux_symbol_count * header.symbol_size)); + t_codec_push_field(ctx, symbol_node, "auxiliary_data", t_hex_from_data(ctx->arena, aux_data)); + } + symbol_idx += 1 + symbol.aux_symbol_count; + primary_symbol_count += 1; + } + t_codec_push_u64(ctx, object, "symbol_count", primary_symbol_count); + return ctx->result; +} + +internal T_Result +t_coff_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out) +{ + String8 data = artifact->data; + MD_Node *root = t_codec_push_node(ctx->arena, 0, str8_lit("coff")); + if (coff_is_obj(data) || coff_is_big_obj(data)) { + MD_Node *object = t_codec_push_node(ctx->arena, root, str8_lit("object")); + T_Result result = t_coff_decode_object(ctx, artifact, data, object); + if (!t_result_is_ok(result)) { return result; } + } else if (coff_is_regular_archive(data)) { + COFF_ArchiveParse archive_parse = coff_regular_archive_parse_from_data(data); + if (archive_parse.error.size != 0) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "invalid COFF archive: %S", archive_parse.error); + } + MD_Node *library = t_codec_push_node(ctx->arena, root, str8_lit("library")); + t_codec_push_field(ctx, library, "second_linker_member", archive_parse.has_second_header ? str8_lit("true") : str8_lit("false")); + t_codec_push_field(ctx, library, "long_names", archive_parse.long_names.size != 0 ? str8_lit("true") : str8_lit("false")); + t_codec_push_u64(ctx, library, "first_symbol_count", archive_parse.first_member.symbol_count); + if (archive_parse.has_second_header) { t_codec_push_u64(ctx, library, "second_symbol_count", archive_parse.second_member.symbol_count); } + MD_Node *members = t_codec_push_node(ctx->arena, library, str8_lit("members")); + U64 member_idx = 0; + U64 offset = coff_regular_archive_member_iter_init(data); + COFF_ArchiveMember member = {0}; + while (coff_regular_archive_member_iter_next(data, &offset, &member)) { + if (str8_match_lit("/", member.header.name, 0) || str8_match_lit("//", member.header.name, 0)) { continue; } + COFF_DataType member_type = coff_data_type_from_data(member.data); + MD_Node *member_node = t_codec_push_node(ctx->arena, members, str8f(ctx->arena, "member_%llu", member_idx)); + t_codec_push_field(ctx, member_node, "name", coff_decode_member_name(archive_parse.long_names, member.header.name)); + t_codec_push_u64(ctx, member_node, "timestamp", member.header.time_stamp); + t_codec_push_u64(ctx, member_node, "user_id", member.header.user_id); + t_codec_push_u64(ctx, member_node, "group_id", member.header.group_id); + t_codec_push_field(ctx, member_node, "mode", member.header.mode); + if (member_type == COFF_DataType_Import) { + COFF_ParsedArchiveImportHeader import = {0}; + COFF_ImportHeader *raw_import = str8_deserial_get_raw_ptr(member.data, 0, sizeof(*raw_import)); + B32 import_is_valid = raw_import != 0 && raw_import->data_size <= member.data.size - sizeof(*raw_import); + if (import_is_valid) { + String8 import_data = str8_substr(member.data, rng_1u64(sizeof(*raw_import), sizeof(*raw_import) + raw_import->data_size)); + String8 ignored = {0}; + U64 cursor = str8_deserial_read_cstr(import_data, 0, &ignored); + cursor += str8_deserial_read_cstr(import_data, cursor, &ignored); + import_is_valid = cursor == import_data.size; + } + if (!import_is_valid || coff_parse_import(member.data, 0, &import) == 0) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "archive member %llu has an invalid import header", member_idx); + } + MD_Node *import_node = t_codec_push_node(ctx->arena, member_node, str8_lit("import")); + t_codec_push_field(ctx, import_node, "machine", coff_string_from_machine_type(import.machine)); + t_codec_push_field(ctx, import_node, "dll", import.dll_name); + t_codec_push_field(ctx, import_node, "name", import.func_name); + t_codec_push_u64(ctx, import_node, "timestamp", import.time_stamp); + t_codec_push_u64(ctx, import_node, "hint_or_ordinal", import.hint_or_ordinal); + t_codec_push_field(ctx, import_node, "type", coff_string_from_import_header_type(import.type)); + t_codec_push_u64(ctx, import_node, "lookup", import.import_by); + } else { + MD_Node *object = t_codec_push_node(ctx->arena, member_node, str8_lit("object")); + T_Result result = t_coff_decode_object(ctx, artifact, member.data, object); + if (!t_result_is_ok(result)) { return result; } + } + member_idx += 1; + } + t_codec_push_u64(ctx, library, "member_count", member_idx); + } else { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("coff"), "artifact is not a supported COFF object or archive"); + } + *semantic_tree_out = root; + return ctx->result; +} diff --git a/src/torture/torture_coff.h b/src/torture/torture_coff.h new file mode 100644 index 000000000..264c2a2b7 --- /dev/null +++ b/src/torture/torture_coff.h @@ -0,0 +1,150 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +typedef enum T_COFF_SymbolKind +{ + T_COFF_SymbolKind_Null, + T_COFF_SymbolKind_External, + T_COFF_SymbolKind_ExternalFunction, + T_COFF_SymbolKind_Static, + T_COFF_SymbolKind_SectionDefinition, + T_COFF_SymbolKind_Weak, + T_COFF_SymbolKind_Absolute, + T_COFF_SymbolKind_Undefined, + T_COFF_SymbolKind_UndefinedFunction, + T_COFF_SymbolKind_UndefinedSection, + T_COFF_SymbolKind_Section, + T_COFF_SymbolKind_Common, +} T_COFF_SymbolKind; + +typedef struct T_COFF_Relocation T_COFF_Relocation; +struct T_COFF_Relocation +{ + T_COFF_Relocation *next; + MD_Node *node; + String8 id; + String8 symbol_id; + U32 offset; + COFF_RelocType type; +}; + +typedef struct T_COFF_Section T_COFF_Section; +struct T_COFF_Section +{ + T_COFF_Section *next; + MD_Node *node; + String8 id; + String8 name; + String8 data; + COFF_SectionFlags flags; + T_COFF_Relocation *first_relocation; + T_COFF_Relocation *last_relocation; + U64 relocation_count; + COFF_ObjSection *encoded; +}; + +typedef struct T_COFF_Symbol T_COFF_Symbol; +struct T_COFF_Symbol +{ + T_COFF_Symbol *next; + MD_Node *node; + String8 id; + String8 name; + String8 section_id; + String8 associate_id; + String8 fallback_id; + U32 value; + U32 size; + T_COFF_SymbolKind kind; + COFF_ComdatSelectType selection; + COFF_WeakExtType weak_search; + COFF_SymStorageClass storage_class; + COFF_ObjSymbol *encoded; +}; + +typedef struct T_COFF_Directive T_COFF_Directive; +struct T_COFF_Directive +{ + T_COFF_Directive *next; + MD_Node *node; + String8 string; +}; + +typedef struct T_COFF_Object T_COFF_Object; +struct T_COFF_Object +{ + MD_Node *node; + COFF_MachineType machine; + COFF_TimeStamp timestamp; + T_COFF_Section *first_section; + T_COFF_Section *last_section; + U64 section_count; + T_COFF_Symbol *first_symbol; + T_COFF_Symbol *last_symbol; + U64 symbol_count; + T_COFF_Directive *first_directive; + T_COFF_Directive *last_directive; + U64 directive_count; +}; + +typedef enum T_COFF_LibraryMemberKind +{ + T_COFF_LibraryMemberKind_Null, + T_COFF_LibraryMemberKind_Object, + T_COFF_LibraryMemberKind_Import, + T_COFF_LibraryMemberKind_DllImport, +} T_COFF_LibraryMemberKind; + +typedef struct T_COFF_LibraryMember T_COFF_LibraryMember; +struct T_COFF_LibraryMember +{ + T_COFF_LibraryMember *next; + MD_Node *node; + String8 id; + String8 path; + T_COFF_LibraryMemberKind kind; + T_COFF_Object *object; + String8 dll; + String8 name; + COFF_MachineType machine; + COFF_TimeStamp timestamp; + COFF_ImportType import_type; + COFF_ImportByType import_by; + U16 hint_or_ordinal; +}; + +typedef struct T_COFF_Library T_COFF_Library; +struct T_COFF_Library +{ + MD_Node *node; + COFF_TimeStamp timestamp; + U16 mode; + B32 second_linker_member; + T_COFF_LibraryMember *first_member; + T_COFF_LibraryMember *last_member; + U64 member_count; +}; + +typedef enum T_COFF_ModelKind +{ + T_COFF_ModelKind_Null, + T_COFF_ModelKind_Object, + T_COFF_ModelKind_Library, +} T_COFF_ModelKind; + +typedef struct T_COFF_Model T_COFF_Model; +struct T_COFF_Model +{ + T_COFF_ModelKind kind; + union + { + T_COFF_Object *object; + T_COFF_Library *library; + }; +}; + +internal T_Result t_coff_validate(T_ParseContext *ctx, T_Artifact *artifact); +internal T_Result t_coff_encode(T_Context *ctx, T_Artifact *artifact); +internal T_Result t_coff_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out); diff --git a/src/torture/torture_controller.c b/src/torture/torture_controller.c new file mode 100644 index 000000000..01ce4c5b5 --- /dev/null +++ b/src/torture/torture_controller.c @@ -0,0 +1,317 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +internal T_ControllerResult +t_controller_result(T_ControllerResultCode code, U64 exit_code) +{ + T_ControllerResult result = {code, exit_code}; + return result; +} + +internal void +t_controller_file_close(File *file) +{ + file_close(*file); + MemoryZeroStruct(file); +} + +internal U64 +t_controller_endt_from_timeout(U64 timeout_us) +{ + if (timeout_us == max_U64) { return max_U64; } + U64 now_us = now_time_us(); + return timeout_us > max_U64 - now_us ? max_U64 : now_us + timeout_us; +} + +internal B32 +t_controller_endt_is_expired(U64 endt_us) +{ + return endt_us != max_U64 && now_time_us() >= endt_us; +} + +internal String8List * +t_controller_capture_from_stream(T_Controller *controller, T_ControllerStream stream) +{ + if (stream == T_ControllerStream_Stdout) { return &controller->stdout_capture; } + if (stream == T_ControllerStream_Stderr) { return controller->merge_outputs ? &controller->stdout_capture : &controller->stderr_capture; } + return 0; +} + +internal U64 * +t_controller_cursor_from_stream(T_Controller *controller, T_ControllerStream stream) +{ + if (stream == T_ControllerStream_Stdout) { return &controller->stdout_cursor; } + if (stream == T_ControllerStream_Stderr) { return &controller->stderr_cursor; } + return 0; +} + +internal B32 +t_controller_capture_has(Arena *arena, String8List *capture, U64 *cursor, String8 needle) +{ + String8 output = str8_list_join(arena, capture, 0); + U64 match_off = str8_find_needle(output, *cursor, needle, 0); + B32 result = match_off < output.size; + if (result) { *cursor = match_off + needle.size; } + return result; +} + +internal T_ControllerResult +t_controller_pump_pipe(T_Controller *controller, File pipe, String8List *capture) +{ + U64 available = file_pipe_bytes_available(pipe); + if (available != 0) { + U64 size = Min(available, KB(64)); + U8 *data = push_array(controller->arena, U8, size); + U64 read_size = file_pipe_read(pipe, data, size); + if (read_size == 0) { return t_controller_result(T_ControllerResultCode_IoFailed, controller->exit_code); } + str8_list_push(controller->arena, capture, str8(data, read_size)); + } + return t_controller_result(T_ControllerResultCode_Ok, controller->exit_code); +} + +internal void +t_controller_drain_pipe(T_Controller *controller, File pipe, String8List *capture) +{ + U64 endt_us = t_controller_endt_from_timeout(10 * 1000000ull); + for (;;) { + t_controller_pump_pipe(controller, pipe, capture); + if (file_pipe_is_end(pipe)) { break; } + if (t_controller_endt_is_expired(endt_us)) { break; } + sleep_ms(1); + } +} + +internal T_ControllerResult +t_controller_poll_exit(T_Controller *controller) +{ + if (controller->launched && !controller->exited) { + U64 exit_code = max_U64; + if (process_poll(controller->process, &exit_code)) { + controller->exited = 1; + controller->exit_code = exit_code; + } + } + return t_controller_result(T_ControllerResultCode_Ok, controller->exit_code); +} + +internal T_ControllerResult +t_controller_launch(Arena *arena, T_Controller *controller, T_ControllerLaunchParams *params) +{ + MemoryZeroStruct(controller); + controller->arena = arena; + controller->exit_code = max_U64; + controller->merge_outputs = params->merge_outputs; + controller->new_console = params->new_console; + + FilePair stdin_pipe = file_pipe_make(1, 0); + FilePair stdout_pipe = file_pipe_make(0, 1); + FilePair stderr_pipe = {0}; + if (!params->merge_outputs) { stderr_pipe = file_pipe_make(0, 1); } + + B32 pipes_are_ok = file_pair_ok(stdin_pipe) && file_pair_ok(stdout_pipe) && + (params->merge_outputs || file_pair_ok(stderr_pipe)); + if (!pipes_are_ok) { goto launch_failed; } + + controller->process_group = process_group_make(1); + if (controller->process_group.u64[0] == 0) { goto launch_failed; } + + ProcessLaunchParams process_params = { + .cmd_line = params->cmd_line, + .path = params->path, + .env = params->env, + .inherit_env = params->inherit_env, + .consoleless = params->consoleless, + .new_console = params->new_console, + .process_group = controller->process_group, + .stdin_file = stdin_pipe.read, + .stdout_file = stdout_pipe.write, + .stderr_file = params->merge_outputs ? stdout_pipe.write : stderr_pipe.write, + }; + controller->process = process_launch(&process_params); + + if (process_match(controller->process, process_zero())) { goto launch_failed; } + + controller->stdin_write = stdin_pipe.write; + controller->stdout_read = stdout_pipe.read; + controller->stderr_read = stderr_pipe.read; + controller->launched = 1; + + MemoryZeroStruct(&stdin_pipe.write); + MemoryZeroStruct(&stdout_pipe.read); + MemoryZeroStruct(&stderr_pipe.read); + + file_pair_close(&stdin_pipe); + file_pair_close(&stdout_pipe); + file_pair_close(&stderr_pipe); + + return t_controller_result(T_ControllerResultCode_Ok, max_U64); + +launch_failed:; + + file_pair_close(&stdin_pipe); + file_pair_close(&stdout_pipe); + file_pair_close(&stderr_pipe); + + process_group_close(controller->process_group); + MemoryZeroStruct(&controller->process_group); + controller->closed = 1; + + return t_controller_result(T_ControllerResultCode_LaunchFailed, max_U64); +} + +internal T_ControllerResult +t_controller_pump(T_Controller *controller) +{ + T_ControllerResult result = t_controller_pump_pipe(controller, controller->stdout_read, &controller->stdout_capture); + if (result.code == T_ControllerResultCode_Ok && !controller->merge_outputs) { result = t_controller_pump_pipe(controller, controller->stderr_read, &controller->stderr_capture); } + if (result.code == T_ControllerResultCode_Ok) { result = t_controller_poll_exit(controller); } + if (result.code == T_ControllerResultCode_Ok && controller->exited) { + result = t_controller_pump_pipe(controller, controller->stdout_read, &controller->stdout_capture); + if (result.code == T_ControllerResultCode_Ok && !controller->merge_outputs) { + result = t_controller_pump_pipe(controller, controller->stderr_read, &controller->stderr_capture); + } + } + return result; +} + +internal T_ControllerResult +t_controller_send(T_Controller *controller, String8 data) +{ + if (controller->exited) { return t_controller_result(T_ControllerResultCode_ProcessExited, controller->exit_code); } + + for (U64 off = 0; off < data.size;) { + U64 write_size = file_pipe_write(controller->stdin_write, data.str + off, data.size - off); + if (write_size == 0) { return t_controller_result(T_ControllerResultCode_IoFailed, controller->exit_code); } + off += write_size; + } + return t_controller_result(T_ControllerResultCode_Ok, controller->exit_code); +} + +internal T_ControllerResult +t_controller_send_line(T_Controller *controller, String8 line) +{ + T_ControllerResult result = t_controller_send(controller, line); + if (result.code == T_ControllerResultCode_Ok) { result = t_controller_send(controller, str8_lit("\n")); } + return result; +} + +internal T_ControllerResult +t_controller_wait(T_Controller *controller, U64 timeout_us) +{ + U64 endt_us = t_controller_endt_from_timeout(timeout_us); + for (;;) { + T_ControllerResult result = t_controller_pump(controller); + if (result.code != T_ControllerResultCode_Ok) { return result; } + if (controller->exited) { return result; } + if (t_controller_endt_is_expired(endt_us)) { return t_controller_result(T_ControllerResultCode_Timeout, controller->exit_code); } + sleep_ms(1); + } +} + +internal T_ControllerResult +t_controller_expect(T_Controller *controller, T_ControllerStream stream, String8 needle, U64 timeout_us) +{ + String8List *capture = t_controller_capture_from_stream(controller, stream); + U64 *cursor = t_controller_cursor_from_stream(controller, stream); + + Temp scratch = scratch_begin(&controller->arena, 1); + U64 endt_us = t_controller_endt_from_timeout(timeout_us); + T_ControllerResult result = t_controller_result(T_ControllerResultCode_Timeout, controller->exit_code); + for (;;) { + result = t_controller_pump(controller); + if (result.code != T_ControllerResultCode_Ok) { break; } + Temp iteration = temp_begin(scratch.arena); + B32 has_match = t_controller_capture_has(scratch.arena, capture, cursor, needle); + temp_end(iteration); + if (has_match) { break; } + if (controller->exited) { + result = t_controller_result(T_ControllerResultCode_ProcessExited, controller->exit_code); + break; + } + if (t_controller_endt_is_expired(endt_us)) { + result = t_controller_result(T_ControllerResultCode_Timeout, controller->exit_code); + break; + } + sleep_ms(1); + } + scratch_end(scratch); + return result; +} + +internal T_ControllerResult +t_controller_wait_until_quiet(T_Controller *controller, U64 quiet_us, U64 timeout_us) +{ + U64 endt_us = t_controller_endt_from_timeout(timeout_us); + U64 quiet_begin_us = now_time_us(); + U64 capture_size = controller->stdout_capture.total_size + controller->stderr_capture.total_size; + for (;;) { + T_ControllerResult result = t_controller_pump(controller); + if (result.code != T_ControllerResultCode_Ok) { return result; } + U64 new_capture_size = controller->stdout_capture.total_size + controller->stderr_capture.total_size; + if (new_capture_size != capture_size) { + capture_size = new_capture_size; + quiet_begin_us = now_time_us(); + } + if (controller->exited) { return t_controller_result(T_ControllerResultCode_ProcessExited, controller->exit_code); } + U64 now_us = now_time_us(); + if (now_us - quiet_begin_us >= quiet_us) { return result; } + if (endt_us != max_U64 && now_us >= endt_us) { return t_controller_result(T_ControllerResultCode_Timeout, controller->exit_code); } + sleep_ms(1); + } +} + +internal T_ControllerResult +t_controller_interrupt(T_Controller *controller) +{ + T_ControllerResultCode code = process_send_ctrl_c(controller->process) ? T_ControllerResultCode_Ok : T_ControllerResultCode_IoFailed; + return t_controller_result(code, controller->exit_code); +} + +internal void +t_controller_close(T_Controller *controller) +{ + if (controller->closed) { return; } + + t_controller_file_close(&controller->stdin_write); + + if (controller->launched) { + t_controller_pump(controller); + + process_group_close(controller->process_group); + MemoryZeroStruct(&controller->process_group); + + if (!process_match(controller->process, process_zero())) { + U64 exit_code = max_U64; + U64 endt_us = t_controller_endt_from_timeout(10 * 1000000ull); + + if (!process_join(controller->process, endt_us, &exit_code)) { + process_kill(controller->process); + process_join(controller->process, max_U64, &exit_code); + } + + MemoryZeroStruct(&controller->process); + controller->exited = 1; + controller->exit_code = exit_code; + } + + t_controller_drain_pipe(controller, controller->stdout_read, &controller->stdout_capture); + if (!controller->merge_outputs) { t_controller_drain_pipe(controller, controller->stderr_read, &controller->stderr_capture); } + } + + t_controller_file_close(&controller->stdout_read); + t_controller_file_close(&controller->stderr_read); + controller->closed = 1; +} + +internal String8 +t_controller_stdout(Arena *arena, T_Controller *controller) +{ + return str8_list_join(arena, &controller->stdout_capture, 0); +} + +internal String8 +t_controller_stderr(Arena *arena, T_Controller *controller) +{ + return str8_list_join(arena, &controller->stderr_capture, 0); +} + diff --git a/src/torture/torture_controller.h b/src/torture/torture_controller.h new file mode 100644 index 000000000..2ad79ab9b --- /dev/null +++ b/src/torture/torture_controller.h @@ -0,0 +1,77 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +// Generic subprocess transport. Suites own command formatting and response semantics. + +typedef enum T_ControllerResultCode +{ + T_ControllerResultCode_Ok, + T_ControllerResultCode_LaunchFailed, + T_ControllerResultCode_IoFailed, + T_ControllerResultCode_Timeout, + T_ControllerResultCode_ProcessExited, +} T_ControllerResultCode; + +typedef struct T_ControllerResult T_ControllerResult; +struct T_ControllerResult +{ + T_ControllerResultCode code; + U64 exit_code; +}; + +typedef enum T_ControllerStream +{ + T_ControllerStream_Stdout, + T_ControllerStream_Stderr, +} T_ControllerStream; + +typedef struct T_ControllerLaunchParams T_ControllerLaunchParams; +struct T_ControllerLaunchParams +{ + String8List cmd_line; + String8 path; + String8List env; + B32 inherit_env; + B32 consoleless; + B32 new_console; + B32 merge_outputs; // Redirect stderr into stdout while preserving one ordered transcript. +}; + +typedef struct T_Controller T_Controller; +struct T_Controller +{ + Arena *arena; + Process process; + ProcessGroup process_group; + + File stdin_write; + File stdout_read; + File stderr_read; + + String8List stdout_capture; + String8List stderr_capture; + U64 stdout_cursor; + U64 stderr_cursor; + + B32 merge_outputs; + B32 new_console; + B32 launched; + B32 exited; + B32 closed; + U64 exit_code; +}; + +internal T_ControllerResult t_controller_launch(Arena *arena, T_Controller *controller, T_ControllerLaunchParams *params); +internal T_ControllerResult t_controller_pump(T_Controller *controller); +internal T_ControllerResult t_controller_send(T_Controller *controller, String8 data); +internal T_ControllerResult t_controller_send_line(T_Controller *controller, String8 line); +internal T_ControllerResult t_controller_wait(T_Controller *controller, U64 timeout_us); +internal T_ControllerResult t_controller_expect(T_Controller *controller, T_ControllerStream stream, String8 needle, U64 timeout_us); +internal T_ControllerResult t_controller_wait_until_quiet(T_Controller *controller, U64 quiet_us, U64 timeout_us); +internal T_ControllerResult t_controller_interrupt(T_Controller *controller); +internal void t_controller_close(T_Controller *controller); + +internal String8 t_controller_stdout(Arena *arena, T_Controller *controller); +internal String8 t_controller_stderr(Arena *arena, T_Controller *controller); diff --git a/src/torture/torture_linker.c b/src/torture/torture_linker.c new file mode 100644 index 000000000..127ec8a3a --- /dev/null +++ b/src/torture/torture_linker.c @@ -0,0 +1,1512 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +internal B32 +t_codec_coff_section_name(String8 string_table, COFF_SectionHeader *header, String8 *name_out) +{ + String8 name = str8_cstring_capped(header->name, header->name + sizeof(header->name)); + if (name.size != 0 && name.str[0] == '/') { + U64 offset = 0; + if (!try_u64_from_str8_c_rules(str8_skip(name, 1), &offset) || offset >= string_table.size) { return 0; } + name = str8_cstring_capped(string_table.str + offset, string_table.str + string_table.size); + if (name.size >= string_table.size - offset) { return 0; } + } + *name_out = name; + return 1; +} + +internal void +t_codec_validate_fields(T_ParseContext *ctx, MD_Node *node, char **allowed) +{ + for + MD_EachNode(child, node->first) + { + B32 is_allowed = 0; + for (U64 i = 0; allowed[i] != 0; i += 1) { + if (str8_match(child->string, str8_cstring(allowed[i]), StringMatchFlag_CaseInsensitive)) { + is_allowed = 1; + break; + } + } + if (!is_allowed) { t_parse_errorf(ctx, T_ResultCode_ValidationError, child, "unknown field '%S'", child->string); } + for (MD_Node *previous = node->first; previous != child; previous = previous->next) { + if (str8_match(previous->string, child->string, StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, child, "duplicate field '%S'", child->string); + break; + } + } + } +} + +internal void +t_codec_validate_scalar(T_ParseContext *ctx, MD_Node *node, char *name, B32 required) +{ + MD_Node *field = t_codec_child(node, name); + if (md_node_is_nil(field)) { + if (required) { t_parse_errorf(ctx, T_ResultCode_ValidationError, node, "operation requires %s", name); } + } else if (md_node_is_nil(field->first) || !md_node_is_nil(field->first->next)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, field, "%s requires exactly one value", name); + } +} + +internal T_Result +t_codec_pe_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out) +{ + String8 data = artifact->data; + if (!pe_check_magic(data)) { return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_zero(), "artifact is not a PE image"); } + PE_BinInfo pe = pe_bin_info_from_data(ctx->arena, data); + MD_Node *root = t_codec_push_node(ctx->arena, 0, str8_lit("pe")); + t_codec_push_field(ctx, root, "is_pe32", pe.is_pe32 ? str8_lit("true") : str8_lit("false")); + t_codec_push_field(ctx, root, "arch", string_from_arch(pe.arch)); + t_codec_push_field(ctx, root, "subsystem", pe_string_from_subsystem(pe.subsystem)); + t_codec_push_u64(ctx, root, "section_count", pe.section_count); + t_codec_push_u64(ctx, root, "section_alignment", pe.virt_section_align); + t_codec_push_u64(ctx, root, "file_alignment", pe.file_section_align); + t_codec_push_u64(ctx, root, "symbol_count", pe.symbol_count); + t_codec_push_u64(ctx, root, "data_directory_count", pe.data_dir_count); + t_codec_push_u64(ctx, root, "entry_point", pe.entry_point); + + COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(data, pe.section_table_range).str; + String8 string_table = str8_substr(data, pe.string_table_range); + MD_Node *section_list = t_codec_push_node(ctx->arena, root, str8_lit("sections")); + MD_Node *indexed_section_list = t_codec_push_node(ctx->arena, root, str8_lit("sections_by_index")); + for + EachIndex(i, pe.section_count) + { + COFF_SectionHeader *header = §ion_table[i]; + String8 section_name = {0}; + if (!t_codec_coff_section_name(string_table, header, §ion_name)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("pe"), "PE section %llu has an invalid name", i + 1); + } + MD_Node *section = t_codec_push_node(ctx->arena, section_list, section_name); + MD_Node *indexed_section = t_codec_push_node(ctx->arena, indexed_section_list, str8f(ctx->arena, "section_%llu", i + 1)); + t_codec_push_field(ctx, indexed_section, "name", section_name); + t_codec_push_u64(ctx, section, "virtual_size", header->vsize); + t_codec_push_u64(ctx, indexed_section, "virtual_size", header->vsize); + t_codec_push_u64(ctx, section, "virtual_offset", header->voff); + t_codec_push_u64(ctx, indexed_section, "virtual_offset", header->voff); + t_codec_push_u64(ctx, section, "file_size", header->fsize); + t_codec_push_u64(ctx, indexed_section, "file_size", header->fsize); + t_codec_push_u64(ctx, section, "file_offset", header->foff); + t_codec_push_u64(ctx, indexed_section, "file_offset", header->foff); + t_codec_push_u64(ctx, section, "alignment", coff_align_size_from_section_flags(header->flags)); + t_codec_push_u64(ctx, indexed_section, "alignment", coff_align_size_from_section_flags(header->flags)); + t_codec_push_u64(ctx, section, "raw_flags", header->flags); + t_codec_push_u64(ctx, indexed_section, "raw_flags", header->flags); + if (header->fsize != 0 && (header->foff > data.size || header->fsize > data.size - header->foff)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("pe"), "PE section %llu has an invalid raw data range", i + 1); + } + U64 section_data_size = Min(header->vsize, header->fsize); + if (section_data_size != 0 && (header->foff > data.size || section_data_size > data.size - header->foff)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_lit("pe"), "PE section %llu has an invalid data range", i + 1); + } + if (section_data_size != 0) { + String8 section_data = str8_substr(data, rng_1u64(header->foff, header->foff + section_data_size)); + t_codec_push_field(ctx, section, "data", t_hex_from_data(ctx->arena, section_data)); + t_codec_push_field(ctx, indexed_section, "data", t_hex_from_data(ctx->arena, section_data)); + } + } + + local_persist char *directory_names[] = {"exports", "imports", "resources", "exceptions", "certificates", "base_relocations", + "debug", "architecture", "global_ptr", "tls", "load_config", "bound_imports", + "import_address", "delay_imports", "com_descriptor", "reserved"}; + MD_Node *directories = t_codec_push_node(ctx->arena, root, str8_lit("data_directories")); + U64 directory_count = Min(pe.data_dir_count, ArrayCount(directory_names)); + for EachIndex(i, directory_count) + { + MD_Node *directory = t_codec_push_node(ctx->arena, directories, str8_cstring(directory_names[i])); + t_codec_push_u64(ctx, directory, "virtual_offset", pe.data_dir_vranges[i].min); + t_codec_push_u64(ctx, directory, "virtual_size", dim_1u64(pe.data_dir_vranges[i])); + t_codec_push_u64(ctx, directory, "file_offset", pe.data_dir_franges[i].min); + t_codec_push_u64(ctx, directory, "file_size", dim_1u64(pe.data_dir_franges[i])); + } + + U16 *optional_magic = str8_deserial_get_raw_ptr(data, pe.optional_header_off, sizeof(*optional_magic)); + U64 optional_header_size = pe.section_table_range.min >= pe.optional_header_off ? pe.section_table_range.min - pe.optional_header_off : 0; + if (optional_magic == 0 || (*optional_magic != PE_PE32_MAGIC && *optional_magic != PE_PE32PLUS_MAGIC)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_zero(), "PE optional header has invalid magic"); + } + if (*optional_magic == PE_PE32PLUS_MAGIC) { + if (optional_header_size < sizeof(PE_OptionalHeader32Plus)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_zero(), "declared PE optional header is truncated"); + } + PE_OptionalHeader32Plus *opt = str8_deserial_get_raw_ptr(data, pe.optional_header_off, sizeof(*opt)); + if (opt == 0) { return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_zero(), "PE optional header is truncated"); } + MD_Node *optional = t_codec_push_node(ctx->arena, root, str8_lit("optional")); + t_codec_push_u64(ctx, optional, "sizeof_code", opt->sizeof_code); + t_codec_push_u64(ctx, optional, "sizeof_initialized_data", opt->sizeof_inited_data); + t_codec_push_u64(ctx, optional, "sizeof_uninitialized_data", opt->sizeof_uninited_data); + t_codec_push_u64(ctx, optional, "code_base", opt->code_base); + t_codec_push_u64(ctx, optional, "image_base", opt->image_base); + t_codec_push_u64(ctx, optional, "major_os_version", opt->major_os_ver); + t_codec_push_u64(ctx, optional, "minor_os_version", opt->minor_os_ver); + t_codec_push_u64(ctx, optional, "major_image_version", opt->major_img_ver); + t_codec_push_u64(ctx, optional, "minor_image_version", opt->minor_img_ver); + t_codec_push_u64(ctx, optional, "major_subsystem_version", opt->major_subsystem_ver); + t_codec_push_u64(ctx, optional, "minor_subsystem_version", opt->minor_subsystem_ver); + t_codec_push_u64(ctx, optional, "win32_version", opt->win32_version_value); + t_codec_push_u64(ctx, optional, "sizeof_image", opt->sizeof_image); + t_codec_push_u64(ctx, optional, "sizeof_headers", opt->sizeof_headers); + t_codec_push_u64(ctx, optional, "dll_characteristics", opt->dll_characteristics); + t_codec_push_u64(ctx, optional, "sizeof_stack_reserve", opt->sizeof_stack_reserve); + t_codec_push_u64(ctx, optional, "sizeof_stack_commit", opt->sizeof_stack_commit); + t_codec_push_u64(ctx, optional, "sizeof_heap_reserve", opt->sizeof_heap_reserve); + t_codec_push_u64(ctx, optional, "sizeof_heap_commit", opt->sizeof_heap_commit); + t_codec_push_u64(ctx, optional, "loader_flags", opt->loader_flags); + } else { + if (optional_header_size < sizeof(PE_OptionalHeader32)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_zero(), "declared PE optional header is truncated"); + } + PE_OptionalHeader32 *opt = str8_deserial_get_raw_ptr(data, pe.optional_header_off, sizeof(*opt)); + if (opt == 0) { return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_zero(), "PE optional header is truncated"); } + MD_Node *optional = t_codec_push_node(ctx->arena, root, str8_lit("optional")); + t_codec_push_u64(ctx, optional, "sizeof_code", opt->sizeof_code); + t_codec_push_u64(ctx, optional, "sizeof_initialized_data", opt->sizeof_inited_data); + t_codec_push_u64(ctx, optional, "sizeof_uninitialized_data", opt->sizeof_uninited_data); + t_codec_push_u64(ctx, optional, "code_base", opt->code_base); + t_codec_push_u64(ctx, optional, "data_base", opt->data_base); + t_codec_push_u64(ctx, optional, "image_base", opt->image_base); + t_codec_push_u64(ctx, optional, "major_os_version", opt->major_os_ver); + t_codec_push_u64(ctx, optional, "minor_os_version", opt->minor_os_ver); + t_codec_push_u64(ctx, optional, "major_image_version", opt->major_img_ver); + t_codec_push_u64(ctx, optional, "minor_image_version", opt->minor_img_ver); + t_codec_push_u64(ctx, optional, "major_subsystem_version", opt->major_subsystem_ver); + t_codec_push_u64(ctx, optional, "minor_subsystem_version", opt->minor_subsystem_ver); + t_codec_push_u64(ctx, optional, "win32_version", opt->win32_version_value); + t_codec_push_u64(ctx, optional, "sizeof_image", opt->sizeof_image); + t_codec_push_u64(ctx, optional, "sizeof_headers", opt->sizeof_headers); + t_codec_push_u64(ctx, optional, "dll_characteristics", opt->dll_characteristics); + t_codec_push_u64(ctx, optional, "sizeof_stack_reserve", opt->sizeof_stack_reserve); + t_codec_push_u64(ctx, optional, "sizeof_stack_commit", opt->sizeof_stack_commit); + t_codec_push_u64(ctx, optional, "sizeof_heap_reserve", opt->sizeof_heap_reserve); + t_codec_push_u64(ctx, optional, "sizeof_heap_commit", opt->sizeof_heap_commit); + t_codec_push_u64(ctx, optional, "loader_flags", opt->loader_flags); + } + + if (pe.data_dir_count > PE_DataDirectoryIndex_EXPORT && dim_1u64(pe.data_dir_vranges[PE_DataDirectoryIndex_EXPORT]) != 0) { + PE_ParsedExportTable table = pe_exports_from_data(ctx->arena, pe.section_count, section_table, data, pe.data_dir_franges[PE_DataDirectoryIndex_EXPORT], + pe.data_dir_vranges[PE_DataDirectoryIndex_EXPORT]); + MD_Node *exports = t_codec_push_node(ctx->arena, root, str8_lit("exports")); + t_codec_push_u64(ctx, exports, "flags", table.flags); + t_codec_push_u64(ctx, exports, "timestamp", table.time_stamp); + t_codec_push_u64(ctx, exports, "major_version", table.major_ver); + t_codec_push_u64(ctx, exports, "minor_version", table.minor_ver); + t_codec_push_u64(ctx, exports, "ordinal_base", table.ordinal_base); + t_codec_push_u64(ctx, exports, "count", table.export_count); + MD_Node *entries = t_codec_push_node(ctx->arena, exports, str8_lit("entries")); + for EachIndex(i, table.export_count) + { + PE_ParsedExport *entry = &table.exports[i]; + MD_Node *entry_node = t_codec_push_node(ctx->arena, entries, str8f(ctx->arena, "export_%llu", i)); + t_codec_push_field(ctx, entry_node, "name", entry->name); + t_codec_push_field(ctx, entry_node, "forwarder", entry->forwarder); + t_codec_push_u64(ctx, entry_node, "virtual_offset", entry->voff); + t_codec_push_u64(ctx, entry_node, "ordinal", entry->ordinal); + } + } + + if (pe.data_dir_count > PE_DataDirectoryIndex_IMPORT && dim_1u64(pe.data_dir_vranges[PE_DataDirectoryIndex_IMPORT]) != 0) { + PE_ParsedStaticImportTable table = pe_static_imports_from_data(ctx->arena, pe.is_pe32, pe.section_count, section_table, data, pe.data_dir_franges[PE_DataDirectoryIndex_IMPORT]); + MD_Node *imports = t_codec_push_node(ctx->arena, root, str8_lit("imports")); + t_codec_push_u64(ctx, imports, "count", table.count); + for EachIndex(dll_idx, table.count) + { + PE_ParsedStaticDLLImport *dll = &table.v[dll_idx]; + MD_Node *dll_node = t_codec_push_node(ctx->arena, imports, str8f(ctx->arena, "dll_%llu", dll_idx)); + t_codec_push_field(ctx, dll_node, "name", dll->name); + t_codec_push_u64(ctx, dll_node, "import_address_table", dll->import_address_table_voff); + t_codec_push_u64(ctx, dll_node, "import_name_table", dll->import_name_table_voff); + t_codec_push_u64(ctx, dll_node, "timestamp", dll->time_stamp); + t_codec_push_u64(ctx, dll_node, "forwarder_chain", dll->forwarder_chain); + t_codec_push_u64(ctx, dll_node, "count", dll->import_count); + MD_Node *entries = t_codec_push_node(ctx->arena, dll_node, str8_lit("entries")); + for EachIndex(import_idx, dll->import_count) + { + PE_ParsedImport *import = &dll->imports[import_idx]; + MD_Node *import_node = t_codec_push_node(ctx->arena, entries, str8f(ctx->arena, "import_%llu", import_idx)); + if (import->type == PE_ParsedImport_Name) { + t_codec_push_field(ctx, import_node, "type", str8_lit("name")); + t_codec_push_field(ctx, import_node, "name", import->u.name.string); + t_codec_push_u64(ctx, import_node, "hint", import->u.name.hint); + } else if (import->type == PE_ParsedImport_Ordinal) { + t_codec_push_field(ctx, import_node, "type", str8_lit("ordinal")); + t_codec_push_u64(ctx, import_node, "ordinal", import->u.ordinal); + } + } + } + } + + if (pe.data_dir_count > PE_DataDirectoryIndex_DELAY_IMPORT && dim_1u64(pe.data_dir_vranges[PE_DataDirectoryIndex_DELAY_IMPORT]) != 0) { + PE_ParsedDelayImportTable table = pe_delay_imports_from_data(ctx->arena, pe.is_pe32, pe.section_count, section_table, data, pe.data_dir_franges[PE_DataDirectoryIndex_DELAY_IMPORT]); + MD_Node *imports = t_codec_push_node(ctx->arena, root, str8_lit("delay_imports")); + t_codec_push_u64(ctx, imports, "count", table.count); + for EachIndex(dll_idx, table.count) + { + PE_ParsedDelayDLLImport *dll = &table.v[dll_idx]; + MD_Node *dll_node = t_codec_push_node(ctx->arena, imports, str8f(ctx->arena, "dll_%llu", dll_idx)); + t_codec_push_field(ctx, dll_node, "name", dll->name); + t_codec_push_u64(ctx, dll_node, "attributes", dll->attributes); + t_codec_push_u64(ctx, dll_node, "module_handle", dll->module_handle_voff); + t_codec_push_u64(ctx, dll_node, "import_address_table", dll->iat_voff); + t_codec_push_u64(ctx, dll_node, "import_name_table", dll->name_table_voff); + t_codec_push_u64(ctx, dll_node, "bound_table", dll->bound_table_voff); + t_codec_push_u64(ctx, dll_node, "unload_table", dll->unload_table_voff); + t_codec_push_u64(ctx, dll_node, "timestamp", dll->time_stamp); + t_codec_push_u64(ctx, dll_node, "bound_count", dll->bound_table_count); + t_codec_push_u64(ctx, dll_node, "unload_count", dll->unload_table_count); + t_codec_push_u64(ctx, dll_node, "count", dll->import_count); + MD_Node *entries = t_codec_push_node(ctx->arena, dll_node, str8_lit("entries")); + for EachIndex(import_idx, dll->import_count) + { + PE_ParsedImport *import = &dll->imports[import_idx]; + MD_Node *import_node = t_codec_push_node(ctx->arena, entries, str8f(ctx->arena, "import_%llu", import_idx)); + if (import->type == PE_ParsedImport_Name) { + t_codec_push_field(ctx, import_node, "type", str8_lit("name")); + t_codec_push_field(ctx, import_node, "name", import->u.name.string); + t_codec_push_u64(ctx, import_node, "hint", import->u.name.hint); + } else if (import->type == PE_ParsedImport_Ordinal) { + t_codec_push_field(ctx, import_node, "type", str8_lit("ordinal")); + t_codec_push_u64(ctx, import_node, "ordinal", import->u.ordinal); + } + } + } + } + + *semantic_tree_out = root; + return ctx->result; +} + +typedef struct T_PdbMsfInfo T_PdbMsfInfo; +struct T_PdbMsfInfo +{ + U64 stream_count; + B32 *stream_present; +}; + +typedef struct T_PdbKindCount T_PdbKindCount; +struct T_PdbKindCount +{ + T_PdbKindCount *next; + U16 kind; + U64 count; +}; + +internal B32 +t_codec_pdb_range_is_valid(U64 off, U64 size, U64 cap) +{ + return off <= cap && size <= cap - off; +} + +internal B32 +t_codec_pdb_cstr_from_offset(String8 data, U64 off, String8 *string_out) +{ + if (off >= data.size) { return 0; } + String8 string = str8_cstring_capped(data.str + off, data.str + data.size); + if (string.size >= data.size - off) { return 0; } + *string_out = string; + return 1; +} + +internal String8 +t_codec_pdb_sym_kind_name(Arena *arena, CV_SymKind kind) +{ + String8 name = cv_string_from_symbol_kind(arena, kind); + if (name.size == 2) { name = str8f(arena, "S_0x%04x", kind); } + return name; +} + +internal String8 +t_codec_pdb_leaf_kind_name(Arena *arena, CV_LeafKind kind) +{ + String8 name = cv_string_from_leaf_name(arena, kind); + if (name.size == 3) { name = str8f(arena, "LF_0x%04x", kind); } + return name; +} + +internal B32 +t_codec_pdb_validate_msf(Arena *arena, String8 data, T_PdbMsfInfo *info, String8 *error) +{ + if (!msf_check_magic_70(data) || data.size < sizeof(MSF_Header70)) { + *error = str8_lit("file is not an MSF 7.0 PDB"); + return 0; + } + MSF_Header70 *header = (MSF_Header70 *)data.str; + U64 page_size = header->page_size; + U64 page_count = header->page_count; + if (page_size < MSF_MIN_PAGE_SIZE || page_size > MSF_MAX_PAGE_SIZE || !IsPow2OrZero(page_size)) { + *error = str8_lit("MSF has an invalid page size"); + return 0; + } + if (page_count == 0) { + *error = str8_lit("MSF has an invalid page count"); + return 0; + } + U64 directory_size = header->stream_table_size; + U64 directory_page_count = CeilIntegerDiv(directory_size, page_size); + U64 directory_map_size = directory_page_count * sizeof(U32); + if (directory_size < sizeof(U32) || directory_size > data.size || directory_map_size > page_size || header->root_pn >= page_count) { + *error = str8_lit("MSF stream directory map is invalid"); + return 0; + } + U64 map_off = (U64)header->root_pn * page_size; + if (!t_codec_pdb_range_is_valid(map_off, directory_map_size, data.size)) { + *error = str8_lit("MSF stream directory map is truncated"); + return 0; + } + U8 *directory = push_array_no_zero(arena, U8, directory_size); + for EachIndex(i, directory_page_count) + { + U32 pn = 0; + MemoryCopy(&pn, data.str + map_off + i * sizeof(pn), sizeof(pn)); + U64 copy_size = Min(page_size, directory_size - i * page_size); + if (pn >= page_count || !t_codec_pdb_range_is_valid((U64)pn * page_size, copy_size, data.size)) { + *error = str8f(arena, "MSF stream directory page %llu is invalid", i); + return 0; + } + MemoryCopy(directory + i * page_size, data.str + (U64)pn * page_size, copy_size); + } + U32 stream_count = 0; + MemoryCopy(&stream_count, directory, sizeof(stream_count)); + if (stream_count > (directory_size - sizeof(U32)) / sizeof(U32)) { + *error = str8_lit("MSF stream size table is truncated"); + return 0; + } + U64 index_cursor = sizeof(U32) + (U64)stream_count * sizeof(U32); + B32 *present = push_array(arena, B32, stream_count); + for EachIndex(i, stream_count) + { + U32 stream_size = 0; + MemoryCopy(&stream_size, directory + sizeof(U32) + i * sizeof(U32), sizeof(stream_size)); + if (stream_size == MSF_DELETED_STREAM_STAMP) { continue; } + present[i] = 1; + U64 stream_page_count = CeilIntegerDiv((U64)stream_size, page_size); + U64 indices_size = stream_page_count * sizeof(U32); + if (!t_codec_pdb_range_is_valid(index_cursor, indices_size, directory_size)) { + *error = str8f(arena, "MSF page list for stream %llu is truncated", i); + return 0; + } + for EachIndex(page_idx, stream_page_count) + { + U32 pn = 0; + MemoryCopy(&pn, directory + index_cursor + page_idx * sizeof(pn), sizeof(pn)); + U64 page_data_size = Min(page_size, (U64)stream_size - page_idx * page_size); + if (pn >= page_count || !t_codec_pdb_range_is_valid((U64)pn * page_size, page_data_size, data.size)) { + *error = str8f(arena, "MSF stream %llu references invalid page %u", i, pn); + return 0; + } + } + index_cursor += indices_size; + } + info->stream_count = stream_count; + info->stream_present = present; + return 1; +} + +internal B32 +t_codec_pdb_symbol_name(CV_Symbol symbol, String8 *name_out) +{ + U64 off = symbol.data.size; + switch (symbol.kind) + { + case CV_SymKind_OBJNAME: off = sizeof(CV_SymObjName); break; + case CV_SymKind_CONSTANT: + { + off = sizeof(CV_SymConstant); + if (off > symbol.data.size) { return 0; } + CV_NumericParsed numeric = {0}; + U64 numeric_size = cv_read_numeric(symbol.data, off, &numeric); + if (numeric_size == 0) { return 0; } + off += numeric_size; + } break; + case CV_SymKind_UDT: off = sizeof(CV_SymUDT); break; + case CV_SymKind_LDATA32: case CV_SymKind_GDATA32: + case CV_SymKind_LTHREAD32: case CV_SymKind_GTHREAD32: off = sizeof(CV_SymData32); break; + case CV_SymKind_PUB32: off = sizeof(CV_SymPub32); break; + case CV_SymKind_LPROC32: case CV_SymKind_GPROC32: + case CV_SymKind_LPROC32_ID: case CV_SymKind_GPROC32_ID: off = sizeof(CV_SymProc32); break; + case CV_SymKind_PROCREF: case CV_SymKind_LPROCREF: case CV_SymKind_DATAREF: off = sizeof(CV_SymRef2); break; + default: return 0; + } + return t_codec_pdb_cstr_from_offset(symbol.data, off, name_out); +} + +internal B32 +t_codec_pdb_read_symbol(String8 data, U64 off, U64 align, CV_Symbol *symbol_out, U64 *read_size_out, String8 *error) +{ + if (!t_codec_pdb_range_is_valid(off, sizeof(CV_SymbolHeader), data.size)) { + *error = str8_lit("CodeView symbol header is truncated"); + return 0; + } + CV_SymbolHeader header = {0}; + MemoryCopy(&header, data.str + off, sizeof(header)); + U64 raw_size = sizeof(CV_SymSize) + header.size; + if (header.size < sizeof(CV_SymKind) || !t_codec_pdb_range_is_valid(off, raw_size, data.size)) { + *error = str8_lit("CodeView symbol record is truncated"); + return 0; + } + U64 read_size = AlignPow2(raw_size, align); + if (read_size < raw_size || !t_codec_pdb_range_is_valid(off, read_size, data.size)) { + *error = str8_lit("CodeView symbol padding is truncated"); + return 0; + } + String8 record_data = str8(data.str + off + sizeof(CV_SymbolHeader), header.size - sizeof(CV_SymKind)); + U64 fixed_size = cv_header_struct_size_from_sym_kind(header.kind); + if (CV_IsProc32(header.kind)) { fixed_size = sizeof(CV_SymProc32); } + if (fixed_size > record_data.size) { + *error = str8_lit("CodeView symbol fixed fields are truncated"); + return 0; + } + CV_Symbol symbol = {0}; + cv_read_symbol(str8_skip(data, off), 0, align, &symbol); + String8 ignored = {0}; + if ((header.kind == CV_SymKind_CONSTANT || header.kind == CV_SymKind_UDT || header.kind == CV_SymKind_PUB32 || header.kind == CV_SymKind_PROCREF || + header.kind == CV_SymKind_LPROCREF || header.kind == CV_SymKind_DATAREF || CV_IsProc32(header.kind) || header.kind == CV_SymKind_LDATA32 || + header.kind == CV_SymKind_GDATA32) && !t_codec_pdb_symbol_name(symbol, &ignored)) { + *error = str8_lit("CodeView symbol name is truncated"); + return 0; + } + *symbol_out = symbol; + *read_size_out = read_size; + return 1; +} + +internal MD_Node * +t_codec_pdb_push_symbol(T_Context *ctx, MD_Node *parent, String8 key, CV_Symbol symbol, U64 off) +{ + MD_Node *node = t_codec_push_node(ctx->arena, parent, key); + t_codec_push_field(ctx, node, "kind", t_codec_pdb_sym_kind_name(ctx->arena, symbol.kind)); + t_codec_push_u64(ctx, node, "kind_value", symbol.kind); + t_codec_push_u64(ctx, node, "offset", off); + t_codec_push_u64(ctx, node, "data_size", symbol.data.size); + String8 name = {0}; + if (t_codec_pdb_symbol_name(symbol, &name)) { t_codec_push_field(ctx, node, "name", name); } + if (symbol.kind == CV_SymKind_PROCREF || symbol.kind == CV_SymKind_LPROCREF || symbol.kind == CV_SymKind_DATAREF) { + CV_SymRef2 value = {0}; MemoryCopy(&value, symbol.data.str, sizeof(value)); + t_codec_push_u64(ctx, node, "suc_name", value.suc_name); + t_codec_push_u64(ctx, node, "sym_off", value.sym_off); + t_codec_push_u64(ctx, node, "imod", value.imod); + } else if (symbol.kind == CV_SymKind_PUB32) { + CV_SymPub32 value = {0}; MemoryCopy(&value, symbol.data.str, sizeof(value)); + t_codec_push_u64(ctx, node, "flags", value.flags); + t_codec_push_u64(ctx, node, "section", value.sec); + t_codec_push_u64(ctx, node, "section_offset", value.off); + } else if (symbol.kind == CV_SymKind_LDATA32 || symbol.kind == CV_SymKind_GDATA32) { + CV_SymData32 value = {0}; MemoryCopy(&value, symbol.data.str, sizeof(value)); + t_codec_push_u64(ctx, node, "type_index", value.itype); + t_codec_push_u64(ctx, node, "section", value.sec); + t_codec_push_u64(ctx, node, "section_offset", value.off); + } else if (CV_IsProc32(symbol.kind)) { + CV_SymProc32 value = {0}; MemoryCopy(&value, symbol.data.str, sizeof(value)); + t_codec_push_u64(ctx, node, "type_index", value.itype); + t_codec_push_u64(ctx, node, "section", value.sec); + t_codec_push_u64(ctx, node, "section_offset", value.off); + t_codec_push_u64(ctx, node, "length", value.len); + } + return node; +} + +internal B32 +t_codec_pdb_push_symbol_stream(T_Context *ctx, MD_Node *parent, String8 data, U64 align, String8 *error) +{ + MD_Node *symbols = t_codec_push_node(ctx->arena, parent, str8_lit("symbols")); + MD_Node *by_name = t_codec_push_node(ctx->arena, parent, str8_lit("by_name")); + T_PdbKindCount *first_count = 0; + U64 count = 0; + U64 proc_stub_count = 0; + U64 public_or_proc_ref_count = 0; + for (U64 cursor = 0; cursor < data.size; count += 1) + { + CV_Symbol symbol = {0}; U64 read_size = 0; + if (!t_codec_pdb_read_symbol(data, cursor, align, &symbol, &read_size, error)) { return 0; } + MD_Node *symbol_node = t_codec_pdb_push_symbol(ctx, symbols, str8f(ctx->arena, "symbol_%llu", count), symbol, cursor); + String8 name = {0}; + if (t_codec_pdb_symbol_name(symbol, &name) && md_node_is_nil(md_child_from_string(by_name, name, 0))) { + MD_Node *named = t_codec_push_node(ctx->arena, by_name, name); + for MD_EachNode(field, symbol_node->first) { MD_Node *copy = t_codec_push_node(ctx->arena, named, field->string); t_codec_push_node(ctx->arena, copy, field->first->string); } + } + T_PdbKindCount *kind_count = first_count; + for (; kind_count != 0 && kind_count->kind != symbol.kind; kind_count = kind_count->next) {} + if (kind_count == 0) { kind_count = push_array(ctx->arena, T_PdbKindCount, 1); kind_count->kind = symbol.kind; kind_count->next = first_count; first_count = kind_count; } + kind_count->count += 1; + proc_stub_count += symbol.kind == CV_SymKind_LPROC32 || symbol.kind == CV_SymKind_END; + public_or_proc_ref_count += symbol.kind == CV_SymKind_PUB32 || symbol.kind == CV_SymKind_LPROCREF; + cursor += read_size; + } + t_codec_push_u64(ctx, parent, "size", data.size); + t_codec_push_u64(ctx, parent, "count", count); + t_codec_push_u64(ctx, parent, "proc_stub_symbol_count", proc_stub_count); + t_codec_push_u64(ctx, parent, "non_proc_stub_symbol_count", count - proc_stub_count); + t_codec_push_u64(ctx, parent, "public_or_proc_ref_symbol_count", public_or_proc_ref_count); + t_codec_push_u64(ctx, parent, "non_public_or_proc_ref_symbol_count", count - public_or_proc_ref_count); + MD_Node *kind_counts = t_codec_push_node(ctx->arena, parent, str8_lit("kind_counts")); + MD_Node *kind_set = t_codec_push_node(ctx->arena, parent, str8_lit("kind_set")); + for (T_PdbKindCount *item = first_count; item != 0; item = item->next) { + String8 name = t_codec_pdb_sym_kind_name(ctx->arena, item->kind); + t_codec_push_u64_s8(ctx, kind_counts, name, item->count); + t_codec_push_node(ctx->arena, kind_set, name); + } + return 1; +} + +internal B32 +t_codec_pdb_push_tpi(T_Context *ctx, MD_Node *root, char *name, MSF_Parsed *msf, MSF_StreamNumber sn, String8 *error) +{ + MD_Node *node = t_codec_push_node(ctx->arena, root, str8_cstring(name)); + String8 data = msf_data_from_stream(msf, sn); + t_codec_push_u64(ctx, node, "stream_index", sn); + t_codec_push_u64(ctx, node, "size", data.size); + if (data.size < sizeof(PDB_TpiHeader)) { *error = str8f(ctx->arena, "%s stream header is truncated", name); return 0; } + PDB_TpiHeader *header = (PDB_TpiHeader *)data.str; + if (header->version != PDB_TpiVersion_IMPV80 || header->header_size < sizeof(*header) || header->header_size > data.size || + header->leaf_data_size > data.size - header->header_size || header->ti_hi < header->ti_lo) { + *error = str8f(ctx->arena, "%s stream header is invalid", name); return 0; + } + U64 expected_count = (U64)header->ti_hi - header->ti_lo; + String8 leaf_data = str8(data.str + header->header_size, header->leaf_data_size); + U64 parsed_count = 0; + for (U64 cursor = 0; cursor < leaf_data.size; parsed_count += 1) { + CV_Leaf leaf = {0}; U64 read_size = cv_read_leaf(str8_skip(leaf_data, cursor), 0, PDB_LEAF_ALIGN, &leaf); + if (read_size == 0 || cv_header_struct_size_from_leaf_kind(leaf.kind) > leaf.data.size) { + *error = str8f(ctx->arena, "%s leaf %llu is malformed", name, parsed_count); return 0; + } + cursor += read_size; + } + if (parsed_count != expected_count) { *error = str8f(ctx->arena, "%s type index range does not match its leaf count", name); return 0; } + PDB_TpiParsed *tpi = pdb_tpi_from_data(ctx->arena, data); + CV_DebugT debug_t = cv_debug_t_from_data(ctx->arena, pdb_leaf_data_from_tpi(tpi), PDB_LEAF_ALIGN); + if (debug_t.count != parsed_count) { *error = str8f(ctx->arena, "%s CodeView leaf parse is inconsistent", name); return 0; } + t_codec_push_u64(ctx, node, "header_size", header->header_size); + t_codec_push_u64(ctx, node, "index_first", tpi->itype_first); + t_codec_push_u64(ctx, node, "index_opl", tpi->itype_opl); + t_codec_push_u64(ctx, node, "leaf_count", parsed_count); + t_codec_push_field(ctx, node, "header_only", parsed_count == 0 && data.size == header->header_size ? str8_lit("true") : str8_lit("false")); + MD_Node *leaves = t_codec_push_node(ctx->arena, node, str8_lit("leaves")); + for EachIndex(i, debug_t.count) + { + CV_Leaf leaf = cv_debug_t_get_leaf(&debug_t, i); + MD_Node *leaf_node = t_codec_push_node(ctx->arena, leaves, str8f(ctx->arena, "leaf_%llu", i)); + t_codec_push_field(ctx, leaf_node, "kind", t_codec_pdb_leaf_kind_name(ctx->arena, leaf.kind)); + t_codec_push_u64(ctx, leaf_node, "kind_value", leaf.kind); + t_codec_push_u64(ctx, leaf_node, "type_index", tpi->itype_first + i); + t_codec_push_u64(ctx, leaf_node, "data_size", leaf.data.size); + if (leaf.kind == CV_LeafKind_POINTER) { + CV_LeafPointer value = {0}; MemoryCopy(&value, leaf.data.str, sizeof(value)); + t_codec_push_u64(ctx, leaf_node, "type", value.itype); t_codec_push_u64(ctx, leaf_node, "attributes", value.attribs); + } else if (leaf.kind == CV_LeafKind_PROCEDURE) { + CV_LeafProcedure value = {0}; MemoryCopy(&value, leaf.data.str, sizeof(value)); + t_codec_push_u64(ctx, leaf_node, "return_type", value.ret_itype); t_codec_push_u64(ctx, leaf_node, "call_kind", value.call_kind); + t_codec_push_u64(ctx, leaf_node, "attributes", value.attribs); t_codec_push_u64(ctx, leaf_node, "argument_count", value.arg_count); + t_codec_push_u64(ctx, leaf_node, "argument_list_type", value.arg_itype); + } + } + return 1; +} + +internal B32 +t_codec_pdb_validate_gsi(T_Context *ctx, String8 data, String8 symbols, String8 *error) +{ + if (data.size < sizeof(PDB_GsiHeader)) { *error = str8_lit("GSI header is truncated"); return 0; } + PDB_GsiHeader *header = (PDB_GsiHeader *)data.str; + U64 bitmap_size = CeilIntegerDiv(4097, 32) * sizeof(U32); + U64 hash_off = sizeof(*header); + U64 bitmap_off = hash_off + header->hash_record_arr_size; + U64 total_size = bitmap_off + header->bucket_data_size; + if (header->signature != PDB_GsiSignature_Basic || header->version != PDB_GsiVersion_V70 || header->hash_record_arr_size % sizeof(PDB_GsiHashRecord) != 0 || + header->bucket_data_size < bitmap_size || total_size != data.size) { + *error = str8_lit("GSI header or ranges are invalid"); return 0; + } + U64 hash_count = header->hash_record_arr_size / sizeof(PDB_GsiHashRecord); + U64 packed_count = (header->bucket_data_size - bitmap_size) / sizeof(U32); + U64 occupied_count = 0; + for EachIndex(i, CeilIntegerDiv(4097, 32)) { U32 bits = 0; MemoryCopy(&bits, data.str + bitmap_off + i * sizeof(bits), sizeof(bits)); occupied_count += count_bits_set32(bits); } + if ((header->bucket_data_size - bitmap_size) % sizeof(U32) != 0 || occupied_count != packed_count) { *error = str8_lit("GSI bucket bitmap and offset array disagree"); return 0; } + U64 offsets_off = bitmap_off + bitmap_size; + U32 previous = 0; + for EachIndex(i, packed_count) { + U32 value = 0; MemoryCopy(&value, data.str + offsets_off + i * sizeof(value), sizeof(value)); + if ((i == 0 && hash_count != 0 && value != 0) || value % sizeof(PDB_GsiHashRecordOffsetCalc) != 0 || + value / sizeof(PDB_GsiHashRecordOffsetCalc) > hash_count || (i != 0 && value < previous)) { + *error = str8_lit("GSI bucket offset is invalid"); return 0; + } + previous = value; + } + if (hash_count != 0 && packed_count == 0) { *error = str8_lit("GSI has hash records but no occupied bucket"); return 0; } + for EachIndex(i, hash_count) { + PDB_GsiHashRecord record = {0}; MemoryCopy(&record, data.str + hash_off + i * sizeof(record), sizeof(record)); + if (record.symbol_off == 0) { *error = str8_lit("GSI references the null symbol offset"); return 0; } + CV_Symbol symbol = {0}; U64 read_size = 0; + if (!t_codec_pdb_read_symbol(symbols, record.symbol_off - 1, 1, &symbol, &read_size, error)) { return 0; } + String8 name = {0}; + if (!t_codec_pdb_symbol_name(symbol, &name)) { *error = str8_lit("GSI references a symbol without a valid name"); return 0; } + } + return 1; +} + +internal B32 +t_codec_pdb_push_gsi(T_Context *ctx, MD_Node *root, char *name, MSF_Parsed *msf, MSF_StreamNumber sn, String8 symbols, B32 is_psi, String8 *error) +{ + MD_Node *node = t_codec_push_node(ctx->arena, root, str8_cstring(name)); + t_codec_push_u64(ctx, node, "stream_index", sn); + if (sn >= msf->stream_count) { t_codec_push_field(ctx, node, "present", str8_lit("false")); return 1; } + String8 stream = msf_data_from_stream(msf, sn); + t_codec_push_field(ctx, node, "present", str8_lit("true")); + t_codec_push_u64(ctx, node, "size", stream.size); + String8 gsi_data = stream; + if (is_psi) { + if (stream.size < sizeof(PDB_PsiHeader)) { *error = str8_lit("PSI header is truncated"); return 0; } + PDB_PsiHeader *header = (PDB_PsiHeader *)stream.str; + U64 ranges_size = sizeof(*header); + U64 range_sizes[] = {header->sym_hash_size, header->addr_map_size}; + for EachElement(i, range_sizes) { if (range_sizes[i] > stream.size - Min(ranges_size, stream.size)) { *error = str8_lit("PSI ranges are truncated"); return 0; } ranges_size += range_sizes[i]; } + if (header->thunk_count != 0 && header->thunk_size > (stream.size - Min(ranges_size, stream.size)) / header->thunk_count) { *error = str8_lit("PSI thunk table is truncated"); return 0; } + ranges_size += (U64)header->thunk_count * header->thunk_size; + if ((U64)header->sec_count > (stream.size - Min(ranges_size, stream.size)) / sizeof(U32)) { *error = str8_lit("PSI section table is truncated"); return 0; } + t_codec_push_u64(ctx, node, "address_map_size", header->addr_map_size); + t_codec_push_u64(ctx, node, "thunk_count", header->thunk_count); + t_codec_push_u64(ctx, node, "section_count", header->sec_count); + gsi_data = str8(stream.str + sizeof(*header), header->sym_hash_size); + } + if (!t_codec_pdb_validate_gsi(ctx, gsi_data, symbols, error)) { return 0; } + PDB_GsiParsed *gsi = pdb_gsi_from_data(ctx->arena, gsi_data); + MD_Node *indexed = t_codec_push_node(ctx->arena, node, str8_lit("symbols")); + MD_Node *kind_counts = t_codec_push_node(ctx->arena, node, str8_lit("kind_counts")); + T_PdbKindCount *first_count = 0; + U64 count = 0; + for EachElement(bucket_idx, gsi->buckets) { + PDB_GsiBucket bucket = gsi->buckets[bucket_idx]; + for EachIndex(i, bucket.count) { + U64 off = bucket.offs[i]; CV_Symbol symbol = {0}; U64 read_size = 0; + if (!t_codec_pdb_read_symbol(symbols, off, 1, &symbol, &read_size, error)) { return 0; } + String8 symbol_name = {0}; t_codec_pdb_symbol_name(symbol, &symbol_name); + if (pdb_gsi_symbol_from_string(gsi, symbols, symbol_name) >= symbols.size) { *error = str8_lit("GSI symbol is not queryable by name"); return 0; } + if (md_node_is_nil(md_child_from_string(indexed, symbol_name, 0))) { t_codec_pdb_push_symbol(ctx, indexed, symbol_name, symbol, off); } + T_PdbKindCount *item = first_count; for (; item != 0 && item->kind != symbol.kind; item = item->next) {} + if (item == 0) { item = push_array(ctx->arena, T_PdbKindCount, 1); item->kind = symbol.kind; item->next = first_count; first_count = item; } + item->count += 1; count += 1; + } + } + for (T_PdbKindCount *item = first_count; item != 0; item = item->next) { String8 kind = t_codec_pdb_sym_kind_name(ctx->arena, item->kind); t_codec_push_u64_s8(ctx, kind_counts, kind, item->count); } + t_codec_push_u64(ctx, node, "indexed_symbol_count", count); + return 1; +} + +internal B32 +t_codec_pdb_push_dbi(T_Context *ctx, MD_Node *root, MSF_Parsed *msf, String8 *error) +{ + String8 data = msf_data_from_stream(msf, PDB_FixedStream_Dbi); + MD_Node *node = t_codec_push_node(ctx->arena, root, str8_lit("dbi")); + t_codec_push_u64(ctx, node, "stream_index", PDB_FixedStream_Dbi); + t_codec_push_u64(ctx, node, "size", data.size); + if (data.size < sizeof(PDB_DbiHeader)) { *error = str8_lit("DBI header is truncated"); return 0; } + PDB_DbiHeader *header = (PDB_DbiHeader *)data.str; + U64 ranges_size = header->module_info_size; + U32 sizes[] = {header->sec_con_size, header->sec_map_size, header->file_info_size, header->tsm_size, header->ec_info_size, header->dbg_header_size}; + for EachElement(i, sizes) { if (sizes[i] > data.size - Min(ranges_size, data.size)) { *error = str8_lit("DBI substream ranges are truncated"); return 0; } ranges_size += sizes[i]; } + if (header->sig != PDB_DbiHeaderSignature_V1 || ranges_size > data.size - sizeof(*header)) { *error = str8_lit("DBI header or substream ranges are invalid"); return 0; } + PDB_DbiParsed *dbi = pdb_dbi_from_data(ctx->arena, data); + t_codec_push_field(ctx, node, "machine", coff_string_from_machine_type(dbi->machine_type)); + t_codec_push_u64(ctx, node, "machine_value", dbi->machine_type); + t_codec_push_u64(ctx, node, "gsi_stream", dbi->gsi_sn); t_codec_push_u64(ctx, node, "psi_stream", dbi->psi_sn); t_codec_push_u64(ctx, node, "symbol_stream", dbi->sym_sn); + String8 module_info = pdb_data_from_dbi_range(dbi, PDB_DbiRange_ModuleInfo); + for (U64 cursor = 0; cursor < module_info.size;) { + if (!t_codec_pdb_range_is_valid(cursor, sizeof(PDB_DbiCompUnitHeader), module_info.size)) { *error = str8_lit("DBI module header is truncated"); return 0; } + PDB_DbiCompUnitHeader *mod = (PDB_DbiCompUnitHeader *)(module_info.str + cursor); + U64 name_off = cursor + sizeof(*mod); String8 first = {0}, second = {0}; + if (!t_codec_pdb_cstr_from_offset(module_info, name_off, &first) || !t_codec_pdb_cstr_from_offset(module_info, name_off + first.size + 1, &second)) { *error = str8_lit("DBI module name is truncated"); return 0; } + cursor = AlignPow2(name_off + first.size + 1 + second.size + 1, 4); + U64 ranges = (U64)mod->symbols_size + mod->c11_lines_size + mod->c13_lines_size; + if ((ranges != 0 && mod->sn >= msf->stream_count) || (mod->sn < msf->stream_count && ranges > msf_data_from_stream(msf, mod->sn).size)) { + *error = str8_lit("DBI module stream ranges are invalid"); return 0; + } + } + PDB_CompUnitArray *modules = pdb_comp_unit_array_from_data(ctx->arena, module_info); + t_codec_push_u64(ctx, node, "module_count", modules->count); + MD_Node *module_nodes = t_codec_push_node(ctx->arena, node, str8_lit("modules")); + for EachIndex(i, modules->count) { + PDB_CompUnit *mod = modules->units[i]; + MD_Node *mod_node = t_codec_push_node(ctx->arena, module_nodes, str8f(ctx->arena, "module_%llu", i)); + t_codec_push_field(ctx, mod_node, "object_name", mod->obj_name); + t_codec_push_field(ctx, mod_node, "object_file_name", str8_skip_last_slash(mod->obj_name)); + t_codec_push_field(ctx, mod_node, "group_name", mod->group_name); + t_codec_push_u64(ctx, mod_node, "stream_index", mod->sn); + String8 sym_data = pdb_data_from_unit_range(msf, mod, PDB_DbiCompUnitRange_Symbols); + String8 c11_data = pdb_data_from_unit_range(msf, mod, PDB_DbiCompUnitRange_C11); + String8 c13_data = pdb_data_from_unit_range(msf, mod, PDB_DbiCompUnitRange_C13); + t_codec_push_u64(ctx, mod_node, "symbol_size", sym_data.size); t_codec_push_u64(ctx, mod_node, "c11_size", c11_data.size); t_codec_push_u64(ctx, mod_node, "c13_size", c13_data.size); + MD_Node *module_symbols = t_codec_push_node(ctx->arena, mod_node, str8_lit("module_symbols")); + if (!t_codec_pdb_push_symbol_stream(ctx, module_symbols, sym_data, PDB_SYMBOL_ALIGN, error)) { return 0; } + } + MD_Node *global_symbols = t_codec_push_node(ctx->arena, root, str8_lit("global_symbols")); + t_codec_push_u64(ctx, global_symbols, "stream_index", dbi->sym_sn); + String8 symbols = {0}; + if (dbi->sym_sn < msf->stream_count) { + t_codec_push_field(ctx, global_symbols, "present", str8_lit("true")); + symbols = msf_data_from_stream(msf, dbi->sym_sn); + if (!t_codec_pdb_push_symbol_stream(ctx, global_symbols, symbols, PDB_SYMBOL_ALIGN, error)) { return 0; } + } else { + t_codec_push_field(ctx, global_symbols, "present", str8_lit("false")); + } + if (!t_codec_pdb_push_gsi(ctx, root, "gsi", msf, dbi->gsi_sn, symbols, 0, error)) { return 0; } + if (!t_codec_pdb_push_gsi(ctx, root, "psi", msf, dbi->psi_sn, symbols, 1, error)) { return 0; } + return 1; +} + +internal T_Result +t_codec_pdb_decode(T_Context *ctx, String8 data, MD_Node **semantic_tree_out) +{ + T_PdbMsfInfo msf_info = {0}; String8 error = {0}; + if (!t_codec_pdb_validate_msf(ctx->arena, data, &msf_info, &error)) { return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_lit("expect_pdb"), "%S", error); } + MSF_Parsed *msf = msf_parsed_from_data(ctx->arena, data); + if (msf == 0 || msf->stream_count != msf_info.stream_count || msf->stream_count <= PDB_FixedStream_Ipi) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_lit("expect_pdb"), "PDB is missing fixed streams"); + } + MD_Node *root = t_codec_push_node(ctx->arena, 0, str8_lit("pdb")); + t_codec_push_u64(ctx, root, "file_size", data.size); t_codec_push_u64(ctx, root, "page_size", msf->page_size); t_codec_push_u64(ctx, root, "page_count", msf->page_count); + t_codec_push_u64(ctx, root, "stream_count", msf->stream_count); + MD_Node *streams = t_codec_push_node(ctx->arena, root, str8_lit("streams")); + for EachIndex(i, msf->stream_count) { + MD_Node *stream = t_codec_push_node(ctx->arena, streams, str8f(ctx->arena, "stream_%llu", i)); + t_codec_push_field(ctx, stream, "present", msf_info.stream_present[i] ? str8_lit("true") : str8_lit("false")); + t_codec_push_u64(ctx, stream, "size", msf->streams[i].size); + } + MD_Node *fixed = t_codec_push_node(ctx->arena, root, str8_lit("fixed_streams")); + struct { char *name; MSF_StreamNumber sn; } fixed_defs[] = {{"info", PDB_FixedStream_Info}, {"tpi", PDB_FixedStream_Tpi}, {"dbi", PDB_FixedStream_Dbi}, {"ipi", PDB_FixedStream_Ipi}}; + for EachElement(i, fixed_defs) { + MD_Node *stream = t_codec_push_node(ctx->arena, fixed, str8_cstring(fixed_defs[i].name)); + t_codec_push_u64(ctx, stream, "index", fixed_defs[i].sn); t_codec_push_field(ctx, stream, "present", msf_info.stream_present[fixed_defs[i].sn] ? str8_lit("true") : str8_lit("false")); + t_codec_push_u64(ctx, stream, "size", msf->streams[fixed_defs[i].sn].size); + if (!msf_info.stream_present[fixed_defs[i].sn]) { return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_lit("expect_pdb"), "PDB fixed stream %s is absent", fixed_defs[i].name); } + } + if (!t_codec_pdb_push_tpi(ctx, root, "tpi", msf, PDB_FixedStream_Tpi, &error) || !t_codec_pdb_push_tpi(ctx, root, "ipi", msf, PDB_FixedStream_Ipi, &error) || + !t_codec_pdb_push_dbi(ctx, root, msf, &error)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, &md_nil_node, str8_lit("expect_pdb"), "%S", error); + } + *semantic_tree_out = root; + return ctx->result; +} + +internal T_Result +t_codec_run_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"path", "args", "expect_exit", "timeout_ms", "stdout_matches", "stderr_matches", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + t_codec_validate_scalar(ctx, arguments, "path", 1); + for (U64 i = 1; allowed[i] != 0; i += 1) { t_codec_validate_scalar(ctx, arguments, allowed[i], 0); } + String8 path = t_codec_scalar(t_codec_child(arguments, "path")); + if (path.size == 0 || str8_find_needle(path, 0, str8_lit("/"), 0) < path.size || str8_find_needle(path, 0, str8_lit("\\"), 0) < path.size || str8_find_needle(path, 0, str8_lit(":"), 0) < path.size) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "path"), "run path must be a relative file name"); + } + String8 expected = t_codec_scalar(t_codec_child(arguments, "expect_exit")); + U64 value = 0; + if (expected.size != 0 && !str8_matchi(expected, str8_lit("nonzero")) && !str8_matchi(expected, str8_lit("any")) && !try_u64_from_str8_c_rules(expected, &value)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "expect_exit"), "invalid expected process exit code"); + } + MD_Node *timeout = t_codec_child(arguments, "timeout_ms"); + if (!md_node_is_nil(timeout) && !try_u64_from_str8_c_rules(t_codec_scalar(timeout), &value)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, timeout, "timeout_ms must be an integer"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_run_execute(T_Context *ctx, MD_Node *arguments) +{ + String8 path = t_make_file_path(ctx->arena, t_codec_scalar(t_codec_child(arguments, "path"))); + U64 timeout_ms = max_U64; + MD_Node *timeout = t_codec_child(arguments, "timeout_ms"); + if (!md_node_is_nil(timeout)) { try_u64_from_str8_c_rules(t_codec_scalar(timeout), &timeout_ms); } + U64 timeout_us = timeout_ms == max_U64 || timeout_ms > max_U64 / 1000 ? max_U64 : timeout_ms * 1000; + if (!t_invoke(path, t_codec_scalar(t_codec_child(arguments, "args")), timeout_us)) { + return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("run"), "unable to launch '%S'", path); + } + if (g_last_exit_code == max_U64) { return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("run"), "process did not exit before timeout"); } + String8 expected = t_codec_scalar(t_codec_child(arguments, "expect_exit")); + B32 any = str8_matchi(expected, str8_lit("any")); + B32 nonzero = str8_matchi(expected, str8_lit("nonzero")); + U64 expected_code = 0; + if (expected.size != 0 && !any && !nonzero) { try_u64_from_str8_c_rules(expected, &expected_code); } + if (!any && (nonzero ? g_last_exit_code == 0 : g_last_exit_code != expected_code)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("run"), "process exited with %llu, expected %S\n%S", g_last_exit_code, + nonzero ? str8_lit("nonzero") : str8f(ctx->arena, "%llu", expected_code), g_errors); + } + StringMatchFlags flags = StringMatchFlag_CaseInsensitive | StringMatchFlag_SlashInsensitive; + String8 stdout_pattern = t_codec_scalar(t_codec_child(arguments, "stdout_matches")); + String8 stderr_pattern = t_codec_scalar(t_codec_child(arguments, "stderr_matches")); + if (stdout_pattern.size != 0 && !str8_match_wildcard(g_output, stdout_pattern, flags)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("run"), "process stdout does not match '%S'\n%S", stdout_pattern, g_output); + } + if (stderr_pattern.size != 0 && !str8_match_wildcard(g_errors, stderr_pattern, flags)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("run"), "process stderr does not match '%S'\n%S", stderr_pattern, g_errors); + } + return ctx->result; +} + +internal B32 +t_codec_is_safe_file_name(String8 name) +{ + return name.size != 0 && str8_find_needle(name, 0, str8_lit("/"), 0) >= name.size && str8_find_needle(name, 0, str8_lit("\\"), 0) >= name.size && + str8_find_needle(name, 0, str8_lit(":"), 0) >= name.size; +} + +internal T_Result +t_codec_clang_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"input", "output", "args", "expect_exit", "timeout_ms", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + for (U64 i = 0; i < 3; i += 1) { t_codec_validate_scalar(ctx, arguments, allowed[i], 1); } + t_codec_validate_scalar(ctx, arguments, "expect_exit", 0); + t_codec_validate_scalar(ctx, arguments, "timeout_ms", 0); + for (U64 i = 0; i < 2; i += 1) { + if (!t_codec_is_safe_file_name(t_codec_scalar(t_codec_child(arguments, allowed[i])))) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, allowed[i]), "clang %s must be a relative file name", allowed[i]); + } + } + U64 value = 0; + MD_Node *expect_exit = t_codec_child(arguments, "expect_exit"); + if (!md_node_is_nil(expect_exit) && !try_u64_from_str8_c_rules(t_codec_scalar(expect_exit), &value)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, expect_exit, "clang expect_exit must be an integer"); + } + MD_Node *timeout = t_codec_child(arguments, "timeout_ms"); + if (!md_node_is_nil(timeout) && !try_u64_from_str8_c_rules(t_codec_scalar(timeout), &value)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, timeout, "clang timeout_ms must be an integer"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_clang_execute(T_Context *ctx, MD_Node *arguments) +{ + String8 input = t_make_file_path(ctx->arena, t_codec_scalar(t_codec_child(arguments, "input"))); + String8 output = t_make_file_path(ctx->arena, t_codec_scalar(t_codec_child(arguments, "output"))); + String8 args = str8f(ctx->arena, "%S -o %S %S", input, output, t_codec_scalar(t_codec_child(arguments, "args"))); + U64 timeout_ms = max_U64; + MD_Node *timeout = t_codec_child(arguments, "timeout_ms"); + if (!md_node_is_nil(timeout)) { try_u64_from_str8_c_rules(t_codec_scalar(timeout), &timeout_ms); } + U64 timeout_us = timeout_ms == max_U64 || timeout_ms > max_U64 / 1000 ? max_U64 : timeout_ms * 1000; + if (!t_invoke(t_clang_path(), args, timeout_us)) { return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("clang"), "unable to launch Clang"); } + U64 expected = 0; + MD_Node *expect_exit = t_codec_child(arguments, "expect_exit"); + if (!md_node_is_nil(expect_exit)) { try_u64_from_str8_c_rules(t_codec_scalar(expect_exit), &expected); } + if (g_last_exit_code != expected) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("clang"), "Clang exited with %llu, expected %llu\n%S", g_last_exit_code, expected, g_errors); + } + String8 data = t_read_file(ctx->arena, t_codec_scalar(t_codec_child(arguments, "output"))); + if (data.size == 0) { return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("clang"), "Clang did not produce '%S'", output); } + return ctx->result; +} + +internal T_Result +t_codec_expect_pe_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"artifact", "expected", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + t_codec_validate_scalar(ctx, arguments, "artifact", 1); + MD_Node *artifact_node = t_codec_child(arguments, "artifact"); + T_Artifact *artifact = t_artifact_from_name(ctx->run, t_codec_scalar(artifact_node)); + if (artifact == 0 || !str8_match(artifact->codec->kind, str8_lit("pe"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, artifact_node, "expect_pe requires a PE artifact"); + } + MD_Node *expected = t_codec_child(arguments, "expected"); + if (md_node_is_nil(expected) || md_node_is_nil(expected->first) || !md_node_is_nil(expected->first->next) || !str8_match(expected->first->string, str8_lit("pe"), 0)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, expected, "expect_pe requires one PE expectation tree"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_expect_pe_execute(T_Context *ctx, MD_Node *arguments) +{ + T_Artifact *artifact = t_artifact_from_name(ctx, t_codec_scalar(t_codec_child(arguments, "artifact"))); + MD_Node *actual = 0; + T_Result result = artifact->codec->decode(ctx, artifact, &actual); + if (!t_result_is_ok(result)) { return result; } + MD_Node *expected = t_codec_child(arguments, "expected")->first; + return t_semantic_match(ctx, expected, actual); +} + +internal T_Result +t_codec_expect_pdb_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"path", "expected", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + t_codec_validate_scalar(ctx, arguments, "path", 1); + String8 path = t_codec_scalar(t_codec_child(arguments, "path")); + if (path.size == 0 || str8_match(path, str8_lit("."), 0) || str8_match(path, str8_lit(".."), 0) || + str8_find_needle(path, 0, str8_lit("/"), 0) < path.size || str8_find_needle(path, 0, str8_lit("\\"), 0) < path.size || + str8_find_needle(path, 0, str8_lit(":"), 0) < path.size) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "path"), "expect_pdb path must be a safe relative file name"); + } + MD_Node *expected = t_codec_child(arguments, "expected"); + if (md_node_is_nil(expected) || md_node_is_nil(expected->first) || !md_node_is_nil(expected->first->next) || !str8_match(expected->first->string, str8_lit("pdb"), 0)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, expected, "expect_pdb requires one PDB expectation tree"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_expect_pdb_execute(T_Context *ctx, MD_Node *arguments) +{ + String8 path = t_codec_scalar(t_codec_child(arguments, "path")); + String8 data = t_read_file(ctx->arena, path); + if (data.size == 0 && !file_path_exists(t_make_file_path(ctx->arena, path))) { + return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("expect_pdb"), "file '%S' does not exist", path); + } + MD_Node *actual = 0; + T_Result result = t_codec_pdb_decode(ctx, data, &actual); + if (!t_result_is_ok(result)) { return result; } + return t_semantic_match(ctx, t_codec_child(arguments, "expected")->first, actual); +} + +internal T_Result +t_codec_expect_coff_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"artifact", "expected", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + t_codec_validate_scalar(ctx, arguments, "artifact", 1); + MD_Node *artifact_node = t_codec_child(arguments, "artifact"); + T_Artifact *artifact = t_artifact_from_name(ctx->run, t_codec_scalar(artifact_node)); + if (artifact == 0 || !str8_match(artifact->codec->kind, str8_lit("coff"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, artifact_node, "expect_coff requires a COFF artifact"); + } + MD_Node *expected = t_codec_child(arguments, "expected"); + if (md_node_is_nil(expected) || md_node_is_nil(expected->first) || !md_node_is_nil(expected->first->next) || !str8_match(expected->first->string, str8_lit("coff"), 0)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, expected, "expect_coff requires one COFF expectation tree"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_expect_coff_execute(T_Context *ctx, MD_Node *arguments) +{ + T_Artifact *artifact = t_artifact_from_name(ctx, t_codec_scalar(t_codec_child(arguments, "artifact"))); + MD_Node *actual = 0; + T_Result result = artifact->codec->decode(ctx, artifact, &actual); + if (!t_result_is_ok(result)) { return result; } + MD_Node *expected = t_codec_child(arguments, "expected")->first; + return t_semantic_match(ctx, expected, actual); +} + +internal T_Result +t_codec_expect_file_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"path", "equals_artifact", "contains", "nonempty", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + t_codec_validate_scalar(ctx, arguments, "path", 1); + t_codec_validate_scalar(ctx, arguments, "equals_artifact", 0); + t_codec_validate_scalar(ctx, arguments, "contains", 0); + t_codec_validate_scalar(ctx, arguments, "nonempty", 0); + String8 path = t_codec_scalar(t_codec_child(arguments, "path")); + if (str8_find_needle(path, 0, str8_lit("/"), 0) < path.size || str8_find_needle(path, 0, str8_lit("\\"), 0) < path.size || str8_find_needle(path, 0, str8_lit(":"), 0) < path.size) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "path"), "expect_file path must be a relative file name"); + } + String8 artifact_name = t_codec_scalar(t_codec_child(arguments, "equals_artifact")); + if (artifact_name.size != 0 && t_artifact_from_name(ctx->run, artifact_name) == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "equals_artifact"), "unknown artifact '%S'", artifact_name); + } + B32 nonempty = 0; + MD_Node *nonempty_node = t_codec_child(arguments, "nonempty"); + if (!md_node_is_nil(nonempty_node) && !t_bool_from_scalar(nonempty_node, &nonempty)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, nonempty_node, "nonempty must be true or false"); + } + if (artifact_name.size == 0 && md_node_is_nil(t_codec_child(arguments, "contains")) && md_node_is_nil(nonempty_node)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments, "expect_file requires equals_artifact, contains, or nonempty"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_expect_file_execute(T_Context *ctx, MD_Node *arguments) +{ + String8 path = t_codec_scalar(t_codec_child(arguments, "path")); + String8 data = t_read_file(ctx->arena, path); + if (data.size == 0 && !file_path_exists(t_make_file_path(ctx->arena, path))) { + return t_context_errorf(ctx, T_ResultCode_IoError, arguments, str8_lit("expect_file"), "file '%S' does not exist", path); + } + String8 artifact_name = t_codec_scalar(t_codec_child(arguments, "equals_artifact")); + if (artifact_name.size != 0) { + T_Artifact *artifact = t_artifact_from_name(ctx, artifact_name); + if (!str8_match(data, artifact->data, 0)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_file"), "file '%S' differs from artifact '%S'", path, artifact_name); + } + } + String8 needle = t_codec_scalar(t_codec_child(arguments, "contains")); + if (needle.size != 0 && str8_find_needle(data, 0, needle, 0) >= data.size) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_file"), "file '%S' does not contain '%S'", path, needle); + } + B32 nonempty = 0; + MD_Node *nonempty_node = t_codec_child(arguments, "nonempty"); + if (!md_node_is_nil(nonempty_node)) { + t_bool_from_scalar(nonempty_node, &nonempty); + if (nonempty && data.size == 0) { return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_file"), "file '%S' is empty", path); } + } + return ctx->result; +} + +internal COFF_SectionHeader * +t_codec_pe_section_from_name(T_Context *ctx, T_Artifact *artifact, PE_BinInfo *pe_out, String8 name) +{ + PE_BinInfo pe = pe_bin_info_from_data(ctx->arena, artifact->data); + COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(artifact->data, pe.section_table_range).str; + String8 string_table = str8_substr(artifact->data, pe.string_table_range); + if (pe_out != 0) { *pe_out = pe; } + return coff_section_header_from_name(string_table, section_table, pe.section_count, name); +} + +internal T_Result +t_codec_expect_pe_word_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"artifact", "section", "offset", "type", "nonzero", "equals", "target_section", + "target_offset", "target_address", "other_section", "other_offset", "other_type", "relation", "modulo", + "remainder", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + char *required[] = {"artifact", "section", "offset", "type", 0}; + for (U64 i = 0; required[i] != 0; i += 1) { t_codec_validate_scalar(ctx, arguments, required[i], 1); } + for (U64 i = 4; allowed[i] != 0; i += 1) { t_codec_validate_scalar(ctx, arguments, allowed[i], 0); } + T_Artifact *artifact = t_artifact_from_name(ctx->run, t_codec_scalar(t_codec_child(arguments, "artifact"))); + if (artifact == 0 || !str8_match(artifact->codec->kind, str8_lit("pe"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "artifact"), "expect_pe_word requires a PE artifact"); + } + String8 type = t_codec_scalar(t_codec_child(arguments, "type")); + if (!str8_match(type, str8_lit("u32"), StringMatchFlag_CaseInsensitive) && !str8_match(type, str8_lit("u64"), StringMatchFlag_CaseInsensitive) && + !str8_match(type, str8_lit("rel32"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "type"), "word type must be u32, u64, or rel32"); + } + String8 relation = t_codec_scalar(t_codec_child(arguments, "relation")); + if (relation.size != 0 && !str8_match(relation, str8_lit("equal"), StringMatchFlag_CaseInsensitive) && + !str8_match(relation, str8_lit("not_equal"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "relation"), "relation must be equal or not_equal"); + } + return ctx->run->result; +} + +internal B32 +t_codec_pe_read_word(T_Context *ctx, T_Artifact *artifact, String8 section_name, U64 offset, String8 type, U64 *value_out) +{ + PE_BinInfo pe = {0}; + COFF_SectionHeader *section = t_codec_pe_section_from_name(ctx, artifact, &pe, section_name); + if (section == 0) { return 0; } + U64 size = str8_match(type, str8_lit("u64"), StringMatchFlag_CaseInsensitive) ? 8 : 4; + if (offset > section->fsize || size > section->fsize - offset || section->foff > artifact->data.size || section->fsize > artifact->data.size - section->foff) { return 0; } + U8 *ptr = artifact->data.str + section->foff + offset; + if (str8_match(type, str8_lit("u64"), StringMatchFlag_CaseInsensitive)) { + MemoryCopy(value_out, ptr, sizeof(U64)); + } else { + U32 raw = 0; + MemoryCopy(&raw, ptr, sizeof(raw)); + if (str8_match(type, str8_lit("rel32"), StringMatchFlag_CaseInsensitive)) { + *value_out = pe.image_base + section->voff + offset + sizeof(S32) + (S32)raw; + } else { + *value_out = raw; + } + } + return 1; +} + +internal T_Result +t_codec_expect_pe_word_execute(T_Context *ctx, MD_Node *arguments) +{ + T_Artifact *artifact = t_artifact_from_name(ctx, t_codec_scalar(t_codec_child(arguments, "artifact"))); + String8 section_name = t_codec_scalar(t_codec_child(arguments, "section")); + String8 type = t_codec_scalar(t_codec_child(arguments, "type")); + U64 offset = 0, value = 0; + try_u64_from_str8_c_rules(t_codec_scalar(t_codec_child(arguments, "offset")), &offset); + if (!t_codec_pe_read_word(ctx, artifact, section_name, offset, type, &value)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "unable to read %S at %S+%llu", type, section_name, offset); + } + MD_Node *nonzero_node = t_codec_child(arguments, "nonzero"); + B32 nonzero = 0; + if (!md_node_is_nil(nonzero_node) && t_bool_from_scalar(nonzero_node, &nonzero) && nonzero && value == 0) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "value at %S+%llu is zero", section_name, offset); + } + MD_Node *equals_node = t_codec_child(arguments, "equals"); + U64 expected = 0; + if (!md_node_is_nil(equals_node) && (!try_u64_from_str8_c_rules(t_codec_scalar(equals_node), &expected) || value != expected)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "value at %S+%llu is %llu, expected %llu", section_name, offset, value, expected); + } + String8 target_section_name = t_codec_scalar(t_codec_child(arguments, "target_section")); + if (target_section_name.size != 0) { + PE_BinInfo pe = {0}; + COFF_SectionHeader *target = t_codec_pe_section_from_name(ctx, artifact, &pe, target_section_name); + U64 target_offset = 0; + try_u64_from_str8_c_rules(t_codec_scalar(t_codec_child(arguments, "target_offset")), &target_offset); + String8 address = t_codec_scalar(t_codec_child(arguments, "target_address")); + if (target != 0) { expected = (str8_match(address, str8_lit("rva"), StringMatchFlag_CaseInsensitive) ? 0 : pe.image_base) + target->voff + target_offset; } + if (target == 0 || value != expected) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "value at %S+%llu does not target %S+%llu", section_name, offset, target_section_name, + target_offset); + } + } + String8 other_section = t_codec_scalar(t_codec_child(arguments, "other_section")); + if (other_section.size != 0) { + U64 other_offset = 0, other_value = 0; + try_u64_from_str8_c_rules(t_codec_scalar(t_codec_child(arguments, "other_offset")), &other_offset); + String8 other_type = t_codec_scalar(t_codec_child(arguments, "other_type")); + if (other_type.size == 0) { other_type = type; } + if (!t_codec_pe_read_word(ctx, artifact, other_section, other_offset, other_type, &other_value)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "unable to read comparison word at %S+%llu", other_section, other_offset); + } + B32 equal = str8_match(t_codec_scalar(t_codec_child(arguments, "relation")), str8_lit("equal"), StringMatchFlag_CaseInsensitive); + if ((equal && value != other_value) || (!equal && value == other_value)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "word relation at %S+%llu and %S+%llu does not hold", section_name, offset, other_section, + other_offset); + } + } + MD_Node *modulo_node = t_codec_child(arguments, "modulo"); + if (!md_node_is_nil(modulo_node)) { + U64 modulo = 0, remainder = 0; + try_u64_from_str8_c_rules(t_codec_scalar(modulo_node), &modulo); + try_u64_from_str8_c_rules(t_codec_scalar(t_codec_child(arguments, "remainder")), &remainder); + if (modulo == 0 || value % modulo != remainder) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_word"), "value at %S+%llu has unexpected remainder", section_name, offset); + } + } + return ctx->result; +} + +internal T_Result +t_codec_expect_pe_bytes_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + char *allowed[] = {"artifact", "section", "offset", "hex", 0}; + t_codec_validate_fields(ctx, arguments, allowed); + for (U64 i = 0; allowed[i] != 0; i += 1) { t_codec_validate_scalar(ctx, arguments, allowed[i], 1); } + T_Artifact *artifact = t_artifact_from_name(ctx->run, t_codec_scalar(t_codec_child(arguments, "artifact"))); + if (artifact == 0 || !str8_match(artifact->codec->kind, str8_lit("pe"), StringMatchFlag_CaseInsensitive)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, t_codec_child(arguments, "artifact"), "expect_pe_bytes requires a PE artifact"); + } + return ctx->run->result; +} + +internal T_Result +t_codec_expect_pe_bytes_execute(T_Context *ctx, MD_Node *arguments) +{ + T_Artifact *artifact = t_artifact_from_name(ctx, t_codec_scalar(t_codec_child(arguments, "artifact"))); + String8 section_name = t_codec_scalar(t_codec_child(arguments, "section")); + COFF_SectionHeader *section = t_codec_pe_section_from_name(ctx, artifact, 0, section_name); + U64 offset = 0; + try_u64_from_str8_c_rules(t_codec_scalar(t_codec_child(arguments, "offset")), &offset); + MD_Node hex_node = {0}; + MD_Node value_node = {0}; + hex_node.kind = MD_NodeKind_Main; + hex_node.string = str8_lit("hex"); + hex_node.first = hex_node.last = &value_node; + value_node.kind = MD_NodeKind_Main; + value_node.string = t_codec_scalar(t_codec_child(arguments, "hex")); + String8 expected = {0}; + T_Result result = t_bytes_from_producer(ctx, &hex_node, &expected); + if (!t_result_is_ok(result)) { return result; } + if (section == 0 || offset > section->fsize || expected.size > section->fsize - offset || section->foff > artifact->data.size || section->fsize > artifact->data.size - section->foff || + !str8_match(str8(artifact->data.str + section->foff + offset, expected.size), expected, 0)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("expect_pe_bytes"), "bytes at %S+%llu do not match", section_name, offset); + } + return ctx->result; +} + +internal T_Result +t_codec_no_arguments_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + if (!md_node_is_nil(arguments->first)) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments->first, "operation takes no arguments"); + } + return ctx->run->result; +} + +#define T_CODEC_INTERNAL_API_CHECK(expr) do { if (!(expr)) { t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, operation, "assertion failed: %s", #expr); } } while (0) + +internal T_Result +t_codec_get_msf_stream_pages_execute(T_Context *ctx, MD_Node *arguments) +{ + Arena *arena = ctx->arena; + String8 operation = str8_lit("get_msf_stream_pages"); + MSF_Context *msf = msf_alloc(MSF_DEFAULT_PAGE_SIZE, MSF_DEFAULT_FPM); + + { + U64 stream_size = MB(150) + 1; + + MSF_StreamNumber sn = msf_stream_alloc_ex(msf, stream_size); + + U8 *test = push_array(arena, U8, stream_size); + MemorySet(test, 0xca, stream_size/2); + MemorySet(test + stream_size/2, 0xbe, stream_size/2); + + String8List stream_data = msf_data_from_sn(arena, msf, sn); + T_CODEC_INTERNAL_API_CHECK(stream_data.total_size == stream_size); + T_CODEC_INTERNAL_API_CHECK(stream_data.node_count == 12); + + String8Array a = str8_array_from_list(arena, &stream_data); + T_CODEC_INTERNAL_API_CHECK(a.v[0].size == 0xffd000); + T_CODEC_INTERNAL_API_CHECK(a.v[1].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[2].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[3].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[4].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[5].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[6].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[7].size == 0xffd000); + T_CODEC_INTERNAL_API_CHECK(a.v[8].size == 0x1000); + T_CODEC_INTERNAL_API_CHECK(a.v[9].size == 0xffe000); + T_CODEC_INTERNAL_API_CHECK(a.v[10].size == 0x613000); + T_CODEC_INTERNAL_API_CHECK(a.v[11].size == 1); + + String8Node buf = *stream_data.first; + U64 buf_pos = 0; + str8_buffer_write(&buf, &buf_pos, str8(test, stream_size)); + + String8 cmp = msf_stream_read_block(arena, msf, sn, stream_size); + T_CODEC_INTERNAL_API_CHECK(cmp.size == stream_size); + T_CODEC_INTERNAL_API_CHECK(MemoryCompare(cmp.str, test, stream_size) == 0); + } + + { + MSF_StreamNumber sn = msf_stream_alloc_ex(msf, 1); + String8List stream_data = msf_data_from_sn(arena, msf, sn); + T_CODEC_INTERNAL_API_CHECK(stream_data.node_count == 1); + T_CODEC_INTERNAL_API_CHECK(stream_data.total_size == 1); + T_CODEC_INTERNAL_API_CHECK(stream_data.first->string.size == 1); + } + + msf_release(msf); + return ctx->result; +} + +internal T_Result +t_codec_data_from_pdb(T_Context *ctx, MD_Node *arguments, PDB_Context *pdb, String8 *data_out) +{ + Arena *arena = ctx->arena; + String8 operation = str8_lit("validate_info_stream"); + TP_Context *tp = tp_alloc(arena, 1, 1, str8_lit("foo")); + TP_Arena *tp_arena = tp_arena_alloc(tp); + pdb_build(tp, tp_arena, pdb, (CV_StringHashTable){0}, 1, 0, 0); + + MSF_Error msf_error = msf_build(pdb->msf); + T_CODEC_INTERNAL_API_CHECK(msf_error == MSF_Error_OK); + String8List raw_msf_list = msf_get_page_data_nodes(arena, pdb->msf); + T_CODEC_INTERNAL_API_CHECK(t_write_file_list(str8_lit("test.pdb"), raw_msf_list)); + + *data_out = str8_list_join(arena, &raw_msf_list, 0); + + tp_arena_release(&tp_arena); + tp_release(tp); + + return ctx->result; +} + +internal T_Result +t_codec_validate_info_stream_execute(T_Context *ctx, MD_Node *arguments) +{ + Arena *arena = ctx->arena; + String8 operation = str8_lit("validate_info_stream"); + COFF_TimeStamp time_stamp = 123; + U32 age = 1; + Guid guid = { .data1 = max_U32, .data2 = max_U16 - 1, .data3 = max_U16 - 2, .data4 = { 1, 2, 3, 4, 5, 6, 7, 8 } }; + PDB_Context *pdb = pdb_alloc(MSF_DEFAULT_PAGE_SIZE, COFF_MachineType_X64, time_stamp, age, guid); + + char *stream_names[] = { "one", "two", "three", "four", "five" }; + MSF_StreamNumber stream_numbers[ArrayCount(stream_names)] = {0}; + + for EachElement(i, stream_names) { + stream_numbers[i] = pdb_push_named_stream(&pdb->info->named_stream_ht, pdb->msf, str8_cstring(stream_names[i])); + T_CODEC_INTERNAL_API_CHECK(stream_numbers[i] != MSF_INVALID_STREAM_NUMBER); + } + + String8 raw_msf = {0}; + t_codec_data_from_pdb(ctx, arguments, pdb, &raw_msf); + MSF_Parsed *msf_parsed = msf_parsed_from_data(arena, raw_msf); + String8 info_data = msf_data_from_stream(msf_parsed, PDB_FixedStream_Info); + +#if 0 + fprintf(stderr, "\n"); + for EachIndex(i, info_data.size) { + fprintf(stderr, "0x%02x, ", info_data.str[i]); + if (i % 19 == 18 && i > 0) { fprintf(stderr, "\n"); } + } +#endif + U8 expected_info_data[] = { + 0x94, 0x2e, 0x31, 0x01, 0x7b, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xfd, + 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x22, 0x00, 0x00, 0x00, 0x6f, 0x6e, 0x65, 0x00, 0x74, 0x77, + 0x6f, 0x00, 0x74, 0x68, 0x72, 0x65, 0x65, 0x00, 0x66, 0x6f, 0x75, 0x72, 0x00, 0x66, 0x69, 0x76, 0x65, 0x00, 0x2f, + 0x4c, 0x69, 0x6e, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x13, + 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x51, 0x33, 0x01, + }; + T_CODEC_INTERNAL_API_CHECK(str8_match(info_data, str8_array_fixed(expected_info_data), 0)); + + pdb_release(pdb); + return ctx->result; +} + +internal T_Result +t_codec_psi_addr_map_radix_sort_execute(T_Context *ctx, MD_Node *arguments) +{ + Arena *arena = ctx->arena; + String8 operation = str8_lit("psi_addr_map_radix_sort"); + String8 names[] = { + str8_lit("alpha"), + str8_lit("bravo"), + str8_lit("charlie"), + str8_lit("delta"), + }; + U64 address_count = (1 << 15) + 1; + U64 record_count = address_count * ArrayCount(names); + + PDB_GsiSortRecord *records = push_array_no_zero(arena, PDB_GsiSortRecord, record_count); + PDB_GsiSortRecord *expected = push_array_no_zero(arena, PDB_GsiSortRecord, record_count); + for EachIndex(i, record_count) { + U64 address_idx = address_count - 1 - i / ArrayCount(names); + records[i].isect_off.isect = 1 + address_idx % 257; + records[i].isect_off.off = address_idx / 257; + records[i].name = names[ArrayCount(names) - 1 - i % ArrayCount(names)]; + records[i].offset = i * sizeof(U32); + } + MemoryCopyTyped(expected, records, record_count); + radsort(expected, record_count, psi_addr_map_compar_is_before); + + TP_Context *tp = tp_alloc(arena, 1, 1, str8_lit("psi addr map sort test")); + U32 *addr_map = psi_addr_map_from_gsi_records(tp, arena, records, record_count); + + for EachIndex(i, record_count) { + T_CODEC_INTERNAL_API_CHECK(addr_map[i] == expected[i].offset); + } + tp_release(tp); + return ctx->result; +} + +internal T_Result +t_codec_u64_array_radix_sort_parallel_execute(T_Context *ctx, MD_Node *arguments) +{ + Arena *arena = ctx->arena; + String8 operation = str8_lit("u64_array_radix_sort_parallel"); + TP_Context *tp = tp_alloc(arena, 1, 1, str8_zero()); + + U64 small_values[] = { max_U64, 0, 7, 1, 7, 0x100000000ull, 2 }; + u64_array_sort_radix_parallel(tp, ArrayCount(small_values), small_values); + for (U64 i = 1; i < ArrayCount(small_values); i += 1) { + T_CODEC_INTERNAL_API_CHECK(small_values[i-1] <= small_values[i]); + } + + U64 count = 200003; + U64 *values = push_array_no_zero(arena, U64, count); + U64 state = 0x9e3779b97f4a7c15ull; + U64 sum_before = 0; + U64 xor_before = 0; + for EachIndex(i, count) { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + values[i] = state * 0x2545f4914f6cdd1dull; + if ((i % 97) == 0) { values[i] = 0; } + if ((i % 193) == 0) { values[i] = max_U64; } + if ((i % 389) == 0) { values[i] = 0x100000001ull; } + sum_before += values[i]; + xor_before ^= values[i]; + } + + u64_array_sort_radix_parallel(tp, count, values); + U64 sum_after = 0; + U64 xor_after = 0; + B32 is_sorted = 1; + for EachIndex(i, count) { + if (i > 0 && values[i-1] > values[i]) { is_sorted = 0; } + sum_after += values[i]; + xor_after ^= values[i]; + } + T_CODEC_INTERNAL_API_CHECK(is_sorted); + T_CODEC_INTERNAL_API_CHECK(sum_before == sum_after); + T_CODEC_INTERNAL_API_CHECK(xor_before == xor_after); + tp_release(tp); + return ctx->result; +} + +internal T_Result +t_codec_patch_cv_symbol_tree_execute(T_Context *ctx, MD_Node *arguments) +{ + Arena *arena = ctx->arena; + String8 operation = str8_lit("patch_cv_symbol_tree"); + String8List raw_symbols = {0}; + str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_OBJNAME, cv_make_obj_name(arena, str8_lit("foo.obj"), 123))); + str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_GPROC32, cv_make_proc32(arena, (CV_SymProc32){0}, str8_lit("Proc")))); + str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_INLINESITE, cv_make_inline_site(arena, (CV_SymInlineSite){0}, str8_zero()))); + str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_INLINESITE_END, cv_make_inline_site_end(arena))); + str8_list_push(arena, &raw_symbols, cv_make_symbol(arena, CV_SymKind_END, cv_make_end(arena))); + + U64 tree_size = cv_patch_symbol_tree_offsets(raw_symbols, sizeof(CV_Signature), 4); + T_CODEC_INTERNAL_API_CHECK(tree_size == 84); + + { + String8Node buf = *raw_symbols.first; + U64 buf_pos = 0; + + CV_SymbolHeader obj_header; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(obj_header), &obj_header) == sizeof(obj_header)); + T_CODEC_INTERNAL_API_CHECK(obj_header.kind == CV_SymKind_OBJNAME); + T_CODEC_INTERNAL_API_CHECK(str8_buffer_skip(&buf, &buf_pos, obj_header.size - sizeof(CV_SymKind))); + + CV_SymbolHeader proc_header; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(proc_header), &proc_header) == sizeof(proc_header)); + T_CODEC_INTERNAL_API_CHECK(proc_header.kind == CV_SymKind_GPROC32); + + CV_SymProc32 proc; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(proc), &proc) == sizeof(proc)); + T_CODEC_INTERNAL_API_CHECK(proc.end == 0x54); + T_CODEC_INTERNAL_API_CHECK(str8_buffer_skip(&buf, &buf_pos, proc_header.size - sizeof(CV_SymKind) - sizeof(proc))); + + CV_SymbolHeader inline_site_header; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(inline_site_header), &inline_site_header) == sizeof(inline_site_header)); + T_CODEC_INTERNAL_API_CHECK(inline_site_header.kind == CV_SymKind_INLINESITE); + + CV_SymInlineSite inline_site; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(inline_site), &inline_site)); + T_CODEC_INTERNAL_API_CHECK(inline_site.parent == 0x14); + T_CODEC_INTERNAL_API_CHECK(inline_site.end == 0x50); + T_CODEC_INTERNAL_API_CHECK(str8_buffer_skip(&buf, &buf_pos, inline_site_header.size - sizeof(CV_SymKind) - sizeof(inline_site))); + + CV_SymbolHeader inline_end_header; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(inline_end_header), &inline_end_header) == sizeof(inline_end_header)); + T_CODEC_INTERNAL_API_CHECK(inline_end_header.kind == CV_SymKind_INLINESITE_END); + + CV_SymbolHeader proc_end_header; + T_CODEC_INTERNAL_API_CHECK(str8_buffer_read(&buf, &buf_pos, sizeof(proc_end_header), &proc_end_header) == sizeof(proc_end_header)); + T_CODEC_INTERNAL_API_CHECK(proc_end_header.kind == CV_SymKind_END); + + T_CODEC_INTERNAL_API_CHECK(buf.string.size == 0); + T_CODEC_INTERNAL_API_CHECK(buf.string.str == 0); + T_CODEC_INTERNAL_API_CHECK(buf_pos == 0); + } + return ctx->result; +} + +#undef T_CODEC_INTERNAL_API_CHECK + +global T_Codec t_codec_script_codecs[] = { + {str8_lit_comp("bytes"), 0, t_codec_bytes_encode, t_codec_bytes_decode}, + {str8_lit_comp("text"), 0, t_codec_text_encode, t_codec_text_decode}, + {str8_lit_comp("coff"), t_coff_validate, t_coff_encode, t_coff_decode}, + {str8_lit_comp("pe"), 0, 0, t_codec_pe_decode}, +}; + +global T_OpSpec t_codec_script_ops[] = { + {str8_lit_comp("compare"), t_op_compare_validate, t_op_compare_execute}, + {str8_lit_comp("compare_file"), t_op_compare_file_validate, t_op_compare_file_execute}, + {str8_lit_comp("repeat"), t_op_repeat_validate, t_op_repeat_execute}, + {str8_lit_comp("run"), t_codec_run_validate, t_codec_run_execute}, + {str8_lit_comp("clang"), t_codec_clang_validate, t_codec_clang_execute}, + {str8_lit_comp("expect_coff"), t_codec_expect_coff_validate, t_codec_expect_coff_execute}, + {str8_lit_comp("expect_pe"), t_codec_expect_pe_validate, t_codec_expect_pe_execute}, + {str8_lit_comp("expect_pdb"), t_codec_expect_pdb_validate, t_codec_expect_pdb_execute}, + {str8_lit_comp("expect_file"), t_codec_expect_file_validate, t_codec_expect_file_execute}, + {str8_lit_comp("expect_pe_word"), t_codec_expect_pe_word_validate, t_codec_expect_pe_word_execute}, + {str8_lit_comp("expect_pe_bytes"), t_codec_expect_pe_bytes_validate, t_codec_expect_pe_bytes_execute}, + {str8_lit_comp("get_msf_stream_pages"), t_codec_no_arguments_validate, t_codec_get_msf_stream_pages_execute}, + {str8_lit_comp("validate_info_stream"), t_codec_no_arguments_validate, t_codec_validate_info_stream_execute}, + {str8_lit_comp("psi_addr_map_radix_sort"), t_codec_no_arguments_validate, t_codec_psi_addr_map_radix_sort_execute}, + {str8_lit_comp("u64_array_radix_sort_parallel"), t_codec_no_arguments_validate, t_codec_u64_array_radix_sort_parallel_execute}, + {str8_lit_comp("patch_cv_symbol_tree"), t_codec_no_arguments_validate, t_codec_patch_cv_symbol_tree_execute}, +}; + +global T_SuiteSpec t_codec_script_suite = { + .name = str8_lit_comp("linker"), + .codecs = t_codec_script_codecs, + .codec_count = ArrayCount(t_codec_script_codecs), + .ops = t_codec_script_ops, + .op_count = ArrayCount(t_codec_script_ops), +}; diff --git a/src/torture/torture_linker.h b/src/torture/torture_linker.h new file mode 100644 index 000000000..895e182d7 --- /dev/null +++ b/src/torture/torture_linker.h @@ -0,0 +1,26 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +internal T_Result t_codec_pe_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out); +internal B32 t_codec_coff_section_name(String8 string_table, COFF_SectionHeader *header, String8 *name_out); +internal T_Result t_codec_run_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_run_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_clang_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_clang_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pe_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pe_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_pdb_decode(T_Context *ctx, String8 data, MD_Node **semantic_tree_out); +internal T_Result t_codec_expect_pdb_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pdb_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_coff_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_coff_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_file_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_file_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pe_word_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pe_word_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pe_bytes_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_codec_expect_pe_bytes_execute(T_Context *ctx, MD_Node *arguments); + +global T_SuiteSpec t_codec_script_suite; diff --git a/src/torture/torture_main.c b/src/torture/torture_main.c index 1b3a6915b..5bebac057 100644 --- a/src/torture/torture_main.c +++ b/src/torture/torture_main.c @@ -102,6 +102,11 @@ #include "linker/lnk_log.h" #include "linker/lnk_debug_helper.h" #include "torture.h" +#include "torture_controller.h" +#include "torture_script.h" +#include "torture_build.h" +#include "torture_coff.h" +#include "torture_linker.h" #include "base/base_inc.c" #include "x64/x64.c" @@ -182,6 +187,12 @@ #include "linker/pdb_ext/pdb_builder.c" #include "linker/lnk_debug_helper.c" #include "torture.c" +#include "torture_base.c" +#include "torture_controller.c" +#include "torture_script.c" +#include "torture_build.c" +#include "torture_linker.c" +#include "torture_coff.c" #include "base/tests/base_tests.c" #include "mdesk/tests/mdesk_tests.c" @@ -191,6 +202,10 @@ #include "rdi_from_pdb/tests/rdi_from_pdb_tests.c" #include "raddbg/tests/raddbg_tests.c" #include "eval2/tests/eval2_tests.c" +#include "torture/tests/torture_subprocess_tests.c" +#include "torture/tests/torture_script_tests.c" +#include "torture/tests/torture_coff_tests.c" +#include "torture/tests/torture_linker_tests.c" internal B32 frame(void) { return 0; } @@ -199,6 +214,29 @@ internal B32 frame(void) { return 0; } internal void entry_point(CmdLine *cmdline) { - t_entry_point(cmdline); + if(cmd_line_has_flag(cmdline, str8_lit("controller_fixture"))) + { + t_controller_fixture(); + } + else if(cmd_line_has_flag(cmdline, str8_lit("controller_interrupt_fixture"))) + { + t_controller_interrupt_fixture(); + } + else if(cmd_line_has_flag(cmdline, str8_lit("run_fixture_ok"))) + { + t_run_operation_fixture(0, 0); + } + else if(cmd_line_has_flag(cmdline, str8_lit("run_fixture_nonzero"))) + { + t_run_operation_fixture(0, 7); + } + else if(cmd_line_has_flag(cmdline, str8_lit("run_fixture_sleep"))) + { + t_run_operation_fixture(1, 0); + } + else + { + t_entry_point(cmdline); + } } diff --git a/src/torture/torture_script.c b/src/torture/torture_script.c new file mode 100644 index 000000000..563fbf10c --- /dev/null +++ b/src/torture/torture_script.c @@ -0,0 +1,747 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +global T_Codec t_codec_bytes = {str8_lit_comp("bytes"), 0, t_codec_bytes_encode, t_codec_bytes_decode}; +global T_Codec t_codec_text = {str8_lit_comp("text"), 0, t_codec_text_encode, t_codec_text_decode}; +global T_OpSpec t_op_compare = {str8_lit_comp("compare"), t_op_compare_validate, t_op_compare_execute}; + +inline internal B32 +t_result_is_ok(T_Result result) +{ + return result.code == T_ResultCode_Ok; +} + +internal TxtPt +t_txt_pt_from_offset(String8 source, U64 offset) +{ + TxtPt result = {1, 1}; + + U64 offset_cap = Min(offset, source.size); + + for (U64 i = 0; i < offset_cap; i += 1) { + if (source.str[i] == '\n') { + result.line += 1; + result.column = 1; + } else { + result.column += 1; + } + } + + return result; +} + +internal void +t_diagnostic_from_md_msg(T_Context *ctx, MD_Msg *msg) +{ + T_Diagnostic *diagnostic = push_array(ctx->arena, T_Diagnostic, 1); + diagnostic->kind = msg->kind; + diagnostic->file_path = ctx->file_path; + diagnostic->location = t_txt_pt_from_offset(ctx->source, msg->node->src_offset); + diagnostic->message = str8_copy(ctx->arena, msg->string); + + SLLQueuePush(ctx->result.diagnostics.first, ctx->result.diagnostics.last, diagnostic); + ctx->result.diagnostics.count += 1; + + if (ctx->result.code == T_ResultCode_Ok && msg->kind >= MD_MsgKind_Error) { + ctx->result.code = T_ResultCode_ParseError; + } +} + +internal T_Result +t_context_errorfv(T_Context *ctx, T_ResultCode code, MD_Node *node, String8 operation, char *fmt, va_list args) +{ + T_Diagnostic *diagnostic = push_array(ctx->arena, T_Diagnostic, 1); + diagnostic->kind = MD_MsgKind_Error; + diagnostic->file_path = ctx->file_path; + diagnostic->location = t_txt_pt_from_offset(ctx->source, md_node_is_nil(node) ? 0 : node->src_offset); + diagnostic->operation = operation; + diagnostic->message = push_str8fv(ctx->arena, fmt, args); + + SLLQueuePush(ctx->result.diagnostics.first, ctx->result.diagnostics.last, diagnostic); + ctx->result.diagnostics.count += 1; + + if (ctx->result.code == T_ResultCode_Ok) { ctx->result.code = code; } + return ctx->result; +} + +inline internal T_Result +t_context_errorf(T_Context *ctx, T_ResultCode code, MD_Node *node, String8 operation, char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + T_Result result = t_context_errorfv(ctx, code, node, operation, fmt, args); + va_end(args); + return result; +} + +inline internal T_Result +t_parse_errorf(T_ParseContext *ctx, T_ResultCode code, MD_Node *node, char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + T_Result result = t_context_errorfv(ctx->run, code, node, ctx->operation, fmt, args); + va_end(args); + return result; +} + +inline internal T_Codec * +t_codec_from_kind(T_SuiteSpec *suite, String8 kind) +{ + for EachIndex(i, suite->codec_count) { + if (str8_match(suite->codecs[i].kind, kind, StringMatchFlag_CaseInsensitive)) { return &suite->codecs[i]; } + } + return 0; +} + +inline internal T_OpSpec * +t_op_spec_from_name(T_SuiteSpec *suite, String8 name) +{ + for EachIndex(i, suite->op_count) { + if (str8_match(suite->ops[i].name, name, StringMatchFlag_CaseInsensitive)) { return &suite->ops[i]; } + } + return 0; +} + +inline internal T_Artifact * +t_artifact_from_name(T_Context *ctx, String8 name) +{ + for (T_Artifact *artifact = ctx->first_artifact; artifact != 0; artifact = artifact->next) { + if (str8_match(artifact->name, name, 0)) { return artifact; } + } + return 0; +} + +inline internal MD_Node * +t_child_from_string(MD_Node *node, char *string) +{ + return md_child_from_string(node, str8_cstring(string), StringMatchFlag_CaseInsensitive); +} + +internal MD_Node * +t_scalar_from_node(MD_Node *node) +{ + return md_node_is_nil(node) ? &md_nil_node : node->first; +} + +internal String8 +t_scalar_string_from_node(MD_Node *node) +{ + MD_Node *scalar = t_scalar_from_node(node); + return md_node_is_nil(scalar) ? str8_zero() : scalar->string; +} + +internal void +t_context_absorb_result(T_Context *ctx, T_Result result) +{ + if (result.diagnostics.first != 0 && + result.diagnostics.first != ctx->result.diagnostics.first) { + if (ctx->result.diagnostics.last == 0) { + ctx->result.diagnostics = result.diagnostics; + } else { + ctx->result.diagnostics.last->next = result.diagnostics.first; + ctx->result.diagnostics.last = result.diagnostics.last; + ctx->result.diagnostics.count += result.diagnostics.count; + } + } + if (ctx->result.code == T_ResultCode_Ok && result.code != T_ResultCode_Ok) { ctx->result.code = result.code; } +} + +internal T_Result +t_script_parse(Arena *arena, TestCtx *test_ctx, T_SuiteSpec *suite, String8 file_path, String8 source, T_Context *ctx_out) +{ + MemoryZeroStruct(ctx_out); + + T_Context *ctx = ctx_out; + ctx->arena = arena; + ctx->test_ctx = test_ctx; + ctx->suite = suite; + ctx->file_path = str8_copy(arena, file_path); + ctx->source = str8_copy(arena, source); + + // tokenize & parse mdesk + MD_TokenizeResult tokenize = md_tokenize_from_text(arena, ctx->source); + MD_ParseResult parse = md_parse_from_text_tokens(arena, ctx->file_path, ctx->source, tokenize.tokens); + + // error check tokenize & parse + for EachNode(msg, MD_Msg, tokenize.msgs.first) { t_diagnostic_from_md_msg(ctx, msg); } + for EachNode(msg, MD_Msg, parse.msgs.first) { t_diagnostic_from_md_msg(ctx, msg); } + if (!t_result_is_ok(ctx->result)) { return ctx->result; } + + ctx->root = parse.root; + + // does script have expected root node? + { + MD_Node *test_node = ctx->root->first; + if (md_node_is_nil(test_node) || !str8_matchi(test_node->string, str8_lit("test"))) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, test_node, str8_zero(), "expected one top-level 'test' node"); + } + if (t_md_node_is_valid(test_node->next)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, test_node->next, str8_zero(), "unexpected top-level node '%S'", test_node->next->string); + } + ctx->test = test_node; + } + + // set up general parse context + T_ParseContext parse_ctx = { arena, ctx, suite, ctx->file_path, ctx->source }; + + // + // @build + // + { + MD_Node *build = t_child_from_string(ctx->test, "build"); + if (t_md_node_is_valid(build)) { + parse_ctx.operation = str8_lit("build"); + T_Result result = t_build_parse(&parse_ctx, build, &ctx->build); + t_context_absorb_result(ctx, result); + } + } + + // + // @artifacts + // + { + MD_Node *artifacts = t_child_from_string(ctx->test, "artifacts"); + + for MD_EachNode(node, artifacts->first) { + + if (t_artifact_from_name(ctx, node->string) != 0) { + t_context_errorf(ctx, T_ResultCode_ValidationError, node, str8_zero(), "duplicate artifact '%S'", node->string); + continue; + } + + MD_Node *definition = &md_nil_node; + String8 file_name = {0}; + B32 has_file_name = 0; + for + MD_EachNode(child, node->first) + { + if (str8_match(child->string, str8_lit("file_name"), StringMatchFlag_CaseInsensitive)) { + if (has_file_name) { t_context_errorf(ctx, T_ResultCode_ValidationError, child, str8_zero(), "artifact '%S' has duplicate file_name", node->string); } + has_file_name = 1; + if (md_node_is_nil(child->first) || !md_node_is_nil(child->first->next) || child->first->string.size == 0) { + t_context_errorf(ctx, T_ResultCode_ValidationError, child, str8_zero(), "artifact '%S' file_name requires one value", node->string); + } else { + file_name = child->first->string; + if (str8_find_needle(file_name, 0, str8_lit("/"), 0) < file_name.size || str8_find_needle(file_name, 0, str8_lit("\\"), 0) < file_name.size || + str8_find_needle(file_name, 0, str8_lit(":"), 0) < file_name.size) { + t_context_errorf(ctx, T_ResultCode_ValidationError, child, str8_zero(), "artifact '%S' file_name must be a relative file name", node->string); + } + } + } else if (md_node_is_nil(definition)) { + definition = child; + } else { + definition = &md_nil_node; + break; + } + } + if (md_node_is_nil(definition)) { + t_context_errorf(ctx, T_ResultCode_ValidationError, node, str8_zero(), "artifact '%S' must contain exactly one codec definition", node->string); + continue; + } + T_Codec *codec = t_codec_from_kind(suite, definition->string); + if (codec == 0) { + t_context_errorf(ctx, T_ResultCode_ValidationError, definition, str8_zero(), "unknown artifact codec '%S'", definition->string); + continue; + } + T_Artifact *artifact = push_array(arena, T_Artifact, 1); + artifact->name = node->string; + artifact->file_name = file_name; + artifact->codec = codec; + artifact->definition = definition; + if (file_name.size != 0) { + for (T_Artifact *other = ctx->first_artifact; other != 0; other = other->next) { + if (other->file_name.size != 0 && str8_match(other->file_name, file_name, StringMatchFlag_CaseInsensitive | StringMatchFlag_SlashInsensitive)) { + t_context_errorf(ctx, T_ResultCode_ValidationError, node, str8_zero(), "artifact file_name '%S' is already used by '%S'", file_name, other->name); + break; + } + } + } + SLLQueuePush(ctx->first_artifact, ctx->last_artifact, artifact); + ctx->artifact_count += 1; + } + + for (T_Artifact *artifact = ctx->first_artifact; artifact != 0; artifact = artifact->next) { + if (artifact->codec->validate != 0) { + T_Result result = artifact->codec->validate(&parse_ctx, artifact); + t_context_absorb_result(ctx, result); + } + artifact->state = t_result_is_ok(ctx->result) ? T_ArtifactState_Validated : T_ArtifactState_Failed; + } + } + + // + // @steps + // + { + MD_Node *steps = t_child_from_string(ctx->test, "steps"); + ctx->command_count = md_node_is_nil(steps) ? 0 : md_child_count_from_node(steps); + ctx->commands = push_array(arena, T_Command, ctx->command_count); + U64 command_index = 0; + for MD_EachNode(node, steps->first) { + T_OpSpec *spec = t_op_spec_from_name(suite, node->string); + if (spec == 0) { + t_context_errorf(ctx, T_ResultCode_ValidationError, node, node->string, "unknown operation '%S'", node->string); + continue; + } + T_Command *command = &ctx->commands[command_index++]; + command->spec = spec; + command->arguments = node; + command->location = t_txt_pt_from_offset(ctx->source, node->src_offset); + command->order = command_index; + if (spec->validate != 0) { + parse_ctx.operation = spec->name; + T_Result result = spec->validate(&parse_ctx, node); + t_context_absorb_result(ctx, result); + } + } + ctx->command_count = command_index; + } + return ctx->result; +} + +internal T_Result +t_script_execute(T_Context *ctx) +{ + if (!t_result_is_ok(ctx->result)) { return ctx->result; } + + for (T_Artifact *artifact = ctx->first_artifact; artifact != 0; artifact = artifact->next) { + if (artifact->codec->encode == 0) { continue; } + T_Result result = artifact->codec->encode(ctx, artifact); + t_context_absorb_result(ctx, result); + if (!t_result_is_ok(ctx->result)) { + artifact->state = T_ArtifactState_Failed; + return ctx->result; + } + artifact->state = T_ArtifactState_Materialized; + if (artifact->file_name.size != 0 && !t_write_file(artifact->file_name, artifact->data)) { + artifact->state = T_ArtifactState_Failed; + return t_context_errorf(ctx, T_ResultCode_IoError, artifact->definition, str8_zero(), "unable to materialize artifact '%S'", artifact->file_name); + } + } + + B32 began = 0; + if (ctx->suite->begin != 0) { + began = 1; + T_Result result = ctx->suite->begin(ctx, ctx->test); + t_context_absorb_result(ctx, result); + if (!t_result_is_ok(ctx->result)) { goto exit; } + } + { + T_Result result = t_build_execute(ctx); + t_context_absorb_result(ctx, result); + if (!t_result_is_ok(ctx->result)) { goto exit; } + } + for EachIndex(i, ctx->command_count) { + T_Command *command = &ctx->commands[i]; + T_Result result = command->spec->execute(ctx, command->arguments); + t_context_absorb_result(ctx, result); + if (!t_result_is_ok(ctx->result)) { break; } + } + +exit:; + if (began && ctx->suite->end != 0) { ctx->suite->end(ctx); } + return ctx->result; +} + +internal T_Result +t_byte_producer_error(T_Context *ctx, MD_Node *node, char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + T_Result result = t_context_errorfv(ctx, T_ResultCode_ValidationError, node, str8_zero(), fmt, args); + va_end(args); + return result; +} + +internal B32 +t_hex_digit_value(U8 c, U8 *value_out) +{ + if (c >= '0' && c <= '9') { + *value_out = c - '0'; + return 1; + } + if (c >= 'a' && c <= 'f') { + *value_out = c - 'a' + 10; + return 1; + } + if (c >= 'A' && c <= 'F') { + *value_out = c - 'A' + 10; + return 1; + } + return 0; +} + +internal T_Result +t_bytes_from_producer_list(T_Context *ctx, MD_Node *first, String8 *data_out) +{ + String8List parts = {0}; + for + MD_EachNode(node, first) + { + String8 part = {0}; + T_Result result = t_bytes_from_producer(ctx, node, &part); + if (!t_result_is_ok(result)) { return result; } + str8_list_push(ctx->arena, &parts, part); + } + *data_out = str8_list_join(ctx->arena, &parts, 0); + return ctx->result; +} + +internal T_Result +t_bytes_from_producer(T_Context *ctx, MD_Node *producer, String8 *data_out) +{ + if (str8_match(producer->string, str8_lit("data"), StringMatchFlag_CaseInsensitive)) { + if (md_node_is_nil(producer->first) || !md_node_is_nil(producer->first->next)) { return t_byte_producer_error(ctx, producer, "data must contain exactly one byte producer"); } + return t_bytes_from_producer(ctx, producer->first, data_out); + } + + if (str8_match(producer->string, str8_lit("concat"), StringMatchFlag_CaseInsensitive)) { return t_bytes_from_producer_list(ctx, producer->first, data_out); } + + U64 size_prefix = 0; + if (str8_match(producer->string, str8_lit("size16le"), StringMatchFlag_CaseInsensitive)) { size_prefix = 2; } + if (str8_match(producer->string, str8_lit("size32le"), StringMatchFlag_CaseInsensitive)) { size_prefix = 4; } + if (size_prefix != 0) { + if (md_node_is_nil(producer->first) || !md_node_is_nil(producer->first->next)) { return t_byte_producer_error(ctx, producer, "size prefix requires one byte producer"); } + String8 payload = {0}; + T_Result result = t_bytes_from_producer(ctx, producer->first, &payload); + U64 max_size = size_prefix == 2 ? max_U16 : max_U32; + if (!t_result_is_ok(result)) { return result; } + if (payload.size > max_size) { return t_byte_producer_error(ctx, producer, "payload is too large for size prefix"); } + U8 *bytes = push_array_no_zero(ctx->arena, U8, size_prefix + payload.size); + for EachIndex(i, size_prefix) { bytes[i] = (U8)(payload.size >> (i * 8)); } + MemoryCopy(bytes + size_prefix, payload.str, payload.size); + *data_out = str8(bytes, size_prefix + payload.size); + return ctx->result; + } + + if (str8_match(producer->string, str8_lit("align4"), StringMatchFlag_CaseInsensitive)) { + if (md_node_is_nil(producer->first) || !md_node_is_nil(producer->first->next)) { return t_byte_producer_error(ctx, producer, "align4 requires one byte producer"); } + String8 payload = {0}; + T_Result result = t_bytes_from_producer(ctx, producer->first, &payload); + if (!t_result_is_ok(result)) { return result; } + U64 aligned_size = AlignPow2(payload.size, 4); + U8 *bytes = push_array(ctx->arena, U8, aligned_size); + MemoryCopy(bytes, payload.str, payload.size); + *data_out = str8(bytes, aligned_size); + return ctx->result; + } + + if (str8_match(producer->string, str8_lit("text"), StringMatchFlag_CaseInsensitive)) { + MD_Node *value = producer->first; + if (md_node_is_nil(value) || !md_node_is_nil(value->next)) { return t_byte_producer_error(ctx, producer, "text requires one value"); } + *data_out = str8_copy(ctx->arena, value->string); + return ctx->result; + } + + if (str8_match(producer->string, str8_lit("work_path"), StringMatchFlag_CaseInsensitive)) { + String8 file_name = t_scalar_string_from_node(producer); + if (file_name.size == 0 || str8_find_needle(file_name, 0, str8_lit("/"), 0) < file_name.size || str8_find_needle(file_name, 0, str8_lit("\\"), 0) < file_name.size || + str8_find_needle(file_name, 0, str8_lit(":"), 0) < file_name.size) { + return t_byte_producer_error(ctx, producer, "work_path requires a relative file name"); + } + *data_out = t_make_file_path(ctx->arena, file_name); + return ctx->result; + } + + if (str8_match(producer->string, str8_lit("hex"), StringMatchFlag_CaseInsensitive)) { + MD_Node *value = producer->first; + if (md_node_is_nil(value) || !md_node_is_nil(value->next)) { return t_byte_producer_error(ctx, producer, "hex requires one string"); } + U8 *bytes = push_array_no_zero(ctx->arena, U8, value->string.size / 2 + 1); + U64 count = 0; + U8 high = 0; + B32 has_high = 0; + for + EachIndex(i, value->string.size) + { + U8 c = value->string.str[i]; + if (char_is_space(c) || c == '_') { continue; } + U8 digit = 0; + if (!t_hex_digit_value(c, &digit)) { return t_byte_producer_error(ctx, value, "invalid hex digit '%c'", c); } + if (!has_high) { + high = digit; + has_high = 1; + } else { + bytes[count++] = (high << 4) | digit; + has_high = 0; + } + } + if (has_high) { return t_byte_producer_error(ctx, value, "hex producer contains an odd number of digits"); } + *data_out = str8(bytes, count); + return ctx->result; + } + + if (str8_match(producer->string, str8_lit("zero"), StringMatchFlag_CaseInsensitive)) { + U64 count = 0; + if (!try_u64_from_str8_c_rules(t_scalar_string_from_node(producer), &count)) { return t_byte_producer_error(ctx, producer, "zero requires a byte count"); } + *data_out = str8(push_array(ctx->arena, U8, count), count); + return ctx->result; + } + + if (str8_match(producer->string, str8_lit("repeat"), StringMatchFlag_CaseInsensitive)) { + MD_Node *value_node = producer->first; + MD_Node *count_node = md_node_is_nil(value_node) ? &md_nil_node : value_node->next; + U64 value = 0; + U64 count = 0; + if (md_node_is_nil(value_node) || md_node_is_nil(count_node) || !md_node_is_nil(count_node->next) || !try_u64_from_str8_c_rules(value_node->string, &value) || value > max_U8 || + !try_u64_from_str8_c_rules(count_node->string, &count)) { + return t_byte_producer_error(ctx, producer, "repeat requires a byte value and count"); + } + U8 *bytes = push_array_no_zero(ctx->arena, U8, count); + MemorySet(bytes, (U8)value, count); + *data_out = str8(bytes, count); + return ctx->result; + } + + U64 integer_size = 0; + if (str8_match(producer->string, str8_lit("u16le"), StringMatchFlag_CaseInsensitive)) { integer_size = 2; } + if (str8_match(producer->string, str8_lit("u32le"), StringMatchFlag_CaseInsensitive)) { integer_size = 4; } + if (str8_match(producer->string, str8_lit("u64le"), StringMatchFlag_CaseInsensitive)) { integer_size = 8; } + if (integer_size != 0) { + U64 value = 0; + if (!try_u64_from_str8_c_rules(t_scalar_string_from_node(producer), &value) || (integer_size < 8 && value >= (1ull << (integer_size * 8)))) { + return t_byte_producer_error(ctx, producer, "%S value is out of range", producer->string); + } + U8 *bytes = push_array_no_zero(ctx->arena, U8, integer_size); + for + EachIndex(i, integer_size) { bytes[i] = (U8)(value >> (i * 8)); } + *data_out = str8(bytes, integer_size); + return ctx->result; + } + + if (md_node_is_nil(producer->first)) { + *data_out = str8_copy(ctx->arena, producer->string); + return ctx->result; + } + return t_byte_producer_error(ctx, producer, "unknown byte producer '%S'", producer->string); +} + +internal MD_Node * +t_push_md_child(Arena *arena, MD_Node *parent, String8 string) +{ + MD_Node *node = md_push_node(arena, MD_NodeKind_Main, MD_NodeFlag_Identifier, string, string, 0); + md_node_push_child(parent, node); + return node; +} + +internal T_Result +t_codec_bytes_encode(T_Context *ctx, T_Artifact *artifact) +{ + MD_Node *data = t_child_from_string(artifact->definition, "data"); + if (md_node_is_nil(data)) { return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_zero(), "bytes artifact requires data"); } + return t_bytes_from_producer(ctx, data, &artifact->data); +} + +internal T_Result +t_codec_bytes_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out) +{ + String8 data = artifact->data; + MD_Node *root = md_push_node(ctx->arena, MD_NodeKind_Main, MD_NodeFlag_Identifier, str8_lit("bytes"), str8_lit("bytes"), 0); + MD_Node *size = t_push_md_child(ctx->arena, root, str8_lit("size")); + t_push_md_child(ctx->arena, size, str8f(ctx->arena, "%llu", data.size)); + MD_Node *hex = t_push_md_child(ctx->arena, root, str8_lit("hex")); + U8 *hex_data = push_array_no_zero(ctx->arena, U8, data.size * 2); + local_persist U8 digits[] = "0123456789abcdef"; + for + EachIndex(i, data.size) + { + hex_data[i * 2 + 0] = digits[data.str[i] >> 4]; + hex_data[i * 2 + 1] = digits[data.str[i] & 15]; + } + t_push_md_child(ctx->arena, hex, str8(hex_data, data.size * 2)); + *semantic_tree_out = root; + return ctx->result; +} + +internal T_Result +t_codec_text_encode(T_Context *ctx, T_Artifact *artifact) +{ + MD_Node *data = t_child_from_string(artifact->definition, "data"); + if (md_node_is_nil(data)) { return t_context_errorf(ctx, T_ResultCode_ValidationError, artifact->definition, str8_zero(), "text artifact requires data"); } + return t_bytes_from_producer(ctx, data, &artifact->data); +} + +internal T_Result +t_codec_text_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out) +{ + String8 data = artifact->data; + MD_Node *root = md_push_node(ctx->arena, MD_NodeKind_Main, MD_NodeFlag_Identifier, str8_lit("text"), str8_lit("text"), 0); + MD_Node *value = t_push_md_child(ctx->arena, root, str8_lit("data")); + t_push_md_child(ctx->arena, value, str8_copy(ctx->arena, data)); + *semantic_tree_out = root; + return ctx->result; +} + +internal T_Result +t_op_compare_validate(T_ParseContext *ctx, MD_Node *arguments) +{ + MD_Node *left = t_child_from_string(arguments, "left"); + MD_Node *right = t_child_from_string(arguments, "right"); + String8 left_name = t_scalar_string_from_node(left); + String8 right_name = t_scalar_string_from_node(right); + if (left_name.size == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments, "compare requires left"); + } else if (t_artifact_from_name(ctx->run, left_name) == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, left, "unknown artifact '%S'", left_name); + } + if (right_name.size == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, arguments, "compare requires right"); + } else if (t_artifact_from_name(ctx->run, right_name) == 0) { + t_parse_errorf(ctx, T_ResultCode_ValidationError, right, "unknown artifact '%S'", right_name); + } + return ctx->run->result; +} + +internal T_Result +t_op_compare_execute(T_Context *ctx, MD_Node *arguments) +{ + String8 left_name = t_scalar_string_from_node(t_child_from_string(arguments, "left")); + String8 right_name = t_scalar_string_from_node(t_child_from_string(arguments, "right")); + T_Artifact *left = t_artifact_from_name(ctx, left_name); + T_Artifact *right = t_artifact_from_name(ctx, right_name); + if (!str8_match(left->data, right->data, 0)) { + U64 mismatch = 0; + U64 common_size = Min(left->data.size, right->data.size); + while (mismatch < common_size && left->data.str[mismatch] == right->data.str[mismatch]) { mismatch += 1; } + return t_context_errorf(ctx, T_ResultCode_Mismatch, arguments, str8_lit("compare"), "artifacts '%S' and '%S' differ at byte %llu (sizes %llu and %llu)", left_name, right_name, + mismatch, left->data.size, right->data.size); + } + return ctx->result; +} + +internal B32 +t_u64_from_scalar(MD_Node *node, U64 *value_out) +{ + return try_u64_from_str8_c_rules(t_scalar_string_from_node(node), value_out); +} + +internal T_Result +t_semantic_match_all(T_Context *ctx, MD_Node *expected, MD_Node *actual, MD_Node *tag, String8 path, String8 field_path) +{ + U64 dot = str8_find_needle(field_path, 0, str8_lit("."), 0); + String8 field = str8_prefix(field_path, dot); + String8 rest = dot < field_path.size ? str8_skip(field_path, dot + 1) : str8_zero(); + if (field.size == 0) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, tag, str8_zero(), "all requires a non-empty field path"); + } + + if (str8_match(field, str8_lit("*"), 0)) { + for MD_EachNode(child, actual->first) { + String8 child_path = str8f(ctx->arena, "%S.%S", path, child->string); + if (rest.size == 0) { + B32 matches = 0; + for (MD_Node *allowed = tag->first->next; !md_node_is_nil(allowed); allowed = allowed->next) { + if (str8_match(t_scalar_string_from_node(child), allowed->string, 0)) { matches = 1; break; } + } + if (!matches) { return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: value is not allowed", child_path); } + } else { + T_Result result = t_semantic_match_all(ctx, expected, child, tag, child_path, rest); + if (!t_result_is_ok(result)) { return result; } + } + } + return ctx->result; + } + + MD_Node *child = md_child_from_string(actual, field, 0); + String8 child_path = str8f(ctx->arena, "%S.%S", path, field); + if (md_node_is_nil(child)) { return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: expected field is missing", child_path); } + if (rest.size != 0) { return t_semantic_match_all(ctx, expected, child, tag, child_path, rest); } + + B32 matches = 0; + for (MD_Node *allowed = tag->first->next; !md_node_is_nil(allowed); allowed = allowed->next) { + if (str8_match(t_scalar_string_from_node(child), allowed->string, 0)) { matches = 1; break; } + } + if (!matches) { return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: value is not allowed", child_path); } + return ctx->result; +} + +internal T_Result +t_semantic_match_node(T_Context *ctx, MD_Node *expected, MD_Node *actual, String8 path) +{ + for MD_EachNode(tag, expected->first_tag) { + if (str8_match(tag->string, str8_lit("exists"), StringMatchFlag_CaseInsensitive)) { continue; } + if (str8_match(tag->string, str8_lit("count"), StringMatchFlag_CaseInsensitive)) { + U64 count = 0; + if (!try_u64_from_str8_c_rules(t_scalar_string_from_node(tag), &count) || md_child_count_from_node(actual) != count) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: child count does not match", path); + } + continue; + } + if (str8_match(tag->string, str8_lit("all"), StringMatchFlag_CaseInsensitive)) { + MD_Node *field_path = tag->first; + if (md_node_is_nil(field_path) || md_node_is_nil(field_path->next)) { + return t_context_errorf(ctx, T_ResultCode_ValidationError, tag, str8_zero(), "all requires a field path and at least one allowed value"); + } + T_Result result = t_semantic_match_all(ctx, expected, actual, tag, path, field_path->string); + if (!t_result_is_ok(result)) { return result; } + continue; + } + if (str8_match(tag->string, str8_lit("contains"), StringMatchFlag_CaseInsensitive)) { + String8 needle = t_scalar_string_from_node(tag); + String8 value = t_scalar_string_from_node(actual); + if (str8_find_needle(value, 0, needle, 0) >= value.size) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: '%S' does not contain '%S'", path, value, needle); + } + continue; + } + if (str8_match(tag->string, str8_lit("starts_with"), StringMatchFlag_CaseInsensitive)) { + String8 prefix = t_scalar_string_from_node(tag); + String8 value = t_scalar_string_from_node(actual); + if (!str8_starts_with(value, prefix)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: '%S' does not start with '%S'", path, value, prefix); + } + continue; + } + if (str8_match(tag->string, str8_lit("aligned"), StringMatchFlag_CaseInsensitive)) { + U64 align = 0; + U64 value = 0; + if (!try_u64_from_str8_c_rules(t_scalar_string_from_node(tag), &align) || align == 0 || !t_u64_from_scalar(actual, &value) || value % align != 0) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: value is not aligned", path); + } + continue; + } + if (str8_match(tag->string, str8_lit("range"), StringMatchFlag_CaseInsensitive)) { + MD_Node *min_node = tag->first; + MD_Node *max_node = md_node_is_nil(min_node) ? &md_nil_node : min_node->next; + U64 min = 0, max = 0, value = 0; + if (md_node_is_nil(max_node) || !md_node_is_nil(max_node->next) || !try_u64_from_str8_c_rules(min_node->string, &min) || + !try_u64_from_str8_c_rules(max_node->string, &max) || !t_u64_from_scalar(actual, &value) || value < min || value > max) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: value is outside expected range", path); + } + continue; + } + if (str8_match(tag->string, str8_lit("bits_set"), StringMatchFlag_CaseInsensitive) || str8_match(tag->string, str8_lit("bits_clear"), StringMatchFlag_CaseInsensitive)) { + U64 mask = 0, value = 0; + B32 bits_set = str8_match(tag->string, str8_lit("bits_set"), StringMatchFlag_CaseInsensitive); + if (!try_u64_from_str8_c_rules(t_scalar_string_from_node(tag), &mask) || !t_u64_from_scalar(actual, &value) || (bits_set ? (value & mask) != mask : (value & mask) != 0)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "%S: required bits are not %s", path, bits_set ? "set" : "clear"); + } + continue; + } + if (str8_match(tag->string, str8_lit("absent"), StringMatchFlag_CaseInsensitive)) { continue; } + return t_context_errorf(ctx, T_ResultCode_ValidationError, tag, str8_zero(), "unknown semantic predicate '%S'", tag->string); + } + + for MD_EachNode(expected_child, expected->first) { + MD_Node *actual_child = md_child_from_string(actual, expected_child->string, 0); + String8 child_path = path.size == 0 ? expected_child->string : str8f(ctx->arena, "%S.%S", path, expected_child->string); + B32 expect_absent = !md_node_is_nil(md_tag_from_string(expected_child, str8_lit("absent"), StringMatchFlag_CaseInsensitive)); + if (expect_absent) { + if (!md_node_is_nil(actual_child)) { return t_context_errorf(ctx, T_ResultCode_Mismatch, expected_child, str8_zero(), "%S: field must be absent", child_path); } + continue; + } + if (md_node_is_nil(actual_child)) { return t_context_errorf(ctx, T_ResultCode_Mismatch, expected_child, str8_zero(), "%S: expected field is missing", child_path); } + T_Result result = t_semantic_match_node(ctx, expected_child, actual_child, child_path); + if (!t_result_is_ok(result)) { return result; } + } + + return ctx->result; +} + +internal T_Result +t_semantic_match(T_Context *ctx, MD_Node *expected, MD_Node *actual) +{ + if (!str8_match(expected->string, actual->string, 0)) { + return t_context_errorf(ctx, T_ResultCode_Mismatch, expected, str8_zero(), "expected '%S', got '%S'", expected->string, actual->string); + } + return t_semantic_match_node(ctx, expected, actual, expected->string); +} diff --git a/src/torture/torture_script.h b/src/torture/torture_script.h new file mode 100644 index 000000000..281ae753e --- /dev/null +++ b/src/torture/torture_script.h @@ -0,0 +1,165 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +typedef struct T_Context T_Context; +typedef struct T_ParseContext T_ParseContext; +typedef struct T_Codec T_Codec; +typedef struct T_OpSpec T_OpSpec; +typedef struct T_SuiteSpec T_SuiteSpec; +typedef struct T_Artifact T_Artifact; +typedef struct T_BuildPlan T_BuildPlan; +typedef struct T_ScriptBinding T_ScriptBinding; + +typedef enum T_ArtifactState +{ + T_ArtifactState_Declared, + T_ArtifactState_Validated, + T_ArtifactState_Materialized, + T_ArtifactState_Failed, +} T_ArtifactState; + +typedef enum T_ResultCode +{ + T_ResultCode_Ok, + T_ResultCode_ParseError, + T_ResultCode_ValidationError, + T_ResultCode_IoError, + T_ResultCode_Mismatch, +} T_ResultCode; + +typedef enum T_DiagnosticKind +{ + T_DiagnosticKind_Null, + T_DiagnosticKind_Note, + T_DiagnosticKind_Warning, + T_DiagnosticKind_Error, +} T_DiagnosticKind; + +typedef struct T_Diagnostic T_Diagnostic; +struct T_Diagnostic +{ + T_Diagnostic *next; + MD_MsgKind kind; + String8 file_path; + TxtPt location; + String8 operation; + String8 message; +}; + +typedef struct T_DiagnosticList T_DiagnosticList; +struct T_DiagnosticList +{ + T_Diagnostic *first; + T_Diagnostic *last; + U64 count; +}; + +typedef struct T_Result T_Result; +struct T_Result +{ + T_ResultCode code; + T_DiagnosticList diagnostics; +}; + +struct T_Codec +{ + String8 kind; + T_Result (*validate)(T_ParseContext *ctx, T_Artifact *artifact); + T_Result (*encode)(T_Context *ctx, T_Artifact *artifact); + T_Result (*decode)(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out); +}; + +struct T_OpSpec +{ + String8 name; + T_Result (*validate)(T_ParseContext *ctx, MD_Node *arguments); + T_Result (*execute)(T_Context *ctx, MD_Node *arguments); +}; + +typedef struct T_Command T_Command; +struct T_Command +{ + T_OpSpec *spec; + MD_Node *arguments; + TxtPt location; + U64 order; +}; + +struct T_SuiteSpec +{ + String8 name; + T_Codec *codecs; + U64 codec_count; + T_OpSpec *ops; + U64 op_count; + T_Result (*begin)(T_Context *ctx, MD_Node *test); + void (*end)(T_Context *ctx); +}; + +struct T_Artifact +{ + T_Artifact *next; + String8 name; + String8 file_name; + T_Codec *codec; + MD_Node *definition; + String8 data; + void *codec_data; + T_ArtifactState state; +}; + +struct T_Context +{ + Arena *arena; + TestCtx *test_ctx; + T_SuiteSpec *suite; + String8 file_path; + String8 source; + MD_Node *root; + MD_Node *test; + T_Artifact *first_artifact; + T_Artifact *last_artifact; + U64 artifact_count; + T_BuildPlan *build; + T_ScriptBinding *bindings; + T_Command *commands; + U64 command_count; + T_Result result; + void *suite_data; +}; + +struct T_ParseContext +{ + Arena *arena; + T_Context *run; + T_SuiteSpec *suite; + String8 file_path; + String8 source; + String8 operation; +}; + +internal B32 t_result_is_ok(T_Result result); +internal T_Result t_context_errorf(T_Context *ctx, T_ResultCode code, MD_Node *node, String8 operation, char *fmt, ...); +internal T_Result t_parse_errorf(T_ParseContext *ctx, T_ResultCode code, MD_Node *node, char *fmt, ...); + +internal T_Codec *t_codec_from_kind(T_SuiteSpec *suite, String8 kind); +internal T_OpSpec *t_op_spec_from_name(T_SuiteSpec *suite, String8 name); +internal T_Artifact *t_artifact_from_name(T_Context *ctx, String8 name); + +internal T_Result t_script_parse(Arena *arena, TestCtx *test_ctx, T_SuiteSpec *suite, String8 file_path, String8 source, T_Context *ctx_out); +internal T_Result t_script_execute(T_Context *ctx); +internal T_Result t_bytes_from_producer(T_Context *ctx, MD_Node *producer, String8 *data_out); + +internal T_Result t_codec_bytes_encode(T_Context *ctx, T_Artifact *artifact); +internal T_Result t_codec_bytes_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out); +internal T_Result t_codec_text_encode(T_Context *ctx, T_Artifact *artifact); +internal T_Result t_codec_text_decode(T_Context *ctx, T_Artifact *artifact, MD_Node **semantic_tree_out); +internal T_Result t_op_compare_validate(T_ParseContext *ctx, MD_Node *arguments); +internal T_Result t_op_compare_execute(T_Context *ctx, MD_Node *arguments); +internal T_Result t_semantic_match(T_Context *ctx, MD_Node *expected, MD_Node *actual); + +global T_Codec t_codec_bytes; +global T_Codec t_codec_text; +global T_OpSpec t_op_compare; From b4c52780377a66cd9590b831fe8524d20ad8bf70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C4=81rti=C5=86=C5=A1=20Mo=C5=BEeiko?= Date: Wed, 2 Sep 2026 18:26:14 -0700 Subject: [PATCH 002/121] blake3: update to 1.8.7 version and use asm implementations Assembly code is >25% faster than intrinsic implementations on msvc. --- .github/workflows/builds.yml | 2 + build.bat | 16 + build.sh | 17 +- src/linker/base_ext/base_blake3.c | 99 +- src/linker/base_ext/base_blake3.h | 2 +- src/third_party/blake3/LICENSE_CC0 | 121 + src/third_party/blake3/asm/LICENSE | 330 --- src/third_party/blake3/asm/blake3.h | 82 - src/third_party/blake3/asm/blake3_dispatch.c | 278 -- src/third_party/blake3/{asm => }/blake3.c | 131 +- src/third_party/blake3/{c => }/blake3.h | 8 +- .../{asm => }/blake3_avx2_x86-64_unix.S | 0 .../blake3_avx2_x86-64_windows_msvc.asm | 0 .../{asm => }/blake3_avx512_x86-64_unix.S | 2239 +++++++++++++++++ .../blake3_avx512_x86-64_windows_msvc.asm | 0 .../blake3/{c => }/blake3_dispatch.c | 74 +- .../blake3/{asm => }/blake3_impl.h | 52 +- .../blake3/{asm => }/blake3_neon.c | 17 +- .../blake3/{asm => }/blake3_portable.c | 0 .../{asm => }/blake3_sse2_x86-64_unix.S | 0 .../blake3_sse2_x86-64_windows_msvc.asm | 0 .../{asm => }/blake3_sse41_x86-64_unix.S | 0 .../blake3_sse41_x86-64_windows_msvc.asm | 0 src/third_party/blake3/c/LICENSE | 330 --- src/third_party/blake3/c/blake3.c | 616 ----- src/third_party/blake3/c/blake3_avx2.c | 312 --- src/third_party/blake3/c/blake3_avx512.c | 1220 --------- src/third_party/blake3/c/blake3_impl.h | 285 --- src/third_party/blake3/c/blake3_neon.c | 368 --- src/third_party/blake3/c/blake3_portable.c | 160 -- src/third_party/blake3/c/blake3_sse2.c | 566 ----- src/third_party/blake3/c/blake3_sse41.c | 560 ----- 32 files changed, 2585 insertions(+), 5300 deletions(-) create mode 100644 src/third_party/blake3/LICENSE_CC0 delete mode 100644 src/third_party/blake3/asm/LICENSE delete mode 100644 src/third_party/blake3/asm/blake3.h delete mode 100644 src/third_party/blake3/asm/blake3_dispatch.c rename src/third_party/blake3/{asm => }/blake3.c (87%) rename src/third_party/blake3/{c => }/blake3.h (90%) rename src/third_party/blake3/{asm => }/blake3_avx2_x86-64_unix.S (100%) rename src/third_party/blake3/{asm => }/blake3_avx2_x86-64_windows_msvc.asm (100%) rename src/third_party/blake3/{asm => }/blake3_avx512_x86-64_unix.S (55%) rename src/third_party/blake3/{asm => }/blake3_avx512_x86-64_windows_msvc.asm (100%) rename src/third_party/blake3/{c => }/blake3_dispatch.c (85%) rename src/third_party/blake3/{asm => }/blake3_impl.h (84%) rename src/third_party/blake3/{asm => }/blake3_neon.c (96%) rename src/third_party/blake3/{asm => }/blake3_portable.c (100%) rename src/third_party/blake3/{asm => }/blake3_sse2_x86-64_unix.S (100%) rename src/third_party/blake3/{asm => }/blake3_sse2_x86-64_windows_msvc.asm (100%) rename src/third_party/blake3/{asm => }/blake3_sse41_x86-64_unix.S (100%) rename src/third_party/blake3/{asm => }/blake3_sse41_x86-64_windows_msvc.asm (100%) delete mode 100644 src/third_party/blake3/c/LICENSE delete mode 100644 src/third_party/blake3/c/blake3.c delete mode 100644 src/third_party/blake3/c/blake3_avx2.c delete mode 100644 src/third_party/blake3/c/blake3_avx512.c delete mode 100644 src/third_party/blake3/c/blake3_impl.h delete mode 100644 src/third_party/blake3/c/blake3_neon.c delete mode 100644 src/third_party/blake3/c/blake3_portable.c delete mode 100644 src/third_party/blake3/c/blake3_sse2.c delete mode 100644 src/third_party/blake3/c/blake3_sse41.c diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 58a208670..f666f2fcb 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -45,6 +45,7 @@ jobs: - name: build shell: bash run: | + [[ "${{ matrix.compiler }}" == "clang-22" ]] && export AR=llvm-ar-22 CC="${{ matrix.compiler }}" ./build.sh "${{ matrix.target }}" "${{ matrix.mode }}" build-windows-2022: @@ -103,6 +104,7 @@ jobs: - name: Run shell: bash run: | + [[ "${{ matrix.compiler }}" == "clang-22" ]] && export AR=llvm-ar-22 CC="${{ matrix.compiler }}" ./run_tests.sh clang "${{ matrix.mode }}" -- \* $TORTURE_FLAGS - name: Upload Artifacts if: failure() diff --git a/build.bat b/build.bat index e51a529eb..6a692d18f 100644 --- a/build.bat +++ b/build.bat @@ -98,6 +98,10 @@ if "%msvc%"=="1" set rc=call rc if "%clang%"=="1" set rc=call llvm-rc if "%msvc%"=="1" set link_dll=/link /DLL if "%clang%"=="1" set link_dll=-Xlinker -DLL +if "%msvc%"=="1" set asm=call ml64 -nologo -c -Zi -Fo +if "%clang%"=="1" set asm=call ml64 -nologo -c -Zi -Fo +if "%msvc%"=="1" set mklib=call lib -nologo +if "%clang%"=="1" set mklib=call llvm-lib :: --- Choose Compile/Link Lines ---------------------------------------------- if "%msvc%"=="1" set compile_debug=%cl_debug% @@ -138,6 +142,18 @@ if "%no_meta%"=="" if exist metagen.exe ( ) popd +:: --- Assemble BLAKE3 -------------------------------------------------------- +if not exist build\blake3.lib if "%VSCMD_ARG_TGT_ARCH%" equ "x64" ( + echo [assembling blake3] + pushd build + %asm% blake3_sse2_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_sse2_x86-64_windows_msvc.asm || exit /b 1 + %asm% blake3_sse41_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_sse41_x86-64_windows_msvc.asm || exit /b 1 + %asm% blake3_avx2_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_avx2_x86-64_windows_msvc.asm || exit /b 1 + %asm% blake3_avx512_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_avx512_x86-64_windows_msvc.asm || exit /b 1 + %mklib% -out:blake3.lib blake3_*_msvc.obj || exit /b 1 + popd +) + :: --- Build Everything (@build_targets) -------------------------------------- pushd build if "%raddbg%"=="1" set didbuild=1 && %compile% ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg.exe || exit /b 1 diff --git a/build.sh b/build.sh index 3724a40d4..a3d119505 100755 --- a/build.sh +++ b/build.sh @@ -42,8 +42,8 @@ fi cc_icon="-DLNX_WM_ICON=1" # --- Choose Compile/Link Lines ----------------------------------------------- -if [[ "${gcc:-0}" == "1" ]]; then compiler="${CC:-gcc} $cc_cflags_gcc"; echo "[gcc compile]"; -elif [[ "${clang:-1}" == "1" ]]; then compiler="${CC:-clang} $cc_cflags_clang"; echo "[clang compile]"; +if [[ "${gcc:-0}" == "1" ]]; then compiler="${CC:-gcc} $cc_cflags_gcc"; ar="${AR:-ar}"; echo "[gcc compile]"; +elif [[ "${clang:-1}" == "1" ]]; then compiler="${CC:-clang} $cc_cflags_clang"; ar="${AR:-llvm-ar}"; echo "[clang compile]"; fi if [[ "${release:-0}" == "1" ]]; then echo "[release mode]"; compile="$compiler $cc_release"; elif [[ "${debug:-1}" == "1" ]]; then echo "[debug mode]"; compile="$compiler $cc_debug"; @@ -62,12 +62,23 @@ then cd .. fi +# --- Assemble BLAKE3 --------------------------------------------------------- +if [[ ! -f "build/blake3.a" ]] +then + echo "[assembling blake3]" + $compiler -c -g -o build/blake3_sse2_x86-64_unix.o src/third_party/blake3/blake3_sse2_x86-64_unix.S + $compiler -c -g -o build/blake3_sse41_x86-64_unix.o src/third_party/blake3/blake3_sse41_x86-64_unix.S + $compiler -c -g -o build/blake3_avx2_x86-64_unix.o src/third_party/blake3/blake3_avx2_x86-64_unix.S + $compiler -c -g -o build/blake3_avx512_x86-64_unix.o src/third_party/blake3/blake3_avx512_x86-64_unix.S + $ar rs build/blake3.a build/blake3_*_unix.o +fi + # --- Build Everything (@build_targets) --------------------------------------- cd build if [[ "${raddbg:-0}" == "1" ]]; then didbuild=1 && $compile ../src/raddbg/raddbg_main.c $cc_icon $cc_link $cc_os_gfx $cc_render $cc_font_provider -o raddbg; fi if [[ "${raddbg_non_graphical:-0}" == "1" ]]; then didbuild=1 && $compile ../src/raddbg/raddbg_main.c -DWM_STUB=1 -DR_BACKEND=R_BACKEND_STUB $cc_link $cc_os_gfx $cc_render $cc_font_provider -o raddbg_non_graphical; fi if [[ "${radbin:-0}" == "1" ]]; then didbuild=1 && $compile ../src/radbin/radbin_main.c $cc_link -o radbin; fi -if [[ "${radlink:-0}" == "1" ]]; then didbuild=1 && $compile ../src/linker/lnk.c $cc_link -o radlink; fi +if [[ "${radlink:-0}" == "1" ]]; then didbuild=1 && $compile ../src/linker/lnk.c $cc_link -o radlink blake3.a; fi if [[ "${torture:-0}" == "1" ]]; then didbuild=1 && $compile ../src/torture/torture_main.c $cc_link $cc_os_gfx $cc_render $cc_font_provider -o torture; fi cd .. diff --git a/src/linker/base_ext/base_blake3.c b/src/linker/base_ext/base_blake3.c index 59f063c42..df9324555 100644 --- a/src/linker/base_ext/base_blake3.c +++ b/src/linker/base_ext/base_blake3.c @@ -1,103 +1,16 @@ // Copyright (c) Epic Games Tools // Licensed under the MIT license (https://opensource.org/license/mit/) -#if defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wmacro-redefined" -#pragma clang diagnostic ignored "-Wsign-conversion" -#elif defined(_MSC_VER) -#pragma warning (push, 0) -#endif - -#include "third_party/blake3/c/blake3_portable.c" - -#if defined(_M_AMD64) || defined(__x86_64__) - -#define round_fn sse2_round_fn -#define compress_pre sse2_compress_pre - -#include "third_party/blake3/c/blake3_sse2.c" -#define loadu sse41_loadu -#define storeu sse41_storeu -#define addv sse41_addv -#define xorv sse41_xorv -#define set1 sse41_set1 -#define set4 sse41_set4 -#define rot16 sse41_rot16 -#define rot12 sse41_rot12 -#define rot8 sse41_rot8 -#define rot7 sse41_rot7 -#define g1 sse41_g1 -#define g2 sse41_g2 -#define diagonalize sse41_diagonalize -#define undiagonalize sse41_undiagonalize -#define compress_pre sse41_compress_pre -#define round_fn sse41_round_fn -#define transpose_vecs sse41_transpose_vecs -#define transpose_msg_vecs sse41_transpose_msg_vecs -#define load_counters sse41_load_counters - -#if defined(__clang__) -#pragma clang attribute push(__attribute__((target("sse4.1"))), apply_to=function) -#endif -#include "third_party/blake3/c/blake3_sse41.c" -#if defined(__clang__) -#pragma clang attribute pop -#endif - -#define loadu avx2_loadu -#define storeu avx2_storeu -#define addv avx2_addv -#define xorv avx2_xorv -#define set1 avx2_set1 -#define rot7 avx2_rot7 -#define rot8 avx2_rot8 -#define rot12 avx2_rot12 -#define rot16 avx2_rot16 -#define round_fn avx2_round_fn -#define transpose_vecs avx2_transpose_vecs -#define transpose_msg_vecs avx2_transpose_msg_vecs -#define load_counters avx2_load_counters - -#if defined(__clang__) -#pragma clang attribute push(__attribute__((target("avx2"))), apply_to=function) -#endif -#include "third_party/blake3/c/blake3_avx2.c" -#if defined(__clang__) -#pragma clang attribute pop -#endif - -#define set4 avx512_set4 -#define g1 avx512_g1 -#define g2 avx512_g2 -#define diagonalize avx512_diagonalize -#define undiagonalize avx512_undiagonalize -#define compress_pre avx512_compress_pre -#define transpose_vecs avx512_transpose_vecs -#define transpose_msg_vecs avx512_transpose_msg_vecs -#define load_counters avx512_load_counters - -#if defined(__clang__) -#pragma clang attribute push(__attribute__((target("avx512f,avx512vl"))), apply_to=function) -#endif -#include "third_party/blake3/c/blake3_avx512.c" -#if defined(__clang__) -#pragma clang attribute pop -#endif - -#endif +#include "third_party/blake3/blake3_portable.c" #if defined(__aarch64__) || defined(_M_ARM64) -#include "third_party/blake3/c/blake3_neon.c" +#include "third_party/blake3/blake3_neon.c" #endif -#include "third_party/blake3/c/blake3_dispatch.c" -#include "third_party/blake3/c/blake3.c" +#include "third_party/blake3/blake3_dispatch.c" +#include "third_party/blake3/blake3.c" -#if defined(__clang__) -#pragma clang diagnostic pop -#elif defined(_MSC_VER) -#pragma warning (pop, 0) +#if defined(_MSC_VER) && defined(_M_AMD64) +#pragma comment (lib, "blake3") #endif - diff --git a/src/linker/base_ext/base_blake3.h b/src/linker/base_ext/base_blake3.h index e80c1e35a..90e7cdfdf 100644 --- a/src/linker/base_ext/base_blake3.h +++ b/src/linker/base_ext/base_blake3.h @@ -6,7 +6,7 @@ #define BLAKE3_API static #define BLAKE3_PRIVATE static -#include "third_party/blake3/c/blake3.h" +#include "third_party/blake3/blake3.h" static void blake3(void* out, size_t outlen, void* in, size_t inlen) diff --git a/src/third_party/blake3/LICENSE_CC0 b/src/third_party/blake3/LICENSE_CC0 new file mode 100644 index 000000000..0e259d42c --- /dev/null +++ b/src/third_party/blake3/LICENSE_CC0 @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/src/third_party/blake3/asm/LICENSE b/src/third_party/blake3/asm/LICENSE deleted file mode 100644 index f5892efc3..000000000 --- a/src/third_party/blake3/asm/LICENSE +++ /dev/null @@ -1,330 +0,0 @@ -This work is released into the public domain with CC0 1.0. Alternatively, it is -licensed under the Apache License 2.0. - -------------------------------------------------------------------------------- - -Creative Commons Legal Code - -CC0 1.0 Universal - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator -and subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for -the purpose of contributing to a commons of creative, cultural and -scientific works ("Commons") that the public can reliably and without fear -of later claims of infringement build upon, modify, incorporate in other -works, reuse and redistribute as freely as possible in any form whatsoever -and for any purposes, including without limitation commercial purposes. -These owners may contribute to the Commons to promote the ideal of a free -culture and the further production of creative, cultural and scientific -works, or to gain reputation or greater distribution for their Work in -part through the use and efforts of others. - -For these and/or other purposes and motivations, and without any -expectation of additional consideration or compensation, the person -associating CC0 with a Work (the "Affirmer"), to the extent that he or she -is an owner of Copyright and Related Rights in the Work, voluntarily -elects to apply CC0 to the Work and publicly distribute the Work under its -terms, with knowledge of his or her Copyright and Related Rights in the -Work and the meaning and intended legal effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not -limited to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, - communicate, and translate a Work; - ii. moral rights retained by the original author(s) and/or performer(s); -iii. publicity and privacy rights pertaining to a person's image or - likeness depicted in a Work; - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - v. rights protecting the extraction, dissemination, use and reuse of data - in a Work; - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation - thereof, including any amended or successor version of such - directive); and -vii. other similar, equivalent or corresponding rights throughout the - world based on applicable law or treaty, and any national - implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention -of, applicable law, Affirmer hereby overtly, fully, permanently, -irrevocably and unconditionally waives, abandons, and surrenders all of -Affirmer's Copyright and Related Rights and associated claims and causes -of action, whether now known or unknown (including existing as well as -future claims and causes of action), in the Work (i) in all territories -worldwide, (ii) for the maximum duration provided by applicable law or -treaty (including future time extensions), (iii) in any current or future -medium and for any number of copies, and (iv) for any purpose whatsoever, -including without limitation commercial, advertising or promotional -purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each -member of the public at large and to the detriment of Affirmer's heirs and -successors, fully intending that such Waiver shall not be subject to -revocation, rescission, cancellation, termination, or any other legal or -equitable action to disrupt the quiet enjoyment of the Work by the public -as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason -be judged legally invalid or ineffective under applicable law, then the -Waiver shall be preserved to the maximum extent permitted taking into -account Affirmer's express Statement of Purpose. In addition, to the -extent the Waiver is so judged Affirmer hereby grants to each affected -person a royalty-free, non transferable, non sublicensable, non exclusive, -irrevocable and unconditional license to exercise Affirmer's Copyright and -Related Rights in the Work (i) in all territories worldwide, (ii) for the -maximum duration provided by applicable law or treaty (including future -time extensions), (iii) in any current or future medium and for any number -of copies, and (iv) for any purpose whatsoever, including without -limitation commercial, advertising or promotional purposes (the -"License"). The License shall be deemed effective as of the date CC0 was -applied by Affirmer to the Work. Should any part of the License for any -reason be judged legally invalid or ineffective under applicable law, such -partial invalidity or ineffectiveness shall not invalidate the remainder -of the License, and in such case Affirmer hereby affirms that he or she -will not (i) exercise any of his or her remaining Copyright and Related -Rights in the Work or (ii) assert any associated claims and causes of -action with respect to the Work, in either case contrary to Affirmer's -express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - b. Affirmer offers the Work as-is and makes no representations or - warranties of any kind concerning the Work, express, implied, - statutory or otherwise, including without limitation warranties of - title, merchantability, fitness for a particular purpose, non - infringement, or the absence of latent or other defects, accuracy, or - the present or absence of errors, whether or not discoverable, all to - the greatest extent permissible under applicable law. - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without - limitation any person's Copyright and Related Rights in the Work. - Further, Affirmer disclaims responsibility for obtaining any necessary - consents, permissions or other rights required for any use of the - Work. - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to - this CC0 or use of the Work. - -------------------------------------------------------------------------------- - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2019 Jack O'Connor and Samuel Neves - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/third_party/blake3/asm/blake3.h b/src/third_party/blake3/asm/blake3.h deleted file mode 100644 index f694dcf27..000000000 --- a/src/third_party/blake3/asm/blake3.h +++ /dev/null @@ -1,82 +0,0 @@ -#ifndef BLAKE3_H -#define BLAKE3_H - -#include -#include - -#if !defined(BLAKE3_API) -# if defined(_WIN32) || defined(__CYGWIN__) -# if defined(BLAKE3_DLL) -# if defined(BLAKE3_DLL_EXPORTS) -# define BLAKE3_API __declspec(dllexport) -# else -# define BLAKE3_API __declspec(dllimport) -# endif -# define BLAKE3_PRIVATE -# else -# define BLAKE3_API -# define BLAKE3_PRIVATE -# endif -# elif __GNUC__ >= 4 -# define BLAKE3_API __attribute__((visibility("default"))) -# define BLAKE3_PRIVATE __attribute__((visibility("hidden"))) -# else -# define BLAKE3_API -# define BLAKE3_PRIVATE -# endif -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#define BLAKE3_VERSION_STRING "1.5.0" -#define BLAKE3_KEY_LEN 32 -#define BLAKE3_OUT_LEN 32 -#define BLAKE3_BLOCK_LEN 64 -#define BLAKE3_CHUNK_LEN 1024 -#define BLAKE3_MAX_DEPTH 54 - -// This struct is a private implementation detail. It has to be here because -// it's part of blake3_hasher below. -typedef struct { - uint32_t cv[8]; - uint64_t chunk_counter; - uint8_t buf[BLAKE3_BLOCK_LEN]; - uint8_t buf_len; - uint8_t blocks_compressed; - uint8_t flags; -} blake3_chunk_state; - -typedef struct { - uint32_t key[8]; - blake3_chunk_state chunk; - uint8_t cv_stack_len; - // The stack size is MAX_DEPTH + 1 because we do lazy merging. For example, - // with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk - // requires a 4th entry, rather than merging everything down to 1, because we - // don't know whether more input is coming. This is different from how the - // reference implementation does things. - uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN]; -} blake3_hasher; - -BLAKE3_API const char *blake3_version(void); -BLAKE3_API void blake3_hasher_init(blake3_hasher *self); -BLAKE3_API void blake3_hasher_init_keyed(blake3_hasher *self, - const uint8_t key[BLAKE3_KEY_LEN]); -BLAKE3_API void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context); -BLAKE3_API void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, - size_t context_len); -BLAKE3_API void blake3_hasher_update(blake3_hasher *self, const void *input, - size_t input_len); -BLAKE3_API void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, - size_t out_len); -BLAKE3_API void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, - uint8_t *out, size_t out_len); -BLAKE3_API void blake3_hasher_reset(blake3_hasher *self); - -#ifdef __cplusplus -} -#endif - -#endif /* BLAKE3_H */ diff --git a/src/third_party/blake3/asm/blake3_dispatch.c b/src/third_party/blake3/asm/blake3_dispatch.c deleted file mode 100644 index 6f814569b..000000000 --- a/src/third_party/blake3/asm/blake3_dispatch.c +++ /dev/null @@ -1,278 +0,0 @@ -#include -#include -#include - -#include "blake3_impl.h" - -#if defined(IS_X86) -#if defined(_MSC_VER) -#include -#elif defined(__GNUC__) -#include -#else -#undef IS_X86 /* Unimplemented! */ -#endif -#endif - -#define MAYBE_UNUSED(x) (void)((x)) - -#if defined(IS_X86) -static uint64_t xgetbv(void) { -#if defined(_MSC_VER) - return _xgetbv(0); -#else - uint32_t eax = 0, edx = 0; - __asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0)); - return ((uint64_t)edx << 32) | eax; -#endif -} - -static void cpuid(uint32_t out[4], uint32_t id) { -#if defined(_MSC_VER) - __cpuid((int *)out, id); -#elif defined(__i386__) || defined(_M_IX86) - __asm__ __volatile__("movl %%ebx, %1\n" - "cpuid\n" - "xchgl %1, %%ebx\n" - : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) - : "a"(id)); -#else - __asm__ __volatile__("cpuid\n" - : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) - : "a"(id)); -#endif -} - -static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) { -#if defined(_MSC_VER) - __cpuidex((int *)out, id, sid); -#elif defined(__i386__) || defined(_M_IX86) - __asm__ __volatile__("movl %%ebx, %1\n" - "cpuid\n" - "xchgl %1, %%ebx\n" - : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) - : "a"(id), "c"(sid)); -#else - __asm__ __volatile__("cpuid\n" - : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) - : "a"(id), "c"(sid)); -#endif -} - -#endif - -enum cpu_feature { - SSE2 = 1 << 0, - SSSE3 = 1 << 1, - SSE41 = 1 << 2, - AVX = 1 << 3, - AVX2 = 1 << 4, - AVX512F = 1 << 5, - AVX512VL = 1 << 6, - /* ... */ - UNDEFINED = 1 << 30 -}; - -#if !defined(BLAKE3_TESTING) -static /* Allow the variable to be controlled manually for testing */ -#endif - volatile int g_cpu_features = UNDEFINED; - -#if !defined(BLAKE3_TESTING) -static -#endif - enum cpu_feature - get_cpu_features(void) { - - /* If TSAN detects a data race here, try compiling with -DBLAKE3_ATOMICS=1 */ - long features = g_cpu_features; - if (features != UNDEFINED) { - return (enum cpu_feature)features; - } else { -#if defined(IS_X86) - uint32_t regs[4] = {0}; - uint32_t *eax = ®s[0], *ebx = ®s[1], *ecx = ®s[2], *edx = ®s[3]; - (void)edx; - features = 0; - cpuid(regs, 0); - const int max_id = *eax; - cpuid(regs, 1); -#if defined(__amd64__) || defined(_M_X64) - features |= SSE2; -#else - if (*edx & (1UL << 26)) - features |= SSE2; -#endif - if (*ecx & (1UL << 9)) - features |= SSSE3; - if (*ecx & (1UL << 19)) - features |= SSE41; - - if (*ecx & (1UL << 27)) { // OSXSAVE - const uint64_t mask = xgetbv(); - if ((mask & 6) == 6) { // SSE and AVX states - if (*ecx & (1UL << 28)) - features |= AVX; - if (max_id >= 7) { - cpuidex(regs, 7, 0); - if (*ebx & (1UL << 5)) - features |= AVX2; - if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm - if (*ebx & (1UL << 31)) - features |= AVX512VL; - if (*ebx & (1UL << 16)) - features |= AVX512F; - } - } - } - } - g_cpu_features = features; - return (enum cpu_feature)features; -#else - /* How to detect NEON? */ - return 0; -#endif - } -} - -void blake3_compress_in_place(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { -#if defined(IS_X86) - const enum cpu_feature features = get_cpu_features(); - MAYBE_UNUSED(features); -#if !defined(BLAKE3_NO_AVX512) - if (features & AVX512VL) { - blake3_compress_in_place_avx512(cv, block, block_len, counter, flags); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE41) - if (features & SSE41) { - blake3_compress_in_place_sse41(cv, block, block_len, counter, flags); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE2) - if (features & SSE2) { - blake3_compress_in_place_sse2(cv, block, block_len, counter, flags); - return; - } -#endif -#endif - blake3_compress_in_place_portable(cv, block, block_len, counter, flags); -} - -void blake3_compress_xof(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags, - uint8_t out[64]) { -#if defined(IS_X86) - const enum cpu_feature features = get_cpu_features(); - MAYBE_UNUSED(features); -#if !defined(BLAKE3_NO_AVX512) - if (features & AVX512VL) { - blake3_compress_xof_avx512(cv, block, block_len, counter, flags, out); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE41) - if (features & SSE41) { - blake3_compress_xof_sse41(cv, block, block_len, counter, flags, out); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE2) - if (features & SSE2) { - blake3_compress_xof_sse2(cv, block, block_len, counter, flags, out); - return; - } -#endif -#endif - blake3_compress_xof_portable(cv, block, block_len, counter, flags, out); -} - -void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { -#if defined(IS_X86) - const enum cpu_feature features = get_cpu_features(); - MAYBE_UNUSED(features); -#if !defined(BLAKE3_NO_AVX512) - if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { - blake3_hash_many_avx512(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); - return; - } -#endif -#if !defined(BLAKE3_NO_AVX2) - if (features & AVX2) { - blake3_hash_many_avx2(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE41) - if (features & SSE41) { - blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE2) - if (features & SSE2) { - blake3_hash_many_sse2(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); - return; - } -#endif -#endif - -#if BLAKE3_USE_NEON == 1 - blake3_hash_many_neon(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, out); - return; -#endif - - blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); -} - -// The dynamically detected SIMD degree of the current platform. -size_t blake3_simd_degree(void) { -#if defined(IS_X86) - const enum cpu_feature features = get_cpu_features(); - MAYBE_UNUSED(features); -#if !defined(BLAKE3_NO_AVX512) - if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { - return 16; - } -#endif -#if !defined(BLAKE3_NO_AVX2) - if (features & AVX2) { - return 8; - } -#endif -#if !defined(BLAKE3_NO_SSE41) - if (features & SSE41) { - return 4; - } -#endif -#if !defined(BLAKE3_NO_SSE2) - if (features & SSE2) { - return 4; - } -#endif -#endif -#if BLAKE3_USE_NEON == 1 - return 4; -#endif - return 1; -} diff --git a/src/third_party/blake3/asm/blake3.c b/src/third_party/blake3/blake3.c similarity index 87% rename from src/third_party/blake3/asm/blake3.c rename to src/third_party/blake3/blake3.c index 692f4b021..00f91f444 100644 --- a/src/third_party/blake3/asm/blake3.c +++ b/src/third_party/blake3/blake3.c @@ -1,6 +1,7 @@ #include #include #include +#include #include "blake3.h" #include "blake3_impl.h" @@ -88,24 +89,30 @@ INLINE void output_chaining_value(const output_t *self, uint8_t cv[32]) { INLINE void output_root_bytes(const output_t *self, uint64_t seek, uint8_t *out, size_t out_len) { + if (out_len == 0) { + return; + } uint64_t output_block_counter = seek / 64; size_t offset_within_block = seek % 64; uint8_t wide_buf[64]; - while (out_len > 0) { - blake3_compress_xof(self->input_cv, self->block, self->block_len, - output_block_counter, self->flags | ROOT, wide_buf); - size_t available_bytes = 64 - offset_within_block; - size_t memcpy_len; - if (out_len > available_bytes) { - memcpy_len = available_bytes; - } else { - memcpy_len = out_len; - } - memcpy(out, wide_buf + offset_within_block, memcpy_len); - out += memcpy_len; - out_len -= memcpy_len; + if(offset_within_block) { + blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); + const size_t available_bytes = 64 - offset_within_block; + const size_t bytes = out_len > available_bytes ? available_bytes : out_len; + memcpy(out, wide_buf + offset_within_block, bytes); + out += bytes; + out_len -= bytes; output_block_counter += 1; - offset_within_block = 0; + } + if(out_len / 64) { + blake3_xof_many(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, out, out_len / 64); + } + output_block_counter += out_len / 64; + out += out_len & -64; + out_len -= out_len & -64; + if(out_len) { + blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); + memcpy(out, wide_buf, out_len); } } @@ -134,9 +141,7 @@ INLINE void chunk_state_update(blake3_chunk_state *self, const uint8_t *input, input_len -= BLAKE3_BLOCK_LEN; } - size_t take = chunk_state_fill_buf(self, input, input_len); - input += take; - input_len -= take; + chunk_state_fill_buf(self, input, input_len); } INLINE output_t chunk_state_output(const blake3_chunk_state *self) { @@ -154,10 +159,10 @@ INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], // Given some input larger than one chunk, return the number of bytes that // should go in the left subtree. This is the largest power-of-2 number of // chunks that leaves at least 1 byte for the right subtree. -INLINE size_t left_len(size_t content_len) { - // Subtract 1 to reserve at least one byte for the right side. content_len +INLINE size_t left_subtree_len(size_t input_len) { + // Subtract 1 to reserve at least one byte for the right side. input_len // should always be greater than BLAKE3_CHUNK_LEN. - size_t full_chunks = (content_len - 1) / BLAKE3_CHUNK_LEN; + size_t full_chunks = (input_len - 1) / BLAKE3_CHUNK_LEN; return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN; } @@ -261,11 +266,10 @@ INLINE size_t compress_parents_parallel(const uint8_t *child_chaining_values, // Why not just have the caller split the input on the first update(), instead // of implementing this special rule? Because we don't want to limit SIMD or // multi-threading parallelism for that update(). -static size_t blake3_compress_subtree_wide(const uint8_t *input, - size_t input_len, - const uint32_t key[8], - uint64_t chunk_counter, - uint8_t flags, uint8_t *out) { +size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out, bool use_tbb) { // Note that the single chunk case does *not* bump the SIMD degree up to 2 // when it is 1. If this implementation adds multi-threading in the future, // this gives us the option of multi-threading even the 2-chunk case, which @@ -279,7 +283,7 @@ static size_t blake3_compress_subtree_wide(const uint8_t *input, // the input into left and right subtrees. (Note that this is only optimal // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree // of 3 or something, we'll need a more complicated strategy.) - size_t left_input_len = left_len(input_len); + size_t left_input_len = left_subtree_len(input_len); size_t right_input_len = input_len - left_input_len; const uint8_t *right_input = &input[left_input_len]; uint64_t right_chunk_counter = @@ -299,12 +303,24 @@ static size_t blake3_compress_subtree_wide(const uint8_t *input, } uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN]; - // Recurse! If this implementation adds multi-threading support in the - // future, this is where it will go. - size_t left_n = blake3_compress_subtree_wide(input, left_input_len, key, - chunk_counter, flags, cv_array); - size_t right_n = blake3_compress_subtree_wide( - right_input, right_input_len, key, right_chunk_counter, flags, right_cvs); + // Recurse! + size_t left_n = SIZE_MAX; + size_t right_n = SIZE_MAX; + +#if defined(BLAKE3_USE_TBB) + blake3_compress_subtree_wide_join_tbb( + key, flags, use_tbb, + // left-hand side + input, left_input_len, chunk_counter, cv_array, &left_n, + // right-hand side + right_input, right_input_len, right_chunk_counter, right_cvs, &right_n); +#else + left_n = blake3_compress_subtree_wide( + input, left_input_len, key, chunk_counter, flags, cv_array, use_tbb); + right_n = blake3_compress_subtree_wide(right_input, right_input_len, key, + right_chunk_counter, flags, right_cvs, + use_tbb); +#endif // BLAKE3_USE_TBB // The special case again. If simd_degree=1, then we'll have left_n=1 and // right_n=1. Rather than compressing them into a single output, return @@ -330,32 +346,37 @@ static size_t blake3_compress_subtree_wide(const uint8_t *input, // // As with compress_subtree_wide(), this function is not used on inputs of 1 // chunk or less. That's a different codepath. -INLINE void compress_subtree_to_parent_node( - const uint8_t *input, size_t input_len, const uint32_t key[8], - uint64_t chunk_counter, uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN]) { +INLINE void +compress_subtree_to_parent_node(const uint8_t *input, size_t input_len, + const uint32_t key[8], uint64_t chunk_counter, + uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN], + bool use_tbb) { #if defined(BLAKE3_TESTING) assert(input_len > BLAKE3_CHUNK_LEN); #endif uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key, - chunk_counter, flags, cv_array); + chunk_counter, flags, cv_array, use_tbb); assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); - - // If MAX_SIMD_DEGREE is greater than 2 and there's enough input, + // The following loop never executes when MAX_SIMD_DEGREE_OR_2 is 2, because + // as we just asserted, num_cvs will always be <=2 in that case. But GCC + // (particularly GCC 8.5) can't tell that it never executes, and if NDEBUG is + // set then it emits incorrect warnings here. We tried a few different + // hacks to silence these, but in the end our hacks just produced different + // warnings (see https://github.com/BLAKE3-team/BLAKE3/pull/380). Out of + // desperation, we ifdef out this entire loop when we know it's not needed. +#if MAX_SIMD_DEGREE_OR_2 > 2 + // If MAX_SIMD_DEGREE_OR_2 is greater than 2 and there's enough input, // compress_subtree_wide() returns more than 2 chaining values. Condense // them into 2 by forming parent nodes repeatedly. uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; - // The second half of this loop condition is always true, and we just - // asserted it above. But GCC can't tell that it's always true, and if NDEBUG - // is set on platforms where MAX_SIMD_DEGREE_OR_2 == 2, GCC emits spurious - // warnings here. GCC 8.5 is particularly sensitive, so if you're changing - // this code, test it against that version. - while (num_cvs > 2 && num_cvs <= MAX_SIMD_DEGREE_OR_2) { + while (num_cvs > 2) { num_cvs = compress_parents_parallel(cv_array, num_cvs, key, flags, out_array); memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); } +#endif memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); } @@ -427,7 +448,7 @@ INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) { // of the whole tree, and it would need to be ROOT finalized. We can't // compress it until we know. // 2) This 64 KiB input might complete a larger tree, whose root node is -// similarly going to be the the root of the whole tree. For example, maybe +// similarly going to be the root of the whole tree. For example, maybe // we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the // node at the root of the 256 KiB subtree until we know how to finalize it. // @@ -452,8 +473,8 @@ INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], self->cv_stack_len += 1; } -void blake3_hasher_update(blake3_hasher *self, const void *input, - size_t input_len) { +INLINE void blake3_hasher_update_base(blake3_hasher *self, const void *input, + size_t input_len, bool use_tbb) { // Explicitly checking for zero avoids causing UB by passing a null pointer // to memcpy. This comes up in practice with things like: // std::vector v; @@ -539,7 +560,7 @@ void blake3_hasher_update(blake3_hasher *self, const void *input, uint8_t cv_pair[2 * BLAKE3_OUT_LEN]; compress_subtree_to_parent_node(input_bytes, subtree_len, self->key, self->chunk.chunk_counter, - self->chunk.flags, cv_pair); + self->chunk.flags, cv_pair, use_tbb); hasher_push_cv(self, cv_pair, self->chunk.chunk_counter); hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN], self->chunk.chunk_counter + (subtree_chunks / 2)); @@ -561,6 +582,20 @@ void blake3_hasher_update(blake3_hasher *self, const void *input, } } +void blake3_hasher_update(blake3_hasher *self, const void *input, + size_t input_len) { + bool use_tbb = false; + blake3_hasher_update_base(self, input, input_len, use_tbb); +} + +#if defined(BLAKE3_USE_TBB) +void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, + size_t input_len) { + bool use_tbb = true; + blake3_hasher_update_base(self, input, input_len, use_tbb); +} +#endif // BLAKE3_USE_TBB + void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, size_t out_len) { blake3_hasher_finalize_seek(self, 0, out, out_len); diff --git a/src/third_party/blake3/c/blake3.h b/src/third_party/blake3/blake3.h similarity index 90% rename from src/third_party/blake3/c/blake3.h rename to src/third_party/blake3/blake3.h index f694dcf27..c5b799870 100644 --- a/src/third_party/blake3/c/blake3.h +++ b/src/third_party/blake3/blake3.h @@ -30,7 +30,7 @@ extern "C" { #endif -#define BLAKE3_VERSION_STRING "1.5.0" +#define BLAKE3_VERSION_STRING "1.8.7" #define BLAKE3_KEY_LEN 32 #define BLAKE3_OUT_LEN 32 #define BLAKE3_BLOCK_LEN 64 @@ -38,7 +38,7 @@ extern "C" { #define BLAKE3_MAX_DEPTH 54 // This struct is a private implementation detail. It has to be here because -// it's part of blake3_hasher below. +// it's part of the blake3_hasher structure defined below. typedef struct { uint32_t cv[8]; uint64_t chunk_counter; @@ -69,6 +69,10 @@ BLAKE3_API void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const voi size_t context_len); BLAKE3_API void blake3_hasher_update(blake3_hasher *self, const void *input, size_t input_len); +#if defined(BLAKE3_USE_TBB) +BLAKE3_API void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, + size_t input_len); +#endif // BLAKE3_USE_TBB BLAKE3_API void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, size_t out_len); BLAKE3_API void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, diff --git a/src/third_party/blake3/asm/blake3_avx2_x86-64_unix.S b/src/third_party/blake3/blake3_avx2_x86-64_unix.S similarity index 100% rename from src/third_party/blake3/asm/blake3_avx2_x86-64_unix.S rename to src/third_party/blake3/blake3_avx2_x86-64_unix.S diff --git a/src/third_party/blake3/asm/blake3_avx2_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_avx2_x86-64_windows_msvc.asm similarity index 100% rename from src/third_party/blake3/asm/blake3_avx2_x86-64_windows_msvc.asm rename to src/third_party/blake3/blake3_avx2_x86-64_windows_msvc.asm diff --git a/src/third_party/blake3/asm/blake3_avx512_x86-64_unix.S b/src/third_party/blake3/blake3_avx512_x86-64_unix.S similarity index 55% rename from src/third_party/blake3/asm/blake3_avx512_x86-64_unix.S rename to src/third_party/blake3/blake3_avx512_x86-64_unix.S index a06aede0f..9642e413a 100644 --- a/src/third_party/blake3/asm/blake3_avx512_x86-64_unix.S +++ b/src/third_party/blake3/blake3_avx512_x86-64_unix.S @@ -19,6 +19,8 @@ .global _blake3_compress_in_place_avx512 .global blake3_compress_xof_avx512 .global _blake3_compress_xof_avx512 +.global blake3_xof_many_avx512 +.global _blake3_xof_many_avx512 #ifdef __APPLE__ .text @@ -2553,6 +2555,2243 @@ blake3_compress_xof_avx512: vmovdqu xmmword ptr [r9+0x30], xmm3 ret +.p2align 6 +blake3_xof_many_avx512: +_blake3_xof_many_avx512: + _CET_ENDBR + mov r10,QWORD PTR [rsp+0x8] + cmp r10,0x1 + ja 2f + vmovdqu xmm0,XMMWORD PTR [rdi] + vmovdqu xmm1,XMMWORD PTR [rdi+0x10] + movzx eax,r8b + movzx edx,dl + shl rax,0x20 + add rdx,rax + vmovq xmm3,rcx + vmovq xmm4,rdx + vpunpcklqdq xmm3,xmm3,xmm4 + vmovaps xmm2,XMMWORD PTR [BLAKE3_IV+rip] + vmovups xmm8,XMMWORD PTR [rsi] + vmovups xmm9,XMMWORD PTR [rsi+0x10] + vshufps xmm4,xmm8,xmm9,0x88 + vshufps xmm5,xmm8,xmm9,0xdd + vmovups xmm8,XMMWORD PTR [rsi+0x20] + vmovups xmm9,XMMWORD PTR [rsi+0x30] + vshufps xmm6,xmm8,xmm9,0x88 + vshufps xmm7,xmm8,xmm9,0xdd + vpshufd xmm6,xmm6,0x93 + vpshufd xmm7,xmm7,0x93 + mov al,0x7 +3: + vpaddd xmm0,xmm0,xmm4 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x10 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0xc + vpaddd xmm0,xmm0,xmm5 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x8 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0x7 + vpshufd xmm0,xmm0,0x93 + vpshufd xmm3,xmm3,0x4e + vpshufd xmm2,xmm2,0x39 + vpaddd xmm0,xmm0,xmm6 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x10 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0xc + vpaddd xmm0,xmm0,xmm7 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x8 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0x7 + vpshufd xmm0,xmm0,0x39 + vpshufd xmm3,xmm3,0x4e + vpshufd xmm2,xmm2,0x93 + dec al + je 3f + vshufps xmm8,xmm4,xmm5,0xd6 + vpshufd xmm9,xmm4,0xf + vpshufd xmm4,xmm8,0x39 + vshufps xmm8,xmm6,xmm7,0xfa + vpblendd xmm9,xmm9,xmm8,0xaa + vpunpcklqdq xmm8,xmm7,xmm5 + vpblendd xmm8,xmm8,xmm6,0x88 + vpshufd xmm8,xmm8,0x78 + vpunpckhdq xmm5,xmm5,xmm7 + vpunpckldq xmm6,xmm6,xmm5 + vpshufd xmm7,xmm6,0x1e + vmovdqa xmm5,xmm9 + vmovdqa xmm6,xmm8 + jmp 3b +3: + vpxor xmm0,xmm0,xmm2 + vpxor xmm1,xmm1,xmm3 + vpxor xmm2,xmm2,XMMWORD PTR [rdi] + vpxor xmm3,xmm3,XMMWORD PTR [rdi+0x10] + vmovdqu XMMWORD PTR [r9],xmm0 + vmovdqu XMMWORD PTR [r9+0x10],xmm1 + vmovdqu XMMWORD PTR [r9+0x20],xmm2 + vmovdqu XMMWORD PTR [r9+0x30],xmm3 + ret +.p2align 6 +2: + push rbp + mov rbp,rsp + sub rsp,0x90 + and rsp,0xffffffffffffffc0 + vpbroadcastd zmm0,ecx + shr rcx,0x20 + vpbroadcastd zmm1,ecx + vpaddd zmm2,zmm0,ZMMWORD PTR [ADD0+rip] + vpcmpltud k1,zmm2,zmm0 + vpaddd zmm1{k1},zmm1,DWORD PTR [ADD1+rip]{1to16} + vmovdqa32 ZMMWORD PTR [rsp],zmm2 + vmovdqa32 ZMMWORD PTR [rsp+0x40],zmm1 + cmp r10,0x10 + jb 2f +3: + vpbroadcastd zmm16,DWORD PTR [rsi] + vpbroadcastd zmm17,DWORD PTR [rsi+0x4] + vpbroadcastd zmm18,DWORD PTR [rsi+0x8] + vpbroadcastd zmm19,DWORD PTR [rsi+0xc] + vpbroadcastd zmm20,DWORD PTR [rsi+0x10] + vpbroadcastd zmm21,DWORD PTR [rsi+0x14] + vpbroadcastd zmm22,DWORD PTR [rsi+0x18] + vpbroadcastd zmm23,DWORD PTR [rsi+0x1c] + vpbroadcastd zmm24,DWORD PTR [rsi+0x20] + vpbroadcastd zmm25,DWORD PTR [rsi+0x24] + vpbroadcastd zmm26,DWORD PTR [rsi+0x28] + vpbroadcastd zmm27,DWORD PTR [rsi+0x2c] + vpbroadcastd zmm28,DWORD PTR [rsi+0x30] + vpbroadcastd zmm29,DWORD PTR [rsi+0x34] + vpbroadcastd zmm30,DWORD PTR [rsi+0x38] + vpbroadcastd zmm31,DWORD PTR [rsi+0x3c] + vpbroadcastd zmm0,DWORD PTR [rdi] + vpbroadcastd zmm1,DWORD PTR [rdi+0x4] + vpbroadcastd zmm2,DWORD PTR [rdi+0x8] + vpbroadcastd zmm3,DWORD PTR [rdi+0xc] + vpbroadcastd zmm4,DWORD PTR [rdi+0x10] + vpbroadcastd zmm5,DWORD PTR [rdi+0x14] + vpbroadcastd zmm6,DWORD PTR [rdi+0x18] + vpbroadcastd zmm7,DWORD PTR [rdi+0x1c] + vpbroadcastd zmm8,DWORD PTR [BLAKE3_IV_0+rip] + vpbroadcastd zmm9,DWORD PTR [BLAKE3_IV_1+rip] + vpbroadcastd zmm10,DWORD PTR [BLAKE3_IV_2+rip] + vpbroadcastd zmm11,DWORD PTR [BLAKE3_IV_3+rip] + vmovdqa32 zmm12,ZMMWORD PTR [rsp] + vmovdqa32 zmm13,ZMMWORD PTR [rsp+0x40] + vpbroadcastd zmm14,edx + vpbroadcastd zmm15,r8d + vpaddd zmm0,zmm0,zmm16 + vpaddd zmm1,zmm1,zmm18 + vpaddd zmm2,zmm2,zmm20 + vpaddd zmm3,zmm3,zmm22 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm17 + vpaddd zmm1,zmm1,zmm19 + vpaddd zmm2,zmm2,zmm21 + vpaddd zmm3,zmm3,zmm23 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm24 + vpaddd zmm1,zmm1,zmm26 + vpaddd zmm2,zmm2,zmm28 + vpaddd zmm3,zmm3,zmm30 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm25 + vpaddd zmm1,zmm1,zmm27 + vpaddd zmm2,zmm2,zmm29 + vpaddd zmm3,zmm3,zmm31 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm18 + vpaddd zmm1,zmm1,zmm19 + vpaddd zmm2,zmm2,zmm23 + vpaddd zmm3,zmm3,zmm20 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm22 + vpaddd zmm1,zmm1,zmm26 + vpaddd zmm2,zmm2,zmm16 + vpaddd zmm3,zmm3,zmm29 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm17 + vpaddd zmm1,zmm1,zmm28 + vpaddd zmm2,zmm2,zmm25 + vpaddd zmm3,zmm3,zmm31 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm27 + vpaddd zmm1,zmm1,zmm21 + vpaddd zmm2,zmm2,zmm30 + vpaddd zmm3,zmm3,zmm24 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm19 + vpaddd zmm1,zmm1,zmm26 + vpaddd zmm2,zmm2,zmm29 + vpaddd zmm3,zmm3,zmm23 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm20 + vpaddd zmm1,zmm1,zmm28 + vpaddd zmm2,zmm2,zmm18 + vpaddd zmm3,zmm3,zmm30 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm22 + vpaddd zmm1,zmm1,zmm25 + vpaddd zmm2,zmm2,zmm27 + vpaddd zmm3,zmm3,zmm24 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm21 + vpaddd zmm1,zmm1,zmm16 + vpaddd zmm2,zmm2,zmm31 + vpaddd zmm3,zmm3,zmm17 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm26 + vpaddd zmm1,zmm1,zmm28 + vpaddd zmm2,zmm2,zmm30 + vpaddd zmm3,zmm3,zmm29 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm23 + vpaddd zmm1,zmm1,zmm25 + vpaddd zmm2,zmm2,zmm19 + vpaddd zmm3,zmm3,zmm31 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm20 + vpaddd zmm1,zmm1,zmm27 + vpaddd zmm2,zmm2,zmm21 + vpaddd zmm3,zmm3,zmm17 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm16 + vpaddd zmm1,zmm1,zmm18 + vpaddd zmm2,zmm2,zmm24 + vpaddd zmm3,zmm3,zmm22 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm28 + vpaddd zmm1,zmm1,zmm25 + vpaddd zmm2,zmm2,zmm31 + vpaddd zmm3,zmm3,zmm30 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm29 + vpaddd zmm1,zmm1,zmm27 + vpaddd zmm2,zmm2,zmm26 + vpaddd zmm3,zmm3,zmm24 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm23 + vpaddd zmm1,zmm1,zmm21 + vpaddd zmm2,zmm2,zmm16 + vpaddd zmm3,zmm3,zmm22 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm18 + vpaddd zmm1,zmm1,zmm19 + vpaddd zmm2,zmm2,zmm17 + vpaddd zmm3,zmm3,zmm20 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm25 + vpaddd zmm1,zmm1,zmm27 + vpaddd zmm2,zmm2,zmm24 + vpaddd zmm3,zmm3,zmm31 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm30 + vpaddd zmm1,zmm1,zmm21 + vpaddd zmm2,zmm2,zmm28 + vpaddd zmm3,zmm3,zmm17 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm29 + vpaddd zmm1,zmm1,zmm16 + vpaddd zmm2,zmm2,zmm18 + vpaddd zmm3,zmm3,zmm20 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm19 + vpaddd zmm1,zmm1,zmm26 + vpaddd zmm2,zmm2,zmm22 + vpaddd zmm3,zmm3,zmm23 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm27 + vpaddd zmm1,zmm1,zmm21 + vpaddd zmm2,zmm2,zmm17 + vpaddd zmm3,zmm3,zmm24 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm31 + vpaddd zmm1,zmm1,zmm16 + vpaddd zmm2,zmm2,zmm25 + vpaddd zmm3,zmm3,zmm22 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm30 + vpaddd zmm1,zmm1,zmm18 + vpaddd zmm2,zmm2,zmm19 + vpaddd zmm3,zmm3,zmm23 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm26 + vpaddd zmm1,zmm1,zmm28 + vpaddd zmm2,zmm2,zmm20 + vpaddd zmm3,zmm3,zmm29 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpxord zmm0,zmm0,zmm8 + vpxord zmm1,zmm1,zmm9 + vpxord zmm2,zmm2,zmm10 + vpxord zmm3,zmm3,zmm11 + vpxord zmm4,zmm4,zmm12 + vpxord zmm5,zmm5,zmm13 + vpxord zmm6,zmm6,zmm14 + vpxord zmm7,zmm7,zmm15 + vpxord zmm8,zmm8,DWORD PTR [rdi]{1to16} + vpxord zmm9,zmm9,DWORD PTR [rdi+0x4]{1to16} + vpxord zmm10,zmm10,DWORD PTR [rdi+0x8]{1to16} + vpxord zmm11,zmm11,DWORD PTR [rdi+0xc]{1to16} + vpxord zmm12,zmm12,DWORD PTR [rdi+0x10]{1to16} + vpxord zmm13,zmm13,DWORD PTR [rdi+0x14]{1to16} + vpxord zmm14,zmm14,DWORD PTR [rdi+0x18]{1to16} + vpxord zmm15,zmm15,DWORD PTR [rdi+0x1c]{1to16} + vpunpckldq zmm16,zmm0,zmm1 + vpunpckhdq zmm17,zmm0,zmm1 + vpunpckldq zmm18,zmm2,zmm3 + vpunpckhdq zmm19,zmm2,zmm3 + vpunpckldq zmm20,zmm4,zmm5 + vpunpckhdq zmm21,zmm4,zmm5 + vpunpckldq zmm22,zmm6,zmm7 + vpunpckhdq zmm23,zmm6,zmm7 + vpunpckldq zmm24,zmm8,zmm9 + vpunpckhdq zmm25,zmm8,zmm9 + vpunpckldq zmm26,zmm10,zmm11 + vpunpckhdq zmm27,zmm10,zmm11 + vpunpckldq zmm28,zmm12,zmm13 + vpunpckhdq zmm29,zmm12,zmm13 + vpunpckldq zmm30,zmm14,zmm15 + vpunpckhdq zmm31,zmm14,zmm15 + vpunpcklqdq zmm0,zmm16,zmm18 + vpunpckhqdq zmm1,zmm16,zmm18 + vpunpcklqdq zmm2,zmm17,zmm19 + vpunpckhqdq zmm3,zmm17,zmm19 + vpunpcklqdq zmm4,zmm20,zmm22 + vpunpckhqdq zmm5,zmm20,zmm22 + vpunpcklqdq zmm6,zmm21,zmm23 + vpunpckhqdq zmm7,zmm21,zmm23 + vpunpcklqdq zmm8,zmm24,zmm26 + vpunpckhqdq zmm9,zmm24,zmm26 + vpunpcklqdq zmm10,zmm25,zmm27 + vpunpckhqdq zmm11,zmm25,zmm27 + vpunpcklqdq zmm12,zmm28,zmm30 + vpunpckhqdq zmm13,zmm28,zmm30 + vpunpcklqdq zmm14,zmm29,zmm31 + vpunpckhqdq zmm15,zmm29,zmm31 + vshufi32x4 zmm16,zmm0,zmm4,0x88 + vshufi32x4 zmm17,zmm1,zmm5,0x88 + vshufi32x4 zmm18,zmm2,zmm6,0x88 + vshufi32x4 zmm19,zmm3,zmm7,0x88 + vshufi32x4 zmm20,zmm0,zmm4,0xdd + vshufi32x4 zmm21,zmm1,zmm5,0xdd + vshufi32x4 zmm22,zmm2,zmm6,0xdd + vshufi32x4 zmm23,zmm3,zmm7,0xdd + vshufi32x4 zmm24,zmm8,zmm12,0x88 + vshufi32x4 zmm25,zmm9,zmm13,0x88 + vshufi32x4 zmm26,zmm10,zmm14,0x88 + vshufi32x4 zmm27,zmm11,zmm15,0x88 + vshufi32x4 zmm28,zmm8,zmm12,0xdd + vshufi32x4 zmm29,zmm9,zmm13,0xdd + vshufi32x4 zmm30,zmm10,zmm14,0xdd + vshufi32x4 zmm31,zmm11,zmm15,0xdd + vshufi32x4 zmm0,zmm16,zmm24,0x88 + vshufi32x4 zmm1,zmm17,zmm25,0x88 + vshufi32x4 zmm2,zmm18,zmm26,0x88 + vshufi32x4 zmm3,zmm19,zmm27,0x88 + vshufi32x4 zmm4,zmm20,zmm28,0x88 + vshufi32x4 zmm5,zmm21,zmm29,0x88 + vshufi32x4 zmm6,zmm22,zmm30,0x88 + vshufi32x4 zmm7,zmm23,zmm31,0x88 + vshufi32x4 zmm8,zmm16,zmm24,0xdd + vshufi32x4 zmm9,zmm17,zmm25,0xdd + vshufi32x4 zmm10,zmm18,zmm26,0xdd + vshufi32x4 zmm11,zmm19,zmm27,0xdd + vshufi32x4 zmm12,zmm20,zmm28,0xdd + vshufi32x4 zmm13,zmm21,zmm29,0xdd + vshufi32x4 zmm14,zmm22,zmm30,0xdd + vshufi32x4 zmm15,zmm23,zmm31,0xdd + vmovdqu32 ZMMWORD PTR [r9],zmm0 + vmovdqu32 ZMMWORD PTR [r9+0x40],zmm1 + vmovdqu32 ZMMWORD PTR [r9+0x80],zmm2 + vmovdqu32 ZMMWORD PTR [r9+0xc0],zmm3 + vmovdqu32 ZMMWORD PTR [r9+0x100],zmm4 + vmovdqu32 ZMMWORD PTR [r9+0x140],zmm5 + vmovdqu32 ZMMWORD PTR [r9+0x180],zmm6 + vmovdqu32 ZMMWORD PTR [r9+0x1c0],zmm7 + vmovdqu32 ZMMWORD PTR [r9+0x200],zmm8 + vmovdqu32 ZMMWORD PTR [r9+0x240],zmm9 + vmovdqu32 ZMMWORD PTR [r9+0x280],zmm10 + vmovdqu32 ZMMWORD PTR [r9+0x2c0],zmm11 + vmovdqu32 ZMMWORD PTR [r9+0x300],zmm12 + vmovdqu32 ZMMWORD PTR [r9+0x340],zmm13 + vmovdqu32 ZMMWORD PTR [r9+0x380],zmm14 + vmovdqu32 ZMMWORD PTR [r9+0x3c0],zmm15 + vmovdqa32 zmm0,ZMMWORD PTR [rsp] + vmovdqa32 zmm1,ZMMWORD PTR [rsp+0x40] + vpaddd zmm2,zmm0,DWORD PTR [ADD16+rip]{1to16} + vpcmpltud k1,zmm2,zmm0 + vpaddd zmm1{k1},zmm1,DWORD PTR [ADD1+rip]{1to16} + vmovdqa32 ZMMWORD PTR [rsp],zmm2 + vmovdqa32 ZMMWORD PTR [rsp+0x40],zmm1 + add r9,0x400 + sub r10,0x10 + cmp r10,0x10 + jae 3b + test r10,r10 + jne 2f +9: + vzeroupper + mov rsp,rbp + pop rbp + ret +2: + test r10,0x8 + je 2f + vpbroadcastd ymm16,DWORD PTR [rsi] + vpbroadcastd ymm17,DWORD PTR [rsi+0x4] + vpbroadcastd ymm18,DWORD PTR [rsi+0x8] + vpbroadcastd ymm19,DWORD PTR [rsi+0xc] + vpbroadcastd ymm20,DWORD PTR [rsi+0x10] + vpbroadcastd ymm21,DWORD PTR [rsi+0x14] + vpbroadcastd ymm22,DWORD PTR [rsi+0x18] + vpbroadcastd ymm23,DWORD PTR [rsi+0x1c] + vpbroadcastd ymm24,DWORD PTR [rsi+0x20] + vpbroadcastd ymm25,DWORD PTR [rsi+0x24] + vpbroadcastd ymm26,DWORD PTR [rsi+0x28] + vpbroadcastd ymm27,DWORD PTR [rsi+0x2c] + vpbroadcastd ymm28,DWORD PTR [rsi+0x30] + vpbroadcastd ymm29,DWORD PTR [rsi+0x34] + vpbroadcastd ymm30,DWORD PTR [rsi+0x38] + vpbroadcastd ymm31,DWORD PTR [rsi+0x3c] + vpbroadcastd ymm0,DWORD PTR [rdi] + vpbroadcastd ymm1,DWORD PTR [rdi+0x4] + vpbroadcastd ymm2,DWORD PTR [rdi+0x8] + vpbroadcastd ymm3,DWORD PTR [rdi+0xc] + vpbroadcastd ymm4,DWORD PTR [rdi+0x10] + vpbroadcastd ymm5,DWORD PTR [rdi+0x14] + vpbroadcastd ymm6,DWORD PTR [rdi+0x18] + vpbroadcastd ymm7,DWORD PTR [rdi+0x1c] + vpbroadcastd ymm8,DWORD PTR [BLAKE3_IV_0+rip] + vpbroadcastd ymm9,DWORD PTR [BLAKE3_IV_1+rip] + vpbroadcastd ymm10,DWORD PTR [BLAKE3_IV_2+rip] + vpbroadcastd ymm11,DWORD PTR [BLAKE3_IV_3+rip] + vmovdqa ymm12,YMMWORD PTR [rsp] + vmovdqa ymm13,YMMWORD PTR [rsp+0x40] + vpbroadcastd ymm14,edx + vpbroadcastd ymm15,r8d + vpaddd ymm0,ymm0,ymm16 + vpaddd ymm1,ymm1,ymm18 + vpaddd ymm2,ymm2,ymm20 + vpaddd ymm3,ymm3,ymm22 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm17 + vpaddd ymm1,ymm1,ymm19 + vpaddd ymm2,ymm2,ymm21 + vpaddd ymm3,ymm3,ymm23 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm24 + vpaddd ymm1,ymm1,ymm26 + vpaddd ymm2,ymm2,ymm28 + vpaddd ymm3,ymm3,ymm30 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm25 + vpaddd ymm1,ymm1,ymm27 + vpaddd ymm2,ymm2,ymm29 + vpaddd ymm3,ymm3,ymm31 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm18 + vpaddd ymm1,ymm1,ymm19 + vpaddd ymm2,ymm2,ymm23 + vpaddd ymm3,ymm3,ymm20 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm22 + vpaddd ymm1,ymm1,ymm26 + vpaddd ymm2,ymm2,ymm16 + vpaddd ymm3,ymm3,ymm29 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm17 + vpaddd ymm1,ymm1,ymm28 + vpaddd ymm2,ymm2,ymm25 + vpaddd ymm3,ymm3,ymm31 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm27 + vpaddd ymm1,ymm1,ymm21 + vpaddd ymm2,ymm2,ymm30 + vpaddd ymm3,ymm3,ymm24 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm19 + vpaddd ymm1,ymm1,ymm26 + vpaddd ymm2,ymm2,ymm29 + vpaddd ymm3,ymm3,ymm23 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm20 + vpaddd ymm1,ymm1,ymm28 + vpaddd ymm2,ymm2,ymm18 + vpaddd ymm3,ymm3,ymm30 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm22 + vpaddd ymm1,ymm1,ymm25 + vpaddd ymm2,ymm2,ymm27 + vpaddd ymm3,ymm3,ymm24 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm21 + vpaddd ymm1,ymm1,ymm16 + vpaddd ymm2,ymm2,ymm31 + vpaddd ymm3,ymm3,ymm17 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm26 + vpaddd ymm1,ymm1,ymm28 + vpaddd ymm2,ymm2,ymm30 + vpaddd ymm3,ymm3,ymm29 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm23 + vpaddd ymm1,ymm1,ymm25 + vpaddd ymm2,ymm2,ymm19 + vpaddd ymm3,ymm3,ymm31 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm20 + vpaddd ymm1,ymm1,ymm27 + vpaddd ymm2,ymm2,ymm21 + vpaddd ymm3,ymm3,ymm17 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm16 + vpaddd ymm1,ymm1,ymm18 + vpaddd ymm2,ymm2,ymm24 + vpaddd ymm3,ymm3,ymm22 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm28 + vpaddd ymm1,ymm1,ymm25 + vpaddd ymm2,ymm2,ymm31 + vpaddd ymm3,ymm3,ymm30 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm29 + vpaddd ymm1,ymm1,ymm27 + vpaddd ymm2,ymm2,ymm26 + vpaddd ymm3,ymm3,ymm24 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm23 + vpaddd ymm1,ymm1,ymm21 + vpaddd ymm2,ymm2,ymm16 + vpaddd ymm3,ymm3,ymm22 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm18 + vpaddd ymm1,ymm1,ymm19 + vpaddd ymm2,ymm2,ymm17 + vpaddd ymm3,ymm3,ymm20 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm25 + vpaddd ymm1,ymm1,ymm27 + vpaddd ymm2,ymm2,ymm24 + vpaddd ymm3,ymm3,ymm31 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm30 + vpaddd ymm1,ymm1,ymm21 + vpaddd ymm2,ymm2,ymm28 + vpaddd ymm3,ymm3,ymm17 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm29 + vpaddd ymm1,ymm1,ymm16 + vpaddd ymm2,ymm2,ymm18 + vpaddd ymm3,ymm3,ymm20 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm19 + vpaddd ymm1,ymm1,ymm26 + vpaddd ymm2,ymm2,ymm22 + vpaddd ymm3,ymm3,ymm23 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm27 + vpaddd ymm1,ymm1,ymm21 + vpaddd ymm2,ymm2,ymm17 + vpaddd ymm3,ymm3,ymm24 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm31 + vpaddd ymm1,ymm1,ymm16 + vpaddd ymm2,ymm2,ymm25 + vpaddd ymm3,ymm3,ymm22 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm30 + vpaddd ymm1,ymm1,ymm18 + vpaddd ymm2,ymm2,ymm19 + vpaddd ymm3,ymm3,ymm23 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm26 + vpaddd ymm1,ymm1,ymm28 + vpaddd ymm2,ymm2,ymm20 + vpaddd ymm3,ymm3,ymm29 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpxor ymm0,ymm0,ymm8 + vpxor ymm1,ymm1,ymm9 + vpxor ymm2,ymm2,ymm10 + vpxor ymm3,ymm3,ymm11 + vpxor ymm4,ymm4,ymm12 + vpxor ymm5,ymm5,ymm13 + vpxor ymm6,ymm6,ymm14 + vpxor ymm7,ymm7,ymm15 + vpxord ymm8,ymm8,DWORD PTR [rdi]{1to8} + vpxord ymm9,ymm9,DWORD PTR [rdi+0x4]{1to8} + vpxord ymm10,ymm10,DWORD PTR [rdi+0x8]{1to8} + vpxord ymm11,ymm11,DWORD PTR [rdi+0xc]{1to8} + vpxord ymm12,ymm12,DWORD PTR [rdi+0x10]{1to8} + vpxord ymm13,ymm13,DWORD PTR [rdi+0x14]{1to8} + vpxord ymm14,ymm14,DWORD PTR [rdi+0x18]{1to8} + vpxord ymm15,ymm15,DWORD PTR [rdi+0x1c]{1to8} + vpunpckldq ymm16,ymm0,ymm1 + vpunpckhdq ymm17,ymm0,ymm1 + vpunpckldq ymm18,ymm2,ymm3 + vpunpckhdq ymm19,ymm2,ymm3 + vpunpckldq ymm20,ymm4,ymm5 + vpunpckhdq ymm21,ymm4,ymm5 + vpunpckldq ymm22,ymm6,ymm7 + vpunpckhdq ymm23,ymm6,ymm7 + vpunpckldq ymm24,ymm8,ymm9 + vpunpckhdq ymm25,ymm8,ymm9 + vpunpckldq ymm26,ymm10,ymm11 + vpunpckhdq ymm27,ymm10,ymm11 + vpunpckldq ymm28,ymm12,ymm13 + vpunpckhdq ymm29,ymm12,ymm13 + vpunpckldq ymm30,ymm14,ymm15 + vpunpckhdq ymm31,ymm14,ymm15 + vpunpcklqdq ymm0,ymm16,ymm18 + vpunpckhqdq ymm1,ymm16,ymm18 + vpunpcklqdq ymm2,ymm17,ymm19 + vpunpckhqdq ymm3,ymm17,ymm19 + vpunpcklqdq ymm4,ymm20,ymm22 + vpunpckhqdq ymm5,ymm20,ymm22 + vpunpcklqdq ymm6,ymm21,ymm23 + vpunpckhqdq ymm7,ymm21,ymm23 + vpunpcklqdq ymm8,ymm24,ymm26 + vpunpckhqdq ymm9,ymm24,ymm26 + vpunpcklqdq ymm10,ymm25,ymm27 + vpunpckhqdq ymm11,ymm25,ymm27 + vpunpcklqdq ymm12,ymm28,ymm30 + vpunpckhqdq ymm13,ymm28,ymm30 + vpunpcklqdq ymm14,ymm29,ymm31 + vpunpckhqdq ymm15,ymm29,ymm31 + vshufi32x4 ymm16,ymm0,ymm4,0x0 + vshufi32x4 ymm17,ymm8,ymm12,0x0 + vshufi32x4 ymm18,ymm1,ymm5,0x0 + vshufi32x4 ymm19,ymm9,ymm13,0x0 + vshufi32x4 ymm20,ymm2,ymm6,0x0 + vshufi32x4 ymm21,ymm10,ymm14,0x0 + vshufi32x4 ymm22,ymm3,ymm7,0x0 + vshufi32x4 ymm23,ymm11,ymm15,0x0 + vshufi32x4 ymm24,ymm0,ymm4,0x3 + vshufi32x4 ymm25,ymm8,ymm12,0x3 + vshufi32x4 ymm26,ymm1,ymm5,0x3 + vshufi32x4 ymm27,ymm9,ymm13,0x3 + vshufi32x4 ymm28,ymm2,ymm6,0x3 + vshufi32x4 ymm29,ymm10,ymm14,0x3 + vshufi32x4 ymm30,ymm3,ymm7,0x3 + vshufi32x4 ymm31,ymm11,ymm15,0x3 + vmovdqu32 YMMWORD PTR [r9],ymm16 + vmovdqu32 YMMWORD PTR [r9+0x20],ymm17 + vmovdqu32 YMMWORD PTR [r9+0x40],ymm18 + vmovdqu32 YMMWORD PTR [r9+0x60],ymm19 + vmovdqu32 YMMWORD PTR [r9+0x80],ymm20 + vmovdqu32 YMMWORD PTR [r9+0xa0],ymm21 + vmovdqu32 YMMWORD PTR [r9+0xc0],ymm22 + vmovdqu32 YMMWORD PTR [r9+0xe0],ymm23 + vmovdqu32 YMMWORD PTR [r9+0x100],ymm24 + vmovdqu32 YMMWORD PTR [r9+0x120],ymm25 + vmovdqu32 YMMWORD PTR [r9+0x140],ymm26 + vmovdqu32 YMMWORD PTR [r9+0x160],ymm27 + vmovdqu32 YMMWORD PTR [r9+0x180],ymm28 + vmovdqu32 YMMWORD PTR [r9+0x1a0],ymm29 + vmovdqu32 YMMWORD PTR [r9+0x1c0],ymm30 + vmovdqu32 YMMWORD PTR [r9+0x1e0],ymm31 + vmovdqa ymm0,YMMWORD PTR [rsp+0x20] + vmovdqa ymm1,YMMWORD PTR [rsp+0x60] + vmovdqa YMMWORD PTR [rsp],ymm0 + vmovdqa YMMWORD PTR [rsp+0x40],ymm1 + add r9,0x200 + sub r10,0x8 +2: + test r10,0x4 + je 2f + vbroadcasti32x4 zmm0,XMMWORD PTR [rdi] + vbroadcasti32x4 zmm1,XMMWORD PTR [rdi+0x10] + vbroadcasti32x4 zmm2,XMMWORD PTR [BLAKE3_IV+rip] + vmovdqa xmm12,XMMWORD PTR [rsp] + vmovdqa xmm13,XMMWORD PTR [rsp+0x40] + vpunpckldq xmm14,xmm12,xmm13 + vpunpckhdq xmm15,xmm12,xmm13 + vpermq ymm14,ymm14,0xdc + vpermq ymm15,ymm15,0xdc + vpbroadcastd zmm12,edx + vinserti64x4 zmm13,zmm14,ymm15,0x1 + mov eax,0x4444 + kmovw k2,eax + vpblendmd zmm13{k2},zmm13,zmm12 + vpbroadcastd zmm15,r8d + mov eax,0x8888 + kmovw k4,eax + vpblendmd zmm3{k4},zmm13,zmm15 + mov eax,0xaaaa + kmovw k3,eax + vbroadcasti32x4 zmm8,XMMWORD PTR [rsi] + vbroadcasti32x4 zmm9,XMMWORD PTR [rsi+0x10] + vshufps zmm4,zmm8,zmm9,0x88 + vshufps zmm5,zmm8,zmm9,0xdd + vbroadcasti32x4 zmm8,XMMWORD PTR [rsi+0x20] + vbroadcasti32x4 zmm9,XMMWORD PTR [rsi+0x30] + vshufps zmm6,zmm8,zmm9,0x88 + vshufps zmm7,zmm8,zmm9,0xdd + vpshufd zmm6,zmm6,0x93 + vpshufd zmm7,zmm7,0x93 + mov al,0x7 +3: + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprord zmm3,zmm3,0x10 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprord zmm1,zmm1,0xc + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprord zmm3,zmm3,0x8 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprord zmm1,zmm1,0x7 + vpshufd zmm0,zmm0,0x93 + vpshufd zmm3,zmm3,0x4e + vpshufd zmm2,zmm2,0x39 + vpaddd zmm0,zmm0,zmm6 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprord zmm3,zmm3,0x10 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprord zmm1,zmm1,0xc + vpaddd zmm0,zmm0,zmm7 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprord zmm3,zmm3,0x8 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprord zmm1,zmm1,0x7 + vpshufd zmm0,zmm0,0x39 + vpshufd zmm3,zmm3,0x4e + vpshufd zmm2,zmm2,0x93 + dec al + je 3f + vshufps zmm8,zmm4,zmm5,0xd6 + vpshufd zmm9,zmm4,0xf + vpshufd zmm4,zmm8,0x39 + vshufps zmm8,zmm6,zmm7,0xfa + vpblendmd zmm9{k3},zmm9,zmm8 + vpunpcklqdq zmm8,zmm7,zmm5 + vpblendmd zmm8{k4},zmm8,zmm6 + vpshufd zmm8,zmm8,0x78 + vpunpckhdq zmm5,zmm5,zmm7 + vpunpckldq zmm6,zmm6,zmm5 + vpshufd zmm7,zmm6,0x1e + vmovdqa32 zmm5,zmm9 + vmovdqa32 zmm6,zmm8 + jmp 3b +3: + vpxord zmm0,zmm0,zmm2 + vpxord zmm1,zmm1,zmm3 + vbroadcasti32x4 zmm8,XMMWORD PTR [rdi] + vbroadcasti32x4 zmm9,XMMWORD PTR [rdi+0x10] + vpxord zmm2,zmm2,zmm8 + vpxord zmm3,zmm3,zmm9 + vmovdqu XMMWORD PTR [r9],xmm0 + vmovdqu XMMWORD PTR [r9+0x10],xmm1 + vmovdqu XMMWORD PTR [r9+0x20],xmm2 + vmovdqu XMMWORD PTR [r9+0x30],xmm3 + vextracti128 XMMWORD PTR [r9+0x40],ymm0,0x1 + vextracti128 XMMWORD PTR [r9+0x50],ymm1,0x1 + vextracti128 XMMWORD PTR [r9+0x60],ymm2,0x1 + vextracti128 XMMWORD PTR [r9+0x70],ymm3,0x1 + vextracti32x4 XMMWORD PTR [r9+0x80],zmm0,0x2 + vextracti32x4 XMMWORD PTR [r9+0x90],zmm1,0x2 + vextracti32x4 XMMWORD PTR [r9+0xa0],zmm2,0x2 + vextracti32x4 XMMWORD PTR [r9+0xb0],zmm3,0x2 + vextracti32x4 XMMWORD PTR [r9+0xc0],zmm0,0x3 + vextracti32x4 XMMWORD PTR [r9+0xd0],zmm1,0x3 + vextracti32x4 XMMWORD PTR [r9+0xe0],zmm2,0x3 + vextracti32x4 XMMWORD PTR [r9+0xf0],zmm3,0x3 + vmovdqa xmm0,XMMWORD PTR [rsp+0x10] + vmovdqa xmm1,XMMWORD PTR [rsp+0x50] + vmovdqa XMMWORD PTR [rsp],xmm0 + vmovdqa XMMWORD PTR [rsp+0x40],xmm1 + add r9,0x100 + sub r10,0x4 +2: + test r10,0x2 + je 2f + vbroadcasti128 ymm0,XMMWORD PTR [rdi] + vbroadcasti128 ymm1,XMMWORD PTR [rdi+0x10] + vmovd xmm13,DWORD PTR [rsp] + vpinsrd xmm13,xmm13,DWORD PTR [rsp+0x40],0x1 + vpinsrd xmm13,xmm13,edx,0x2 + vmovd xmm14,DWORD PTR [rsp+0x4] + vpinsrd xmm14,xmm14,DWORD PTR [rsp+0x44],0x1 + vpinsrd xmm14,xmm14,edx,0x2 + vinserti128 ymm13,ymm13,xmm14,0x1 + vbroadcasti128 ymm2,XMMWORD PTR [BLAKE3_IV+rip] + vpbroadcastd ymm8,r8d + vpblendd ymm3,ymm13,ymm8,0x88 + vbroadcasti128 ymm8,XMMWORD PTR [rsi] + vbroadcasti128 ymm9,XMMWORD PTR [rsi+0x10] + vshufps ymm4,ymm8,ymm9,0x88 + vshufps ymm5,ymm8,ymm9,0xdd + vbroadcasti128 ymm8,XMMWORD PTR [rsi+0x20] + vbroadcasti128 ymm9,XMMWORD PTR [rsi+0x30] + vshufps ymm6,ymm8,ymm9,0x88 + vshufps ymm7,ymm8,ymm9,0xdd + vpshufd ymm6,ymm6,0x93 + vpshufd ymm7,ymm7,0x93 + mov al,0x7 +3: + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm0,ymm0,ymm1 + vpxord ymm3,ymm3,ymm0 + vprord ymm3,ymm3,0x10 + vpaddd ymm2,ymm2,ymm3 + vpxord ymm1,ymm1,ymm2 + vprord ymm1,ymm1,0xc + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm0,ymm0,ymm1 + vpxord ymm3,ymm3,ymm0 + vprord ymm3,ymm3,0x8 + vpaddd ymm2,ymm2,ymm3 + vpxord ymm1,ymm1,ymm2 + vprord ymm1,ymm1,0x7 + vpshufd ymm0,ymm0,0x93 + vpshufd ymm3,ymm3,0x4e + vpshufd ymm2,ymm2,0x39 + vpaddd ymm0,ymm0,ymm6 + vpaddd ymm0,ymm0,ymm1 + vpxord ymm3,ymm3,ymm0 + vprord ymm3,ymm3,0x10 + vpaddd ymm2,ymm2,ymm3 + vpxord ymm1,ymm1,ymm2 + vprord ymm1,ymm1,0xc + vpaddd ymm0,ymm0,ymm7 + vpaddd ymm0,ymm0,ymm1 + vpxord ymm3,ymm3,ymm0 + vprord ymm3,ymm3,0x8 + vpaddd ymm2,ymm2,ymm3 + vpxord ymm1,ymm1,ymm2 + vprord ymm1,ymm1,0x7 + vpshufd ymm0,ymm0,0x39 + vpshufd ymm3,ymm3,0x4e + vpshufd ymm2,ymm2,0x93 + dec al + je 3f + vshufps ymm8,ymm4,ymm5,0xd6 + vpshufd ymm9,ymm4,0xf + vpshufd ymm4,ymm8,0x39 + vshufps ymm8,ymm6,ymm7,0xfa + vpblendd ymm9,ymm9,ymm8,0xaa + vpunpcklqdq ymm8,ymm7,ymm5 + vpblendd ymm8,ymm8,ymm6,0x88 + vpshufd ymm8,ymm8,0x78 + vpunpckhdq ymm5,ymm5,ymm7 + vpunpckldq ymm6,ymm6,ymm5 + vpshufd ymm7,ymm6,0x1e + vmovdqa ymm5,ymm9 + vmovdqa ymm6,ymm8 + jmp 3b +3: + vpxor ymm0,ymm0,ymm2 + vpxor ymm1,ymm1,ymm3 + vbroadcasti128 ymm8,XMMWORD PTR [rdi] + vbroadcasti128 ymm9,XMMWORD PTR [rdi+0x10] + vpxor ymm2,ymm2,ymm8 + vpxor ymm3,ymm3,ymm9 + vmovdqu XMMWORD PTR [r9],xmm0 + vmovdqu XMMWORD PTR [r9+0x10],xmm1 + vmovdqu XMMWORD PTR [r9+0x20],xmm2 + vmovdqu XMMWORD PTR [r9+0x30],xmm3 + vextracti128 XMMWORD PTR [r9+0x40],ymm0,0x1 + vextracti128 XMMWORD PTR [r9+0x50],ymm1,0x1 + vextracti128 XMMWORD PTR [r9+0x60],ymm2,0x1 + vextracti128 XMMWORD PTR [r9+0x70],ymm3,0x1 + vmovdqu xmm0,XMMWORD PTR [rsp+0x8] + vmovdqu xmm1,XMMWORD PTR [rsp+0x48] + vmovdqa XMMWORD PTR [rsp],xmm0 + vmovdqa XMMWORD PTR [rsp+0x40],xmm1 + add r9,0x80 + sub r10,0x2 +2: + test r10,0x1 + je 9b + vmovdqu xmm0,XMMWORD PTR [rdi] + vmovdqu xmm1,XMMWORD PTR [rdi+0x10] + vmovd xmm14,DWORD PTR [rsp] + vpinsrd xmm14,xmm14,DWORD PTR [rsp+0x40],0x1 + vpinsrd xmm14,xmm14,edx,0x2 + vmovdqa xmm2,XMMWORD PTR [BLAKE3_IV+rip] + vpinsrd xmm3,xmm14,r8d,0x3 + vmovups xmm8,XMMWORD PTR [rsi] + vmovups xmm9,XMMWORD PTR [rsi+0x10] + vshufps xmm4,xmm8,xmm9,0x88 + vshufps xmm5,xmm8,xmm9,0xdd + vmovups xmm8,XMMWORD PTR [rsi+0x20] + vmovups xmm9,XMMWORD PTR [rsi+0x30] + vshufps xmm6,xmm8,xmm9,0x88 + vshufps xmm7,xmm8,xmm9,0xdd + vpshufd xmm6,xmm6,0x93 + vpshufd xmm7,xmm7,0x93 + mov al,0x7 +3: + vpaddd xmm0,xmm0,xmm4 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x10 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0xc + vpaddd xmm0,xmm0,xmm5 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x8 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0x7 + vpshufd xmm0,xmm0,0x93 + vpshufd xmm3,xmm3,0x4e + vpshufd xmm2,xmm2,0x39 + vpaddd xmm0,xmm0,xmm6 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x10 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0xc + vpaddd xmm0,xmm0,xmm7 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x8 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0x7 + vpshufd xmm0,xmm0,0x39 + vpshufd xmm3,xmm3,0x4e + vpshufd xmm2,xmm2,0x93 + dec al + je 3f + vshufps xmm8,xmm4,xmm5,0xd6 + vpshufd xmm9,xmm4,0xf + vpshufd xmm4,xmm8,0x39 + vshufps xmm8,xmm6,xmm7,0xfa + vpblendd xmm9,xmm9,xmm8,0xaa + vpunpcklqdq xmm8,xmm7,xmm5 + vpblendd xmm8,xmm8,xmm6,0x88 + vpshufd xmm8,xmm8,0x78 + vpunpckhdq xmm5,xmm5,xmm7 + vpunpckldq xmm6,xmm6,xmm5 + vpshufd xmm7,xmm6,0x1e + vmovdqa xmm5,xmm9 + vmovdqa xmm6,xmm8 + jmp 3b +3: + vpxor xmm0,xmm0,xmm2 + vpxor xmm1,xmm1,xmm3 + vpxor xmm2,xmm2,XMMWORD PTR [rdi] + vpxor xmm3,xmm3,XMMWORD PTR [rdi+0x10] + vmovdqu XMMWORD PTR [r9],xmm0 + vmovdqu XMMWORD PTR [r9+0x10],xmm1 + vmovdqu XMMWORD PTR [r9+0x20],xmm2 + vmovdqu XMMWORD PTR [r9+0x30],xmm3 + jmp 9b + + #ifdef __APPLE__ .static_data #else diff --git a/src/third_party/blake3/asm/blake3_avx512_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm similarity index 100% rename from src/third_party/blake3/asm/blake3_avx512_x86-64_windows_msvc.asm rename to src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm diff --git a/src/third_party/blake3/c/blake3_dispatch.c b/src/third_party/blake3/blake3_dispatch.c similarity index 85% rename from src/third_party/blake3/c/blake3_dispatch.c rename to src/third_party/blake3/blake3_dispatch.c index 88f6bc88d..0092a36c0 100644 --- a/src/third_party/blake3/c/blake3_dispatch.c +++ b/src/third_party/blake3/blake3_dispatch.c @@ -4,43 +4,23 @@ #include "blake3_impl.h" -#if defined(IS_X86) -#if defined(_MSC_VER) -#include -#elif defined(__GNUC__) -#include -#else -#undef IS_X86 /* Unimplemented! */ -#endif -#endif +#if defined(IS_X86) -#if !defined(BLAKE3_ATOMICS) -#if defined(__has_include) -#if __has_include() && !defined(_MSC_VER) -#define BLAKE3_ATOMICS 1 +#if defined(__GNUC__) || defined(__clang__) +#define ATOMIC_INT int +#define ATOMIC_LOAD(x) __atomic_load_n(&x, __ATOMIC_RELAXED) +#define ATOMIC_STORE(x, y) __atomic_store_n(&x, y, __ATOMIC_RELAXED) #else -#define BLAKE3_ATOMICS 0 -#endif /* __has_include() && !defined(_MSC_VER) */ -#else -#define BLAKE3_ATOMICS 0 -#endif /* defined(__has_include) */ -#endif /* BLAKE3_ATOMICS */ +#include +#define ATOMIC_INT int +#define ATOMIC_LOAD(x) __iso_volatile_load32(&x) +#define ATOMIC_STORE(x, y) __iso_volatile_store32(&x, y) +#endif -#if BLAKE3_ATOMICS -#define ATOMIC_INT _Atomic int -#define ATOMIC_LOAD(x) x -#define ATOMIC_STORE(x, y) x = y -#elif defined(_MSC_VER) -#define ATOMIC_INT long -#define ATOMIC_LOAD(x) _InterlockedOr(&x, 0) -#define ATOMIC_STORE(x, y) _InterlockedExchange(&x, y) -#else -#define ATOMIC_INT int -#define ATOMIC_LOAD(x) x -#define ATOMIC_STORE(x, y) x = y #endif + #define MAYBE_UNUSED(x) (void)((x)) #if defined(IS_X86) @@ -86,7 +66,6 @@ static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) { #endif } -#endif enum cpu_feature { SSE2 = 1 << 0, @@ -112,9 +91,9 @@ static get_cpu_features(void) { /* If TSAN detects a data race here, try compiling with -DBLAKE3_ATOMICS=1 */ - long features = ATOMIC_LOAD(g_cpu_features); + enum cpu_feature features = ATOMIC_LOAD(g_cpu_features); if (features != UNDEFINED) { - return (enum cpu_feature)features; + return features; } else { #if defined(IS_X86) uint32_t regs[4] = {0}; @@ -154,13 +133,14 @@ static } } ATOMIC_STORE(g_cpu_features, features); - return (enum cpu_feature)features; + return features; #else /* How to detect NEON? */ return 0; #endif } } +#endif void blake3_compress_in_place(uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], @@ -220,6 +200,30 @@ void blake3_compress_xof(const uint32_t cv[8], blake3_compress_xof_portable(cv, block, block_len, counter, flags, out); } + +void blake3_xof_many(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64], size_t outblocks) { + if (outblocks == 0) { + // The current assembly implementation always outputs at least 1 block. + return; + } +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(_WIN32) && !defined(__CYGWIN__) && !defined(BLAKE3_NO_AVX512) + if (features & AVX512VL) { + blake3_xof_many_avx512(cv, block, block_len, counter, flags, out, outblocks); + return; + } +#endif +#endif + for(size_t i = 0; i < outblocks; ++i) { + blake3_compress_xof(cv, block, block_len, counter + i, flags, out + 64*i); + } +} + void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, diff --git a/src/third_party/blake3/asm/blake3_impl.h b/src/third_party/blake3/blake3_impl.h similarity index 84% rename from src/third_party/blake3/asm/blake3_impl.h rename to src/third_party/blake3/blake3_impl.h index beab5cf53..88e71e41e 100644 --- a/src/third_party/blake3/asm/blake3_impl.h +++ b/src/third_party/blake3/blake3_impl.h @@ -9,6 +9,10 @@ #include "blake3.h" +#ifdef __cplusplus +extern "C" { +#endif + // internal flags enum blake3_flags { CHUNK_START = 1 << 0, @@ -28,7 +32,13 @@ enum blake3_flags { #define INLINE static inline __attribute__((always_inline)) #endif -#if defined(__x86_64__) || defined(_M_X64) +#ifdef __cplusplus +#define NOEXCEPT noexcept +#else +#define NOEXCEPT +#endif + +#if (defined(__x86_64__) || defined(_M_X64)) && !defined(_M_ARM64EC) #define IS_X86 #define IS_X86_64 #endif @@ -38,7 +48,7 @@ enum blake3_flags { #define IS_X86_32 #endif -#if defined(__aarch64__) || defined(_M_ARM64) +#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) #define IS_AARCH64 #endif @@ -162,6 +172,13 @@ INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], key_words[7] = load32(&key[7 * 4]); } +INLINE void load_block_words(const uint8_t block[BLAKE3_BLOCK_LEN], + uint32_t block_words[16]) { + for (size_t i = 0; i < 16; i++) { + block_words[i] = load32(&block[i * 4]); + } +} + INLINE void store32(void *dst, uint32_t w) { uint8_t *p = (uint8_t *)dst; p[0] = (uint8_t)(w >> 0); @@ -191,6 +208,11 @@ void blake3_compress_xof(const uint32_t cv[8], uint8_t block_len, uint64_t counter, uint8_t flags, uint8_t out[64]); +void blake3_xof_many(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64], size_t outblocks); + void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, @@ -198,6 +220,22 @@ void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, size_t blake3_simd_degree(void); +BLAKE3_PRIVATE size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out, bool use_tbb); + +#if defined(BLAKE3_USE_TBB) +BLAKE3_PRIVATE void blake3_compress_subtree_wide_join_tbb( + // shared params + const uint32_t key[8], uint8_t flags, bool use_tbb, + // left-hand side params + const uint8_t *l_input, size_t l_input_len, uint64_t l_chunk_counter, + uint8_t *l_cvs, size_t *l_n, + // right-hand side params + const uint8_t *r_input, size_t r_input_len, uint64_t r_chunk_counter, + uint8_t *r_cvs, size_t *r_n) NOEXCEPT; +#endif // Declarations for implementation-specific functions. void blake3_compress_in_place_portable(uint32_t cv[8], @@ -270,6 +308,13 @@ void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, uint64_t counter, bool increment_counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out); + +#if !defined(_WIN32) && !defined(__CYGWIN__) +void blake3_xof_many_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t* out, size_t outblocks); +#endif #endif #endif @@ -281,5 +326,8 @@ void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, uint8_t flags_end, uint8_t *out); #endif +#ifdef __cplusplus +} +#endif #endif /* BLAKE3_IMPL_H */ diff --git a/src/third_party/blake3/asm/blake3_neon.c b/src/third_party/blake3/blake3_neon.c similarity index 96% rename from src/third_party/blake3/asm/blake3_neon.c rename to src/third_party/blake3/blake3_neon.c index 8a818fc78..794ea80f4 100644 --- a/src/third_party/blake3/asm/blake3_neon.c +++ b/src/third_party/blake3/blake3_neon.c @@ -10,14 +10,12 @@ INLINE uint32x4_t loadu_128(const uint8_t src[16]) { // vld1q_u32 has alignment requirements. Don't use it. - uint32x4_t x; - memcpy(&x, src, 16); - return x; + return vreinterpretq_u32_u8(vld1q_u8(src)); } INLINE void storeu_128(uint32x4_t src, uint8_t dest[16]) { // vst1q_u32 has alignment requirements. Don't use it. - memcpy(dest, &src, 16); + vst1q_u8(dest, vreinterpretq_u8_u32(src)); } INLINE uint32x4_t add_128(uint32x4_t a, uint32x4_t b) { @@ -36,7 +34,7 @@ INLINE uint32x4_t set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { } INLINE uint32x4_t rot16_128(uint32x4_t x) { - // The straightfoward implementation would be two shifts and an or, but that's + // The straightforward implementation would be two shifts and an or, but that's // slower on microarchitectures we've tested. See // https://github.com/BLAKE3-team/BLAKE3/pull/319. // return vorrq_u32(vshrq_n_u32(x, 16), vshlq_n_u32(x, 32 - 16)); @@ -245,10 +243,11 @@ INLINE void load_counters4(uint64_t counter, bool increment_counter, counter_high(counter + (mask & 2)), counter_high(counter + (mask & 3))); } -void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { +static void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, + uint8_t *out) { uint32x4_t h_vecs[8] = { set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), diff --git a/src/third_party/blake3/asm/blake3_portable.c b/src/third_party/blake3/blake3_portable.c similarity index 100% rename from src/third_party/blake3/asm/blake3_portable.c rename to src/third_party/blake3/blake3_portable.c diff --git a/src/third_party/blake3/asm/blake3_sse2_x86-64_unix.S b/src/third_party/blake3/blake3_sse2_x86-64_unix.S similarity index 100% rename from src/third_party/blake3/asm/blake3_sse2_x86-64_unix.S rename to src/third_party/blake3/blake3_sse2_x86-64_unix.S diff --git a/src/third_party/blake3/asm/blake3_sse2_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm similarity index 100% rename from src/third_party/blake3/asm/blake3_sse2_x86-64_windows_msvc.asm rename to src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm diff --git a/src/third_party/blake3/asm/blake3_sse41_x86-64_unix.S b/src/third_party/blake3/blake3_sse41_x86-64_unix.S similarity index 100% rename from src/third_party/blake3/asm/blake3_sse41_x86-64_unix.S rename to src/third_party/blake3/blake3_sse41_x86-64_unix.S diff --git a/src/third_party/blake3/asm/blake3_sse41_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_sse41_x86-64_windows_msvc.asm similarity index 100% rename from src/third_party/blake3/asm/blake3_sse41_x86-64_windows_msvc.asm rename to src/third_party/blake3/blake3_sse41_x86-64_windows_msvc.asm diff --git a/src/third_party/blake3/c/LICENSE b/src/third_party/blake3/c/LICENSE deleted file mode 100644 index f5892efc3..000000000 --- a/src/third_party/blake3/c/LICENSE +++ /dev/null @@ -1,330 +0,0 @@ -This work is released into the public domain with CC0 1.0. Alternatively, it is -licensed under the Apache License 2.0. - -------------------------------------------------------------------------------- - -Creative Commons Legal Code - -CC0 1.0 Universal - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator -and subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for -the purpose of contributing to a commons of creative, cultural and -scientific works ("Commons") that the public can reliably and without fear -of later claims of infringement build upon, modify, incorporate in other -works, reuse and redistribute as freely as possible in any form whatsoever -and for any purposes, including without limitation commercial purposes. -These owners may contribute to the Commons to promote the ideal of a free -culture and the further production of creative, cultural and scientific -works, or to gain reputation or greater distribution for their Work in -part through the use and efforts of others. - -For these and/or other purposes and motivations, and without any -expectation of additional consideration or compensation, the person -associating CC0 with a Work (the "Affirmer"), to the extent that he or she -is an owner of Copyright and Related Rights in the Work, voluntarily -elects to apply CC0 to the Work and publicly distribute the Work under its -terms, with knowledge of his or her Copyright and Related Rights in the -Work and the meaning and intended legal effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not -limited to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, - communicate, and translate a Work; - ii. moral rights retained by the original author(s) and/or performer(s); -iii. publicity and privacy rights pertaining to a person's image or - likeness depicted in a Work; - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - v. rights protecting the extraction, dissemination, use and reuse of data - in a Work; - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation - thereof, including any amended or successor version of such - directive); and -vii. other similar, equivalent or corresponding rights throughout the - world based on applicable law or treaty, and any national - implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention -of, applicable law, Affirmer hereby overtly, fully, permanently, -irrevocably and unconditionally waives, abandons, and surrenders all of -Affirmer's Copyright and Related Rights and associated claims and causes -of action, whether now known or unknown (including existing as well as -future claims and causes of action), in the Work (i) in all territories -worldwide, (ii) for the maximum duration provided by applicable law or -treaty (including future time extensions), (iii) in any current or future -medium and for any number of copies, and (iv) for any purpose whatsoever, -including without limitation commercial, advertising or promotional -purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each -member of the public at large and to the detriment of Affirmer's heirs and -successors, fully intending that such Waiver shall not be subject to -revocation, rescission, cancellation, termination, or any other legal or -equitable action to disrupt the quiet enjoyment of the Work by the public -as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason -be judged legally invalid or ineffective under applicable law, then the -Waiver shall be preserved to the maximum extent permitted taking into -account Affirmer's express Statement of Purpose. In addition, to the -extent the Waiver is so judged Affirmer hereby grants to each affected -person a royalty-free, non transferable, non sublicensable, non exclusive, -irrevocable and unconditional license to exercise Affirmer's Copyright and -Related Rights in the Work (i) in all territories worldwide, (ii) for the -maximum duration provided by applicable law or treaty (including future -time extensions), (iii) in any current or future medium and for any number -of copies, and (iv) for any purpose whatsoever, including without -limitation commercial, advertising or promotional purposes (the -"License"). The License shall be deemed effective as of the date CC0 was -applied by Affirmer to the Work. Should any part of the License for any -reason be judged legally invalid or ineffective under applicable law, such -partial invalidity or ineffectiveness shall not invalidate the remainder -of the License, and in such case Affirmer hereby affirms that he or she -will not (i) exercise any of his or her remaining Copyright and Related -Rights in the Work or (ii) assert any associated claims and causes of -action with respect to the Work, in either case contrary to Affirmer's -express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - b. Affirmer offers the Work as-is and makes no representations or - warranties of any kind concerning the Work, express, implied, - statutory or otherwise, including without limitation warranties of - title, merchantability, fitness for a particular purpose, non - infringement, or the absence of latent or other defects, accuracy, or - the present or absence of errors, whether or not discoverable, all to - the greatest extent permissible under applicable law. - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without - limitation any person's Copyright and Related Rights in the Work. - Further, Affirmer disclaims responsibility for obtaining any necessary - consents, permissions or other rights required for any use of the - Work. - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to - this CC0 or use of the Work. - -------------------------------------------------------------------------------- - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2019 Jack O'Connor and Samuel Neves - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/third_party/blake3/c/blake3.c b/src/third_party/blake3/c/blake3.c deleted file mode 100644 index 692f4b021..000000000 --- a/src/third_party/blake3/c/blake3.c +++ /dev/null @@ -1,616 +0,0 @@ -#include -#include -#include - -#include "blake3.h" -#include "blake3_impl.h" - -const char *blake3_version(void) { return BLAKE3_VERSION_STRING; } - -INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8], - uint8_t flags) { - memcpy(self->cv, key, BLAKE3_KEY_LEN); - self->chunk_counter = 0; - memset(self->buf, 0, BLAKE3_BLOCK_LEN); - self->buf_len = 0; - self->blocks_compressed = 0; - self->flags = flags; -} - -INLINE void chunk_state_reset(blake3_chunk_state *self, const uint32_t key[8], - uint64_t chunk_counter) { - memcpy(self->cv, key, BLAKE3_KEY_LEN); - self->chunk_counter = chunk_counter; - self->blocks_compressed = 0; - memset(self->buf, 0, BLAKE3_BLOCK_LEN); - self->buf_len = 0; -} - -INLINE size_t chunk_state_len(const blake3_chunk_state *self) { - return (BLAKE3_BLOCK_LEN * (size_t)self->blocks_compressed) + - ((size_t)self->buf_len); -} - -INLINE size_t chunk_state_fill_buf(blake3_chunk_state *self, - const uint8_t *input, size_t input_len) { - size_t take = BLAKE3_BLOCK_LEN - ((size_t)self->buf_len); - if (take > input_len) { - take = input_len; - } - uint8_t *dest = self->buf + ((size_t)self->buf_len); - memcpy(dest, input, take); - self->buf_len += (uint8_t)take; - return take; -} - -INLINE uint8_t chunk_state_maybe_start_flag(const blake3_chunk_state *self) { - if (self->blocks_compressed == 0) { - return CHUNK_START; - } else { - return 0; - } -} - -typedef struct { - uint32_t input_cv[8]; - uint64_t counter; - uint8_t block[BLAKE3_BLOCK_LEN]; - uint8_t block_len; - uint8_t flags; -} output_t; - -INLINE output_t make_output(const uint32_t input_cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - output_t ret; - memcpy(ret.input_cv, input_cv, 32); - memcpy(ret.block, block, BLAKE3_BLOCK_LEN); - ret.block_len = block_len; - ret.counter = counter; - ret.flags = flags; - return ret; -} - -// Chaining values within a given chunk (specifically the compress_in_place -// interface) are represented as words. This avoids unnecessary bytes<->words -// conversion overhead in the portable implementation. However, the hash_many -// interface handles both user input and parent node blocks, so it accepts -// bytes. For that reason, chaining values in the CV stack are represented as -// bytes. -INLINE void output_chaining_value(const output_t *self, uint8_t cv[32]) { - uint32_t cv_words[8]; - memcpy(cv_words, self->input_cv, 32); - blake3_compress_in_place(cv_words, self->block, self->block_len, - self->counter, self->flags); - store_cv_words(cv, cv_words); -} - -INLINE void output_root_bytes(const output_t *self, uint64_t seek, uint8_t *out, - size_t out_len) { - uint64_t output_block_counter = seek / 64; - size_t offset_within_block = seek % 64; - uint8_t wide_buf[64]; - while (out_len > 0) { - blake3_compress_xof(self->input_cv, self->block, self->block_len, - output_block_counter, self->flags | ROOT, wide_buf); - size_t available_bytes = 64 - offset_within_block; - size_t memcpy_len; - if (out_len > available_bytes) { - memcpy_len = available_bytes; - } else { - memcpy_len = out_len; - } - memcpy(out, wide_buf + offset_within_block, memcpy_len); - out += memcpy_len; - out_len -= memcpy_len; - output_block_counter += 1; - offset_within_block = 0; - } -} - -INLINE void chunk_state_update(blake3_chunk_state *self, const uint8_t *input, - size_t input_len) { - if (self->buf_len > 0) { - size_t take = chunk_state_fill_buf(self, input, input_len); - input += take; - input_len -= take; - if (input_len > 0) { - blake3_compress_in_place( - self->cv, self->buf, BLAKE3_BLOCK_LEN, self->chunk_counter, - self->flags | chunk_state_maybe_start_flag(self)); - self->blocks_compressed += 1; - self->buf_len = 0; - memset(self->buf, 0, BLAKE3_BLOCK_LEN); - } - } - - while (input_len > BLAKE3_BLOCK_LEN) { - blake3_compress_in_place(self->cv, input, BLAKE3_BLOCK_LEN, - self->chunk_counter, - self->flags | chunk_state_maybe_start_flag(self)); - self->blocks_compressed += 1; - input += BLAKE3_BLOCK_LEN; - input_len -= BLAKE3_BLOCK_LEN; - } - - size_t take = chunk_state_fill_buf(self, input, input_len); - input += take; - input_len -= take; -} - -INLINE output_t chunk_state_output(const blake3_chunk_state *self) { - uint8_t block_flags = - self->flags | chunk_state_maybe_start_flag(self) | CHUNK_END; - return make_output(self->cv, self->buf, self->buf_len, self->chunk_counter, - block_flags); -} - -INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], - const uint32_t key[8], uint8_t flags) { - return make_output(key, block, BLAKE3_BLOCK_LEN, 0, flags | PARENT); -} - -// Given some input larger than one chunk, return the number of bytes that -// should go in the left subtree. This is the largest power-of-2 number of -// chunks that leaves at least 1 byte for the right subtree. -INLINE size_t left_len(size_t content_len) { - // Subtract 1 to reserve at least one byte for the right side. content_len - // should always be greater than BLAKE3_CHUNK_LEN. - size_t full_chunks = (content_len - 1) / BLAKE3_CHUNK_LEN; - return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN; -} - -// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE chunks at the same time -// on a single thread. Write out the chunk chaining values and return the -// number of chunks hashed. These chunks are never the root and never empty; -// those cases use a different codepath. -INLINE size_t compress_chunks_parallel(const uint8_t *input, size_t input_len, - const uint32_t key[8], - uint64_t chunk_counter, uint8_t flags, - uint8_t *out) { -#if defined(BLAKE3_TESTING) - assert(0 < input_len); - assert(input_len <= MAX_SIMD_DEGREE * BLAKE3_CHUNK_LEN); -#endif - - const uint8_t *chunks_array[MAX_SIMD_DEGREE]; - size_t input_position = 0; - size_t chunks_array_len = 0; - while (input_len - input_position >= BLAKE3_CHUNK_LEN) { - chunks_array[chunks_array_len] = &input[input_position]; - input_position += BLAKE3_CHUNK_LEN; - chunks_array_len += 1; - } - - blake3_hash_many(chunks_array, chunks_array_len, - BLAKE3_CHUNK_LEN / BLAKE3_BLOCK_LEN, key, chunk_counter, - true, flags, CHUNK_START, CHUNK_END, out); - - // Hash the remaining partial chunk, if there is one. Note that the empty - // chunk (meaning the empty message) is a different codepath. - if (input_len > input_position) { - uint64_t counter = chunk_counter + (uint64_t)chunks_array_len; - blake3_chunk_state chunk_state; - chunk_state_init(&chunk_state, key, flags); - chunk_state.chunk_counter = counter; - chunk_state_update(&chunk_state, &input[input_position], - input_len - input_position); - output_t output = chunk_state_output(&chunk_state); - output_chaining_value(&output, &out[chunks_array_len * BLAKE3_OUT_LEN]); - return chunks_array_len + 1; - } else { - return chunks_array_len; - } -} - -// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE parents at the same time -// on a single thread. Write out the parent chaining values and return the -// number of parents hashed. (If there's an odd input chaining value left over, -// return it as an additional output.) These parents are never the root and -// never empty; those cases use a different codepath. -INLINE size_t compress_parents_parallel(const uint8_t *child_chaining_values, - size_t num_chaining_values, - const uint32_t key[8], uint8_t flags, - uint8_t *out) { -#if defined(BLAKE3_TESTING) - assert(2 <= num_chaining_values); - assert(num_chaining_values <= 2 * MAX_SIMD_DEGREE_OR_2); -#endif - - const uint8_t *parents_array[MAX_SIMD_DEGREE_OR_2]; - size_t parents_array_len = 0; - while (num_chaining_values - (2 * parents_array_len) >= 2) { - parents_array[parents_array_len] = - &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN]; - parents_array_len += 1; - } - - blake3_hash_many(parents_array, parents_array_len, 1, key, - 0, // Parents always use counter 0. - false, flags | PARENT, - 0, // Parents have no start flags. - 0, // Parents have no end flags. - out); - - // If there's an odd child left over, it becomes an output. - if (num_chaining_values > 2 * parents_array_len) { - memcpy(&out[parents_array_len * BLAKE3_OUT_LEN], - &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN], - BLAKE3_OUT_LEN); - return parents_array_len + 1; - } else { - return parents_array_len; - } -} - -// The wide helper function returns (writes out) an array of chaining values -// and returns the length of that array. The number of chaining values returned -// is the dynamically detected SIMD degree, at most MAX_SIMD_DEGREE. Or fewer, -// if the input is shorter than that many chunks. The reason for maintaining a -// wide array of chaining values going back up the tree, is to allow the -// implementation to hash as many parents in parallel as possible. -// -// As a special case when the SIMD degree is 1, this function will still return -// at least 2 outputs. This guarantees that this function doesn't perform the -// root compression. (If it did, it would use the wrong flags, and also we -// wouldn't be able to implement extendable output.) Note that this function is -// not used when the whole input is only 1 chunk long; that's a different -// codepath. -// -// Why not just have the caller split the input on the first update(), instead -// of implementing this special rule? Because we don't want to limit SIMD or -// multi-threading parallelism for that update(). -static size_t blake3_compress_subtree_wide(const uint8_t *input, - size_t input_len, - const uint32_t key[8], - uint64_t chunk_counter, - uint8_t flags, uint8_t *out) { - // Note that the single chunk case does *not* bump the SIMD degree up to 2 - // when it is 1. If this implementation adds multi-threading in the future, - // this gives us the option of multi-threading even the 2-chunk case, which - // can help performance on smaller platforms. - if (input_len <= blake3_simd_degree() * BLAKE3_CHUNK_LEN) { - return compress_chunks_parallel(input, input_len, key, chunk_counter, flags, - out); - } - - // With more than simd_degree chunks, we need to recurse. Start by dividing - // the input into left and right subtrees. (Note that this is only optimal - // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree - // of 3 or something, we'll need a more complicated strategy.) - size_t left_input_len = left_len(input_len); - size_t right_input_len = input_len - left_input_len; - const uint8_t *right_input = &input[left_input_len]; - uint64_t right_chunk_counter = - chunk_counter + (uint64_t)(left_input_len / BLAKE3_CHUNK_LEN); - - // Make space for the child outputs. Here we use MAX_SIMD_DEGREE_OR_2 to - // account for the special case of returning 2 outputs when the SIMD degree - // is 1. - uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; - size_t degree = blake3_simd_degree(); - if (left_input_len > BLAKE3_CHUNK_LEN && degree == 1) { - // The special case: We always use a degree of at least two, to make - // sure there are two outputs. Except, as noted above, at the chunk - // level, where we allow degree=1. (Note that the 1-chunk-input case is - // a different codepath.) - degree = 2; - } - uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN]; - - // Recurse! If this implementation adds multi-threading support in the - // future, this is where it will go. - size_t left_n = blake3_compress_subtree_wide(input, left_input_len, key, - chunk_counter, flags, cv_array); - size_t right_n = blake3_compress_subtree_wide( - right_input, right_input_len, key, right_chunk_counter, flags, right_cvs); - - // The special case again. If simd_degree=1, then we'll have left_n=1 and - // right_n=1. Rather than compressing them into a single output, return - // them directly, to make sure we always have at least two outputs. - if (left_n == 1) { - memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); - return 2; - } - - // Otherwise, do one layer of parent node compression. - size_t num_chaining_values = left_n + right_n; - return compress_parents_parallel(cv_array, num_chaining_values, key, flags, - out); -} - -// Hash a subtree with compress_subtree_wide(), and then condense the resulting -// list of chaining values down to a single parent node. Don't compress that -// last parent node, however. Instead, return its message bytes (the -// concatenated chaining values of its children). This is necessary when the -// first call to update() supplies a complete subtree, because the topmost -// parent node of that subtree could end up being the root. It's also necessary -// for extended output in the general case. -// -// As with compress_subtree_wide(), this function is not used on inputs of 1 -// chunk or less. That's a different codepath. -INLINE void compress_subtree_to_parent_node( - const uint8_t *input, size_t input_len, const uint32_t key[8], - uint64_t chunk_counter, uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN]) { -#if defined(BLAKE3_TESTING) - assert(input_len > BLAKE3_CHUNK_LEN); -#endif - - uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; - size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key, - chunk_counter, flags, cv_array); - assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); - - // If MAX_SIMD_DEGREE is greater than 2 and there's enough input, - // compress_subtree_wide() returns more than 2 chaining values. Condense - // them into 2 by forming parent nodes repeatedly. - uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; - // The second half of this loop condition is always true, and we just - // asserted it above. But GCC can't tell that it's always true, and if NDEBUG - // is set on platforms where MAX_SIMD_DEGREE_OR_2 == 2, GCC emits spurious - // warnings here. GCC 8.5 is particularly sensitive, so if you're changing - // this code, test it against that version. - while (num_cvs > 2 && num_cvs <= MAX_SIMD_DEGREE_OR_2) { - num_cvs = - compress_parents_parallel(cv_array, num_cvs, key, flags, out_array); - memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); - } - memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); -} - -INLINE void hasher_init_base(blake3_hasher *self, const uint32_t key[8], - uint8_t flags) { - memcpy(self->key, key, BLAKE3_KEY_LEN); - chunk_state_init(&self->chunk, key, flags); - self->cv_stack_len = 0; -} - -void blake3_hasher_init(blake3_hasher *self) { hasher_init_base(self, IV, 0); } - -void blake3_hasher_init_keyed(blake3_hasher *self, - const uint8_t key[BLAKE3_KEY_LEN]) { - uint32_t key_words[8]; - load_key_words(key, key_words); - hasher_init_base(self, key_words, KEYED_HASH); -} - -void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, - size_t context_len) { - blake3_hasher context_hasher; - hasher_init_base(&context_hasher, IV, DERIVE_KEY_CONTEXT); - blake3_hasher_update(&context_hasher, context, context_len); - uint8_t context_key[BLAKE3_KEY_LEN]; - blake3_hasher_finalize(&context_hasher, context_key, BLAKE3_KEY_LEN); - uint32_t context_key_words[8]; - load_key_words(context_key, context_key_words); - hasher_init_base(self, context_key_words, DERIVE_KEY_MATERIAL); -} - -void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context) { - blake3_hasher_init_derive_key_raw(self, context, strlen(context)); -} - -// As described in hasher_push_cv() below, we do "lazy merging", delaying -// merges until right before the next CV is about to be added. This is -// different from the reference implementation. Another difference is that we -// aren't always merging 1 chunk at a time. Instead, each CV might represent -// any power-of-two number of chunks, as long as the smaller-above-larger stack -// order is maintained. Instead of the "count the trailing 0-bits" algorithm -// described in the spec, we use a "count the total number of 1-bits" variant -// that doesn't require us to retain the subtree size of the CV on top of the -// stack. The principle is the same: each CV that should remain in the stack is -// represented by a 1-bit in the total number of chunks (or bytes) so far. -INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) { - size_t post_merge_stack_len = (size_t)popcnt(total_len); - while (self->cv_stack_len > post_merge_stack_len) { - uint8_t *parent_node = - &self->cv_stack[(self->cv_stack_len - 2) * BLAKE3_OUT_LEN]; - output_t output = parent_output(parent_node, self->key, self->chunk.flags); - output_chaining_value(&output, parent_node); - self->cv_stack_len -= 1; - } -} - -// In reference_impl.rs, we merge the new CV with existing CVs from the stack -// before pushing it. We can do that because we know more input is coming, so -// we know none of the merges are root. -// -// This setting is different. We want to feed as much input as possible to -// compress_subtree_wide(), without setting aside anything for the chunk_state. -// If the user gives us 64 KiB, we want to parallelize over all 64 KiB at once -// as a single subtree, if at all possible. -// -// This leads to two problems: -// 1) This 64 KiB input might be the only call that ever gets made to update. -// In this case, the root node of the 64 KiB subtree would be the root node -// of the whole tree, and it would need to be ROOT finalized. We can't -// compress it until we know. -// 2) This 64 KiB input might complete a larger tree, whose root node is -// similarly going to be the the root of the whole tree. For example, maybe -// we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the -// node at the root of the 256 KiB subtree until we know how to finalize it. -// -// The second problem is solved with "lazy merging". That is, when we're about -// to add a CV to the stack, we don't merge it with anything first, as the -// reference impl does. Instead we do merges using the *previous* CV that was -// added, which is sitting on top of the stack, and we put the new CV -// (unmerged) on top of the stack afterwards. This guarantees that we never -// merge the root node until finalize(). -// -// Solving the first problem requires an additional tool, -// compress_subtree_to_parent_node(). That function always returns the top -// *two* chaining values of the subtree it's compressing. We then do lazy -// merging with each of them separately, so that the second CV will always -// remain unmerged. (That also helps us support extendable output when we're -// hashing an input all-at-once.) -INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], - uint64_t chunk_counter) { - hasher_merge_cv_stack(self, chunk_counter); - memcpy(&self->cv_stack[self->cv_stack_len * BLAKE3_OUT_LEN], new_cv, - BLAKE3_OUT_LEN); - self->cv_stack_len += 1; -} - -void blake3_hasher_update(blake3_hasher *self, const void *input, - size_t input_len) { - // Explicitly checking for zero avoids causing UB by passing a null pointer - // to memcpy. This comes up in practice with things like: - // std::vector v; - // blake3_hasher_update(&hasher, v.data(), v.size()); - if (input_len == 0) { - return; - } - - const uint8_t *input_bytes = (const uint8_t *)input; - - // If we have some partial chunk bytes in the internal chunk_state, we need - // to finish that chunk first. - if (chunk_state_len(&self->chunk) > 0) { - size_t take = BLAKE3_CHUNK_LEN - chunk_state_len(&self->chunk); - if (take > input_len) { - take = input_len; - } - chunk_state_update(&self->chunk, input_bytes, take); - input_bytes += take; - input_len -= take; - // If we've filled the current chunk and there's more coming, finalize this - // chunk and proceed. In this case we know it's not the root. - if (input_len > 0) { - output_t output = chunk_state_output(&self->chunk); - uint8_t chunk_cv[32]; - output_chaining_value(&output, chunk_cv); - hasher_push_cv(self, chunk_cv, self->chunk.chunk_counter); - chunk_state_reset(&self->chunk, self->key, self->chunk.chunk_counter + 1); - } else { - return; - } - } - - // Now the chunk_state is clear, and we have more input. If there's more than - // a single chunk (so, definitely not the root chunk), hash the largest whole - // subtree we can, with the full benefits of SIMD (and maybe in the future, - // multi-threading) parallelism. Two restrictions: - // - The subtree has to be a power-of-2 number of chunks. Only subtrees along - // the right edge can be incomplete, and we don't know where the right edge - // is going to be until we get to finalize(). - // - The subtree must evenly divide the total number of chunks up until this - // point (if total is not 0). If the current incomplete subtree is only - // waiting for 1 more chunk, we can't hash a subtree of 4 chunks. We have - // to complete the current subtree first. - // Because we might need to break up the input to form powers of 2, or to - // evenly divide what we already have, this part runs in a loop. - while (input_len > BLAKE3_CHUNK_LEN) { - size_t subtree_len = round_down_to_power_of_2(input_len); - uint64_t count_so_far = self->chunk.chunk_counter * BLAKE3_CHUNK_LEN; - // Shrink the subtree_len until it evenly divides the count so far. We know - // that subtree_len itself is a power of 2, so we can use a bitmasking - // trick instead of an actual remainder operation. (Note that if the caller - // consistently passes power-of-2 inputs of the same size, as is hopefully - // typical, this loop condition will always fail, and subtree_len will - // always be the full length of the input.) - // - // An aside: We don't have to shrink subtree_len quite this much. For - // example, if count_so_far is 1, we could pass 2 chunks to - // compress_subtree_to_parent_node. Since we'll get 2 CVs back, we'll still - // get the right answer in the end, and we might get to use 2-way SIMD - // parallelism. The problem with this optimization, is that it gets us - // stuck always hashing 2 chunks. The total number of chunks will remain - // odd, and we'll never graduate to higher degrees of parallelism. See - // https://github.com/BLAKE3-team/BLAKE3/issues/69. - while ((((uint64_t)(subtree_len - 1)) & count_so_far) != 0) { - subtree_len /= 2; - } - // The shrunken subtree_len might now be 1 chunk long. If so, hash that one - // chunk by itself. Otherwise, compress the subtree into a pair of CVs. - uint64_t subtree_chunks = subtree_len / BLAKE3_CHUNK_LEN; - if (subtree_len <= BLAKE3_CHUNK_LEN) { - blake3_chunk_state chunk_state; - chunk_state_init(&chunk_state, self->key, self->chunk.flags); - chunk_state.chunk_counter = self->chunk.chunk_counter; - chunk_state_update(&chunk_state, input_bytes, subtree_len); - output_t output = chunk_state_output(&chunk_state); - uint8_t cv[BLAKE3_OUT_LEN]; - output_chaining_value(&output, cv); - hasher_push_cv(self, cv, chunk_state.chunk_counter); - } else { - // This is the high-performance happy path, though getting here depends - // on the caller giving us a long enough input. - uint8_t cv_pair[2 * BLAKE3_OUT_LEN]; - compress_subtree_to_parent_node(input_bytes, subtree_len, self->key, - self->chunk.chunk_counter, - self->chunk.flags, cv_pair); - hasher_push_cv(self, cv_pair, self->chunk.chunk_counter); - hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN], - self->chunk.chunk_counter + (subtree_chunks / 2)); - } - self->chunk.chunk_counter += subtree_chunks; - input_bytes += subtree_len; - input_len -= subtree_len; - } - - // If there's any remaining input less than a full chunk, add it to the chunk - // state. In that case, also do a final merge loop to make sure the subtree - // stack doesn't contain any unmerged pairs. The remaining input means we - // know these merges are non-root. This merge loop isn't strictly necessary - // here, because hasher_push_chunk_cv already does its own merge loop, but it - // simplifies blake3_hasher_finalize below. - if (input_len > 0) { - chunk_state_update(&self->chunk, input_bytes, input_len); - hasher_merge_cv_stack(self, self->chunk.chunk_counter); - } -} - -void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, - size_t out_len) { - blake3_hasher_finalize_seek(self, 0, out, out_len); -} - -void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, - uint8_t *out, size_t out_len) { - // Explicitly checking for zero avoids causing UB by passing a null pointer - // to memcpy. This comes up in practice with things like: - // std::vector v; - // blake3_hasher_finalize(&hasher, v.data(), v.size()); - if (out_len == 0) { - return; - } - - // If the subtree stack is empty, then the current chunk is the root. - if (self->cv_stack_len == 0) { - output_t output = chunk_state_output(&self->chunk); - output_root_bytes(&output, seek, out, out_len); - return; - } - // If there are any bytes in the chunk state, finalize that chunk and do a - // roll-up merge between that chunk hash and every subtree in the stack. In - // this case, the extra merge loop at the end of blake3_hasher_update - // guarantees that none of the subtrees in the stack need to be merged with - // each other first. Otherwise, if there are no bytes in the chunk state, - // then the top of the stack is a chunk hash, and we start the merge from - // that. - output_t output; - size_t cvs_remaining; - if (chunk_state_len(&self->chunk) > 0) { - cvs_remaining = self->cv_stack_len; - output = chunk_state_output(&self->chunk); - } else { - // There are always at least 2 CVs in the stack in this case. - cvs_remaining = self->cv_stack_len - 2; - output = parent_output(&self->cv_stack[cvs_remaining * 32], self->key, - self->chunk.flags); - } - while (cvs_remaining > 0) { - cvs_remaining -= 1; - uint8_t parent_block[BLAKE3_BLOCK_LEN]; - memcpy(parent_block, &self->cv_stack[cvs_remaining * 32], 32); - output_chaining_value(&output, &parent_block[32]); - output = parent_output(parent_block, self->key, self->chunk.flags); - } - output_root_bytes(&output, seek, out, out_len); -} - -void blake3_hasher_reset(blake3_hasher *self) { - chunk_state_reset(&self->chunk, self->key, 0); - self->cv_stack_len = 0; -} diff --git a/src/third_party/blake3/c/blake3_avx2.c b/src/third_party/blake3/c/blake3_avx2.c deleted file mode 100644 index 46b5cecfd..000000000 --- a/src/third_party/blake3/c/blake3_avx2.c +++ /dev/null @@ -1,312 +0,0 @@ -#include "blake3_impl.h" - -#include - -#define DEGREE 8 - -INLINE __m256i loadu(const uint8_t src[32]) { - return _mm256_loadu_si256((const __m256i *)src); -} - -INLINE void storeu(__m256i src, uint8_t dest[16]) { - _mm256_storeu_si256((__m256i *)dest, src); -} - -INLINE __m256i addv(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); } - -// Note that clang-format doesn't like the name "xor" for some reason. -INLINE __m256i xorv(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); } - -INLINE __m256i set1(uint32_t x) { return _mm256_set1_epi32((int32_t)x); } - -INLINE __m256i rot16(__m256i x) { - return _mm256_shuffle_epi8( - x, _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, - 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)); -} - -INLINE __m256i rot12(__m256i x) { - return _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 32 - 12)); -} - -INLINE __m256i rot8(__m256i x) { - return _mm256_shuffle_epi8( - x, _mm256_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1, - 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)); -} - -INLINE __m256i rot7(__m256i x) { - return _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 32 - 7)); -} - -INLINE void round_fn(__m256i v[16], __m256i m[16], size_t r) { - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[15] = rot16(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot12(v[4]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[15] = rot8(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot7(v[4]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot16(v[15]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[4] = rot12(v[4]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot8(v[15]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - v[4] = rot7(v[4]); -} - -INLINE void transpose_vecs(__m256i vecs[DEGREE]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high - // is 22/33/66/77. - __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]); - __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]); - __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]); - __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]); - __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]); - __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]); - __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]); - __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]); - - // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is - // 11/33. - __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145); - __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145); - __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367); - __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367); - __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145); - __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145); - __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367); - __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367); - - // Interleave 128-bit lanes. - vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20); - vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20); - vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20); - vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20); - vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31); - vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31); - vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31); - vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31); -} - -INLINE void transpose_msg_vecs(const uint8_t *const *inputs, - size_t block_offset, __m256i out[16]) { - out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m256i)]); - out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m256i)]); - out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m256i)]); - out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m256i)]); - out[4] = loadu(&inputs[4][block_offset + 0 * sizeof(__m256i)]); - out[5] = loadu(&inputs[5][block_offset + 0 * sizeof(__m256i)]); - out[6] = loadu(&inputs[6][block_offset + 0 * sizeof(__m256i)]); - out[7] = loadu(&inputs[7][block_offset + 0 * sizeof(__m256i)]); - out[8] = loadu(&inputs[0][block_offset + 1 * sizeof(__m256i)]); - out[9] = loadu(&inputs[1][block_offset + 1 * sizeof(__m256i)]); - out[10] = loadu(&inputs[2][block_offset + 1 * sizeof(__m256i)]); - out[11] = loadu(&inputs[3][block_offset + 1 * sizeof(__m256i)]); - out[12] = loadu(&inputs[4][block_offset + 1 * sizeof(__m256i)]); - out[13] = loadu(&inputs[5][block_offset + 1 * sizeof(__m256i)]); - out[14] = loadu(&inputs[6][block_offset + 1 * sizeof(__m256i)]); - out[15] = loadu(&inputs[7][block_offset + 1 * sizeof(__m256i)]); - for (size_t i = 0; i < 8; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs(&out[0]); - transpose_vecs(&out[8]); -} - -INLINE void load_counters(uint64_t counter, bool increment_counter, - __m256i *out_lo, __m256i *out_hi) { - const __m256i mask = _mm256_set1_epi32(-(int32_t)increment_counter); - const __m256i add0 = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); - const __m256i add1 = _mm256_and_si256(mask, add0); - __m256i l = _mm256_add_epi32(_mm256_set1_epi32((int32_t)counter), add1); - __m256i carry = _mm256_cmpgt_epi32(_mm256_xor_si256(add1, _mm256_set1_epi32(0x80000000)), - _mm256_xor_si256( l, _mm256_set1_epi32(0x80000000))); - __m256i h = _mm256_sub_epi32(_mm256_set1_epi32((int32_t)(counter >> 32)), carry); - *out_lo = l; - *out_hi = h; -} - -static -void blake3_hash8_avx2(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m256i h_vecs[8] = { - set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), - set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), - }; - __m256i counter_low_vec, counter_high_vec; - load_counters(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m256i block_len_vec = set1(BLAKE3_BLOCK_LEN); - __m256i block_flags_vec = set1(block_flags); - __m256i msg_vecs[16]; - transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m256i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn(v, msg_vecs, 0); - round_fn(v, msg_vecs, 1); - round_fn(v, msg_vecs, 2); - round_fn(v, msg_vecs, 3); - round_fn(v, msg_vecs, 4); - round_fn(v, msg_vecs, 5); - round_fn(v, msg_vecs, 6); - h_vecs[0] = xorv(v[0], v[8]); - h_vecs[1] = xorv(v[1], v[9]); - h_vecs[2] = xorv(v[2], v[10]); - h_vecs[3] = xorv(v[3], v[11]); - h_vecs[4] = xorv(v[4], v[12]); - h_vecs[5] = xorv(v[5], v[13]); - h_vecs[6] = xorv(v[6], v[14]); - h_vecs[7] = xorv(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs(h_vecs); - storeu(h_vecs[0], &out[0 * sizeof(__m256i)]); - storeu(h_vecs[1], &out[1 * sizeof(__m256i)]); - storeu(h_vecs[2], &out[2 * sizeof(__m256i)]); - storeu(h_vecs[3], &out[3 * sizeof(__m256i)]); - storeu(h_vecs[4], &out[4 * sizeof(__m256i)]); - storeu(h_vecs[5], &out[5 * sizeof(__m256i)]); - storeu(h_vecs[6], &out[6 * sizeof(__m256i)]); - storeu(h_vecs[7], &out[7 * sizeof(__m256i)]); -} - -void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= DEGREE) { - blake3_hash8_avx2(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += DEGREE; - } - inputs += DEGREE; - num_inputs -= DEGREE; - out = &out[DEGREE * BLAKE3_OUT_LEN]; - } -#if !defined(BLAKE3_NO_SSE41) - blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, out); -#else - blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); -#endif -} diff --git a/src/third_party/blake3/c/blake3_avx512.c b/src/third_party/blake3/c/blake3_avx512.c deleted file mode 100644 index f969bf37f..000000000 --- a/src/third_party/blake3/c/blake3_avx512.c +++ /dev/null @@ -1,1220 +0,0 @@ -#include "blake3_impl.h" - -#include - -#define _mm_shuffle_ps2(a, b, c) \ - (_mm_castps_si128( \ - _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) - -INLINE __m128i loadu_128(const uint8_t src[16]) { - return _mm_loadu_si128((const __m128i *)src); -} - -INLINE __m256i loadu_256(const uint8_t src[32]) { - return _mm256_loadu_si256((const __m256i *)src); -} - -INLINE __m512i loadu_512(const uint8_t src[64]) { - return _mm512_loadu_si512((const __m512i *)src); -} - -INLINE void storeu_128(__m128i src, uint8_t dest[16]) { - _mm_storeu_si128((__m128i *)dest, src); -} - -INLINE void storeu_256(__m256i src, uint8_t dest[16]) { - _mm256_storeu_si256((__m256i *)dest, src); -} - -INLINE __m128i add_128(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } - -INLINE __m256i add_256(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); } - -INLINE __m512i add_512(__m512i a, __m512i b) { return _mm512_add_epi32(a, b); } - -INLINE __m128i xor_128(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } - -INLINE __m256i xor_256(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); } - -INLINE __m512i xor_512(__m512i a, __m512i b) { return _mm512_xor_si512(a, b); } - -INLINE __m128i set1_128(uint32_t x) { return _mm_set1_epi32((int32_t)x); } - -INLINE __m256i set1_256(uint32_t x) { return _mm256_set1_epi32((int32_t)x); } - -INLINE __m512i set1_512(uint32_t x) { return _mm512_set1_epi32((int32_t)x); } - -INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); -} - -INLINE __m128i rot16_128(__m128i x) { return _mm_ror_epi32(x, 16); } - -INLINE __m256i rot16_256(__m256i x) { return _mm256_ror_epi32(x, 16); } - -INLINE __m512i rot16_512(__m512i x) { return _mm512_ror_epi32(x, 16); } - -INLINE __m128i rot12_128(__m128i x) { return _mm_ror_epi32(x, 12); } - -INLINE __m256i rot12_256(__m256i x) { return _mm256_ror_epi32(x, 12); } - -INLINE __m512i rot12_512(__m512i x) { return _mm512_ror_epi32(x, 12); } - -INLINE __m128i rot8_128(__m128i x) { return _mm_ror_epi32(x, 8); } - -INLINE __m256i rot8_256(__m256i x) { return _mm256_ror_epi32(x, 8); } - -INLINE __m512i rot8_512(__m512i x) { return _mm512_ror_epi32(x, 8); } - -INLINE __m128i rot7_128(__m128i x) { return _mm_ror_epi32(x, 7); } - -INLINE __m256i rot7_256(__m256i x) { return _mm256_ror_epi32(x, 7); } - -INLINE __m512i rot7_512(__m512i x) { return _mm512_ror_epi32(x, 7); } - -/* - * ---------------------------------------------------------------------------- - * compress_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = add_128(add_128(*row0, m), *row1); - *row3 = xor_128(*row3, *row0); - *row3 = rot16_128(*row3); - *row2 = add_128(*row2, *row3); - *row1 = xor_128(*row1, *row2); - *row1 = rot12_128(*row1); -} - -INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = add_128(add_128(*row0, m), *row1); - *row3 = xor_128(*row3, *row0); - *row3 = rot8_128(*row3); - *row2 = add_128(*row2, *row3); - *row1 = xor_128(*row1, *row2); - *row1 = rot7_128(*row1); -} - -// Note the optimization here of leaving row1 as the unrotated row, rather than -// row0. All the message loads below are adjusted to compensate for this. See -// discussion at https://github.com/sneves/blake2-avx2/pull/4 -INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); -} - -INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); -} - -INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags) { - rows[0] = loadu_128((uint8_t *)&cv[0]); - rows[1] = loadu_128((uint8_t *)&cv[4]); - rows[2] = set4(IV[0], IV[1], IV[2], IV[3]); - rows[3] = set4(counter_low(counter), counter_high(counter), - (uint32_t)block_len, (uint32_t)flags); - - __m128i m0 = loadu_128(&block[sizeof(__m128i) * 0]); - __m128i m1 = loadu_128(&block[sizeof(__m128i) * 1]); - __m128i m2 = loadu_128(&block[sizeof(__m128i) * 2]); - __m128i m3 = loadu_128(&block[sizeof(__m128i) * 3]); - - __m128i t0, t1, t2, t3, tt; - - // Round 1. The first round permutes the message words from the original - // input order, into the groups that get mixed in parallel. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 - t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 - t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 2. This round and all following rounds apply a fixed permutation - // to the message words from the round before. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 3 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 4 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 5 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 6 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 7 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); -} - -void blake3_compress_xof_avx512(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu_128(xor_128(rows[0], rows[2]), &out[0]); - storeu_128(xor_128(rows[1], rows[3]), &out[16]); - storeu_128(xor_128(rows[2], loadu_128((uint8_t *)&cv[0])), &out[32]); - storeu_128(xor_128(rows[3], loadu_128((uint8_t *)&cv[4])), &out[48]); -} - -void blake3_compress_in_place_avx512(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu_128(xor_128(rows[0], rows[2]), (uint8_t *)&cv[0]); - storeu_128(xor_128(rows[1], rows[3]), (uint8_t *)&cv[4]); -} - -/* - * ---------------------------------------------------------------------------- - * hash4_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void round_fn4(__m128i v[16], __m128i m[16], size_t r) { - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = add_128(v[0], v[4]); - v[1] = add_128(v[1], v[5]); - v[2] = add_128(v[2], v[6]); - v[3] = add_128(v[3], v[7]); - v[12] = xor_128(v[12], v[0]); - v[13] = xor_128(v[13], v[1]); - v[14] = xor_128(v[14], v[2]); - v[15] = xor_128(v[15], v[3]); - v[12] = rot16_128(v[12]); - v[13] = rot16_128(v[13]); - v[14] = rot16_128(v[14]); - v[15] = rot16_128(v[15]); - v[8] = add_128(v[8], v[12]); - v[9] = add_128(v[9], v[13]); - v[10] = add_128(v[10], v[14]); - v[11] = add_128(v[11], v[15]); - v[4] = xor_128(v[4], v[8]); - v[5] = xor_128(v[5], v[9]); - v[6] = xor_128(v[6], v[10]); - v[7] = xor_128(v[7], v[11]); - v[4] = rot12_128(v[4]); - v[5] = rot12_128(v[5]); - v[6] = rot12_128(v[6]); - v[7] = rot12_128(v[7]); - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = add_128(v[0], v[4]); - v[1] = add_128(v[1], v[5]); - v[2] = add_128(v[2], v[6]); - v[3] = add_128(v[3], v[7]); - v[12] = xor_128(v[12], v[0]); - v[13] = xor_128(v[13], v[1]); - v[14] = xor_128(v[14], v[2]); - v[15] = xor_128(v[15], v[3]); - v[12] = rot8_128(v[12]); - v[13] = rot8_128(v[13]); - v[14] = rot8_128(v[14]); - v[15] = rot8_128(v[15]); - v[8] = add_128(v[8], v[12]); - v[9] = add_128(v[9], v[13]); - v[10] = add_128(v[10], v[14]); - v[11] = add_128(v[11], v[15]); - v[4] = xor_128(v[4], v[8]); - v[5] = xor_128(v[5], v[9]); - v[6] = xor_128(v[6], v[10]); - v[7] = xor_128(v[7], v[11]); - v[4] = rot7_128(v[4]); - v[5] = rot7_128(v[5]); - v[6] = rot7_128(v[6]); - v[7] = rot7_128(v[7]); - - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = add_128(v[0], v[5]); - v[1] = add_128(v[1], v[6]); - v[2] = add_128(v[2], v[7]); - v[3] = add_128(v[3], v[4]); - v[15] = xor_128(v[15], v[0]); - v[12] = xor_128(v[12], v[1]); - v[13] = xor_128(v[13], v[2]); - v[14] = xor_128(v[14], v[3]); - v[15] = rot16_128(v[15]); - v[12] = rot16_128(v[12]); - v[13] = rot16_128(v[13]); - v[14] = rot16_128(v[14]); - v[10] = add_128(v[10], v[15]); - v[11] = add_128(v[11], v[12]); - v[8] = add_128(v[8], v[13]); - v[9] = add_128(v[9], v[14]); - v[5] = xor_128(v[5], v[10]); - v[6] = xor_128(v[6], v[11]); - v[7] = xor_128(v[7], v[8]); - v[4] = xor_128(v[4], v[9]); - v[5] = rot12_128(v[5]); - v[6] = rot12_128(v[6]); - v[7] = rot12_128(v[7]); - v[4] = rot12_128(v[4]); - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = add_128(v[0], v[5]); - v[1] = add_128(v[1], v[6]); - v[2] = add_128(v[2], v[7]); - v[3] = add_128(v[3], v[4]); - v[15] = xor_128(v[15], v[0]); - v[12] = xor_128(v[12], v[1]); - v[13] = xor_128(v[13], v[2]); - v[14] = xor_128(v[14], v[3]); - v[15] = rot8_128(v[15]); - v[12] = rot8_128(v[12]); - v[13] = rot8_128(v[13]); - v[14] = rot8_128(v[14]); - v[10] = add_128(v[10], v[15]); - v[11] = add_128(v[11], v[12]); - v[8] = add_128(v[8], v[13]); - v[9] = add_128(v[9], v[14]); - v[5] = xor_128(v[5], v[10]); - v[6] = xor_128(v[6], v[11]); - v[7] = xor_128(v[7], v[8]); - v[4] = xor_128(v[4], v[9]); - v[5] = rot7_128(v[5]); - v[6] = rot7_128(v[6]); - v[7] = rot7_128(v[7]); - v[4] = rot7_128(v[4]); -} - -INLINE void transpose_vecs_128(__m128i vecs[4]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is - // 22/33. Note that this doesn't split the vector into two lanes, as the - // AVX2 counterparts do. - __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); - __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); - __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); - __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); - - // Interleave 64-bit lanes. - __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); - __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); - __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); - __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); - - vecs[0] = abcd_0; - vecs[1] = abcd_1; - vecs[2] = abcd_2; - vecs[3] = abcd_3; -} - -INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, - size_t block_offset, __m128i out[16]) { - out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(__m128i)]); - out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(__m128i)]); - out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(__m128i)]); - out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(__m128i)]); - out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(__m128i)]); - out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(__m128i)]); - out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(__m128i)]); - out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(__m128i)]); - out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(__m128i)]); - out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(__m128i)]); - out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(__m128i)]); - out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(__m128i)]); - out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(__m128i)]); - out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(__m128i)]); - out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(__m128i)]); - out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(__m128i)]); - for (size_t i = 0; i < 4; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs_128(&out[0]); - transpose_vecs_128(&out[4]); - transpose_vecs_128(&out[8]); - transpose_vecs_128(&out[12]); -} - -INLINE void load_counters4(uint64_t counter, bool increment_counter, - __m128i *out_lo, __m128i *out_hi) { - uint64_t mask = (increment_counter ? ~0 : 0); - __m256i mask_vec = _mm256_set1_epi64x(mask); - __m256i deltas = _mm256_setr_epi64x(0, 1, 2, 3); - deltas = _mm256_and_si256(mask_vec, deltas); - __m256i counters = - _mm256_add_epi64(_mm256_set1_epi64x((int64_t)counter), deltas); - *out_lo = _mm256_cvtepi64_epi32(counters); - *out_hi = _mm256_cvtepi64_epi32(_mm256_srli_epi64(counters, 32)); -} - -static -void blake3_hash4_avx512(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m128i h_vecs[8] = { - set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), - set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), - }; - __m128i counter_low_vec, counter_high_vec; - load_counters4(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m128i block_len_vec = set1_128(BLAKE3_BLOCK_LEN); - __m128i block_flags_vec = set1_128(block_flags); - __m128i msg_vecs[16]; - transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m128i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_128(IV[0]), set1_128(IV[1]), set1_128(IV[2]), set1_128(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn4(v, msg_vecs, 0); - round_fn4(v, msg_vecs, 1); - round_fn4(v, msg_vecs, 2); - round_fn4(v, msg_vecs, 3); - round_fn4(v, msg_vecs, 4); - round_fn4(v, msg_vecs, 5); - round_fn4(v, msg_vecs, 6); - h_vecs[0] = xor_128(v[0], v[8]); - h_vecs[1] = xor_128(v[1], v[9]); - h_vecs[2] = xor_128(v[2], v[10]); - h_vecs[3] = xor_128(v[3], v[11]); - h_vecs[4] = xor_128(v[4], v[12]); - h_vecs[5] = xor_128(v[5], v[13]); - h_vecs[6] = xor_128(v[6], v[14]); - h_vecs[7] = xor_128(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs_128(&h_vecs[0]); - transpose_vecs_128(&h_vecs[4]); - // The first four vecs now contain the first half of each output, and the - // second four vecs contain the second half of each output. - storeu_128(h_vecs[0], &out[0 * sizeof(__m128i)]); - storeu_128(h_vecs[4], &out[1 * sizeof(__m128i)]); - storeu_128(h_vecs[1], &out[2 * sizeof(__m128i)]); - storeu_128(h_vecs[5], &out[3 * sizeof(__m128i)]); - storeu_128(h_vecs[2], &out[4 * sizeof(__m128i)]); - storeu_128(h_vecs[6], &out[5 * sizeof(__m128i)]); - storeu_128(h_vecs[3], &out[6 * sizeof(__m128i)]); - storeu_128(h_vecs[7], &out[7 * sizeof(__m128i)]); -} - -/* - * ---------------------------------------------------------------------------- - * hash8_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void round_fn8(__m256i v[16], __m256i m[16], size_t r) { - v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = add_256(v[0], v[4]); - v[1] = add_256(v[1], v[5]); - v[2] = add_256(v[2], v[6]); - v[3] = add_256(v[3], v[7]); - v[12] = xor_256(v[12], v[0]); - v[13] = xor_256(v[13], v[1]); - v[14] = xor_256(v[14], v[2]); - v[15] = xor_256(v[15], v[3]); - v[12] = rot16_256(v[12]); - v[13] = rot16_256(v[13]); - v[14] = rot16_256(v[14]); - v[15] = rot16_256(v[15]); - v[8] = add_256(v[8], v[12]); - v[9] = add_256(v[9], v[13]); - v[10] = add_256(v[10], v[14]); - v[11] = add_256(v[11], v[15]); - v[4] = xor_256(v[4], v[8]); - v[5] = xor_256(v[5], v[9]); - v[6] = xor_256(v[6], v[10]); - v[7] = xor_256(v[7], v[11]); - v[4] = rot12_256(v[4]); - v[5] = rot12_256(v[5]); - v[6] = rot12_256(v[6]); - v[7] = rot12_256(v[7]); - v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = add_256(v[0], v[4]); - v[1] = add_256(v[1], v[5]); - v[2] = add_256(v[2], v[6]); - v[3] = add_256(v[3], v[7]); - v[12] = xor_256(v[12], v[0]); - v[13] = xor_256(v[13], v[1]); - v[14] = xor_256(v[14], v[2]); - v[15] = xor_256(v[15], v[3]); - v[12] = rot8_256(v[12]); - v[13] = rot8_256(v[13]); - v[14] = rot8_256(v[14]); - v[15] = rot8_256(v[15]); - v[8] = add_256(v[8], v[12]); - v[9] = add_256(v[9], v[13]); - v[10] = add_256(v[10], v[14]); - v[11] = add_256(v[11], v[15]); - v[4] = xor_256(v[4], v[8]); - v[5] = xor_256(v[5], v[9]); - v[6] = xor_256(v[6], v[10]); - v[7] = xor_256(v[7], v[11]); - v[4] = rot7_256(v[4]); - v[5] = rot7_256(v[5]); - v[6] = rot7_256(v[6]); - v[7] = rot7_256(v[7]); - - v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = add_256(v[0], v[5]); - v[1] = add_256(v[1], v[6]); - v[2] = add_256(v[2], v[7]); - v[3] = add_256(v[3], v[4]); - v[15] = xor_256(v[15], v[0]); - v[12] = xor_256(v[12], v[1]); - v[13] = xor_256(v[13], v[2]); - v[14] = xor_256(v[14], v[3]); - v[15] = rot16_256(v[15]); - v[12] = rot16_256(v[12]); - v[13] = rot16_256(v[13]); - v[14] = rot16_256(v[14]); - v[10] = add_256(v[10], v[15]); - v[11] = add_256(v[11], v[12]); - v[8] = add_256(v[8], v[13]); - v[9] = add_256(v[9], v[14]); - v[5] = xor_256(v[5], v[10]); - v[6] = xor_256(v[6], v[11]); - v[7] = xor_256(v[7], v[8]); - v[4] = xor_256(v[4], v[9]); - v[5] = rot12_256(v[5]); - v[6] = rot12_256(v[6]); - v[7] = rot12_256(v[7]); - v[4] = rot12_256(v[4]); - v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = add_256(v[0], v[5]); - v[1] = add_256(v[1], v[6]); - v[2] = add_256(v[2], v[7]); - v[3] = add_256(v[3], v[4]); - v[15] = xor_256(v[15], v[0]); - v[12] = xor_256(v[12], v[1]); - v[13] = xor_256(v[13], v[2]); - v[14] = xor_256(v[14], v[3]); - v[15] = rot8_256(v[15]); - v[12] = rot8_256(v[12]); - v[13] = rot8_256(v[13]); - v[14] = rot8_256(v[14]); - v[10] = add_256(v[10], v[15]); - v[11] = add_256(v[11], v[12]); - v[8] = add_256(v[8], v[13]); - v[9] = add_256(v[9], v[14]); - v[5] = xor_256(v[5], v[10]); - v[6] = xor_256(v[6], v[11]); - v[7] = xor_256(v[7], v[8]); - v[4] = xor_256(v[4], v[9]); - v[5] = rot7_256(v[5]); - v[6] = rot7_256(v[6]); - v[7] = rot7_256(v[7]); - v[4] = rot7_256(v[4]); -} - -INLINE void transpose_vecs_256(__m256i vecs[8]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high - // is 22/33/66/77. - __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]); - __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]); - __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]); - __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]); - __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]); - __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]); - __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]); - __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]); - - // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is - // 11/33. - __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145); - __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145); - __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367); - __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367); - __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145); - __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145); - __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367); - __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367); - - // Interleave 128-bit lanes. - vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20); - vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20); - vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20); - vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20); - vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31); - vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31); - vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31); - vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31); -} - -INLINE void transpose_msg_vecs8(const uint8_t *const *inputs, - size_t block_offset, __m256i out[16]) { - out[0] = loadu_256(&inputs[0][block_offset + 0 * sizeof(__m256i)]); - out[1] = loadu_256(&inputs[1][block_offset + 0 * sizeof(__m256i)]); - out[2] = loadu_256(&inputs[2][block_offset + 0 * sizeof(__m256i)]); - out[3] = loadu_256(&inputs[3][block_offset + 0 * sizeof(__m256i)]); - out[4] = loadu_256(&inputs[4][block_offset + 0 * sizeof(__m256i)]); - out[5] = loadu_256(&inputs[5][block_offset + 0 * sizeof(__m256i)]); - out[6] = loadu_256(&inputs[6][block_offset + 0 * sizeof(__m256i)]); - out[7] = loadu_256(&inputs[7][block_offset + 0 * sizeof(__m256i)]); - out[8] = loadu_256(&inputs[0][block_offset + 1 * sizeof(__m256i)]); - out[9] = loadu_256(&inputs[1][block_offset + 1 * sizeof(__m256i)]); - out[10] = loadu_256(&inputs[2][block_offset + 1 * sizeof(__m256i)]); - out[11] = loadu_256(&inputs[3][block_offset + 1 * sizeof(__m256i)]); - out[12] = loadu_256(&inputs[4][block_offset + 1 * sizeof(__m256i)]); - out[13] = loadu_256(&inputs[5][block_offset + 1 * sizeof(__m256i)]); - out[14] = loadu_256(&inputs[6][block_offset + 1 * sizeof(__m256i)]); - out[15] = loadu_256(&inputs[7][block_offset + 1 * sizeof(__m256i)]); - for (size_t i = 0; i < 8; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs_256(&out[0]); - transpose_vecs_256(&out[8]); -} - -INLINE void load_counters8(uint64_t counter, bool increment_counter, - __m256i *out_lo, __m256i *out_hi) { - uint64_t mask = (increment_counter ? ~0 : 0); - __m512i mask_vec = _mm512_set1_epi64(mask); - __m512i deltas = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); - deltas = _mm512_and_si512(mask_vec, deltas); - __m512i counters = - _mm512_add_epi64(_mm512_set1_epi64((int64_t)counter), deltas); - *out_lo = _mm512_cvtepi64_epi32(counters); - *out_hi = _mm512_cvtepi64_epi32(_mm512_srli_epi64(counters, 32)); -} - -static -void blake3_hash8_avx512(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m256i h_vecs[8] = { - set1_256(key[0]), set1_256(key[1]), set1_256(key[2]), set1_256(key[3]), - set1_256(key[4]), set1_256(key[5]), set1_256(key[6]), set1_256(key[7]), - }; - __m256i counter_low_vec, counter_high_vec; - load_counters8(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m256i block_len_vec = set1_256(BLAKE3_BLOCK_LEN); - __m256i block_flags_vec = set1_256(block_flags); - __m256i msg_vecs[16]; - transpose_msg_vecs8(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m256i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_256(IV[0]), set1_256(IV[1]), set1_256(IV[2]), set1_256(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn8(v, msg_vecs, 0); - round_fn8(v, msg_vecs, 1); - round_fn8(v, msg_vecs, 2); - round_fn8(v, msg_vecs, 3); - round_fn8(v, msg_vecs, 4); - round_fn8(v, msg_vecs, 5); - round_fn8(v, msg_vecs, 6); - h_vecs[0] = xor_256(v[0], v[8]); - h_vecs[1] = xor_256(v[1], v[9]); - h_vecs[2] = xor_256(v[2], v[10]); - h_vecs[3] = xor_256(v[3], v[11]); - h_vecs[4] = xor_256(v[4], v[12]); - h_vecs[5] = xor_256(v[5], v[13]); - h_vecs[6] = xor_256(v[6], v[14]); - h_vecs[7] = xor_256(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs_256(h_vecs); - storeu_256(h_vecs[0], &out[0 * sizeof(__m256i)]); - storeu_256(h_vecs[1], &out[1 * sizeof(__m256i)]); - storeu_256(h_vecs[2], &out[2 * sizeof(__m256i)]); - storeu_256(h_vecs[3], &out[3 * sizeof(__m256i)]); - storeu_256(h_vecs[4], &out[4 * sizeof(__m256i)]); - storeu_256(h_vecs[5], &out[5 * sizeof(__m256i)]); - storeu_256(h_vecs[6], &out[6 * sizeof(__m256i)]); - storeu_256(h_vecs[7], &out[7 * sizeof(__m256i)]); -} - -/* - * ---------------------------------------------------------------------------- - * hash16_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void round_fn16(__m512i v[16], __m512i m[16], size_t r) { - v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = add_512(v[0], v[4]); - v[1] = add_512(v[1], v[5]); - v[2] = add_512(v[2], v[6]); - v[3] = add_512(v[3], v[7]); - v[12] = xor_512(v[12], v[0]); - v[13] = xor_512(v[13], v[1]); - v[14] = xor_512(v[14], v[2]); - v[15] = xor_512(v[15], v[3]); - v[12] = rot16_512(v[12]); - v[13] = rot16_512(v[13]); - v[14] = rot16_512(v[14]); - v[15] = rot16_512(v[15]); - v[8] = add_512(v[8], v[12]); - v[9] = add_512(v[9], v[13]); - v[10] = add_512(v[10], v[14]); - v[11] = add_512(v[11], v[15]); - v[4] = xor_512(v[4], v[8]); - v[5] = xor_512(v[5], v[9]); - v[6] = xor_512(v[6], v[10]); - v[7] = xor_512(v[7], v[11]); - v[4] = rot12_512(v[4]); - v[5] = rot12_512(v[5]); - v[6] = rot12_512(v[6]); - v[7] = rot12_512(v[7]); - v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = add_512(v[0], v[4]); - v[1] = add_512(v[1], v[5]); - v[2] = add_512(v[2], v[6]); - v[3] = add_512(v[3], v[7]); - v[12] = xor_512(v[12], v[0]); - v[13] = xor_512(v[13], v[1]); - v[14] = xor_512(v[14], v[2]); - v[15] = xor_512(v[15], v[3]); - v[12] = rot8_512(v[12]); - v[13] = rot8_512(v[13]); - v[14] = rot8_512(v[14]); - v[15] = rot8_512(v[15]); - v[8] = add_512(v[8], v[12]); - v[9] = add_512(v[9], v[13]); - v[10] = add_512(v[10], v[14]); - v[11] = add_512(v[11], v[15]); - v[4] = xor_512(v[4], v[8]); - v[5] = xor_512(v[5], v[9]); - v[6] = xor_512(v[6], v[10]); - v[7] = xor_512(v[7], v[11]); - v[4] = rot7_512(v[4]); - v[5] = rot7_512(v[5]); - v[6] = rot7_512(v[6]); - v[7] = rot7_512(v[7]); - - v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = add_512(v[0], v[5]); - v[1] = add_512(v[1], v[6]); - v[2] = add_512(v[2], v[7]); - v[3] = add_512(v[3], v[4]); - v[15] = xor_512(v[15], v[0]); - v[12] = xor_512(v[12], v[1]); - v[13] = xor_512(v[13], v[2]); - v[14] = xor_512(v[14], v[3]); - v[15] = rot16_512(v[15]); - v[12] = rot16_512(v[12]); - v[13] = rot16_512(v[13]); - v[14] = rot16_512(v[14]); - v[10] = add_512(v[10], v[15]); - v[11] = add_512(v[11], v[12]); - v[8] = add_512(v[8], v[13]); - v[9] = add_512(v[9], v[14]); - v[5] = xor_512(v[5], v[10]); - v[6] = xor_512(v[6], v[11]); - v[7] = xor_512(v[7], v[8]); - v[4] = xor_512(v[4], v[9]); - v[5] = rot12_512(v[5]); - v[6] = rot12_512(v[6]); - v[7] = rot12_512(v[7]); - v[4] = rot12_512(v[4]); - v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = add_512(v[0], v[5]); - v[1] = add_512(v[1], v[6]); - v[2] = add_512(v[2], v[7]); - v[3] = add_512(v[3], v[4]); - v[15] = xor_512(v[15], v[0]); - v[12] = xor_512(v[12], v[1]); - v[13] = xor_512(v[13], v[2]); - v[14] = xor_512(v[14], v[3]); - v[15] = rot8_512(v[15]); - v[12] = rot8_512(v[12]); - v[13] = rot8_512(v[13]); - v[14] = rot8_512(v[14]); - v[10] = add_512(v[10], v[15]); - v[11] = add_512(v[11], v[12]); - v[8] = add_512(v[8], v[13]); - v[9] = add_512(v[9], v[14]); - v[5] = xor_512(v[5], v[10]); - v[6] = xor_512(v[6], v[11]); - v[7] = xor_512(v[7], v[8]); - v[4] = xor_512(v[4], v[9]); - v[5] = rot7_512(v[5]); - v[6] = rot7_512(v[6]); - v[7] = rot7_512(v[7]); - v[4] = rot7_512(v[4]); -} - -// 0b10001000, or lanes a0/a2/b0/b2 in little-endian order -#define LO_IMM8 0x88 - -INLINE __m512i unpack_lo_128(__m512i a, __m512i b) { - return _mm512_shuffle_i32x4(a, b, LO_IMM8); -} - -// 0b11011101, or lanes a1/a3/b1/b3 in little-endian order -#define HI_IMM8 0xdd - -INLINE __m512i unpack_hi_128(__m512i a, __m512i b) { - return _mm512_shuffle_i32x4(a, b, HI_IMM8); -} - -INLINE void transpose_vecs_512(__m512i vecs[16]) { - // Interleave 32-bit lanes. The _0 unpack is lanes - // 0/0/1/1/4/4/5/5/8/8/9/9/12/12/13/13, and the _2 unpack is lanes - // 2/2/3/3/6/6/7/7/10/10/11/11/14/14/15/15. - __m512i ab_0 = _mm512_unpacklo_epi32(vecs[0], vecs[1]); - __m512i ab_2 = _mm512_unpackhi_epi32(vecs[0], vecs[1]); - __m512i cd_0 = _mm512_unpacklo_epi32(vecs[2], vecs[3]); - __m512i cd_2 = _mm512_unpackhi_epi32(vecs[2], vecs[3]); - __m512i ef_0 = _mm512_unpacklo_epi32(vecs[4], vecs[5]); - __m512i ef_2 = _mm512_unpackhi_epi32(vecs[4], vecs[5]); - __m512i gh_0 = _mm512_unpacklo_epi32(vecs[6], vecs[7]); - __m512i gh_2 = _mm512_unpackhi_epi32(vecs[6], vecs[7]); - __m512i ij_0 = _mm512_unpacklo_epi32(vecs[8], vecs[9]); - __m512i ij_2 = _mm512_unpackhi_epi32(vecs[8], vecs[9]); - __m512i kl_0 = _mm512_unpacklo_epi32(vecs[10], vecs[11]); - __m512i kl_2 = _mm512_unpackhi_epi32(vecs[10], vecs[11]); - __m512i mn_0 = _mm512_unpacklo_epi32(vecs[12], vecs[13]); - __m512i mn_2 = _mm512_unpackhi_epi32(vecs[12], vecs[13]); - __m512i op_0 = _mm512_unpacklo_epi32(vecs[14], vecs[15]); - __m512i op_2 = _mm512_unpackhi_epi32(vecs[14], vecs[15]); - - // Interleave 64-bit lanes. The _0 unpack is lanes - // 0/0/0/0/4/4/4/4/8/8/8/8/12/12/12/12, the _1 unpack is lanes - // 1/1/1/1/5/5/5/5/9/9/9/9/13/13/13/13, the _2 unpack is lanes - // 2/2/2/2/6/6/6/6/10/10/10/10/14/14/14/14, and the _3 unpack is lanes - // 3/3/3/3/7/7/7/7/11/11/11/11/15/15/15/15. - __m512i abcd_0 = _mm512_unpacklo_epi64(ab_0, cd_0); - __m512i abcd_1 = _mm512_unpackhi_epi64(ab_0, cd_0); - __m512i abcd_2 = _mm512_unpacklo_epi64(ab_2, cd_2); - __m512i abcd_3 = _mm512_unpackhi_epi64(ab_2, cd_2); - __m512i efgh_0 = _mm512_unpacklo_epi64(ef_0, gh_0); - __m512i efgh_1 = _mm512_unpackhi_epi64(ef_0, gh_0); - __m512i efgh_2 = _mm512_unpacklo_epi64(ef_2, gh_2); - __m512i efgh_3 = _mm512_unpackhi_epi64(ef_2, gh_2); - __m512i ijkl_0 = _mm512_unpacklo_epi64(ij_0, kl_0); - __m512i ijkl_1 = _mm512_unpackhi_epi64(ij_0, kl_0); - __m512i ijkl_2 = _mm512_unpacklo_epi64(ij_2, kl_2); - __m512i ijkl_3 = _mm512_unpackhi_epi64(ij_2, kl_2); - __m512i mnop_0 = _mm512_unpacklo_epi64(mn_0, op_0); - __m512i mnop_1 = _mm512_unpackhi_epi64(mn_0, op_0); - __m512i mnop_2 = _mm512_unpacklo_epi64(mn_2, op_2); - __m512i mnop_3 = _mm512_unpackhi_epi64(mn_2, op_2); - - // Interleave 128-bit lanes. The _0 unpack is - // 0/0/0/0/8/8/8/8/0/0/0/0/8/8/8/8, the _1 unpack is - // 1/1/1/1/9/9/9/9/1/1/1/1/9/9/9/9, and so on. - __m512i abcdefgh_0 = unpack_lo_128(abcd_0, efgh_0); - __m512i abcdefgh_1 = unpack_lo_128(abcd_1, efgh_1); - __m512i abcdefgh_2 = unpack_lo_128(abcd_2, efgh_2); - __m512i abcdefgh_3 = unpack_lo_128(abcd_3, efgh_3); - __m512i abcdefgh_4 = unpack_hi_128(abcd_0, efgh_0); - __m512i abcdefgh_5 = unpack_hi_128(abcd_1, efgh_1); - __m512i abcdefgh_6 = unpack_hi_128(abcd_2, efgh_2); - __m512i abcdefgh_7 = unpack_hi_128(abcd_3, efgh_3); - __m512i ijklmnop_0 = unpack_lo_128(ijkl_0, mnop_0); - __m512i ijklmnop_1 = unpack_lo_128(ijkl_1, mnop_1); - __m512i ijklmnop_2 = unpack_lo_128(ijkl_2, mnop_2); - __m512i ijklmnop_3 = unpack_lo_128(ijkl_3, mnop_3); - __m512i ijklmnop_4 = unpack_hi_128(ijkl_0, mnop_0); - __m512i ijklmnop_5 = unpack_hi_128(ijkl_1, mnop_1); - __m512i ijklmnop_6 = unpack_hi_128(ijkl_2, mnop_2); - __m512i ijklmnop_7 = unpack_hi_128(ijkl_3, mnop_3); - - // Interleave 128-bit lanes again for the final outputs. - vecs[0] = unpack_lo_128(abcdefgh_0, ijklmnop_0); - vecs[1] = unpack_lo_128(abcdefgh_1, ijklmnop_1); - vecs[2] = unpack_lo_128(abcdefgh_2, ijklmnop_2); - vecs[3] = unpack_lo_128(abcdefgh_3, ijklmnop_3); - vecs[4] = unpack_lo_128(abcdefgh_4, ijklmnop_4); - vecs[5] = unpack_lo_128(abcdefgh_5, ijklmnop_5); - vecs[6] = unpack_lo_128(abcdefgh_6, ijklmnop_6); - vecs[7] = unpack_lo_128(abcdefgh_7, ijklmnop_7); - vecs[8] = unpack_hi_128(abcdefgh_0, ijklmnop_0); - vecs[9] = unpack_hi_128(abcdefgh_1, ijklmnop_1); - vecs[10] = unpack_hi_128(abcdefgh_2, ijklmnop_2); - vecs[11] = unpack_hi_128(abcdefgh_3, ijklmnop_3); - vecs[12] = unpack_hi_128(abcdefgh_4, ijklmnop_4); - vecs[13] = unpack_hi_128(abcdefgh_5, ijklmnop_5); - vecs[14] = unpack_hi_128(abcdefgh_6, ijklmnop_6); - vecs[15] = unpack_hi_128(abcdefgh_7, ijklmnop_7); -} - -INLINE void transpose_msg_vecs16(const uint8_t *const *inputs, - size_t block_offset, __m512i out[16]) { - out[0] = loadu_512(&inputs[0][block_offset]); - out[1] = loadu_512(&inputs[1][block_offset]); - out[2] = loadu_512(&inputs[2][block_offset]); - out[3] = loadu_512(&inputs[3][block_offset]); - out[4] = loadu_512(&inputs[4][block_offset]); - out[5] = loadu_512(&inputs[5][block_offset]); - out[6] = loadu_512(&inputs[6][block_offset]); - out[7] = loadu_512(&inputs[7][block_offset]); - out[8] = loadu_512(&inputs[8][block_offset]); - out[9] = loadu_512(&inputs[9][block_offset]); - out[10] = loadu_512(&inputs[10][block_offset]); - out[11] = loadu_512(&inputs[11][block_offset]); - out[12] = loadu_512(&inputs[12][block_offset]); - out[13] = loadu_512(&inputs[13][block_offset]); - out[14] = loadu_512(&inputs[14][block_offset]); - out[15] = loadu_512(&inputs[15][block_offset]); - for (size_t i = 0; i < 16; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs_512(out); -} - -INLINE void load_counters16(uint64_t counter, bool increment_counter, - __m512i *out_lo, __m512i *out_hi) { - const __m512i mask = _mm512_set1_epi32(-(int32_t)increment_counter); - const __m512i deltas = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); - const __m512i masked_deltas = _mm512_and_si512(deltas, mask); - const __m512i low_words = _mm512_add_epi32( - _mm512_set1_epi32((int32_t)counter), - masked_deltas); - // The carry bit is 1 if the high bit of the word was 1 before addition and is - // 0 after. - // NOTE: It would be a bit more natural to use _mm512_cmp_epu32_mask to - // compute the carry bits here, and originally we did, but that intrinsic is - // broken under GCC 5.4. See https://github.com/BLAKE3-team/BLAKE3/issues/271. - const __m512i carries = _mm512_srli_epi32( - _mm512_andnot_si512( - low_words, // 0 after (gets inverted by andnot) - _mm512_set1_epi32((int32_t)counter)), // and 1 before - 31); - const __m512i high_words = _mm512_add_epi32( - _mm512_set1_epi32((int32_t)(counter >> 32)), - carries); - *out_lo = low_words; - *out_hi = high_words; -} - -static -void blake3_hash16_avx512(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, - uint8_t *out) { - __m512i h_vecs[8] = { - set1_512(key[0]), set1_512(key[1]), set1_512(key[2]), set1_512(key[3]), - set1_512(key[4]), set1_512(key[5]), set1_512(key[6]), set1_512(key[7]), - }; - __m512i counter_low_vec, counter_high_vec; - load_counters16(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m512i block_len_vec = set1_512(BLAKE3_BLOCK_LEN); - __m512i block_flags_vec = set1_512(block_flags); - __m512i msg_vecs[16]; - transpose_msg_vecs16(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m512i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_512(IV[0]), set1_512(IV[1]), set1_512(IV[2]), set1_512(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn16(v, msg_vecs, 0); - round_fn16(v, msg_vecs, 1); - round_fn16(v, msg_vecs, 2); - round_fn16(v, msg_vecs, 3); - round_fn16(v, msg_vecs, 4); - round_fn16(v, msg_vecs, 5); - round_fn16(v, msg_vecs, 6); - h_vecs[0] = xor_512(v[0], v[8]); - h_vecs[1] = xor_512(v[1], v[9]); - h_vecs[2] = xor_512(v[2], v[10]); - h_vecs[3] = xor_512(v[3], v[11]); - h_vecs[4] = xor_512(v[4], v[12]); - h_vecs[5] = xor_512(v[5], v[13]); - h_vecs[6] = xor_512(v[6], v[14]); - h_vecs[7] = xor_512(v[7], v[15]); - - block_flags = flags; - } - - // transpose_vecs_512 operates on a 16x16 matrix of words, but we only have 8 - // state vectors. Pad the matrix with zeros. After transposition, store the - // lower half of each vector. - __m512i padded[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_512(0), set1_512(0), set1_512(0), set1_512(0), - set1_512(0), set1_512(0), set1_512(0), set1_512(0), - }; - transpose_vecs_512(padded); - _mm256_mask_storeu_epi32(&out[0 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[0])); - _mm256_mask_storeu_epi32(&out[1 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[1])); - _mm256_mask_storeu_epi32(&out[2 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[2])); - _mm256_mask_storeu_epi32(&out[3 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[3])); - _mm256_mask_storeu_epi32(&out[4 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[4])); - _mm256_mask_storeu_epi32(&out[5 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[5])); - _mm256_mask_storeu_epi32(&out[6 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[6])); - _mm256_mask_storeu_epi32(&out[7 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[7])); - _mm256_mask_storeu_epi32(&out[8 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[8])); - _mm256_mask_storeu_epi32(&out[9 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[9])); - _mm256_mask_storeu_epi32(&out[10 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[10])); - _mm256_mask_storeu_epi32(&out[11 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[11])); - _mm256_mask_storeu_epi32(&out[12 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[12])); - _mm256_mask_storeu_epi32(&out[13 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[13])); - _mm256_mask_storeu_epi32(&out[14 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[14])); - _mm256_mask_storeu_epi32(&out[15 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[15])); -} - -/* - * ---------------------------------------------------------------------------- - * hash_many_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void hash_one_avx512(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - blake3_compress_in_place_avx512(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - memcpy(out, cv, BLAKE3_OUT_LEN); -} - -void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= 16) { - blake3_hash16_avx512(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += 16; - } - inputs += 16; - num_inputs -= 16; - out = &out[16 * BLAKE3_OUT_LEN]; - } - while (num_inputs >= 8) { - blake3_hash8_avx512(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += 8; - } - inputs += 8; - num_inputs -= 8; - out = &out[8 * BLAKE3_OUT_LEN]; - } - while (num_inputs >= 4) { - blake3_hash4_avx512(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += 4; - } - inputs += 4; - num_inputs -= 4; - out = &out[4 * BLAKE3_OUT_LEN]; - } - while (num_inputs > 0) { - hash_one_avx512(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/third_party/blake3/c/blake3_impl.h b/src/third_party/blake3/c/blake3_impl.h deleted file mode 100644 index beab5cf53..000000000 --- a/src/third_party/blake3/c/blake3_impl.h +++ /dev/null @@ -1,285 +0,0 @@ -#ifndef BLAKE3_IMPL_H -#define BLAKE3_IMPL_H - -#include -#include -#include -#include -#include - -#include "blake3.h" - -// internal flags -enum blake3_flags { - CHUNK_START = 1 << 0, - CHUNK_END = 1 << 1, - PARENT = 1 << 2, - ROOT = 1 << 3, - KEYED_HASH = 1 << 4, - DERIVE_KEY_CONTEXT = 1 << 5, - DERIVE_KEY_MATERIAL = 1 << 6, -}; - -// This C implementation tries to support recent versions of GCC, Clang, and -// MSVC. -#if defined(_MSC_VER) -#define INLINE static __forceinline -#else -#define INLINE static inline __attribute__((always_inline)) -#endif - -#if defined(__x86_64__) || defined(_M_X64) -#define IS_X86 -#define IS_X86_64 -#endif - -#if defined(__i386__) || defined(_M_IX86) -#define IS_X86 -#define IS_X86_32 -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#define IS_AARCH64 -#endif - -#if defined(IS_X86) -#if defined(_MSC_VER) -#include -#endif -#endif - -#if !defined(BLAKE3_USE_NEON) - // If BLAKE3_USE_NEON not manually set, autodetect based on AArch64ness - #if defined(IS_AARCH64) - #if defined(__ARM_BIG_ENDIAN) - #define BLAKE3_USE_NEON 0 - #else - #define BLAKE3_USE_NEON 1 - #endif - #else - #define BLAKE3_USE_NEON 0 - #endif -#endif - -#if defined(IS_X86) -#define MAX_SIMD_DEGREE 16 -#elif BLAKE3_USE_NEON == 1 -#define MAX_SIMD_DEGREE 4 -#else -#define MAX_SIMD_DEGREE 1 -#endif - -// There are some places where we want a static size that's equal to the -// MAX_SIMD_DEGREE, but also at least 2. -#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2) - -static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, - 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL, - 0x1F83D9ABUL, 0x5BE0CD19UL}; - -static const uint8_t MSG_SCHEDULE[7][16] = { - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, - {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, - {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, - {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, - {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, - {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13}, -}; - -/* Find index of the highest set bit */ -/* x is assumed to be nonzero. */ -static unsigned int highest_one(uint64_t x) { -#if defined(__GNUC__) || defined(__clang__) - return 63 ^ (unsigned int)__builtin_clzll(x); -#elif defined(_MSC_VER) && defined(IS_X86_64) - unsigned long index; - _BitScanReverse64(&index, x); - return index; -#elif defined(_MSC_VER) && defined(IS_X86_32) - if(x >> 32) { - unsigned long index; - _BitScanReverse(&index, (unsigned long)(x >> 32)); - return 32 + index; - } else { - unsigned long index; - _BitScanReverse(&index, (unsigned long)x); - return index; - } -#else - unsigned int c = 0; - if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } - if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } - if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } - if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } - if(x & 0x000000000000000cULL) { x >>= 2; c += 2; } - if(x & 0x0000000000000002ULL) { c += 1; } - return c; -#endif -} - -// Count the number of 1 bits. -INLINE unsigned int popcnt(uint64_t x) { -#if defined(__GNUC__) || defined(__clang__) - return (unsigned int)__builtin_popcountll(x); -#else - unsigned int count = 0; - while (x != 0) { - count += 1; - x &= x - 1; - } - return count; -#endif -} - -// Largest power of two less than or equal to x. As a special case, returns 1 -// when x is 0. -INLINE uint64_t round_down_to_power_of_2(uint64_t x) { - return 1ULL << highest_one(x | 1); -} - -INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; } - -INLINE uint32_t counter_high(uint64_t counter) { - return (uint32_t)(counter >> 32); -} - -INLINE uint32_t load32(const void *src) { - const uint8_t *p = (const uint8_t *)src; - return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | - ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24); -} - -INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], - uint32_t key_words[8]) { - key_words[0] = load32(&key[0 * 4]); - key_words[1] = load32(&key[1 * 4]); - key_words[2] = load32(&key[2 * 4]); - key_words[3] = load32(&key[3 * 4]); - key_words[4] = load32(&key[4 * 4]); - key_words[5] = load32(&key[5 * 4]); - key_words[6] = load32(&key[6 * 4]); - key_words[7] = load32(&key[7 * 4]); -} - -INLINE void store32(void *dst, uint32_t w) { - uint8_t *p = (uint8_t *)dst; - p[0] = (uint8_t)(w >> 0); - p[1] = (uint8_t)(w >> 8); - p[2] = (uint8_t)(w >> 16); - p[3] = (uint8_t)(w >> 24); -} - -INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) { - store32(&bytes_out[0 * 4], cv_words[0]); - store32(&bytes_out[1 * 4], cv_words[1]); - store32(&bytes_out[2 * 4], cv_words[2]); - store32(&bytes_out[3 * 4], cv_words[3]); - store32(&bytes_out[4 * 4], cv_words[4]); - store32(&bytes_out[5 * 4], cv_words[5]); - store32(&bytes_out[6 * 4], cv_words[6]); - store32(&bytes_out[7 * 4], cv_words[7]); -} - -void blake3_compress_in_place(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); - -void blake3_compress_xof(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags, - uint8_t out[64]); - -void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out); - -size_t blake3_simd_degree(void); - - -// Declarations for implementation-specific functions. -void blake3_compress_in_place_portable(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); - -void blake3_compress_xof_portable(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]); - -void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); - -#if defined(IS_X86) -#if !defined(BLAKE3_NO_SSE2) -void blake3_compress_in_place_sse2(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); -void blake3_compress_xof_sse2(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]); -void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif -#if !defined(BLAKE3_NO_SSE41) -void blake3_compress_in_place_sse41(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); -void blake3_compress_xof_sse41(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]); -void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif -#if !defined(BLAKE3_NO_AVX2) -void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif -#if !defined(BLAKE3_NO_AVX512) -void blake3_compress_in_place_avx512(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); - -void blake3_compress_xof_avx512(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]); - -void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif -#endif - -#if BLAKE3_USE_NEON == 1 -void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif - - -#endif /* BLAKE3_IMPL_H */ diff --git a/src/third_party/blake3/c/blake3_neon.c b/src/third_party/blake3/c/blake3_neon.c deleted file mode 100644 index 8a818fc78..000000000 --- a/src/third_party/blake3/c/blake3_neon.c +++ /dev/null @@ -1,368 +0,0 @@ -#include "blake3_impl.h" - -#include - -#ifdef __ARM_BIG_ENDIAN -#error "This implementation only supports little-endian ARM." -// It might be that all we need for big-endian support here is to get the loads -// and stores right, but step zero would be finding a way to test it in CI. -#endif - -INLINE uint32x4_t loadu_128(const uint8_t src[16]) { - // vld1q_u32 has alignment requirements. Don't use it. - uint32x4_t x; - memcpy(&x, src, 16); - return x; -} - -INLINE void storeu_128(uint32x4_t src, uint8_t dest[16]) { - // vst1q_u32 has alignment requirements. Don't use it. - memcpy(dest, &src, 16); -} - -INLINE uint32x4_t add_128(uint32x4_t a, uint32x4_t b) { - return vaddq_u32(a, b); -} - -INLINE uint32x4_t xor_128(uint32x4_t a, uint32x4_t b) { - return veorq_u32(a, b); -} - -INLINE uint32x4_t set1_128(uint32_t x) { return vld1q_dup_u32(&x); } - -INLINE uint32x4_t set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - uint32_t array[4] = {a, b, c, d}; - return vld1q_u32(array); -} - -INLINE uint32x4_t rot16_128(uint32x4_t x) { - // The straightfoward implementation would be two shifts and an or, but that's - // slower on microarchitectures we've tested. See - // https://github.com/BLAKE3-team/BLAKE3/pull/319. - // return vorrq_u32(vshrq_n_u32(x, 16), vshlq_n_u32(x, 32 - 16)); - return vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x))); -} - -INLINE uint32x4_t rot12_128(uint32x4_t x) { - // See comment in rot16_128. - // return vorrq_u32(vshrq_n_u32(x, 12), vshlq_n_u32(x, 32 - 12)); - return vsriq_n_u32(vshlq_n_u32(x, 32-12), x, 12); -} - -INLINE uint32x4_t rot8_128(uint32x4_t x) { - // See comment in rot16_128. - // return vorrq_u32(vshrq_n_u32(x, 8), vshlq_n_u32(x, 32 - 8)); -#if defined(__clang__) - return vreinterpretq_u32_u8(__builtin_shufflevector(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12)); -#elif __GNUC__ * 10000 + __GNUC_MINOR__ * 100 >=40700 - static const uint8x16_t r8 = {1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12}; - return vreinterpretq_u32_u8(__builtin_shuffle(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), r8)); -#else - return vsriq_n_u32(vshlq_n_u32(x, 32-8), x, 8); -#endif -} - -INLINE uint32x4_t rot7_128(uint32x4_t x) { - // See comment in rot16_128. - // return vorrq_u32(vshrq_n_u32(x, 7), vshlq_n_u32(x, 32 - 7)); - return vsriq_n_u32(vshlq_n_u32(x, 32-7), x, 7); -} - -// TODO: compress_neon - -// TODO: hash2_neon - -/* - * ---------------------------------------------------------------------------- - * hash4_neon - * ---------------------------------------------------------------------------- - */ - -INLINE void round_fn4(uint32x4_t v[16], uint32x4_t m[16], size_t r) { - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = add_128(v[0], v[4]); - v[1] = add_128(v[1], v[5]); - v[2] = add_128(v[2], v[6]); - v[3] = add_128(v[3], v[7]); - v[12] = xor_128(v[12], v[0]); - v[13] = xor_128(v[13], v[1]); - v[14] = xor_128(v[14], v[2]); - v[15] = xor_128(v[15], v[3]); - v[12] = rot16_128(v[12]); - v[13] = rot16_128(v[13]); - v[14] = rot16_128(v[14]); - v[15] = rot16_128(v[15]); - v[8] = add_128(v[8], v[12]); - v[9] = add_128(v[9], v[13]); - v[10] = add_128(v[10], v[14]); - v[11] = add_128(v[11], v[15]); - v[4] = xor_128(v[4], v[8]); - v[5] = xor_128(v[5], v[9]); - v[6] = xor_128(v[6], v[10]); - v[7] = xor_128(v[7], v[11]); - v[4] = rot12_128(v[4]); - v[5] = rot12_128(v[5]); - v[6] = rot12_128(v[6]); - v[7] = rot12_128(v[7]); - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = add_128(v[0], v[4]); - v[1] = add_128(v[1], v[5]); - v[2] = add_128(v[2], v[6]); - v[3] = add_128(v[3], v[7]); - v[12] = xor_128(v[12], v[0]); - v[13] = xor_128(v[13], v[1]); - v[14] = xor_128(v[14], v[2]); - v[15] = xor_128(v[15], v[3]); - v[12] = rot8_128(v[12]); - v[13] = rot8_128(v[13]); - v[14] = rot8_128(v[14]); - v[15] = rot8_128(v[15]); - v[8] = add_128(v[8], v[12]); - v[9] = add_128(v[9], v[13]); - v[10] = add_128(v[10], v[14]); - v[11] = add_128(v[11], v[15]); - v[4] = xor_128(v[4], v[8]); - v[5] = xor_128(v[5], v[9]); - v[6] = xor_128(v[6], v[10]); - v[7] = xor_128(v[7], v[11]); - v[4] = rot7_128(v[4]); - v[5] = rot7_128(v[5]); - v[6] = rot7_128(v[6]); - v[7] = rot7_128(v[7]); - - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = add_128(v[0], v[5]); - v[1] = add_128(v[1], v[6]); - v[2] = add_128(v[2], v[7]); - v[3] = add_128(v[3], v[4]); - v[15] = xor_128(v[15], v[0]); - v[12] = xor_128(v[12], v[1]); - v[13] = xor_128(v[13], v[2]); - v[14] = xor_128(v[14], v[3]); - v[15] = rot16_128(v[15]); - v[12] = rot16_128(v[12]); - v[13] = rot16_128(v[13]); - v[14] = rot16_128(v[14]); - v[10] = add_128(v[10], v[15]); - v[11] = add_128(v[11], v[12]); - v[8] = add_128(v[8], v[13]); - v[9] = add_128(v[9], v[14]); - v[5] = xor_128(v[5], v[10]); - v[6] = xor_128(v[6], v[11]); - v[7] = xor_128(v[7], v[8]); - v[4] = xor_128(v[4], v[9]); - v[5] = rot12_128(v[5]); - v[6] = rot12_128(v[6]); - v[7] = rot12_128(v[7]); - v[4] = rot12_128(v[4]); - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = add_128(v[0], v[5]); - v[1] = add_128(v[1], v[6]); - v[2] = add_128(v[2], v[7]); - v[3] = add_128(v[3], v[4]); - v[15] = xor_128(v[15], v[0]); - v[12] = xor_128(v[12], v[1]); - v[13] = xor_128(v[13], v[2]); - v[14] = xor_128(v[14], v[3]); - v[15] = rot8_128(v[15]); - v[12] = rot8_128(v[12]); - v[13] = rot8_128(v[13]); - v[14] = rot8_128(v[14]); - v[10] = add_128(v[10], v[15]); - v[11] = add_128(v[11], v[12]); - v[8] = add_128(v[8], v[13]); - v[9] = add_128(v[9], v[14]); - v[5] = xor_128(v[5], v[10]); - v[6] = xor_128(v[6], v[11]); - v[7] = xor_128(v[7], v[8]); - v[4] = xor_128(v[4], v[9]); - v[5] = rot7_128(v[5]); - v[6] = rot7_128(v[6]); - v[7] = rot7_128(v[7]); - v[4] = rot7_128(v[4]); -} - -INLINE void transpose_vecs_128(uint32x4_t vecs[4]) { - // Individually transpose the four 2x2 sub-matrices in each corner. - uint32x4x2_t rows01 = vtrnq_u32(vecs[0], vecs[1]); - uint32x4x2_t rows23 = vtrnq_u32(vecs[2], vecs[3]); - - // Swap the top-right and bottom-left 2x2s (which just got transposed). - vecs[0] = - vcombine_u32(vget_low_u32(rows01.val[0]), vget_low_u32(rows23.val[0])); - vecs[1] = - vcombine_u32(vget_low_u32(rows01.val[1]), vget_low_u32(rows23.val[1])); - vecs[2] = - vcombine_u32(vget_high_u32(rows01.val[0]), vget_high_u32(rows23.val[0])); - vecs[3] = - vcombine_u32(vget_high_u32(rows01.val[1]), vget_high_u32(rows23.val[1])); -} - -INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, - size_t block_offset, uint32x4_t out[16]) { - out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(uint32x4_t)]); - out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(uint32x4_t)]); - out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(uint32x4_t)]); - out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(uint32x4_t)]); - out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(uint32x4_t)]); - out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(uint32x4_t)]); - out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(uint32x4_t)]); - out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(uint32x4_t)]); - out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(uint32x4_t)]); - out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(uint32x4_t)]); - out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(uint32x4_t)]); - out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(uint32x4_t)]); - out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(uint32x4_t)]); - out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(uint32x4_t)]); - out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(uint32x4_t)]); - out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(uint32x4_t)]); - transpose_vecs_128(&out[0]); - transpose_vecs_128(&out[4]); - transpose_vecs_128(&out[8]); - transpose_vecs_128(&out[12]); -} - -INLINE void load_counters4(uint64_t counter, bool increment_counter, - uint32x4_t *out_low, uint32x4_t *out_high) { - uint64_t mask = (increment_counter ? ~0 : 0); - *out_low = set4( - counter_low(counter + (mask & 0)), counter_low(counter + (mask & 1)), - counter_low(counter + (mask & 2)), counter_low(counter + (mask & 3))); - *out_high = set4( - counter_high(counter + (mask & 0)), counter_high(counter + (mask & 1)), - counter_high(counter + (mask & 2)), counter_high(counter + (mask & 3))); -} - -void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - uint32x4_t h_vecs[8] = { - set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), - set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), - }; - uint32x4_t counter_low_vec, counter_high_vec; - load_counters4(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - uint32x4_t block_len_vec = set1_128(BLAKE3_BLOCK_LEN); - uint32x4_t block_flags_vec = set1_128(block_flags); - uint32x4_t msg_vecs[16]; - transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - uint32x4_t v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_128(IV[0]), set1_128(IV[1]), set1_128(IV[2]), set1_128(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn4(v, msg_vecs, 0); - round_fn4(v, msg_vecs, 1); - round_fn4(v, msg_vecs, 2); - round_fn4(v, msg_vecs, 3); - round_fn4(v, msg_vecs, 4); - round_fn4(v, msg_vecs, 5); - round_fn4(v, msg_vecs, 6); - h_vecs[0] = xor_128(v[0], v[8]); - h_vecs[1] = xor_128(v[1], v[9]); - h_vecs[2] = xor_128(v[2], v[10]); - h_vecs[3] = xor_128(v[3], v[11]); - h_vecs[4] = xor_128(v[4], v[12]); - h_vecs[5] = xor_128(v[5], v[13]); - h_vecs[6] = xor_128(v[6], v[14]); - h_vecs[7] = xor_128(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs_128(&h_vecs[0]); - transpose_vecs_128(&h_vecs[4]); - // The first four vecs now contain the first half of each output, and the - // second four vecs contain the second half of each output. - storeu_128(h_vecs[0], &out[0 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[4], &out[1 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[1], &out[2 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[5], &out[3 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[2], &out[4 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[6], &out[5 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[3], &out[6 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[7], &out[7 * sizeof(uint32x4_t)]); -} - -/* - * ---------------------------------------------------------------------------- - * hash_many_neon - * ---------------------------------------------------------------------------- - */ - -void blake3_compress_in_place_portable(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); - -INLINE void hash_one_neon(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, uint8_t flags_end, - uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - // TODO: Implement compress_neon. However note that according to - // https://github.com/BLAKE2/BLAKE2/commit/7965d3e6e1b4193438b8d3a656787587d2579227, - // compress_neon might not be any faster than compress_portable. - blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - memcpy(out, cv, BLAKE3_OUT_LEN); -} - -void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= 4) { - blake3_hash4_neon(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += 4; - } - inputs += 4; - num_inputs -= 4; - out = &out[4 * BLAKE3_OUT_LEN]; - } - while (num_inputs > 0) { - hash_one_neon(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/third_party/blake3/c/blake3_portable.c b/src/third_party/blake3/c/blake3_portable.c deleted file mode 100644 index 062dd1b47..000000000 --- a/src/third_party/blake3/c/blake3_portable.c +++ /dev/null @@ -1,160 +0,0 @@ -#include "blake3_impl.h" -#include - -INLINE uint32_t rotr32(uint32_t w, uint32_t c) { - return (w >> c) | (w << (32 - c)); -} - -INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d, - uint32_t x, uint32_t y) { - state[a] = state[a] + state[b] + x; - state[d] = rotr32(state[d] ^ state[a], 16); - state[c] = state[c] + state[d]; - state[b] = rotr32(state[b] ^ state[c], 12); - state[a] = state[a] + state[b] + y; - state[d] = rotr32(state[d] ^ state[a], 8); - state[c] = state[c] + state[d]; - state[b] = rotr32(state[b] ^ state[c], 7); -} - -INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) { - // Select the message schedule based on the round. - const uint8_t *schedule = MSG_SCHEDULE[round]; - - // Mix the columns. - g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); - g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); - g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); - g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); - - // Mix the rows. - g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); - g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); - g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); - g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); -} - -INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags) { - uint32_t block_words[16]; - block_words[0] = load32(block + 4 * 0); - block_words[1] = load32(block + 4 * 1); - block_words[2] = load32(block + 4 * 2); - block_words[3] = load32(block + 4 * 3); - block_words[4] = load32(block + 4 * 4); - block_words[5] = load32(block + 4 * 5); - block_words[6] = load32(block + 4 * 6); - block_words[7] = load32(block + 4 * 7); - block_words[8] = load32(block + 4 * 8); - block_words[9] = load32(block + 4 * 9); - block_words[10] = load32(block + 4 * 10); - block_words[11] = load32(block + 4 * 11); - block_words[12] = load32(block + 4 * 12); - block_words[13] = load32(block + 4 * 13); - block_words[14] = load32(block + 4 * 14); - block_words[15] = load32(block + 4 * 15); - - state[0] = cv[0]; - state[1] = cv[1]; - state[2] = cv[2]; - state[3] = cv[3]; - state[4] = cv[4]; - state[5] = cv[5]; - state[6] = cv[6]; - state[7] = cv[7]; - state[8] = IV[0]; - state[9] = IV[1]; - state[10] = IV[2]; - state[11] = IV[3]; - state[12] = counter_low(counter); - state[13] = counter_high(counter); - state[14] = (uint32_t)block_len; - state[15] = (uint32_t)flags; - - round_fn(state, &block_words[0], 0); - round_fn(state, &block_words[0], 1); - round_fn(state, &block_words[0], 2); - round_fn(state, &block_words[0], 3); - round_fn(state, &block_words[0], 4); - round_fn(state, &block_words[0], 5); - round_fn(state, &block_words[0], 6); -} - -void blake3_compress_in_place_portable(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - uint32_t state[16]; - compress_pre(state, cv, block, block_len, counter, flags); - cv[0] = state[0] ^ state[8]; - cv[1] = state[1] ^ state[9]; - cv[2] = state[2] ^ state[10]; - cv[3] = state[3] ^ state[11]; - cv[4] = state[4] ^ state[12]; - cv[5] = state[5] ^ state[13]; - cv[6] = state[6] ^ state[14]; - cv[7] = state[7] ^ state[15]; -} - -void blake3_compress_xof_portable(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]) { - uint32_t state[16]; - compress_pre(state, cv, block, block_len, counter, flags); - - store32(&out[0 * 4], state[0] ^ state[8]); - store32(&out[1 * 4], state[1] ^ state[9]); - store32(&out[2 * 4], state[2] ^ state[10]); - store32(&out[3 * 4], state[3] ^ state[11]); - store32(&out[4 * 4], state[4] ^ state[12]); - store32(&out[5 * 4], state[5] ^ state[13]); - store32(&out[6 * 4], state[6] ^ state[14]); - store32(&out[7 * 4], state[7] ^ state[15]); - store32(&out[8 * 4], state[8] ^ cv[0]); - store32(&out[9 * 4], state[9] ^ cv[1]); - store32(&out[10 * 4], state[10] ^ cv[2]); - store32(&out[11 * 4], state[11] ^ cv[3]); - store32(&out[12 * 4], state[12] ^ cv[4]); - store32(&out[13 * 4], state[13] ^ cv[5]); - store32(&out[14 * 4], state[14] ^ cv[6]); - store32(&out[15 * 4], state[15] ^ cv[7]); -} - -INLINE void hash_one_portable(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - store_cv_words(out, cv); -} - -void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs > 0) { - hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/third_party/blake3/c/blake3_sse2.c b/src/third_party/blake3/c/blake3_sse2.c deleted file mode 100644 index 86bb17aba..000000000 --- a/src/third_party/blake3/c/blake3_sse2.c +++ /dev/null @@ -1,566 +0,0 @@ -#include "blake3_impl.h" - -#include - -#define DEGREE 4 - -#define _mm_shuffle_ps2(a, b, c) \ - (_mm_castps_si128( \ - _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) - -INLINE __m128i loadu(const uint8_t src[16]) { - return _mm_loadu_si128((const __m128i *)src); -} - -INLINE void storeu(__m128i src, uint8_t dest[16]) { - _mm_storeu_si128((__m128i *)dest, src); -} - -INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } - -// Note that clang-format doesn't like the name "xor" for some reason. -INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } - -INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); } - -INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); -} - -INLINE __m128i rot16(__m128i x) { - return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, 0xB1), 0xB1); -} - -INLINE __m128i rot12(__m128i x) { - return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12)); -} - -INLINE __m128i rot8(__m128i x) { - return xorv(_mm_srli_epi32(x, 8), _mm_slli_epi32(x, 32 - 8)); -} - -INLINE __m128i rot7(__m128i x) { - return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7)); -} - -INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = addv(addv(*row0, m), *row1); - *row3 = xorv(*row3, *row0); - *row3 = rot16(*row3); - *row2 = addv(*row2, *row3); - *row1 = xorv(*row1, *row2); - *row1 = rot12(*row1); -} - -INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = addv(addv(*row0, m), *row1); - *row3 = xorv(*row3, *row0); - *row3 = rot8(*row3); - *row2 = addv(*row2, *row3); - *row1 = xorv(*row1, *row2); - *row1 = rot7(*row1); -} - -// Note the optimization here of leaving row1 as the unrotated row, rather than -// row0. All the message loads below are adjusted to compensate for this. See -// discussion at https://github.com/sneves/blake2-avx2/pull/4 -INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); -} - -INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); -} - -INLINE __m128i blend_epi16(__m128i a, __m128i b, const int16_t imm8) { - const __m128i bits = _mm_set_epi16(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01); - __m128i mask = _mm_set1_epi16(imm8); - mask = _mm_and_si128(mask, bits); - mask = _mm_cmpeq_epi16(mask, bits); - return _mm_or_si128(_mm_and_si128(mask, b), _mm_andnot_si128(mask, a)); -} - -INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags) { - rows[0] = loadu((uint8_t *)&cv[0]); - rows[1] = loadu((uint8_t *)&cv[4]); - rows[2] = set4(IV[0], IV[1], IV[2], IV[3]); - rows[3] = set4(counter_low(counter), counter_high(counter), - (uint32_t)block_len, (uint32_t)flags); - - __m128i m0 = loadu(&block[sizeof(__m128i) * 0]); - __m128i m1 = loadu(&block[sizeof(__m128i) * 1]); - __m128i m2 = loadu(&block[sizeof(__m128i) * 2]); - __m128i m3 = loadu(&block[sizeof(__m128i) * 3]); - - __m128i t0, t1, t2, t3, tt; - - // Round 1. The first round permutes the message words from the original - // input order, into the groups that get mixed in parallel. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 - t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 - t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 2. This round and all following rounds apply a fixed permutation - // to the message words from the round before. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 3 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 4 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 5 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 6 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 7 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); -} - -void blake3_compress_in_place_sse2(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]); - storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]); -} - -void blake3_compress_xof_sse2(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu(xorv(rows[0], rows[2]), &out[0]); - storeu(xorv(rows[1], rows[3]), &out[16]); - storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]); - storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]); -} - -INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) { - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[15] = rot16(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot12(v[4]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[15] = rot8(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot7(v[4]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot16(v[15]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[4] = rot12(v[4]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot8(v[15]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - v[4] = rot7(v[4]); -} - -INLINE void transpose_vecs(__m128i vecs[DEGREE]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is - // 22/33. Note that this doesn't split the vector into two lanes, as the - // AVX2 counterparts do. - __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); - __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); - __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); - __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); - - // Interleave 64-bit lanes. - __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); - __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); - __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); - __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); - - vecs[0] = abcd_0; - vecs[1] = abcd_1; - vecs[2] = abcd_2; - vecs[3] = abcd_3; -} - -INLINE void transpose_msg_vecs(const uint8_t *const *inputs, - size_t block_offset, __m128i out[16]) { - out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]); - out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]); - out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]); - out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]); - out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]); - out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]); - out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]); - out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]); - out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]); - out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]); - out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]); - out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]); - out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]); - out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]); - out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); - out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); - for (size_t i = 0; i < 4; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs(&out[0]); - transpose_vecs(&out[4]); - transpose_vecs(&out[8]); - transpose_vecs(&out[12]); -} - -INLINE void load_counters(uint64_t counter, bool increment_counter, - __m128i *out_lo, __m128i *out_hi) { - const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); - const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); - const __m128i add1 = _mm_and_si128(mask, add0); - __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); - __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), - _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); - __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); - *out_lo = l; - *out_hi = h; -} - -static -void blake3_hash4_sse2(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m128i h_vecs[8] = { - set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), - set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), - }; - __m128i counter_low_vec, counter_high_vec; - load_counters(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN); - __m128i block_flags_vec = set1(block_flags); - __m128i msg_vecs[16]; - transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m128i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn(v, msg_vecs, 0); - round_fn(v, msg_vecs, 1); - round_fn(v, msg_vecs, 2); - round_fn(v, msg_vecs, 3); - round_fn(v, msg_vecs, 4); - round_fn(v, msg_vecs, 5); - round_fn(v, msg_vecs, 6); - h_vecs[0] = xorv(v[0], v[8]); - h_vecs[1] = xorv(v[1], v[9]); - h_vecs[2] = xorv(v[2], v[10]); - h_vecs[3] = xorv(v[3], v[11]); - h_vecs[4] = xorv(v[4], v[12]); - h_vecs[5] = xorv(v[5], v[13]); - h_vecs[6] = xorv(v[6], v[14]); - h_vecs[7] = xorv(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs(&h_vecs[0]); - transpose_vecs(&h_vecs[4]); - // The first four vecs now contain the first half of each output, and the - // second four vecs contain the second half of each output. - storeu(h_vecs[0], &out[0 * sizeof(__m128i)]); - storeu(h_vecs[4], &out[1 * sizeof(__m128i)]); - storeu(h_vecs[1], &out[2 * sizeof(__m128i)]); - storeu(h_vecs[5], &out[3 * sizeof(__m128i)]); - storeu(h_vecs[2], &out[4 * sizeof(__m128i)]); - storeu(h_vecs[6], &out[5 * sizeof(__m128i)]); - storeu(h_vecs[3], &out[6 * sizeof(__m128i)]); - storeu(h_vecs[7], &out[7 * sizeof(__m128i)]); -} - -INLINE void hash_one_sse2(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - blake3_compress_in_place_sse2(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - memcpy(out, cv, BLAKE3_OUT_LEN); -} - -void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= DEGREE) { - blake3_hash4_sse2(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += DEGREE; - } - inputs += DEGREE; - num_inputs -= DEGREE; - out = &out[DEGREE * BLAKE3_OUT_LEN]; - } - while (num_inputs > 0) { - hash_one_sse2(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/third_party/blake3/c/blake3_sse41.c b/src/third_party/blake3/c/blake3_sse41.c deleted file mode 100644 index b28ae13b9..000000000 --- a/src/third_party/blake3/c/blake3_sse41.c +++ /dev/null @@ -1,560 +0,0 @@ -#include "blake3_impl.h" - -#include - -#define DEGREE 4 - -#define _mm_shuffle_ps2(a, b, c) \ - (_mm_castps_si128( \ - _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) - -INLINE __m128i loadu(const uint8_t src[16]) { - return _mm_loadu_si128((const __m128i *)src); -} - -INLINE void storeu(__m128i src, uint8_t dest[16]) { - _mm_storeu_si128((__m128i *)dest, src); -} - -INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } - -// Note that clang-format doesn't like the name "xor" for some reason. -INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } - -INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); } - -INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); -} - -INLINE __m128i rot16(__m128i x) { - return _mm_shuffle_epi8( - x, _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)); -} - -INLINE __m128i rot12(__m128i x) { - return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12)); -} - -INLINE __m128i rot8(__m128i x) { - return _mm_shuffle_epi8( - x, _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)); -} - -INLINE __m128i rot7(__m128i x) { - return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7)); -} - -INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = addv(addv(*row0, m), *row1); - *row3 = xorv(*row3, *row0); - *row3 = rot16(*row3); - *row2 = addv(*row2, *row3); - *row1 = xorv(*row1, *row2); - *row1 = rot12(*row1); -} - -INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = addv(addv(*row0, m), *row1); - *row3 = xorv(*row3, *row0); - *row3 = rot8(*row3); - *row2 = addv(*row2, *row3); - *row1 = xorv(*row1, *row2); - *row1 = rot7(*row1); -} - -// Note the optimization here of leaving row1 as the unrotated row, rather than -// row0. All the message loads below are adjusted to compensate for this. See -// discussion at https://github.com/sneves/blake2-avx2/pull/4 -INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); -} - -INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); -} - -INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags) { - rows[0] = loadu((uint8_t *)&cv[0]); - rows[1] = loadu((uint8_t *)&cv[4]); - rows[2] = set4(IV[0], IV[1], IV[2], IV[3]); - rows[3] = set4(counter_low(counter), counter_high(counter), - (uint32_t)block_len, (uint32_t)flags); - - __m128i m0 = loadu(&block[sizeof(__m128i) * 0]); - __m128i m1 = loadu(&block[sizeof(__m128i) * 1]); - __m128i m2 = loadu(&block[sizeof(__m128i) * 2]); - __m128i m3 = loadu(&block[sizeof(__m128i) * 3]); - - __m128i t0, t1, t2, t3, tt; - - // Round 1. The first round permutes the message words from the original - // input order, into the groups that get mixed in parallel. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 - t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 - t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 2. This round and all following rounds apply a fixed permutation - // to the message words from the round before. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 3 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 4 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 5 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 6 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 7 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); -} - -void blake3_compress_in_place_sse41(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]); - storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]); -} - -void blake3_compress_xof_sse41(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu(xorv(rows[0], rows[2]), &out[0]); - storeu(xorv(rows[1], rows[3]), &out[16]); - storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]); - storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]); -} - -INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) { - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[15] = rot16(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot12(v[4]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[15] = rot8(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot7(v[4]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot16(v[15]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[4] = rot12(v[4]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot8(v[15]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - v[4] = rot7(v[4]); -} - -INLINE void transpose_vecs(__m128i vecs[DEGREE]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is - // 22/33. Note that this doesn't split the vector into two lanes, as the - // AVX2 counterparts do. - __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); - __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); - __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); - __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); - - // Interleave 64-bit lanes. - __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); - __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); - __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); - __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); - - vecs[0] = abcd_0; - vecs[1] = abcd_1; - vecs[2] = abcd_2; - vecs[3] = abcd_3; -} - -INLINE void transpose_msg_vecs(const uint8_t *const *inputs, - size_t block_offset, __m128i out[16]) { - out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]); - out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]); - out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]); - out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]); - out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]); - out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]); - out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]); - out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]); - out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]); - out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]); - out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]); - out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]); - out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]); - out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]); - out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); - out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); - for (size_t i = 0; i < 4; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs(&out[0]); - transpose_vecs(&out[4]); - transpose_vecs(&out[8]); - transpose_vecs(&out[12]); -} - -INLINE void load_counters(uint64_t counter, bool increment_counter, - __m128i *out_lo, __m128i *out_hi) { - const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); - const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); - const __m128i add1 = _mm_and_si128(mask, add0); - __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); - __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), - _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); - __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); - *out_lo = l; - *out_hi = h; -} - -static -void blake3_hash4_sse41(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m128i h_vecs[8] = { - set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), - set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), - }; - __m128i counter_low_vec, counter_high_vec; - load_counters(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN); - __m128i block_flags_vec = set1(block_flags); - __m128i msg_vecs[16]; - transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m128i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn(v, msg_vecs, 0); - round_fn(v, msg_vecs, 1); - round_fn(v, msg_vecs, 2); - round_fn(v, msg_vecs, 3); - round_fn(v, msg_vecs, 4); - round_fn(v, msg_vecs, 5); - round_fn(v, msg_vecs, 6); - h_vecs[0] = xorv(v[0], v[8]); - h_vecs[1] = xorv(v[1], v[9]); - h_vecs[2] = xorv(v[2], v[10]); - h_vecs[3] = xorv(v[3], v[11]); - h_vecs[4] = xorv(v[4], v[12]); - h_vecs[5] = xorv(v[5], v[13]); - h_vecs[6] = xorv(v[6], v[14]); - h_vecs[7] = xorv(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs(&h_vecs[0]); - transpose_vecs(&h_vecs[4]); - // The first four vecs now contain the first half of each output, and the - // second four vecs contain the second half of each output. - storeu(h_vecs[0], &out[0 * sizeof(__m128i)]); - storeu(h_vecs[4], &out[1 * sizeof(__m128i)]); - storeu(h_vecs[1], &out[2 * sizeof(__m128i)]); - storeu(h_vecs[5], &out[3 * sizeof(__m128i)]); - storeu(h_vecs[2], &out[4 * sizeof(__m128i)]); - storeu(h_vecs[6], &out[5 * sizeof(__m128i)]); - storeu(h_vecs[3], &out[6 * sizeof(__m128i)]); - storeu(h_vecs[7], &out[7 * sizeof(__m128i)]); -} - -INLINE void hash_one_sse41(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - blake3_compress_in_place_sse41(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - memcpy(out, cv, BLAKE3_OUT_LEN); -} - -void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= DEGREE) { - blake3_hash4_sse41(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += DEGREE; - } - inputs += DEGREE; - num_inputs -= DEGREE; - out = &out[DEGREE * BLAKE3_OUT_LEN]; - } - while (num_inputs > 0) { - hash_one_sse41(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} From 9c4c63088ea704d3348d94faa90c6f66f1211263 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 17 Jun 2026 11:14:28 -0700 Subject: [PATCH 003/121] radlink: release the ~1GB image buffer early so reclaim overlaps the run The image buffer was push'd on the shared link arena and only reclaimed in the single-threaded process rundown at exit -- a multi-second kernel page-reclaim tail (observed: one thread 100% in-kernel, zero user frames). Allocate it as a standalone reserve_memory/commit_memory region and release_memory() it the instant the background image-write thread joins (image is on disk, no later reader). VirtualFree(MEM_RELEASE) returns fast; the kernel zeroes the ~1GB on its background thread, overlapping the parallel input-view release + exit instead of blocking rundown. Discard early so the kernel cleans up while the app still runs -- don't defer to exit. Gated 65/65 linker torture (determ_test + p2r_determinism: image correct). Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit e7a0e5ef45d7165ad0f28b552a353a90242f89ea) --- src/linker/lnk.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index dbf4f577e..a1c219cce 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -5881,7 +5881,11 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT ProfBeginV("Alloc Image Buffer [%M]", lnk_section_table_total_fsize(sectab)); image_data.size = lnk_section_table_total_fsize(sectab) + image_string_table.total_size; - image_data.str = push_array_no_zero(arena->v[0], U8, image_data.size); + // Standalone reservation (not the shared link arena) so it can be released the instant the image + // is written to disk -- VirtualFree returns fast and the kernel zeroes this ~1GB on its background + // thread, overlapping the rest of the run, instead of in the single-threaded exit rundown. + image_data.str = reserve_memory(image_data.size); + commit_memory(image_data.str, image_data.size); ProfEnd(); ProfBegin("Fill Align Bytes"); @@ -6711,6 +6715,9 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // wait for the thread to finish writing image to disk thread_join(image_write_thread, -1); + // image is on disk and no longer read by anyone -- release its ~1GB now so the kernel reclaims it + // concurrently with the remaining work + exit, not single-threaded in the process rundown. + release_memory(image_ctx.image_data.str, image_ctx.image_data.size); // // Timers // From cdb3becec4ae9b4e97a558a20baa33c3245d8293 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 09:31:29 -0700 Subject: [PATCH 004/121] radlink: /OPT:GCTYPES -- garbage-collect unreferenced CodeView types before PDB emit (opt-in, default off) After type merging, prune any merged TPI/IPI record not transitively reachable from a surviving symbol. Roots are the type indices referenced by the symbols that survive /OPT:REF (plus inlinee call-site types); the type graph is then closed over and everything unreached is dropped before the streams are written. Runs only under /OPT:REF -- it is the debug-info analogue of dead-section stripping, and is otherwise transparent (no visible type is removed). Implementation notes: - parallel transitive closure (bulk-synchronous rounds, atomic mark/expand) - fwdref<->definition pairing via a per-unique-name ring so a live forward reference keeps its definition (and vice-versa) - compaction is in place with the remap kept in scratch, so peak memory is unchanged Numbers (UnrealEditorFortnite-Engine.dll, /OPT:REF /OPT:ICF, hashing NONE): PDB 5315 -> 5081 MB (type-GC alone: -234 MB) Default OFF: shrinks the PDB but a pruned type can't be cast-to in the debugger watch window. Co-Authored-By: Claude Opus 4.8 (1M context) Co-Authored-By: Claude Fable 5 (cherry picked from commit 024f1ed19b6bc5867753eb5f92342c6e0ce35ce6) --- src/linker/lnk.c | 7 + src/linker/lnk_config.c | 4 + src/linker/lnk_config.h | 1 + src/linker/lnk_debug_info.c | 290 ++++++++++++++++++++++++++++++++++++ src/linker/lnk_debug_info.h | 1 + 5 files changed, 303 insertions(+) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index a1c219cce..26a9d0d8b 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -6588,6 +6588,13 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) LNK_CodeViewInput cv = lnk_make_code_view_input(tp, arena, config, debug_info_objs_count, debug_info_objs, rrt_input); LNK_MergedTypes cv_types = lnk_merge_types(tp, arena, &cv, 0); + // prune merged types not reachable from any surviving symbol (PDB-size win). OFF by default: + // it removes types that a debugger can still legitimately cast to in the watch window + // (reachable-from-symbols is a subset of castable-types). Opt in with /OPT:GCTYPES. + if (config->opt_gc_types == LNK_SwitchState_Yes) { + lnk_gc_types(tp, arena->v[0], &cv, &cv_types); + } + // // Debug Info // diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index 4e0993319..02716a1ce 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -1727,6 +1727,10 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val config->opt_lbr = LNK_SwitchState_Yes; } else if (str8_match_lit("nolibr", param, StringMatchFlag_CaseInsensitive)) { config->opt_lbr = LNK_SwitchState_No; + } else if (str8_match_lit("gctypes", param, StringMatchFlag_CaseInsensitive)) { + config->opt_gc_types = LNK_SwitchState_Yes; + } else if (str8_match_lit("nogctypes", param, StringMatchFlag_CaseInsensitive)) { + config->opt_gc_types = LNK_SwitchState_No; } else { lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "unknown option \"%S\"", param); } diff --git a/src/linker/lnk_config.h b/src/linker/lnk_config.h index 5ad6bb20c..83a524d61 100644 --- a/src/linker/lnk_config.h +++ b/src/linker/lnk_config.h @@ -324,6 +324,7 @@ typedef struct LNK_Config LNK_SwitchState opt_ref; LNK_SwitchState opt_icf; LNK_SwitchState opt_lbr; + LNK_SwitchState opt_gc_types; // /OPT:GCTYPES -- prune unreferenced CodeView types. Default OFF: shrinks PDB but a pruned type can't be cast-to in the debugger watch window. U64 opt_iter_count; LNK_SwitchState import_table_emit_biat; LNK_SwitchState import_table_emit_uiat; diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index d18791cbb..0ae9eb629 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3031,6 +3031,296 @@ lnk_pdb_output_enqueue_remaining(LNK_PdbOutput *output, MSF_Context *msf) scratch_end(scratch); } +//////////////////////////////// +// Type Garbage Collection +// +// After type merging, prune merged TPI/IPI leaves that are not reachable from any surviving +// symbol record (the GC roots), compact them, and remap all type indices. link.exe keeps a +// large unreferenced-type set; pruning it is a transparent PDB-size win (debug-info only -- the +// image is untouched). Runs on the final post-fixup type indices in place. + +typedef struct LNK_GCTypes +{ + LNK_CodeViewInput *cv; + U64 min [CV_TypeIndexSource_COUNT]; // first type index per source + U64 orig_n[CV_TypeIndexSource_COUNT]; // pre-GC leaf count per source + U8 *mark [CV_TypeIndexSource_COUNT]; // reachable bitmap, indexed by (ti - min) + U8 *expanded[CV_TypeIndexSource_COUNT]; // transitive-closure: already-visited bitmap + CV_TypeIndex *remap [CV_TypeIndexSource_COUNT]; // old leaf idx -> new type index + U8 **leaf_v [CV_TypeIndexSource_COUNT]; // original leaf pointer arrays + U32 *udt_next; // TPI fwdref<->definition unique_name ring + U32 *changed; // set when a closure round marks something new + Rng1U64 *sym_ranges; + B32 do_rewrite; // 0 = mark roots, 1 = rewrite to compacted indices + // per-source scratch (set before dispatch) + CV_TypeIndexSource cur_source; + U8 **cur_leaf_v; + Rng1U64 *cur_ranges; +} LNK_GCTypes; + +typedef struct LNK_GCNamePair { U64 hash; U32 idx; } LNK_GCNamePair; + +internal int +lnk_gc_name_pair_is_before(void *raw_a, void *raw_b) +{ + LNK_GCNamePair *a = raw_a, *b = raw_b; + return a->hash != b->hash ? (a->hash < b->hash) : (a->idx < b->idx); +} + +internal void +lnk_gc_mark_ti(LNK_GCTypes *g, CV_TypeIndexSource s, CV_TypeIndex ti) +{ + U64 lo = g->min[s]; + if (ti >= lo) { U64 idx = ti - lo; if (idx < g->orig_n[s]) { g->mark[s][idx] = 1; } } +} + +// walk a record's type-index sites; mark roots (do_rewrite==0) or rewrite to compacted indices (==1) +internal void +lnk_gc_visit_offsets(LNK_GCTypes *g, String8 data, CV_TypeIndexInfoList ti_info_list) +{ + for EachNode(n, CV_TypeIndexInfo, ti_info_list.first) { + U8 *p = data.str + n->offset; + CV_TypeIndex ti = memory_read32(p); + if (g->do_rewrite) { + U64 lo = g->min[n->source]; + if (ti >= lo) { U64 idx = ti - lo; if (idx < g->orig_n[n->source]) { memory_write32(p, g->remap[n->source][idx]); } } + } else { + lnk_gc_mark_ti(g, n->source, ti); + } + } +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_syms_task) +{ + LNK_GCTypes *g = raw_task; + Temp scratch = scratch_begin(0, 0); + for EachInRange(i, g->sym_ranges[task_id]) { + LNK_SymbolInput symbols = g->cv->symbol_inputs[i]; + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { + Temp temp = temp_begin(scratch.arena); + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); + CV_TypeIndexInfoList l = cv_get_symbol_type_index_offsets(temp.arena, symbol.kind, symbol.data); + lnk_gc_visit_offsets(g, symbol.data, l); + temp_end(temp); + } + } + scratch_end(scratch); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_inlines_task) +{ + LNK_GCTypes *g = raw_task; + U64 obj_idx = task_id; + Temp scratch = scratch_begin(0, 0); + String8List inlinee_lines = cv_sub_section_from_debug_s(g->cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); + for EachNode(dn, String8Node, inlinee_lines.first) { + Temp temp = temp_begin(scratch.arena); + CV_TypeIndexInfoList l = cv_get_inlinee_type_index_offsets(temp.arena, dn->string); + lnk_gc_visit_offsets(g, dn->string, l); + temp_end(temp); + } + scratch_end(scratch); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_rewrite_leaves_task) +{ + LNK_GCTypes *g = raw_task; + Temp scratch = scratch_begin(0, 0); + for EachInRange(i, g->cur_ranges[task_id]) { + Temp temp = temp_begin(scratch.arena); + CV_Leaf leaf = cv_leaf_from_ptr(g->cur_leaf_v[i]); + CV_TypeIndexInfoList l = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); + lnk_gc_visit_offsets(g, leaf.data, l); + temp_end(temp); + } + scratch_end(scratch); +} + +typedef struct LNK_GCRingTask +{ + U8 **leaf_v; + Rng1U64 *ranges; + U64 *counts; + U64 *offsets; + LNK_GCNamePair *pairs; +} LNK_GCRingTask; + +// parallel: count UDT leaves with a unique_name per range (pass 0) / emit (hash,idx) pairs (pass 1) +internal +THREAD_POOL_TASK_FUNC(lnk_gc_ring_count_task) +{ + LNK_GCRingTask *t = raw_task; + U64 n = 0; + for EachInRange(i, t->ranges[task_id]) { + CV_Leaf leaf = cv_leaf_from_ptr(t->leaf_v[i]); + if (cv_is_udt(leaf.kind)) { + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (ui.props & CV_TypeProp_HasUniqueName) { n += 1; } + } + } + t->counts[task_id] = n; +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_ring_fill_task) +{ + LNK_GCRingTask *t = raw_task; + U64 cur = t->offsets[task_id]; + for EachInRange(i, t->ranges[task_id]) { + CV_Leaf leaf = cv_leaf_from_ptr(t->leaf_v[i]); + if (cv_is_udt(leaf.kind)) { + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (ui.props & CV_TypeProp_HasUniqueName) { + U64 h = 14695981039346656037ull; + for EachIndex(c, ui.unique_name.size) { h = (h ^ ui.unique_name.str[c]) * 0x100000001b3ull; } + t->pairs[cur].hash = h; t->pairs[cur].idx = (U32)i; cur += 1; + } + } + } +} + +// one bulk-synchronous round of transitive closure: visit each marked-but-unexpanded leaf in +// this source, mark the leaves it references (and its unique_name UDT counterparts). Parallel; +// `expanded` is grabbed atomically so each leaf is visited once, `changed` flags progress. +internal +THREAD_POOL_TASK_FUNC(lnk_gc_expand_task) +{ + LNK_GCTypes *g = raw_task; + CV_TypeIndexSource s = g->cur_source; + Temp scratch = scratch_begin(0, 0); + for EachInRange(i, g->cur_ranges[task_id]) { + if (!g->mark[s][i]) { continue; } + if (ins_atomic_u8_eval_assign(&g->expanded[s][i], 1)) { continue; } // already visited + + Temp temp = temp_begin(scratch.arena); + CV_Leaf leaf = cv_leaf_from_ptr(g->leaf_v[s][i]); + CV_TypeIndexInfoList l = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); + for EachNode(n, CV_TypeIndexInfo, l.first) { + CV_TypeIndex ti = memory_read32(leaf.data.str + n->offset); + U64 lo = g->min[n->source]; + if (ti >= lo) { + U64 ci = ti - lo; + if (ci < g->orig_n[n->source] && !g->mark[n->source][ci]) { g->mark[n->source][ci] = 1; ins_atomic_u32_eval_assign(g->changed, 1); } + } + } + temp_end(temp); + + if (s == CV_TypeIndexSource_TPI) { + for (U32 j = g->udt_next[i]; j != i; j = g->udt_next[j]) { + if (!g->mark[CV_TypeIndexSource_TPI][j]) { g->mark[CV_TypeIndexSource_TPI][j] = 1; ins_atomic_u32_eval_assign(g->changed, 1); } + } + } + } + scratch_end(scratch); +} + +internal void +lnk_gc_types(TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedTypes *types) +{ + ProfBeginFunction(); + Temp scratch = scratch_begin(&arena, 1); + + LNK_GCTypes g = {0}; + g.cv = cv; + U64 total_leaves = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.min[s] = types->min_type_indices[s]; + g.orig_n[s] = types->count[s]; + g.leaf_v[s] = types->v[s]; + g.mark[s] = push_array(scratch.arena, U8, g.orig_n[s] ? g.orig_n[s] : 1); // zeroed + total_leaves += g.orig_n[s]; + } + + // mark roots: every type index referenced by a surviving symbol / inlinee record + g.do_rewrite = 0; + g.sym_ranges = tp_divide_work(scratch.arena, cv->symbol_input_count, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_syms_task, &g); + tp_for_parallel(tp, 0, cv->obj_count, lnk_gc_inlines_task, &g); + + // link UDT leaves that share a unique_name into rings, so that marking any one (e.g. a + // forward ref reached as a member-pointer target) also keeps its full definition -- needed + // for the debugger to complete types referenced only by name. TPI only (IPI has no UDTs). + U64 n_tpi = g.orig_n[CV_TypeIndexSource_TPI]; + g.udt_next = push_array_no_zero(scratch.arena, U32, n_tpi ? n_tpi : 1); + for EachIndex(i, n_tpi) { g.udt_next[i] = (U32)i; } + { + LNK_GCRingTask rt = {0}; + rt.leaf_v = g.leaf_v[CV_TypeIndexSource_TPI]; + rt.ranges = tp_divide_work(scratch.arena, n_tpi, tp->worker_count); + rt.counts = push_array(scratch.arena, U64, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_ring_count_task, &rt); + rt.offsets = offsets_from_counts_array_u64(scratch.arena, rt.counts, tp->worker_count); + U64 np = sum_array_u64(tp->worker_count, rt.counts); + rt.pairs = push_array_no_zero(scratch.arena, LNK_GCNamePair, np ? np : 1); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_ring_fill_task, &rt); + + radsort(rt.pairs, np, lnk_gc_name_pair_is_before); + for (U64 a = 0; a < np; ) { + U64 b = a + 1; + while (b < np && rt.pairs[b].hash == rt.pairs[a].hash) { b += 1; } + for (U64 k = a; k < b; k += 1) { g.udt_next[rt.pairs[k].idx] = rt.pairs[(k + 1 < b) ? (k + 1) : a].idx; } + a = b; + } + } + + // transitive closure (parallel, bulk-synchronous): repeat rounds that visit each + // marked-but-unexpanded leaf and mark what it references, until a round marks nothing new + U32 changed = 0; + g.changed = &changed; + Rng1U64 *expand_ranges[CV_TypeIndexSource_COUNT]; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.expanded[s] = push_array(scratch.arena, U8, g.orig_n[s] ? g.orig_n[s] : 1); + expand_ranges[s] = tp_divide_work(scratch.arena, g.orig_n[s], tp->worker_count); + } + do { + changed = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.cur_source = (CV_TypeIndexSource)s; + g.cur_ranges = expand_ranges[s]; + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_expand_task, &g); + } + } while (changed); + + // compact each source: assign new contiguous type indices to the kept leaves. The leaf + // pointer array is compacted IN PLACE (kept count only shrinks, so the write cursor never + // passes the read cursor) and remap lives in scratch -- so the GC adds nothing to the arena + // that survives into the (peak) PDB build. + U64 kept_total = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.remap[s] = push_array_no_zero(scratch.arena, CV_TypeIndex, g.orig_n[s] ? g.orig_n[s] : 1); + U8 **v = g.leaf_v[s]; + U64 new_n = 0; + for EachIndex(idx, g.orig_n[s]) { + if (g.mark[s][idx]) { g.remap[s][idx] = (CV_TypeIndex)(g.min[s] + new_n); v[new_n++] = v[idx]; } + else { g.remap[s][idx] = 0; /* T_NOTYPE; never referenced by a kept record */ } + } + types->count[s] = new_n; + kept_total += new_n; + } + + // rewrite all type-index references to the compacted indices + g.do_rewrite = 1; + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_syms_task, &g); + tp_for_parallel(tp, 0, cv->obj_count, lnk_gc_inlines_task, &g); + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.cur_source = (CV_TypeIndexSource)s; + g.cur_leaf_v = types->v[s]; + g.cur_ranges = tp_divide_work(scratch.arena, types->count[s], tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_rewrite_leaves_task, &g); + } + + if (lnk_get_log_status(LNK_Log_Debug)) { + lnk_log(LNK_Log_Debug, "type GC: kept %llu of %llu leaves (pruned %llu)", kept_total, total_leaves, total_leaves - kept_total); + } + + scratch_end(scratch); + ProfEnd(); +} + internal LNK_FileArtifact lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags) { diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 573fc3b53..bbf8b6603 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -272,4 +272,5 @@ internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP //////////////////////////////// // PDB +internal void lnk_gc_types (TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedTypes *types); internal LNK_FileArtifact lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags); From 4109234b9b0cd66f6b135a16139332eb84754074 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 10:10:58 -0700 Subject: [PATCH 005/121] radlink: frontier worklist for type-GC transitive closure (perf) The closure re-scanned every merged type each round (O(rounds * total types)) to find marked-but-unexpanded leaves. In a full-link trace of UnrealEditorFortnite that round-rescan dominated the type-GC: lnk_gc_expand_task ~13.3 s of CPU. Replace it with a frontier worklist: the atomic mark now gates a single append per leaf, and each round expands only the slice newly marked by the previous round, so total work is O(reachable types) instead of O(rounds * total types). Drops the per-round `expanded` bitmap and full-array sweeps. Output is unchanged -- same reachable set, PDB byte-identical (5081 MB on the same input), and debugger-fidelity checks (addr->symbol 100%, core types resolve) match the pre-change build. Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 58d0c27c4eac8d2d5d2785d1928fbc181c16f8e0) --- src/linker/lnk_debug_info.c | 69 ++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 24 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 0ae9eb629..fcb8f4b14 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3045,17 +3045,20 @@ typedef struct LNK_GCTypes U64 min [CV_TypeIndexSource_COUNT]; // first type index per source U64 orig_n[CV_TypeIndexSource_COUNT]; // pre-GC leaf count per source U8 *mark [CV_TypeIndexSource_COUNT]; // reachable bitmap, indexed by (ti - min) - U8 *expanded[CV_TypeIndexSource_COUNT]; // transitive-closure: already-visited bitmap CV_TypeIndex *remap [CV_TypeIndexSource_COUNT]; // old leaf idx -> new type index U8 **leaf_v [CV_TypeIndexSource_COUNT]; // original leaf pointer arrays U32 *udt_next; // TPI fwdref<->definition unique_name ring - U32 *changed; // set when a closure round marks something new Rng1U64 *sym_ranges; B32 do_rewrite; // 0 = mark roots, 1 = rewrite to compacted indices + // transitive-closure frontier: indices marked but not yet expanded. Each leaf is appended once + // (the atomic mark gates it), so frontier[s] is sized orig_n[s] and fcount[s] is its atomic tail. + U32 *frontier[CV_TypeIndexSource_COUNT]; + U32 *fcount [CV_TypeIndexSource_COUNT]; // atomic append cursor per source // per-source scratch (set before dispatch) CV_TypeIndexSource cur_source; U8 **cur_leaf_v; Rng1U64 *cur_ranges; + U64 round_begin, round_end; // frontier slice processed this round } LNK_GCTypes; typedef struct LNK_GCNamePair { U64 hash; U32 idx; } LNK_GCNamePair; @@ -3183,18 +3186,29 @@ THREAD_POOL_TASK_FUNC(lnk_gc_ring_fill_task) } } -// one bulk-synchronous round of transitive closure: visit each marked-but-unexpanded leaf in -// this source, mark the leaves it references (and its unique_name UDT counterparts). Parallel; -// `expanded` is grabbed atomically so each leaf is visited once, `changed` flags progress. +// mark a leaf reachable and, if this is its first mark, append it to its source's frontier so a +// later round expands it. Atomic mark gates the append, so each leaf lands on the frontier once. +internal void +lnk_gc_mark_enqueue(LNK_GCTypes *g, CV_TypeIndexSource ns, U64 ci) +{ + if (ci < g->orig_n[ns] && !ins_atomic_u8_eval_assign(&g->mark[ns][ci], 1)) { + U32 pos = ins_atomic_u32_inc_eval(g->fcount[ns]) - 1; + g->frontier[ns][pos] = (U32)ci; + } +} + +// one bulk-synchronous round of transitive closure: expand the frontier slice [round_begin, +// round_end) of cur_source -- visit each leaf and enqueue the leaves it references (and its +// unique_name UDT counterparts). Frontier-driven, so total work is O(reachable leaves), not +// O(rounds * total leaves). internal THREAD_POOL_TASK_FUNC(lnk_gc_expand_task) { LNK_GCTypes *g = raw_task; CV_TypeIndexSource s = g->cur_source; Temp scratch = scratch_begin(0, 0); - for EachInRange(i, g->cur_ranges[task_id]) { - if (!g->mark[s][i]) { continue; } - if (ins_atomic_u8_eval_assign(&g->expanded[s][i], 1)) { continue; } // already visited + for EachInRange(local, g->cur_ranges[task_id]) { + U32 i = g->frontier[s][g->round_begin + local]; Temp temp = temp_begin(scratch.arena); CV_Leaf leaf = cv_leaf_from_ptr(g->leaf_v[s][i]); @@ -3202,16 +3216,13 @@ THREAD_POOL_TASK_FUNC(lnk_gc_expand_task) for EachNode(n, CV_TypeIndexInfo, l.first) { CV_TypeIndex ti = memory_read32(leaf.data.str + n->offset); U64 lo = g->min[n->source]; - if (ti >= lo) { - U64 ci = ti - lo; - if (ci < g->orig_n[n->source] && !g->mark[n->source][ci]) { g->mark[n->source][ci] = 1; ins_atomic_u32_eval_assign(g->changed, 1); } - } + if (ti >= lo) { lnk_gc_mark_enqueue(g, n->source, ti - lo); } } temp_end(temp); if (s == CV_TypeIndexSource_TPI) { for (U32 j = g->udt_next[i]; j != i; j = g->udt_next[j]) { - if (!g->mark[CV_TypeIndexSource_TPI][j]) { g->mark[CV_TypeIndexSource_TPI][j] = 1; ins_atomic_u32_eval_assign(g->changed, 1); } + lnk_gc_mark_enqueue(g, CV_TypeIndexSource_TPI, j); } } } @@ -3269,21 +3280,31 @@ lnk_gc_types(TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedType // transitive closure (parallel, bulk-synchronous): repeat rounds that visit each // marked-but-unexpanded leaf and mark what it references, until a round marks nothing new - U32 changed = 0; - g.changed = &changed; - Rng1U64 *expand_ranges[CV_TypeIndexSource_COUNT]; + // seed the frontier with the root-marked leaves (one O(total leaves) scan), then expand + // frontier slices until both sources drain. Each leaf is expanded exactly once. + U32 fcount[CV_TypeIndexSource_COUNT] = {0}; + U64 start [CV_TypeIndexSource_COUNT] = {0}; for EachIndex(s, CV_TypeIndexSource_COUNT) { - g.expanded[s] = push_array(scratch.arena, U8, g.orig_n[s] ? g.orig_n[s] : 1); - expand_ranges[s] = tp_divide_work(scratch.arena, g.orig_n[s], tp->worker_count); + g.frontier[s] = push_array_no_zero(scratch.arena, U32, g.orig_n[s] ? g.orig_n[s] : 1); + g.fcount[s] = &fcount[s]; + for EachIndex(i, g.orig_n[s]) { if (g.mark[s][i]) { g.frontier[s][fcount[s]++] = (U32)i; } } } - do { - changed = 0; + for (;;) { + B32 any = 0; for EachIndex(s, CV_TypeIndexSource_COUNT) { - g.cur_source = (CV_TypeIndexSource)s; - g.cur_ranges = expand_ranges[s]; - tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_expand_task, &g); + U64 begin = start[s], end = fcount[s]; // fcount may grow during the round (cross-source enqueues) + if (begin < end) { + any = 1; + g.cur_source = (CV_TypeIndexSource)s; + g.round_begin = begin; + g.round_end = end; + g.cur_ranges = tp_divide_work(scratch.arena, end - begin, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_expand_task, &g); + start[s] = end; + } } - } while (changed); + if (!any) { break; } + } // compact each source: assign new contiguous type indices to the kept leaves. The leaf // pointer array is compacted IN PLACE (kept count only shrinks, so the write cursor never From c24649929794eb2d23d02f48031fc5969a5dcafd Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 10:28:17 -0700 Subject: [PATCH 006/121] radlink: skip the GC mark atomic on the already-reachable fast path The frontier mark did an interlocked op on every reference edge. Add a plain non-atomic check first (the mark bit only ever goes 0->1, so a stale "already set" read is safe), so the interlocked op runs once per leaf at its 0->1 transition instead of once per edge. Output identical (PDB 5081 MB). Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 9872144b21c667cd1900a29297da42b18a21d6ec) --- src/linker/lnk_debug_info.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index fcb8f4b14..3a76f96c9 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3191,7 +3191,11 @@ THREAD_POOL_TASK_FUNC(lnk_gc_ring_fill_task) internal void lnk_gc_mark_enqueue(LNK_GCTypes *g, CV_TypeIndexSource ns, U64 ci) { - if (ci < g->orig_n[ns] && !ins_atomic_u8_eval_assign(&g->mark[ns][ci], 1)) { + if (ci >= g->orig_n[ns]) { return; } + if (g->mark[ns][ci]) { return; } // fast non-atomic skip: already reachable (the common edge) + // only the worker that wins the 0->1 transition appends, so the atomic runs once per leaf (not + // once per reference edge). + if (!ins_atomic_u8_eval_assign(&g->mark[ns][ci], 1)) { U32 pos = ins_atomic_u32_inc_eval(g->fcount[ns]) - 1; g->frontier[ns][pos] = (U32)ci; } From e4b3ed3c07defa91f02bf32099f5464447456d38 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 13 Jul 2026 14:23:51 -0700 Subject: [PATCH 007/121] radlink: header-unit IFC debug-record resolution Resolve MSVC C++ header-unit IFC debug records (LF_IFC_RECORD 0x1522) by merging the .ifc .msvc.trait.debug-records CodeView stream and redirecting each record to its real type -> fixes VS debugger AV stepping into header- unit code (BAD 921->0, 44/44 MSVC name-match, live debug verified). (cherry picked from commit 4b7f8bf8, ICF leader-keying half dropped: superseded by upstream d0d8b596's own /OPT:ICF) Co-Authored-By: Claude Opus 4.8 (1M context) Co-Authored-By: Claude Fable 5 --- src/linker/codeview_ext/ifc.c | 107 +++++++++++++ src/linker/codeview_ext/ifc.h | 43 +++++ src/linker/lnk.c | 2 + src/linker/lnk_config.c | 13 ++ src/linker/lnk_config.h | 4 + src/linker/lnk_debug_info.c | 284 ++++++++++++++++++++++++++++++++++ src/linker/lnk_debug_info.h | 10 ++ 7 files changed, 463 insertions(+) create mode 100644 src/linker/codeview_ext/ifc.c create mode 100644 src/linker/codeview_ext/ifc.h diff --git a/src/linker/codeview_ext/ifc.c b/src/linker/codeview_ext/ifc.c new file mode 100644 index 000000000..e5693740f --- /dev/null +++ b/src/linker/codeview_ext/ifc.c @@ -0,0 +1,107 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +read_only global U8 g_ifc_signature[4] = { 0x54, 0x51, 0x45, 0x1A }; +read_only global U8 g_uba_signature[4] = { 0x55, 0x42, 0x41, 0x01 }; // "UBA\x01" + +internal IFC_File +ifc_file_read(Arena *arena, String8 path, String8 *error_out) +{ + IFC_File ifc = {0}; + ifc.path = push_str8_copy(arena, path); + + String8 data = lnk_read_data_from_file_path(arena, 0, path); + ifc.data = data; + if (data.size < 4) { + *error_out = push_str8f(arena, "IFC '%S' is too small (%llu bytes)", path, data.size); + return ifc; + } + + // detect UBA-compressed input (out of scope) + if (MemoryMatch(data.str, g_uba_signature, sizeof(g_uba_signature))) { + *error_out = push_str8f(arena, "IFC '%S' is UBA-compressed (magic 'UBA\\x01'); materialize a raw .ifc (UBA decompress unsupported)", path); + return ifc; + } + + // validate signature + if ( ! MemoryMatch(data.str, g_ifc_signature, sizeof(g_ifc_signature))) { + *error_out = push_str8f(arena, "IFC '%S' has bad signature (expected 54 51 45 1A)", path); + return ifc; + } + + // --- parse header --- + U64 off = 4; + if (off + 32 > data.size) { goto truncated; } + MemoryCopy(ifc.content_hash, data.str + off, 32); + off += 32; + + if (off + 4 > data.size) { goto truncated; } + U8 major = data.str[off+0]; + U8 minor = data.str[off+1]; + U8 abi = data.str[off+2]; (void)abi; + U8 arch = data.str[off+3]; + off += 4; + + // assert version 0.44 + x64; error otherwise (encoding proven only for these) + if ( ! (major == 0 && minor == 44)) { + *error_out = push_str8f(arena, "IFC '%S' unsupported version %u.%u (expected 0.44)", path, major, minor); + return ifc; + } + if (arch != 2) { + *error_out = push_str8f(arena, "IFC '%S' unsupported architecture %u (expected 2 == x64)", path, arch); + return ifc; + } + + U32 cplusplus; off += str8_deserial_read_struct(data, off, &cplusplus); (void)cplusplus; + U32 string_table_bytes; off += str8_deserial_read_struct(data, off, &string_table_bytes); + U32 string_table_size; off += str8_deserial_read_struct(data, off, &string_table_size); + U32 unit; off += str8_deserial_read_struct(data, off, &unit); (void)unit; + U32 src_path; off += str8_deserial_read_struct(data, off, &src_path); (void)src_path; + U32 global_scope; off += str8_deserial_read_struct(data, off, &global_scope); (void)global_scope; + U32 toc; off += str8_deserial_read_struct(data, off, &toc); + U32 partition_count; off += str8_deserial_read_struct(data, off, &partition_count); + if (off > data.size) { goto truncated; } + + // string table + if ((U64)string_table_bytes + string_table_size > data.size) { + *error_out = push_str8f(arena, "IFC '%S' string table out of bounds", path); + return ifc; + } + String8 string_table = str8(data.str + string_table_bytes, string_table_size); + + // --- partition summary table --- + String8 needle = str8_lit(".msvc.trait.debug-records"); + U64 po = toc; + for (U32 i = 0; i < partition_count; ++i, po += 16) { + if (po + 16 > data.size) { goto truncated; } + U32 name_off, p_off, count, entity_size; + str8_deserial_read_struct(data, po + 0, &name_off); + str8_deserial_read_struct(data, po + 4, &p_off); + str8_deserial_read_struct(data, po + 8, &count); + str8_deserial_read_struct(data, po + 12, &entity_size); + + if (name_off >= string_table.size) { continue; } + String8 name = str8_cstring((char *)string_table.str + name_off); + if (str8_match(name, needle, 0)) { + // entity_size == 1 -> count is a byte length + if ((U64)p_off + count > data.size) { + *error_out = push_str8f(arena, "IFC '%S' debug-records partition out of bounds", path); + return ifc; + } + ifc.debug_records = str8(data.str + p_off, count); + break; + } + } + + if (ifc.debug_records.size == 0) { + *error_out = push_str8f(arena, "IFC '%S' has no '.msvc.trait.debug-records' partition", path); + return ifc; + } + + ifc.is_valid = 1; + return ifc; + +truncated: + *error_out = push_str8f(arena, "IFC '%S' is truncated", path); + return ifc; +} diff --git a/src/linker/codeview_ext/ifc.h b/src/linker/codeview_ext/ifc.h new file mode 100644 index 000000000..fb3169b75 --- /dev/null +++ b/src/linker/codeview_ext/ifc.h @@ -0,0 +1,43 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +//////////////////////////////// +// MSVC IFC (header-unit module interface) reader +// +// radlink consumes the `.msvc.trait.debug-records` partition embedded in an +// MSVC `.ifc` (C++20 module / header-unit interface) file. That partition is a +// raw CodeView type-leaf stream (entity_size == 1, NO u32 signature, first leaf +// at offset 0, TI base 0x1000). A consuming `.obj` references this stream via +// LF_IFC_RECORD (0x1522) leaves -- see lnk_debug_info.c. +// +// File layout (microsoft/ifc-spec, FileHeader): +// u8[4] signature = { 0x54,0x51,0x45,0x1A } ("TQE\x1a") +// u8[32] content_hash (sha256) -- record.GUID(16)++record.hash(16) == first 32 bytes here +// u8 major, minor -- assert 0.44 +// u8 abi +// u8 arch -- 2 == x64 +// u32 cplusplus +// u32 string_table_bytes (off), u32 string_table_size +// u32 unit +// u32 src_path (textoffset) +// u32 global_scope +// u32 toc -- offset to partition summary table +// u32 partition_count +// u8 internal_partition +// Partition summary entry (16 bytes): { u32 name(textoffset); u32 offset; u32 count; u32 entity_size } + +typedef struct IFC_File +{ + String8 data; // whole .ifc bytes (owning view into arena) + String8 path; // .ifc path (copied) + U8 content_hash[32]; + String8 debug_records; // .msvc.trait.debug-records blob {ptr,size}; size 0 if absent + B32 is_valid; +} IFC_File; + +// Reads `path`, validates magic/version/arch, locates `.msvc.trait.debug-records`. +// On error fills *error_out and returns is_valid=0. Detects UBA-compressed inputs +// (magic "UBA\x01") and reports them (decompression is out of scope). +internal IFC_File ifc_file_read(Arena *arena, String8 path, String8 *error_out); diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 26a9d0d8b..fec3e9bec 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -114,6 +114,7 @@ #include "lnk_debug_helper.h" #include "lnk_obj.h" #include "lnk_lib.h" +#include "codeview_ext/ifc.h" #include "lnk_debug_info.h" #include "lnk.h" @@ -128,6 +129,7 @@ #include "lnk_obj.c" #include "lnk_debug_helper.c" #include "lnk_lib.c" +#include "codeview_ext/ifc.c" #include "lnk_debug_info.c" // ----------------------------------------------------------------------------- diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index 02716a1ce..47083874b 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -113,6 +113,8 @@ global read_only LNK_CmdSwitch g_cmd_switch_map[] = { LNK_CmdSwitch_RadTypeServer, 0, LNK_CmdValueKind_Scalar, "RAD_TYPE_SERVER", ":FILENAME", "Merge types and store them in the specified file. The filename must have the .rrt extension." }, { LNK_CmdSwitch_LLVM_AddrSig, 0, LNK_CmdValueKind_Scalar, "LLVM_ADDRSIG", "[:NO]", "Use .llvm_addrsig to guide ICF." }, + { LNK_CmdSwitch_IfcMap, 1, LNK_CmdValueKind_Scalar, "IFCMAP", ":FILENAME", "Map a header-unit module interface (.ifc) for debug-record resolution (TOML)." }, + { LNK_CmdSwitch_IfcDebugRecords, 0, LNK_CmdValueKind_Scalar, "IFCDEBUGRECORDS", "[:NO]", "Resolve MSVC header-unit IFC debug records into real CodeView types." }, { LNK_CmdSwitch_Help, 0, LNK_CmdValueKind_Null, "HELP", "", "" }, { LNK_CmdSwitch_Help, 0, LNK_CmdValueKind_Null, "?", "", "" }, @@ -2336,6 +2338,17 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val case LNK_CmdSwitch_LLVM_AddrSig: { lnk_cmd_switch_parse_flag(obj, cmd_switch, value, &config->llvm_addrsig); } break; + case LNK_CmdSwitch_IfcMap: { + // collect .toml paths (header-unit -> .ifc); parsed lazily during debug-info build + String8List copy = str8_list_copy(config->arena, &value_strings); + str8_list_concat_in_place(&config->ifc_map_list, ©); + } break; + case LNK_CmdSwitch_IfcDebugRecords: { + LNK_SwitchState state = LNK_SwitchState_Null; + if (lnk_cmd_switch_parse_flag(obj, cmd_switch, value_strings, &state)) { + config->ifc_debug_records = state; + } + } break; } scratch_end(scratch); diff --git a/src/linker/lnk_config.h b/src/linker/lnk_config.h index 83a524d61..8f9052afc 100644 --- a/src/linker/lnk_config.h +++ b/src/linker/lnk_config.h @@ -149,6 +149,8 @@ typedef enum LNK_CmdSwitch_RadTypeServer_MatchObj, LNK_CmdSwitch_LLVM_AddrSig, + LNK_CmdSwitch_IfcMap, + LNK_CmdSwitch_IfcDebugRecords, LNK_CmdSwitch_Help, @@ -426,6 +428,8 @@ typedef struct LNK_Config LNK_SwitchState type_server; LNK_SwitchState sort_imports; LNK_SwitchState llvm_addrsig; + LNK_SwitchState ifc_debug_records; // resolve LF_IFC_RECORD (0x1522) into real CodeView types + String8List ifc_map_list; // .toml paths from /ifcMap (header-unit -> .ifc) } LNK_Config; // --- MSVC Error Codes -------------------------------------------------------- diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 3a76f96c9..190086f82 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -656,6 +656,270 @@ lnk_rrt_array_from_config(Arena *arena, LNK_Config *config) return rrt_arr; } +//////////////////////////////// +// IFC header-unit debug-record resolution + +typedef struct LNK_IfcMapEntry +{ + String8 ifc_path; // absolute .ifc path +} LNK_IfcMapEntry; + +// Parse the trivial /ifcMap TOML by hand: +// [[header-unit]] +// name = ["quote", ''] +// ifc = "" +// Registers basename(header-unit-path) -> .ifc path in `hm` (hash_map of path->raw LNK_IfcMapEntry*). +internal void +lnk_parse_ifc_map_toml(Arena *arena, HashMap *hm, String8 toml_data) +{ + U64 cursor = 0; + String8 cur_name = {0}; + while (cursor < toml_data.size) { + // read a line + U64 line_end = cursor; + while (line_end < toml_data.size && toml_data.str[line_end] != '\n') { line_end += 1; } + String8 line = str8_skip_chop_whitespace(str8_substr(toml_data, r1u64(cursor, line_end))); + cursor = line_end + 1; + + if (line.size == 0 || line.str[0] == '#') { continue; } + + if (str8_match(str8_prefix(line, 4), str8_lit("name"), 0)) { + // name = ["quote", ''] -- extract the last single-quoted token + U64 q0 = str8_find_needle(line, 0, str8_lit("'"), 0); + if (q0 < line.size) { + U64 q1 = str8_find_needle(line, q0 + 1, str8_lit("'"), 0); + if (q1 < line.size) { + cur_name = str8_substr(line, r1u64(q0 + 1, q1)); + } + } + } else if (str8_match(str8_prefix(line, 3), str8_lit("ifc"), 0)) { + U64 q0 = str8_find_needle(line, 0, str8_lit("\""), 0); + if (q0 < line.size && cur_name.size) { + U64 q1 = str8_find_needle(line, q0 + 1, str8_lit("\""), 0); + if (q1 < line.size) { + String8 ifc_path = str8_substr(line, r1u64(q0 + 1, q1)); + // key by basename of the header-unit path (matches LF_IFC_RECORD header_unit_path basename) + String8 base = str8_skip_last_slash(cur_name); + // header-unit paths use backslashes; normalize to last path component + U64 bs = str8_find_needle_reverse(base, 0, str8_lit("\\"), 0); + if (bs) { base = str8_skip(base, bs); } + LNK_IfcMapEntry *e = push_array(arena, LNK_IfcMapEntry, 1); + e->ifc_path = push_str8_copy(arena, ifc_path); + hash_map_push_string_raw(arena, hm, push_str8_copy(arena, base), e); + } + } + cur_name = str8_zero(); + } + } +} + +// Reads every /ifcMap toml, materializes the union of header-unit basename -> .ifc path. +internal HashMap +lnk_build_ifc_map(Arena *arena, LNK_Config *config) +{ + HashMap hm = {0}; + Temp scratch = scratch_begin(&arena, 1); + for EachNode(n, String8Node, config->ifc_map_list.first) { + String8 toml = lnk_read_data_from_file_path(scratch.arena, 0, n->string); + if (toml.size == 0) { + lnk_error(LNK_Error_Cmdl, "/ifcMap: unable to read TOML '%S'", n->string); + continue; + } + lnk_parse_ifc_map_toml(arena, &hm, toml); + } + scratch_end(scratch); + return hm; +} + +// LF_IFC_RECORD (0x1522) body layout (header {len,kind} already stripped from leaf.data): +// u16 version (==2); u32 ifc_type_index X; u8[16] guid; u8[16] hash; char[] header_unit_path NUL +typedef struct LNK_IfcRecord +{ + U32 ifc_type_index; // X: TI into the .ifc debug-records blob (base 0x1000) + U8 guid[16]; + U8 hash[16]; + String8 header_unit_path; + B32 is_valid; +} LNK_IfcRecord; + +internal LNK_IfcRecord +lnk_parse_ifc_record(String8 leaf_data) +{ + LNK_IfcRecord rec = {0}; + if (leaf_data.size < 2 + 4 + 16 + 16) { return rec; } + U64 off = 0; + U16 version; off += str8_deserial_read_struct(leaf_data, off, &version); + off += str8_deserial_read_struct(leaf_data, off, &rec.ifc_type_index); + MemoryCopy(rec.guid, leaf_data.str + off, 16); off += 16; + MemoryCopy(rec.hash, leaf_data.str + off, 16); off += 16; + rec.header_unit_path = str8_cstring_capped(leaf_data.str + off, leaf_data.str + leaf_data.size); + rec.is_valid = 1; + return rec; +} + +// Injects referenced .ifc debug-records blobs as extra "objs" in `input`, scans every +// consuming obj's .debug$T for LF_IFC_RECORD (0x1522) leaves, registers each placeholder +// local TI -> blob leaf redirect, and rewrites the 0x1522 leaf to NOTYPE so it is excluded +// from the output TPI. Must run after .debug$T is parsed and before min-type-index / symbol +// setup (which iterate input->count). +internal void +lnk_apply_ifc_debug_records(TP_Arena *tp_arena, LNK_CodeViewInput *input, LNK_Config *config) +{ + ProfBeginFunction(); + Temp scratch = scratch_begin(&tp_arena->v[0], 1); + Arena *arena = tp_arena->v[0]; + + // basename -> .ifc path + HashMap ifc_map_hm = lnk_build_ifc_map(scratch.arena, config); + + // first pass: discover which .ifc files are actually referenced and by which records. + // de-dup .ifc reads by path; parse blob to a CV_DebugT once per .ifc. + HashMap ifc_path_to_blobidx = {0}; // path -> (blob slot index + 1) + IFC_File *ifc_files = push_array(scratch.arena, IFC_File, 256); + U64 ifc_file_count = 0; + CV_DebugT blob_debug_t[256] = {0}; + + // scan all consuming objs for 0x1522 leaves; collect the .ifc paths needed + for EachIndex(obj_idx, input->obj_count) { + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + for EachIndex(leaf_idx, debug_t->count) { + CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); + if (hdr->kind != 0x1522) { continue; } + + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + LNK_IfcRecord rec = lnk_parse_ifc_record(leaf.data); + if (!rec.is_valid) { continue; } + + String8 base = str8_skip_last_slash(rec.header_unit_path); + U64 bs = str8_find_needle_reverse(base, 0, str8_lit("\\"), 0); + if (bs) { base = str8_skip(base, bs); } + + LNK_IfcMapEntry *e = hash_map_search_string_raw(&ifc_map_hm, base); + if (e == 0) { continue; } + + if (hash_map_search_string_u64(&ifc_path_to_blobidx, e->ifc_path) == 0) { + if (ifc_file_count >= 256) { continue; } + String8 err = {0}; + IFC_File f = ifc_file_read(arena, e->ifc_path, &err); + if (!f.is_valid) { + lnk_error(LNK_Error_Cmdl, "/ifcDebugRecords: %S", err); + // still reserve a slot so we don't retry + hash_map_push_string_u64(scratch.arena, &ifc_path_to_blobidx, e->ifc_path, ifc_file_count + 1); + ifc_files[ifc_file_count] = f; + blob_debug_t[ifc_file_count] = (CV_DebugT){0}; + ifc_file_count += 1; + continue; + } + // parse the raw CV leaf stream (no signature, TI base 0x1000) + CV_DebugT dt = cv_debug_t_from_data(arena, f.debug_records, 1); + hash_map_push_string_u64(scratch.arena, &ifc_path_to_blobidx, e->ifc_path, ifc_file_count + 1); + ifc_files[ifc_file_count] = f; + blob_debug_t[ifc_file_count] = dt; + ifc_file_count += 1; + } + } + } + + if (ifc_file_count == 0) { goto done; } + + // --- inject blob objs into the parallel arrays (like type servers, but in ifc_obj_range) --- + U64 prev_count = input->count; + U64 new_count = prev_count + ifc_file_count; + + LNK_Obj **obj_arr2 = push_array(arena, LNK_Obj *, new_count); + CV_DebugS *debug_s_arr2 = push_array(arena, CV_DebugS, new_count); + CV_DebugT *debug_t_arr2 = push_array(arena, CV_DebugT, new_count); + CV_DebugH *debug_h_arr2 = push_array(arena, CV_DebugH, new_count); + U64 *obj_to_ts2 = push_array(arena, U64, new_count); + + MemoryCopyTyped(obj_arr2, input->obj_arr, prev_count); + MemoryCopyTyped(debug_s_arr2, input->debug_s_arr, prev_count); + MemoryCopyTyped(debug_t_arr2, input->debug_t_arr, prev_count); + MemoryCopyTyped(debug_h_arr2, input->debug_h_arr, prev_count); + MemoryCopyTyped(obj_to_ts2, input->obj_to_ts, prev_count); + MemorySet(obj_to_ts2 + prev_count, 0xff, ifc_file_count * sizeof(U64)); // blobs are not type servers + + // blob obj indices + index list for hash-deep / dedup + U32Array ifc_indices = { .v = push_array(arena, U32, ifc_file_count) }; + for EachIndex(i, ifc_file_count) { + U64 blob_obj_idx = prev_count + i; + LNK_Obj *blob_obj = push_array(arena, LNK_Obj, 1); + blob_obj->path = ifc_files[i].path; + obj_arr2[blob_obj_idx] = blob_obj; + debug_t_arr2[blob_obj_idx] = blob_debug_t[i]; + ifc_indices.v[ifc_indices.count++] = (U32)blob_obj_idx; + } + + input->count = new_count; + input->obj_arr = obj_arr2; + input->debug_s_arr = debug_s_arr2; + input->debug_t_arr = debug_t_arr2; + input->debug_h_arr = debug_h_arr2; + input->obj_to_ts = obj_to_ts2; + input->ifc_obj_range = r1u64(prev_count, new_count); + input->ifc_indices = ifc_indices; // hashed + deduped before int objs (see lnk_merge_types) + + // --- second pass: register redirects + NOTYPE the 0x1522 leaves --- + input->has_ifc_redirects = 1; + U64 redirect_count = 0; + for EachIndex(obj_idx, input->obj_count) { + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + for EachIndex(leaf_idx, debug_t->count) { + CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); + if (hdr->kind != 0x1522) { continue; } + + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + LNK_IfcRecord rec = lnk_parse_ifc_record(leaf.data); + + // the leaf's own local TI K (the placeholder other leaves reference) + CV_TypeIndex K = cv_ti_from_leaf_idx(debug_t, CV_TypeIndexSource_TPI, leaf_idx); + + B32 resolved = 0; + if (rec.is_valid) { + String8 base = str8_skip_last_slash(rec.header_unit_path); + U64 bs = str8_find_needle_reverse(base, 0, str8_lit("\\"), 0); + if (bs) { base = str8_skip(base, bs); } + LNK_IfcMapEntry *e = hash_map_search_string_raw(&ifc_map_hm, base); + if (e) { + U64 *slot = hash_map_search_string_u64(&ifc_path_to_blobidx, e->ifc_path); + if (slot) { + U64 blob_i = *slot - 1; + IFC_File *f = &ifc_files[blob_i]; + U64 blob_obj_idx = input->ifc_obj_range.min + blob_i; + CV_DebugT *bdt = &input->debug_t_arr[blob_obj_idx]; + + // verify GUID++hash matches this .ifc's content hash (32 bytes) + B32 hash_ok = MemoryMatch(rec.guid, f->content_hash, 16) && + MemoryMatch(rec.hash, f->content_hash + 16, 16); + + if (f->is_valid && hash_ok) { + U64 blob_leaf_idx = cv_leaf_idx_from_ti(bdt, CV_TypeIndexSource_TPI, rec.ifc_type_index); + if (blob_leaf_idx < bdt->count) { + hash_map_push_u64_u64(arena, &input->ifc_redirect_hm, + Compose64Bit(obj_idx, K), + Compose64Bit(blob_obj_idx, blob_leaf_idx)); + redirect_count += 1; + resolved = 1; + } + } + } + } + } + (void)resolved; + + // exclude the placeholder leaf from output regardless: rewrite to NOTYPE. + // NOTYPE routes to CV_TypeIndexSource_NULL which is never emitted to TPI/IPI. + memory_write16(MemberFromPtr(CV_LeafHeader, hdr, kind), (U16)CV_LeafKind_NOTYPE); + } + } + + lnk_log(LNK_Log_Debug, "[IFC] injected %llu .ifc blob(s), %llu record redirect(s)", ifc_file_count, redirect_count); + +done: + scratch_end(scratch); + ProfEnd(); +} + internal LNK_CodeViewInput lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 obj_count, LNK_Obj **obj_arr, LNK_RRT_Array rrt_input) { @@ -1099,6 +1363,12 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, } ProfEnd(); + // resolve MSVC header-unit IFC debug records (LF_IFC_RECORD 0x1522) -> real CodeView types. + // injects .ifc debug-records blobs as extra objs and registers placeholder-TI redirects. + if (config->ifc_debug_records == LNK_SwitchState_Yes && config->ifc_map_list.node_count) { + lnk_apply_ifc_debug_records(tp_arena, &input, config); + } + // set default min type index for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { input.min_type_indices[ti_source] = CV_MinComplexTypeIndex; } @@ -1213,6 +1483,16 @@ lnk_leaf_ref_leaf_idx(LNK_LeafRef ref) internal LNK_LeafRef lnk_leaf_ref_from_ti(LNK_CodeViewInput *input, U32 obj_idx, CV_TypeIndexSource source, CV_TypeIndex ti) { + // IFC redirect: a consuming obj's local LF_IFC_RECORD placeholder TI is mapped + // to a leaf inside an injected .ifc debug-records blob obj. The blob leaves then + // dedup/hash/fixup natively through the rest of this function. + if (input->has_ifc_redirects && source == CV_TypeIndexSource_TPI) { + U64 *packed = hash_map_search_u64_u64(&input->ifc_redirect_hm, Compose64Bit(obj_idx, ti)); + if (packed) { + return (LNK_LeafRef){ (U32)(*packed >> 32), (U32)(*packed & max_U32) }; + } + } + // ti range: external type server U64 ts_idx = input->obj_to_ts[obj_idx]; if (ts_idx != max_U64) { @@ -1882,6 +2162,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK U32Array indices; U32Array hash_indices; } hash_targets[] = { + { lnk_hash_debug_t_deep_task, input->ifc_indices }, // hash .ifc blobs first: int-obj leaves redirect into them { lnk_hash_debug_t_task, input->debug_p_indices }, // hash .debug$P first so we can mix in hashes for precompiled sub leaves when hashing leaves in .debug$T { lnk_hash_debug_t_task, input->int_obj_indices }, { lnk_hash_debug_t_deep_task, input->type_server_indices }, @@ -1991,6 +2272,9 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.indices = dedup_type_server_indices; tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, "Type Servers"); + + task.indices = input->ifc_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, "IFC Blobs"); ProfEnd(); ProfBegin("Extract present buckets from the leaf hash tables"); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index bbf8b6603..53974fa71 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -108,6 +108,16 @@ typedef struct Rng1U64 *symbol_input_ranges; // [worker_count] U64 symbol_patch_task_count; // LNK_SymbolInputTask *symbol_patch_task; // [symbol_patch_task_count] + + // IFC (header-unit debug-record) resolution: + // redirects a consuming obj's local LF_IFC_RECORD placeholder TI to a leaf in + // an injected .ifc debug-records blob "obj". Consulted first in lnk_leaf_ref_from_ti. + // key = Compose64Bit(obj_idx, local_ti) + // value = Compose64Bit(blob_obj_idx, blob_leaf_idx) + B32 has_ifc_redirects; + HashMap ifc_redirect_hm; + Rng1U64 ifc_obj_range; // [min,max) range of injected blob objs in the parallel arrays + U32Array ifc_indices; // obj indices of injected .ifc blob objs (hashed/deduped first) } LNK_CodeViewInput; typedef struct From 636cf4320b308d05ae95f8bb070d920d8654162c Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 17 Jul 2026 17:22:21 -0700 Subject: [PATCH 008/121] linker: parallelize apply_ifc nonblob_complete set merge + ifc parse The IFC apply phase merged the per-worker nonblob_complete sets and parsed .ifc inputs serially on the main thread. Run the set merge and the ifc parse as pool tasks. Applied record set and output bytes unchanged (gated byte-identical). Co-Authored-By: Claude Opus 4.8 (1M context) Co-Authored-By: Claude Fable 5 (cherry picked from commit 86c48d32f40314a644a67d3905cf9623605ed432) radlink: header-unit IFC debug-record resolution Resolve MSVC C++ header-unit IFC debug records (LF_IFC_RECORD 0x1522) by merging the .ifc .msvc.trait.debug-records CodeView stream and redirecting each record to its real type -> fixes VS debugger AV stepping into header- unit code (BAD 921->0, 44/44 MSVC name-match, live debug verified). (cherry picked from commit 4b7f8bf8, ICF leader-keying half dropped: superseded by upstream d0d8b596's own /OPT:ICF) Co-Authored-By: Claude Opus 4.8 (1M context) Co-Authored-By: Claude Fable 5 radlink: skip redundant library re-searches in the resolution fixpoint lnk_link_inputs resolves libraries to a fixpoint: an outer pass loops over every library, and each library is re-searched (a full tp_for_parallel over all workers, scanning every undefined/weak symbol in search_chunks) once per drained input batch until nothing new resolves. On the UE editor link that is ~2733 dispatches, each waking and joining ~60 workers -- and the phase is barrier-bound, so that wake/join is the cost, not the scan. Most re-searches are redundant: search_chunks only grows during the loop (symbols are never removed until the end) and member-queue dedup is idempotent, so a re-search can only queue new members if the undefined/weak symbol set grew or anti-dep searching was just enabled since this library was last searched. Stamp each LNK_Lib with the search_chunks symbol count + anti-dep mode at its last search and skip the dispatch when neither changed. ~24% fewer dispatches (2733 -> ~2089) and ~0.2s of wake/join wall-time removed. Output is byte-identical and reproducible (which dispatch coalesces is timing dependent, but a skipped one provably queued nothing, so the result is unchanged -- verified relink-twice byte-identical across many runs). Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 3054138a4a333f743cd214178bcf084e83c1cff8) radlink: cache symbol interp so the lib search stops re-parsing resolved symbols (squashed with: exact per-obj bitset filter for ifc_redirect_hm lookups; parallelize the IFC apply discovery/read+parse/resolution -- the three were one logical change split by a rebase) --- src/linker/lnk_debug_info.c | 630 ++++++++++++++++++++++++++++++------ src/linker/lnk_debug_info.h | 5 + 2 files changed, 542 insertions(+), 93 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 190086f82..0d65f6dc4 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -661,7 +661,9 @@ lnk_rrt_array_from_config(Arena *arena, LNK_Config *config) typedef struct LNK_IfcMapEntry { - String8 ifc_path; // absolute .ifc path + String8 ifc_path; // absolute .ifc path + U64 blob_slot; // resolved blob slot + 1; 0 = not yet resolved (filled by the serial + // discovery replay in lnk_apply_ifc_debug_records, memoizes path lookups) } LNK_IfcMapEntry; // Parse the trivial /ifcMap TOML by hand: @@ -757,70 +759,440 @@ lnk_parse_ifc_record(String8 leaf_data) return rec; } +// Per-blob closure + NOTYPE-prune (third pass of lnk_apply_ifc_debug_records). Each blob is fully +// independent: it reads/writes only its own ref_bits[blob_i] and its own blob DebugT leaves, so the +// work parallelizes across the ~13 blobs with no shared state. The worklist scratch comes from the +// per-worker arena. Closure counts are written per-blob and summed afterward (order-independent). +// Open-addressing U64 hash set keyed by unique_name hash. Used to record which UDT unique_names +// already have a COMPLETE definition in a non-blob (consuming) obj, so the blob prune can keep a +// blob's complete definition only for names that NO normal obj completes (blob-only types). cap is +// a power of two; 0-hash is reserved as the empty sentinel (we OR in a bit so a real 0 can't occur). +typedef struct LNK_U64Set { U64 *slots; U64 cap; } LNK_U64Set; + +internal U64 +lnk_uname_hash(String8 s) +{ + U64 h = 5381; + for EachIndex(c, s.size) { h = ((h << 5) + h) ^ (U64)s.str[c]; } + return h | 1; // never 0 (0 is the empty sentinel) +} + +internal void +lnk_u64set_add(LNK_U64Set *set, U64 h) +{ + U64 i = h & (set->cap - 1); + for (;;) { + if (set->slots[i] == 0) { set->slots[i] = h; return; } + if (set->slots[i] == h) { return; } + i = (i + 1) & (set->cap - 1); + } +} + +// Thread-safe insert mirroring lnk_icf_map_put_atomic (lnk.c). The empty sentinel is 0 (not +// LNK_ICF_EMPTY); h is guaranteed nonzero by lnk_uname_hash (`| 1`) so a real key can never be 0. +// Claim an empty slot with an atomic CAS 0->h; the CAS winner owns it. On a lost race re-read the +// same slot (it may now hold our h via a duplicate, or another key) before advancing. Duplicate +// keys across objs are legal and idempotent (a slot already holding h returns), so any insertion +// order yields the identical final membership -- the set is read-only (lnk_u64set_has) afterward. +internal void +lnk_u64set_add_atomic(LNK_U64Set *set, U64 h) +{ + U64 i = h & (set->cap - 1); + for (;;) { + if (set->slots[i] == 0) { + if (ins_atomic_u64_eval_cond_assign(&set->slots[i], h, 0) == 0) { return; } + continue; + } + if (set->slots[i] == h) { return; } + i = (i + 1) & (set->cap - 1); + } +} + +internal B32 +lnk_u64set_has(LNK_U64Set *set, U64 h) +{ + U64 i = h & (set->cap - 1); + for (;;) { + if (set->slots[i] == 0) { return 0; } + if (set->slots[i] == h) { return 1; } + i = (i + 1) & (set->cap - 1); + } +} + +// Parallel collect of complete-definition unique_name hashes per non-blob obj. Each obj is scanned +// independently (read-only over its .debug$T leaves) and emits its hashes into a per-obj list; a +// serial pass then adds them to the shared open-addressing set. Moves the ~1.25s serial cv_get_udt_info +// scan off the main thread. Determinism: set membership is order-independent, serial-add reproduces. +typedef struct LNK_IfcCompleteScanTask +{ + LNK_CodeViewInput *input; + U64 **out_hashes; // per-obj hash array (allocated by task) + U64 *out_counts; // per-obj count +} LNK_IfcCompleteScanTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_complete_scan_task) +{ + LNK_IfcCompleteScanTask *task = raw_task; + U64 obj_idx = task_id; + CV_DebugT *dt = &task->input->debug_t_arr[obj_idx]; + U64 *hashes = push_array_no_zero(arena, U64, dt->count ? dt->count : 1); + U64 n = 0; + for EachIndex(leaf_idx, dt->count) { + CV_Leaf leaf = cv_debug_t_get_leaf(dt, leaf_idx); + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (!(ui.props & CV_TypeProp_HasUniqueName) || ui.unique_name.size == 0) { continue; } + if (ui.props & CV_TypeProp_FwdRef) { continue; } + hashes[n++] = lnk_uname_hash(ui.unique_name); + } + task->out_hashes[obj_idx] = hashes; + task->out_counts[obj_idx] = n; +} + +// Parallel merge of the per-obj complete-def hash lists into the shared set. Replaces the serial +// lnk_u64set_add loop (the ~914ms hotspot -- 801ms of it first-touch KiPageFault on a single thread +// faulting a 128MB+ set). lnk_u64set_add_atomic spreads both the random-scatter probes AND the +// page faults across the pool. Determinism: keys may legitimately duplicate across objs, but the +// insert is idempotent and the set is read-only afterward (lnk_u64set_has), so insertion order +// cannot change the final membership -- output is bit-identical to the serial merge. +typedef struct LNK_IfcSetMergeTask +{ + Rng1U64 *ranges; + U64 **out_hashes; + U64 *out_counts; + LNK_U64Set *set; + U64 nonblob_count; +} LNK_IfcSetMergeTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_set_merge_task) +{ + LNK_IfcSetMergeTask *task = raw_task; + for EachInRange(obj_idx, task->ranges[task_id]) { + U64 *h = task->out_hashes[obj_idx]; + U64 n = task->out_counts[obj_idx]; + for EachIndex(t, n) { lnk_u64set_add_atomic(task->set, h[t]); } + } +} + +// Fused discovery+redirect scan (passes 1+2 of lnk_apply_ifc_debug_records): each consuming +// obj's .debug$T is swept ONCE, in parallel, for 0x1522 (LF_IFC_RECORD) leaves. The worker +// parses each record, resolves its header-unit basename against the read-only ifc_map_hm, and +// emits one raw record per 0x1522 leaf in ascending leaf_idx order. Workers write NOTHING +// (no NOTYPE, no discovery, no redirects): every order-sensitive effect -- .ifc first-encounter +// slot assignment, NOTYPE rewrites, redirect hash-map push order, ref_bits seeding -- is +// replayed SERIALLY from these records in ascending obj_idx, then ascending leaf_idx: the exact +// order of the original serial passes, so output is byte-for-byte identical. +typedef struct LNK_IfcRawRec +{ + U64 leaf_idx; // 0x1522 leaf index inside the consuming obj + CV_TypeIndex K; // consuming obj's local placeholder TI + U32 ifc_type_index; // X: TI into the .ifc blob (base 0x1000) + U8 guid[16]; + U8 hash[16]; + LNK_IfcMapEntry *entry; // basename -> map entry (0: invalid record or no map hit) + U64 blob_i_plus1; // resolved blob slot + 1 (serial replay fills; 0 = unresolved) + U64 blob_leaf_idx; // resolved leaf inside the blob (serial replay fills) +} LNK_IfcRawRec; + +typedef struct LNK_IfcScanTask +{ + LNK_CodeViewInput *input; + HashMap *ifc_map_hm; // read-only in workers + LNK_IfcRawRec **out_recs; // per-obj ordered raw records (allocated by task) + U64 *out_counts; // per-obj record count +} LNK_IfcScanTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_scan_task) +{ + LNK_IfcScanTask *task = raw_task; + U64 obj_idx = task_id; + LNK_CodeViewInput *input = task->input; + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + + // count 0x1522 leaves first to size the per-obj record array + U64 ifc_leaf_count = 0; + for EachIndex(leaf_idx, debug_t->count) { + CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); + if (hdr->kind == 0x1522) { ifc_leaf_count += 1; } + } + if (ifc_leaf_count == 0) { task->out_recs[obj_idx] = 0; task->out_counts[obj_idx] = 0; return; } + + LNK_IfcRawRec *recs = push_array_no_zero(arena, LNK_IfcRawRec, ifc_leaf_count); + U64 n = 0; + + for EachIndex(leaf_idx, debug_t->count) { + CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); + if (hdr->kind != 0x1522) { continue; } + + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + LNK_IfcRecord rec = lnk_parse_ifc_record(leaf.data); + + LNK_IfcRawRec *r = &recs[n++]; + r->leaf_idx = leaf_idx; + r->K = cv_ti_from_leaf_idx(debug_t, CV_TypeIndexSource_TPI, leaf_idx); + r->ifc_type_index = rec.ifc_type_index; + r->entry = 0; + r->blob_i_plus1 = 0; + r->blob_leaf_idx = 0; + + if (rec.is_valid) { + MemoryCopy(r->guid, rec.guid, 16); + MemoryCopy(r->hash, rec.hash, 16); + String8 base = str8_skip_last_slash(rec.header_unit_path); + U64 bs = str8_find_needle_reverse(base, 0, str8_lit("\\"), 0); + if (bs) { base = str8_skip(base, bs); } + r->entry = hash_map_search_string_raw(task->ifc_map_hm, base); + } + } + + task->out_recs[obj_idx] = recs; + task->out_counts[obj_idx] = n; +} + +// Parallel per-obj record resolution + placeholder NOTYPE (runs after discovery/read/injection, +// when entry->blob_slot, ifc_files, and the injected blob debug_t entries are all frozen/read-only). +// Each worker fills its own obj's raw records in place (blob_i_plus1/blob_leaf_idx), rewrites its +// own 0x1522 leaves to NOTYPE (per-obj disjoint, constant value -- order-free), and reports the +// resolved count + K range. The serial replay below then only pushes redirects in the original +// (obj_idx, leaf_idx) order, so hash-map push order and all outputs stay bit-identical. +typedef struct LNK_IfcResolveTask +{ + LNK_CodeViewInput *input; + IFC_File *ifc_files; + LNK_IfcRawRec **recs; // per-obj raw records from the scan + U64 *counts; // per-obj record count + U64 *res_counts; // out: per-obj resolved record count + U64 *k_first; // out: first resolved K (valid when res_counts != 0) + U64 *k_last; // out: last resolved K (valid when res_counts != 0) +} LNK_IfcResolveTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_resolve_task) +{ + LNK_IfcResolveTask *task = raw_task; + U64 obj_idx = task_id; + LNK_CodeViewInput *input = task->input; + LNK_IfcRawRec *recs = task->recs[obj_idx]; + U64 n = task->counts[obj_idx]; + if (n == 0) { task->res_counts[obj_idx] = 0; return; } + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + + U64 res_count = 0; + U64 k_first = 0, k_last = 0; + for EachIndex(t, n) { + LNK_IfcRawRec *r = &recs[t]; + CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, r->leaf_idx); + // exclude the placeholder leaf from output regardless: rewrite to NOTYPE + memory_write16(MemberFromPtr(CV_LeafHeader, hdr, kind), (U16)CV_LeafKind_NOTYPE); + if (r->entry == 0 || r->entry->blob_slot == 0) { continue; } + U64 blob_i = r->entry->blob_slot - 1; + IFC_File *f = &task->ifc_files[blob_i]; + B32 hash_ok = MemoryMatch(r->guid, f->content_hash, 16) && + MemoryMatch(r->hash, f->content_hash + 16, 16); + if (!f->is_valid || !hash_ok) { continue; } + CV_DebugT *bdt = &input->debug_t_arr[input->ifc_obj_range.min + blob_i]; + U64 blob_leaf_idx = cv_leaf_idx_from_ti(bdt, CV_TypeIndexSource_TPI, r->ifc_type_index); + if (blob_leaf_idx >= bdt->count) { continue; } + r->blob_i_plus1 = blob_i + 1; + r->blob_leaf_idx = blob_leaf_idx; + if (res_count == 0) { k_first = r->K; } + k_last = r->K; + res_count += 1; + } + task->res_counts[obj_idx] = res_count; + task->k_first[obj_idx] = k_first; + task->k_last[obj_idx] = k_last; +} + +// Parallel .ifc read + `.msvc.trait.debug-records` parse into PRE-ASSIGNED slots. Slot order +// (== blob obj order == output order) is fixed by the serial discovery replay before any file +// is read, so going wide here cannot reorder anything. Workers do not call lnk_error: read +// failures are collected per slot and reported serially in slot order afterward (identical +// message order to the old serial read; LNK_Error_Cmdl stops the link either way). Worker-arena +// allocations (file bytes + leaf offsets) are long-lived, same as the parallel .debug$T parse +// (lnk_parse_debug_t_task pattern). +typedef struct LNK_IfcReadTask +{ + String8 *paths; // per-slot .ifc path + IFC_File *ifc_files; // per-slot output + CV_DebugT *blob_debug_t; // per-slot output + String8 *errors; // per-slot read error (size 0 = ok) +} LNK_IfcReadTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_read_task) +{ + LNK_IfcReadTask *task = raw_task; + U64 slot = task_id; + String8 err = {0}; + IFC_File f = ifc_file_read(arena, task->paths[slot], &err); + task->ifc_files[slot] = f; + task->errors[slot] = err; + if (f.is_valid) { + // parse the raw CV leaf stream (no signature, TI base 0x1000) + task->blob_debug_t[slot] = cv_debug_t_from_data(arena, f.debug_records, 1); + } else { + MemoryZeroStruct(&task->blob_debug_t[slot]); + } +} + +typedef struct LNK_IfcCloseTask +{ + LNK_CodeViewInput *input; + U8 **ref_bits; + U64 *closure_leaves; // per-blob output: # leaves surviving in closure + LNK_U64Set *nonblob_complete; // unique_name hashes completed by some non-blob obj +} LNK_IfcCloseTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_close_blob_task) +{ + LNK_IfcCloseTask *task = raw_task; + U64 blob_i = task_id; + LNK_CodeViewInput *input = task->input; + U64 blob_obj_idx = input->ifc_obj_range.min + blob_i; + CV_DebugT *bdt = &input->debug_t_arr[blob_obj_idx]; + U8 *bits = task->ref_bits[blob_i]; + U64 closure = 0; + if (bdt->count == 0) { task->closure_leaves[blob_i] = 0; return; } + + Temp wtemp = temp_begin(arena); + + // Extra closure roots for forward-ref completion: in CodeView a forward-ref UDT is completed by + // ANY same-unique_name complete definition in the PDB. A consuming obj typically emits only a + // forward-ref of a header-unit type; the full-merge build incidentally kept the matching complete + // definition from the .ifc blob, so the debugger could complete it. On-demand would drop that + // definition (nothing references it by TI), leaving the type incomplete vs full-merge. To preserve + // fidelity WITHOUT dragging the whole blob, root every blob complete-def UDT whose unique_name has + // NO complete definition in any non-blob obj (i.e. blob-only types -- trait/delegate marker structs + // etc.). Common types (FString, FGuid, ...) are completed by normal objs, so their redundant blob + // copies stay pruned. Members of the kept defs are pulled by the closure walk below. + for EachIndex(leaf_idx, bdt->count) { + if (bits[leaf_idx >> 3] & (1u << (leaf_idx & 7))) { continue; } // already a root + CV_Leaf leaf = cv_debug_t_get_leaf(bdt, leaf_idx); + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (!(ui.props & CV_TypeProp_HasUniqueName) || ui.unique_name.size == 0) { continue; } + if (ui.props & CV_TypeProp_FwdRef) { continue; } // only complete definitions + U64 h = lnk_uname_hash(ui.unique_name); + if (lnk_u64set_has(task->nonblob_complete, h)) { continue; } // a normal obj already completes it + bits[leaf_idx >> 3] |= (U8)(1u << (leaf_idx & 7)); + } + + U64 *worklist = push_array_no_zero(wtemp.arena, U64, bdt->count); + U64 wl_count = 0; + for EachIndex(leaf_idx, bdt->count) { + if (bits[leaf_idx >> 3] & (1u << (leaf_idx & 7))) { worklist[wl_count++] = leaf_idx; } + } + + while (wl_count) { + U64 leaf_idx = worklist[--wl_count]; + CV_Leaf leaf = cv_debug_t_get_leaf(bdt, leaf_idx); + Temp itemp = temp_begin(wtemp.arena); + CV_TypeIndexInfoList ti_list = cv_get_leaf_type_index_offsets(itemp.arena, leaf.kind, leaf.data); + for EachNode(ti_info, CV_TypeIndexInfo, ti_list.first) { + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(leaf.data, ti_info->offset, sizeof(*ti_ptr)); + if (ti_ptr == 0) { continue; } + CV_TypeIndex sub_ti = memory_read32(ti_ptr); + if (sub_ti < bdt->ti_ranges[ti_info->source].min || + sub_ti >= bdt->ti_ranges[ti_info->source].max) { continue; } + U64 sub_leaf_idx = cv_leaf_idx_from_ti(bdt, ti_info->source, sub_ti); + if (sub_leaf_idx >= bdt->count) { continue; } + if (bits[sub_leaf_idx >> 3] & (1u << (sub_leaf_idx & 7))) { continue; } + bits[sub_leaf_idx >> 3] |= (U8)(1u << (sub_leaf_idx & 7)); + worklist[wl_count++] = sub_leaf_idx; + } + temp_end(itemp); + } + temp_end(wtemp); + + for EachIndex(leaf_idx, bdt->count) { + if (bits[leaf_idx >> 3] & (1u << (leaf_idx & 7))) { closure += 1; continue; } + CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(bdt, leaf_idx); + if (hdr->kind == CV_LeafKind_NOTYPE) { continue; } + memory_write16(MemberFromPtr(CV_LeafHeader, hdr, kind), (U16)CV_LeafKind_NOTYPE); + memory_write16(MemberFromPtr(CV_LeafHeader, hdr, size), (U16)sizeof(CV_LeafKind)); + } + task->closure_leaves[blob_i] = closure; +} + // Injects referenced .ifc debug-records blobs as extra "objs" in `input`, scans every // consuming obj's .debug$T for LF_IFC_RECORD (0x1522) leaves, registers each placeholder // local TI -> blob leaf redirect, and rewrites the 0x1522 leaf to NOTYPE so it is excluded // from the output TPI. Must run after .debug$T is parsed and before min-type-index / symbol // setup (which iterate input->count). internal void -lnk_apply_ifc_debug_records(TP_Arena *tp_arena, LNK_CodeViewInput *input, LNK_Config *config) +lnk_apply_ifc_debug_records(TP_Context *tp, TP_Arena *tp_arena, LNK_CodeViewInput *input, LNK_Config *config) { ProfBeginFunction(); + U64 apply_begin_us = now_time_us(); Temp scratch = scratch_begin(&tp_arena->v[0], 1); Arena *arena = tp_arena->v[0]; // basename -> .ifc path HashMap ifc_map_hm = lnk_build_ifc_map(scratch.arena, config); - - // first pass: discover which .ifc files are actually referenced and by which records. - // de-dup .ifc reads by path; parse blob to a CV_DebugT once per .ifc. + U64 discover_begin_us = now_time_us(); + + // --- fused scan (old passes 1+2, parallel): ONE sweep of every consuming obj's .debug$T + // emits per-obj raw 0x1522 records in ascending leaf_idx order (record parse + basename -> + // ifc_map_hm entry resolution happen in the workers; nothing is written). Every + // order-sensitive effect is replayed serially from these records below. --- + LNK_IfcScanTask scan = {0}; + scan.input = input; + scan.ifc_map_hm = &ifc_map_hm; + scan.out_recs = push_array(scratch.arena, LNK_IfcRawRec *, input->obj_count); + scan.out_counts = push_array(scratch.arena, U64, input->obj_count); + tp_for_parallel(tp, tp_arena, input->obj_count, lnk_ifc_scan_task, &scan); + U64 scan_end_us = now_time_us(); + lnk_log(LNK_Log_Timers, "[IFC] parallel scan in %.2f ms", (F64)(scan_end_us - discover_begin_us) / 1000.0); + + // --- serial discovery replay: assign .ifc blob slots in first-encounter order (ascending + // obj_idx, then ascending leaf_idx -- identical to the old serial pass) WITHOUT reading any + // file, so the reads can go wide below. De-dup by path; entry->blob_slot memoizes the path + // lookup. 256-slot cap semantics preserved: on overflow the entry stays unresolved and its + // records never redirect. --- HashMap ifc_path_to_blobidx = {0}; // path -> (blob slot index + 1) IFC_File *ifc_files = push_array(scratch.arena, IFC_File, 256); U64 ifc_file_count = 0; CV_DebugT blob_debug_t[256] = {0}; - - // scan all consuming objs for 0x1522 leaves; collect the .ifc paths needed + String8 slot_paths[256] = {0}; for EachIndex(obj_idx, input->obj_count) { - CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; - for EachIndex(leaf_idx, debug_t->count) { - CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); - if (hdr->kind != 0x1522) { continue; } - - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); - LNK_IfcRecord rec = lnk_parse_ifc_record(leaf.data); - if (!rec.is_valid) { continue; } - - String8 base = str8_skip_last_slash(rec.header_unit_path); - U64 bs = str8_find_needle_reverse(base, 0, str8_lit("\\"), 0); - if (bs) { base = str8_skip(base, bs); } - - LNK_IfcMapEntry *e = hash_map_search_string_raw(&ifc_map_hm, base); - if (e == 0) { continue; } - - if (hash_map_search_string_u64(&ifc_path_to_blobidx, e->ifc_path) == 0) { + LNK_IfcRawRec *recs = scan.out_recs[obj_idx]; + U64 n = scan.out_counts[obj_idx]; + for EachIndex(t, n) { + LNK_IfcMapEntry *e = recs[t].entry; + if (e == 0 || e->blob_slot) { continue; } + U64 *slot = hash_map_search_string_u64(&ifc_path_to_blobidx, e->ifc_path); + if (slot == 0) { if (ifc_file_count >= 256) { continue; } - String8 err = {0}; - IFC_File f = ifc_file_read(arena, e->ifc_path, &err); - if (!f.is_valid) { - lnk_error(LNK_Error_Cmdl, "/ifcDebugRecords: %S", err); - // still reserve a slot so we don't retry - hash_map_push_string_u64(scratch.arena, &ifc_path_to_blobidx, e->ifc_path, ifc_file_count + 1); - ifc_files[ifc_file_count] = f; - blob_debug_t[ifc_file_count] = (CV_DebugT){0}; - ifc_file_count += 1; - continue; - } - // parse the raw CV leaf stream (no signature, TI base 0x1000) - CV_DebugT dt = cv_debug_t_from_data(arena, f.debug_records, 1); hash_map_push_string_u64(scratch.arena, &ifc_path_to_blobidx, e->ifc_path, ifc_file_count + 1); - ifc_files[ifc_file_count] = f; - blob_debug_t[ifc_file_count] = dt; + slot_paths[ifc_file_count] = e->ifc_path; ifc_file_count += 1; + slot = hash_map_search_string_u64(&ifc_path_to_blobidx, e->ifc_path); } + e->blob_slot = *slot; } } if (ifc_file_count == 0) { goto done; } + U64 discover_end_us = now_time_us(); + lnk_log(LNK_Log_Timers, "[IFC] discover replay in %.2f ms", (F64)(discover_end_us - scan_end_us) / 1000.0); + + // --- parallel .ifc read + debug-records parse into the pre-assigned slots; report read + // errors serially in slot order (identical message order to the old serial read). --- + { + LNK_IfcReadTask read = {0}; + read.paths = slot_paths; + read.ifc_files = ifc_files; + read.blob_debug_t = blob_debug_t; + read.errors = push_array(scratch.arena, String8, ifc_file_count); + tp_for_parallel(tp, tp_arena, ifc_file_count, lnk_ifc_read_task, &read); + for EachIndex(i, ifc_file_count) { + if (!ifc_files[i].is_valid) { lnk_error(LNK_Error_Cmdl, "/ifcDebugRecords: %S", read.errors[i]); } + } + lnk_log(LNK_Log_Timers, "[IFC] read+parse %llu blob(s) in %.2f ms", ifc_file_count, (F64)(now_time_us() - discover_end_us) / 1000.0); + } // --- inject blob objs into the parallel arrays (like type servers, but in ifc_obj_range) --- U64 prev_count = input->count; @@ -859,64 +1231,127 @@ lnk_apply_ifc_debug_records(TP_Arena *tp_arena, LNK_CodeViewInput *input, LNK_Co input->ifc_obj_range = r1u64(prev_count, new_count); input->ifc_indices = ifc_indices; // hashed + deduped before int objs (see lnk_merge_types) - // --- second pass: register redirects + NOTYPE the 0x1522 leaves --- - input->has_ifc_redirects = 1; - U64 redirect_count = 0; + // --- on-demand pruning state: per blob, a "referenced" bitset of leaf indices that + // are reachable from some consuming obj's 0x1522 redirect (the closure roots). Only these + // + their transitive blob-internal deps get merged; the rest are rewritten to NOTYPE so the + // hash/dedup pipeline skips ~all of the ~1.5M blob leaves that nothing references. --- + U8 **ref_bits = push_array(scratch.arena, U8 *, ifc_file_count); + for EachIndex(i, ifc_file_count) { + U64 c = blob_debug_t[i].count; + ref_bits[i] = push_array(scratch.arena, U8, (c + 7) / 8); // zero-init -> nothing referenced yet + } + + // --- parallel record resolution + placeholder NOTYPE: per-obj disjoint, order-free (see + // lnk_ifc_resolve_task). All inputs (entry->blob_slot, ifc_files, blob debug_t) are frozen + // after the discovery/read/injection steps above. --- + input->has_ifc_redirects = 1; + input->ifc_redirect_bits = push_array(arena, U64 *, input->count); + input->ifc_redirect_ti_rng = push_array(arena, Rng1U64, input->count); + U64 redirect_count = 0; + U64 resolve_begin_us = now_time_us(); + LNK_IfcResolveTask resolve = {0}; + resolve.input = input; + resolve.ifc_files = ifc_files; + resolve.recs = scan.out_recs; + resolve.counts = scan.out_counts; + resolve.res_counts = push_array(scratch.arena, U64, input->obj_count); + resolve.k_first = push_array(scratch.arena, U64, input->obj_count); + resolve.k_last = push_array(scratch.arena, U64, input->obj_count); + tp_for_parallel(tp, 0, input->obj_count, lnk_ifc_resolve_task, &resolve); + lnk_log(LNK_Log_Timers, "[IFC] parallel resolve in %.2f ms", (F64)(now_time_us() - resolve_begin_us) / 1000.0); + + // --- serial redirect replay: push redirects in ascending obj_idx, then ascending leaf_idx -- + // the exact original serial order -- so the redirect hash-map push order, ref_bits seeding, + // and redirect_count are bit-for-bit identical to the serial code. + U64 merge_begin_us = now_time_us(); for EachIndex(obj_idx, input->obj_count) { - CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; - for EachIndex(leaf_idx, debug_t->count) { - CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); - if (hdr->kind != 0x1522) { continue; } - - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); - LNK_IfcRecord rec = lnk_parse_ifc_record(leaf.data); - - // the leaf's own local TI K (the placeholder other leaves reference) - CV_TypeIndex K = cv_ti_from_leaf_idx(debug_t, CV_TypeIndexSource_TPI, leaf_idx); - - B32 resolved = 0; - if (rec.is_valid) { - String8 base = str8_skip_last_slash(rec.header_unit_path); - U64 bs = str8_find_needle_reverse(base, 0, str8_lit("\\"), 0); - if (bs) { base = str8_skip(base, bs); } - LNK_IfcMapEntry *e = hash_map_search_string_raw(&ifc_map_hm, base); - if (e) { - U64 *slot = hash_map_search_string_u64(&ifc_path_to_blobidx, e->ifc_path); - if (slot) { - U64 blob_i = *slot - 1; - IFC_File *f = &ifc_files[blob_i]; - U64 blob_obj_idx = input->ifc_obj_range.min + blob_i; - CV_DebugT *bdt = &input->debug_t_arr[blob_obj_idx]; - - // verify GUID++hash matches this .ifc's content hash (32 bytes) - B32 hash_ok = MemoryMatch(rec.guid, f->content_hash, 16) && - MemoryMatch(rec.hash, f->content_hash + 16, 16); - - if (f->is_valid && hash_ok) { - U64 blob_leaf_idx = cv_leaf_idx_from_ti(bdt, CV_TypeIndexSource_TPI, rec.ifc_type_index); - if (blob_leaf_idx < bdt->count) { - hash_map_push_u64_u64(arena, &input->ifc_redirect_hm, - Compose64Bit(obj_idx, K), - Compose64Bit(blob_obj_idx, blob_leaf_idx)); - redirect_count += 1; - resolved = 1; - } - } - } - } - } - (void)resolved; + LNK_IfcRawRec *recs = scan.out_recs[obj_idx]; + U64 n = scan.out_counts[obj_idx]; + if (n == 0 || resolve.res_counts[obj_idx] == 0) { continue; } + + // exact key filter range: records are K-ascending (the scan emits leaf_idx ascending and + // cv_ti_from_leaf_idx is monotonic), so [k_first, k_last] spans all resolved keys. + Rng1U64 krng = r1u64(resolve.k_first[obj_idx], resolve.k_last[obj_idx] + 1); + input->ifc_redirect_ti_rng[obj_idx] = krng; + input->ifc_redirect_bits[obj_idx] = push_array(arena, U64, (dim_1u64(krng) + 63) / 64); + for EachIndex(t, n) { + LNK_IfcRawRec *r = &recs[t]; + if (r->blob_i_plus1 == 0) { continue; } + U64 blob_i = r->blob_i_plus1 - 1; + U64 blob_obj_idx = input->ifc_obj_range.min + blob_i; + hash_map_push_u64_u64(arena, &input->ifc_redirect_hm, + Compose64Bit(obj_idx, r->K), + Compose64Bit(blob_obj_idx, r->blob_leaf_idx)); + U64 rel = r->K - krng.min; + input->ifc_redirect_bits[obj_idx][rel >> 6] |= (1ull << (rel & 63)); + redirect_count += 1; + // seed closure root: this blob leaf is referenced + ref_bits[blob_i][r->blob_leaf_idx >> 3] |= (U8)(1u << (r->blob_leaf_idx & 7)); + } + } - // exclude the placeholder leaf from output regardless: rewrite to NOTYPE. - // NOTYPE routes to CV_TypeIndexSource_NULL which is never emitted to TPI/IPI. - memory_write16(MemberFromPtr(CV_LeafHeader, hdr, kind), (U16)CV_LeafKind_NOTYPE); + lnk_log(LNK_Log_Timers, "[IFC] redirect replay in %.2f ms", (F64)(now_time_us() - merge_begin_us) / 1000.0); + + // --- third pass: per blob, close the referenced set over blob-internal sub-TIs, then + // NOTYPE every leaf not in the closure. cv_leaf_idx_from_ti on a raw blob is source-agnostic + // (source_offsets are 0, all ti_ranges == [0x1000, 0x1000+count)) so a sub-TI maps directly to + // leaf_idx = ti - 0x1000 regardless of its CV_TypeIndexSource label. Walk is iterative (worklist). + U64 total_blob_leaves = 0, total_closure_leaves = 0; + U64 closure_begin_us = now_time_us(); + for EachIndex(blob_i, ifc_file_count) { + total_blob_leaves += input->debug_t_arr[input->ifc_obj_range.min + blob_i].count; + } + if (ifc_file_count) { + // Build the set of unique_names that already have a COMPLETE definition in some non-blob obj. + // The blob prune keeps a blob complete-def only when its name is absent here (blob-only type), + // so forward-refs that no normal obj can complete still get their definition (full-merge fidelity) + // while redundant blob copies of normally-defined types stay pruned. Size to ~2x the non-blob + // complete-def count, rounded up to a power of two, for low load factor. + U64 nonblob_complete_estimate = 0; + for EachIndex(obj_idx, input->ifc_obj_range.min) { + nonblob_complete_estimate += input->debug_t_arr[obj_idx].source_counts[CV_TypeIndexSource_TPI]; + } + LNK_U64Set nonblob_complete = {0}; + nonblob_complete.cap = 1; + while (nonblob_complete.cap < (nonblob_complete_estimate * 2 + 16)) { nonblob_complete.cap <<= 1; } + nonblob_complete.slots = push_array(scratch.arena, U64, nonblob_complete.cap); + // parallel scan: each non-blob obj emits its complete-def hashes; serial merge adds to the set. + U64 nonblob_count = input->ifc_obj_range.min; + if (nonblob_count) { + LNK_IfcCompleteScanTask scan = {0}; + scan.input = input; + scan.out_hashes = push_array(scratch.arena, U64 *, nonblob_count); + scan.out_counts = push_array(scratch.arena, U64, nonblob_count); + tp_for_parallel(tp, tp_arena, nonblob_count, lnk_ifc_complete_scan_task, &scan); + // parallel atomic-CAS merge (replaces the serial lnk_u64set_add loop): output-identical + // because set membership is order-independent + idempotent (see lnk_u64set_add_atomic). + LNK_IfcSetMergeTask merge = {0}; + merge.ranges = tp_divide_work(scratch.arena, nonblob_count, tp->worker_count); + merge.out_hashes = scan.out_hashes; + merge.out_counts = scan.out_counts; + merge.set = &nonblob_complete; + merge.nonblob_count = nonblob_count; + tp_for_parallel(tp, 0, tp->worker_count, lnk_ifc_set_merge_task, &merge); } + + LNK_IfcCloseTask close_task = {0}; + close_task.input = input; + close_task.ref_bits = ref_bits; + close_task.closure_leaves = push_array(scratch.arena, U64, ifc_file_count); + close_task.nonblob_complete = &nonblob_complete; + tp_for_parallel(tp, tp_arena, ifc_file_count, lnk_ifc_close_blob_task, &close_task); + for EachIndex(blob_i, ifc_file_count) { total_closure_leaves += close_task.closure_leaves[blob_i]; } } + (void)tp; + lnk_log(LNK_Log_Timers, "[IFC] closure pass in %.2f ms", (F64)(now_time_us() - closure_begin_us) / 1000.0); - lnk_log(LNK_Log_Debug, "[IFC] injected %llu .ifc blob(s), %llu record redirect(s)", ifc_file_count, redirect_count); + lnk_log(LNK_Log_Debug, "[IFC] injected %llu .ifc blob(s), %llu record redirect(s); on-demand closure %llu / %llu blob leaves (%.1f%%)", + ifc_file_count, redirect_count, total_closure_leaves, total_blob_leaves, + total_blob_leaves ? (100.0 * (F64)total_closure_leaves / (F64)total_blob_leaves) : 0.0); done: scratch_end(scratch); + lnk_log(LNK_Log_Timers, "[IFC] apply total in %.2f ms", (F64)(now_time_us() - apply_begin_us) / 1000.0); ProfEnd(); } @@ -1366,7 +1801,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, // resolve MSVC header-unit IFC debug records (LF_IFC_RECORD 0x1522) -> real CodeView types. // injects .ifc debug-records blobs as extra objs and registers placeholder-TI redirects. if (config->ifc_debug_records == LNK_SwitchState_Yes && config->ifc_map_list.node_count) { - lnk_apply_ifc_debug_records(tp_arena, &input, config); + lnk_apply_ifc_debug_records(tp, tp_arena, &input, config); } // set default min type index @@ -1487,9 +1922,18 @@ lnk_leaf_ref_from_ti(LNK_CodeViewInput *input, U32 obj_idx, CV_TypeIndexSource s // to a leaf inside an injected .ifc debug-records blob obj. The blob leaves then // dedup/hash/fixup natively through the rest of this function. if (input->has_ifc_redirects && source == CV_TypeIndexSource_TPI) { - U64 *packed = hash_map_search_u64_u64(&input->ifc_redirect_hm, Compose64Bit(obj_idx, ti)); - if (packed) { - return (LNK_LeafRef){ (U32)(*packed >> 32), (U32)(*packed & max_U32) }; + // exact per-obj bitset filter: bit set iff Compose64Bit(obj_idx, ti) is a key in + // ifc_redirect_hm. skips the (miss-dominated) per-call key hash + map walk; on a set + // bit the original map search runs unchanged, so behavior is bit-identical. + U64 *bits = input->ifc_redirect_bits[obj_idx]; + if (bits != 0 && contains_1u64(input->ifc_redirect_ti_rng[obj_idx], ti)) { + U64 rel = ti - input->ifc_redirect_ti_rng[obj_idx].min; + if (bits[rel >> 6] & (1ull << (rel & 63))) { + U64 *packed = hash_map_search_u64_u64(&input->ifc_redirect_hm, Compose64Bit(obj_idx, ti)); + if (packed) { + return (LNK_LeafRef){ (U32)(*packed >> 32), (U32)(*packed & max_U32) }; + } + } } } diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 53974fa71..169ec6a3a 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -118,6 +118,11 @@ typedef struct HashMap ifc_redirect_hm; Rng1U64 ifc_obj_range; // [min,max) range of injected blob objs in the parallel arrays U32Array ifc_indices; // obj indices of injected .ifc blob objs (hashed/deduped first) + // exact per-obj key filter for ifc_redirect_hm: bit set iff Compose64Bit(obj_idx, ti) was pushed. + // lets lnk_leaf_ref_from_ti skip the (miss-dominated) hash-map search entirely; on a set bit the + // original map is searched unchanged, so results are bit-identical to always searching. + U64 **ifc_redirect_bits; // [count]; null == obj has no redirect keys + Rng1U64 *ifc_redirect_ti_rng; // [count]; [min,max) local-TI span covered by the obj's bitset } LNK_CodeViewInput; typedef struct From 492e2858fb0ab04309e76b081d367bcf83803fbb Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Sun, 21 Jun 2026 09:45:42 -0700 Subject: [PATCH 009/121] linker: parallelize the serial setup loops in lnk_make_code_view_input Several O(input) per-obj setup sweeps at the head of lnk_make_code_view_input ran serially on the main thread while the pool idled between the parallel parse phases. Dispatch them over the pool; results land in per-obj slots, so downstream iteration order and output bytes are unchanged (gated byte-identical). Co-Authored-By: Claude Opus 4.8 (1M context) Co-Authored-By: Claude Fable 5 (cherry picked from commit 69672ab6dbb850cf30b5179cffbdeb00d7a382e2) --- src/linker/lnk_debug_info.c | 223 +++++++++++++++++++++++++----------- 1 file changed, 158 insertions(+), 65 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 0d65f6dc4..6229e8829 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -1355,6 +1355,86 @@ lnk_apply_ifc_debug_records(TP_Context *tp, TP_Arena *tp_arena, LNK_CodeViewInpu ProfEnd(); } +//////////////////////////////// +// parallel setup tasks for lnk_make_code_view_input + +// Loop 3 (PCH/ext/int classification). The expensive predicates -- the read-only rrt_hm lookup +// and cv_debug_t_is_type_server_ref -- run in parallel per obj. Each obj's class tag and PCH-merge +// mutation are fully independent, so this pass is data-parallel. The ordered 3-array compaction +// and the MultipleDebugTAndDebugP warning are then replayed SERIALLY in obj_idx order, so output +// (array contents/order + warning order + discarded set) is byte-identical to the serial loop. +typedef struct LNK_CvClassifyTask +{ + LNK_CodeViewInput *input; + CV_DebugT *debug_p_arr; + HashMap *rrt_hm; // read-only after build + LNK_Obj **obj_arr; + U8 *class_tag; // 0=debug_p, 1=ext, 2=int + U8 *warn_multi; +} LNK_CvClassifyTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_cv_classify_task) +{ + LNK_CvClassifyTask *t = raw_task; + U64 obj_idx = task_id; + CV_DebugT *debug_t = &t->input->debug_t_arr[obj_idx]; + CV_DebugT *debug_p = &t->debug_p_arr[obj_idx]; + + // classify (same predicate order/precedence as the serial loop) + U8 tag; + if (hash_map_search_path_u64(t->rrt_hm, t->obj_arr[obj_idx]->path)) { tag = 1; } + else if (debug_p->count > 0 && debug_t->count == 0) { tag = 0; } + else if (cv_debug_t_is_type_server_ref(debug_t)) { tag = 1; } + else { tag = 2; } + t->class_tag[obj_idx] = tag; + + // per-obj independent debug_t mutation (identical to serial) + if (debug_t->count == 0 && debug_p->count > 0) { + *debug_t = *debug_p; + } else if (debug_t->count && debug_p->count) { + t->warn_multi[obj_idx] = 1; // defer warning to serial obj-order replay + MemoryZeroStruct(debug_t); + MemoryZeroStruct(debug_p); + } +} + +// Loop 4 (Make Symbol Inputs) count pass. cv_sub_section_from_debug_s is a pure read of the +// already-parsed data_list, so caching each obj's Symbols sub-section list in parallel is safe. +typedef struct LNK_CvSymTask +{ + LNK_CodeViewInput *input; + String8List *per_obj_syms; + U64 *counts; // per-obj node_count (count pass) + U64 *offsets; // per-obj symbol_inputs offset (fill pass) +} LNK_CvSymTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_cv_sym_count_task) +{ + LNK_CvSymTask *t = raw_task; + U64 obj_idx = task_id; + t->per_obj_syms[obj_idx] = cv_sub_section_from_debug_s(t->input->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); + t->counts[obj_idx] = t->per_obj_syms[obj_idx].node_count; +} + +// Loop 4 fill pass. Each obj writes a disjoint, contiguous range of symbol_inputs starting at its +// prefix-sum offset, in node order -- byte-identical to the serial append (which walked obj_idx +// ascending, each obj's nodes in list order). +internal +THREAD_POOL_TASK_FUNC(lnk_cv_sym_fill_task) +{ + LNK_CvSymTask *t = raw_task; + U64 obj_idx = task_id; + U64 cur = t->offsets[obj_idx]; + String8List s = t->per_obj_syms[obj_idx]; + for EachNode(n, String8Node, s.first) { + LNK_SymbolInput *in = &t->input->symbol_inputs[cur++]; + in->obj_idx = obj_idx; + in->raw_symbols = n->string; + } +} + internal LNK_CodeViewInput lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 obj_count, LNK_Obj **obj_arr, LNK_RRT_Array rrt_input) { @@ -1375,47 +1455,51 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, ProfBegin("Apply RRT to Objs"); - // hash map (obj path, obj idx) + // hash map (obj path, obj idx). Kept SERIAL: HashMap is a 4-ary trie whose insert mutates shared + // child pointers + arena-allocates nodes -> not safe for concurrent insert. Only built (and + // consulted) when there is at least one input RRT; the monolithic Engine.dll link has none. HashMap obj_path_hm = {0}; - for EachIndex(obj_idx, obj_count) { - hash_map_push_path_u64(scratch.arena, &obj_path_hm, obj_arr[obj_idx]->path, obj_idx); - } + if (rrt_input.count) { + for EachIndex(obj_idx, obj_count) { + hash_map_push_path_u64(scratch.arena, &obj_path_hm, obj_arr[obj_idx]->path, obj_idx); + } - for EachIndex(obj_idx, obj_count) { - LNK_Obj *obj = obj_arr[obj_idx]; - U64 *packed_rrt_idx = hash_map_search_path_u64(&rrt_hm, obj->path); + for EachIndex(obj_idx, obj_count) { + LNK_Obj *obj = obj_arr[obj_idx]; + U64 *packed_rrt_idx = hash_map_search_path_u64(&rrt_hm, obj->path); - // obj is not part of any input RRT - if (packed_rrt_idx == 0) { continue; } + // obj is not part of any input RRT + if (packed_rrt_idx == 0) { continue; } - // unpack index - U32 rrt_idx = *packed_rrt_idx >> 32; - U32 rrt_obj_idx = *packed_rrt_idx & max_U32; - LNK_RRT *rrt = &rrt_input.v[rrt_idx]; + // unpack index + U32 rrt_idx = *packed_rrt_idx >> 32; + U32 rrt_obj_idx = *packed_rrt_idx & max_U32; + LNK_RRT *rrt = &rrt_input.v[rrt_idx]; - // obj was recompiled, do not apply RRT indirection - FileProperties obj_file_props = properties_from_file_path(obj->path); - if (rrt->obj_time_stamps[rrt_obj_idx] != obj_file_props.modified) { continue; } + // obj was recompiled, do not apply RRT indirection + FileProperties obj_file_props = properties_from_file_path(obj->path); + if (rrt->obj_time_stamps[rrt_obj_idx] != obj_file_props.modified) { continue; } // invalidate debug section pointers obj->coff.debug_t_section_number = 0; obj->coff.debug_p_section_number = 0; obj->coff.debug_h_section_number = 0; - // apply type index map - obj->ti_range = rrt->obj_ti_ranges[rrt_obj_idx]; - obj->ti_map = rrt->obj_ti_maps [rrt_obj_idx]; - - // apply PCH info - U32 rrt_pch_obj_idx = rrt->obj_pch_indices[rrt_obj_idx]; - if (rrt_pch_obj_idx < rrt->obj_count) { - String8 rrt_pch_obj_path = rrt->obj_paths.v[rrt_pch_obj_idx]; - U64 pch_obj_idx = *hash_map_search_path_u64(&obj_path_hm, rrt_pch_obj_path); - obj->pch_ti_range = rrt->obj_pch_ti_ranges[rrt_obj_idx]; - obj->pch_obj_idx = pch_obj_idx; - } else { - obj->pch_ti_range = r1u64(0,0); - obj->pch_obj_idx = ~0; + // apply type index map + obj->ti_range = rrt->obj_ti_ranges[rrt_obj_idx]; + obj->ti_map = rrt->obj_ti_maps [rrt_obj_idx]; + + // apply PCH info + U32 rrt_pch_obj_idx = rrt->obj_pch_indices[rrt_obj_idx]; + if (rrt_pch_obj_idx < rrt->obj_count) { + String8 rrt_pch_obj_path = rrt->obj_paths.v[rrt_pch_obj_idx]; + U64 pch_obj_idx = *hash_map_search_path_u64(&obj_path_hm, rrt_pch_obj_path); + obj->pch_ti_range = rrt->obj_pch_ti_ranges[rrt_obj_idx]; + obj->pch_obj_idx = pch_obj_idx; + } else { + obj->pch_ti_range = r1u64(0,0); + obj->pch_obj_idx = ~0; + } } } ProfEnd(); @@ -1520,25 +1604,35 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, ProfEnd(); // sort objs based on type: PCH, /Zi (external), /Z7 (internal) - input.debug_p_indices.v = push_array(tp_arena->v[0], U32, obj_count); + input.debug_p_indices.v = push_array(tp_arena->v[0], U32, obj_count); input.ext_obj_indices.v = push_array(tp_arena->v[0], U32, obj_count); input.int_obj_indices.v = push_array(tp_arena->v[0], U32, obj_count); - for EachIndex(obj_idx, obj_count) { - CV_DebugT *debug_t = &input.debug_t_arr[obj_idx]; - CV_DebugT *debug_p = &debug_p_arr[obj_idx]; - U32Array *arr_ptr; - if (hash_map_search_path_u64(&rrt_hm, obj_arr[obj_idx]->path)) { arr_ptr = &input.ext_obj_indices; } - else if (debug_p->count > 0 && debug_t->count == 0) { arr_ptr = &input.debug_p_indices; } - else if (cv_debug_t_is_type_server_ref(debug_t)) { arr_ptr = &input.ext_obj_indices; } - else { arr_ptr = &input.int_obj_indices; } - arr_ptr->v[arr_ptr->count++] = obj_idx; - - if (debug_t->count == 0 && debug_p->count > 0) { - *debug_t = *debug_p; - } else if (debug_t->count && debug_p->count) { - lnk_error_obj(LNK_Warning_MultipleDebugTAndDebugP, obj_arr[obj_idx], "multiple sections with debug types detected, obj must have either .debug$T or .debug$P; discarding both sections"); - MemoryZeroStruct(debug_t); - MemoryZeroStruct(debug_p); + ProfScope("Classify Objs") + { + // parallel: classify each obj + apply per-obj debug_t mutation (see lnk_cv_classify_task) + LNK_CvClassifyTask classify = {0}; + classify.input = &input; + classify.debug_p_arr = debug_p_arr; + classify.rrt_hm = &rrt_hm; + classify.obj_arr = obj_arr; + classify.class_tag = push_array(scratch.arena, U8, obj_count ? obj_count : 1); + classify.warn_multi = push_array(scratch.arena, U8, obj_count ? obj_count : 1); + tp_for_parallel_prof(tp, 0, obj_count, lnk_cv_classify_task, &classify, "Classify Objs (parallel)"); + + // serial obj-order compaction into the 3 ordered arrays + deterministic warning emission. + // Cache-linear single pass; preserves the exact element order + warning order of the old loop. + for EachIndex(obj_idx, obj_count) { + U32Array *arr_ptr; + switch (classify.class_tag[obj_idx]) { + case 0: arr_ptr = &input.debug_p_indices; break; + case 1: arr_ptr = &input.ext_obj_indices; break; + default: arr_ptr = &input.int_obj_indices; break; + } + arr_ptr->v[arr_ptr->count++] = obj_idx; + + if (classify.warn_multi[obj_idx]) { + lnk_error_obj(LNK_Warning_MultipleDebugTAndDebugP, obj_arr[obj_idx], "multiple sections with debug types detected, obj must have either .debug$T or .debug$P; discarding both sections"); + } } } @@ -1820,25 +1914,24 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, ProfBegin("Make Symbol Inputs"); { - // count symbol blocks - for EachIndex(obj_idx, input.count) { - String8List s = cv_sub_section_from_debug_s(input.debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); - input.symbol_input_count += s.node_count; - } - + // count symbol blocks (cache each obj's Symbols sub-section list so the fill pass below + // does not re-decode .debug$S a second time -- cv_sub_section_from_debug_s walks subsections). + String8List *per_obj_syms = push_array(scratch.arena, String8List, input.count ? input.count : 1); + LNK_CvSymTask sym_task = {0}; + sym_task.input = &input; + sym_task.per_obj_syms = per_obj_syms; + sym_task.counts = push_array(scratch.arena, U64, input.count ? input.count : 1); + + // parallel: cache each obj's Symbols sub-section list + count nodes + tp_for_parallel_prof(tp, 0, input.count, lnk_cv_sym_count_task, &sym_task, "Count Symbol Inputs"); + input.symbol_input_count = sum_array_u64(input.count, sym_task.counts); + sym_task.offsets = offsets_from_counts_array_u64(scratch.arena, sym_task.counts, input.count); + // alloc block pointers - input.symbol_inputs = push_array_no_zero(tp_arena->v[0], LNK_SymbolInput, input.symbol_input_count); - - U64 symbol_input_count = 0; - for EachIndex(obj_idx, input.count) { - String8List s = cv_sub_section_from_debug_s(input.debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); - for EachNode(n, String8Node, s.first) { - Assert(symbol_input_count < input.symbol_input_count); - LNK_SymbolInput *in = &input.symbol_inputs[symbol_input_count++]; - in->obj_idx = obj_idx; - in->raw_symbols = n->string; - } - } + input.symbol_inputs = push_array_no_zero(tp_arena->v[0], LNK_SymbolInput, input.symbol_input_count ? input.symbol_input_count : 1); + + // parallel fill into disjoint per-obj ranges at prefix-sum offsets (byte-identical order) + tp_for_parallel_prof(tp, 0, input.count, lnk_cv_sym_fill_task, &sym_task, "Fill Symbol Inputs"); ProfBegin("Make Ranges"); From 6d653d1e7c53d45b90667e5d4abfd5e96a42cca3 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Sun, 21 Jun 2026 13:07:40 -0700 Subject: [PATCH 010/121] radlink perf: decommit idle tctx scratch pages before PDB build After merge-types reaches the per-thread scratch high-water (~9GB of tctx scratch arenas stay committed but idle through the PDB peak), release the committed-but-unused scratch pages back to the OS before the PDB build re-grows them. Drops recorded peak working set on the monolithic UnrealEditorFortnite-Engine.dll link. - arena_decommit_unused(): decommit committed pages strictly above each block's live pos in the active chain, and the unused bodies of free-list blocks (keeping the header page). Reservation kept; push path re-commits on demand, so reuse is transparent and output byte-identical. - tctx_scratch_decommit(): decommit the calling thread's two equipped scratch arenas. - lnk_scratch_decommit_worker + tp_for_parallel(worker_count) with an in-task barrier: every worker (worker 0 IS the main thread) decommits its own scratch exactly once between lnk_merge_types and lnk_build_pdb. Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit b1d8a4abeed4bc7fbb8ea607073c43ad13761e24) --- src/base/base_arena.c | 48 ++++++++++++++++++++++++++++++++++ src/base/base_arena.h | 3 +++ src/base/base_thread_context.c | 15 +++++++++++ src/base/base_thread_context.h | 1 + src/linker/lnk.c | 29 ++++++++++++++++++++ 5 files changed, 96 insertions(+) diff --git a/src/base/base_arena.c b/src/base/base_arena.c index d4376c105..2753f603a 100644 --- a/src/base/base_arena.c +++ b/src/base/base_arena.c @@ -353,6 +353,54 @@ arena_pop_to(Arena *arena, U64 pos) } +//- rjf: arena decommit of unused (rewound/free) pages + +internal void +arena_decommit_unused(Arena *arena) +{ + // NOTE(perf): decommit committed-but-unused pages so they stop counting against + // working set, while keeping the reservation. Only touches pages strictly above + // the live `pos` high-water of each block in the active chain, and the unused + // bodies of free-list blocks. Live data (<= pos) is never touched. The push path + // re-commits on demand (arena_push grows `cmt`), so reuse is transparent. + if(arena->flags & ArenaFlag_LargePages) + { + // large pages cannot be partially decommitted safely; skip. + return; + } + U64 page_size = get_system_info()->page_size; + + // rjf: active chain -- decommit committed region above each block's live pos + for(Arena *n = arena->current; n != 0; n = n->prev) + { + U64 pos_aligned = AlignPow2(n->pos, page_size); + if(pos_aligned < n->cmt) + { + U8 *decommit_ptr = (U8 *)n + pos_aligned; + U64 decommit_size = n->cmt - pos_aligned; + AsanPoisonMemoryRegion(decommit_ptr, decommit_size); + decommit_memory(decommit_ptr, decommit_size); + n->cmt = pos_aligned; + } + } + +#if ARENA_FREE_LIST + // rjf: free chain -- decommit everything above the first (header) page + for(Arena *n = arena->free_last; n != 0; n = n->prev) + { + U64 keep = AlignPow2(ARENA_HEADER_SIZE, page_size); + if(keep < n->cmt) + { + U8 *decommit_ptr = (U8 *)n + keep; + U64 decommit_size = n->cmt - keep; + AsanPoisonMemoryRegion(decommit_ptr, decommit_size); + decommit_memory(decommit_ptr, decommit_size); + n->cmt = keep; + } + } +#endif +} + //- rjf: arena push/pop helpers internal void diff --git a/src/base/base_arena.h b/src/base/base_arena.h index b3ba37827..a61f5bd3f 100644 --- a/src/base/base_arena.h +++ b/src/base/base_arena.h @@ -76,6 +76,9 @@ internal void *arena_push(Arena *arena, U64 size, U64 align, B32 zero); internal U64 arena_pos(Arena *arena); internal void arena_pop_to(Arena *arena, U64 pos); +//- rjf: arena decommit of unused (rewound/free) pages +internal void arena_decommit_unused(Arena *arena); + //- rjf: arena push/pop helpers internal void arena_clear(Arena *arena); internal void arena_pop(Arena *arena, U64 amt); diff --git a/src/base/base_thread_context.c b/src/base/base_thread_context.c index 421636101..d42be68ed 100644 --- a/src/base/base_thread_context.c +++ b/src/base/base_thread_context.c @@ -82,6 +82,21 @@ tctx_get_scratch(Arena **conflicts, U64 count) return result; } +//- rjf: scratch decommit (release committed-but-unused scratch pages back to OS) + +internal void +tctx_scratch_decommit(void) +{ + TCTX *tctx = tctx_selected(); + for(U64 i = 0; i < ArrayCount(tctx->arenas); i += 1) + { + if(tctx->arenas[i] != 0) + { + arena_decommit_unused(tctx->arenas[i]); + } + } +} + //- rjf: lane metadata internal LaneCtx diff --git a/src/base/base_thread_context.h b/src/base/base_thread_context.h index 494d1804f..e52b36476 100644 --- a/src/base/base_thread_context.h +++ b/src/base/base_thread_context.h @@ -90,6 +90,7 @@ internal TCTX *tctx_selected(void); //- rjf: scratch arenas internal Arena *tctx_get_scratch(Arena **conflicts, U64 count); +internal void tctx_scratch_decommit(void); #define scratch_begin(conflicts, count) temp_begin(tctx_get_scratch((conflicts), (count))) #define scratch_end(scratch) temp_end(scratch) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index fec3e9bec..27ce8778d 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -6453,6 +6453,21 @@ lnk_log_timers(void) scratch_end(scratch); } +internal +THREAD_POOL_TASK_FUNC(lnk_scratch_decommit_worker) +{ + // Each worker decommits the committed-but-unused pages of its OWN equipped + // tctx scratch arenas. Runs on the worker thread, so tctx_selected() yields + // that worker's scratch. No cross-thread arena access. + // + // The barrier (dispatched with task_count == worker_count) guarantees every + // worker runs the body exactly once -- otherwise the work-stealing loop could + // let one fast worker grab several tasks and leave other workers' scratch + // committed. All worker_count threads are woken, so all must reach the barrier. + tctx_scratch_decommit(); + barrier_wait(tp->barrier); +} + internal THREAD_POOL_TASK_FUNC(lnk_p2r_worker) { @@ -6597,6 +6612,20 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) lnk_gc_types(tp, arena->v[0], &cv, &cv_types); } + // merge-types reached the scratch high-water (~9GB of per-thread tctx scratch + // stays committed but idle). Release those unused scratch pages back to the OS + // before the PDB build re-grows, dropping the recorded peak working set. Each + // worker decommits its own scratch; do the main thread's scratch too. Only + // pages strictly above each arena's live `pos` are touched, so output stays + // byte-identical and the push path re-commits on demand during PDB build. + { + ProfBegin("Decommit Scratch"); + // task_count == worker_count + the in-worker barrier => every worker + // (worker 0 IS the main thread) runs exactly once, covering main's scratch. + tp_for_parallel(tp, 0, tp->worker_count, lnk_scratch_decommit_worker, 0); + ProfEnd(); + } + // // Debug Info // From 21506227fcfa2fda1338faf905e03ffbff7e6854 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Sun, 21 Jun 2026 17:13:24 -0700 Subject: [PATCH 011/121] linker: free leaf bucket_arr probe tables before the merge-types commit peak The per-source leaf-dedup probe tables (bucket_arr) are dead once the unique-leaf set has been extracted, but their pages stayed committed through the merge-types commit peak. Release them as soon as extraction is done so they come off the peak footprint. Memory-release timing only; output bytes unchanged (gated byte-identical). Co-Authored-By: Claude Opus 4.8 (1M context) Co-Authored-By: Claude Fable 5 (cherry picked from commit c03287a3698a08c6ba33c45d0146541f6ec97825) --- src/linker/lnk_debug_info.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 6229e8829..d458744b4 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -2756,6 +2756,14 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } ProfEnd(); + // bucket_arr (the ~1.3x-total-leaf-count probe tables) is only live through the dedup + extract + // phases: its last read is in lnk_get_present_buckets_task ("Copy present buckets") which copies + // bucket pointers into unique_leaf_refs. Allocate it in a dedicated arena so we can release that + // multi-GB working set immediately after the extract loop, before the merge-types/PDB-build peak. + // (A temp_begin on scratch.arena would not work: many surviving allocations -- unique_leaf_refs, + // assigned_ti, radix scratch -- land in scratch.arena after bucket_arr.) + Arena *bucket_arena = arena_alloc(.name = "LEAF_BUCKETS"); + ProfBegin("Leaf Hash Table Init"); for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { U64 total_count = 0; @@ -2763,7 +2771,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.leaf_ht_arr[ti_source].cap = total_count; task.leaf_ht_arr[ti_source].cap = 1 + ((task.leaf_ht_arr[ti_source].cap * 13) / 10); // * 1.3 - task.leaf_ht_arr[ti_source].bucket_arr = push_array_no_zero(scratch.arena, LNK_LeafRef, task.leaf_ht_arr[ti_source].cap); + task.leaf_ht_arr[ti_source].bucket_arr = push_array_no_zero(bucket_arena, LNK_LeafRef, task.leaf_ht_arr[ti_source].cap); MemorySet(task.leaf_ht_arr[ti_source].bucket_arr, 0xff, sizeof(LNK_LeafRef) * task.leaf_ht_arr[ti_source].cap); #if PROFILE_TELEMETRY @@ -2833,6 +2841,11 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } } + // bucket_arr is fully consumed (copied into unique_leaf_refs / sorted) -- release the probe tables + // now so this multi-GB working set is gone before the merge-types/PDB-build peak. + arena_release(bucket_arena); + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { task.leaf_ht_arr[ti_source].bucket_arr = 0; } + #if PROFILE_TELEMETRY tmMessage(0, TMMF_ICON_NOTE, "TPI Count: %.*s", str8_varg(str8_from_count(scratch.arena, task.unique_leaf_refs_arr[CV_TypeIndexSource_TPI].count))); tmMessage(0, TMMF_ICON_NOTE, "IPI Count: %.*s", str8_varg(str8_from_count(scratch.arena, task.unique_leaf_refs_arr[CV_TypeIndexSource_IPI].count))); @@ -2848,10 +2861,11 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.assigned_ti_arr[ti_source].cap = ((task.unique_leaf_refs_arr[ti_source].count * 13) / 10); task.assigned_ti_arr[ti_source].ti_arr = push_array(scratch.arena, CV_TypeIndex, task.assigned_ti_arr[ti_source].cap); - // unique extraction is complete, so the dedup bucket slots can back the - // direct hash table without increasing peak memory - Assert(task.assigned_ti_arr[ti_source].cap <= task.leaf_ht_arr[ti_source].cap); - task.assigned_ti_arr[ti_source].hash_arr = task.leaf_ht_arr[ti_source].bucket_arr; + // bucket_arr used to back hash_arr here to avoid a fresh allocation, but the bucket + // arena is released right after the extract loop (above) so the probe tables' multi-GB + // working set is gone before the merge-types commit peak; allocate hash_arr fresh -- + // it is sized by the (much smaller) unique count, not the total-based bucket cap + task.assigned_ti_arr[ti_source].hash_arr = push_array(scratch.arena, U64, task.assigned_ti_arr[ti_source].cap); task.min_type_indices[ti_source] = CV_MinComplexTypeIndex; task.ranges = tp_divide_work(scratch.arena, task.unique_leaf_refs_arr[ti_source].count, tp->worker_count); From 86ee226ce4c21299cbac8a9668dfaa1ecd1c3a4b Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Sun, 21 Jun 2026 18:35:23 -0700 Subject: [PATCH 012/121] radlink: env-gated RADLINK_PHASE_LOG raw per-phase micros file (diagnostic, byte-neutral) When RADLINK_PHASE_LOG is set, lnk_log_timers writes machine-parseable raw per-phase microseconds (Image/PDB/RDI/Lib/Debug + TOTAL) to that path, for automated perf A/B. Env-unset -> identical code path, DLL/PDB byte-identical. Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit f4915498fc856892fd696f07bb80fdefc2141a1c) --- src/linker/lnk.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 27ce8778d..ed76c10bc 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -6449,7 +6449,21 @@ lnk_log_timers(void) StringJoin new_line_join = { str8_lit_comp(""), str8_lit_comp("\n"), str8_lit_comp("") }; String8 output = str8_list_join(scratch.arena, &output_list, &new_line_join); lnk_log(LNK_Log_Timers, "%S\n", output); - + + // Diagnostic: when RADLINK_PHASE_LOG is set, also write machine-parseable raw + // per-phase micros to that file (for automated perf A/B). Env-unset -> no-op, + // so normal/validation links are byte-identical; this never touches DLL/PDB bytes. + char *phase_log_path = getenv("RADLINK_PHASE_LOG"); + if (phase_log_path != 0 && phase_log_path[0] != 0) { + String8List raw_list = {0}; + for (U64 i = 0; i < LNK_Timer_Count; ++i) { + str8_list_pushf(scratch.arena, &raw_list, "%S %llu\n", lnk_string_from_timer_type(i), g_timers[i].end - g_timers[i].begin); + } + str8_list_pushf(scratch.arena, &raw_list, "TOTAL %llu\n", total_build_time_micro); + String8 raw_str = str8_list_join(scratch.arena, &raw_list, 0); + lnk_write_data_to_file_path(str8_cstring(phase_log_path), str8_zero(), raw_str); + } + scratch_end(scratch); } @@ -6759,8 +6773,11 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // // Timers // - if (lnk_get_log_status(LNK_Log_Timers)) { - lnk_log_timers(); + { + char *phase_log_env = getenv("RADLINK_PHASE_LOG"); + if (lnk_get_log_status(LNK_Log_Timers) || (phase_log_env != 0 && phase_log_env[0] != 0)) { + lnk_log_timers(); + } } scratch_end(scratch); From e1cc21cdc628189b05856cb0650c70cb5676be8d Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 24 Jun 2026 22:06:13 -0700 Subject: [PATCH 013/121] linker: /RAD_SHARED_THREAD_POOL -- cross-process shared thread pool (governor + fair-share cohorts) Share one worker budget across concurrent radlink processes: a cross-process governor grants budget slots fairly (fair-share cohorts for barrier passes), capping total running workers near core count during farm convoys instead of workers x links oversubscription. The pool is dual-path: without the switch the upstream barrier implementation runs verbatim, so default behavior is unchanged. Soak-validated at 3216 concurrent shared-pool links with zero hangs; output byte-identical in both modes. Squashed from 2 series commits; their original messages follow. * radlink: cross-process shared thread pool (governor + fair-share barrier passes) Collapsed net of our 5-commit pool series (9fd1db9a -> d28970dc revert -> 74347207 reapply -> e373f039 fair-share -> 35703bdb grant-race fix), reconciled onto upstream's reworked thread pool (9b63131f "ditch semaphores in favor of barriers"). Upstream rewrote tp_run_tasks to bracket each pass with an entry+exit barrier_wait(pool->barrier) sized to the full worker_count, and dropped main_semaphore/task_semaphore. That model is incompatible with our shared governor, where a path-A pass wakes only a governor-chosen SUBSET of workers (main always runs, woken workers return their budget slot on drain): a full-width entry barrier would block forever waiting for parked workers. We therefore keep our semaphore-completion tp_run_tasks (no built-in barrier) which already implements BOTH modes correctly, and discard upstream's barrier-bracket internals. This is transparent to callers -- tp_broadcast/tp_sum_u64/barrier_wait semantics are preserved, and barrier passes rendezvous a cohort-sized barrier. Net design (= e373f039 + 35703bdb final): - per-process governor thread + NAMED budget_semaphore (machine core budget) - parked workers woken one-per-granted-slot; each returns its slot on drain - path A (tp_for_parallel): main always runs, governor opportunistically recruits workers as budget frees; granted++ is published BEFORE the pass_active re-check (no grant-race hang), aborting with granted-- if the pass already ended - path B (tp_for_parallel_reserve + tp_barrier_begin/end): fair-share cohort = 1 + budget slots free RIGHT NOW (never amasses the machine, deadlock-free); output is width-independent so any cohort is byte-identical to full width - off-by-default: non-shared mode is the prior semaphore pool, zero overhead Carries the base_threads helpers the pool needs (semaphore_drop_n, semaphore_drop_if_room, semaphore_take_n on win32 + linux); upstream's semaphore_drop_count/semaphore_drop shim are kept untouched alongside. Caller conversions (lnk.c, lnk_debug_info.c): the ICF-refine region, /OPT:REF mark, scratch-decommit, p2r, rrt type-data, and PDB move-globals/write-modules barrier passes now use tp_for_parallel_reserve inside a tp_barrier_begin/end cohort bracket; lnk_icf_refine_region_task indexes per-lane scratch by task_id (the contiguous lane), and PDB obj indices are redistributed per pass to the pinned cohort C via lnk_build_pdb_distribute_obj_indices. * radlink: dual-path thread pool (non-shared=upstream barrier, shared=governor) Reconcile the thread pool into two independent code paths selected by pool->is_shared (== /RAD_SHARED_THREAD_POOL name.size>0): NON-SHARED: restore origin/dev's barrier implementation VERBATIM. Workers park on a kernel barrier between passes (zero-syscall steady state). tp_run_tasks brackets the work loop in barrier_wait at entry+exit; tp_worker_main loops calling it; tp_for_parallel just inits state and joins as worker 0. No governor thread, no budget/wake/governor/main semaphores allocated -- zero overhead, zero new threads vs upstream. SHARED: keep OUR cross-process governor. Parked workers woken one-per-grant by a per-process governor thread borrowing a NAMED global budget semaphore; each woken path-A worker returns its slot on drain; granted++ published before the pass_active re-check (grant-race fix); main always runs; main_semaphore completion. Path-B barrier passes use the fair-share cohort (tp_for_parallel_reserve + tp_barrier_begin/end: cohort = 1 + budget-free-now, pinned, width-independent), NOT a full-width barrier. Dispatch branches on is_shared: tp_for_parallel runs the upstream barrier dispatch for non-shared and the governor dispatch for shared; the work loop splits into tp_run_tasks (upstream barrier, non-shared) and tp_run_tasks_shared (main_semaphore completion, shared). tp_alloc allocates governor/budget state and launches the governor thread only when is_shared. tp_barrier_begin/end and tp_for_parallel_reserve are no-ops in non-shared mode (degrade to the upstream full-width barrier pass), so the lnk.c / lnk_debug_info.c cohort callers stay transparent there. Validated: build clean. Non-shared output content-identical to the d566da33 baseline (soak_big input set; only the irreducible /Brepro debug-GUID + PE stamp/checksum bytes differ, equal to the same-binary control floor); no governor thread spawned in non-shared. Shared re-soak 3216 concurrent shared-pool links (48-way, mixed big/small, 64-core budget), zero hangs; shared output (64-way and 8-way) content-identical to non-shared (width-independent). Pool name is copied into the config arena at parse (rsp-backed string lifetime). Co-Authored-By: Claude Opus 4.8 (1M context) Co-Authored-By: Claude Fable 5 --- src/base/base_threads.h | 3 + src/linker/lnk.c | 41 ++- src/linker/lnk_config.c | 5 +- src/linker/lnk_debug_info.c | 55 ++- src/linker/thread_pool/thread_pool.c | 492 ++++++++++++++++++++++++--- src/linker/thread_pool/thread_pool.h | 39 +++ src/linux/base/linux_base.c | 34 ++ src/win32/base/win32_base.c | 49 +++ 8 files changed, 653 insertions(+), 65 deletions(-) diff --git a/src/base/base_threads.h b/src/base/base_threads.h index 3aae2ea6c..c84bf26e2 100644 --- a/src/base/base_threads.h +++ b/src/base/base_threads.h @@ -128,8 +128,11 @@ internal void semaphore_release(Semaphore semaphore); internal Semaphore semaphore_open(String8 name); internal void semaphore_close(Semaphore semaphore); internal B32 semaphore_take(Semaphore semaphore, U64 endt_us); +internal B32 semaphore_take_n(Semaphore semaphore, U32 count, U64 endt_us); // blocking acquire of `count` permits (off hot path) internal void semaphore_drop(Semaphore semaphore); internal void semaphore_drop_count(Semaphore semaphore, U64 drop_count); +internal void semaphore_drop_if_room(Semaphore semaphore); // best-effort post; no-op if already at max +internal void semaphore_drop_n(Semaphore semaphore, U32 count); // release `count` permits in one syscall //- rjf: barriers internal Barrier barrier_alloc(U64 count); diff --git a/src/linker/lnk.c b/src/linker/lnk.c index ed76c10bc..644885f0c 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -2903,9 +2903,15 @@ lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj { ProfBegin("/OPT:REF"); Temp scratch = scratch_begin(0,0); - U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, tp->worker_count, objs, objs_count); + // BARRIER pass (path B): the task synchronizes with barrier_wait(tp->barrier)/tp_broadcast, + // so under /RAD_SHARED_THREAD_POOL it must run at a pinned cohort via the reserve path -- + // a plain tp_for_parallel admits workers incrementally and the barrier never fills (deadlock). + // Pin the cohort BEFORE sizing the per-lane obj distribution so both agree on the width. + U32 C = tp_barrier_begin(tp); + U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, C, objs, objs_count); LNK_OptTask task = { .symtab = symtab, .config = config, .objs = objs, .objs_count = objs_count, .obj_indices = obj_indices }; - tp_for_parallel(tp, 0, tp->worker_count, lnk_opt_ref_task, &task); + tp_for_parallel_reserve(tp, 0, C, lnk_opt_ref_task, &task); // BARRIER pass (path B) + tp_barrier_end(tp); scratch_end(scratch); ProfEnd(); } @@ -3788,9 +3794,16 @@ lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj Temp scratch = scratch_begin(0,0); lnk_log(LNK_Log_Debug, "/OPT:ICF:"); - U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, tp->worker_count, objs, objs_count); + + // BARRIER pass (path B): the task synchronizes with barrier_wait(tp->barrier)/tp_broadcast, + // so under /RAD_SHARED_THREAD_POOL it must run at a pinned cohort via the reserve path -- + // a plain tp_for_parallel admits workers incrementally and the barrier never fills (deadlock). + // Pin the cohort BEFORE sizing the per-lane obj distribution so both agree on the width. + U32 C = tp_barrier_begin(tp); + U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, C, objs, objs_count); LNK_OptTask task = { .symtab = symtab, .config = config, .objs = objs, .objs_count = objs_count, .obj_indices = obj_indices }; - tp_for_parallel(tp, 0, tp->worker_count, lnk_opt_icf_task, &task); + tp_for_parallel_reserve(tp, 0, C, lnk_opt_icf_task, &task); // BARRIER pass (path B) + tp_barrier_end(tp); scratch_end(scratch); ProfEnd(); @@ -5637,10 +5650,18 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT ProfScope("Gather Sections") { TP_Temp temp = tp_temp_begin(arena); + // BARRIER pass (path B): the task synchronizes with barrier_wait(tp->barrier), so under + // /RAD_SHARED_THREAD_POOL it must run at a pinned cohort via the reserve path -- a plain + // tp_for_parallel admits workers incrementally and the barrier never fills (deadlock). + // Pin the cohort BEFORE sizing the per-lane ranges/defns so everything agrees on the width. + U32 C = tp_barrier_begin(tp); task.u.gather_sects.arena = arena->v[0]; - task.u.gather_sects.ranges = tp_divide_work(arena->v[0], objs_count, tp->worker_count); - task.u.gather_sects.defns = push_array(arena->v[0], HashTable *, tp->worker_count); - tp_for_parallel_prof(tp, arena, tp->worker_count, lnk_gather_sections_task, &task, "Gather Sections"); + task.u.gather_sects.ranges = tp_divide_work(arena->v[0], objs_count, C); + task.u.gather_sects.defns = push_array(arena->v[0], HashTable *, C); + ProfBegin("Gather Sections"); + tp_for_parallel_reserve(tp, arena, C, lnk_gather_sections_task, &task); // BARRIER pass (path B) + ProfEnd(); + tp_barrier_end(tp); tp_temp_end(temp); } @@ -6636,7 +6657,7 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) ProfBegin("Decommit Scratch"); // task_count == worker_count + the in-worker barrier => every worker // (worker 0 IS the main thread) runs exactly once, covering main's scratch. - tp_for_parallel(tp, 0, tp->worker_count, lnk_scratch_decommit_worker, 0); + tp_for_parallel_reserve(tp, 0, tp->worker_count, lnk_scratch_decommit_worker, 0); // BARRIER pass (path B) ProfEnd(); } @@ -6671,7 +6692,7 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) lnk_timer_begin(LNK_Timer_Rdi); LNK_P2R p2r = { .config = config, .pdb_data = lnk_data_from_file_artifact(lnk_get_huge_arena(), &pdb_artifact), .image_data = image_ctx.image_data }; - tp_for_parallel(tp, arena, tp->worker_count, lnk_p2r_worker, &p2r); + tp_for_parallel_reserve(tp, arena, tp->worker_count, lnk_p2r_worker, &p2r); // BARRIER pass (path B) String8List rdi_blobs = rdim_file_blobs_from_section_bundle(scratch.arena, &p2r.bake_results.section_bundle); lnk_write_data_list_to_file_path(config->rad_debug_name, config->temp_rad_debug_name, rdi_blobs); @@ -6953,7 +6974,7 @@ lnk_run_type_server(TP_Context *tp, TP_Arena *arena, LNK_Config *config) ProfScope("Pack Type Data & Data Ranges") { LNK_RRTTypeDataSerializer task = { &cv_types, &rrt.type_data_raw, rrt.type_data_ranges }; - tp_for_parallel(tp, arena, tp->worker_count, lnk_serialize_rrt_type_data_task, &task); + tp_for_parallel_reserve(tp, arena, tp->worker_count, lnk_serialize_rrt_type_data_task, &task); // BARRIER pass (path B) // pack type index ranges for EachIndex(i, CV_TypeIndexSource_COUNT) { diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index 47083874b..51d22f6a2 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -2204,7 +2204,10 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val if (value.size == 0) { config->shared_thread_pool_name = str8_lit(LNK_DEFAULT_THREAD_POOL_NAME); } else { - lnk_cmd_switch_parse_string(obj, cmd_switch, value, &config->shared_thread_pool_name); + // NOTE: must copy into the config arena -- the parsed string points into + // response-file/cmdline scratch that is freed long before late consumers + // (pool init, summary) read it + lnk_cmd_switch_parse_string_copy(config->arena, obj, cmd_switch, value, &config->shared_thread_pool_name); if (config->shared_thread_pool_name.size == 0) { lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "invalid empty string for thread pool name"); } diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index d458744b4..f7f21f582 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4181,6 +4181,23 @@ lnk_gc_types(TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedType ProfEnd(); } +// FAIR-SHARE: round-robin cv->obj_count objs across `worker_count` lane buckets. +// Rebuilt per barrier pass so the distribution matches the cohort C that pass +// runs at (lnk_move_global_symbols_to_gsi / lnk_write_pdb_modules read +// task->obj_indices[task_id] for lanes [0,C)). Output is width-independent, so any +// C produces byte-identical PDB bytes; only the per-lane partitioning changes. +internal void +lnk_build_pdb_distribute_obj_indices(Arena *arena, LNK_BuildPdb *task, U64 obj_count, U32 worker_count) +{ + U64 objs_per_worker = CeilIntegerDiv(obj_count, worker_count); + task->obj_indices = push_array(arena, U32Array, worker_count); + for EachIndex(i, worker_count) { task->obj_indices[i].v = push_array(arena, U32, objs_per_worker ? objs_per_worker : 1); } + for EachIndex(obj_idx, obj_count) { + U32Array *obj_indices = &task->obj_indices[obj_idx % worker_count]; + obj_indices->v[obj_indices->count++] = (U32)obj_idx; + } +} + internal LNK_FileArtifact lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags) { @@ -4227,16 +4244,13 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config // set min type indices for EachElement(ti_source, cv_types.min_type_indices) { task.pdb->type_servers[ti_source]->ti_lo = cv_types.min_type_indices[ti_source]; } - // per worker obj indices - { - U64 objs_per_worker = CeilIntegerDiv(cv->obj_count, tp->worker_count); - task.obj_indices = push_array(scratch.arena, U32Array, tp->worker_count); - for EachIndex(i, tp->worker_count) { task.obj_indices[i].v = push_array(scratch.arena, U32, objs_per_worker); } - for EachIndex(obj_idx, cv->obj_count) { - U32Array *obj_indices = &task.obj_indices[obj_idx % tp->worker_count]; - obj_indices->v[obj_indices->count++] = obj_idx; - } - } + // per-worker obj indices are (re)distributed per barrier pass to the cohort + // that pass actually runs at (FAIR-SHARE: tp->worker_count is pinned to the + // cohort C inside each tp_barrier_begin/end bracket, and the lnk_move_global_ + // symbols_to_gsi / lnk_write_pdb_modules tasks read task.obj_indices[task_id] + // for lanes [0,C)). Distributing to the full worker_count up front would leave + // objs in buckets [C,worker_count) unprocessed when Cdbi, cv->obj_arr[obj_idx]->path, lnk_obj_get_lib_path(cv->obj_arr[obj_idx])); } - ProfScope("Write Modules") tp_for_parallel(tp, 0, tp->worker_count, lnk_write_pdb_modules, &task); + ProfScope("Write Modules") + { + // FAIR-SHARE: pin the cohort, distribute objs over exactly the cohort lanes, + // then run the barrier pass at that cohort. tp_barrier_begin sets + // tp->worker_count := C for the bracket. + U32 C = tp_barrier_begin(tp); + lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C); + tp_for_parallel_reserve(tp, 0, C, lnk_write_pdb_modules, &task); // BARRIER pass (path B): barrier_wait/tp_broadcast + tp_barrier_end(tp); + } if (output_ptr != 0) { for EachIndex(obj_idx, cv->obj_count) { lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.mod_arr[obj_idx]->sn); } } - ProfScope("Move Global Symbols") tp_for_parallel(tp, 0, tp->worker_count, lnk_move_global_symbols_to_gsi, &task); - ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); + ProfScope("Move Global Symbols") + { + U32 C = tp_barrier_begin(tp); + lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C); + tp_for_parallel_reserve(tp, 0, C, lnk_move_global_symbols_to_gsi, &task); // BARRIER pass (path B): tp_sum_u64/tp_broadcast/barrier_wait + tp_barrier_end(tp); + } + ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); if (output_ptr != 0) { lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->publics_sn); lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->globals_sn); diff --git a/src/linker/thread_pool/thread_pool.c b/src/linker/thread_pool/thread_pool.c index 3dc2c1467..d59216898 100644 --- a/src/linker/thread_pool/thread_pool.c +++ b/src/linker/thread_pool/thread_pool.c @@ -1,6 +1,30 @@ // Copyright (c) Epic Games Tools // Licensed under the MIT license (https://opensource.org/license/mit/) +// +// DUAL-PATH thread pool. +// +// NON-SHARED mode (no /RAD_SHARED_THREAD_POOL): UPSTREAM's barrier +// implementation, VERBATIM. Workers park on a kernel barrier between passes +// (zero-syscall steady state, no per-pass semaphore traffic). tp_run_tasks +// brackets the work loop in barrier_wait at entry+exit; tp_worker_main loops +// calling it; tp_for_parallel just inits state and joins as worker 0. No +// governor thread, no budget/wake/governor semaphores, no main_semaphore. +// +// SHARED mode (/RAD_SHARED_THREAD_POOL): OUR cross-process governor. Workers +// are PARKED on wake_semaphore and woken one-per-grant; a per-process +// governor thread borrows global budget slots and wakes parked workers; +// completion is signalled via main_semaphore; path-B barrier passes run a +// fair-share cohort (tp_barrier_begin/end + tp_for_parallel_reserve). +// +// Dispatch and the worker entry point branch on pool->is_shared (== name.size>0). +// All governor/budget state is allocated only when is_shared, so non-shared mode +// has zero extra threads and zero extra synchronization objects vs upstream. +// + +//////////////////////////////////////////////////////////////////////////////// +//~ NON-SHARED (upstream barrier) path -- VERBATIM from origin/dev. + internal void tp_run_tasks(TP_Context *pool, TP_Worker *worker) { @@ -36,60 +60,226 @@ tp_worker_main(void *raw_worker) } } +//////////////////////////////////////////////////////////////////////////////// +//~ SHARED (cross-process governor) path -- OURS. + +internal void +tp_for_parallel_init_state(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskFunc *task_func, void *task_data) +{ + pool->task_arena = task_arena; + pool->task_func = task_func; + pool->task_data = task_data; + pool->task_count = task_count; + pool->task_done = 0; + ins_atomic_u64_eval_assign(&pool->task_left, task_count); +} + +// +// SHARED work loop. Semaphore-completion model (no barrier bracket): pure +// work-stealing on the atomic task_left decrement; the last finisher pings +// main_semaphore so the dispatching main thread can return. +// +internal void +tp_run_tasks_shared(TP_Context *pool, TP_Worker *worker) +{ + for (;;) { + S64 task_left = ins_atomic_u64_dec_eval(&pool->task_left); + + // are there any tasks left to run? + if (task_left < 0) { + break; + } + + // run task + Arena *arena = pool->task_arena ? pool->task_arena->v[worker->id] : 0; + U64 task_id = pool->task_count - (task_left+1); + pool->task_func(arena, worker->id, task_id, pool->task_data, pool); + + // cache task count so we dont touch pool memory after atomic inc + U64 task_count = pool->task_count; + + // on last task ping main thread (main_semaphore is null when worker_count==1, + // in which case main runs everything inline and never waits) + U64 task_done = ins_atomic_u64_inc_eval(&pool->task_done); + if (task_done == task_count && pool->worker_count > 1) { + semaphore_drop(pool->main_semaphore); + } + } +} + +// +// SHARED worker. Parked on wake_semaphore. Woken one-per-grant. Two wake kinds: +// - path A (barrier-free, governor-driven): the worker was woken because the +// governor acquired a global budget slot for it. When the worker drains +// (tp_run_tasks_shared returns), it RETURNS that slot: release(budget) + +// granted--, so the slot can flow to another process mid-pass. +// - path B (barrier pass): the dispatching thread reserved cohort slots up +// front and woke exactly the cohort's workers. The worker just runs the pass +// and re-parks; the dispatcher releases the slots in bulk afterwards. The +// worker must NOT touch the budget here (cohort must stay live for the pass). +// internal void tp_worker_main_shared(void *raw_worker) { TP_Worker *worker = raw_worker; TP_Context *pool = worker->pool; for (; pool->is_live; ) { - if (semaphore_take(pool->exec_semaphore, max_U64)) { - tp_run_tasks(pool, worker); + if (!semaphore_take(pool->wake_semaphore, max_U64)) { + continue; + } + if (!pool->is_live) { + break; + } + // capture pass kind at wake time (only one pass kind is active at once) + B32 barrier_pass = pool->barrier_pass; + + tp_run_tasks_shared(pool, worker); + + if (!barrier_pass) { + // path A: hand my budget slot back so another process can use it + ins_atomic_u64_dec_eval(&pool->granted); + semaphore_drop(pool->budget_semaphore); + } + } +} + +// +// Per-process governor. Sleeps until main signals a path-A pass is active, then +// acquires global budget slots (only while THIS process has pending demand) and +// wakes one local parked worker per slot. Slots are returned by the workers +// themselves when they drain, so the governor only ever ACQUIRES. +// +internal void +tp_governor_main(void *raw_pool) +{ + TP_Context *pool = raw_pool; + for (; pool->is_live; ) { + // wait for a pass to begin (or for shutdown) + if (!semaphore_take(pool->governor_semaphore, max_U64)) { + continue; + } + if (!pool->is_live) { + break; + } + + // Grant slots while the pass is live and there is demand. Cap total live + // grants at worker_count-1 (main/worker 0 is the worker_count-th runner and + // never consumes a slot). `granted` is decremented by workers as they drain. + for (; ins_atomic_u32_eval(&pool->pass_active); ) { + S64 task_left = ins_atomic_u64_eval((U64 *)&pool->task_left); + S64 demand = task_left > 0 ? task_left : 0; + S64 cap = (S64)pool->worker_count - 1; + S64 live = ins_atomic_u64_eval((U64 *)&pool->granted); + S64 want = Min(cap, demand) - live; + + if (want > 0) { + // Bounded wait so we re-check pass_active/demand and never block forever + // on budget that may never free if the pass ends first. + if (semaphore_take(pool->budget_semaphore, now_time_us() + 1000)) { + // Publish the grant (granted++) BEFORE checking pass_active, and ABORT + // with granted-- if the pass already ended. This makes main's path-A + // drain-spin (waits granted==0) observe any in-flight grant and block + // until the governor resolves it -- so main cannot exit tp_for_parallel + // and start a path-B barrier pass (which sets barrier_pass=1) while a + // grant is pending. Hence a woken worker ALWAYS captures barrier_pass==0 + // for a path-A grant and does its paired granted--. + // + // The earlier "check pass_active, then granted++" ordering was NOT + // atomic: the governor could pass the check, get preempted while main + // ended the pass + drained granted to 0 + started a path-B pass, then + // wake a worker that captured barrier_pass==1, skipped granted--, and + // wedged granted>0 forever (observed: main spinning in tp_for_parallel, + // all workers parked). granted++ first closes that window. + ins_atomic_u64_inc_eval(&pool->granted); + if (ins_atomic_u32_eval(&pool->pass_active)) { + semaphore_drop(pool->wake_semaphore); + } else { + ins_atomic_u64_dec_eval(&pool->granted); // abort: pass ended + semaphore_drop(pool->budget_semaphore); // give the slot back + } + } + } else { + // No demand right now (work drained or fully covered). Briefly idle; the + // pass-end ping will release us promptly via the outer take when we loop. + sleep_ms(0); + } } } } -internal TP_Context * +//////////////////////////////////////////////////////////////////////////////// +//~ Alloc / release (dual). + +internal TP_Context * tp_alloc(Arena *arena, U32 worker_count, U32 max_worker_count, String8 name) { ProfBeginDynamic("Alloc Thread Pool [Worker Count: %u]", worker_count); AssertAlways(worker_count > 0); - B32 is_shared = (name.size > 0); + B32 is_shared = (name.size > 0); + Temp scratch = scratch_begin(&arena, 1); + + // init pool + TP_Context *pool = push_array(arena, TP_Context, 1); + pool->run_barrier = barrier_alloc(worker_count); + pool->barrier = barrier_alloc(worker_count); + pool->is_live = 1; + pool->is_shared = is_shared; + pool->worker_count = worker_count; + pool->worker_arr = push_array(arena, TP_Worker, worker_count); // alloc semaphores - Semaphore exec_semaphore = {0}; - if (worker_count > 1) { - if (is_shared) { + if (is_shared) { + // SHARED: governor + budget + wake + completion. Only allocated here, so + // non-shared mode pays for none of it. + if (worker_count > 1) { AssertAlways(worker_count <= max_worker_count); - exec_semaphore = semaphore_alloc(0, worker_count, str8_zero()); + + pool->main_semaphore = semaphore_alloc(0, 1, str8_zero()); + + // ONE NAMED cross-process semaphore. CreateSemaphoreW on an existing name + // returns the existing object (first process inits with this count; later + // processes attach and the supplied count is ignored by the OS), so all + // processes share one BUDGET. + // BUDGET: init=max=max_worker_count (the machine core budget). + // FAIR-SHARE: there is no longer a barrier-lock. A barrier pass (path B) + // does NOT amass the full cohort; it runs at whatever budget is free right + // now (best-effort), so multiple processes can run barrier passes + // concurrently and none can deadlock waiting to amass the machine. + String8 budget_name = push_str8f(scratch.arena, "%S.budget", name); + pool->budget_semaphore = semaphore_alloc(max_worker_count, max_worker_count, budget_name); + + // local wake/governor signalling. governor_semaphore is a 0/1 "at least one + // pending pass" flag: main pings it with semaphore_drop_if_room (a redundant + // ping while one is already pending is a harmless no-op, since the pending + // signal will make the governor re-evaluate the current pass_active anyway). + pool->wake_semaphore = semaphore_alloc(0, worker_count, str8_zero()); + pool->governor_semaphore = semaphore_alloc(0, 1, str8_zero()); } } // pick entry point for the workers void *worker_entry = is_shared ? tp_worker_main_shared : tp_worker_main; - // init pool - TP_Context *pool = push_array(arena, TP_Context, 1); - pool->exec_semaphore = exec_semaphore; - pool->run_barrier = barrier_alloc(worker_count); - pool->barrier = barrier_alloc(worker_count); - pool->is_live = 1; - pool->worker_count = worker_count; - pool->worker_arr = push_array(arena, TP_Worker, worker_count); - // init worker data for (U64 i = 0; i < worker_count; i += 1) { TP_Worker *worker = &pool->worker_arr[i]; worker->id = i; worker->pool = pool; } - + // launch worker threads for (U64 i = 1; i < worker_count; i += 1) { TP_Worker *worker = &pool->worker_arr[i]; worker->handle = thread_launch(worker_entry, worker); } - + + // launch the per-process governor (shared mode only) + if (is_shared && worker_count > 1) { + pool->governor_handle = thread_launch(tp_governor_main, pool); + } + + scratch_end(scratch); ProfEnd(); return pool; } @@ -99,24 +289,45 @@ tp_release(TP_Context *pool) { pool->is_live = 0; - B32 is_shared = pool->exec_semaphore.u64[0] != 0; - if (is_shared) { - for EachIndex(i, pool->worker_count) { - semaphore_drop(pool->exec_semaphore); + if (pool->is_shared) { + if (pool->worker_count > 1) { + // wake governor so it observes !is_live and exits (a pending ping is fine) + semaphore_drop_if_room(pool->governor_semaphore); + // wake every parked worker so each observes !is_live and exits. Wakes here + // are NOT path-A grants (no budget was taken), so mark barrier_pass to keep + // workers from touching the budget on their way out. + pool->barrier_pass = 1; + for (U64 i = 1; i < pool->worker_count; i += 1) { + semaphore_drop(pool->wake_semaphore); + } + } + for (U64 i = 1; i < pool->worker_count; i += 1) { + thread_detach(pool->worker_arr[i].handle); + } + if (pool->worker_count > 1) { + thread_detach(pool->governor_handle); + semaphore_release(pool->budget_semaphore); + semaphore_release(pool->wake_semaphore); + semaphore_release(pool->governor_semaphore); + semaphore_release(pool->main_semaphore); + } + } else { + // NON-SHARED: upstream verbatim. Workers are parked on the barrier; flipping + // is_live and waking the barrier lets each observe !is_live and exit. + for (U64 i = 1; i < pool->worker_count; i += 1) { + thread_detach(pool->worker_arr[i].handle); } } - for (U64 i = 1; i < pool->worker_count; i += 1) { - thread_detach(pool->worker_arr[i].handle); - } - if (is_shared) { - semaphore_release(pool->exec_semaphore); - } + barrier_release(pool->run_barrier); barrier_release(pool->barrier); MemoryZeroStruct(pool); } +//////////////////////////////////////////////////////////////////////////////// +//~ Arenas / temps -- shared by both modes (unchanged). + internal TP_Arena * tp_arena_alloc(TP_Context *pool) { @@ -179,11 +390,23 @@ tp_temp_end(TP_Temp temp) ProfEnd(); } +//////////////////////////////////////////////////////////////////////////////// +//~ Dispatch (dual). + internal void tp_for_parallel(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskFunc *task_func, void *task_data) { - if (task_count) { - // init run + if (task_count == 0) { + return; + } + + if (!pool->is_shared) { + // + // NON-SHARED: UPSTREAM verbatim. Init state, then join the barrier as worker + // 0; the already-parked workers (looping in tp_run_tasks) rendezvous at the + // entry barrier, steal tasks, and rendezvous again at the exit barrier. No + // semaphores, no governor. + // pool->task_arena = task_arena; pool->task_func = task_func; pool->task_data = task_data; @@ -191,26 +414,214 @@ tp_for_parallel(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskF pool->task_done = 0; pool->task_left = task_count; - // if we are in shared mode -> ping - if (*pool->exec_semaphore.u64) { - U64 drop_count64 = pool->worker_count - 1; - U32 drop_count = safe_cast_u32(drop_count64); - semaphore_drop_count(pool->exec_semaphore, drop_count); + // run tasks on main worker + tp_run_tasks(pool, &pool->worker_arr[0]); + return; + } + + // + // SHARED: OUR governor dispatch. + // + tp_for_parallel_init_state(pool, task_arena, task_count, task_func, task_data); + + if (pool->worker_count == 1) { + // no workers: main runs everything inline + tp_run_tasks_shared(pool, &pool->worker_arr[0]); + return; + } + + // PATH A: barrier-free dispatch (the common case). Pure work-stealing via the + // atomic task_left decrement in tp_run_tasks_shared. Main (worker 0) ALWAYS + // runs and never consumes a global budget slot -> per-process forward-progress + // guarantee. The governor opportunistically borrows budget slots and wakes + // local parked workers; each woken worker returns its slot when it drains. + + // announce a path-A pass and let the governor recruit workers as budget frees + pool->barrier_pass = 0; + ins_atomic_u32_eval_assign(&pool->pass_active, 1); + semaphore_drop_if_room(pool->governor_semaphore); + + // main always runs (no slot consumed) + tp_run_tasks_shared(pool, &pool->worker_arr[0]); + + // all tasks done (last finisher pinged main_semaphore) + semaphore_take(pool->main_semaphore, max_U64); + + // End the pass so the governor stops issuing new grants. + ins_atomic_u32_eval_assign(&pool->pass_active, 0); + + // CRITICAL: before returning we must guarantee that no woken worker is still + // inside tp_run_tasks_shared. Otherwise the next pass's init_state (which resets + // task_left/task_done) would race a straggler still looping on this pass and + // corrupt the counters / lose the completion ping -> deadlock. + // + // Every governor grant is paired with exactly one wake permit and one worker + // that, on draining, does `granted--; drop(budget)`. Even grants issued in the + // tiny window before pass_active was cleared have a pending wake permit that a + // worker will consume, drain immediately (task_left<0), and account for. So + // `granted` monotonically drains to 0 once the governor has stopped; spin + // until it does. This is brief (workers see task_left<0 and exit at once). + for (; ins_atomic_u64_eval((U64 *)&pool->granted) != 0; ) { + sleep_ms(0); + } +} + +//////////////////////////////////////////////////////////////////////////////// +//~ FAIR-SHARE barrier-pass cohort bracket (SHARED path B; no-op in non-shared). +// +// A barrier pass (path B) runs at the cohort this process currently holds, NOT +// the full machine. tp_barrier_begin grabs whatever budget slots are FREE RIGHT +// NOW (best-effort, never blocking to amass), up to worker_count-1, and pins the +// pool to cohort C = 1 (main) + grabbed slots for the pass duration: +// - pool->worker_count := C (so every tp->worker_count read -- divide_work, +// lane_count, per-worker array sizing in the +// caller's setup -- sees the cohort) +// - pool->barrier := a fresh C-sized barrier (so barrier_wait/broadcast/sum +// rendezvous exactly the C participants) +// - the grabbed slots are HELD until tp_barrier_end (cohort stays live; the +// governor only touches budget during a path-A pass, which cannot overlap a +// barrier pass within this process). +// +// Deadlock-freedom: tp_barrier_begin NEVER blocks on budget. If the machine is +// busy and zero slots are free, C == 1 and the pass runs serially on main. A +// process therefore ALWAYS makes progress (>= main) and never waits to amass -> +// no starvation, no deadlock, no barrier-lock. Output is width-independent +// (proven: w1 == w64), so a cohort-C pass is byte-identical to a full-width pass. +// +// In NON-SHARED mode tp_barrier_begin/end are no-ops (return pool->worker_count / +// early-out) and tp_for_parallel_reserve degrades to the plain upstream +// full-width barrier pass via tp_for_parallel. +// +internal U32 +tp_barrier_begin(TP_Context *pool) +{ + if (!pool->is_shared || pool->worker_count == 1) { + return pool->worker_count; // no-op: non-shared / single-worker + } + if (pool->barrier_depth > 0) { + pool->barrier_depth += 1; // nested: cohort already pinned + return pool->worker_count; + } + + // best-effort grab: take as many free budget slots as we can WITHOUT blocking + // (endt_us==0 -> WaitForSingleObject(.,0) non-blocking poll). Stop at the first + // empty take or when we hold worker_count-1. + U32 want = pool->worker_count - 1; + U32 extra = 0; + for (; extra < want; ) { + if (semaphore_take(pool->budget_semaphore, 0)) { + extra += 1; + } else { + break; // no more free slots right now } + } - // run tasks on main worker - tp_run_tasks(pool, pool->worker_arr); - Assert(pool->task_done == task_count); + U32 cohort = 1 + extra; // main + grabbed workers + + pool->barrier_saved_workers = pool->worker_count; + pool->barrier_cohort_extra = extra; + pool->barrier_saved = pool->barrier; + pool->barrier = barrier_alloc(cohort); + pool->worker_count = cohort; + pool->barrier_pass = 1; + pool->barrier_depth = 1; + return cohort; +} + +internal void +tp_barrier_end(TP_Context *pool) +{ + if (!pool->is_shared || pool->barrier_saved_workers == 0) { + return; // no-op: non-shared / single-worker / not in a bracket } + pool->barrier_depth -= 1; + if (pool->barrier_depth > 0) { + return; // nested: outer bracket still owns the cohort + } + + U32 extra = pool->barrier_cohort_extra; + + // restore the full-width pool + the original barrier + barrier_release(pool->barrier); + pool->barrier = pool->barrier_saved; + pool->worker_count = pool->barrier_saved_workers; + pool->barrier_pass = 0; + + pool->barrier_saved_workers = 0; + pool->barrier_cohort_extra = 0; + MemoryZeroStruct(&pool->barrier_saved); + + // hand the grabbed budget slots back to the machine + semaphore_drop_n(pool->budget_semaphore, extra); } +// +// PATH B: barrier-pass dispatch (fair-share). Runs the task once per lane on the +// CURRENT cohort (main + woken workers). If the caller has not already opened a +// tp_barrier_begin/end bracket, this opens one (cohort = whatever is free now), +// runs, and closes it. The passed task_count is ignored for sizing -- the pass +// always runs exactly pool->worker_count tasks (== cohort). +// +// In NON-SHARED mode this degrades to the plain upstream full-width barrier pass +// (tp_for_parallel), so non-shared barrier passes behave exactly as upstream. +// +internal void +tp_for_parallel_reserve(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskFunc *task_func, void *task_data) +{ + if (task_count == 0) { + return; + } + + if (!pool->is_shared || pool->worker_count == 1) { + // non-shared (or single worker): identical to the plain dispatch (upstream + // full-width barrier pass in non-shared mode) + tp_for_parallel(pool, task_arena, task_count, task_func, task_data); + return; + } + + // open a cohort bracket unless the caller already pinned one + B32 opened = 0; + if (pool->barrier_depth == 0) { + tp_barrier_begin(pool); + opened = 1; + } + + U32 cohort = pool->worker_count; // pinned for the whole pass + + if (cohort == 1) { + // machine fully busy: run serially on main (byte-identical -- width independent) + tp_for_parallel_init_state(pool, task_arena, cohort, task_func, task_data); + tp_run_tasks_shared(pool, &pool->worker_arr[0]); + } else { + tp_for_parallel_init_state(pool, task_arena, cohort, task_func, task_data); + + // wake exactly the cohort's workers (ids 1..cohort-1). These are barrier-pass + // wakes: workers must NOT return budget on drain -- the slots are held by the + // bracket and released in tp_barrier_end so the cohort stays live for the pass. + semaphore_drop_n(pool->wake_semaphore, cohort - 1); + + // main is the cohort-th participant (lane 0) + tp_run_tasks_shared(pool, &pool->worker_arr[0]); + + // wait for the cohort to finish + semaphore_take(pool->main_semaphore, max_U64); + } + + if (opened) { + tp_barrier_end(pool); + } +} + +//////////////////////////////////////////////////////////////////////////////// +//~ Helpers -- shared by both modes (unchanged). + internal Rng1U64 * tp_divide_work(Arena *arena, U64 item_count, U32 worker_count) { U64 per_count = CeilIntegerDiv(item_count, worker_count); Rng1U64 *range_arr = push_array_no_zero(arena, Rng1U64, worker_count + 1); for (U64 i = 0; i < worker_count; i += 1) { - range_arr[i] = rng_1u64(Min(item_count, i * per_count), + range_arr[i] = rng_1u64(Min(item_count, i * per_count), Min(item_count, i * per_count + per_count)); } @@ -250,4 +661,3 @@ tp_sum_u64(TP_Context *tp, U64 task_id, U64 v) barrier_wait(tp->barrier); return result; } - diff --git a/src/linker/thread_pool/thread_pool.h b/src/linker/thread_pool/thread_pool.h index f89f17852..f3a1904e8 100644 --- a/src/linker/thread_pool/thread_pool.h +++ b/src/linker/thread_pool/thread_pool.h @@ -38,6 +38,25 @@ typedef struct TP_Context U64 broadcast_size; U64 sum; + // shared (cross-process) governor mode; all zero in non-shared mode + B32 is_shared; + Semaphore budget_semaphore; // NAMED: global core budget; init=max=max_worker_count + Semaphore wake_semaphore; // local: governor/dispatcher wakes one parked worker per drop + Semaphore governor_semaphore; // local: main pings governor that a path-A pass is active + Thread governor_handle; + volatile U32 pass_active; // 1 while a path-A (barrier-free) pass is in flight + volatile U32 barrier_pass; // 1 while the current wake cohort is a path-B barrier pass + volatile S64 granted; // budget slots currently held by woken path-A workers + + // FAIR-SHARE barrier-pass cohort state (path B). A barrier pass runs at the + // cohort the governor currently allows this process to hold: main + however + // many budget slots are free RIGHT NOW (best-effort, never amassed). Pinned + // for the pass duration. See tp_barrier_begin/tp_barrier_end. + U32 barrier_depth; // >0 while inside a tp_barrier_begin/end bracket (re-entrant guard) + U32 barrier_saved_workers; // worker_count to restore at tp_barrier_end + U32 barrier_cohort_extra; // budget slots held for this barrier pass (cohort = 1 + this) + Barrier barrier_saved; // pool->barrier to restore at tp_barrier_end + U32 worker_count; TP_Worker *worker_arr; @@ -57,5 +76,25 @@ internal TP_Temp tp_temp_begin(TP_Arena *arena); internal void tp_temp_end(TP_Temp temp); #define tp_for_parallel_prof(pool, arena, task_count, task_func, task_data, zone_name) ProfBegin(zone_name); tp_for_parallel(pool, arena, task_count, task_func, task_data); ProfEnd(); internal void tp_for_parallel(TP_Context *pool, TP_Arena *arena, U64 task_count, TP_TaskFunc *task_func, void *task_data); +// FAIR-SHARE barrier-pass cohort bracket. Between tp_barrier_begin and +// tp_barrier_end, pool->worker_count is PINNED to the cohort this process +// currently holds (1 + budget slots free right now, capped at the full count), +// and pool->barrier is sized to that cohort. Returns the cohort count C. A caller +// that pre-distributes work by tp->worker_count (sizes per-worker arrays, builds +// divide_work ranges, sets a task-data .worker_count) MUST do that setup inside +// the bracket so it sees C, then call tp_for_parallel_reserve with task_count==C. +// Re-entrant: nested begins just return the pinned C. In non-shared mode (or +// worker_count==1) it is a no-op that returns worker_count. +internal U32 tp_barrier_begin(TP_Context *pool); +internal void tp_barrier_end(TP_Context *pool); +// Barrier-pass dispatch: task_func uses barrier_wait/tp_broadcast/tp_sum_u64. The +// cohort is whatever this process currently holds (fair-share): if not already +// inside a tp_barrier_begin/end bracket this opens one itself, runs the pass at +// the pinned cohort, and closes it. Output is width-independent so any cohort +// (down to 1 = main only) produces byte-identical results. The passed task_count +// is IGNORED for sizing; the pass always runs exactly pool->worker_count (==cohort) +// tasks, one per lane. In non-shared mode it is identical to tp_for_parallel. +internal void tp_for_parallel_reserve(TP_Context *pool, TP_Arena *arena, U64 task_count, TP_TaskFunc *task_func, void *task_data); +#define tp_for_parallel_reserve_prof(pool, arena, task_count, task_func, task_data, zone_name) ProfBegin(zone_name); tp_for_parallel_reserve(pool, arena, task_count, task_func, task_data); ProfEnd(); internal Rng1U64 * tp_divide_work(Arena *arena, U64 item_count, U32 worker_count); #define tp_broadcast(p) tp_broadcast_(tp, task_id, p, sizeof(*p)) diff --git a/src/linux/base/linux_base.c b/src/linux/base/linux_base.c index 3daa94f79..55a9a2413 100644 --- a/src/linux/base/linux_base.c +++ b/src/linux/base/linux_base.c @@ -711,6 +711,40 @@ semaphore_drop_count(Semaphore semaphore, U64 count) } } +internal void +semaphore_drop_if_room(Semaphore semaphore) +{ + // POSIX sem_t is unbounded, so a post can never be rejected for being "full". + semaphore_drop(semaphore); +} + +internal void +semaphore_drop_n(Semaphore semaphore, U32 count) +{ + if(semaphore.u64[0] != 0) + { + for(U32 i = 0; i < count; i += 1) + { + int err = LNX_RETRY_ON_EINTR(sem_post((sem_t*)semaphore.u64[0])); + Assert(err == 0); + } + } +} + +internal B32 +semaphore_take_n(Semaphore semaphore, U32 count, U64 endt_us) +{ + for(U32 i = 0; i < count; i += 1) + { + if(!semaphore_take(semaphore, endt_us)) + { + semaphore_drop_n(semaphore, i); + return 0; + } + } + return 1; +} + //- rjf: barriers internal Barrier diff --git a/src/win32/base/win32_base.c b/src/win32/base/win32_base.c index a67c204c6..ee7cd2bfe 100644 --- a/src/win32/base/win32_base.c +++ b/src/win32/base/win32_base.c @@ -698,6 +698,55 @@ semaphore_drop_count(Semaphore semaphore, U64 drop_count) ReleaseSemaphore((HANDLE)*semaphore.u64, drop_count, 0); } +// Best-effort post: succeed if there is room, silently no-op if the count is +// already at max (ERROR_TOO_MANY_POSTS). Use ONLY for "at least one pending +// signal" wakeups (e.g. the governor ping) where redundant posts are harmless. +// Any OTHER failure is still a hard error. +internal void +semaphore_drop_if_room(Semaphore semaphore) +{ + HANDLE handle = (HANDLE)semaphore.u64[0]; + BOOL ok = ReleaseSemaphore(handle, 1, 0); + if (!ok) { + DWORD err = GetLastError(); + AssertAlways(err == ERROR_TOO_MANY_POSTS); + } +} + +internal void +semaphore_drop_n(Semaphore semaphore, U32 count) +{ + if (count > 0) { + HANDLE handle = (HANDLE)semaphore.u64[0]; + BOOL ok = ReleaseSemaphore(handle, count, 0); + if (!ok) { + // The non-shared thread pool intentionally batches a wake of up to + // worker_count permits onto a semaphore that may still hold un-retaken + // permits from a prior pass; the surplus clamps at the max and the OS + // returns ERROR_TOO_MANY_POSTS. That is a benign over-wake (workers are + // already runnable), so tolerate it -- but ONLY it. Any other failure + // (e.g. a bad handle) is a real bug and must not be swallowed. + DWORD err = GetLastError(); + AssertAlways(err == ERROR_TOO_MANY_POSTS); + } + } +} + +internal B32 +semaphore_take_n(Semaphore semaphore, U32 count, U64 endt_us) +{ + // Blocking acquire of `count` permits, one at a time. Off the hot path only: + // used by the shared thread-pool barrier-reserve path to gather budget slots. + for (U32 i = 0; i < count; i += 1) { + if (!semaphore_take(semaphore, endt_us)) { + // partial failure: give back what we took so we don't leak permits + semaphore_drop_n(semaphore, i); + return 0; + } + } + return 1; +} + //- rjf: barriers internal Barrier From 04871b283ebbab9e9ed555887b63898e38091cf8 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 24 Jun 2026 23:15:08 -0700 Subject: [PATCH 014/121] radlink: sort image relocs by apply_off for monotone-forward patch writes The reloc patcher (lnk_obj_reloc_patcher) walked each section's relocs in on-disk COFF table order, scattering RMW patch writes randomly across the ~739MB image buffer (no HW prefetch, poor write locality). Sort each section's relocs by apply_off (orig_idx tiebreak for a deterministic total order) into a per-section scratch copy before patching, so the write stream is monotone forward. Per-section relocs are independent and each writes its own disjoint field, so final bytes are unchanged. byte-identical: soak_big DLL+PDB diff == base-vs-base control floor (34/18, GUID/timestamp/checksum churn only; no .text/.data/reloc-region diffs). Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 57484d8bc09a71a047e288431625d63675b73eff) --- src/linker/lnk.c | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 644885f0c..71c8adaff 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -4470,10 +4470,34 @@ lnk_compute_win32_image_header_size(LNK_Config *config, U64 sect_count) return image_header_size; } +// reloc patch ordering: sort each section's relocs by apply_off so the RMW +// write stream into the image is monotone-forward (HW-prefetchable) instead of +// scattered in on-disk table order. apply_off is the primary key; orig_idx is a +// tiebreak so the total order is deterministic and the final bytes are identical +// to the unsorted patch order (each reloc writes its own disjoint field; only a +// pathological same-apply_off overlap could depend on order, and the orig_idx +// tiebreak preserves the original sequence there too). +typedef struct LNK_RelocSortKey +{ + COFF_Reloc reloc; + U32 orig_idx; +} LNK_RelocSortKey; + +internal int +lnk_reloc_sort_key_is_before(void *raw_a, void *raw_b) +{ + LNK_RelocSortKey *a = raw_a, *b = raw_b; + if (a->reloc.apply_off != b->reloc.apply_off) { + return a->reloc.apply_off < b->reloc.apply_off; + } + return a->orig_idx < b->orig_idx; +} + internal THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) { ProfBeginFunction(); + Temp scratch = scratch_begin(0, 0); LNK_ObjRelocPatcher *task = raw_task; LNK_Obj *obj = task->objs[task_id]; @@ -4481,10 +4505,6 @@ THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) COFF_FileHeaderInfo obj_header = obj->coff.header; String8 string_table = lnk_coff_string_table_from_obj(obj); - U32 closest_sect = 0; - U32 closest_reloc = 0; - U32 closest_foff = max_U32; - for LNK_EachCoffSection(it, obj) { COFF_SectionHeader *section_header = it.v.header; COFF_SectionFlags section_flags = *it.v.flags; @@ -4497,10 +4517,17 @@ THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) Rng1U64 section_frange = rng_1u64(section_header->foff, section_header->foff + section_header->fsize); String8 section_data = section_flags & LNK_SECTION_FLAG_DEBUG ? lnk_obj_section_data_from_number(obj, it.v.section_number) : str8_substr(task->image_data, section_frange); - // apply relocs + // apply relocs (sorted by apply_off for monotone-forward image writes) COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(obj, section_header); + Temp reloc_temp = temp_begin(scratch.arena); + LNK_RelocSortKey *sorted_relocs = push_array_no_zero(reloc_temp.arena, LNK_RelocSortKey, relocs.count); for EachIndex(reloc_idx, relocs.count) { - COFF_Reloc *reloc = &relocs.v[reloc_idx]; + sorted_relocs[reloc_idx].reloc = relocs.v[reloc_idx]; + sorted_relocs[reloc_idx].orig_idx = (U32)reloc_idx; + } + radsort(sorted_relocs, relocs.count, lnk_reloc_sort_key_is_before); + for EachIndex(reloc_idx, relocs.count) { + COFF_Reloc *reloc = &sorted_relocs[reloc_idx].reloc; // error check relocation if (obj->coff.header.machine == COFF_MachineType_X64) { @@ -4573,8 +4600,10 @@ THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) // commit new reloc value MemoryCopy(section_data.str + reloc->apply_off, &reloc_result, reloc_value.size); } + temp_end(reloc_temp); } + scratch_end(scratch); ProfEnd(); } From fed13c807cbdcab2fc8c3ad3f4c1076814ed2010 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 24 Jun 2026 23:17:26 -0700 Subject: [PATCH 015/121] radlink: non-temporal stores for write-once image fill The ~1GB image buffer is filled (align/zero fill + per-contrib body copies), then streamed straight to disk; the filling threads never re-read it. Normal stores pollute L2/L3 with ~1GB of write-once data. Use SSE2 non-temporal stores (_mm_stream_si128, baseline on x86-64 -- no -mavx needed) for the large (>=256B) fills via lnk_stream_set / lnk_stream_copy, with a scalar head/tail and a MemoryCopy/MemorySet fallback for small or unaligned spans (identical bytes either way). _mm_sfence() after the align-fill and at the end of each fill worker makes the NT stores globally visible before the reloc-patch / checksum passes read the image back. byte-identical: soak_big DLL+PDB diff == base-vs-base control floor (34/18, GUID/timestamp/checksum churn only; identical diff offsets, no image-content diffs). Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit b7359285976c732d8f6f7cab892f59e256cebeb8) --- src/linker/lnk.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 71c8adaff..c5c707419 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -4436,6 +4436,48 @@ THREAD_POOL_TASK_FUNC(lnk_patch_weak_symbols_task) lnk_patch_obj_symtab(task->symtab, task->objs[task_id], task->u.patch_symtabs.was_symbol_patched[task_id], COFF_SymbolValueInterp_Weak); } +// Non-temporal (streaming) stores for the write-once image buffer. The image is +// filled, then streamed straight to disk; these bytes are not re-read by the +// filling thread, so NT stores avoid polluting L2/L3 with ~1GB of write-once +// data. A later pass (lnk_obj_reloc_patcher) DOES read the image back, so every +// caller must _mm_sfence() before that pass runs to make the NT stores globally +// visible. NT stores require 32B alignment; the unaligned head/tail and small +// (<256B) copies fall back to MemoryCopy/MemorySet (identical bytes either way). +#define LNK_STREAM_MIN_SIZE 256 + +// SSE2 (baseline on x86-64, no -mavx required) 16B non-temporal stores. +internal void +lnk_stream_copy(void *dst, void *src, U64 size) +{ + if (size < LNK_STREAM_MIN_SIZE) { MemoryCopy(dst, src, size); return; } + U8 *d = (U8 *)dst, *s = (U8 *)src; + U64 head = (U64)(0x10 - ((U64)d & 0xf)) & 0xf; // bytes to reach 16B-aligned dst + if (head) { MemoryCopy(d, s, head); d += head; s += head; size -= head; } + U64 vec = size & ~(U64)0xf; + for (U64 i = 0; i < vec; i += 0x10) { + __m128i v = _mm_loadu_si128((__m128i const *)(s + i)); + _mm_stream_si128((__m128i *)(d + i), v); + } + U64 tail = size - vec; + if (tail) { MemoryCopy(d + vec, s + vec, tail); } +} + +internal void +lnk_stream_set(void *dst, U8 byte, U64 size) +{ + if (size < LNK_STREAM_MIN_SIZE) { MemorySet(dst, byte, size); return; } + U8 *d = (U8 *)dst; + U64 head = (U64)(0x10 - ((U64)d & 0xf)) & 0xf; + if (head) { MemorySet(d, byte, head); d += head; size -= head; } + __m128i v = _mm_set1_epi8((char)byte); + U64 vec = size & ~(U64)0xf; + for (U64 i = 0; i < vec; i += 0x10) { + _mm_stream_si128((__m128i *)(d + i), v); + } + U64 tail = size - vec; + if (tail) { MemorySet(d + vec, byte, tail); } +} + internal THREAD_POOL_TASK_FUNC(lnk_image_fill_task) { @@ -4445,15 +4487,26 @@ THREAD_POOL_TASK_FUNC(lnk_image_fill_task) for EachNode(n, LNK_ImageFillNode, task->u.image_fill.fill_nodes[task_id]) { for EachIndex(i, n->sc_count) { LNK_SectionContrib *sc = n->sc[i]; + // fast-path: the vast majority of contribs are a single data-node -> one direct copy, skipping + // the list-walk + cursor bookkeeping on the hot 739MB image-write loop. + if (sc->first_data_node.next == 0) { + U64 image_off = sc->u.off + n->base_foff; + Assert(image_off + sc->first_data_node.string.size <= image_data.size); + lnk_stream_copy(image_data.str + image_off, sc->first_data_node.string.str, sc->first_data_node.string.size); + continue; + } U64 cursor = 0; for EachNode(data_n, String8Node, &sc->first_data_node) { U64 image_off = sc->u.off + n->base_foff + cursor; Assert(image_off + data_n->string.size <= image_data.size); - MemoryCopyStr8(image_data.str + image_off, data_n->string); + lnk_stream_copy(image_data.str + image_off, data_n->string.str, data_n->string.size); cursor += data_n->string.size; } } } + // NT stores above are not ordered wrt later normal reads on other cores; the + // reloc-patch pass reads the image back. Make these stores globally visible. + _mm_sfence(); ProfEnd(); } @@ -5945,9 +5998,13 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT LNK_Section *sect = §_n->data; ProfBeginV("Section: %S Size: %M", sect->name, sect->fsize); U8 fill_byte = sect->flags & COFF_SectionFlag_CntCode ? coff_code_align_byte_from_machine(config->machine) : 0; - MemorySet(image_data.str + sect->foff, fill_byte, sect->fsize); + // write-once into the image buffer -> stream past the cache (see lnk_stream_set) + lnk_stream_set(image_data.str + sect->foff, fill_byte, sect->fsize); ProfEnd(); } + // flush NT stores before the parallel contrib-fill / reloc passes read or + // re-write these lines + _mm_sfence(); ProfEnd(); Temp temp = temp_begin(scratch.arena); From 8e6a9318293123b1b24a40f9d976463bd3c24a7c Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 24 Jun 2026 23:47:14 -0700 Subject: [PATCH 016/121] radlink: false-sharing pads on per-worker [worker_count] arrays ICF refine region: chunk_nc/chunk_kp/chunk_sc/chunk_max[worker_count] are written per-lane as chunk_X[wid] (24B/64B stride); adjacent lanes shared a cache line on every per-round scan/radix reduction. Stride each lane entry to a full 64B line (LNK_ICF_CL_STRIDE_U64, index wid*stride) so each lane owns its line; chunk_max's [0..2] round-total scratch stays inside lane-0's line. Symbol table: pad LNK_SymbolHashTrieChunkList to 64B so symtab->chunks / search_chunks (per-worker arrays indexed [worker_id]) no longer false-share on the parallel insert. Pure layout change (more memory, no logic change); indexed values identical. byte-identical, soak_big cmp 0 (DLL 0 / PDB 0). Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 79c98e390f940457f761d6eaeec719504e204f2f) --- src/linker/lnk_symbol_table.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/linker/lnk_symbol_table.h b/src/linker/lnk_symbol_table.h index 4fe2f24cd..2e2c1dc13 100644 --- a/src/linker/lnk_symbol_table.h +++ b/src/linker/lnk_symbol_table.h @@ -79,6 +79,10 @@ typedef struct LNK_SymbolHashTrieChunkList U64 count; LNK_SymbolHashTrieChunk *first; LNK_SymbolHashTrieChunk *last; + // false-sharing pad: symtab->chunks / search_chunks are [worker_count] arrays indexed + // [worker_id]; at 24B/entry adjacent workers share a cache line on the parallel insert. + // Pad each entry to a full 64B line so each worker owns its line. Pure layout -> byte-identical. + U8 pad_[64 - 3*8]; } LNK_SymbolHashTrieChunkList; // --- Symbol Table ------------------------------------------------------------ From 327407b03f415e764c6cec3b9b2e531d9cf5952a Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 24 Jun 2026 23:48:31 -0700 Subject: [PATCH 017/121] radlink: symbol trie inline-hash fast-reject LNK_SymbolHashTrie descent (insert + search) dereferenced name -> String8 -> name bytes (2-3 cache-line misses) per level just to call str8_match. Store the full key hash in the node at insert and fast-reject on hash mismatch before touching the name string; str8_match still gates the real match, so hash is fast-reject only. Same trie topology / contents / match results. byte-identical, soak_big cmp 0 (DLL 0 / PDB 0). Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 85cf482c28c01ed4f6720b15802944d8824f6c15) --- src/linker/lnk_symbol_table.c | 6 ++++-- src/linker/lnk_symbol_table.h | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index 83ab62384..aea705b97 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -441,6 +441,7 @@ lnk_symbol_hash_trie_insert_or_replace(Arena *arena, LNK_SymbolHashTrie *new_trie = lnk_symbol_hash_trie_chunk_list_push(arena, chunks, 0x1000); new_trie->name = &symbol->name; new_trie->symbol = symbol; + new_trie->hash = hash; MemoryZeroArray(new_trie->child); // try to insert new node @@ -461,7 +462,8 @@ lnk_symbol_hash_trie_insert_or_replace(Arena *arena, // load current symbol String8 *curr_name = ins_atomic_ptr_eval(&curr_trie->name); - if (curr_name && str8_match(*curr_name, symbol->name, 0)) { + // fast-reject on stored hash before touching the name string (str8_match still gates) + if (curr_name && curr_trie->hash == hash && str8_match(*curr_name, symbol->name, 0)) { for (LNK_Symbol *src = symbol;;) { // try replacing current symbol with zero, otherwise loop back and retry LNK_Symbol *leader = ins_atomic_ptr_eval_assign(&curr_trie->symbol, 0); @@ -507,7 +509,7 @@ lnk_symbol_hash_trie_search(LNK_SymbolHashTrie *trie, U64 hash, String8 name) if (curr == 0) { break; } - if (curr->name && str8_match(*curr->name, name, 0)) { + if (curr->name && curr->hash == hash && str8_match(*curr->name, name, 0)) { result = curr; break; } diff --git a/src/linker/lnk_symbol_table.h b/src/linker/lnk_symbol_table.h index 2e2c1dc13..1b2e5827e 100644 --- a/src/linker/lnk_symbol_table.h +++ b/src/linker/lnk_symbol_table.h @@ -63,6 +63,10 @@ typedef struct LNK_SymbolHashTrie { String8 *name; LNK_Symbol *symbol; + // full key hash stored at insert; descent fast-rejects on hash mismatch BEFORE + // dereferencing name -> String8 -> name bytes (saves 2-3 line misses/level). + // str8_match still gates the real match, so this is fast-reject only -> byte-identical. + U64 hash; struct LNK_SymbolHashTrie *child[4]; } LNK_SymbolHashTrie; From 31b59e3f7e4ea0148c02bae02b3a98c6e247de48 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 1 Jul 2026 15:27:51 -0700 Subject: [PATCH 018/121] radlink: reap merge-types bucket arena on a background thread The multi-GB LEAF_BUCKETS probe-table arena release at the end of type dedup cost ~370-950ms of SERIAL main-thread kernel time (MiDeleteVaDirect/MiDecommitFreePage under VirtualFree(MEM_RELEASE)), sitting on the critical path between dedup and the type-index fixup passes. Hand the arena to a background reaper thread instead; the main thread proceeds into the fixups while the kernel tears the pages down concurrently. Rejected alternative (measured): chunked MEM_DECOMMIT across the thread pool does not help -- decommit serializes in the kernel on the process address-space lock (~14 GB/s aggregate regardless of thread count; 8 GiB took 860 ms chunked-parallel vs 371 ms serial release). Same reason the per-worker scratch-decommit pass before the PDB build stays as-is: redistributing it is not faster, and backgrounding it is unsafe because workers re-push into those scratch arenas during the PDB build (documented in lnk_scratch_decommit_worker). Reaper protocol: all pointers into the arena are dropped before launch (bucket_arr slots zeroed), at most one reaper is in flight (joined before relaunch and at end of link next to the image-write join), and the handle is not detached at launch (thread_detach right after thread_launch frees the W32_Entity the entry point is about to read). Editor-scale (UnrealEditorFortnite-Engine.dll, /OPT:ICFSTATIC /BREPRO /RAD_WORKERS:64): - main-thread MiDeleteVaDirect 909 -> 163 ms (Superluminal, whole link) - main-thread NtFreeVirtualMemory incl 1219 -> 553 ms - lnk_merge_types main-thread incl 6.48 -> 5.10 s - Debug phase 13.88/13.92 -> 12.97/13.63 s (RADLINK_PHASE_LOG, 2 warm runs each); reaper carries the 947 ms release off-thread - PeakWorkingSet64 unchanged (49.91 GB both) -- peak is during dedup while the buckets are still live, so the delayed release is past it - byte-identity: DLL base-vs-head diff == control (18B /Brepro timestamp+GUID band), OTHER=0; PDB streams identical in count, size, and name (MSF page placement shuffles, as it already does run-to-run) Co-Authored-By: Claude Fable 5 (cherry picked from commit ca8c7f137bc47fdc4178b3f4fb38d3a48223b02d) --- src/linker/lnk.c | 16 +++++++++++++ src/linker/lnk_debug_info.c | 45 +++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index c5c707419..e59181992 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -6585,6 +6585,14 @@ THREAD_POOL_TASK_FUNC(lnk_scratch_decommit_worker) // worker runs the body exactly once -- otherwise the work-stealing loop could // let one fast worker grab several tasks and leave other workers' scratch // committed. All worker_count threads are woken, so all must reach the barrier. + // + // NOTE(perf): redistributing these decommits in chunks across the pool does + // NOT help: MEM_DECOMMIT serializes in the kernel on the process address-space + // lock (~14 GB/s aggregate no matter the thread count; measured 9.3 GiB in + // 677 ms chunked-parallel vs ~500 ms with this per-worker scheme). And handing + // the decommit to a background thread is UNSAFE here: workers push to these + // scratch arenas as soon as the PDB build starts, and a push would re-commit + // pages that the background decommit then rips out. tctx_scratch_decommit(); barrier_wait(tp->barrier); } @@ -6741,9 +6749,11 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // byte-identical and the push path re-commits on demand during PDB build. { ProfBegin("Decommit Scratch"); + U64 decommit_begin_us = now_time_us(); // task_count == worker_count + the in-worker barrier => every worker // (worker 0 IS the main thread) runs exactly once, covering main's scratch. tp_for_parallel_reserve(tp, 0, tp->worker_count, lnk_scratch_decommit_worker, 0); // BARRIER pass (path B) + lnk_log(LNK_Log_Timers, "[teardown] scratch decommit pass in %.2f ms", (F64)(now_time_us() - decommit_begin_us) / 1000.0); ProfEnd(); } @@ -6874,6 +6884,12 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // wait for the thread to finish writing image to disk thread_join(image_write_thread, -1); + // reap the background arena-release thread, if one is still in flight + if (g_arena_reaper_thread.u64[0] != 0) { + thread_join(g_arena_reaper_thread, max_U64); + MemoryZeroStruct(&g_arena_reaper_thread); + } + // image is on disk and no longer read by anyone -- release its ~1GB now so the kernel reclaims it // concurrently with the remaining work + exit, not single-threaded in the process rundown. release_memory(image_ctx.image_data.str, image_ctx.image_data.size); diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index f7f21f582..4dd0520ca 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -12,6 +12,42 @@ lnk_get_huge_arena(void) return g_huge_arena; } +// Handle of the in-flight background arena-release thread (at most one). Joined +// (a) before launching the next reaper and (b) at the end of the link, next to +// the image-write-thread join. NOTE: do NOT thread_detach right after +// thread_launch -- that releases the W32_Entity the new thread's entry point is +// about to read (startup race). +static Thread g_arena_reaper_thread = {0}; + +internal void +lnk_arena_release_thread(void *raw_arena) +{ + // REAPER: releasing a huge arena costs ~50-100ms/GB of committed pages in the + // kernel (MiDeleteVaDirect/MiDecommitFreePage walk every PTE under + // VirtualFree(MEM_RELEASE)), and that work serializes on the process + // address-space lock -- chunking it across the thread pool does NOT make it + // faster (measured: 8 GiB in 860 ms chunked-parallel vs 371 ms serial). So + // instead take it off the critical path entirely: release on a background + // thread while the main thread proceeds. Caller must hand over EXCLUSIVE + // ownership -- no reference to the arena (or memory inside it) may survive + // the thread_launch. + ProfBeginFunction(); + U64 begin_us = now_time_us(); + Arena *arena = raw_arena; + + U64 committed_size = 0; + for (Arena *n = arena->current; n != 0; n = n->prev) { committed_size += n->cmt; } +#if ARENA_FREE_LIST + for (Arena *n = arena->free_last; n != 0; n = n->prev) { committed_size += n->cmt; } +#endif + + arena_release(arena); + + lnk_log(LNK_Log_Timers, "[teardown] background release of %llu MiB arena took %.2f ms (off main thread)", + committed_size / MB(1), (F64)(now_time_us() - begin_us) / 1000.0); + ProfEnd(); +} + internal void lnk_discard_cv_debug_info(LNK_CodeViewInput *input, U64 obj_idx) { @@ -2842,9 +2878,14 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } // bucket_arr is fully consumed (copied into unique_leaf_refs / sorted) -- release the probe tables - // now so this multi-GB working set is gone before the merge-types/PDB-build peak. - arena_release(bucket_arena); + // now so this multi-GB working set is gone before the merge-types/PDB-build peak. Handed to a + // background reaper thread: a serial VirtualFree(MEM_RELEASE) of these multi-GB committed blocks + // costs ~350ms+ of main-thread kernel time (MiDeleteVaDirect/MiDecommitFreePage), which otherwise + // sits on the critical path between dedup and the type-index fixup passes. All pointers into the + // arena are dropped below before the launch returns ownership to the reaper. for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { task.leaf_ht_arr[ti_source].bucket_arr = 0; } + if (g_arena_reaper_thread.u64[0] != 0) { thread_join(g_arena_reaper_thread, max_U64); } + g_arena_reaper_thread = thread_launch(lnk_arena_release_thread, bucket_arena); #if PROFILE_TELEMETRY tmMessage(0, TMMF_ICON_NOTE, "TPI Count: %.*s", str8_varg(str8_from_count(scratch.arena, task.unique_leaf_refs_arr[CV_TypeIndexSource_TPI].count))); From 9bc6eb55183592208f8945959b7ed09e3f1d29ed Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 1 Jul 2026 14:47:28 -0700 Subject: [PATCH 019/121] radlink: patch debug-section relocs on private copies (keep CoW input pages clean) Debug sections (.debug$*) are not copied into the image, so lnk_obj_reloc_patcher used to commit relocations directly into the memory-mapped input (FILE_MAP_COPY), dirtying copy-on-write pages the kernel then has to reclaim at exit. Instead, copy each reloc-patched debug section of debug-info-surviving objs into private memory and patch the copy; readers fetch section bytes through lnk_obj_get_sect_data, which prefers the copy. Objs excluded from debug info never reach the PDB/RDI path, so their debug relocs are skipped entirely instead of patched pointlessly. Sections without relocs are consumed straight from the (clean) input view. Same reloc math, same phase order; GSI dedup and PDB module copies see identical post-fixup bytes, so DLL and PDB outputs are byte-identical by construction. Co-Authored-By: Claude Fable 5 (cherry picked from commit 074e5c219ad38d6d4d39e0257673b7e6d4925f81) --- src/linker/lnk.c | 25 ++++++++++++++++++++++--- src/linker/lnk_obj.c | 3 +++ src/linker/lnk_obj.h | 5 +++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index e59181992..92b641803 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -4566,12 +4566,31 @@ THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) if (section_flags & COFF_SectionFlag_LnkRemove) { continue; } if (section_flags & COFF_SectionFlag_CntUninitializedData) { continue; } + COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(obj, section_header); + // get section bytes (special case debug info because it is not copied to the image) Rng1U64 section_frange = rng_1u64(section_header->foff, section_header->foff + section_header->fsize); - String8 section_data = section_flags & LNK_SECTION_FLAG_DEBUG ? lnk_obj_section_data_from_number(obj, it.v.section_number) : str8_substr(task->image_data, section_frange); + String8 section_data; + if (section_flags & LNK_SECTION_FLAG_DEBUG) { + // Objs excluded from debug output have no later consumer for these bytes. + if (obj->exclude_from_debug_info) { continue; } + + // A relocation-free debug section can stay on the clean input view. + if (relocs.count == 0) { continue; } + + if (obj->section_data_copies == 0) { + obj->section_data_copies = push_array(arena, String8, obj->coff.sections.count_no_null + 1); + } + String8 src = str8_substr(obj->coff.data, section_frange); + U8 *copy = push_array_no_zero(arena, U8, src.size); + MemoryCopy(copy, src.str, src.size); + obj->section_data_copies[it.v.section_number] = str8(copy, src.size); + section_data = obj->section_data_copies[it.v.section_number]; + } else { + section_data = str8_substr(task->image_data, section_frange); + } // apply relocs (sorted by apply_off for monotone-forward image writes) - COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(obj, section_header); Temp reloc_temp = temp_begin(scratch.arena); LNK_RelocSortKey *sorted_relocs = push_array_no_zero(reloc_temp.arena, LNK_RelocSortKey, relocs.count); for EachIndex(reloc_idx, relocs.count) { @@ -6069,7 +6088,7 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT // patch relocs { LNK_ObjRelocPatcher task = { .image_data = image_data, .objs = objs, .image_base = pe.image_base, .image_section_table = image_section_table }; - tp_for_parallel_prof(tp, 0, objs_count, lnk_obj_reloc_patcher, &task, "Patch Relocs"); + tp_for_parallel_prof(tp, arena, objs_count, lnk_obj_reloc_patcher, &task, "Patch Relocs"); } // patch load config diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index b79edf31b..aa743be3b 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -677,6 +677,9 @@ internal String8 lnk_obj_section_data_from_number(LNK_Obj *obj, U64 section_number) { Assert(1 <= section_number && section_number <= obj->coff.sections.count_no_null); + if (obj->section_data_copies != 0 && obj->section_data_copies[section_number].size != 0) { + return obj->section_data_copies[section_number]; + } COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(obj->coff.data, obj->coff.header.section_table_range).str; COFF_SectionHeader *section = §ion_table[section_number-1]; return str8_substr(obj->coff.data, r1u64s(section->foff, section->fsize)); diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index e3d50cdba..8d9519e42 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -59,6 +59,11 @@ typedef struct LNK_Obj struct LNK_LibMemberRef *link_member; struct LNK_ObjNode *self; + // Reloc-patched private copies of debug sections, indexed by COFF section + // number (slot zero is null). Keeping patches out of the copy-on-write input + // view avoids committing private pages for the mapped OBJ. + String8 *section_data_copies; + // @type_server Rng1U64 ti_range; CV_TypeIndex *ti_map; From f832c09358575e84c4a5d337ff90ff04d59a0c13 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 1 Jul 2026 14:52:35 -0700 Subject: [PATCH 020/121] radlink: /PDBSTRIPPED strips proc itype in the destination buffer, not the source $$S The stripped-PDB pass zeroed CV_SymProc32.itype in the SOURCE symbol record and then copied it out, mutating $$S bytes (and dirtying their backing pages) after the full PDB was already built. Write the record first and zero the itype field in the destination copy instead; the source stays untouched. Stripped-PDB output bytes are unchanged. Co-Authored-By: Claude Fable 5 (cherry picked from commit acc3929165f33bf0e28924bd44b57bfffab1d3c1) --- src/linker/lnk.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 92b641803..f0c7340f7 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -6859,9 +6859,11 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); if (symbol.kind == CV_SymKind_SKIP) { continue; } if (cv_is_lproc(symbol)) { - CV_SymProc32 *src_proc = str8_deserial_get_raw_ptr(symbol.data, 0, sizeof(*src_proc)); - memory_write32(&src_proc->itype, 0); // strip type index + // strip the type index in the DESTINATION copy -- the source $$S stays untouched + // (patching the source would dirty its private/CoW backing pages for no reason) + U64 rec_off = buffer_cursor; buffer_cursor += cv_write_symbol(buffer, buffer_cursor, buffer_size, &symbol, CV_SymbolAlign); + memory_write32(buffer + rec_off + sizeof(CV_SymbolHeader) + OffsetOf(CV_SymProc32, itype), 0); buffer_cursor += cv_write_symbol(buffer, buffer_cursor, buffer_size, &(CV_Symbol){ .kind = CV_SymKind_END }, CV_SymbolAlign); } } From a21bd75279e0858bb8d88aae5b8e4eec73b6af56 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 1 Jul 2026 18:10:30 -0700 Subject: [PATCH 021/121] linker: size leaf dedup tables from a deterministic unique-count estimate On dup-heavy input (PCH/type-server fan-out) the unique leaf count is a small fraction of the total, and sizing the dedup probe tables from the total leaf count oversizes them 5-10x, costing tens of seconds of demand-zero page faults on 64B-apart random probes. Estimate the distinct-hash count per CV_TypeIndexSource from the already produced debug_h hashes (presence bitmap + linear counting) and size the tables from that, clamped to the old total-based caps. The bitmaps are filled with commutative atomic ORs over deterministic input hashes, so the estimate -- and therefore the caps -- are identical run to run. Winner selection per hash class (min lnk_leaf_ref_compare) and the sorted extraction are cap-independent: only probe sequences change, not probe logic, so output bytes are unchanged. Overflow safety: the table-full case (probe wraps without a slot, which previously only fired a debug Assert) now sets a shared flag; the passes bail out early and the whole dedup is redone once with the always sufficient total-based caps. Whether that happens is a pure function of the input, so the retry path is deterministic too. Co-Authored-By: Claude Fable 5 (cherry picked from commit ffe2aa1f007e9f1c245acf3a7e5735c5b59bafc3) --- src/linker/lnk_debug_info.c | 187 +++++++++++++++++++++++++++++++----- src/linker/lnk_debug_info.h | 11 +++ 2 files changed, 172 insertions(+), 26 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 4dd0520ca..e7f9fa393 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -2360,6 +2360,32 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_deep_task) ProfEnd(); } +internal +THREAD_POOL_TASK_FUNC(lnk_estimate_unique_leaves_task) +{ + ProfBeginFunction(); + LNK_MergeTypes *task = raw_task; + U64 obj_idx = task->indices.v[task_id]; + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + CV_DebugH *debug_h = &task->input->debug_h_arr[obj_idx]; + // same prune rule as lnk_leaf_dedup_task: NOTYPE'd IFC blob leaves were never hashed and are + // never inserted, so they must not contribute to the estimate either + B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); + for EachIndex(leaf_idx, debug_t->count) { + CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); + CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); + if (is_ifc_blob && kind == CV_LeafKind_NOTYPE) { continue; } + CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); + U64 bit_idx = debug_h->v[leaf_idx] & (task->estimate_bitmap_bits[leaf_source] - 1); + U32 *word = &task->estimate_bitmap[leaf_source][bit_idx / 32]; + U32 bit = 1u << (bit_idx % 32); + // atomic OR is commutative -> final bitmap contents are schedule-independent (deterministic); + // pre-check skips the interlocked op for already-set bits (the common case on dup-heavy input) + if ((ins_atomic_u32_eval(word) & bit) == 0) { ins_atomic_u32_or(word, bit); } + } + ProfEnd(); +} + internal THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) { @@ -2370,8 +2396,12 @@ THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) CV_DebugH *debug_h = &task->input->debug_h_arr[task->pop_obj_idx]; for EachInRange(leaf_idx, task->pop_range[task_id]) { - - LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); + // another worker overflowed an estimate-sized table -- the whole dedup result is discarded + // and retried with the total-based caps, so bail out early + if (ins_atomic_u32_eval(&task->leaf_ht_overflow) != 0) { break; } + + LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); + B32 is_inserted_or_updated = 1; CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); // leaf index -> leaf header CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); // leaf header -> leaf kind CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); // leaf kind -> type stream @@ -2391,8 +2421,15 @@ THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) // advance to next bucket idx = ((idx + 1) == leaf_ht->cap ? 0 : (idx + 1)); } while (idx != best_idx); - InvalidPath; + is_inserted_or_updated = 0; exit:; + if (!is_inserted_or_updated && leaf_source != CV_TypeIndexSource_NULL) { + // TPI/IPI table is full (estimate undershot) -- flag for a deterministic retry with the + // total-based caps. the NULL-source table is deliberately undersized and silently drops + // leaves that do not fit (pre-existing behavior; they are never emitted). + ins_atomic_u32_eval_assign(&task->leaf_ht_overflow, 1); + break; + } } } @@ -2403,10 +2440,15 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) U64 obj_idx = task->indices.v[task_id]; CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; CV_DebugH *debug_h = &task->input->debug_h_arr[obj_idx]; + B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); ProfBeginDynamic("dedup in obj 0x%llx (%.*s) leaf count %llu", obj_idx, str8_varg(task->input->obj_arr[obj_idx]->path), debug_t->count); for EachIndex(leaf_idx, debug_t->count) { + // another worker overflowed an estimate-sized table -- the whole dedup result is discarded + // and retried with the total-based caps, so bail out early + if (ins_atomic_u32_eval(&task->leaf_ht_overflow) != 0) { break; } + if (is_ifc_blob && cv_debug_t_get_leaf_header(debug_t, leaf_idx)->kind == CV_LeafKind_NOTYPE) { continue; } B32 is_inserted_or_updated = 1; @@ -2444,7 +2486,13 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) is_inserted_or_updated = 0; exit:; - Assert(is_inserted_or_updated); + if (!is_inserted_or_updated && leaf_source != CV_TypeIndexSource_NULL) { + // TPI/IPI table is full (estimate undershot) -- flag for a deterministic retry with the + // total-based caps. the NULL-source table is deliberately undersized and silently drops + // leaves that do not fit (pre-existing behavior; they are never emitted). + ins_atomic_u32_eval_assign(&task->leaf_ht_overflow, 1); + break; + } } ProfEnd(); @@ -2801,26 +2849,92 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK Arena *bucket_arena = arena_alloc(.name = "LEAF_BUCKETS"); ProfBegin("Leaf Hash Table Init"); - for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { - U64 total_count = 0; - for EachIndex(obj_idx, input->count) { total_count += input->debug_t_arr[obj_idx].source_counts[ti_source]; } + // fallback caps derived from TOTAL (pre-dedup, pre-prune) leaf counts. total >= unique always, so + // these caps can never overflow; they are also the caps the estimate-based sizing clamps against + // and retries with. NOTE: the NULL-source cap is deliberately derived from the pre-prune + // source_counts (see the IFC pruning comment in lnk_leaf_dedup_task) -- pruned NOTYPE leaves are + // never inserted, so pre-prune totals always cover the insert set with slack. + U64 leaf_ht_cap_fallback[CV_TypeIndexSource_COUNT] = {0}; + { + U64 total_counts[CV_TypeIndexSource_COUNT] = {0}; + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + for EachIndex(obj_idx, input->count) { total_counts[ti_source] += input->debug_t_arr[obj_idx].source_counts[ti_source]; } + // pow2 cap so bucket index is hash & (cap-1) (mask) instead of hash % cap (a 64-bit DIV in the + // densest dedup probe loop). u64_up_to_pow2(1.3*count) keeps load factor <= ~0.65. + leaf_ht_cap_fallback[ti_source] = u64_up_to_pow2(1 + ((total_counts[ti_source] * 13) / 10)); // * 1.3, pow2 + } - task.leaf_ht_arr[ti_source].cap = total_count; - task.leaf_ht_arr[ti_source].cap = 1 + ((task.leaf_ht_arr[ti_source].cap * 13) / 10); // * 1.3 - task.leaf_ht_arr[ti_source].bucket_arr = push_array_no_zero(bucket_arena, LNK_LeafRef, task.leaf_ht_arr[ti_source].cap); - MemorySet(task.leaf_ht_arr[ti_source].bucket_arr, 0xff, sizeof(LNK_LeafRef) * task.leaf_ht_arr[ti_source].cap); + // On dup-heavy input (PCH/type-server fan-out) unique count is a small fraction of total, and a + // total-sized probe table wastes multi-GB of demand-zero page faults on 64B-apart random probes. + // Estimate the distinct-hash count from the already-produced debug_h hashes (Produce Hashes + // completes above) with a per-source presence bitmap + linear counting, and size the tables from + // that instead. Everything here is a pure function of the input hashes, so the caps -- and the + // overflow/retry decision below -- are identical run to run. + ProfBegin("Estimate Unique Leaves"); + { + // sweep exactly the objs whose leaves get inserted: prepopulate + the four dedup passes + U32Array sweep_arrs[] = { input->debug_p_indices, input->int_obj_indices, input->type_server_indices, input->ifc_indices }; + U32Array sweep_indices = {0}; + for EachElement(i, sweep_arrs) { sweep_indices.count += sweep_arrs[i].count; } + sweep_indices.v = push_array_no_zero(scratch.arena, U32, sweep_indices.count); + sweep_indices.count = 0; + for EachElement(i, sweep_arrs) { + MemoryCopy(sweep_indices.v + sweep_indices.count, sweep_arrs[i].v, sizeof(U32) * sweep_arrs[i].count); + sweep_indices.count += sweep_arrs[i].count; + } + + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + // bits >= total >= unique keeps the bitmap load < 1, where linear counting is accurate; + // clamp keeps the transient bitmap allocation bounded (16MB per source at the top end) + task.estimate_bitmap_bits[ti_source] = u64_up_to_pow2(Clamp(1ull << 16, total_counts[ti_source], 1ull << 27)); + task.estimate_bitmap [ti_source] = push_array(scratch.arena, U32, task.estimate_bitmap_bits[ti_source] / 32); + } + + task.indices = sweep_indices; + tp_for_parallel(tp, 0, task.indices.count, lnk_estimate_unique_leaves_task, &task); + + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + U64 bit_count = task.estimate_bitmap_bits[ti_source]; + U64 word_count = bit_count / 32; + U64 set_count = 0; + for EachIndex(word_idx, word_count) { set_count += count_bits_set32(task.estimate_bitmap[ti_source][word_idx]); } + + U64 cap = leaf_ht_cap_fallback[ti_source]; + U64 zero_count = bit_count - set_count; + // the NULL-source table keeps the historic total-based cap: objs synthesized outside the + // parse path never populate source_counts, so its cap can be far below the number of + // NULL-source leaves thrown at it. that has always been tolerated -- NULL-source leaves are + // never emitted, the table just drops what does not fit (see the fall-through handling in + // lnk_leaf_dedup_task) -- so it must not participate in estimate sizing or overflow retry. + if (ti_source != CV_TypeIndexSource_NULL && set_count > 0 && zero_count > 0) { + // linear counting: distinct ~= m * ln(m / zeros) + F64 estimate = (F64)bit_count * log((F64)bit_count / (F64)zero_count); + if (estimate < (F64)set_count) { estimate = (F64)set_count; } + // 1.9x safety keeps the load factor <= ~0.55 even before pow2 rounding; overflow (only + // possible if the estimate undershoots by >1.8x) is caught and retried deterministically + U64 target = (U64)(estimate * 1.9) + 4096; + cap = Min(u64_up_to_pow2(target), leaf_ht_cap_fallback[ti_source]); + } + task.leaf_ht_arr[ti_source].cap = cap; + } + } + ProfEnd(); + + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + task.leaf_ht_arr[ti_source].bucket_arr = push_array_no_zero(bucket_arena, LNK_LeafRef, task.leaf_ht_arr[ti_source].cap); + MemorySet(task.leaf_ht_arr[ti_source].bucket_arr, 0xff, sizeof(LNK_LeafRef) * task.leaf_ht_arr[ti_source].cap); #if PROFILE_TELEMETRY - tmMessage(0, TMMF_ICON_NOTE, "%.*s Bucket Count: %.*s", str8_varg(cv_string_from_type_index_source(ti_source)), str8_varg(str8_from_count(scratch.arena, task.leaf_ht_arr[ti_source].cap))); + tmMessage(0, TMMF_ICON_NOTE, "%.*s Bucket Count: %.*s", str8_varg(cv_string_from_type_index_source(ti_source)), str8_varg(str8_from_count(scratch.arena, task.leaf_ht_arr[ti_source].cap))); #endif + } } ProfEnd(); U32Array dedup_type_server_indices = input->type_server_indices; - ProfBegin("Prepopulate hash table with largest type-set"); + LNK_TypeServer *largest_ts = 0; { - LNK_TypeServer *largest_ts = 0; for EachIndex(i, input->ts_arr.count) { LNK_TypeServer *ts = &input->ts_arr.v[i]; if (ts->rrt == 0) { continue; } @@ -2832,7 +2946,6 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK if (largest_ts) { task.pop_obj_idx = input->ts_obj_range.min + largest_ts->ts_idx; task.pop_range = tp_divide_work(scratch.arena, task.input->debug_t_arr[task.pop_obj_idx].count, tp->worker_count); - tp_for_parallel(tp, tp_temp, tp->worker_count, lnk_populate_leaf_ht, &task); U32Array new_dedup_type_server_indices = { .v = push_array(scratch.arena, U32, input->type_server_indices.count) }; for EachIndex(i, input->type_server_indices.count) { @@ -2842,21 +2955,43 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK dedup_type_server_indices = new_dedup_type_server_indices; } } - ProfEnd(); - ProfBegin("Leaf Dedup"); - task.indices = input->debug_p_indices; - tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, ".debug$P"); + for (U64 attempt = 0; ; attempt += 1) { + ProfBegin("Prepopulate hash table with largest type-set"); + if (largest_ts) { + tp_for_parallel(tp, tp_temp, tp->worker_count, lnk_populate_leaf_ht, &task); + } + ProfEnd(); - task.indices = input->int_obj_indices; - tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, ".debug$T"); + ProfBegin("Leaf Dedup"); + task.indices = input->debug_p_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, ".debug$P"); - task.indices = dedup_type_server_indices; - tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, "Type Servers"); + task.indices = input->int_obj_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, ".debug$T"); - task.indices = input->ifc_indices; - tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, "IFC Blobs"); - ProfEnd(); + task.indices = dedup_type_server_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, "Type Servers"); + + task.indices = input->ifc_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, "IFC Blobs"); + ProfEnd(); + + if (ins_atomic_u32_eval(&task.leaf_ht_overflow) == 0) { break; } + + // an estimate-sized table overflowed: rebuild every probe table at the always-sufficient + // total-based caps and redo the passes. the input hashes are deterministic, so this branch is + // taken (or not) identically every run, and the retried result is what the total-based sizing + // would have produced in the first place. the fallback caps cannot overflow, so at most one retry. + AssertAlways(attempt == 0); + lnk_log(LNK_Log_Debug, "leaf dedup: unique-count estimate overflowed, retrying with total-based table caps"); + ins_atomic_u32_eval_assign(&task.leaf_ht_overflow, 0); + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + task.leaf_ht_arr[ti_source].cap = leaf_ht_cap_fallback[ti_source]; + task.leaf_ht_arr[ti_source].bucket_arr = push_array_no_zero(bucket_arena, LNK_LeafRef, leaf_ht_cap_fallback[ti_source]); + MemorySet(task.leaf_ht_arr[ti_source].bucket_arr, 0xff, sizeof(LNK_LeafRef) * leaf_ht_cap_fallback[ti_source]); + } + } ProfBegin("Extract present buckets from the leaf hash tables"); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 169ec6a3a..a963a7b3f 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -208,6 +208,17 @@ typedef struct U64 pop_obj_idx; Rng1U64 *pop_range; + // deterministic unique-leaf estimate: distinct-hash bitmaps (per ti source) filled with + // commutative atomic ORs over the precomputed debug_h hashes -> same input, same bits, same + // estimate every run. sized pow2 so bit index is hash & (bits-1). + U32 *estimate_bitmap [CV_TypeIndexSource_COUNT]; // [estimate_bitmap_bits/32] + U64 estimate_bitmap_bits[CV_TypeIndexSource_COUNT]; // pow2 + + // set when a probe wraps without finding a slot (estimate-sized table overflowed); dedup is + // retried once with the always-sufficient total-based caps. deterministic: overflow happens + // iff the unique count exceeds cap, which is a function of the input alone. + U32 leaf_ht_overflow; + LNK_MergedTypes result; } LNK_MergeTypes; From 636412afd041d6bb7c97ac6994dda677c086b608 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 1 Jul 2026 18:13:16 -0700 Subject: [PATCH 022/121] linker: shard GSI/PSI symbol inserts by bucket range lnk_move_global_symbols_to_gsi hashed globals, proc refs, and publics in parallel but then funneled every gsi_push_ through a task 0 serial loop while the other workers parked on the barrier. Shard the inserts instead: worker i owns buckets [i*B/W, (i+1)*B/W) and walks the full symbol sequence in global order, inserting only symbols whose hash % bucket_count lands in its range. Each bucket has exactly one owner and receives its inserts in global sequence order, so per-chain order -- which is serialized into the PDB -- is byte-identical to the serial loop, for any worker count. No locks or atomics; gsi->symbol_count is bumped once by task 0 behind a barrier. The publics walk previously consumed the per-worker lists in place (clearing node->next during iteration), which would race with concurrent shard walkers; flatten the lists into a global-order node/hash array first, then shard-insert from that. Barrier passes walk the fixed lane partition strided by the pinned cohort so no lane is dropped under /RAD_SHARED_THREAD_POOL fair-share. Co-Authored-By: Claude Fable 5 (cherry picked from commit c820b64b14bcfc2111b17e9f0aa29e496455e11b) --- src/linker/lnk_debug_info.c | 238 +++++++++++++++++++++++------------- src/linker/lnk_debug_info.h | 6 +- 2 files changed, 161 insertions(+), 83 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index e7f9fa393..0bb2ed5d9 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -1976,7 +1976,8 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 max_weight = CeilIntegerDiv(total_input_size, tp->worker_count); U64 cursor = 0; - input.symbol_input_ranges = push_array(tp_arena->v[0], Rng1U64, tp->worker_count); + input.symbol_input_ranges = push_array(tp_arena->v[0], Rng1U64, tp->worker_count); + input.symbol_input_range_count = tp->worker_count; for EachIndex(i, tp->worker_count) { if (cursor >= input.symbol_input_count) { break; } U64 begin = cursor; @@ -3322,23 +3323,30 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) ProfBegin("Global Symbols"); { + // FAIR-SHARE: symbol_input_ranges is a FIXED [symbol_input_range_count] partition built at + // full pool width, but this barrier pass runs at the pinned cohort C == tp->worker_count + // (C <= fixed). Walk the fixed lanes strided by the cohort so every fixed lane is processed + // exactly once for any C; at C == fixed this degenerates to lane == task_id (one lane each, + // identical to the old direct indexing). VoidList global_symbols = {0}; - for EachInRange(i, task->cv->symbol_input_ranges[task_id]) { - LNK_SymbolInput symbols = task->cv->symbol_inputs[i]; - for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); + for (U64 lane = task_id; lane < task->cv->symbol_input_range_count; lane += tp->worker_count) { + for EachInRange(i, task->cv->symbol_input_ranges[lane]) { + LNK_SymbolInput symbols = task->cv->symbol_inputs[i]; + for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); - if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { - void *ptr = cv_ptr_from_symbol(symbol); - void_list_push(scratch.arena, &global_symbols, ptr); - } + if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { + void *ptr = cv_ptr_from_symbol(symbol); + void_list_push(scratch.arena, &global_symbols, ptr); + } - if (cv_is_scope_symbol(symbol.kind)) { - depth += 1; - } else if (cv_is_end_symbol(symbol.kind)) { - if (depth == 0) { Assert(0 && "malformed symbol stream"); break; } - depth -= 1; + if (cv_is_scope_symbol(symbol.kind)) { + depth += 1; + } else if (cv_is_end_symbol(symbol.kind)) { + if (depth == 0) { Assert(0 && "malformed symbol stream"); break; } + depth -= 1; + } } } } @@ -3393,17 +3401,31 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } barrier_wait(tp->barrier); - // push global symbols + // push global symbols, sharded by bucket range: worker i owns buckets [i*B/W, (i+1)*B/W) and + // walks the FULL symbol sequence in global order, inserting only symbols whose bucket lands in + // its range. each bucket has a single owner and receives its inserts in global sequence order, + // so per-chain order (which is serialized into the PDB) is byte-identical to a serial loop, for + // any worker count -- no locks, no atomics. + CV_SymbolNode *global_nodes = 0; if (task_id == 0) { - CV_SymbolNode *nodes = push_array_no_zero(gsi->arena, CV_SymbolNode, symbol_count); + global_nodes = push_array_no_zero(gsi->arena, CV_SymbolNode, symbol_count); + } + tp_broadcast(&global_nodes); + { + U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; + U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; for EachIndex(i, symbol_count) { - CV_SymbolNode *n = &nodes[i]; + U64 bucket_idx = symbol_hashes[i] % gsi->bucket_count; + if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } + CV_SymbolNode *n = &global_nodes[i]; n->prev = n->next = 0; n->data = cv_symbol_from_ptr(symbol_arr[i]); n->data.offset = i; - gsi_push_(gsi, symbol_hashes[i], n); + cv_symbol_list_push_node(&gsi->bucket_arr[bucket_idx], n); } } + barrier_wait(tp->barrier); + if (task_id == 0) { gsi->symbol_count += symbol_count; } } ProfEnd(); @@ -3495,59 +3517,76 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } barrier_wait(tp->barrier); - // push proc refs - if (task_id == 0) { - U64 total_proc_ref_count = sum_array_u64(tp->worker_count, proc_ref_counts); - for EachIndex(i, total_proc_ref_count) { gsi_push_(gsi, proc_ref_hashes[i], &proc_ref_nodes[i]); } + // push proc refs, sharded by bucket range (single owner per bucket, inserts in global node + // order -> per-chain order identical to a serial loop for any worker count) + { + U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; + U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; + for EachIndex(i, total_proc_ref_count) { + U64 bucket_idx = (U32)proc_ref_hashes[i] % gsi->bucket_count; + if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } + cv_symbol_list_push_node(&gsi->bucket_arr[bucket_idx], &proc_ref_nodes[i]); + } } barrier_wait(tp->barrier); + if (task_id == 0) { gsi->symbol_count += total_proc_ref_count; } } ProfEnd(); ProfBegin("Public Symbols"); { - U64 *public_symbol_sizes = 0; // [worker_count] - U64 *public_symbol_node_counts = 0; // [worker_count] + // FAIR-SHARE: task->symtab->chunks is a FIXED [symtab->arena->count] partition built at + // full pool width, but this barrier pass runs at the pinned cohort C == tp->worker_count + // (C <= fixed). Walk the fixed lanes strided by the cohort so every fixed lane is + // processed exactly once for any C, and keep every per-lane array in FIXED-lane order so + // the flattened global order below (which feeds the sharded PSI insert) is byte-identical + // to a full-width run. At C == fixed the strided loops degenerate to lane == task_id. + U64 fixed_lane_count = task->symtab->arena->count; + + U64 *public_symbol_sizes = 0; // [fixed_lane_count] + U64 *public_symbol_node_counts = 0; // [fixed_lane_count] if (task_id == 0) { - public_symbol_sizes = push_array(scratch.arena, U64, tp->worker_count); - public_symbol_node_counts = push_array(scratch.arena, U64, tp->worker_count); + public_symbol_sizes = push_array(scratch.arena, U64, fixed_lane_count); + public_symbol_node_counts = push_array(scratch.arena, U64, fixed_lane_count); } tp_broadcast(&public_symbol_sizes); tp_broadcast(&public_symbol_node_counts); // compute buffer size for CV public symbols - LNK_SymbolHashTrieChunkList symbol_chunks = task->symtab->chunks[task_id]; - U64 public_symbol_size = 0; - U64 public_symbol_count = 0; - for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { - for EachIndex(i, chunk->count) { - LNK_Symbol *symbol = chunk->v[i].symbol; - LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); - - if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } - COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); - if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } - - public_symbol_size += AlignPow2(sizeof(CV_SymPub32) + symbol->name.size + 1, sizeof(void *)); - public_symbol_count += 1; - public_symbol_node_counts[task_id] += 1; + for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { + LNK_SymbolHashTrieChunkList symbol_chunks = task->symtab->chunks[lane]; + U64 public_symbol_size = 0; + U64 public_symbol_count = 0; + for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { + for EachIndex(i, chunk->count) { + LNK_Symbol *symbol = chunk->v[i].symbol; + LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); + + if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } + COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); + if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } + + public_symbol_size += AlignPow2(sizeof(CV_SymPub32) + symbol->name.size + 1, sizeof(void *)); + public_symbol_count += 1; + public_symbol_node_counts[lane] += 1; + } } + public_symbol_sizes [lane] += public_symbol_size; + public_symbol_node_counts[lane] += public_symbol_count; } - public_symbol_sizes [task_id] += public_symbol_size; - public_symbol_node_counts[task_id] += public_symbol_count; barrier_wait(tp->barrier); Arena **public_symbol_arenas = 0; Arena **public_symbol_node_arenas = 0; - CV_SymbolList *public_symbols = 0; // [worker_count] - U32 **public_symbol_hashes = 0; // [worker_count][public_symbol.count] + CV_SymbolList *public_symbols = 0; // [fixed_lane_count] + U32 **public_symbol_hashes = 0; // [fixed_lane_count][public_symbol.count] if (task_id == 0) { - U64 public_symbol_total_count = sum_array_u64(tp->worker_count, public_symbol_node_counts); - public_symbol_arenas = alloc_arena_many(psi->gsi->arena, tp->worker_count, public_symbol_sizes); - public_symbol_node_arenas = alloc_arena_array(psi->gsi->arena, tp->worker_count, public_symbol_node_counts, CV_SymbolNode); - public_symbols = push_array(scratch.arena, CV_SymbolList, tp->worker_count); - public_symbol_hashes = push_array(scratch.arena, U32 *, tp->worker_count); + U64 public_symbol_total_count = sum_array_u64(fixed_lane_count, public_symbol_node_counts); + public_symbol_arenas = alloc_arena_many(psi->gsi->arena, fixed_lane_count, public_symbol_sizes); + public_symbol_node_arenas = alloc_arena_array(psi->gsi->arena, fixed_lane_count, public_symbol_node_counts, CV_SymbolNode); + public_symbols = push_array(scratch.arena, CV_SymbolList, fixed_lane_count); + public_symbol_hashes = push_array(scratch.arena, U32 *, fixed_lane_count); } tp_broadcast(&public_symbol_arenas); tp_broadcast(&public_symbol_node_arenas); @@ -3555,52 +3594,87 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) tp_broadcast(&public_symbol_hashes); // make CV public symbols - Arena *public_symbol_arena = public_symbol_arenas [task_id]; - Arena *public_symbol_node_arena = public_symbol_node_arenas[task_id]; - CV_SymbolList *public_symbol_list = &public_symbols [task_id]; - for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { - for EachIndex(i, chunk->count) { - LNK_Symbol *symbol = chunk->v[i].symbol; - LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); - - // discard removed and non-section symbols - if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } - COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); - if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } - - CV_Pub32Flags flags = COFF_SymbolType_IsFunc(symbol_parsed.type) ? CV_Pub32Flag_Function : 0; - ISectOff sc = lnk_sc_from_symbol(symbol); - CV_Symbol pub_symbol = cv_make_pub32(public_symbol_arena, flags, safe_cast_u32(sc.off), safe_cast_u16(sc.isect), symbol->name); - cv_symbol_list_push(public_symbol_node_arena, public_symbol_list, pub_symbol); + for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { + LNK_SymbolHashTrieChunkList symbol_chunks = task->symtab->chunks[lane]; + Arena *public_symbol_arena = public_symbol_arenas [lane]; + Arena *public_symbol_node_arena = public_symbol_node_arenas[lane]; + CV_SymbolList *public_symbol_list = &public_symbols [lane]; + for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { + for EachIndex(i, chunk->count) { + LNK_Symbol *symbol = chunk->v[i].symbol; + LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); + + // discard removed and non-section symbols + if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } + COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); + if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } + + CV_Pub32Flags flags = COFF_SymbolType_IsFunc(symbol_parsed.type) ? CV_Pub32Flag_Function : 0; + ISectOff sc = lnk_sc_from_symbol(symbol); + CV_Symbol pub_symbol = cv_make_pub32(public_symbol_arena, flags, safe_cast_u32(sc.off), safe_cast_u16(sc.isect), symbol->name); + cv_symbol_list_push(public_symbol_node_arena, public_symbol_list, pub_symbol); + } } } barrier_wait(tp->barrier); // hash public symbols - { + for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { U64 hash_idx = 0; - U32 *hashes = push_array(scratch.arena, U32, public_symbols[task_id].count); - for EachNode(n, CV_SymbolNode, public_symbols[task_id].first) { + U32 *hashes = push_array(scratch.arena, U32, public_symbols[lane].count); + for EachNode(n, CV_SymbolNode, public_symbols[lane].first) { String8 name = cv_name_from_symbol(n->data.kind, n->data.data); hashes[hash_idx++] = gsi_hash(gsi, name); } - public_symbol_hashes[task_id] = hashes; + public_symbol_hashes[lane] = hashes; } barrier_wait(tp->barrier); - // insert public symbols into PSI + // flatten the per-worker symbol lists (in worker order, matching the old serial walk) into one + // global-order node/hash array, so the sharded insert below can walk it without racing on the + // list links + U64 public_symbol_total_count = 0; + U64 *public_symbol_offsets = 0; // [fixed_lane_count] + CV_SymbolNode **public_symbol_flat_nodes = 0; // [public_symbol_total_count] + U32 *public_symbol_flat_hashes = 0; // [public_symbol_total_count] if (task_id == 0) { - for EachIndex(i, tp->worker_count) { - U64 k = 0; - for (CV_SymbolNode *curr = public_symbols[i].first, *next = 0; curr != 0; curr = next, k += 1) { - next = curr->next; - curr->next = 0; - gsi_push_(psi->gsi, public_symbol_hashes[i][k], curr); - } + U64 *list_counts = push_array_no_zero(scratch.arena, U64, fixed_lane_count); + for EachIndex(i, fixed_lane_count) { list_counts[i] = public_symbols[i].count; } + public_symbol_offsets = offsets_from_counts_array_u64(scratch.arena, list_counts, fixed_lane_count); + public_symbol_total_count = sum_array_u64(fixed_lane_count, list_counts); + public_symbol_flat_nodes = push_array_no_zero(scratch.arena, CV_SymbolNode *, public_symbol_total_count); + public_symbol_flat_hashes = push_array_no_zero(scratch.arena, U32, public_symbol_total_count); + } + tp_broadcast(&public_symbol_total_count); + tp_broadcast(&public_symbol_offsets); + tp_broadcast(&public_symbol_flat_nodes); + tp_broadcast(&public_symbol_flat_hashes); + for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { + U64 cursor = public_symbol_offsets[lane]; + U64 k = 0; + for (CV_SymbolNode *curr = public_symbols[lane].first; curr != 0; curr = curr->next, k += 1) { + public_symbol_flat_nodes [cursor] = curr; + public_symbol_flat_hashes[cursor] = public_symbol_hashes[lane][k]; + cursor += 1; + } + } + barrier_wait(tp->barrier); + + // insert public symbols into PSI, sharded by bucket range (single owner per bucket, inserts in + // global order -> per-chain order identical to a serial loop for any worker count) + { + PDB_GsiContext *pub_gsi = psi->gsi; + U64 shard_min = (task_id * pub_gsi->bucket_count) / tp->worker_count; + U64 shard_max = ((task_id + 1) * pub_gsi->bucket_count) / tp->worker_count; + for EachIndex(i, public_symbol_total_count) { + U64 bucket_idx = public_symbol_flat_hashes[i] % pub_gsi->bucket_count; + if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } + cv_symbol_list_push_node(&pub_gsi->bucket_arr[bucket_idx], public_symbol_flat_nodes[i]); } } barrier_wait(tp->barrier); + if (task_id == 0) { psi->gsi->symbol_count += public_symbol_total_count; } } ProfEnd(); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index a963a7b3f..6ae4b527c 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -105,9 +105,13 @@ typedef struct U64 symbol_input_count; LNK_SymbolInput *symbol_inputs; // [symbol_input_count] - Rng1U64 *symbol_input_ranges; // [worker_count] + Rng1U64 *symbol_input_ranges; // [symbol_input_range_count] U64 symbol_patch_task_count; // LNK_SymbolInputTask *symbol_patch_task; // [symbol_patch_task_count] + // FAIR-SHARE: fixed lane count symbol_input_ranges was built for (full pool width at build + // time). Barrier passes may run at a pinned cohort C < this; they must walk lanes + // [task_id, symbol_input_range_count) strided by the cohort, NOT index by task_id alone. + U64 symbol_input_range_count; // IFC (header-unit debug-record) resolution: // redirects a consuming obj's local LF_IFC_RECORD placeholder TI to a leaf in From 8d54f4eceb2397face56ead5bec090dc9bbb31b4 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 1 Jul 2026 18:17:21 -0700 Subject: [PATCH 023/121] codeview: replace per-TI linked-list type-index offsets with static descriptor tables cv_get_leaf/symbol_type_index_offsets built an arena-pushed CV_TypeIndexInfo node per type index per record, and hot loops (leaf hashing, TI fixup, type GC) pointer-chased those lists; the two functions showed ~13s combined exclusive CPU across workers on editor-scale links. New CV_TiOffsets view returns, with zero allocation for the common kinds: - fixed-shape leaves/symbols: pointer into a static per-kind {source,offset} table (POINTER picks between two static variants off attribs) - count-stride kinds (ARGLIST, SUBSTR_LIST, BUILDINFO, VFTPATH, CALLERS/CALLEES/INLINEES): inline {run_base, run_count} descriptor, offset(i) = run_base + i*4 - member-walk kinds (FIELDLIST/METHODLIST/inlinee lines) keep their walk and materialize a flat CV_TiOff array (doubling growth, footprint comparable to the old per-node list) Emission order is preserved exactly (incl. FUNC_ID IPI-before-TPI and UDT_SRC_LINE TPI-then-IPI asymmetries); the blake3 leaf-hash stream and all consumers see identical offset sequences, so output bytes are unchanged. All hot consumers (lnk_hash_cv_leaf(_deep), lnk_fixup_cv_type_indices, lnk_gc_visit_offsets/expand, ifc closure, pdb_builder TI patch) now iterate the flat view; legacy list API kept as a thin shim over the new one. Co-Authored-By: Claude Fable 5 (cherry picked from commit b36f595830d52dc3f59b70b72d5d9d04b4f69533) --- src/codeview/codeview.h | 26 +++ src/codeview/codeview_parse.c | 387 ++++++++++++++++++++----------- src/codeview/codeview_parse.h | 6 + src/linker/lnk_debug_info.c | 171 +++++++------- src/linker/lnk_debug_info.h | 4 +- src/linker/pdb_ext/pdb_builder.c | 15 +- 6 files changed, 389 insertions(+), 220 deletions(-) diff --git a/src/codeview/codeview.h b/src/codeview/codeview.h index 196e19960..b433af228 100644 --- a/src/codeview/codeview.h +++ b/src/codeview/codeview.h @@ -2701,6 +2701,32 @@ struct CV_TypeIndexInfoList CV_TypeIndexInfo *last; }; +typedef struct CV_TiOff CV_TiOff; +struct CV_TiOff +{ + CV_TypeIndexSource source; + U32 offset; +}; + +// Flat, allocation-free view over a record's type-index sites. For any record kind at +// most one of the two parts is populated: +// - `arr`: fixed-shape kinds point at a static per-kind table (zero allocation); +// member-walk kinds (FIELDLIST/METHODLIST/inlinee lines) point at an arena- +// materialized array. +// - homogeneous run (count-stride kinds: ARGLIST, SUBSTR_LIST, BUILDINFO, VFTPATH, +// CALLERS/CALLEES/INLINEES): offset(i) = run_base + i*sizeof(CV_TypeIndex). +// Emission order (arr order, then ascending run) matches the legacy +// CV_TypeIndexInfoList push order exactly; hash streams depend on it. +typedef struct CV_TiOffsets CV_TiOffsets; +struct CV_TiOffsets +{ + const CV_TiOff *arr; + U32 arr_count; + CV_TypeIndexSource run_source; + U32 run_base; + U32 run_count; +}; + typedef struct CV_TypeIndexArray CV_TypeIndexArray; struct CV_TypeIndexArray { diff --git a/src/codeview/codeview_parse.c b/src/codeview/codeview_parse.c index 3997d4a47..bb0b84c5e 100644 --- a/src/codeview/codeview_parse.c +++ b/src/codeview/codeview_parse.c @@ -588,180 +588,269 @@ cv_symbol_type_index_info_push(Arena *arena, CV_TypeIndexInfoList *list, CV_Type SLLQueuePush(list->first, list->last, info); list->count += 1; - + return info; } -internal CV_TypeIndexInfoList -cv_get_symbol_type_index_offsets(Arena *arena, CV_SymKind kind, String8 data) +//////////////////////////////// +//~ Type-index offset descriptors +// +// Static per-kind tables for fixed-shape records; homogeneous runs for +// count-stride records. Table entry order mirrors the legacy +// cv_symbol_type_index_info_push order exactly (incl. FUNC_ID IPI-before-TPI +// and UDT_SRC_LINE TPI-then-IPI asymmetries) -- leaf hash streams depend on it. + +#define CV_TIOFF(s, o) { CV_TypeIndexSource_##s, (U32)(o) } + +// leaves +read_only global CV_TiOff cv_tioffs_lf_modifier[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafModifier, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_pointer[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafPointer, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_pointer_ex[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafPointer, itype)), CV_TIOFF(TPI, sizeof(CV_LeafPointer) + 0) }; +read_only global CV_TiOff cv_tioffs_lf_array[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafArray, entry_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafArray, index_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_struct[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafStruct, field_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafStruct, derived_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafStruct, vshape_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_struct2[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafStruct2, field_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafStruct2, derived_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafStruct2, vshape_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_union[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafUnion, field_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_alias[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafAlias, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_func_id[] = { CV_TIOFF(IPI, OffsetOf(CV_LeafFuncId, scope_string_id)), CV_TIOFF(TPI, OffsetOf(CV_LeafFuncId, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_mfunc_id[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafMFuncId, owner_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafMFuncId, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_string_id[] = { CV_TIOFF(IPI, OffsetOf(CV_LeafStringId, substr_list_id)) }; +read_only global CV_TiOff cv_tioffs_lf_udt_src_line[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafUDTSrcLine, udt_itype)), CV_TIOFF(IPI, OffsetOf(CV_LeafUDTSrcLine, src_string_id)) }; +read_only global CV_TiOff cv_tioffs_lf_udt_mod_src_line[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafUDTModSrcLine, udt_itype)), CV_TIOFF(IPI, OffsetOf(CV_LeafUDTModSrcLine, src_string_id)) }; +read_only global CV_TiOff cv_tioffs_lf_enum[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafEnum, base_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafEnum, field_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_procedure[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafProcedure, ret_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafProcedure, arg_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_mfunction[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafMFunction, ret_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafMFunction, class_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafMFunction, this_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafMFunction, arg_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_vftable[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafVFTable, owner_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafVFTable, base_table_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_skip[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafSkip, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_method[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafMethod, list_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_onemethod[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafOneMethod, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_bitfield[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafBitField, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_index[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafIndex, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_member[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafMember, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_vfunctab[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafVFuncTab, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_vfuncoff[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafVFuncOff, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_nesttype[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafNestType, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_nesttypeex[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafNestTypeEx, itype)) }; + +// symbols +read_only global CV_TiOff cv_tioffs_s_buildinfo[] = { CV_TIOFF(IPI, OffsetOf(CV_SymBuildInfo, id)) }; +read_only global CV_TiOff cv_tioffs_s_data32[] = { CV_TIOFF(TPI, OffsetOf(CV_SymData32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_proc32_id[] = { CV_TIOFF(IPI, OffsetOf(CV_SymProc32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_proc32[] = { CV_TIOFF(TPI, OffsetOf(CV_SymProc32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_udt[] = { CV_TIOFF(TPI, OffsetOf(CV_SymUDT, itype)) }; +read_only global CV_TiOff cv_tioffs_s_thread32[] = { CV_TIOFF(TPI, OffsetOf(CV_SymThread32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_filestatic[] = { CV_TIOFF(TPI, OffsetOf(CV_SymFileStatic, itype)) }; +read_only global CV_TiOff cv_tioffs_s_local[] = { CV_TIOFF(TPI, OffsetOf(CV_SymLocal, itype)) }; +read_only global CV_TiOff cv_tioffs_s_regrel32[] = { CV_TIOFF(TPI, OffsetOf(CV_SymRegrel32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_register[] = { CV_TIOFF(TPI, OffsetOf(CV_SymRegister, itype)) }; +read_only global CV_TiOff cv_tioffs_s_constant[] = { CV_TIOFF(TPI, OffsetOf(CV_SymConstant, itype)) }; +read_only global CV_TiOff cv_tioffs_s_callsiteinfo[] = { CV_TIOFF(TPI, OffsetOf(CV_SymCallSiteInfo, itype)) }; +read_only global CV_TiOff cv_tioffs_s_inlinesite[] = { CV_TIOFF(IPI, OffsetOf(CV_SymInlineSite, inlinee)) }; +read_only global CV_TiOff cv_tioffs_s_heapallocsite[] = { CV_TIOFF(TPI, OffsetOf(CV_SymHeapAllocSite, itype)) }; + +#undef CV_TIOFF + +#define CV_TIOFFS_FIXED(result, table) do { (result).arr = (table); (result).arr_count = ArrayCount(table); } while (0) + +internal U64 +cv_ti_offsets_count(const CV_TiOffsets *offs) { - CV_TypeIndexInfoList list = {0}; + return (U64)offs->arr_count + (U64)offs->run_count; +} + +internal CV_TiOff +cv_ti_offset_at(const CV_TiOffsets *offs, U64 idx) +{ + CV_TiOff result; + if (idx < offs->arr_count) { + result = offs->arr[idx]; + } else { + result.source = offs->run_source; + result.offset = offs->run_base + (U32)(idx - offs->arr_count) * (U32)sizeof(CV_TypeIndex); + } + return result; +} + +// grow-by-doubling scratch for member-walk kinds (FIELDLIST/METHODLIST/inlinee +// lines); memory footprint stays comparable to the legacy per-node list +typedef struct CV_TiOffBuilder +{ + Arena *arena; + CV_TiOff *v; + U32 count; + U32 cap; +} CV_TiOffBuilder; + +internal void +cv_tioff_builder_put(CV_TiOffBuilder *b, CV_TypeIndexSource source, U64 offset) +{ + if (b->count == b->cap) { + U32 new_cap = b->cap ? b->cap * 2 : 16; + CV_TiOff *new_v = push_array_no_zero(b->arena, CV_TiOff, new_cap); + MemoryCopy(new_v, b->v, sizeof(CV_TiOff) * b->count); + b->v = new_v; + b->cap = new_cap; + } + b->v[b->count].source = source; + b->v[b->count].offset = (U32)offset; + b->count += 1; +} + +internal CV_TiOffsets +cv_symbol_ti_offsets(CV_SymKind kind, String8 data) +{ + CV_TiOffsets result = {0}; switch (kind) { case CV_SymKind_BUILDINFO: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_SymBuildInfo, id)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_buildinfo); } break; case CV_SymKind_GDATA32: case CV_SymKind_LDATA32: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymData32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_data32); } break; case CV_SymKind_LPROC32_ID: - case CV_SymKind_GPROC32_ID: + case CV_SymKind_GPROC32_ID: case CV_SymKind_LPROC32_DPC_ID: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_SymProc32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_proc32_id); } break; case CV_SymKind_GPROC32: - case CV_SymKind_LPROC32: + case CV_SymKind_LPROC32: case CV_SymKind_LPROC32_DPC: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymProc32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_proc32); } break; case CV_SymKind_UDT: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymUDT, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_udt); } break; case CV_SymKind_GTHREAD32: case CV_SymKind_LTHREAD32: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymThread32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_thread32); } break; case CV_SymKind_FILESTATIC: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymFileStatic, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_filestatic); } break; case CV_SymKind_LOCAL: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymLocal, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_local); } break; - case CV_SymKind_REGREL32: + case CV_SymKind_REGREL32: case CV_SymKind_BPREL32: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymRegrel32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_regrel32); } break; case CV_SymKind_REGISTER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymRegister, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_register); } break; case CV_SymKind_CONSTANT: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymConstant, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_constant); } break; case CV_SymKind_CALLSITEINFO: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymCallSiteInfo, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_callsiteinfo); } break; case CV_SymKind_CALLERS: case CV_SymKind_CALLEES: case CV_SymKind_INLINEES: { Assert(data.size >= sizeof(CV_SymFunctionList)); CV_SymFunctionList *func_list = (CV_SymFunctionList*)data.str; - for (U64 i = 0; i < func_list->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, sizeof(CV_SymFunctionList) + i * sizeof(CV_TypeIndex)); - } + result.run_source = CV_TypeIndexSource_IPI; + result.run_base = sizeof(CV_SymFunctionList); + result.run_count = func_list->count; } break; case CV_SymKind_INLINESITE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_SymInlineSite, inlinee)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_inlinesite); } break; case CV_SymKind_HEAPALLOCSITE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymHeapAllocSite, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_heapallocsite); } break; } - return list; + return result; } -internal CV_TypeIndexInfoList -cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data) +internal CV_TiOffsets +cv_leaf_ti_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data) { - CV_TypeIndexInfoList list = {0}; + CV_TiOffsets result = {0}; switch (leaf_kind) { case CV_LeafKind_NOTYPE: case CV_LeafKind_VTSHAPE: case CV_LeafKind_LABEL: - case CV_LeafKind_NULL: + case CV_LeafKind_NULL: case CV_LeafKind_NOTTRAN: { // no type indices } break; case CV_LeafKind_MODIFIER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafModifier, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_modifier); } break; case CV_LeafKind_POINTER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafPointer, itype)); - CV_LeafPointer *ptr = (CV_LeafPointer *)data.str; - CV_PointerKind ptr_kind = CV_PointerAttribs_Extract_Kind(ptr->attribs); + CV_LeafPointer *ptr = (CV_LeafPointer *)data.str; + CV_PointerKind ptr_kind = CV_PointerAttribs_Extract_Kind(ptr->attribs); if (ptr_kind == CV_PointerKind_BaseType) { // TODO: add CV_LeafPointerBaseType - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, sizeof(CV_LeafPointer) + 0); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_pointer_ex); } else { CV_PointerMode ptr_mode = CV_PointerAttribs_Extract_Mode(ptr->attribs); if (ptr_mode == CV_PointerMode_PtrMem || ptr_mode == CV_PointerMode_PtrMethod) { // TODO: add type for the CvLeafPointerMember to syms_cv.mc - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, sizeof(CV_LeafPointer) + 0); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_pointer_ex); + } else { + CV_TIOFFS_FIXED(result, cv_tioffs_lf_pointer); } } } break; case CV_LeafKind_ARRAY: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafArray, entry_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafArray, index_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_array); } break; - case CV_LeafKind_CLASS: + case CV_LeafKind_CLASS: case CV_LeafKind_STRUCTURE: case CV_LeafKind_INTERFACE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct, field_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct, derived_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct, vshape_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_struct); } break; case CV_LeafKind_CLASS2: case CV_LeafKind_STRUCT2: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct2, field_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct2, derived_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct2, vshape_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_struct2); } break; case CV_LeafKind_UNION: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafUnion, field_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_union); } break; case CV_LeafKind_ALIAS: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafAlias, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_alias); } break; case CV_LeafKind_FUNC_ID: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_LeafFuncId, scope_string_id)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafFuncId, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_func_id); } break; case CV_LeafKind_MFUNC_ID: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFuncId, owner_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFuncId, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_mfunc_id); } break; case CV_LeafKind_STRING_ID: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_LeafStringId, substr_list_id)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_string_id); } break; case CV_LeafKind_UDT_SRC_LINE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafUDTSrcLine, udt_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_LeafUDTSrcLine, src_string_id)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_udt_src_line); } break; case CV_LeafKind_UDT_MOD_SRC_LINE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafUDTModSrcLine, udt_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_LeafUDTModSrcLine, src_string_id)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_udt_mod_src_line); } break; case CV_LeafKind_BUILDINFO: { Assert(data.size >= sizeof(CV_LeafBuildInfo)); CV_LeafBuildInfo *build_info = (CV_LeafBuildInfo *)data.str; - for (U16 i = 0; i < build_info->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, sizeof(CV_LeafBuildInfo) + i * sizeof(CV_ItemId)); - } + result.run_source = CV_TypeIndexSource_IPI; + result.run_base = sizeof(CV_LeafBuildInfo); + result.run_count = build_info->count; } break; case CV_LeafKind_ENUM: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafEnum, base_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafEnum, field_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_enum); } break; case CV_LeafKind_PROCEDURE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafProcedure, ret_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafProcedure, arg_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_procedure); } break; case CV_LeafKind_MFUNCTION: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFunction, ret_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFunction, class_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFunction, this_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFunction, arg_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_mfunction); } break; case CV_LeafKind_VFTABLE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafVFTable, owner_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafVFTable, base_table_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_vftable); } break; case CV_LeafKind_VFTPATH: { Assert(sizeof(CV_LeafVFPath) <= data.size); CV_LeafVFPath *vfpath = (CV_LeafVFPath *)data.str; - for (U32 i = 0; i < vfpath->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, sizeof(CV_LeafVFPath) + i * sizeof(CV_TypeId)); - } + result.run_source = CV_TypeIndexSource_TPI; + result.run_base = sizeof(CV_LeafVFPath); + result.run_count = vfpath->count; } break; case CV_LeafKind_TYPESERVER: case CV_LeafKind_TYPESERVER2: @@ -769,76 +858,77 @@ cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data // no type indices } break; case CV_LeafKind_SKIP: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafSkip, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_skip); } break; case CV_LeafKind_SUBSTR_LIST: { Assert(sizeof(CV_LeafArgList) <= data.size); CV_LeafArgList *arg_list = (CV_LeafArgList*)data.str; - for (U32 i = 0; i < arg_list->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, sizeof(CV_LeafArgList) + i * sizeof(CV_TypeIndex)); - } + result.run_source = CV_TypeIndexSource_IPI; + result.run_base = sizeof(CV_LeafArgList); + result.run_count = arg_list->count; } break; case CV_LeafKind_ARGLIST: { Assert(sizeof(CV_LeafArgList) <= data.size); CV_LeafArgList *arg_list = (CV_LeafArgList*)data.str; - for (U32 i = 0; i < arg_list->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, sizeof(CV_LeafArgList) + i * sizeof(CV_TypeIndex)); - } + result.run_source = CV_TypeIndexSource_TPI; + result.run_base = sizeof(CV_LeafArgList); + result.run_count = arg_list->count; } break; - case CV_LeafKind_LIST: + case CV_LeafKind_LIST: case CV_LeafKind_FIELDLIST: { + CV_TiOffBuilder b = { arena }; for (U64 cursor = 0; cursor < data.size; ) { CV_LeafKind list_member_kind = 0; U64 read_size = str8_deserial_read_struct(data, cursor, &list_member_kind); - + if(read_size != sizeof(list_member_kind)) { Assert(!"malformed LF_FIELDLIST"); break; } cursor += read_size; - + switch (list_member_kind) { default: Assert(!"TODO: handle malformed field member"); break; case CV_LeafKind_INDEX: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafIndex, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafIndex, itype)); cursor += sizeof(CV_LeafIndex); } break; case CV_LeafKind_MEMBER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMember, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMember, itype)); cursor += sizeof(CV_LeafMember); - + CV_NumericParsed size; cursor += cv_read_numeric(data, cursor, &size); - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_STMEMBER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafStMember, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafStMember, itype)); cursor += sizeof(CV_LeafStMember); - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_METHOD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMethod, list_itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMethod, list_itype)); cursor += sizeof(CV_LeafMethod); - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_ONEMETHOD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafOneMethod, itype)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafOneMethod, itype)); + CV_LeafOneMethod onemethod; cursor += str8_deserial_read_struct(data, cursor, &onemethod); - + CV_MethodProp prop = CV_FieldAttribs_Extract_MethodProp(onemethod.attribs); if(prop == CV_MethodProp_PureIntro || prop == CV_MethodProp_Intro) { cursor += sizeof(U32); // virtoff } - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; @@ -851,134 +941,140 @@ cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_NESTTYPE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafNestType, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafNestType, itype)); cursor += sizeof(CV_LeafNestType); - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_NESTTYPEEX: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafNestTypeEx, itype)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafNestTypeEx, itype)); + cursor += sizeof(CV_LeafNestTypeEx); String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_BCLASS: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafBClass, itype)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafBClass, itype)); + cursor += sizeof(CV_LeafBClass); CV_NumericParsed offset; cursor += cv_read_numeric(data, cursor, &offset); } break; case CV_LeafKind_VBCLASS: case CV_LeafKind_IVBCLASS: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVBClass, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVBClass, itype)); cursor += sizeof(CV_LeafVBClass); - + CV_NumericParsed virtual_base_pointer; cursor += cv_read_numeric(data, cursor, &virtual_base_pointer); - + CV_NumericParsed virtual_base_offset; cursor += cv_read_numeric(data, cursor, &virtual_base_offset); } break; case CV_LeafKind_VFUNCTAB: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVFuncTab, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVFuncTab, itype)); cursor += sizeof(CV_LeafVFuncTab); } break; case CV_LeafKind_VFUNCOFF: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVFuncOff, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVFuncOff, itype)); cursor += sizeof(CV_LeafVFuncOff); } break; } cursor = AlignPow2(cursor, 4); } + result.arr = b.v; + result.arr_count = b.count; } break; case CV_LeafKind_METHOD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMethod, list_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_method); } break; case CV_LeafKind_METHODLIST: { + CV_TiOffBuilder b = { arena }; for (U64 cursor = 0; cursor < data.size; ) { // read method CV_LeafMethodListMember method; U64 read_size = str8_deserial_read_struct(data, cursor, &method); - + // error check read if (read_size != sizeof(method)) { Assert(!"malformed LF_METHODLIST"); break; } - + // push type index offset - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMethodListMember, itype)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMethodListMember, itype)); + // take into account intro virtual offset CV_MethodProp mprop = CV_FieldAttribs_Extract_MethodProp(method.attribs); if (mprop == CV_MethodProp_Intro || mprop == CV_MethodProp_PureIntro) { read_size += sizeof(U32); } - + // advance cursor += read_size; } + result.arr = b.v; + result.arr_count = b.count; } break; case CV_LeafKind_ONEMETHOD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafOneMethod, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_onemethod); } break; case CV_LeafKind_BITFIELD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafBitField, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_bitfield); } break; case CV_LeafKind_PRECOMP: case CV_LeafKind_REFSYM: { // no type indices } break; case CV_LeafKind_INDEX: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafIndex, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_index); } break; case CV_LeafKind_MEMBER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMember, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_member); } break; case CV_LeafKind_VFUNCTAB: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafVFuncTab, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_vfunctab); } break; case CV_LeafKind_VFUNCOFF: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafVFuncOff, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_vfuncoff); } break; case CV_LeafKind_NESTTYPE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafNestType, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_nesttype); } break; case CV_LeafKind_NESTTYPEEX: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafNestTypeEx, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_nesttypeex); } break; default: { NotImplemented; } break; } - return list; + return result; } -internal CV_TypeIndexInfoList -cv_get_inlinee_type_index_offsets(Arena *arena, String8 raw_data) +internal CV_TiOffsets +cv_inlinee_ti_offsets(Arena *arena, String8 raw_data) { - CV_TypeIndexInfoList list = {0}; - + CV_TiOffsets result = {0}; + CV_TiOffBuilder b = { arena }; + U64 cursor = 0; - + // first four bytes are always signature CV_C13InlineeLinesSig sig = max_U32; cursor += str8_deserial_read_struct(raw_data, cursor, &sig); - + while(cursor < raw_data.size) { // read header CV_C13InlineeSourceLineHeader *header = (CV_C13InlineeSourceLineHeader *) str8_deserial_get_raw_ptr(raw_data, cursor, sizeof(CV_C13InlineeSourceLineHeader)); - + // store type index offset - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, cursor + OffsetOf(CV_C13InlineeSourceLineHeader, inlinee)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_IPI, cursor + OffsetOf(CV_C13InlineeSourceLineHeader, inlinee)); + // advance past header cursor += sizeof(*header); - + // skip extra files B32 has_extra_files = (sig == CV_C13InlineeLinesSig_EXTRA_FILES); if (has_extra_files) @@ -988,10 +1084,41 @@ cv_get_inlinee_type_index_offsets(Arena *arena, String8 raw_data) cursor += /* file id: */ sizeof(U32) * file_count; } } - + + result.arr = b.v; + result.arr_count = b.count; + return result; +} + +internal CV_TypeIndexInfoList +cv_ti_offsets_to_list(Arena *arena, CV_TiOffsets offs) +{ + CV_TypeIndexInfoList list = {0}; + for (U64 i = 0, count = cv_ti_offsets_count(&offs); i < count; i += 1) { + CV_TiOff ti = cv_ti_offset_at(&offs, i); + cv_symbol_type_index_info_push(arena, &list, ti.source, ti.offset); + } return list; } +internal CV_TypeIndexInfoList +cv_get_symbol_type_index_offsets(Arena *arena, CV_SymKind kind, String8 data) +{ + return cv_ti_offsets_to_list(arena, cv_symbol_ti_offsets(kind, data)); +} + +internal CV_TypeIndexInfoList +cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data) +{ + return cv_ti_offsets_to_list(arena, cv_leaf_ti_offsets(arena, leaf_kind, data)); +} + +internal CV_TypeIndexInfoList +cv_get_inlinee_type_index_offsets(Arena *arena, String8 raw_data) +{ + return cv_ti_offsets_to_list(arena, cv_inlinee_ti_offsets(arena, raw_data)); +} + internal String8Array cv_get_data_around_type_indices(Arena *arena, CV_TypeIndexInfoList ti_list, String8 data) { diff --git a/src/codeview/codeview_parse.h b/src/codeview/codeview_parse.h index c33377988..b922f7eaa 100644 --- a/src/codeview/codeview_parse.h +++ b/src/codeview/codeview_parse.h @@ -291,6 +291,12 @@ internal B32 cv_is_leaf_type_server(CV_LeafKind kind); internal B32 cv_is_leaf_pch(CV_LeafKind kind); internal CV_TypeIndexSource cv_type_index_source_from_leaf_kind(CV_LeafKind leaf_kind); +internal CV_TiOffsets cv_symbol_ti_offsets(CV_SymKind kind, String8 data); // never allocates +internal CV_TiOffsets cv_leaf_ti_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data); // allocates only for FIELDLIST/METHODLIST +internal CV_TiOffsets cv_inlinee_ti_offsets(Arena *arena, String8 raw_data); +internal U64 cv_ti_offsets_count(const CV_TiOffsets *offs); +internal CV_TiOff cv_ti_offset_at(const CV_TiOffsets *offs, U64 idx); + internal CV_TypeIndexInfoList cv_get_symbol_type_index_offsets(Arena *arena, CV_SymKind kind, String8 data); internal CV_TypeIndexInfoList cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data); internal CV_TypeIndexInfoList cv_get_inlinee_type_index_offsets(Arena *arena, String8 raw_data); diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 0bb2ed5d9..c3687a62f 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -1126,14 +1126,15 @@ THREAD_POOL_TASK_FUNC(lnk_ifc_close_blob_task) U64 leaf_idx = worklist[--wl_count]; CV_Leaf leaf = cv_debug_t_get_leaf(bdt, leaf_idx); Temp itemp = temp_begin(wtemp.arena); - CV_TypeIndexInfoList ti_list = cv_get_leaf_type_index_offsets(itemp.arena, leaf.kind, leaf.data); - for EachNode(ti_info, CV_TypeIndexInfo, ti_list.first) { - CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(leaf.data, ti_info->offset, sizeof(*ti_ptr)); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(itemp.arena, leaf.kind, leaf.data); + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff ti_info = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(leaf.data, ti_info.offset, sizeof(*ti_ptr)); if (ti_ptr == 0) { continue; } CV_TypeIndex sub_ti = memory_read32(ti_ptr); - if (sub_ti < bdt->ti_ranges[ti_info->source].min || - sub_ti >= bdt->ti_ranges[ti_info->source].max) { continue; } - U64 sub_leaf_idx = cv_leaf_idx_from_ti(bdt, ti_info->source, sub_ti); + if (sub_ti < bdt->ti_ranges[ti_info.source].min || + sub_ti >= bdt->ti_ranges[ti_info.source].max) { continue; } + U64 sub_leaf_idx = cv_leaf_idx_from_ti(bdt, ti_info.source, sub_ti); if (sub_leaf_idx >= bdt->count) { continue; } if (bits[sub_leaf_idx >> 3] & (1u << (sub_leaf_idx & 7))) { continue; } bits[sub_leaf_idx >> 3] |= (U8)(1u << (sub_leaf_idx & 7)); @@ -2131,7 +2132,7 @@ lnk_match_leaf_ref(LNK_CodeViewInput *input, LNK_LeafRef a, LNK_LeafRef b) } internal U64 -lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list, B32 discard_cycles) +lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs, B32 discard_cycles) { U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); @@ -2139,6 +2140,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); CV_TypeIndexSource curr_ti_source = cv_type_index_source_from_leaf_kind(leaf.kind); CV_TypeIndex curr_ti = cv_ti_from_leaf_idx(debug_t, curr_ti_source, leaf_idx); + U64 ti_count = cv_ti_offsets_count(&ti_offs); // init hasher LNK_Hasher hasher; @@ -2147,11 +2149,12 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf // hash bytes around indices { U64 last_ti_off = 0; - for EachNode(ti_info, CV_TypeIndexInfo, ti_info_list.first) { + for (U64 ti_idx = 0; ti_idx < ti_count; ti_idx += 1) { + CV_TiOff ti_info = cv_ti_offset_at(&ti_offs, ti_idx); U8 *bytes = leaf.data.str + last_ti_off; - U64 size = ti_info->offset - last_ti_off; + U64 size = ti_info.offset - last_ti_off; lnk_hasher_update(&hasher, bytes, size); - last_ti_off = ti_info->offset + sizeof(CV_TypeIndex); + last_ti_off = ti_info.offset + sizeof(CV_TypeIndex); } Assert(leaf.data.size >= last_ti_off); @@ -2161,17 +2164,18 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf } // mix-in sub leaf hashes - for EachNode(sub_ti_n, CV_TypeIndexInfo, ti_info_list.first) { - CV_TypeIndex *sub_ti_ptr = str8_deserial_get_raw_ptr(leaf.data, sub_ti_n->offset, sizeof(*sub_ti_ptr)); + for (U64 ti_idx = 0; ti_idx < ti_count; ti_idx += 1) { + CV_TiOff sub_ti_n = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *sub_ti_ptr = str8_deserial_get_raw_ptr(leaf.data, sub_ti_n.offset, sizeof(*sub_ti_ptr)); CV_TypeIndex sub_ti = memory_read32(sub_ti_ptr); - - // simple indices are stable across compile units - if (sub_ti < debug_t->ti_ranges[sub_ti_n->source].min) { + + // simple indices are stable across compile units + if (sub_ti < debug_t->ti_ranges[sub_ti_n.source].min) { lnk_hasher_update_struct(&hasher, &sub_ti); continue; } - if (sub_ti >= debug_t->ti_ranges[sub_ti_n->source].max) { + if (sub_ti >= debug_t->ti_ranges[sub_ti_n.source].max) { // discard type U32 leaf_idx = curr_ti - debug_t->ti_ranges[curr_ti_source].min; U8 *leaf_header = debug_t->data.str + debug_t->offsets[leaf_idx]; @@ -2184,7 +2188,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf // log error Temp scratch = scratch_begin(0,0); String8 leaf_kind_str = cv_string_from_leaf_kind(leaf.kind); - String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) out of bounds type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, sub_ti_n->offset); + String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) out of bounds type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, (U64)sub_ti_n.offset); lnk_error_obj(LNK_Error_InvalidTypeIndex, input->obj_arr[obj_idx], "%S", error_msg); scratch_end(scratch); @@ -2206,7 +2210,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf // log error Temp scratch = scratch_begin(0,0); String8 leaf_kind_str = cv_string_from_leaf_kind(leaf.kind); - String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) forward refs member type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, sub_ti_n->offset); + String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) forward refs member type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, (U64)sub_ti_n.offset); lnk_error_obj(LNK_Error_InvalidTypeIndex, input->obj_arr[obj_idx], "%S", error_msg); scratch_end(scratch); @@ -2214,7 +2218,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf } // type index -> hash - LNK_LeafRef sub_ref = lnk_leaf_ref_from_ti(input, obj_idx, sub_ti_n->source, sub_ti); + LNK_LeafRef sub_ref = lnk_leaf_ref_from_ti(input, obj_idx, sub_ti_n.source, sub_ti); U64 sub_hash = input->debug_h_arr[lnk_leaf_ref_obj_idx(sub_ref)].v[lnk_leaf_ref_leaf_idx(sub_ref)]; // mix-in sub-type hash @@ -2240,15 +2244,16 @@ internal void lnk_hash_cv_leaf_deep(Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef root_leaf_ref, - CV_TypeIndexInfoList root_ti_info_list) + CV_TiOffsets root_ti_offs) { Temp temp = temp_begin(arena); typedef struct HashStack { struct HashStack *next; LNK_LeafRef leaf_ref; - CV_TypeIndexInfoList ti_info_list; - CV_TypeIndexInfo *ti_info; + CV_TiOffsets ti_offs; + U64 ti_next; + U64 ti_count; CV_Leaf leaf; CV_TypeIndex ti; CV_TypeIndexSource ti_source; @@ -2259,29 +2264,30 @@ lnk_hash_cv_leaf_deep(Arena *arena, CV_DebugT *root_debug_t = &input->debug_t_arr[root_obj_idx]; HashStack *root_frame = push_array(temp.arena, HashStack, 1); root_frame->leaf_ref = root_leaf_ref; - root_frame->ti_info_list = root_ti_info_list; - root_frame->ti_info = root_ti_info_list.first; + root_frame->ti_offs = root_ti_offs; + root_frame->ti_next = 0; + root_frame->ti_count = cv_ti_offsets_count(&root_ti_offs); root_frame->leaf = cv_debug_t_get_leaf(root_debug_t, lnk_leaf_ref_leaf_idx(root_leaf_ref)); root_frame->ti_source = cv_type_index_source_from_leaf_kind(root_frame->leaf.kind); root_frame->ti = cv_ti_from_leaf_idx(root_debug_t, root_frame->ti_source, lnk_leaf_ref_leaf_idx(root_leaf_ref)); HashStack *stack = root_frame; while (stack) { - while (stack->ti_info) { - CV_TypeIndexInfo *ti_info = stack->ti_info; + while (stack->ti_next < stack->ti_count) { + CV_TiOff ti_info = cv_ti_offset_at(&stack->ti_offs, stack->ti_next); // advance iterator - stack->ti_info = stack->ti_info->next; + stack->ti_next += 1; // get type index info - CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(stack->leaf.data, ti_info->offset, sizeof(*ti_ptr)); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(stack->leaf.data, ti_info.offset, sizeof(*ti_ptr)); CV_TypeIndex ti = memory_read32(ti_ptr); // skip out of bounds indices - if ( ! contains_1u64(input->debug_t_arr[root_obj_idx].ti_ranges[ti_info->source], ti)) { continue; } + if ( ! contains_1u64(input->debug_t_arr[root_obj_idx].ti_ranges[ti_info.source], ti)) { continue; } // skip hashed types - LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(input, root_obj_idx, ti_info->source, ti); + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(input, root_obj_idx, ti_info.source, ti); U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); if (input->debug_h_arr[obj_idx].v[leaf_idx] != 0) { continue; } @@ -2291,17 +2297,18 @@ lnk_hash_cv_leaf_deep(Arena *arena, HashStack *frame = push_array(temp.arena, HashStack, 1); frame->leaf_ref = leaf_ref; frame->leaf = cv_debug_t_get_leaf(&input->debug_t_arr[obj_idx], leaf_idx); - frame->ti_info_list = cv_get_leaf_type_index_offsets(temp.arena, frame->leaf.kind, frame->leaf.data); - frame->ti_info = frame->ti_info_list.first; + frame->ti_offs = cv_leaf_ti_offsets(temp.arena, frame->leaf.kind, frame->leaf.data); + frame->ti_next = 0; + frame->ti_count = cv_ti_offsets_count(&frame->ti_offs); frame->ti = ti; - frame->ti_source = ti_info->source; + frame->ti_source = ti_info.source; SLLStackPush(stack, frame); break; } // no more type indices, pop frame - if ( ! stack->ti_info) { - lnk_hash_cv_leaf(input, stack->leaf_ref, stack->ti_info_list, 0); + if (stack->ti_next >= stack->ti_count) { + lnk_hash_cv_leaf(input, stack->leaf_ref, stack->ti_offs, 0); SLLStackPop(stack); } } @@ -2334,10 +2341,10 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_task) U32 obj_idx = task->indices.v[task_id]; CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; for EachIndex(leaf_idx, debug_t->count) { - Temp temp = temp_begin(task->fixed_arenas[worker_id]); - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); - CV_TypeIndexInfoList ti_list = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - lnk_hash_cv_leaf(task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_list, 1); + Temp temp = temp_begin(task->fixed_arenas[worker_id]); + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + lnk_hash_cv_leaf(task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_offs, 1); temp_end(temp); } ProfEnd(); @@ -2350,12 +2357,14 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_deep_task) LNK_MergeTypes *task = raw_task; U64 obj_idx = task->indices.v[task_id]; CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); for EachIndex(leaf_idx, debug_t->count) { if (task->input->debug_h_arr[obj_idx].v[leaf_idx] != 0) { continue; } - Temp temp = temp_begin(task->fixed_arenas[worker_id]); - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); - CV_TypeIndexInfoList ti_list = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - lnk_hash_cv_leaf_deep(temp.arena, task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_list); + if (is_ifc_blob && cv_debug_t_get_leaf_header(debug_t, leaf_idx)->kind == CV_LeafKind_NOTYPE) { continue; } + Temp temp = temp_begin(task->fixed_arenas[worker_id]); + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + lnk_hash_cv_leaf_deep(temp.arena, task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_offs); temp_end(temp); } ProfEnd(); @@ -2574,17 +2583,18 @@ THREAD_POOL_TASK_FUNC(lnk_assign_type_indices_task) } internal void -lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_TypeIndexInfoList ti_info_list) +lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_TiOffsets ti_offs) { - for EachNode(n, CV_TypeIndexInfo, ti_info_list.first) { - CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(data, n->offset, sizeof(*ti_ptr)); + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(data, n.offset, sizeof(*ti_ptr)); CV_TypeIndex ti = memory_read32(ti_ptr); // skip basic types - if (ti < ctx->input->min_type_indices[n->source]) { continue; } + if (ti < ctx->input->min_type_indices[n.source]) { continue; } - LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n->source, ti); - CV_TypeIndex final_ti = lnk_assigned_ti_hash_search(&ctx->assigned_ti_arr[n->source], ctx->input, leaf_ref); + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n.source, ti); + CV_TypeIndex final_ti = lnk_assigned_ti_hash_search(&ctx->assigned_ti_arr[n.source], ctx->input, leaf_ref); memory_write32(ti_ptr, final_ti); #if LNK_PARANOID @@ -2604,15 +2614,11 @@ THREAD_POOL_TASK_FUNC(lnk_cv_patcher_symbols_task) for EachInRange(i, range) { LNK_SymbolInput symbols = task->input->symbol_inputs[i]; for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { - Temp temp = temp_begin(task->fixed_arenas[worker_id]); - CV_Symbol symbol = {0}; TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); - CV_TypeIndexInfoList ti_info_list = cv_get_symbol_type_index_offsets(temp.arena, symbol.kind, symbol.data); - lnk_fixup_cv_type_indices(task, symbols.obj_idx, symbol.data, ti_info_list); - - temp_end(temp); + CV_TiOffsets ti_offs = cv_symbol_ti_offsets(symbol.kind, symbol.data); + lnk_fixup_cv_type_indices(task, symbols.obj_idx, symbol.data, ti_offs); } } ProfEnd(); @@ -2628,8 +2634,8 @@ THREAD_POOL_TASK_FUNC(lnk_cv_patcher_inlines_task) Arena *fixed_arena = task->fixed_arenas[worker_id]; for EachNode(inline_data_n, String8Node, inlinee_lines.first) { Temp temp = temp_begin(fixed_arena); - CV_TypeIndexInfoList ti_info_list = cv_get_inlinee_type_index_offsets(temp.arena, inline_data_n->string); - lnk_fixup_cv_type_indices(task, obj_idx, inline_data_n->string, ti_info_list); + CV_TiOffsets ti_offs = cv_inlinee_ti_offsets(temp.arena, inline_data_n->string); + lnk_fixup_cv_type_indices(task, obj_idx, inline_data_n->string, ti_offs); temp_end(temp); } ProfEnd(); @@ -2644,12 +2650,13 @@ THREAD_POOL_TASK_FUNC(lnk_cv_patcher_leaves_task) Arena *fixed_arena = task->fixed_arenas[task_id]; for EachInRange(leaf_ref_idx, range) { Temp temp = temp_begin(fixed_arena); - LNK_LeafRef patch = task->unique_leaf_refs_arr[task->ti_source].v[leaf_ref_idx]; - U32 obj_idx = lnk_leaf_ref_obj_idx(patch); - CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, lnk_leaf_ref_leaf_idx(patch)); - CV_TypeIndexInfoList ti_info_list = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - lnk_fixup_cv_type_indices(task, obj_idx, leaf.data, ti_info_list); + LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[leaf_ref_idx]; + U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + lnk_fixup_cv_type_indices(task, obj_idx, leaf.data, ti_offs); temp_end(temp); } ProfEnd(); @@ -2773,7 +2780,10 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK Temp scratch = temp_begin(lnk_get_huge_arena()); LNK_MergeTypes task = { .input = input }; - U64 max_ti_list_size = sizeof(CV_TypeIndexInfo) * (max_U16 / sizeof(CV_TypeIndex)); + // scratch bound: CV_TiOff arrays for member-walk leaves are built with doubling growth + // (sum of caps <= ~4x entry count, entries <= max_U16/8 per leaf => <= ~512KB), plus + // deep-hash stack frames; 2x the legacy per-node list bound keeps comfortable headroom + U64 max_ti_list_size = 2 * sizeof(CV_TypeIndexInfo) * (max_U16 / sizeof(CV_TypeIndex)); task.fixed_arenas = alloc_fixed_size_arena_array(scratch.arena, tp->worker_count, max_ti_list_size, max_ti_list_size); ProfBegin("Produce Hashes"); @@ -4164,16 +4174,17 @@ lnk_gc_mark_ti(LNK_GCTypes *g, CV_TypeIndexSource s, CV_TypeIndex ti) // walk a record's type-index sites; mark roots (do_rewrite==0) or rewrite to compacted indices (==1) internal void -lnk_gc_visit_offsets(LNK_GCTypes *g, String8 data, CV_TypeIndexInfoList ti_info_list) +lnk_gc_visit_offsets(LNK_GCTypes *g, String8 data, CV_TiOffsets ti_offs) { - for EachNode(n, CV_TypeIndexInfo, ti_info_list.first) { - U8 *p = data.str + n->offset; + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&ti_offs, ti_idx); + U8 *p = data.str + n.offset; CV_TypeIndex ti = memory_read32(p); if (g->do_rewrite) { - U64 lo = g->min[n->source]; - if (ti >= lo) { U64 idx = ti - lo; if (idx < g->orig_n[n->source]) { memory_write32(p, g->remap[n->source][idx]); } } + U64 lo = g->min[n.source]; + if (ti >= lo) { U64 idx = ti - lo; if (idx < g->orig_n[n.source]) { memory_write32(p, g->remap[n.source][idx]); } } } else { - lnk_gc_mark_ti(g, n->source, ti); + lnk_gc_mark_ti(g, n.source, ti); } } } @@ -4186,12 +4197,9 @@ THREAD_POOL_TASK_FUNC(lnk_gc_syms_task) for EachInRange(i, g->sym_ranges[task_id]) { LNK_SymbolInput symbols = g->cv->symbol_inputs[i]; for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { - Temp temp = temp_begin(scratch.arena); CV_Symbol symbol = {0}; TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); - CV_TypeIndexInfoList l = cv_get_symbol_type_index_offsets(temp.arena, symbol.kind, symbol.data); - lnk_gc_visit_offsets(g, symbol.data, l); - temp_end(temp); + lnk_gc_visit_offsets(g, symbol.data, cv_symbol_ti_offsets(symbol.kind, symbol.data)); } } scratch_end(scratch); @@ -4206,7 +4214,7 @@ THREAD_POOL_TASK_FUNC(lnk_gc_inlines_task) String8List inlinee_lines = cv_sub_section_from_debug_s(g->cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); for EachNode(dn, String8Node, inlinee_lines.first) { Temp temp = temp_begin(scratch.arena); - CV_TypeIndexInfoList l = cv_get_inlinee_type_index_offsets(temp.arena, dn->string); + CV_TiOffsets l = cv_inlinee_ti_offsets(temp.arena, dn->string); lnk_gc_visit_offsets(g, dn->string, l); temp_end(temp); } @@ -4221,7 +4229,7 @@ THREAD_POOL_TASK_FUNC(lnk_gc_rewrite_leaves_task) for EachInRange(i, g->cur_ranges[task_id]) { Temp temp = temp_begin(scratch.arena); CV_Leaf leaf = cv_leaf_from_ptr(g->cur_leaf_v[i]); - CV_TypeIndexInfoList l = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); + CV_TiOffsets l = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); lnk_gc_visit_offsets(g, leaf.data, l); temp_end(temp); } @@ -4301,11 +4309,12 @@ THREAD_POOL_TASK_FUNC(lnk_gc_expand_task) Temp temp = temp_begin(scratch.arena); CV_Leaf leaf = cv_leaf_from_ptr(g->leaf_v[s][i]); - CV_TypeIndexInfoList l = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - for EachNode(n, CV_TypeIndexInfo, l.first) { - CV_TypeIndex ti = memory_read32(leaf.data.str + n->offset); - U64 lo = g->min[n->source]; - if (ti >= lo) { lnk_gc_mark_enqueue(g, n->source, ti - lo); } + CV_TiOffsets l = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&l); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&l, ti_idx); + CV_TypeIndex ti = memory_read32(leaf.data.str + n.offset); + U64 lo = g->min[n.source]; + if (ti >= lo) { lnk_gc_mark_enqueue(g, n.source, ti - lo); } } temp_end(temp); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 6ae4b527c..2d8c55fca 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -293,8 +293,8 @@ internal LNK_CodeViewInput lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp internal int lnk_leaf_ref_compare (LNK_LeafRef a, LNK_LeafRef b); internal B32 lnk_match_leaf_ref (LNK_CodeViewInput *input, LNK_LeafRef a, LNK_LeafRef b); -internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list, B32 discard_cycles); -internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list); +internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs, B32 discard_cycles); +internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs); internal CV_TypeIndex lnk_assigned_ti_hash_search (LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); internal LNK_MergedTypes lnk_merge_types (TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK_MergeTypeFlags merge_flags); internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP_Arena *arena, U64 leaf_count, U8 **leaf_arr, LNK_TypeNameHashMode mode, U64 hash_length, String8 map_name); diff --git a/src/linker/pdb_ext/pdb_builder.c b/src/linker/pdb_ext/pdb_builder.c index fbe2c9aed..71898a54d 100644 --- a/src/linker/pdb_ext/pdb_builder.c +++ b/src/linker/pdb_ext/pdb_builder.c @@ -1349,14 +1349,15 @@ pdb_load_types_from_leaf_list(PDB_TypeServer **type_server_arr, CV_LeafList leaf // get offsets for type indices in data blob CV_Leaf *leaf = &node->data; - CV_TypeIndexInfoList ti_info_list = cv_get_leaf_type_index_offsets(temp.arena, leaf->kind, leaf->data); - - for (CV_TypeIndexInfo *ti_info = ti_info_list.first; ti_info != 0; ti_info = ti_info->next) { - Assert(ti_info->offset + sizeof(CV_TypeIndex) <= leaf->data.size); - CV_TypeIndex *ti_ptr = (CV_TypeIndex *)(leaf->data.str + ti_info->offset); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf->kind, leaf->data); + + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff ti_info = cv_ti_offset_at(&ti_offs, ti_idx); + Assert(ti_info.offset + sizeof(CV_TypeIndex) <= leaf->data.size); + CV_TypeIndex *ti_ptr = (CV_TypeIndex *)(leaf->data.str + ti_info.offset); CV_TypeIndex external_ti = *ti_ptr; - - B32 is_complex_type = external_ti >= ti_map->min_itype[ti_info->source]; + + B32 is_complex_type = external_ti >= ti_map->min_itype[ti_info.source]; if (is_complex_type) { // search external type index CV_TypeIndex internal_tpi_idx = pdb_type_index_map_search(ti_map, CV_TypeIndexSource_TPI, external_ti); From ff830b73190fb7f15f669e1f4c33c9f895a73051 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 1 Jul 2026 18:04:04 -0700 Subject: [PATCH 024/121] linker: parallelize image align-byte fill (first-touch) across the pool The align-byte fill is the first touch of the freshly committed ~751MB image buffer (every page a demand-zero fault) and ran serially on the main thread while all workers parked. Range-split the fill into a flat (dst, byte, size) task list and dispatch it via tp_for_parallel: sections are cut into ~4MB chunks with PAGE-ALIGNED split points inside the page-aligned image reservation, so no two workers ever touch the same 4K page. Each task keeps the NT-store lnk_stream_set fill and sfences its own stores before signalling completion, so everything is globally visible after the join (replacing the single main-thread sfence). Measured on a FN editor DLL link (751MB image, 64 workers), via a throwaway timer around the fill block: serial 24.5-29.1ms -> parallel 4.7-5.0ms, i.e. about -20ms wall. Smaller than the profile-estimated 0.3-0.8s: Windows fault clustering + NT-store bandwidth make the serial first-touch much cheaper here than the estimate assumed. Writes are value-identical to the serial loop and byte-disjoint -> output is byte-identical by construction (verified: base-vs-head diff is the PE checksum byte + debug-directory GUID/timestamp band only, matching the same-binary control). Co-Authored-By: Claude Fable 5 (cherry picked from commit 51af7452a83d7a15184e5733baa160b82146e98b) --- src/linker/lnk.c | 71 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index f0c7340f7..54aa0fa9e 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -4510,6 +4510,25 @@ THREAD_POOL_TASK_FUNC(lnk_image_fill_task) ProfEnd(); } +typedef struct +{ + U8 *dst; + U64 size; + U8 byte; +} LNK_FillAlignRange; + +internal +THREAD_POOL_TASK_FUNC(lnk_fill_align_bytes_task) +{ + ProfBeginFunction(); + LNK_FillAlignRange *range = &((LNK_FillAlignRange *)raw_task)[task_id]; + lnk_stream_set(range->dst, range->byte, range->size); + // make this task's NT stores globally visible before the completion counter is + // bumped, so every thread past the join (contrib-fill / reloc passes) sees them + _mm_sfence(); + ProfEnd(); +} + internal U64 lnk_compute_win32_image_header_size(LNK_Config *config, U64 sect_count) { @@ -6013,17 +6032,49 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT ProfEnd(); ProfBegin("Fill Align Bytes"); - for EachNode(sect_n, LNK_SectionNode, sectab->list.first) { - LNK_Section *sect = §_n->data; - ProfBeginV("Section: %S Size: %M", sect->name, sect->fsize); - U8 fill_byte = sect->flags & COFF_SectionFlag_CntCode ? coff_code_align_byte_from_machine(config->machine) : 0; - // write-once into the image buffer -> stream past the cache (see lnk_stream_set) - lnk_stream_set(image_data.str + sect->foff, fill_byte, sect->fsize); - ProfEnd(); + { + // This is the first touch of the freshly committed ~image-size buffer: every + // page is a demand-zero fault. Serial on main, that soft-fault storm (plus the + // stream-set itself) parks all workers for the duration; range-split it across + // the pool instead. Split points are PAGE-ALIGNED in the image buffer (the + // reservation is page-aligned) so no two workers ever touch the same 4K page. + // Writes are value-identical to the serial loop and byte-disjoint -> byte-safe. + Temp fill_temp = temp_begin(scratch.arena); + U64 range_quantum = MB(4); + + // upper bound on range count + U64 range_cap = 0; + for EachNode(sect_n, LNK_SectionNode, sectab->list.first) { + range_cap += CeilIntegerDiv(sect_n->data.fsize, range_quantum) + 1; + } + + LNK_FillAlignRange *ranges = push_array_no_zero(fill_temp.arena, LNK_FillAlignRange, range_cap); + U64 range_count = 0; + for EachNode(sect_n, LNK_SectionNode, sectab->list.first) { + LNK_Section *sect = §_n->data; + U8 fill_byte = sect->flags & COFF_SectionFlag_CntCode ? coff_code_align_byte_from_machine(config->machine) : 0; + U64 pos = sect->foff; + U64 end = sect->foff + sect->fsize; + for (; pos < end; ) { + U64 next = AlignDownPow2(pos + range_quantum, KB(4)); + next = ClampTop(next, end); + if (next <= pos) { next = end; } + Assert(range_count < range_cap); + LNK_FillAlignRange *range = &ranges[range_count++]; + range->dst = image_data.str + pos; + range->size = next - pos; + range->byte = fill_byte; + pos = next; + } + } + + // write-once into the image buffer -> stream past the cache (see lnk_stream_set); + // each task sfences its own NT stores before signalling completion, so after the + // join every fill below is globally visible to the contrib-fill / reloc passes + tp_for_parallel(tp, 0, range_count, lnk_fill_align_bytes_task, ranges); + + temp_end(fill_temp); } - // flush NT stores before the parallel contrib-fill / reloc passes read or - // re-write these lines - _mm_sfence(); ProfEnd(); Temp temp = temp_begin(scratch.arena); From 2ffe6b3b75356f82be3b8cc0a554acbde2dbc86c Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 1 Jul 2026 18:12:57 -0700 Subject: [PATCH 025/121] arena: 2MB commit quantum for the link's high-churn arenas Bump the commit quantum from the 64KB default to 2MB for the three arena families that take essentially all of a 50GB editor link's commit churn: - thread-pool per-worker arenas (tp_arena_alloc) - per-thread scratch arenas (tctx_alloc) - the linker's huge debug-info arena (lnk_get_huge_arena) via their arena_alloc params only -- the global 64KB default is untouched, so small arenas do not bloat. arena_decommit_unused is page-granular and tracks cmt exactly, so it stays correct with the larger quantum. Measured on a FN editor DLL link (751MB image, ~50.7GB committed, 64 workers), via a throwaway atomic counter in commit_memory: VirtualAlloc(MEM_COMMIT) calls: 80.5-81.1K -> 8.3K (-90%) committed bytes: 50.7GB -> 51.3GB (+0.6GB quantum slack) peak working set: 56.4GB -> 56.8GB (+0.3-0.5GB) kernel time (GetProcessTimes): no detectable movement (210-282s run noise swamps it; faults, not commit syscalls, dominate kernel time on this link) So this is a syscall/address-space-lock relief change, not a measurable wall/kernel win on this workload; kept because the slack cost is bounded and small relative to the 56GB peak. Commit sizes only affect when pages are committed, never what is written -> output is byte-identical (verified: base-vs-head diff is the PE checksum byte + debug-directory GUID/timestamp band only, same as the same-binary control). Co-Authored-By: Claude Fable 5 (cherry picked from commit 11ac6b17a890256fab2a65fd7f29171cba9778db) --- src/base/base_thread_context.c | 14 ++++++++++---- src/linker/lnk_debug_info.c | 6 +++++- src/linker/thread_pool/thread_pool.c | 7 ++++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/base/base_thread_context.c b/src/base/base_thread_context.c index d42be68ed..df2ec71e5 100644 --- a/src/base/base_thread_context.c +++ b/src/base/base_thread_context.c @@ -17,15 +17,21 @@ C_LINKAGE thread_static TCTX *tctx_thread_local = 0; internal TCTX * tctx_alloc(void) { + // 2MB commit quantum for scratch arenas (vs the 64KB default): scratch takes + // heavy churn on every thread (the linker pushes tens of GB through these); + // the larger quantum cuts VirtualAlloc(MEM_COMMIT) calls -- all serialized on + // the process address-space lock -- ~32x. Slack is <= 2MB per scratch arena + // past its high-water mark (2 arenas per thread), and arena_decommit_unused + // still trims page-granular, independent of the commit quantum. #if PROFILE_TELEMETRY thread_static static char name[2][1024]; raddbg_snprintf(name[0], sizeof(name[0]), "Scratch/0[TID:%u]", tid()); raddbg_snprintf(name[1], sizeof(name[1]), "Scratch/1[TID:%u]", tid()); - Arena *arena_0 = arena_alloc(.name = name[0]); - Arena *arena_1 = arena_alloc(.name = name[1]); + Arena *arena_0 = arena_alloc(.commit_size = MB(2), .name = name[0]); + Arena *arena_1 = arena_alloc(.commit_size = MB(2), .name = name[1]); #else - Arena *arena_0 = arena_alloc(); - Arena *arena_1 = arena_alloc(); + Arena *arena_0 = arena_alloc(.commit_size = MB(2)); + Arena *arena_1 = arena_alloc(.commit_size = MB(2)); #endif TCTX *tctx = push_array(arena_0, TCTX, 1); tctx->arenas[0] = arena_0; diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index c3687a62f..b51c12b74 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -7,7 +7,11 @@ internal Arena * lnk_get_huge_arena(void) { if (g_huge_arena == 0) { - g_huge_arena = arena_alloc(.name = "HUGE"); + // 2MB commit quantum (vs the 64KB default): this arena backs multi-GB debug + // info merges; the larger quantum cuts VirtualAlloc(MEM_COMMIT) syscalls + // (all serialized on the process address-space lock) ~32x for at most 2MB + // of slack past the high-water mark. + g_huge_arena = arena_alloc(.commit_size = MB(2), .name = "HUGE"); } return g_huge_arena; } diff --git a/src/linker/thread_pool/thread_pool.c b/src/linker/thread_pool/thread_pool.c index d59216898..0eedd1b8a 100644 --- a/src/linker/thread_pool/thread_pool.c +++ b/src/linker/thread_pool/thread_pool.c @@ -335,7 +335,12 @@ tp_arena_alloc(TP_Context *pool) Temp scratch = scratch_begin(0,0); Arena **arr = push_array(scratch.arena, Arena *, pool->worker_count); for (U64 i = 0; i < pool->worker_count; ++i) { - arr[i] = arena_alloc("THREAD_POOL"); + // 2MB commit quantum: these per-worker arenas take the bulk of the link's + // ~50GB of MEM_COMMIT growth; the default 64KB quantum turns that into + // ~800K NtAllocateVirtualMemory calls from 64 threads serialized on the + // process address-space lock. Slack is bounded by workers x live arenas x + // quantum (single-digit MBs per worker), far below the syscall cost. + arr[i] = arena_alloc(.commit_size = MB(2), .name = "THREAD_POOL"); } Arena **dst = push_array(arr[0], Arena *, pool->worker_count); MemoryCopy(dst, arr, sizeof(Arena*) * pool->worker_count); From 7a754f6edcdbdf3fabaf51ca00641c8920e4fd46 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 1 Jul 2026 22:02:21 -0700 Subject: [PATCH 026/121] linker: sample the unique-leaf estimator at every 8th leaf position The pre-dedup unique-leaf estimator swept every debug_h hash into a presence bitmap (~6.6s of worker thread-time on editor-scale links). Sample every 8th leaf POSITION per obj instead: position-based sampling is a pure function of the input (schedule-independent), the presence bitmap shrinks 8x (cheaper cache footprint per update), and the sampled distinct count is scaled back up by a calibrated factor before the existing 1.9x safety + Min(fallback-cap) clamp. SCALE=5.0 satisfies SCALE*1.9 >= K=8, covering the worst-case sampled-to-true ratio for every duplication pattern, so the deterministic overflow-retry stays off; an undershoot would still be caught by that retry (exercised live during calibration at SCALE=1). Measured on the FN editor-scale link: estimate block 182.6 -> ~38 ms wall, caps byte-for-byte identical to the unsampled estimator (64M TPI / 16M IPI, load factors 0.351 / 0.387), output byte-identical. Also logs the estimate/caps/load factors under /RAD_LOG:TIMERS (output-neutral). Co-Authored-By: Claude Fable 5 (cherry picked from commit 857a941dc26432ea7cc44355c427ff58fb7ee25b) --- src/linker/lnk_debug_info.c | 38 ++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index b51c12b74..139590266 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -2374,6 +2374,25 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_deep_task) ProfEnd(); } +// Deterministic sampling for the unique-leaf estimator: process every K-th leaf POSITION per obj. +// Position-based (leaf_idx % K), never value-based, so the sampled set -- and therefore the +// estimate and the table caps -- is a pure function of the input, schedule-independent. The +// dominant duplication pattern is whole-stream duplication (the same PCH/type-server leaf sequence +// repeated across objs), where a unique hash sits at the SAME position in every copy: the sampled +// distinct count then scales ~1/K, which LNK_ESTIMATE_SAMPLE_SCALE compensates for. The scale is +// calibrated (see the estimate block in lnk_merge_types); an undershoot is caught by the existing +// deterministic overflow-retry at total-based caps, an overshoot is clamped by Min(fallback cap). +// +// SCALE calibration: sampled-distinct is between distinct (fully position-scattered duplication) +// and distinct/K (whole-stream duplication or unique-heavy input), so the true ratio is in [1, K]. +// SCALE * 1.9 (the downstream safety factor) must cover the worst-case ratio K to keep the +// overflow-retry off for every duplication pattern: SCALE = 5.0 gives 5.0*1.9 = 9.5 >= K = 8 +// (1.19x margin over the bound, which also absorbs linear-counting noise). Measured on the FN +// editor-scale link: ratio 3.99 (TPI) / 6.13 (IPI); SCALE = 5.0 reproduces the unsampled +// estimator's caps exactly (64M/16M) at load factors 0.35/0.39. +#define LNK_ESTIMATE_SAMPLE_STRIDE 8 +#define LNK_ESTIMATE_SAMPLE_SCALE 5.0 + internal THREAD_POOL_TASK_FUNC(lnk_estimate_unique_leaves_task) { @@ -2385,7 +2404,7 @@ THREAD_POOL_TASK_FUNC(lnk_estimate_unique_leaves_task) // same prune rule as lnk_leaf_dedup_task: NOTYPE'd IFC blob leaves were never hashed and are // never inserted, so they must not contribute to the estimate either B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); - for EachIndex(leaf_idx, debug_t->count) { + for (U64 leaf_idx = 0; leaf_idx < debug_t->count; leaf_idx += LNK_ESTIMATE_SAMPLE_STRIDE) { CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); if (is_ifc_blob && kind == CV_LeafKind_NOTYPE) { continue; } @@ -2886,6 +2905,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK // that instead. Everything here is a pure function of the input hashes, so the caps -- and the // overflow/retry decision below -- are identical run to run. ProfBegin("Estimate Unique Leaves"); + U64 estimate_begin_us = now_time_us(); { // sweep exactly the objs whose leaves get inserted: prepopulate + the four dedup passes U32Array sweep_arrs[] = { input->debug_p_indices, input->int_obj_indices, input->type_server_indices, input->ifc_indices }; @@ -2899,9 +2919,11 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { - // bits >= total >= unique keeps the bitmap load < 1, where linear counting is accurate; + // at most ceil(total/K) hashes are inserted under K-th-position sampling, so bits >= + // total/K >= sampled-unique keeps the bitmap load < 1, where linear counting is accurate; // clamp keeps the transient bitmap allocation bounded (16MB per source at the top end) - task.estimate_bitmap_bits[ti_source] = u64_up_to_pow2(Clamp(1ull << 16, total_counts[ti_source], 1ull << 27)); + U64 sampled_total = (total_counts[ti_source] + LNK_ESTIMATE_SAMPLE_STRIDE - 1) / LNK_ESTIMATE_SAMPLE_STRIDE; + task.estimate_bitmap_bits[ti_source] = u64_up_to_pow2(Clamp(1ull << 16, sampled_total, 1ull << 27)); task.estimate_bitmap [ti_source] = push_array(scratch.arena, U32, task.estimate_bitmap_bits[ti_source] / 32); } @@ -2925,14 +2947,20 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK // linear counting: distinct ~= m * ln(m / zeros) F64 estimate = (F64)bit_count * log((F64)bit_count / (F64)zero_count); if (estimate < (F64)set_count) { estimate = (F64)set_count; } + // scale the sampled distinct count back up to a full-population estimate (see the + // sampling comment above lnk_estimate_unique_leaves_task) + estimate *= LNK_ESTIMATE_SAMPLE_SCALE; // 1.9x safety keeps the load factor <= ~0.55 even before pow2 rounding; overflow (only // possible if the estimate undershoots by >1.8x) is caught and retried deterministically U64 target = (U64)(estimate * 1.9) + 4096; cap = Min(u64_up_to_pow2(target), leaf_ht_cap_fallback[ti_source]); + lnk_log(LNK_Log_Timers, "[typededup] estimate src=%llu: set=%llu est=%.0f cap=%llu (fallback %llu)", + ti_source, set_count, estimate, cap, leaf_ht_cap_fallback[ti_source]); } task.leaf_ht_arr[ti_source].cap = cap; } } + lnk_log(LNK_Log_Timers, "[typededup] unique-leaf estimate in %.2f ms", (F64)(now_time_us() - estimate_begin_us) / 1000.0); ProfEnd(); for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { @@ -3021,6 +3049,10 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.offsets[ti_source] = offsets_from_counts_array_u64(scratch.arena, task.counts[ti_source], tp->worker_count); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_get_present_buckets_task, &task, "Copy present buckets"); + lnk_log(LNK_Log_Timers, "[typededup] src=%llu: unique=%llu cap=%llu load=%.3f", + ti_source, task.unique_leaf_refs_arr[ti_source].count, task.leaf_ht_arr[ti_source].cap, + task.leaf_ht_arr[ti_source].cap ? (F64)task.unique_leaf_refs_arr[ti_source].count / (F64)task.leaf_ht_arr[ti_source].cap : 0.0); + // sort output leaves based on { location index, leaf index } to guarantee determinism ProfScope("Radix Sort") { u64_array_sort_radix_parallel(tp, task.unique_leaf_refs_arr[ti_source].count, task.unique_leaf_refs_arr[ti_source].v); From 40170b4b8bcf93f1f7f04ab3451ef487160b7a12 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 1 Jul 2026 22:05:37 -0700 Subject: [PATCH 027/121] linker: size-weighted obj distribution for the PDB barrier passes lnk_build_pdb_distribute_obj_indices round-robined objs across lanes by index, ignoring per-obj debug$S size, so a lane drawing several giant objs held the barrier pass at its final barrier while other lanes idled. Distribute by greedy LPT instead: objs taken in weight-descending order (obj_idx tie-break), each assigned to the least-loaded lane. Weights are O(1) per obj -- symbols-subsection total_size for the GSI pass, total debug$S size for the module-write pass. The partition is output-neutral: per-obj results land in per-obj module streams or in GSI bucket chains that are content-sorted at serialization (gsi_symbol_is_before), so any deterministic assignment produces byte-identical PDB bytes. Measured on the FN editor-scale link (6 quiet interleaved samples): Write Modules wall 174-191 -> 151-168 ms (~-12%, samples fully separated); Move Global Symbols wall unchanged (~352 ms) -- its heavy sub-phases are partitioned by symbol_input_ranges/symtab chunks, not obj_indices. Also logs both phase walls under /RAD_LOG:TIMERS (output-neutral). Co-Authored-By: Claude Fable 5 (cherry picked from commit 5690f9965f6fd8862bc62e4bd0cf0ae9019ef2af) --- src/linker/lnk_debug_info.c | 92 ++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 11 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 139590266..afa17506a 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4476,20 +4476,74 @@ lnk_gc_types(TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedType ProfEnd(); } -// FAIR-SHARE: round-robin cv->obj_count objs across `worker_count` lane buckets. +typedef struct +{ + U64 weight; + U32 obj_idx; +} LNK_ObjDistWeight; + +force_inline int +lnk_obj_dist_weight_is_before(void *raw_a, void *raw_b) +{ + LNK_ObjDistWeight *a = raw_a, *b = raw_b; + if (a->weight != b->weight) { return a->weight > b->weight; } + return a->obj_idx < b->obj_idx; // deterministic total order +} + +// FAIR-SHARE: distribute cv->obj_count objs across `worker_count` lane buckets. // Rebuilt per barrier pass so the distribution matches the cohort C that pass // runs at (lnk_move_global_symbols_to_gsi / lnk_write_pdb_modules read -// task->obj_indices[task_id] for lanes [0,C)). Output is width-independent, so any -// C produces byte-identical PDB bytes; only the per-lane partitioning changes. +// task->obj_indices[task_id] for lanes [0,C)). Output is width- and +// assignment-independent -- per-obj results land in per-obj slots (module +// streams) or in GSI bucket chains that are content-sorted at serialization +// (gsi_symbol_is_before radsorts every chain) -- so any deterministic partition +// produces byte-identical PDB bytes; only the per-lane balance changes. +// +// `weights` (optional, [obj_count]) upgrades the round-robin to a greedy LPT +// (longest-processing-time) assignment: objs are taken in weight-descending +// order (obj_idx tie-break -> deterministic) and each goes to the least-loaded +// lane. Round-robin ignores per-obj symbol-stream size, so a lane that draws +// several giant objs holds the whole barrier pass at the final barrier while +// the other lanes idle. internal void -lnk_build_pdb_distribute_obj_indices(Arena *arena, LNK_BuildPdb *task, U64 obj_count, U32 worker_count) +lnk_build_pdb_distribute_obj_indices(Arena *arena, LNK_BuildPdb *task, U64 obj_count, U32 worker_count, U64 *weights) { - U64 objs_per_worker = CeilIntegerDiv(obj_count, worker_count); task->obj_indices = push_array(arena, U32Array, worker_count); - for EachIndex(i, worker_count) { task->obj_indices[i].v = push_array(arena, U32, objs_per_worker ? objs_per_worker : 1); } - for EachIndex(obj_idx, obj_count) { - U32Array *obj_indices = &task->obj_indices[obj_idx % worker_count]; - obj_indices->v[obj_indices->count++] = (U32)obj_idx; + if (weights == 0) { + U64 objs_per_worker = CeilIntegerDiv(obj_count, worker_count); + for EachIndex(i, worker_count) { task->obj_indices[i].v = push_array(arena, U32, objs_per_worker ? objs_per_worker : 1); } + for EachIndex(obj_idx, obj_count) { + U32Array *obj_indices = &task->obj_indices[obj_idx % worker_count]; + obj_indices->v[obj_indices->count++] = (U32)obj_idx; + } + } else { + Temp scratch = scratch_begin(&arena, 1); + + LNK_ObjDistWeight *order = push_array_no_zero(scratch.arena, LNK_ObjDistWeight, obj_count); + for EachIndex(obj_idx, obj_count) { order[obj_idx] = (LNK_ObjDistWeight){ .weight = weights[obj_idx], .obj_idx = (U32)obj_idx }; } + radsort(order, obj_count, lnk_obj_dist_weight_is_before); + + U64 *loads = push_array(scratch.arena, U64, worker_count); + U32 *assign = push_array_no_zero(scratch.arena, U32, obj_count); + for EachIndex(i, obj_count) { + U32 min_lane = 0; + for (U32 lane = 1; lane < worker_count; lane += 1) { if (loads[lane] < loads[min_lane]) { min_lane = lane; } } + assign[order[i].obj_idx] = min_lane; + loads[min_lane] += order[i].weight + 1; // +1 spreads zero-weight objs too + task->obj_indices[min_lane].count += 1; + } + + for EachIndex(lane, worker_count) { + task->obj_indices[lane].v = push_array_no_zero(arena, U32, task->obj_indices[lane].count); + task->obj_indices[lane].count = 0; + } + // fill in ascending obj order per lane (deterministic, cache-friendly iteration) + for EachIndex(obj_idx, obj_count) { + U32Array *obj_indices = &task->obj_indices[assign[obj_idx]]; + obj_indices->v[obj_indices->count++] = (U32)obj_idx; + } + + scratch_end(scratch); } } @@ -4574,13 +4628,23 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config ProfScope("Write Modules") { + U64 phase_begin_us = now_time_us(); // FAIR-SHARE: pin the cohort, distribute objs over exactly the cohort lanes, // then run the barrier pass at that cohort. tp_barrier_begin sets // tp->worker_count := C for the bracket. U32 C = tp_barrier_begin(tp); - lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C); + // weight = total debug$S byte size: the module-stream write walks every + // subsection of the obj (symbols + lines + checksums + ...) + U64 *weights = push_array_no_zero(scratch.arena, U64, cv->obj_count); + for EachIndex(obj_idx, cv->obj_count) { + U64 total = 0; + for EachIndex(k, CV_C13SubSectionIdxKind_COUNT) { total += cv->debug_s_arr[obj_idx].data_list[k].total_size; } + weights[obj_idx] = total; + } + lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C, weights); tp_for_parallel_reserve(tp, 0, C, lnk_write_pdb_modules, &task); // BARRIER pass (path B): barrier_wait/tp_broadcast tp_barrier_end(tp); + lnk_log(LNK_Log_Timers, "[pdb] write modules in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); } if (output_ptr != 0) { for EachIndex(obj_idx, cv->obj_count) { @@ -4589,10 +4653,16 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config } ProfScope("Move Global Symbols") { + U64 phase_begin_us = now_time_us(); U32 C = tp_barrier_begin(tp); - lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C); + // weight = per-obj symbols-subsection byte size: both proc-refs passes walk + // exactly these bytes per obj, and String8List.total_size makes it O(1) + U64 *weights = push_array_no_zero(scratch.arena, U64, cv->obj_count); + for EachIndex(obj_idx, cv->obj_count) { weights[obj_idx] = cv_sub_section_from_debug_s(cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols).total_size; } + lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C, weights); tp_for_parallel_reserve(tp, 0, C, lnk_move_global_symbols_to_gsi, &task); // BARRIER pass (path B): tp_sum_u64/tp_broadcast/barrier_wait tp_barrier_end(tp); + lnk_log(LNK_Log_Timers, "[pdb] move global symbols in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); } ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); if (output_ptr != 0) { From ff81610a0bd68b9bd2599bc742acc993a1ad2ece Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 1 Jul 2026 23:35:39 -0700 Subject: [PATCH 028/121] linker: schedule-independent symrec order for same-name global symbols The per-bucket GSI sort tie-breaks same-name globals on CV_Symbol.offset, which was the compacted deduper slot index -- CAS-arrival order in cv_symbol_deduper_insert_or_update. Same-name different-content records (duplicate S_UDTs with distinct type indices) could swap symrec positions whenever the lane->worker schedule changed (fair-share cohorts under /RAD_SHARED_THREAD_POOL) or probe chains contended. Key on the content hash of the full raw record instead: order becomes a pure function of record bytes (hash -> kind -> data-bytes fallback in gsi_symbol_is_before; byte-identical records are folded by the deduper before the sort). Co-Authored-By: Claude Fable 5 (cherry picked from commit 59862fa485c01e5fb18a7082ea3075dbb4818ce7) --- src/linker/lnk_debug_info.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index afa17506a..c59d081e0 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3466,7 +3466,16 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) CV_SymbolNode *n = &global_nodes[i]; n->prev = n->next = 0; n->data = cv_symbol_from_ptr(symbol_arr[i]); - n->data.offset = i; + // deterministic same-name tie-break for the per-bucket sort in gsi_serialize_symbols_task + // (gsi_symbol_is_before compares name -> offset -> kind -> data bytes): key on the content + // hash of the full raw record instead of the compacted deduper slot index. Slot order is + // CAS-arrival order in cv_symbol_deduper_insert_or_update and can flip between same-name + // different-content records (e.g. duplicate S_UDTs with distinct type indices) whenever the + // lane->worker schedule changes (shared-pool cohorts) or probe chains contend, permuting + // symrec bytes run-to-run. The hash is schedule-independent; on collision the comparator's + // kind/data-bytes fallback stays content-deterministic, and byte-identical records cannot + // reach the sort (the deduper folds them), so the pointer tiebreaker stays unreachable. + n->data.offset = u64_hash_from_str8(cv_raw_from_symbol(symbol_arr[i])); cv_symbol_list_push_node(&gsi->bucket_arr[bucket_idx], n); } } From 16811db4a0329071c4aef799a067c18fab146fcb Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 2 Jul 2026 00:33:23 -0700 Subject: [PATCH 029/121] linker: /OPT:REF walk: memoize reloc-symbol resolution, drop per-reloc tree HashMaps, skip name decodes - per-worker lossy open-addressing cache (obj input idx, symbol idx) -> final resolved ref; the resolve chain (interp parse + trie search per hop) repeated per referencing reloc - cycle detection + per-walk visited-section set: flat arrays with linear scan instead of arena-backed tree HashMap nodes (same first-revisit semantics) - lnk_resolve_symbol + walk unpack sites use lnk_parsed_symbol_from_coff_symbol_idx_no_name; name (string-table decode + strlen) only where a by-name symbol-table search happens - test-and-test-and-set on is_live flags to keep already-live cachelines in shared state Live-section set and warning behavior are byte-for-byte unchanged by construction. Co-Authored-By: Claude Fable 5 (cherry picked from commit c990a45a6d3a9b9b19f64ab63768d66d077b10f1) --- src/linker/lnk.c | 149 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 140 insertions(+), 9 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 54aa0fa9e..3099fa03b 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -2725,6 +2725,17 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) tp_broadcast(&global_batch_list); tp_broadcast(&active_thread_count); + // Per-worker memo: (obj input idx << 32 | coff symbol idx) -> final ref of the reloc-symbol + // resolve chain below. A symbol is referenced by one reloc per call site, so the chain + // (interp parse + symbol-table trie search per hop) otherwise repeats for identical inputs + // millions of times. Open-addressing and lossy (a collision past the probe window evicts); + // a miss only costs the recompute -- the cached value is a pure function of the key because + // the symbol table and parsed_symbols are read-only during /OPT:REF. + typedef struct { U64 key; LNK_Obj *obj; U64 symbol_idx; } LNK_RefResolveSlot; + U64 resolve_cache_mask = (1ull << 20) - 1; + LNK_RefResolveSlot *resolve_cache = push_array_no_zero(scratch.arena, LNK_RefResolveSlot, resolve_cache_mask + 1); + MemorySet(resolve_cache, 0xff, sizeof(resolve_cache[0]) * (resolve_cache_mask + 1)); + LNK_RelocRefsBatchList free_list = {0}; for (;;) { // update active thread count @@ -2743,30 +2754,149 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) // reloc -> symbol LNK_ObjSymbolRef ref_symbol = (LNK_ObjSymbolRef){ .obj = batch->v[i].obj, .symbol_idx = reloc->isymbol }; - lnk_resolve_reloc_target_symbol(scratch2.arena, symtab, ref_symbol, str8_lit("/OPT:REF"), &ref_symbol); + { + // resolve-cache lookup + U64 cache_key = ((U64)ref_symbol.obj->input_idx << 32ull) | (U64)ref_symbol.symbol_idx; + U64 cache_hash = cache_key * 0x9E3779B97F4A7C15ull; cache_hash ^= cache_hash >> 32; + U64 cache_slot = max_U64; + B32 cache_hit = 0; + for (U64 probe_idx = 0; probe_idx < 8; probe_idx += 1) { + U64 slot = (cache_hash + probe_idx) & resolve_cache_mask; + if (resolve_cache[slot].key == cache_key) { + ref_symbol = (LNK_ObjSymbolRef){ .obj = resolve_cache[slot].obj, .symbol_idx = (U32)resolve_cache[slot].symbol_idx }; + cache_hit = 1; + break; + } + if (resolve_cache[slot].key == max_U64) { cache_slot = slot; break; } + } + + if (!cache_hit) { + // cycle detection via linear scan of the visited chain: chains are 1-3 hops in + // practice, and this keeps the tree HashMap + per-hop arena pushes off the hot path + // (exact same first-revisit semantics) + U64 chain_fixed[64]; + U64 *chain = chain_fixed; + U64 chain_count = 0; + U64 chain_cap = ArrayCount(chain_fixed); + B32 was_cyclic = 0; + + Temp temp = temp_begin(scratch2.arena); + B32 keep_walking = 1; + do { + // detect cyclic chains + U64 symbol_key = ((U64)ref_symbol.obj->input_idx << 32ull) | (U64)ref_symbol.symbol_idx; + B32 was_seen = 0; + for EachIndex(chain_idx, chain_count) { + if (chain[chain_idx] == symbol_key) { was_seen = 1; break; } + } + if (!was_seen) { + if (chain_count == chain_cap) { + U64 *new_chain = push_array_no_zero(temp.arena, U64, chain_cap * 2); + MemoryCopy(new_chain, chain, sizeof(chain[0]) * chain_count); + chain = new_chain; chain_cap *= 2; + } + chain[chain_count++] = symbol_key; + } else { + COFF_ParsedSymbol reloc_parsed = lnk_parsed_symbol_from_coff_symbol_idx(batch->v[i].obj, reloc->isymbol); + lnk_error_obj(LNK_Warning_CyclicSymbol, batch->v[i].obj, "symbol %S forms a cyclic chain (/OPT:REF)", reloc_parsed.name); + MemoryZeroStruct(&ref_symbol); + was_cyclic = 1; + break; + } + + // unpack symbol (interp needs no name decode) + COFF_ParsedSymbol ref_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(ref_symbol.obj, ref_symbol.symbol_idx); + COFF_SymbolValueInterpType ref_interp = coff_interp_from_parsed_symbol(ref_parsed); + + // resolve symbol + LNK_ObjSymbolRef next_ref = {0}; + if (lnk_resolve_symbol(symtab, ref_symbol, &next_ref)) { + keep_walking = (ref_interp == COFF_SymbolValueInterp_Weak || ref_interp == COFF_SymbolValueInterp_Undefined); + ref_symbol = next_ref; + } else { + keep_walking = 0; + } + } while (keep_walking); + temp_end(temp); + + // memoize (skip the cyclic-warning path so the warning replays per reloc as before) + if (!was_cyclic) { + if (cache_slot == max_U64) { cache_slot = cache_hash & resolve_cache_mask; } + resolve_cache[cache_slot] = (LNK_RefResolveSlot){ .key = cache_key, .obj = ref_symbol.obj, .symbol_idx = ref_symbol.symbol_idx }; + } + } + } // skip unresolved symbol if (ref_symbol.obj == 0) { continue; } - // unpack resolved symbol + // unpack resolved symbol (only interp + section_number are used -- skip the name decode) COFF_ParsedSymbol ref_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(ref_symbol.obj, ref_symbol.symbol_idx); COFF_SymbolValueInterpType ref_interp = coff_interp_from_parsed_symbol(ref_parsed); if (ref_interp == COFF_SymbolValueInterp_Regular) { Temp temp = temp_begin(scratch2.arena); - U32List associated_sections = lnk_obj_collect_associated_section_numbers(temp.arena, ref_symbol.obj, ref_parsed.section_number, 0); + LNK_Obj *walk_obj = ref_symbol.obj; + U32 seed_sn = ref_parsed.section_number; + + // per-walk visited set + walk stack: flat arrays with linear-scan membership -- + // associative groups are a handful of sections, and the tree HashMap + per-node + // arena pushes dominated this walk + U32 visited_fixed[64]; + U32 *visited = visited_fixed; + U64 visited_count = 0; + U64 visited_cap = ArrayCount(visited_fixed); + U32 stack_fixed[64]; + U32 *stack = stack_fixed; + U64 stack_count = 0; + U64 stack_cap = ArrayCount(stack_fixed); + stack[stack_count++] = seed_sn; + do { + U32 section_number = stack[--stack_count]; + + // detect cyclic associative sections + { + B32 was_seen = 0; + for EachIndex(visited_idx, visited_count) { + if (visited[visited_idx] == section_number) { was_seen = 1; break; } + } + if (was_seen) { continue; } + if (visited_count == visited_cap) { + U32 *new_visited = push_array_no_zero(temp.arena, U32, visited_cap * 2); + MemoryCopy(new_visited, visited, sizeof(visited[0]) * visited_count); + visited = new_visited; visited_cap *= 2; + } + visited[visited_count++] = section_number; + } - // visit root section - u32_list_push(temp.arena, &associated_sections, ref_parsed.section_number); + // push associated section + for EachNode(associated_n, U32Node, walk_obj->coff.sections.associated_section_numbers[section_number]) { + U32 assoc_sn = associated_n->data; - for EachNode(section_n, U32Node, associated_sections.first) { - U32 section_number = section_n->data; + + { + B32 assoc_seen = 0; + for EachIndex(visited_idx, visited_count) { + if (visited[visited_idx] == assoc_sn) { assoc_seen = 1; break; } + } + if (assoc_seen) { continue; } + } + if (stack_count == stack_cap) { + U32 *new_stack = push_array_no_zero(temp.arena, U32, stack_cap * 2); + MemoryCopy(new_stack, stack, sizeof(stack[0]) * stack_count); + stack = new_stack; stack_cap *= 2; + } + stack[stack_count++] = assoc_sn; + } COFF_SectionFlags section_flags = ref_symbol.obj->coff.sections.headers[section_number].flags; // on first section visit, set live flag and enqueue section - U8 was_visited = ins_atomic_u8_eval_assign(&is_live[ref_symbol.obj->input_idx][section_number], 1); + // (plain read first -- most targets are already live; the read keeps the flag + // cacheline shared instead of dirtying it with an unconditional exchange) + U8 was_visited = *(volatile U8 *)&is_live[walk_obj->input_idx][section_number]; + if (!was_visited) { was_visited = ins_atomic_u8_eval_assign(&is_live[walk_obj->input_idx][section_number], 1); } if (was_visited) { continue; } // is section eligible for walking? @@ -2791,7 +2921,8 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) } batch->v[batch->count++] = refs; - } + + } while (stack_count); temp_end(temp); } From 6ac907fa37fb8a7683fdf30c53d38ca714f2d092 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 2 Jul 2026 00:40:41 -0700 Subject: [PATCH 030/121] linker: /OPT:REF: parallelize unreachable-section removal The remove pass was a serial O(all sections) walk on task 0 (self-labeled TODO: thread). Section flags are per-obj so writes are disjoint; stride the obj list across tasks via objs_by_idx. Stats accumulate per task and reduce on task 0, keeping the /OPT:REF debug-log totals identical regardless of cohort width or schedule. Co-Authored-By: Claude Fable 5 (cherry picked from commit 2f8650749add3849cfcc7e74953ed06d3cf51240) --- src/linker/lnk.c | 120 +++++++++++++++++++++++------------------------ 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 3099fa03b..13ed5d64a 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -2649,20 +2649,29 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) LNK_Obj **objs = task->objs; U64 objs_count = task->objs_count; - U8 **is_live = 0; - U64 *active_thread_count = 0; + // "Remove Unreachable Sections" per-task stat accumulators (reduced on task 0 for the log) + typedef struct { U64 vsize; U64 fsize; U64 section_count; } LNK_OptRefStat; + enum { LNK_OptRefStat_Null, LNK_OptRefStat_Code, LNK_OptRefStat_Data, LNK_OptRefStat_Debug, LNK_OptRefStat_Count }; + + U8 **is_live = 0; + LNK_Obj **objs_by_idx = 0; // input_idx -> obj, for the strided removal pass + U64 *active_thread_count = 0; LNK_RelocRefsBatchList *global_batch_list = 0; + LNK_OptRefStat *remove_stats = 0; if (task_id == 0) { - active_thread_count = push_array(scratch.arena, U64, 1); + remove_stats = push_array(scratch.arena, LNK_OptRefStat, LNK_OptRefStat_Count * tp->worker_count); + active_thread_count = push_array(scratch.arena, U64, 1); global_batch_list = push_array(scratch.arena, LNK_RelocRefsBatchList, 1); // alloc live flags and set live status on every non-COMDAT section - is_live = push_array_no_zero(scratch.arena, U8 *, objs_count); + is_live = push_array_no_zero(scratch.arena, U8 *, objs_count); + objs_by_idx = push_array_no_zero(scratch.arena, LNK_Obj *, objs_count ? objs_count : 1); { for EachIndex(obj_idx, objs_count) { LNK_Obj *obj = objs[obj_idx]; - is_live[obj_idx] = push_array(scratch.arena, U8, obj->coff.sections.count_no_null + 1); + is_live[obj_idx] = push_array(scratch.arena, U8, obj->coff.sections.count_no_null + 1); + objs_by_idx[obj_idx] = obj; for LNK_EachCoffSection(it, obj) { is_live[obj_idx][it.v.section_number] = !(*it.v.flags & COFF_SectionFlag_LnkCOMDAT); @@ -2722,8 +2731,10 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) } } tp_broadcast(&is_live); + tp_broadcast(&objs_by_idx); tp_broadcast(&global_batch_list); tp_broadcast(&active_thread_count); + tp_broadcast(&remove_stats); // Per-worker memo: (obj input idx << 32 | coff symbol idx) -> final ref of the reloc-symbol // resolve chain below. A symbol is referenced by one reloc per call site, so the chain @@ -2955,74 +2966,63 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) exit:; barrier_wait(tp->barrier); - // TODO: thread - if (task_id == 0) { + // Remove unreachable sections. Section flags are per-obj (disjoint writes), so the obj list is + // strided across tasks via objs_by_idx; stats accumulate per task and are reduced on task 0, so + // the debug log totals are identical regardless of cohort width or schedule. + { ProfBegin("Remove Unreachable Sections"); + LNK_OptRefStat *stats = remove_stats + task_id * LNK_OptRefStat_Count; + for (U64 obj_idx = task_id; obj_idx < objs_count; obj_idx += tp->worker_count) { + LNK_Obj *obj = objs_by_idx[obj_idx]; - for EachIndex(obj_idx, objs_count) { - LNK_Obj *obj = objs[obj_idx]; for LNK_EachCoffSection(it, obj) { - COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(obj, it.v.section_number); - if ( ! is_live[obj->input_idx][it.v.section_number]) { - *it.v.flags |= COFF_SectionFlag_LnkRemove; - } - } - } + U32 section_number = it.v.section_number; + if (is_live[obj->input_idx][section_number]) { continue; } - if (lnk_get_log_status(LNK_Log_Debug)) { - typedef struct { U64 vsize; U64 fsize; U64 section_count; U64 live_count; U64 live_fsize; U64 live_vsize; } Stat; - enum { Stat_Null, Stat_Code, Stat_Data, Stat_Debug, Stat_Count }; - Stat stats[Stat_Count] = {0}; + COFF_SectionHeader *section_header = it.v.header; + *it.v.flags |= COFF_SectionFlag_LnkRemove; + COFF_SectionFlags section_flags = *it.v.flags; - for EachIndex(obj_idx, objs_count) { - LNK_Obj *obj = objs[obj_idx]; + U64 stat_kind = LNK_OptRefStat_Null; + if (section_flags & LNK_SECTION_FLAG_DEBUG) { stat_kind = LNK_OptRefStat_Debug; } + else if (section_flags & COFF_SectionFlag_CntCode) { stat_kind = LNK_OptRefStat_Code; } + else { stat_kind = LNK_OptRefStat_Data; } - for LNK_EachCoffSection(it, obj) { - COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(obj, it.v.section_number); - - U64 stat_kind = Stat_Null; - if (*it.v.flags & LNK_SECTION_FLAG_DEBUG) { stat_kind = Stat_Debug; } - else if (*it.v.flags & COFF_SectionFlag_CntCode) { stat_kind = Stat_Code; } - else { stat_kind = Stat_Data; } - - if (is_live[obj->input_idx][it.v.section_number]) { - stats[stat_kind].live_count += 1; - if (*it.v.flags & COFF_SectionFlag_CntUninitializedData) { - stats[stat_kind].live_vsize += section_header->vsize; - } else { - stats[stat_kind].live_fsize += section_header->fsize; - } - } else { - if (*it.v.flags & COFF_SectionFlag_CntUninitializedData) { - stats[stat_kind].vsize += section_header->vsize; - } else { - stats[stat_kind].fsize += section_header->fsize; - } - stats[stat_kind].section_count += 1; - } + if (section_flags & COFF_SectionFlag_CntUninitializedData) { + stats[stat_kind].vsize += section_header->vsize; + } else { + stats[stat_kind].fsize += section_header->fsize; } + stats[stat_kind].section_count += 1; } + } + ProfEnd(); + } + barrier_wait(tp->barrier); - U64 total_fsize = 0, total_section_count = 0; - U64 total_fsize_live = 0, total_section_count_live = 0; - for EachElement(i, stats) { - total_fsize += stats[i].fsize; - total_section_count += stats[i].section_count; - total_fsize_live += stats[i].live_fsize; - total_section_count_live += stats[i].live_count; + if (task_id == 0 && lnk_get_log_status(LNK_Log_Debug)) { + LNK_OptRefStat stats[LNK_OptRefStat_Count] = {0}; + for EachIndex(reduce_task_idx, tp->worker_count) { + for EachIndex(stat_idx, (U64)LNK_OptRefStat_Count) { + stats[stat_idx].vsize += remove_stats[reduce_task_idx * LNK_OptRefStat_Count + stat_idx].vsize; + stats[stat_idx].fsize += remove_stats[reduce_task_idx * LNK_OptRefStat_Count + stat_idx].fsize; + stats[stat_idx].section_count += remove_stats[reduce_task_idx * LNK_OptRefStat_Count + stat_idx].section_count; } - String8List stat_list = {0}; - str8_list_pushf(scratch.arena, &stat_list, "Code : removed %M, %S sections; live %M, %S sections", stats[Stat_Code].fsize, str8_from_count(scratch.arena, stats[Stat_Code].section_count ), stats[Stat_Code].live_fsize, str8_from_count(scratch.arena, stats[Stat_Code].live_count)); - str8_list_pushf(scratch.arena, &stat_list, "Data : removed %M, %S sections; live %M, %S sections", stats[Stat_Data].fsize, str8_from_count(scratch.arena, stats[Stat_Data].section_count ), stats[Stat_Data].live_fsize, str8_from_count(scratch.arena, stats[Stat_Data].live_count)); - str8_list_pushf(scratch.arena, &stat_list, "Debug: removed %M, %S sections; live %M, %S sections", stats[Stat_Debug].fsize, str8_from_count(scratch.arena, stats[Stat_Debug].section_count), stats[Stat_Debug].live_fsize, str8_from_count(scratch.arena, stats[Stat_Debug].live_count)); - str8_list_pushf(scratch.arena, &stat_list, "Total: removed %M, %S sections; live %M, %S sections", total_fsize, str8_from_count(scratch.arena, total_section_count), total_fsize_live, str8_from_count(scratch.arena, total_section_count_live)); - String8 stat_str = str8_list_join(scratch.arena, &stat_list, &(StringJoin){.pre = str8_lit(" "), .sep = str8_lit("\n ")}); - lnk_log(LNK_Log_Debug, "/OPT:REF Stats:\n%S", stat_str); } - ProfEnd(); + U64 total_fsize = 0, total_section_count = 0; + for EachElement(i, stats) { + total_fsize += stats[i].fsize; + total_section_count += stats[i].section_count; + } + String8List stat_list = {0}; + str8_list_pushf(scratch.arena, &stat_list, "Code : %M, %S sections", stats[LNK_OptRefStat_Code].fsize, str8_from_count(scratch.arena, stats[LNK_OptRefStat_Code].section_count )); + str8_list_pushf(scratch.arena, &stat_list, "Data : %M, %S sections", stats[LNK_OptRefStat_Data].fsize, str8_from_count(scratch.arena, stats[LNK_OptRefStat_Data].section_count )); + str8_list_pushf(scratch.arena, &stat_list, "Debug: %M, %S sections", stats[LNK_OptRefStat_Debug].fsize, str8_from_count(scratch.arena, stats[LNK_OptRefStat_Debug].section_count)); + str8_list_pushf(scratch.arena, &stat_list, "Total: %M, %S sections", total_fsize, str8_from_count(scratch.arena, total_section_count)); + String8 stat_str = str8_list_join(scratch.arena, &stat_list, &(StringJoin){.pre = str8_lit(" "), .sep = str8_lit("\n ")}); + lnk_log(LNK_Log_Debug, "/OPT:REF Stats:\n%S", stat_str); } - barrier_wait(tp->barrier); scratch_end(scratch2); scratch_end(scratch); From 43318eb72217a98c38e900888036ba47db05d7a0 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 2 Jul 2026 00:25:18 -0700 Subject: [PATCH 031/121] linker: packed-discriminator pre-filter for archive symbol-dir bsearch Archive symbol-dir names (lib->symbol_names.v[]) point into the mapped archive's string table, so every bsearch probe's MemCompare chases .str into scattered archive bytes; sorted order != memory order, so probes have no locality. Build a parallel U64 disc[] at lib parse time packing each name's first 8 bytes big-endian (zero-padded): integer compare of discriminators decides str8_compar_case_sensitive order exactly -- including the shorter-prefix-precedes size tie-break -- whenever they differ, and equal discriminators fall through to the full compare. Probe sequence and result are identical to str8_array_bsearch; probes now read the contiguous disc[] array and touch archive string-table bytes only on discriminator ties. Co-Authored-By: Claude Fable 5 (cherry picked from commit 82252854274b93b72603d131426d7fd557139d2e) --- src/linker/lnk_lib.c | 82 ++++++++++++++++++++++++++++++++++++++++++-- src/linker/lnk_lib.h | 9 +++++ 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/src/linker/lnk_lib.c b/src/linker/lnk_lib.c index 50bc4bd25..506cc4468 100644 --- a/src/linker/lnk_lib.c +++ b/src/linker/lnk_lib.c @@ -20,6 +20,25 @@ lnk_first_member_sort_key_is_before(void *raw_a, void *raw_b) return str8_is_before_case_sensitive(&a->symbol_name, &b->symbol_name); } +internal force_inline U64 +lnk_symbol_name_disc(String8 name) +{ + // pack the first 8 bytes big-endian so an integer compare orders exactly like memcmp; pad short + // names with zeros. this is faithful to str8_compar_case_sensitive INCLUDING the size tie-break + // (base_strings.c: shorter prefix precedes): at the first differing padded byte either both + // strings have a real byte there (== memcmp order), or the shorter string's zero pad compares + // below the longer string's next real byte (== shorter-prefix-precedes). a zero pad byte can tie + // only with another pad byte or an embedded NUL, and any such tie leaves the discriminators + // marching in lockstep until a real difference or full equality -- so disc inequality ALWAYS + // decides the compare, and disc equality falls through to the full str8_compar. + U64 disc = 0; + U64 n = Min(name.size, 8); + for (U64 i = 0; i < n; i += 1) { + disc |= (U64)name.str[i] << (56 - i*8); + } + return disc; +} + internal B32 lnk_lib_from_data(Arena *arena, String8 data, String8 path, U64 input_idx, LNK_Lib *lib_out) { @@ -128,7 +147,14 @@ lnk_lib_from_data(Arena *arena, String8 data, String8 path, U64 input_idx, LNK_L scratch_end(scratch); } - + + // build packed discriminators parallel to the (sorted) symbol name dir; bsearch probes read this + // contiguous array and touch archive string-table bytes only on discriminator ties + U64 *symbol_discs = push_array_no_zero(arena, U64, symbol_names.count); + for EachIndex(symbol_idx, symbol_names.count) { + symbol_discs[symbol_idx] = lnk_symbol_name_disc(symbol_names.v[symbol_idx]); + } + // init lib lib_out->path = push_str8_copy(arena, path); lib_out->data = data; @@ -138,6 +164,7 @@ lnk_lib_from_data(Arena *arena, String8 data, String8 path, U64 input_idx, LNK_L lib_out->member_offsets = member_offsets; lib_out->symbol_indices = symbol_indices; lib_out->symbol_names = symbol_names; + lib_out->symbol_discs = symbol_discs; lib_out->long_names = parse.long_names; lib_out->input_idx = input_idx; @@ -216,10 +243,61 @@ lnk_lib_list_push_parallel(TP_Context *tp, TP_Arena *arena, LNK_LibList *list, U return result; } +internal force_inline int +lnk_disc_str8_compar(U64 a_disc, String8 *a, U64 b_disc, String8 *b) +{ + // discriminator inequality decides the compare without touching string bytes (see + // lnk_symbol_name_disc for the order-fidelity argument); ties fall through to the full compare + if (a_disc != b_disc) { + return a_disc < b_disc ? -1 : +1; + } + return str8_compar_case_sensitive(a, b); +} + +// str8_array_bsearch with a packed-discriminator pre-filter: identical probe sequence and result +// (each probe's compare outcome is identical), but probes read the contiguous disc[] array instead +// of chasing symbol_names.v[].str into scattered archive string-table bytes +internal U64 +lnk_lib_bsearch_symbol_name(LNK_Lib *lib, String8 value) +{ + String8Array arr = lib->symbol_names; + U64 *disc = lib->symbol_discs; + if (arr.count > 1) { + U64 value_disc = lnk_symbol_name_disc(value); + + int lo_compar = lnk_disc_str8_compar(value_disc, &value, disc[0], &arr.v[0]); + if (lo_compar == 0) { + return 0; + } + + int hi_compar = lnk_disc_str8_compar(value_disc, &value, disc[arr.count-1], &arr.v[arr.count-1]); + if (hi_compar == 0) { + return arr.count-1; + } + + if (lo_compar > 0 && hi_compar < 0) { + for (U64 l = 0, r = arr.count-1; l <= r; ) { + U64 m = l + (r - l) / 2; + int cmp = lnk_disc_str8_compar(disc[m], &arr.v[m], value_disc, &value); + if (cmp == 0) { + return m; + } else if (cmp < 0) { + l = m + 1; + } else { + r = m - 1; + } + } + } + } else if (arr.count == 1 && str8_match(arr.v[0], value, 0)) { + return 0; + } + return max_U64; +} + internal force_inline B32 lnk_search_lib(LNK_Lib *lib, String8 symbol_name, U32 *member_idx_out) { - U64 symbol_idx = str8_array_bsearch(lib->symbol_names, symbol_name); + U64 symbol_idx = lnk_lib_bsearch_symbol_name(lib, symbol_name); if (symbol_idx < lib->symbol_count) { if (member_idx_out) { *member_idx_out = lib->symbol_indices[symbol_idx]-1; diff --git a/src/linker/lnk_lib.h b/src/linker/lnk_lib.h index 6db16e293..bce3b9290 100644 --- a/src/linker/lnk_lib.h +++ b/src/linker/lnk_lib.h @@ -13,6 +13,14 @@ typedef struct LNK_Lib U32 *member_offsets; U16 *symbol_indices; String8Array symbol_names; + + // symbol-dir bsearch pre-filter: symbol_names.v[].str points into the mapped archive's string + // table, so every bsearch probe's MemCompare chases scattered archive bytes (sorted order != + // memory order -> no locality). symbol_discs[i] packs the first 8 bytes of symbol_names.v[i] + // big-endian (zero-padded), so an integer compare of discriminators decides str8_compar order + // whenever they differ; probes touch archive bytes only on discriminator ties. Parallel to + // symbol_names, built once at parse time. + U64 *symbol_discs; String8 long_names; U64 input_idx; @@ -70,5 +78,6 @@ internal LNK_Lib ** lnk_array_from_lib_list(Arena *arena, LNK_LibList list internal void lnk_lib_list_push_node(LNK_LibList *list, LNK_LibNode *node); internal LNK_LibNodeArray lnk_lib_list_push_parallel(TP_Context *tp, TP_Arena *arena, LNK_LibList *list, U64 inputs_count, struct LNK_Input **inputs); +internal U64 lnk_lib_bsearch_symbol_name(LNK_Lib *lib, String8 value); internal force_inline B32 lnk_search_lib(LNK_Lib *lib, String8 symbol_name, U32 *member_idx_out); From a30ee1344f765116169c9b828e008b24e7931ec5 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 2 Jul 2026 00:59:56 -0700 Subject: [PATCH 032/121] linker: balance GSI deduper inserts and parallelize bucket compaction Per-barrier arrival stamps on the lnk_move_global_symbols_to_gsi barrier pass (FN editor-scale link, cohort 64) showed two real stalls in the Global Symbols section: - the deduper-insert phase had a ~2x arrival skew (47ms spread on a ~90ms phase): the collect partition is weighted by raw symbol bytes, but insert cost follows global-symbol COUNT, whose density varies per lane. Flatten the per-worker collect lists (in lane order) into one array and re-divide the inserts evenly. The deduper is CAS-based and content-keyed, so any insert partition produces the same deduped content set; downstream order is already schedule-independent (per-chain radsort keyed on the content hash written into n->data.offset). - "Compact Buckets" ran serially on task 0 (~38ms sweep over the ~1.3x global-symbol-count slot array) while the other 63 workers idled at the next barrier. Compact in parallel instead: each worker counts occupied slots in its contiguous slot range, then copies them to its prefix-sum offset. Concatenated ranges preserve ascending slot order, so symbol_arr is identical to the serial compaction. Output is byte-identical (DLL 0-byte diff, PDB 0 differing streams, determinism re-link clean). Co-Authored-By: Claude Fable 5 (cherry picked from commit 72563e5a67ddf967ae389821bf39f0de4133f131) --- src/linker/lnk_debug_info.c | 89 +++++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 19 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index c59d081e0..654356737 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3402,42 +3402,93 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) U64 bucket_cap; void **buckets; + U64 *collect_counts; // [worker_count] + void **flat_symbols; // [global_symbol_count] if (task_id == 0) { - bucket_cap = global_symbol_count * 13 / 10; - buckets = push_array(scratch.arena, void *, bucket_cap); + bucket_cap = global_symbol_count * 13 / 10; + buckets = push_array(scratch.arena, void *, bucket_cap); + collect_counts = push_array(scratch.arena, U64, tp->worker_count); + flat_symbols = push_array_no_zero(scratch.arena, void *, global_symbol_count ? global_symbol_count : 1); } tp_broadcast(&bucket_cap); tp_broadcast(&buckets); + tp_broadcast(&collect_counts); + tp_broadcast(&flat_symbols); + + // BALANCE: per-lane global-symbol density varies a lot (the collect partition above is + // weighted by raw symbol bytes, not by global-symbol count), which skewed the insert phase + // by ~2x. Flatten the per-worker lists (in worker order) into one array and re-divide the + // INSERTS evenly. The deduper is CAS-based and content-keyed: any insert partition yields + // the same deduped content set, and downstream order is already schedule-independent (the + // per-chain sort keys on the content hash written into n->data.offset below), so this only + // changes who performs an insert, never the output. + ProfBegin("Insert Global Symbols"); + collect_counts[task_id] = global_symbols.count; + barrier_wait(tp->barrier); + { + U64 flat_off = 0; + for (U64 w = 0; w < task_id; w += 1) { flat_off += collect_counts[w]; } + for EachNode(n, VoidNode, global_symbols.first) { flat_symbols[flat_off++] = n->v; } + } + barrier_wait(tp->barrier); - // insert symbols into hash table - for EachNode(n, VoidNode, global_symbols.first) { - String8 raw = cv_raw_from_symbol(n->v); - U64 hash = u64_hash_from_str8(raw); - cv_symbol_deduper_insert_or_update(buckets, bucket_cap, hash, n->v); + // insert symbols into hash table (even split of the flattened array) + { + U64 ins_lo = (task_id * global_symbol_count) / tp->worker_count; + U64 ins_hi = ((task_id + 1) * global_symbol_count) / tp->worker_count; + for (U64 i = ins_lo; i < ins_hi; i += 1) { + String8 raw = cv_raw_from_symbol(flat_symbols[i]); + U64 hash = u64_hash_from_str8(raw); + cv_symbol_deduper_insert_or_update(buckets, bucket_cap, hash, flat_symbols[i]); + } } barrier_wait(tp->barrier); + ProfEnd(); - U64 symbol_count = 0; - void **symbol_arr = 0; // [symbol_count] - Rng1U64 *symbol_ranges = 0; // [worker_count] - U32 *symbol_hashes = 0; // [symbol_count] + // compact buckets in parallel: each worker owns a contiguous slot range, counts its + // occupied slots, then copies them to its prefix-sum offset. Concatenated ranges preserve + // ascending slot order, so symbol_arr is byte-identical to the old task-0-serial compaction + // (which stalled the other workers at the next barrier for the whole bucket_cap sweep). + U64 symbol_count = 0; + void **symbol_arr = 0; // [symbol_count] + Rng1U64 *symbol_ranges = 0; // [worker_count] + U32 *symbol_hashes = 0; // [symbol_count] + U64 *compact_counts = 0; // [worker_count] if (task_id == 0) { - ProfBeginV("Compact Buckets [bucket_cap %llu]", bucket_cap); - for EachIndex(src, bucket_cap) { - buckets[symbol_count] = buckets[src]; - symbol_count += buckets[src] != 0; - } - ProfEnd(); + compact_counts = push_array(scratch.arena, U64, tp->worker_count); + } + tp_broadcast(&compact_counts); - symbol_arr = buckets; + ProfBeginV("Compact Buckets [bucket_cap %llu]", bucket_cap); + U64 slot_lo = (task_id * bucket_cap) / tp->worker_count; + U64 slot_hi = ((task_id + 1) * bucket_cap) / tp->worker_count; + { + U64 c = 0; + for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { c += buckets[slot] != 0; } + compact_counts[task_id] = c; + } + barrier_wait(tp->barrier); + + for EachIndex(w, tp->worker_count) { symbol_count += compact_counts[w]; } // same value on every worker + if (task_id == 0) { + symbol_arr = push_array_no_zero(scratch.arena, void *, symbol_count ? symbol_count : 1); symbol_ranges = tp_divide_work(scratch.arena, symbol_count, tp->worker_count); symbol_hashes = push_array_no_zero(scratch.arena, U32, symbol_count); } - tp_broadcast(&symbol_count); tp_broadcast(&symbol_arr); tp_broadcast(&symbol_ranges); tp_broadcast(&symbol_hashes); + { + U64 dst = 0; + for (U64 w = 0; w < task_id; w += 1) { dst += compact_counts[w]; } + for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { + if (buckets[slot]) { symbol_arr[dst++] = buckets[slot]; } + } + } + barrier_wait(tp->barrier); + ProfEnd(); + // hash symbols Rng1U64 symbol_range = symbol_ranges[task_id]; for EachInRange(i, symbol_range) { From d6a5899e28ec794d5ec589669efe571752f54d2e Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 2 Jul 2026 17:51:52 -0700 Subject: [PATCH 033/121] linker: release scratch free-list blocks off the main thread The scratch decommit pass between type merge and PDB build was a near-dead ~0.9 s main-thread window (kernel-serialized MEM_DECOMMIT, whole pool parked at the barrier). ~84% of the decommitted bytes (9.4 of 11.3 GiB on the FN editor link) live in arena FREE-LIST blocks that hold no live data, so each worker now detaches its free chains (same-thread pointer ops) onto a global list and a background thread releases them while the PDB build runs; only the active-chain pages above the live pos are still decommitted synchronously. Window: 866 -> ~45 ms. Outputs unaffected (memory ops only); the reaper is joined next to the existing arena reaper before exit. Co-Authored-By: Claude Fable 5 (cherry picked from commit 4bf14ad27942311f2833ae90e135d6fc4a9ede34) --- src/linker/lnk.c | 75 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 13ed5d64a..14b14feb9 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -6775,6 +6775,11 @@ lnk_log_timers(void) scratch_end(scratch); } +// scratch free-list blocks detached during the decommit pass, released on a +// background thread (see lnk_scratch_decommit_worker) +global Arena *g_detached_scratch_blocks = 0; +global Thread g_scratch_freelist_reaper = {0}; + internal THREAD_POOL_TASK_FUNC(lnk_scratch_decommit_worker) { @@ -6791,13 +6796,66 @@ THREAD_POOL_TASK_FUNC(lnk_scratch_decommit_worker) // NOT help: MEM_DECOMMIT serializes in the kernel on the process address-space // lock (~14 GB/s aggregate no matter the thread count; measured 9.3 GiB in // 677 ms chunked-parallel vs ~500 ms with this per-worker scheme). And handing - // the decommit to a background thread is UNSAFE here: workers push to these - // scratch arenas as soon as the PDB build starts, and a push would re-commit - // pages that the background decommit then rips out. - tctx_scratch_decommit(); + // the ACTIVE-CHAIN decommit to a background thread is UNSAFE here: workers push + // to these scratch arenas as soon as the PDB build starts, and a push would + // re-commit pages that the background decommit then rips out. + // + // The FREE-LIST blocks are different: they hold no live data and are only + // touched again when a grow pops them. On the editor link ~84% of the + // decommitted bytes (9.4 of 11.3 GiB) sit in free-list blocks, so instead of + // decommitting them here (serialized kernel work on the critical path), each + // worker DETACHES its arenas' free chains (pointer ops, same thread => safe) + // onto a global list that a background thread releases while the PDB build + // runs. A post-detach grow simply sees an empty free list and reserves a + // fresh block -- same cost as the re-commit it would have paid anyway. + TCTX *tctx = tctx_selected(); + for EachIndex(arena_idx, ArrayCount(tctx->arenas)) { + Arena *arena = tctx->arenas[arena_idx]; + if (arena == 0) { continue; } +#if ARENA_FREE_LIST + // detach this arena's free chain and publish the blocks for background release + for (Arena *block = arena->free_last, *block_next = 0; block != 0; block = block_next) { + block_next = block->prev; + for (;;) { + Arena *head = (Arena *)ins_atomic_u64_eval(&g_detached_scratch_blocks); + block->prev = head; + if ((Arena *)ins_atomic_u64_eval_cond_assign((U64 *)&g_detached_scratch_blocks, (U64)block, (U64)head) == head) { break; } + } + } + arena->free_last = 0; +#endif + // decommit the committed-but-unused pages above the live pos (active chain) + arena_decommit_unused(arena); + } barrier_wait(tp->barrier); } +// Releases the scratch free-list blocks detached by lnk_scratch_decommit_worker. +// Runs in the background: MEM_RELEASE serializes on the process address-space +// lock in the kernel, so on the main thread this would extend the decommit +// window 1:1; off the main thread it overlaps the PDB build. +internal void +lnk_detached_scratch_release_thread(void *raw) +{ + ProfBeginFunction(); + U64 begin_us = now_time_us(); + + U64 released_bytes = 0; + U64 released_count = 0; + Arena *chain = (Arena *)ins_atomic_u64_eval_assign((U64 *)&g_detached_scratch_blocks, 0); + for (Arena *block = chain, *block_next = 0; block != 0; block = block_next) { + block_next = block->prev; + released_bytes += block->cmt; + released_count += 1; + AsanUnpoisonMemoryRegion(block, block->cmt); + release_memory(block, block->res); + } + + lnk_log(LNK_Log_Timers, "[teardown] background release of %llu detached scratch blocks (%llu MiB committed) took %.2f ms (off main thread)", + released_count, released_bytes / MB(1), (F64)(now_time_us() - begin_us) / 1000.0); + ProfEnd(); +} + internal THREAD_POOL_TASK_FUNC(lnk_p2r_worker) { @@ -6954,6 +7012,9 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // task_count == worker_count + the in-worker barrier => every worker // (worker 0 IS the main thread) runs exactly once, covering main's scratch. tp_for_parallel_reserve(tp, 0, tp->worker_count, lnk_scratch_decommit_worker, 0); // BARRIER pass (path B) + if (g_detached_scratch_blocks != 0) { + g_scratch_freelist_reaper = thread_launch(lnk_detached_scratch_release_thread, 0); + } lnk_log(LNK_Log_Timers, "[teardown] scratch decommit pass in %.2f ms", (F64)(now_time_us() - decommit_begin_us) / 1000.0); ProfEnd(); } @@ -7093,6 +7154,12 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) MemoryZeroStruct(&g_arena_reaper_thread); } + // reap the background scratch free-list release thread, if one was launched + if (g_scratch_freelist_reaper.u64[0] != 0) { + thread_join(g_scratch_freelist_reaper, max_U64); + MemoryZeroStruct(&g_scratch_freelist_reaper); + } + // image is on disk and no longer read by anyone -- release its ~1GB now so the kernel reclaims it // concurrently with the remaining work + exit, not single-threaded in the process rundown. release_memory(image_ctx.image_data.str, image_ctx.image_data.size); From a53bbf932b4beb24e494f4dc456c9f923a90b558 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 2 Jul 2026 15:56:03 -0700 Subject: [PATCH 034/121] linker: warn visibly on unknown /RAD_* command-line switches The RAD_ switch namespace is owned by radlink, so an unknown /RAD_ switch on the command line means the build system expects a feature this binary does not have. Previously this fell into LNK_Warning_UnknownSwitch, which the release-default /RAD_IGNORE mutes -- the switch was silently dropped with no trace in the build log. Newer build scripts must keep working against older radlink binaries (forward compatibility), so this must not fail the link either: warn once per unknown switch via LNK_Warning_Cmdl (not muted by the release default), ignore the switch, and continue. Unknown non-RAD switches and obj-directive switches keep the old warning. Also register RAD_TYPEHASHALG as an alias of RAD_TPYE_HASH_ALG [sic]: UnrealBuildTool passes /RAD_TypeHashAlg:BLAKE3, which never matched the misspelled table entry and was silently ignored. Honoring it is a no-op (BLAKE3 is already the internal default pushed via /RAD_TPYE_HASH_ALG:BLAKE3), verified by byte-identical Engine.dll A/B. Co-Authored-By: Claude Fable 5 (cherry picked from commit 2b45cc0269fd1cdbe285795734088f68a25e4c32) --- src/linker/lnk_config.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index 51d22f6a2..4533d67ce 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -104,6 +104,7 @@ global read_only LNK_CmdSwitch g_cmd_switch_map[] = { LNK_CmdSwitch_Rad_WriteTempFiles, 0, LNK_CmdValueKind_Scalar, "RAD_WRITE_TEMP_FILES", "[:NO]", "When speicifed linker writes image and debug info to temporary files and renames after link is done." }, { LNK_CmdSwitch_Rad_TimeStamp, 0, LNK_CmdValueKind_Scalar, "RAD_TIME_STAMP", ":#", "Time stamp embeded in EXE and PDB." }, { LNK_CmdSwitch_Rad_DebugTypeHash, 0, LNK_CmdValueKind_Scalar, "RAD_DEBUG_TYPE_HASH", ":{BLAKE3|XXHASH}", "Sets hashing algorithm for debug type merging." }, + { LNK_CmdSwitch_Rad_DebugTypeHash, 0, LNK_CmdValueKind_Scalar, "RAD_TYPEHASHALG", ":{BLAKE3|XXHASH}", "Alias of RAD_DEBUG_TYPE_HASH (spelling used by UnrealBuildTool)." }, { LNK_CmdSwitch_Rad_UnresolvedSymbolLimit, 0, LNK_CmdValueKind_Scalar, "RAD_UNRESOLVED_SYMBOL_LIMIT", ":#", "Limits number of unresolved symbol errors linker reports." }, { LNK_CmdSwitch_Rad_UnresolvedSymbolRefLimit, 0, LNK_CmdValueKind_Scalar, "RAD_UNRESOLVED_SYMBOL_REF_LIMIT", ":#", "Limit number of unresolved symbol references linker reports." }, { LNK_CmdSwitch_Rad_Version, 0, LNK_CmdValueKind_Null, "RAD_VERSION", "", "Print version and exit." }, @@ -1210,7 +1211,17 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val switch (cmd_switch) { case LNK_CmdSwitch_Null: { - lnk_error_obj(LNK_Warning_UnknownSwitch, obj, "unknown switch: \"/%S%s%S\"", cmd_name, value.size ? ":" : "", value); + // Unknown /RAD_* switches on the command line warn and are ignored: the + // RAD_ namespace is owned by this linker, but newer build scripts must + // keep working against older radlink binaries (forward compatibility), + // so an unrecognized /RAD_* switch must not fail the link. Use + // LNK_Warning_Cmdl so the warning stays visible even though the + // release-default /RAD_IGNORE mutes LNK_Warning_UnknownSwitch. + if (obj == 0 && str8_match_lit("RAD_", str8_prefix(cmd_name, 4), StringMatchFlag_CaseInsensitive)) { + lnk_error(LNK_Warning_Cmdl, "unknown switch \"/%S%s%S\"; this radlink build does not support it -- switch ignored", cmd_name, value.size ? ":" : "", value); + } else { + lnk_error_obj(LNK_Warning_UnknownSwitch, obj, "unknown switch: \"/%S%s%S\"", cmd_name, value.size ? ":" : "", value); + } } break; default: break; From 2515a8f62dea0b29411aacb1ecf318039bff27fa Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 3 Jul 2026 12:46:49 -0700 Subject: [PATCH 035/121] linker: opt-in one-line [radlink summary] (v=3) for production triage Per-link line behind /RAD_LOG:Summary: wall/user/kern, peak ws, peak commit charge (cm=), CoW-promoted pages (cowp=), page faults, io, mem= available-physical samples (t0/pdb/t1), per-phase wall/user/kernel/faults quadruples with dbgg[]/pdbg[] sub-buckets and other= residuals (local cross-check: sum + residual == phase within 0.2%), pool grant_avg/park and procs=now/peak via a named semaphore (UBA virtualizes named sections per-process; semaphores pass through). The shared-pool cross-process counter detach runs unconditionally on every exit path -- it is the only decrement site (the linker leaves through _exit). This line is what root-caused every production fault-storm from build logs alone. Co-Authored-By: Claude Fable 5 (cherry picked from commit 2223534c7f83e5118248ddb108fb67a220e55611) --- src/base/base_threads.h | 1 + src/linker/lnk.c | 320 ++++++++++++++++++++++++++- src/linker/lnk.h | 6 + src/linker/lnk_debug_info.c | 18 ++ src/linker/lnk_log.c | 3 + src/linker/lnk_log.h | 3 +- src/linker/lnk_timer.c | 71 +++++- src/linker/lnk_timer.h | 55 +++++ src/linker/thread_pool/thread_pool.c | 128 ++++++++++- src/linker/thread_pool/thread_pool.h | 54 +++++ src/linux/base/linux_base.c | 14 ++ src/win32/base/win32_base.c | 9 + 12 files changed, 675 insertions(+), 7 deletions(-) diff --git a/src/base/base_threads.h b/src/base/base_threads.h index c84bf26e2..b40c10f9f 100644 --- a/src/base/base_threads.h +++ b/src/base/base_threads.h @@ -133,6 +133,7 @@ internal void semaphore_drop(Semaphore semaphore); internal void semaphore_drop_count(Semaphore semaphore, U64 drop_count); internal void semaphore_drop_if_room(Semaphore semaphore); // best-effort post; no-op if already at max internal void semaphore_drop_n(Semaphore semaphore, U32 count); // release `count` permits in one syscall +internal B32 semaphore_drop_prev(Semaphore semaphore, U32 *prev_count_out); // release 1 permit + report the pre-release count (win32: exact, from ReleaseSemaphore; posix: best-effort sem_getvalue) //- rjf: barriers internal Barrier barrier_alloc(U64 count); diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 14b14feb9..fc0199c9c 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -54,6 +54,10 @@ #include "llvm/llvm.c" #include "dwarf/x64/dwarf_x64.c" +#if OS_WINDOWS +# include // GetProcessMemoryInfo for the end-of-link summary line +#endif + // --- Third Party ------------------------------------------------------------- #include "base_ext/base_blake3.h" @@ -1224,6 +1228,7 @@ internal void lnk_load_inputs(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer *inputer, LNK_SymbolTable *symtab, LNK_Link *link) { ProfBeginFunction(); + lnk_summary_phase_begin(LNK_SummaryPhase_Input); Temp scratch = scratch_begin(arena->v, arena->count); U64 obj_id_base = link->objs.count; @@ -1445,6 +1450,7 @@ lnk_load_inputs(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer } scratch_end(scratch); + lnk_summary_phase_end(LNK_SummaryPhase_Input); ProfEnd(); } @@ -1637,6 +1643,14 @@ lnk_link_inputs(TP_Context *tp, ProfBeginFunction(); Temp scratch = scratch_begin(arena->v, arena->count); + // summary: this function is Input (lnk_load_inputs rounds, accumulated on its + // own bucket) + Resolve (lib search / member resolution / directives, i.e. + // everything else); attribute the remainder to Resolve at the bottom. The + // same subtraction works for every counter: they are monotonic and the Input + // brackets nest strictly inside this window + LNK_SummaryCounters summary_begin = lnk_summary_counters_now(); + LNK_SummaryCounters summary_input_at = g_summary_phase[LNK_SummaryPhase_Input]; + HashMap imports_hm = {0}; LNK_LibMemberRefList *member_ref_lists = push_array(scratch.arena, LNK_LibMemberRefList, tp->worker_count); @@ -1919,6 +1933,17 @@ lnk_link_inputs(TP_Context *tp, if (resolved_members_count == 0) { break; } } + { + LNK_SummaryCounters now = lnk_summary_counters_now(); + LNK_SummaryCounters window = lnk_summary_counters_sub_sat(now, summary_begin); + LNK_SummaryCounters input_delta = lnk_summary_counters_sub_sat(g_summary_phase[LNK_SummaryPhase_Input], summary_input_at); + LNK_SummaryCounters resolve = lnk_summary_counters_sub_sat(window, input_delta); + g_summary_phase[LNK_SummaryPhase_Resolve].wall_us += resolve.wall_us; + g_summary_phase[LNK_SummaryPhase_Resolve].user_us += resolve.user_us; + g_summary_phase[LNK_SummaryPhase_Resolve].kern_us += resolve.kern_us; + g_summary_phase[LNK_SummaryPhase_Resolve].faults += resolve.faults; + } + scratch_end(scratch); ProfEnd(); } @@ -2443,7 +2468,9 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer // if (config->opt_ref == LNK_SwitchState_Yes) { if (objs == 0) { objs = lnk_array_from_obj_list(scratch.arena, link->objs); } + lnk_summary_phase_begin(LNK_SummaryPhase_Ref); lnk_opt_ref(tp, symtab, config, objs, link->objs.count); + lnk_summary_phase_end(LNK_SummaryPhase_Ref); } // @@ -2451,7 +2478,9 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer // if (config->opt_icf == LNK_SwitchState_Yes) { if (objs == 0) { objs = lnk_array_from_obj_list(scratch.arena, link->objs); } + lnk_summary_phase_begin(LNK_SummaryPhase_Icf); lnk_opt_icf(tp, symtab, config, objs, link->objs.count); + lnk_summary_phase_end(LNK_SummaryPhase_Icf); } } @@ -6723,11 +6752,269 @@ internal void lnk_write_thread(void *raw_ctx) { ProfBeginFunction(); + lnk_summary_phase_begin(LNK_SummaryPhase_Write); LNK_WriteThreadContext *ctx = raw_ctx; lnk_write_data_to_file_path(ctx->path, ctx->temp_path, ctx->data); + lnk_summary_phase_end(LNK_SummaryPhase_Write); ProfEnd(); } +//////////////////////////////////////////////////////////////////////////////// +//~ One-line end-of-link summary (always on; production triage). Everything +// needed at print time is stashed in this global as the link progresses, so +// the line can be emitted best-effort from ANY exit path (lnk_exit on error, +// entry_point on success) with whatever was known by then. + +typedef struct LNK_SummaryInfo +{ + volatile U32 printed; // print-exactly-once latch + U64 start_us; // set first thing in entry_point + U64 t0_ms; // UTC ms epoch at link start (t1 is stamped at print time) + U64 worker_count; + U64 objs_count; + U64 input_bytes; // sum of obj data sizes (lib members count their slice) + U64 libs_count; + // physical-memory samples (GlobalMemoryStatusEx): storm triage -- prod storms + // show pdb-phase kernel time exploding 54x for the same fault count, fitting + // free-list exhaustion / page-repurpose; these 3 samples prove/refute that + U64 mem_avail_t0; // ullAvailPhys at link start + U64 mem_avail_pdb; // ullAvailPhys at pdb-phase start (0 = phase never ran) + U32 mem_load_max; // max dwMemoryLoad seen across the samples + // name COPIES: config strings parsed out of an @rsp point into the response + // file buffer, whose scratch dies right after config parse -- capture the + // bytes here instead of keeping String8s into freed memory + U64 out_name_size; + U64 pool_name_size; // non-zero => /RAD_SHARED_THREAD_POOL + U8 out_name [128]; + U8 pool_name[128]; +} LNK_SummaryInfo; + +global LNK_SummaryInfo g_summary_info; + +internal void +lnk_summary_copy_name(U8 *dst, U64 dst_cap, U64 *dst_size_out, String8 name) +{ + U64 size = Min(name.size, dst_cap); + MemoryCopy(dst, name.str, size); + *dst_size_out = size; +} + +internal U64 +lnk_summary_us_from_timer(LNK_TimerType timer) +{ + // guard against a fatal exit mid-phase (begin stamped, end still zero) + return g_timers[timer].end > g_timers[timer].begin ? g_timers[timer].end - g_timers[timer].begin : 0; +} + +internal LNK_SummaryCounters +lnk_summary_counters_from_timer(LNK_TimerType timer) +{ + LNK_SummaryCounters zero = {0}; + // same mid-phase guard as lnk_summary_us_from_timer + if (g_timers[timer].end <= g_timers[timer].begin) { return zero; } + return lnk_summary_counters_sub_sat(g_timer_counters_end[timer], g_timer_counters_begin[timer]); +} + +// one GlobalMemoryStatusEx sample for the summary line: returns available +// physical bytes and folds dwMemoryLoad into the running max. 1 syscall per +// call, called 3x per link (link start, pdb-phase start, print time). +internal U64 +lnk_summary_sample_mem(void) +{ + U64 avail = 0; +#if OS_WINDOWS + MEMORYSTATUSEX msx = { sizeof(msx) }; + if (GlobalMemoryStatusEx(&msx)) { + avail = msx.ullAvailPhys; + if (msx.dwMemoryLoad > g_summary_info.mem_load_max) { g_summary_info.mem_load_max = msx.dwMemoryLoad; } + } +#endif + return avail; +} + +internal U64 +lnk_summary_utc_ms(void) +{ +#if OS_WINDOWS + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + U64 t100 = ((U64)ft.dwHighDateTime << 32) | ft.dwLowDateTime; + return (t100 - 116444736000000000ULL) / 10000; // FILETIME epoch -> unix ms epoch +#else + return 0; +#endif +} + +// one phase bucket -> "wall-ms/user-ms/kernel-ms/faults-K" (process-wide deltas +// at the bucket's boundaries; user can exceed wall on parallel phases, and a +// bucket that overlaps another thread's work counts that work too) +internal String8 +lnk_summary_str_from_counters(Arena *arena, LNK_SummaryCounters c) +{ + return push_str8f(arena, "%llu/%llu/%llu/%llu", c.wall_us / 1000, c.user_us / 1000, c.kern_us / 1000, c.faults / 1000); +} + +internal void +lnk_print_summary(int exit_code) +{ + // run exactly once, no matter which exit path gets here first + if (ins_atomic_u32_eval_cond_assign(&g_summary_info.printed, 1, 0) != 0) { + return; + } + + // detach from the shared-pool cross-process counter on every exit path, even + // when the summary line is off -- the linker leaves through _exit and never + // runs tp_release, so this is the only place the counter gets decremented + F64 pool_grant_avg = 0, pool_park_seconds = 0; + U32 pool_procs_now = 0, pool_procs_peak = 0; + B32 pool_on = (g_summary_info.pool_name_size > 0); + if (pool_on) { + tp_stats_snapshot(&pool_grant_avg, &pool_park_seconds); + tp_procs_snapshot(&pool_procs_now, &pool_procs_peak); + tp_procs_detach(); + } + + // the line itself is opt-in (/RAD_LOG:Summary) -- always-on turned out to be + // noise in build logs; farm convoy triage passes the switch explicitly + if (!lnk_get_log_status(LNK_Log_Summary)) { + return; + } + + Temp scratch = scratch_begin(0, 0); + + F64 wall = g_summary_info.start_us ? (F64)(now_time_us() - g_summary_info.start_us) / 1000000.0 : 0; + + // process CPU + memory counters + F64 user_time = 0, kernel_time = 0, peak_ws_gib = 0, page_faults_m = 0, peak_commit_gib = 0; + U32 cow_promoted_pages = 0; +#if OS_WINDOWS + { + FILETIME create_ft, exit_ft, kernel_ft, user_ft; + if (GetProcessTimes(GetCurrentProcess(), &create_ft, &exit_ft, &kernel_ft, &user_ft)) { + user_time = (F64)(((U64)user_ft.dwHighDateTime << 32) | user_ft.dwLowDateTime) / 10000000.0; + kernel_time = (F64)(((U64)kernel_ft.dwHighDateTime << 32) | kernel_ft.dwLowDateTime) / 10000000.0; + } + PROCESS_MEMORY_COUNTERS pmc = { (DWORD)sizeof(pmc) }; + if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) { + peak_ws_gib = (F64)pmc.PeakWorkingSetSize / (F64)GB(1); + page_faults_m = (F64)pmc.PageFaultCount / 1000000.0; + // peak pagefile-backed commit charge -- the number build-farm memory admission + // sees; with read-only input views this tracks ws minus the mapped input set + peak_commit_gib = (F64)pmc.PeakPagefileUsage / (F64)GB(1); + } + cow_promoted_pages = (U32)g_lnk_cow_promoted_pages; + } +#endif + + // process IO totals: hard page-ins on mapped inputs surface as read bytes, + // UBA-detoured output writes as write bytes + U64 io_read_mb = 0, io_write_mb = 0; +#if OS_WINDOWS + { + IO_COUNTERS ioc = {0}; + if (GetProcessIoCounters(GetCurrentProcess(), &ioc)) { + io_read_mb = ioc.ReadTransferCount / MB(1); + io_write_mb = ioc.WriteTransferCount / MB(1); + } + } +#endif + + // phase triplets. img/dbg/pdb come from the /RAD_LOG:TIMERS stamps; dbg is + // the debug-info umbrella minus the PDB/RDI sub-phases it contains. + LNK_SummaryCounters img_c = lnk_summary_counters_from_timer(LNK_Timer_Image); + LNK_SummaryCounters pdb_c = lnk_summary_counters_from_timer(LNK_Timer_Pdb); + LNK_SummaryCounters rdi_c = lnk_summary_counters_from_timer(LNK_Timer_Rdi); + LNK_SummaryCounters dbg_c = lnk_summary_counters_sub_sat(lnk_summary_counters_sub_sat(lnk_summary_counters_from_timer(LNK_Timer_Debug), pdb_c), rdi_c); + + // residual catch-alls: umbrella bucket minus the sum of its printed + // sub-buckets, clamped at 0 per field (a /PDBSTRIPPED link runs the pdb + // sub-phases a second time OUTSIDE the Timer_Pdb bracket, which can push the + // sub-bucket sum past the umbrella -- clamp instead of printing garbage). + // Storm triage: prod shows the pdbg sub-buckets covering only ~19% of pdb + // kernel time in a storm window vs ~96% locally -- other= pins the + // uncovered span without waiting for a local repro. + LNK_SummaryCounters pdb_other, dbg_other; + { + LNK_SummaryCounters pdbg_sum = g_summary_phase[LNK_SummaryPhase_PdbGsi]; + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbHsh]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbIni]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbSym]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbMod]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbTpi]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbStr]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbSc]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbMsf]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbWr]); + pdb_other = lnk_summary_counters_sub_sat(pdb_c, pdbg_sum); + + LNK_SummaryCounters dbgg_sum = lnk_summary_counters_add(g_summary_phase[LNK_SummaryPhase_DbgMcvi], g_summary_phase[LNK_SummaryPhase_DbgMerge]); + dbg_other = lnk_summary_counters_sub_sat(dbg_c, dbgg_sum); + } + + // governor stats snapshotted above, before the detach + String8 pool_stats = str8_zero(); + if (pool_on) { + pool_stats = push_str8f(scratch.arena, " pool=%S grant_avg=%.1f park=%.1f procs=%u/%u", + str8(g_summary_info.pool_name, g_summary_info.pool_name_size), pool_grant_avg, pool_park_seconds, pool_procs_now, pool_procs_peak); + } + + // final memory sample (t1) -- 3rd and last GlobalMemoryStatusEx of the link + U64 mem_avail_t1 = lnk_summary_sample_mem(); + + lnk_fprintf(stdout, + "[radlink summary] v=3 out=%S exit=%d t0=%llu t1=%llu wall=%.1f user=%.1f kern=%.1f ws=%.1fG cm=%.1fG cowp=%u pf=%.1fM io=%llu/%lluMB mem=%.1f/%.1f/%.1f/%u workers=%llu%S" + " in=%lluo/%.1fG libs=%llu" + " ph[inp=%S res=%S icf=%S ref=%S img=%S dbg=%S pdb=%S wr=%S]" + " dbgg[mcvi=%S merge=%S other=%S]" + " pdbg[hsh=%S ini=%S gsi=%S sym=%S mod=%S tpi=%S str=%S sc=%S msf=%S wr=%S other=%S]\n", + g_summary_info.out_name_size ? str8(g_summary_info.out_name, g_summary_info.out_name_size) : str8_lit("-"), + exit_code, + g_summary_info.t0_ms, + lnk_summary_utc_ms(), + wall, + user_time, + kernel_time, + peak_ws_gib, + peak_commit_gib, + cow_promoted_pages, + page_faults_m, + io_read_mb, + io_write_mb, + (F64)g_summary_info.mem_avail_t0 / (F64)GB(1), + (F64)g_summary_info.mem_avail_pdb / (F64)GB(1), + (F64)mem_avail_t1 / (F64)GB(1), + g_summary_info.mem_load_max, + g_summary_info.worker_count, + pool_stats, + g_summary_info.objs_count, + (F64)g_summary_info.input_bytes / (F64)GB(1), + g_summary_info.libs_count, + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Input]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Resolve]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Icf]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Ref]), + lnk_summary_str_from_counters(scratch.arena, img_c), + lnk_summary_str_from_counters(scratch.arena, dbg_c), + lnk_summary_str_from_counters(scratch.arena, pdb_c), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Write]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_DbgMcvi]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_DbgMerge]), + lnk_summary_str_from_counters(scratch.arena, dbg_other), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbHsh]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbIni]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbGsi]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbSym]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbMod]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbTpi]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbStr]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbSc]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbMsf]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbWr]), + lnk_summary_str_from_counters(scratch.arena, pdb_other)); + + scratch_end(scratch); +} + internal void lnk_log_timers(void) { @@ -6938,6 +7225,15 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) LNK_Obj **objs = lnk_array_from_obj_list(scratch.arena, link.objs); LNK_Lib **libs = lnk_array_from_lib_list(scratch.arena, link.libs); + // summary: input volume (lib members count their member slice) + { + U64 input_bytes = 0; + for EachIndex(obj_idx, objs_count) { input_bytes += objs[obj_idx]->coff.data.size; } + g_summary_info.objs_count = objs_count; + g_summary_info.libs_count = libs_count; + g_summary_info.input_bytes = input_bytes; + } + // // Layout Image // @@ -6990,8 +7286,12 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // CodeView // LNK_RRT_Array rrt_input = lnk_rrt_array_from_config(arena->v[0], config); + lnk_summary_phase_begin(LNK_SummaryPhase_DbgMcvi); LNK_CodeViewInput cv = lnk_make_code_view_input(tp, arena, config, debug_info_objs_count, debug_info_objs, rrt_input); + lnk_summary_phase_end(LNK_SummaryPhase_DbgMcvi); + lnk_summary_phase_begin(LNK_SummaryPhase_DbgMerge); LNK_MergedTypes cv_types = lnk_merge_types(tp, arena, &cv, 0); + lnk_summary_phase_end(LNK_SummaryPhase_DbgMerge); // prune merged types not reachable from any surviving symbol (PDB-size win). OFF by default: // it removes types that a debugger can still legitimately cast to in the watch window @@ -7027,8 +7327,9 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) if (config->debug_mode == LNK_DebugMode_Full || config->rad_debug == LNK_SwitchState_Yes) { LNK_FileArtifact pdb_artifact = {0}; { + g_summary_info.mem_avail_pdb = lnk_summary_sample_mem(); lnk_timer_begin(LNK_Timer_Pdb); - + lnk_summary_phase_begin(LNK_SummaryPhase_PdbHsh); if (config->pdb_hash_type_names != LNK_TypeNameHashMode_None) { lnk_replace_type_names_with_hashes(tp, arena, @@ -7038,10 +7339,12 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) config->pdb_hash_type_name_length, config->pdb_hash_type_name_map); } - + lnk_summary_phase_end(LNK_SummaryPhase_PdbHsh); pdb_writer.output_path = config->debug_mode == LNK_DebugMode_Full ? config->pdb_name : str8_zero(); pdb_writer.temp_output_path = config->debug_mode == LNK_DebugMode_Full ? config->temp_pdb_name : str8_zero(); + lnk_summary_phase_begin(LNK_SummaryPhase_PdbWr); pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &cv, cv_types, pdb_writer, LNK_PDB_BuilderFlag_All); + lnk_summary_phase_end(LNK_SummaryPhase_PdbWr); lnk_timer_end(LNK_Timer_Pdb); } @@ -7127,7 +7430,9 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) stripped_cv.symbol_input_ranges = push_array(scratch.arena, Rng1U64, tp->worker_count); LNK_FileArtifact pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &stripped_cv, (LNK_MergedTypes){0}, (LNK_PdbWriter){0}, LNK_PDB_BuilderFlag_All); + lnk_summary_phase_begin(LNK_SummaryPhase_PdbWr); lnk_write_data_list_to_file_path(config->pdb_stripped_name, str8f(scratch.arena, "%S.tmp", config->pdb_stripped_name), pdb_artifact.data); + lnk_summary_phase_end(LNK_SummaryPhase_PdbWr); } lnk_timer_end(LNK_Timer_Debug); @@ -7432,11 +7737,20 @@ internal void entry_point(CmdLine *cmdline) { Temp scratch = scratch_begin(0,0); + g_summary_info.start_us = now_time_us(); + g_summary_info.t0_ms = lnk_summary_utc_ms(); + g_summary_info.mem_avail_t0 = lnk_summary_sample_mem(); lnk_log_begin(); // init config from the command line LNK_Config *config = lnk_config_init(cmdline->argc, cmdline->argv); + // Snapshot summary identity immediately after command-line parsing, before + // pool initialization and later scratch allocations. + lnk_summary_copy_name(g_summary_info.out_name, sizeof(g_summary_info.out_name), &g_summary_info.out_name_size, str8_skip_last_slash(config->out_path)); + lnk_summary_copy_name(g_summary_info.pool_name, sizeof(g_summary_info.pool_name), &g_summary_info.pool_name_size, config->shared_thread_pool_name); + g_summary_info.worker_count = config->worker_count; + if (lnk_get_log_status(LNK_Log_Debug)) { lnk_fprintf(stderr, "--------------------------------------------------------------------------------\n"); lnk_fprintf(stderr, "Command Line: %S\n", config->raw_cmd_line); @@ -7454,6 +7768,8 @@ entry_point(CmdLine *cmdline) case LNK_BootMode_TypeServer: lnk_run_type_server(tp, tp_arena, config); break; } + lnk_print_summary(0); + lnk_log_end(); scratch_end(scratch); } diff --git a/src/linker/lnk.h b/src/linker/lnk.h index 59b3d9a27..556693f5f 100644 --- a/src/linker/lnk.h +++ b/src/linker/lnk.h @@ -409,3 +409,9 @@ internal LNK_ImageContext lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_C internal void lnk_log_link_stats(LNK_ObjList obj_list, LNK_LibList *lib_index, LNK_SectionTable *sectab); internal void lnk_log_timers(void); + +// One-line end-of-link summary for production triage (always on). Prints +// exactly once; safe to call from any exit path (values best-effort on early +// error exits). Defined in lnk.c; called from lnk_exit and entry_point. +internal void lnk_print_summary(int exit_code); + diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 654356737..aaf402c46 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4617,6 +4617,11 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config builder_flags = ~0; } + // ini= bucket: pdb_alloc_ commits the MSF + type-server tables (fresh pages, + // ~132K faults on the editor link) -- under a storm every fresh commit pays + // the page-repurpose path, so this span needs its own attribution + lnk_summary_phase_begin(LNK_SummaryPhase_PdbIni); + LNK_BuildPdb task = { .image_data = image_data, .symtab = symtab, @@ -4661,18 +4666,24 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config // objs in buckets [C,worker_count) unprocessed when Ctype_servers[CV_TypeIndexSource_IPI], cv_types.count[CV_TypeIndexSource_IPI], cv_types.v[CV_TypeIndexSource_IPI]); } if (builder_flags & LNK_PDB_BuilderFlag_Tpi) { pdb_type_server_push_parallel(tp, task.pdb->type_servers[CV_TypeIndexSource_TPI], cv_types.count[CV_TypeIndexSource_TPI], cv_types.v[CV_TypeIndexSource_TPI]); } + lnk_summary_phase_end(LNK_SummaryPhase_PdbTpi); + lnk_summary_phase_begin(LNK_SummaryPhase_PdbStr); ProfBegin("Merge String Tables"); task.string_ht = cv_dedup_string_tables(tp_arena, tp, cv->obj_count, cv->debug_s_arr); cv_string_hash_table_assign_buffer_offsets(tp, task.string_ht); ProfEnd(); + lnk_summary_phase_end(LNK_SummaryPhase_PdbStr); task.string_table_base_offset = task.pdb->info->strtab.size; ProfBegin("Add string tables"); @@ -4688,6 +4699,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config ProfScope("Write Modules") { + lnk_summary_phase_begin(LNK_SummaryPhase_PdbGsi); U64 phase_begin_us = now_time_us(); // FAIR-SHARE: pin the cohort, distribute objs over exactly the cohort lanes, // then run the barrier pass at that cohort. tp_barrier_begin sets @@ -4705,6 +4717,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config tp_for_parallel_reserve(tp, 0, C, lnk_write_pdb_modules, &task); // BARRIER pass (path B): barrier_wait/tp_broadcast tp_barrier_end(tp); lnk_log(LNK_Log_Timers, "[pdb] write modules in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); + lnk_summary_phase_end(LNK_SummaryPhase_PdbMod); } if (output_ptr != 0) { for EachIndex(obj_idx, cv->obj_count) { @@ -4733,6 +4746,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config } if (builder_flags & LNK_PDB_BuilderFlag_SC) { + lnk_summary_phase_begin(LNK_SummaryPhase_PdbSc); ProfBegin("Build Section Contrib Map"); { ProfBegin("Build DBI Section Headers"); @@ -4774,6 +4788,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config sec_contribs->cap = new_count; } ProfEnd(); + lnk_summary_phase_end(LNK_SummaryPhase_PdbSc); } if (builder_flags & LNK_PDB_BuilderFlag_NATVIS) { @@ -4809,6 +4824,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config ProfEnd(); } + lnk_summary_phase_begin(LNK_SummaryPhase_PdbMsf); pdb_build_dbi_info(tp, task.pdb, task.string_ht, 0, cv->is_stripped, &build_hooks); MSF_Error msf_err = msf_build(task.pdb->msf); @@ -4827,6 +4843,8 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config if (output_ptr != 0) { lnk_background_file_writer_end_file(output_ptr->writer, output_ptr->file, artifact.data.total_size); } + lnk_summary_phase_end(LNK_SummaryPhase_PdbMsf); + // NOTE: linker is about to exit so we can skip memory release // and let windows free memory since it does this faster diff --git a/src/linker/lnk_log.c b/src/linker/lnk_log.c index 5dce5875d..d8700d3c1 100644 --- a/src/linker/lnk_log.c +++ b/src/linker/lnk_log.c @@ -28,6 +28,9 @@ lnk_fprintf(FILE *f, char *fmt, ...) internal void lnk_exit(int code) { + // one-line summary must reach the build log on error exits too (best-effort; + // takes/drops g_log_mutex internally, so print BEFORE taking it here) + lnk_print_summary(code); mutex_take(g_log_mutex); fflush(stdout); fflush(stderr); diff --git a/src/linker/lnk_log.h b/src/linker/lnk_log.h index b10c836b5..01bf5fae2 100644 --- a/src/linker/lnk_log.h +++ b/src/linker/lnk_log.h @@ -13,7 +13,8 @@ X(SizeBreakdown) \ X(LinkStats) \ X(Timers) \ - X(Links) + X(Links) \ + X(Summary) typedef enum { diff --git a/src/linker/lnk_timer.c b/src/linker/lnk_timer.c index b94817d78..748c643bb 100644 --- a/src/linker/lnk_timer.c +++ b/src/linker/lnk_timer.c @@ -3,16 +3,83 @@ global LNK_Timer g_timers[LNK_Timer_Count]; +// summary (v2): every timer/phase boundary also stamps process-wide CPU + +// fault counters, so each bucket reports wall/user/kernel/faults +global LNK_SummaryCounters g_timer_counters_begin[LNK_Timer_Count]; +global LNK_SummaryCounters g_timer_counters_end [LNK_Timer_Count]; + +internal LNK_SummaryCounters +lnk_summary_counters_now(void) +{ + LNK_SummaryCounters c = { .wall_us = now_time_us() }; +#if OS_WINDOWS + FILETIME create_ft, exit_ft, kernel_ft, user_ft; + if (GetProcessTimes(GetCurrentProcess(), &create_ft, &exit_ft, &kernel_ft, &user_ft)) { + c.user_us = (((U64)user_ft.dwHighDateTime << 32) | user_ft.dwLowDateTime) / 10; + c.kern_us = (((U64)kernel_ft.dwHighDateTime << 32) | kernel_ft.dwLowDateTime) / 10; + } + PROCESS_MEMORY_COUNTERS pmc = { (DWORD)sizeof(pmc) }; + if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) { + c.faults = pmc.PageFaultCount; + } +#endif + return c; +} + +internal LNK_SummaryCounters +lnk_summary_counters_sub_sat(LNK_SummaryCounters a, LNK_SummaryCounters b) +{ + LNK_SummaryCounters c; + c.wall_us = a.wall_us > b.wall_us ? a.wall_us - b.wall_us : 0; + c.user_us = a.user_us > b.user_us ? a.user_us - b.user_us : 0; + c.kern_us = a.kern_us > b.kern_us ? a.kern_us - b.kern_us : 0; + c.faults = a.faults > b.faults ? a.faults - b.faults : 0; + return c; +} + +internal LNK_SummaryCounters +lnk_summary_counters_add(LNK_SummaryCounters a, LNK_SummaryCounters b) +{ + LNK_SummaryCounters c; + c.wall_us = a.wall_us + b.wall_us; + c.user_us = a.user_us + b.user_us; + c.kern_us = a.kern_us + b.kern_us; + c.faults = a.faults + b.faults; + return c; +} + internal void lnk_timer_begin(LNK_TimerType timer) { - g_timers[timer].begin = now_time_us(); + g_timer_counters_begin[timer] = lnk_summary_counters_now(); + g_timers[timer].begin = g_timer_counters_begin[timer].wall_us; } internal void lnk_timer_end(LNK_TimerType timer) { - g_timers[timer].end = now_time_us(); + g_timer_counters_end[timer] = lnk_summary_counters_now(); + g_timers[timer].end = g_timer_counters_end[timer].wall_us; +} + +global LNK_SummaryCounters g_summary_phase [LNK_SummaryPhase_Count]; +global LNK_SummaryCounters g_summary_phase_start[LNK_SummaryPhase_Count]; + +internal void +lnk_summary_phase_begin(LNK_SummaryPhase phase) +{ + g_summary_phase_start[phase] = lnk_summary_counters_now(); +} + +internal void +lnk_summary_phase_end(LNK_SummaryPhase phase) +{ + // atomic adds: the Write bracket runs on the background image-write thread + LNK_SummaryCounters now = lnk_summary_counters_now(); + ins_atomic_u64_add_eval(&g_summary_phase[phase].wall_us, now.wall_us - g_summary_phase_start[phase].wall_us); + ins_atomic_u64_add_eval(&g_summary_phase[phase].user_us, now.user_us - g_summary_phase_start[phase].user_us); + ins_atomic_u64_add_eval(&g_summary_phase[phase].kern_us, now.kern_us - g_summary_phase_start[phase].kern_us); + ins_atomic_u64_add_eval(&g_summary_phase[phase].faults, now.faults - g_summary_phase_start[phase].faults); } internal String8 diff --git a/src/linker/lnk_timer.h b/src/linker/lnk_timer.h index 9e5864549..5ea02c59f 100644 --- a/src/linker/lnk_timer.h +++ b/src/linker/lnk_timer.h @@ -22,3 +22,58 @@ typedef struct LNK_Timer internal void lnk_timer_begin(LNK_TimerType timer); internal void lnk_timer_end(LNK_TimerType timer); +// Per-phase counter snapshot for the end-of-link summary line (v2). Each +// boundary stamp is wall (QPC) + process-wide user/kernel CPU (GetProcessTimes) +// + process-wide soft+hard fault count (GetProcessMemoryInfo.PageFaultCount): +// 2 syscalls per boundary, negligible. Deltas are PROCESS-WIDE, so a phase that +// overlaps a concurrent thread's work (e.g. the image-write thread overlapping +// the debug-info phases) counts that work too -- attribution, not accounting. +typedef struct LNK_SummaryCounters +{ + U64 wall_us; + U64 user_us; // process user CPU, all threads + U64 kern_us; // process kernel CPU, all threads + U64 faults; // process page faults (soft+hard) +} LNK_SummaryCounters; + +internal LNK_SummaryCounters lnk_summary_counters_now(void); +internal LNK_SummaryCounters lnk_summary_counters_sub_sat(LNK_SummaryCounters a, LNK_SummaryCounters b); // per-field saturating a-b +internal LNK_SummaryCounters lnk_summary_counters_add(LNK_SummaryCounters a, LNK_SummaryCounters b); // per-field a+b + +// Phase accumulators for the end-of-link summary line. Unlike LNK_Timer +// (single begin/end shot), these ACCUMULATE across repeated brackets (e.g. +// lnk_load_inputs runs once per input round; the PDB sub-phases run again for +// the /PDBSTRIPPED build). Always measured. Image/Debug/PDB/RDI buckets come +// from g_timers (which stamp the same counters); these cover the phases that +// had no timer, plus the dbg/pdb sub-buckets. +typedef enum LNK_SummaryPhase +{ + LNK_SummaryPhase_Input, // lnk_load_inputs (parse/load objs+libs), all rounds + LNK_SummaryPhase_Resolve, // lnk_link_inputs minus contained Input time (lib search + member resolution) + LNK_SummaryPhase_Icf, // lnk_opt_icf + LNK_SummaryPhase_Ref, // lnk_opt_ref + LNK_SummaryPhase_Write, // image write thread (overlaps debug info) + + // dbg umbrella sub-buckets (printed as dbgg[...]) + LNK_SummaryPhase_DbgMcvi, // lnk_make_code_view_input + LNK_SummaryPhase_DbgMerge, // lnk_merge_types + + // pdb sub-buckets (printed as pdbg[...]); brackets sit on the pre-existing + // Prof/timer boundaries inside lnk_build_pdb + the write at its call site + LNK_SummaryPhase_PdbHsh, // lnk_replace_type_names_with_hashes (/RAD_PDB_HASH_TYPE_NAMES): parallel rewrite touching every merged TPI leaf -- storm re-fault amplifier + LNK_SummaryPhase_PdbIni, // lnk_build_pdb task init: pdb_alloc_ (MSF + type-server tables, ~132K fresh commits on the editor link) + LNK_SummaryPhase_PdbGsi, // lnk_move_global_symbols_to_gsi barrier pass ("Move Global Symbols") + LNK_SummaryPhase_PdbSym, // pdb_build_gsi_psi ("Build GSI and PSI": symrec + GSI/PSI hash streams) + LNK_SummaryPhase_PdbMod, // lnk_write_pdb_modules barrier pass ("Write Modules") + LNK_SummaryPhase_PdbTpi, // pdb_type_server_push_parallel TPI+IPI + pdb_type_server_build TPI/IPI + LNK_SummaryPhase_PdbStr, // string tables: cv_dedup_string_tables + offset assign + strtab add ("Merge String Tables"/"Add string tables") + LNK_SummaryPhase_PdbSc, // "Build Section Contrib Map" (per-obj section contribs + DBI section headers) + LNK_SummaryPhase_PdbMsf, // dbi_build + pdb_info_build + msf_build + page-node gather + LNK_SummaryPhase_PdbWr, // PDB file write (lnk_write_data_list_to_file_path in lnk_io) + + LNK_SummaryPhase_Count +} LNK_SummaryPhase; + +internal void lnk_summary_phase_begin(LNK_SummaryPhase phase); +internal void lnk_summary_phase_end(LNK_SummaryPhase phase); + diff --git a/src/linker/thread_pool/thread_pool.c b/src/linker/thread_pool/thread_pool.c index 0eedd1b8a..2482b109e 100644 --- a/src/linker/thread_pool/thread_pool.c +++ b/src/linker/thread_pool/thread_pool.c @@ -60,6 +60,100 @@ tp_worker_main(void *raw_worker) } } +//////////////////////////////////////////////////////////////////////////////// +//~ SHARED-mode governor stats (summary line). Zero cost when the pool is off: +// every call site below is on a shared-mode-only path. + +global TP_SharedStats g_tp_shared_stats; + +internal void +tp_stats_level_add(S64 delta) +{ + // transitions are rare (one per grant/release, thousands per link), so a tiny + // spinlock around the integrator is cheaper than any clever lock-free scheme + for (; ins_atomic_u64_eval_cond_assign(&g_tp_shared_stats.lock, 1, 0) != 0; ) { } + U64 now_us = now_time_us(); + g_tp_shared_stats.area_us += (U64)(g_tp_shared_stats.level * (S64)(now_us - g_tp_shared_stats.last_us)); + g_tp_shared_stats.last_us = now_us; + g_tp_shared_stats.level += delta; + ins_atomic_u64_eval_assign(&g_tp_shared_stats.lock, 0); +} + +internal void +tp_stats_park_add(U64 worker_us) +{ + ins_atomic_u64_add_eval(&g_tp_shared_stats.park_us, worker_us); +} + +internal void +tp_stats_snapshot(F64 *grant_avg_out, F64 *park_seconds_out) +{ + *grant_avg_out = 0; + *park_seconds_out = 0; + if (g_tp_shared_stats.begin_us != 0) { + tp_stats_level_add(0); // finalize the integral up to now + U64 wall_us = g_tp_shared_stats.last_us - g_tp_shared_stats.begin_us; + if (wall_us > 0) { + *grant_avg_out = (F64)g_tp_shared_stats.area_us / (F64)wall_us; + } + *park_seconds_out = (F64)g_tp_shared_stats.park_us / 1000000.0; + } +} + +//////////////////////////////////////////////////////////////////////////////// +//~ SHARED-mode cross-process attach counter (summary line procs=). Counter +// SEMAPHORE, not a named section -- see the ".nproc.v3" comment in +// thread_pool.h for why (UBA virtualizes named sections per-process). + +global Semaphore g_tp_procs_sem; // zero handle = not attached +global U32 g_tp_procs_maxseen; // process-local max of observed n + +internal void +tp_procs_attach(Arena *scratch_arena, String8 name) +{ + String8 sem_name = push_str8f(scratch_arena, "%S.nproc." TP_NPROC_V, name); + Semaphore sem = semaphore_alloc(0, TP_NPROC_MAX, sem_name); // create-or-open, count starts at 0 + if (sem.u64[0] == 0) { + return; // best-effort: no semaphore, procs= prints 0/0 + } + U32 prev = 0; + if (!semaphore_drop_prev(sem, &prev)) { // attach: hold one permit + semaphore_release(sem); + return; + } + g_tp_procs_sem = sem; + g_tp_procs_maxseen = prev + 1; +} + +internal void +tp_procs_snapshot(U32 *attached_out, U32 *maxseen_out) +{ + *attached_out = 0; + *maxseen_out = 0; + if (g_tp_procs_sem.u64[0] != 0) { + U32 prev = 0; + if (semaphore_drop_prev(g_tp_procs_sem, &prev)) { // read: +1 ... + semaphore_take(g_tp_procs_sem, 0); // ... then undo (0-timeout, count > 0 by construction) + // prev = count BEFORE the transient release = #attached, which already + // includes THIS process's attach permit -- no +1 (unlike attach) + U32 n = prev; + g_tp_procs_maxseen = Max(g_tp_procs_maxseen, n); + *attached_out = n; + } + *maxseen_out = g_tp_procs_maxseen; + } +} + +internal void +tp_procs_detach(void) +{ + if (g_tp_procs_sem.u64[0] != 0) { + semaphore_take(g_tp_procs_sem, 0); // give the attach permit back (0-timeout: never block an exit path) + semaphore_release(g_tp_procs_sem); + MemoryZeroStruct(&g_tp_procs_sem); + } +} + //////////////////////////////////////////////////////////////////////////////// //~ SHARED (cross-process governor) path -- OURS. @@ -137,6 +231,7 @@ tp_worker_main_shared(void *raw_worker) if (!barrier_pass) { // path A: hand my budget slot back so another process can use it + tp_stats_level_add(-1); ins_atomic_u64_dec_eval(&pool->granted); semaphore_drop(pool->budget_semaphore); } @@ -175,7 +270,11 @@ tp_governor_main(void *raw_pool) if (want > 0) { // Bounded wait so we re-check pass_active/demand and never block forever // on budget that may never free if the pass ends first. - if (semaphore_take(pool->budget_semaphore, now_time_us() + 1000)) { + U64 wait_begin_us = now_time_us(); + B32 got_slot = semaphore_take(pool->budget_semaphore, wait_begin_us + 1000); + // stats: while we waited here, `want` runnable workers sat parked on budget + tp_stats_park_add((now_time_us() - wait_begin_us) * (U64)want); + if (got_slot) { // Publish the grant (granted++) BEFORE checking pass_active, and ABORT // with granted-- if the pass already ended. This makes main's path-A // drain-spin (waits granted==0) observe any in-flight grant and block @@ -192,6 +291,7 @@ tp_governor_main(void *raw_pool) // all workers parked). granted++ first closes that window. ins_atomic_u64_inc_eval(&pool->granted); if (ins_atomic_u32_eval(&pool->pass_active)) { + tp_stats_level_add(+1); semaphore_drop(pool->wake_semaphore); } else { ins_atomic_u64_dec_eval(&pool->granted); // abort: pass ended @@ -246,7 +346,10 @@ tp_alloc(Arena *arena, U32 worker_count, U32 max_worker_count, String8 name) // does NOT amass the full cohort; it runs at whatever budget is free right // now (best-effort), so multiple processes can run barrier passes // concurrently and none can deadlock waiting to amass the machine. - String8 budget_name = push_str8f(scratch.arena, "%S.budget", name); + // ".v2" LAYOUT-VERSION suffix: see TP_SharedBlock in thread_pool.h -- old + // exes ("%S.budget", no procs section) and new exes must never share + // kernel objects for the same pool name + String8 budget_name = push_str8f(scratch.arena, "%S.budget." TP_SHARED_V, name); pool->budget_semaphore = semaphore_alloc(max_worker_count, max_worker_count, budget_name); // local wake/governor signalling. governor_semaphore is a 0/1 "at least one @@ -279,6 +382,12 @@ tp_alloc(Arena *arena, U32 worker_count, U32 max_worker_count, String8 name) pool->governor_handle = thread_launch(tp_governor_main, pool); } + // stats: start the grant_avg integration window (shared mode only) + if (is_shared) { + g_tp_shared_stats.begin_us = g_tp_shared_stats.last_us = now_time_us(); + tp_procs_attach(scratch.arena, name); // summary line procs= (attach counter + peak watermark) + } + scratch_end(scratch); ProfEnd(); return pool; @@ -530,6 +639,13 @@ tp_barrier_begin(TP_Context *pool) pool->worker_count = cohort; pool->barrier_pass = 1; pool->barrier_depth = 1; + + // stats: cohort slots are held for the whole bracket; the shortfall is the + // budget we wanted but could not grab (parked lanes while this pass runs) + pool->barrier_begin_us = now_time_us(); + pool->barrier_shortfall = want - extra; + if (extra > 0) { tp_stats_level_add((S64)extra); } + return cohort; } @@ -546,6 +662,14 @@ tp_barrier_end(TP_Context *pool) U32 extra = pool->barrier_cohort_extra; + // stats: release the held slots from the integral; account parked lanes + if (extra > 0) { tp_stats_level_add(-(S64)extra); } + if (pool->barrier_shortfall > 0) { + tp_stats_park_add((now_time_us() - pool->barrier_begin_us) * (U64)pool->barrier_shortfall); + } + pool->barrier_begin_us = 0; + pool->barrier_shortfall = 0; + // restore the full-width pool + the original barrier barrier_release(pool->barrier); pool->barrier = pool->barrier_saved; diff --git a/src/linker/thread_pool/thread_pool.h b/src/linker/thread_pool/thread_pool.h index f3a1904e8..143d23d78 100644 --- a/src/linker/thread_pool/thread_pool.h +++ b/src/linker/thread_pool/thread_pool.h @@ -56,6 +56,8 @@ typedef struct TP_Context U32 barrier_saved_workers; // worker_count to restore at tp_barrier_end U32 barrier_cohort_extra; // budget slots held for this barrier pass (cohort = 1 + this) Barrier barrier_saved; // pool->barrier to restore at tp_barrier_end + U64 barrier_begin_us; // stats: bracket open time (for park accounting) + U32 barrier_shortfall; // stats: budget slots we wanted but could not grab for this pass U32 worker_count; TP_Worker *worker_arr; @@ -98,3 +100,55 @@ internal void tp_for_parallel_reserve(TP_Context *pool, TP_Arena *arena, #define tp_for_parallel_reserve_prof(pool, arena, task_count, task_func, task_data, zone_name) ProfBegin(zone_name); tp_for_parallel_reserve(pool, arena, task_count, task_func, task_data); ProfEnd(); internal Rng1U64 * tp_divide_work(Arena *arena, U64 item_count, U32 worker_count); #define tp_broadcast(p) tp_broadcast_(tp, task_id, p, sizeof(*p)) + +// SHARED-mode governor stats (for the end-of-link summary line). All counters +// are only ever touched from shared-mode-only code paths, so the non-shared +// pool pays zero cost. `level` is the count of global budget slots this +// process currently HOLDS (path-A grants + path-B cohort extras); it is +// integrated over time (area_us = sum level x dt, QPC-stamped on every +// grant/release transition) so grant_avg = area/wall. `park_us` accumulates +// worker-microseconds spent waiting on budget while this process had pending +// demand (path A: governor budget waits x wanted-worker count; path B: pass +// duration x cohort shortfall). +typedef struct TP_SharedStats +{ + volatile U64 lock; // spinlock for the {last_us, level, area_us} integrator + U64 begin_us; // pool alloc time (grant_avg denominator start) + U64 last_us; // last transition stamp + S64 level; // budget slots currently held + U64 area_us; // integral of level over time + volatile U64 park_us; // worker-us parked on budget while work was available +} TP_SharedStats; + +internal void tp_stats_level_add(S64 delta); +internal void tp_stats_park_add(U64 worker_us); +internal void tp_stats_snapshot(F64 *grant_avg_out, F64 *park_seconds_out); + +// SHARED-mode cross-process attach counter (summary line: procs=/). +// Lives in a named counter SEMAPHORE ".nproc.v3"; the budget +// semaphore is ".budget.v2". Names carry a version suffix so an old +// radlink pointed at the SAME /RAD_SHARED_THREAD_POOL name never shares kernel +// objects with a new one: mixed old/new farms run independent pools instead of +// corrupting one. +// +// WHY A SEMAPHORE: UBA detours CreateFileMapping and VIRTUALIZES named +// sections per-process, so the previous ".procs.v2" shared-memory +// block reported procs=1/1 under UBA. Named SEMAPHORES pass through the detour +// (the budget semaphore is demonstrably shared in prod: fair grants across +// processes), so the counter is the semaphore's own count: +// attach = release(+1, &prev) -> n = prev+1 (each attached process holds one permit) +// detach = 0-timeout wait (-1) +// read = release(+1, &prev) -> n = prev (prev already counts our own +// permit), then 0-timeout wait to undo +// The transient read +1 can inflate a concurrent reader's n by 1 -- advisory +// counter, acceptable. Peak cannot be tracked exactly cross-process without +// shm, so `maxseen` is this process's local max of n observed at attach and at +// the summary read. Best-effort: a process that dies without detaching leaves +// the count high until the semaphore object itself dies with its last handle. +#define TP_SHARED_V "v2" +#define TP_NPROC_V "v3" +#define TP_NPROC_MAX (1u << 20) // far above any plausible concurrent-link count + +internal void tp_procs_snapshot(U32 *attached_out, U32 *maxseen_out); // 0/0 when no shared pool +internal void tp_procs_detach(void); // give the permit back + close (idempotent) + diff --git a/src/linux/base/linux_base.c b/src/linux/base/linux_base.c index 55a9a2413..842ef5491 100644 --- a/src/linux/base/linux_base.c +++ b/src/linux/base/linux_base.c @@ -718,6 +718,20 @@ semaphore_drop_if_room(Semaphore semaphore) semaphore_drop(semaphore); } +internal B32 +semaphore_drop_prev(Semaphore semaphore, U32 *prev_count_out) +{ + // best-effort: sem_getvalue+sem_post is not atomic (unlike win32 + // ReleaseSemaphore's lpPreviousCount); callers use this for advisory + // counters only + *prev_count_out = 0; + if(semaphore.u64[0] == 0) { return 0; } + int value = 0; + if(sem_getvalue((sem_t*)*semaphore.u64, &value) == 0 && value > 0) { *prev_count_out = (U32)value; } + int err = LNX_RETRY_ON_EINTR(sem_post((sem_t*)*semaphore.u64)); + return err == 0; +} + internal void semaphore_drop_n(Semaphore semaphore, U32 count) { diff --git a/src/win32/base/win32_base.c b/src/win32/base/win32_base.c index ee7cd2bfe..e68b51bb0 100644 --- a/src/win32/base/win32_base.c +++ b/src/win32/base/win32_base.c @@ -698,6 +698,15 @@ semaphore_drop_count(Semaphore semaphore, U64 drop_count) ReleaseSemaphore((HANDLE)*semaphore.u64, drop_count, 0); } +internal B32 +semaphore_drop_prev(Semaphore semaphore, U32 *prev_count_out) +{ + LONG prev = 0; + BOOL ok = ReleaseSemaphore((HANDLE)*semaphore.u64, 1, &prev); + *prev_count_out = ok ? (U32)prev : 0; + return !!ok; +} + // Best-effort post: succeed if there is room, silently no-op if the count is // already at max (ERROR_TOO_MANY_POSTS). Use ONLY for "at least one pending // signal" wakeups (e.g. the governor ping) where redundant posts are harmless. From 65b25cbc6021b836682a7a238bb311ddb0ff3377 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 3 Jul 2026 14:45:53 -0700 Subject: [PATCH 036/121] linker: batch-prefetch mapped debug input, fanned out over the pool -- serial ~0.9 s stall to parallel burst Production (504-link convoy, 126 concurrent, 64 cores): dbg phase burns 682s kernel vs 303s user; mcvi alone traps 42M page faults (~10us each = the kernel time), merge another 12M. Every process first-touches its mapped .debug$S/$T input one 4K fault at a time and the machine goes unresponsive. Batch-populate those ranges with PrefetchVirtualMemory (chunked + coalesced WIN32_MEMORY_RANGE_ENTRY arrays) right before the parse/hash walks: lnk_make_code_view_input prefetches every obj's .debug$S/$T/$P(/$H) section data ahead of the parse loops; lnk_merge_types prefetches the scheduled objs' .debug$T leaf data ahead of the hash tasks. Pure paging hint: output bytes unaffected, resolved via GetProcAddress with silent fallthrough on pre-Win8, already-resident pages cost a no-op. The mcvi prefetch issued ~630 serial PrefetchVirtualMemory calls covering 14 GiB (~0.9 s) plus ~0.5 s in merge, a measurable single-link wall cost (the kernel's per-page population work dominates the syscall overhead). Chunk the coalesced entry array into 256-entry batches and run them as pool tasks; serial fallback when there is no pool or only one batch. Advisory syscalls with no output -- any batch interleaving is fine. Co-Authored-By: Claude Fable 5 (cherry picked from commit 45824bdde728df5f9ae813e2f26ac14865d77fd7) --- src/linker/lnk_debug_info.c | 183 ++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index aaf402c46..3f07829f7 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -52,6 +52,117 @@ lnk_arena_release_thread(void *raw_arena) ProfEnd(); } +//////////////////////////////////////////////////////////////////////////////// +//~ Fault-storm mitigation: batched PrefetchVirtualMemory over mapped input +// ranges. The .debug$S/$T parse and type-merge loops first-touch tens of GB of +// memory-mapped obj sections one 4K page fault at a time; with ~100+ links in +// flight on a build farm those per-page traps saturate the kernel machine-wide +// (prod, 126 concurrent links: dbg phase 682s kernel vs 303s user, 42M faults +// in mcvi alone). PrefetchVirtualMemory populates the ranges in bulk (large MM +// batches, no per-page trap), so issue it over each phase's input ranges right +// before the parse walk. Purely a paging hint: no output byte depends on it, +// failure is silently ignored (pre-Win8 OS / memory pressure), and prefetching +// an already-resident page is a cheap no-op -- so no residency tracking, +// blanket prefetch per phase. + +#if OS_WINDOWS +// declared locally so we do not depend on the SDK's _WIN32_WINNT gate for +// WIN32_MEMORY_RANGE_ENTRY; layout matches memoryapi.h exactly +typedef struct LNK_Win32MemoryRangeEntry +{ + void *VirtualAddress; + SIZE_T NumberOfBytes; +} LNK_Win32MemoryRangeEntry; +typedef BOOL LNK_Win32PrefetchVirtualMemoryFunc(HANDLE process, ULONG_PTR count, LNK_Win32MemoryRangeEntry *ranges, ULONG flags); // WINAPI omitted: x64-only convention + +// entries per task: the kernel's per-page population work dominates the +// syscall overhead, so small batches fanned out over the pool parallelize the +// MM work (14 GiB of mcvi input: ~0.9 s serial -> a wide parallel burst) +#define LNK_PREFETCH_BATCH_SIZE 256 + +typedef struct +{ + LNK_Win32PrefetchVirtualMemoryFunc *proc; + U64 entry_count; + LNK_Win32MemoryRangeEntry *entries; +} LNK_PrefetchTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_prefetch_task) +{ + LNK_PrefetchTask *task = raw_task; + U64 lo = task_id * LNK_PREFETCH_BATCH_SIZE; + U64 hi = Min(lo + LNK_PREFETCH_BATCH_SIZE, task->entry_count); + if (lo < hi) { + task->proc(GetCurrentProcess(), (ULONG_PTR)(hi - lo), task->entries + lo, 0); + } +} +#endif + +internal void +lnk_prefetch_ranges(TP_Context *tp, U64 range_count, Rng1U64 *ranges) +{ +#if OS_WINDOWS + // resolve once (Win8+; on older OS fall through silently). Only called from + // serial phase-setup code, so the local_persist init has no race. + local_persist LNK_Win32PrefetchVirtualMemoryFunc *prefetch_proc = 0; + local_persist B32 prefetch_proc_resolved = 0; + if (!prefetch_proc_resolved) { + prefetch_proc_resolved = 1; + HMODULE kernel32 = GetModuleHandleW(L"kernel32.dll"); + if (kernel32 != 0) { + prefetch_proc = (LNK_Win32PrefetchVirtualMemoryFunc *)GetProcAddress(kernel32, "PrefetchVirtualMemory"); + } + } + if (prefetch_proc == 0 || range_count == 0) { return; } + + Temp scratch = scratch_begin(0,0); + + // Coalesce page-aligned neighbors with a single linear pass: ranges arrive + // obj-by-obj in file-offset order, so adjacent sections of the same mapped + // obj (the common case by far) fold into one entry. No sort -- the API does + // not require ordered or disjoint ranges, overlap just costs a cheap re-walk. + LNK_Win32MemoryRangeEntry *entries = push_array_no_zero(scratch.arena, LNK_Win32MemoryRangeEntry, range_count); + U64 entry_count = 0; + U64 pending_min = 0, pending_max = 0; + for EachIndex(range_idx, range_count) { + if (ranges[range_idx].min >= ranges[range_idx].max) { continue; } + U64 min = AlignDownPow2(ranges[range_idx].min, KB(4)); + U64 max = AlignPow2 (ranges[range_idx].max, KB(4)); + if (pending_max != 0 && min <= pending_max && max >= pending_min) { + pending_min = Min(pending_min, min); + pending_max = Max(pending_max, max); + continue; + } + if (pending_max != 0) { + entries[entry_count].VirtualAddress = (void *)pending_min; + entries[entry_count].NumberOfBytes = (SIZE_T)(pending_max - pending_min); + entry_count += 1; + } + pending_min = min; + pending_max = max; + } + if (pending_max != 0) { + entries[entry_count].VirtualAddress = (void *)pending_min; + entries[entry_count].NumberOfBytes = (SIZE_T)(pending_max - pending_min); + entry_count += 1; + } + + // fan the batches out over the pool: population is per-page kernel work, so + // this turns a serial ~1 s stall into a wide parallel burst. Purely advisory + // syscalls with no output -- any batch interleaving is fine. + LNK_PrefetchTask task = { .proc = prefetch_proc, .entry_count = entry_count, .entries = entries }; + U64 batch_count = CeilIntegerDiv(entry_count, LNK_PREFETCH_BATCH_SIZE); + if (tp != 0 && batch_count > 1) { + tp_for_parallel(tp, 0, batch_count, lnk_prefetch_task, &task); + } else { + for EachIndex(batch_idx, batch_count) { lnk_prefetch_task(0, 0, batch_idx, &task, 0); } + } + + scratch_end(scratch); +#endif +} + internal void lnk_discard_cv_debug_info(LNK_CodeViewInput *input, U64 obj_idx) { @@ -1549,6 +1660,50 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, input.debug_s_list_arr = lnk_collect_obj_sections(tp, tp_arena, obj_count, obj_arr, str8_lit(".debug$S"), 0); ProfEnd(); + // batch-populate the mapped .debug$S/$T/$P/$H input ranges before the parse + // loops below first-touch them page by page (see lnk_prefetch_ranges) + ProfScope("Prefetch CodeView") + { + Temp temp = temp_begin(scratch.arena); + + U64 range_cap = 3 * obj_count; // debug$T + debug$P + debug$H + for EachIndex(obj_idx, obj_count) { range_cap += input.debug_s_list_arr[obj_idx].node_count; } + + Rng1U64 *ranges = push_array_no_zero(temp.arena, Rng1U64, range_cap); + U64 range_count = 0; + for EachIndex(obj_idx, obj_count) { + LNK_Obj *obj = obj_arr[obj_idx]; + + for EachNode(n, String8Node, input.debug_s_list_arr[obj_idx].first) { + if (n->string.size) { ranges[range_count++] = rng_1u64((U64)n->string.str, (U64)n->string.str + n->string.size); } + } + if (obj->debug_t_sect_idx < obj->header.section_count_no_null) { + LNK_ObjSection sect = lnk_obj_section_from_sect_idx(obj, obj->debug_t_sect_idx); + String8 data = lnk_obj_get_sect_data(obj, obj->debug_t_sect_idx, sect.frange); + if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } + } + if (obj->debug_p_sect_idx < obj->header.section_count_no_null) { + LNK_ObjSection sect = lnk_obj_section_from_sect_idx(obj, obj->debug_p_sect_idx); + String8 data = lnk_obj_get_sect_data(obj, obj->debug_p_sect_idx, sect.frange); + if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } + } + if (config->ghash && obj->debug_h_sect_idx < obj->header.section_count_no_null) { + LNK_ObjSection sect = lnk_obj_section_from_sect_idx(obj, obj->debug_h_sect_idx); + String8 data = lnk_obj_get_sect_data(obj, obj->debug_h_sect_idx, sect.frange); + if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } + } + } + Assert(range_count <= range_cap); + U64 prefetch_begin_us = now_time_us(); + U64 prefetch_bytes = 0; + for EachIndex(range_idx, range_count) { prefetch_bytes += dim_1u64(ranges[range_idx]); } + lnk_prefetch_ranges(tp, range_count, ranges); + lnk_log(LNK_Log_Timers, "[mcvi] prefetched %llu debug section ranges (%llu MiB) in %.2f ms", + range_count, prefetch_bytes / MB(1), (F64)(now_time_us() - prefetch_begin_us) / 1000.0); + + temp_end(temp); + } + // profiler info if (lnk_get_log_status(LNK_Log_Debug) || PROFILE_TELEMETRY) { U64 total_debug_s_size = 0, total_debug_t_size = 0, total_debug_p_size = 0, total_debug_h_size = 0; @@ -2848,6 +3003,34 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } ProfEnd(); + // batch-populate the .debug$T/$P leaf data the hashers below walk leaf by + // leaf; under farm-wide memory pressure these mapped pages were trimmed + // since the parse phase touched them (see lnk_prefetch_ranges) + ProfScope("Prefetch Type Data") + { + Temp temp = temp_begin(scratch.arena); + + U64 range_cap = 0; + for EachElement(i, hash_targets) { range_cap += hash_targets[i].hash_indices.count; } + + Rng1U64 *ranges = push_array_no_zero(temp.arena, Rng1U64, range_cap); + U64 range_count = 0; + for EachElement(i, hash_targets) { + for EachIndex(k, hash_targets[i].hash_indices.count) { + String8 data = input->debug_t_arr[hash_targets[i].hash_indices.v[k]].data; + if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } + } + } + U64 prefetch_begin_us = now_time_us(); + U64 prefetch_bytes = 0; + for EachIndex(range_idx, range_count) { prefetch_bytes += dim_1u64(ranges[range_idx]); } + lnk_prefetch_ranges(tp, range_count, ranges); + lnk_log(LNK_Log_Timers, "[merge] prefetched %llu type data ranges (%llu MiB) in %.2f ms", + range_count, prefetch_bytes / MB(1), (F64)(now_time_us() - prefetch_begin_us) / 1000.0); + + temp_end(temp); + } + for EachElement(i, hash_targets) { task.indices = hash_targets[i].hash_indices; ProfBegin("Hash [Count: %.*s]", str8_varg(str8_from_count(scratch.arena, task.indices.count))); From c26064fb5f31db74604d3c5fb7c5a9fa9b21133f Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 6 Jul 2026 08:52:05 -0700 Subject: [PATCH 037/121] linker: map input file views read-only -- commit charge per dirtied page, not per view Input files were mapped FILE_MAP_COPY, which charges pagefile commit for the ENTIRE view at map time even for pages never written: 22.4 GiB of a 49.7 GiB peak commit on a large editor DLL link, and N concurrent links multiply it (4-overlap: 205 GB aggregate commit demand), feeding build-farm memory admission for pages that are 99.9% never dirtied. Views now map FILE_MAP_READ from the same PAGE_WRITECOPY section (zero commit at map time). The few writers that still patch input bytes in place go through two mechanisms: - lnk_cow_promote_range: one VirtualProtect(PAGE_WRITECOPY) over a known hot range (used by the obj section-header patch tasks; ~20 pages per obj, ~240K pages per big link -- per-page faults here cost ~25s kernel) - lnk_cow_page_promote_veh: vectored-exception fallback that promotes single faulting pages (IFC 0x1522 pokes, LF_ENDPRECOMP removal, debug$S TI fixups; ~4K pages per big link), so any in-place writer stays correct without per-site plumbing Also stop dirtying archive pages in lnk_lib_from_data: first-linker-member big-endian offsets are now converted on a private copy instead of in place. Write semantics are unchanged from FILE_MAP_COPY: first write makes the page private, input files are never modified. READ_WRITE and no-map modes are unaffected (their pages never write-fault and their sections refuse PAGE_WRITECOPY). Same classic Win32 APIs as before, so UBA detours see nothing new. Big editor DLL link: peak commit 50.9 -> 28.8 GB (-22.1 GB), wall/user/kernel parity, outputs byte-identical. 4-concurrent: aggregate peak commit 205 -> 116 GB. Co-Authored-By: Claude Fable 5 (cherry picked from commit 4b247bb4cc9f2d8a4669855915e98624287a7358) --- src/linker/lnk_io.c | 51 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk_io.c b/src/linker/lnk_io.c index e5318d09c..f16b984d0 100644 --- a/src/linker/lnk_io.c +++ b/src/linker/lnk_io.c @@ -217,6 +217,50 @@ THREAD_POOL_TASK_FUNC(lnk_data_from_file_path_task) } } +#if OS_WINDOWS +// Input views are mapped from PAGE_WRITECOPY sections with FILE_MAP_READ access, so an +// untouched view carries no pagefile commit charge. Mapping with FILE_MAP_COPY instead +// would charge commit for the ENTIRE view at map time -- even for pages never written -- +// so N concurrent big links would hold input-set-sized commit for their whole runtime +// (measured 22.4 GiB of a 49.7 GiB peak on a large editor DLL link) and feed build-farm +// memory admission limits for no benefit. +// +// The few remaining writers that patch input bytes in place (IFC 0x1522 LF_IFC_RECORD +// NOTYPE pokes, LF_ENDPRECOMP removal in debug$P; a couple of bytes per page, a handful +// of pages per link) hit this vectored handler, which promotes JUST the faulting page to +// PAGE_WRITECOPY and retries. Commit is then charged per dirtied page instead of per +// view. Write semantics are identical to the old FILE_MAP_COPY mapping: the first write +// makes the page private, the input file is never modified. Pages of ordinary allocations +// or of read-write mappings never reach the handler (they do not fault on write), so +// /RAD_MEMORY_MAP_FILES:READ_WRITE and no-map modes are unaffected. +global volatile LONG g_lnk_cow_veh_installed; +global volatile LONG g_lnk_cow_promoted_pages; + +internal LONG NTAPI +lnk_cow_page_promote_veh(EXCEPTION_POINTERS *info) +{ + EXCEPTION_RECORD *er = info->ExceptionRecord; + if (er->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && er->NumberParameters >= 2 && er->ExceptionInformation[0] == 1) { + void *addr = (void *)er->ExceptionInformation[1]; + MEMORY_BASIC_INFORMATION mbi = {0}; + if (VirtualQuery(addr, &mbi, sizeof(mbi)) >= sizeof(mbi) && mbi.Type == MEM_MAPPED) { + if (mbi.Protect == PAGE_READONLY) { + void *page = (void *)((UINT_PTR)addr & ~(UINT_PTR)(KB(4) - 1)); + DWORD old_protect = 0; + if (VirtualProtect(page, KB(4), PAGE_WRITECOPY, &old_protect)) { + InterlockedIncrement(&g_lnk_cow_promoted_pages); + return EXCEPTION_CONTINUE_EXECUTION; + } + } else if (mbi.Protect == PAGE_WRITECOPY || mbi.Protect == PAGE_READWRITE) { + // another thread promoted this page between our fault and the query; retry the write + return EXCEPTION_CONTINUE_EXECUTION; + } + } + } + return EXCEPTION_CONTINUE_SEARCH; +} +#endif + internal THREAD_POOL_TASK_FUNC(lnk_memory_map_file_task) { @@ -254,11 +298,16 @@ THREAD_POOL_TASK_FUNC(lnk_memory_map_file_task) } else { HANDLE file_handle = CreateFileW(path16.str, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); if (file_handle != INVALID_HANDLE_VALUE) { + if (InterlockedCompareExchange(&g_lnk_cow_veh_installed, 1, 0) == 0) { + AddVectoredExceptionHandler(1, lnk_cow_page_promote_veh); + } HANDLE mapping_handle = CreateFileMappingA(file_handle, 0, PAGE_WRITECOPY, 0, 0, 0); if (mapping_handle != INVALID_HANDLE_VALUE) { LARGE_INTEGER file_size = {0}; GetFileSizeEx(file_handle, &file_size); - void *file_data = MapViewOfFile(mapping_handle, FILE_MAP_COPY, 0, 0, file_size.QuadPart); + // FILE_MAP_READ view of a WRITECOPY section: zero commit charge at map time; + // pages become writable one at a time through lnk_cow_page_promote_veh + void *file_data = MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_size.QuadPart); if (file_data) { task->data_arr.v[task_id] = str8(file_data, file_size.QuadPart); if (task->was_read) { From 2f0435a0a64c4ec413892bc468aedf77cb5ea3db Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 9 Jul 2026 10:08:29 -0700 Subject: [PATCH 038/121] linker: /OPT:ICF -- folded functions keep bindable debug info (Lines at the leader RVA; full record tree for different-source folds with locals) Record external folds in the per-section fold map (is_extern, excluded from /OPT:REF redirects), redirect dead followers' sect_map to the leader contrib, and keep each folded function's associated .debug$S: by default Lines-only (reloc-patched to the leader RVA -- source breakpoints on folded bodies bind; symbol records stay dropped, the bulk of link.exe's module-stream cost for the same feature), escalated to the FULL record tree when the fold joins a different source location (identity = FILECHKSMS content hash + first line of the Lines fragment, compared follower-vs-leader at mark time, O(1) per fold) and the tree has locals -- the watch window then labels folded frames with the right source's variable names, like link.exe. Measured on the FN editor DLL: ~6.5% of folds differ in source, mostly empty virtuals; gate DLL cost +2.6% PDB (mod_sym +48.8MB), lines ~1.5%. Co-Authored-By: Claude Fable 5 --- src/linker/lnk.c | 223 +++++++++++++++++++++++++++++++++++- src/linker/lnk.h | 4 +- src/linker/lnk_debug_info.c | 40 +++++-- src/linker/lnk_obj.h | 21 +++- 4 files changed, 272 insertions(+), 16 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index fc0199c9c..f5803c864 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -2479,9 +2479,17 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer if (config->opt_icf == LNK_SwitchState_Yes) { if (objs == 0) { objs = lnk_array_from_obj_list(scratch.arena, link->objs); } lnk_summary_phase_begin(LNK_SummaryPhase_Icf); - lnk_opt_icf(tp, symtab, config, objs, link->objs.count); + lnk_opt_icf(tp, arena->v[0], symtab, config, objs, link->objs.count); lnk_summary_phase_end(LNK_SummaryPhase_Icf); } + + // + // keep line info for ICF-folded functions (bound to the leader RVA) -- see task comment + // + if (config->opt_icf == LNK_SwitchState_Yes && config->opt_ref == LNK_SwitchState_Yes) { + if (objs == 0) { objs = lnk_array_from_obj_list(scratch.arena, link->objs); } + lnk_icf_mark_folded_lines(tp, arena, objs, link->objs.count); + } } // @@ -3845,6 +3853,13 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) temp_end(assoc_temp); } + // record the fold for debug aliasing (lnk_icf_mark_folded_lines): unlike the symlink + // redirect, this distinguishes an ICF fold (different-named section joined to a leader) + // from same-name COMDAT selection and /OPT:REF removal + if (contrib_obj->icf_fold) { + contrib_obj->icf_fold[contrib->section_number] = (LNK_ICFFold){ .leader_obj_idx = (U32)leader->obj_idx, .leader_sn = leader->section_number, .set = 1 }; + } + #if LNK_PARANOID String8 section_name = lnk_obj_section_name_from_section_number(contrib_obj, contrib->section_number); String8 leader_name = lnk_obj_section_name_from_section_number(leader_obj, leader->section_number); @@ -3948,13 +3963,23 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) } internal void -lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count) +lnk_opt_icf(TP_Context *tp, Arena *perm, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count) { ProfBegin("/OPT:ICF"); - Temp scratch = scratch_begin(0,0); - + Temp scratch = scratch_begin(&perm, 1); + lnk_log(LNK_Log_Debug, "/OPT:ICF:"); + // per-section fold map, consumed by the debug-aliasing pass after /OPT:REF + // (lnk_icf_mark_folded_lines); allocated only when that pass will run + if (config->opt_ref == LNK_SwitchState_Yes) { + ProfScope("Alloc fold maps") { + for EachIndex(obj_idx, objs_count) { + objs[obj_idx]->icf_fold = push_array(perm, LNK_ICFFold, objs[obj_idx]->coff.sections.count_no_null + 1); + } + } + } + // BARRIER pass (path B): the task synchronizes with barrier_wait(tp->barrier)/tp_broadcast, // so under /RAD_SHARED_THREAD_POOL it must run at a pinned cohort via the reserve path -- // a plain tp_for_parallel admits workers incrementally and the barrier never fills (deadlock). @@ -4011,6 +4036,190 @@ lnk_should_gather_section(LNK_Obj *obj, U64 section_number, COFF_SectionHeader * return 1; } +typedef struct +{ + LNK_Obj **objs; // indexed by input_idx (== task_id) +} LNK_ICFMarkFoldedLinesTask; + +// Find the COMDAT-associative .debug$S child that carries a function section's +// per-function CodeView records. +internal U32 +lnk_icf_debug_s_child_from_section(LNK_Obj *obj, U32 fn_sn) +{ + if (fn_sn == 0 || fn_sn > obj->coff.sections.count_no_null) { return 0; } + for EachNode(assoc_n, U32Node, obj->coff.sections.associated_section_numbers[fn_sn]) { + U32 sn = assoc_n->data; + if (sn == 0 || sn > obj->coff.sections.count_no_null) { continue; } + if (~obj->coff.sections.headers[sn].flags & LNK_SECTION_FLAG_DEBUG) { continue; } + if (str8_match(lnk_obj_section_name_from_section_number(obj, sn), str8_lit(".debug$S"), 0)) { return sn; } + } + return 0; +} + + +// FILECHKSMS of the obj-wide (non-COMDAT) .debug$S -- the table every per-function +// Lines fragment's file_off indexes into +internal String8 +lnk_icf_obj_file_chksms(Arena *scratch, LNK_Obj *obj) +{ + for LNK_EachCoffSection(it, obj) { + COFF_SectionFlags flags = *it.v.flags; + if (~flags & LNK_SECTION_FLAG_DEBUG) { continue; } + if ( flags & COFF_SectionFlag_LnkCOMDAT) { continue; } + if (!str8_match(lnk_obj_section_name_from_section_number(obj, it.v.section_number), str8_lit(".debug$S"), 0)) { continue; } + String8 raw = lnk_obj_section_data_from_number(obj, it.v.section_number); + CV_DebugS ds = cv_debug_s_from_data(scratch, raw); + String8List ck = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_FileChksms); + if (ck.node_count) { return ck.first->string; } + } + return str8_zero(); +} + +// source identity of a function: (checksum of its file, first line). Two ICF fold members +// with equal keys are the same source text (template twins) -- their locals/labels are +// identical and the leader's record tree serves both. +typedef struct +{ + B32 valid; + U32 line; + U8 chksum_kind; + String8 chksum; +} LNK_ICFSrcKey; + +internal LNK_ICFSrcKey +lnk_icf_src_key_from_fn(Arena *scratch, LNK_Obj *obj, U32 fn_sn, String8 chksms) +{ + LNK_ICFSrcKey key = {0}; + U32 child_sn = lnk_icf_debug_s_child_from_section(obj, fn_sn); + if (child_sn == 0 || chksms.size == 0) { return key; } + String8 raw = lnk_obj_section_data_from_number(obj, child_sn); + CV_DebugS ds = cv_debug_s_from_data(scratch, raw); + String8List lines = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Lines); + if (lines.node_count == 0) { return key; } + String8 frag = lines.first->string; + if (frag.size < sizeof(CV_C13SubSecLinesHeader) + sizeof(CV_C13File) + sizeof(CV_C13Line)) { return key; } + CV_C13File *file = (CV_C13File *)(frag.str + sizeof(CV_C13SubSecLinesHeader)); + CV_C13Line *l0 = (CV_C13Line *)((U8 *)file + sizeof(CV_C13File)); + if ((U64)file->file_off + sizeof(CV_C13Checksum) > chksms.size) { return key; } + CV_C13Checksum *ck = (CV_C13Checksum *)(chksms.str + file->file_off); + if ((U64)file->file_off + sizeof(CV_C13Checksum) + ck->len > chksms.size) { return key; } + key.valid = 1; + key.line = (U32)(l0->flags & 0xFFFFFF); + key.chksum_kind = ck->kind; + key.chksum = str8(chksms.str + file->file_off + sizeof(CV_C13Checksum), ck->len); + return key; +} + +// does the function's record tree have anything a watch window would show? +internal B32 +lnk_icf_debug_s_has_locals(Arena *scratch, LNK_Obj *obj, U32 child_sn) +{ + String8 raw = lnk_obj_section_data_from_number(obj, child_sn); + CV_DebugS ds = cv_debug_s_from_data(scratch, raw); + String8List syms = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Symbols); + for EachNode(n, String8Node, syms.first) { + String8 s = n->string; + for (U64 o = 0; o + 4 <= s.size; ) { + U16 len, kind; + MemoryCopy(&len, s.str + o, sizeof(len)); + MemoryCopy(&kind, s.str + o + 2, sizeof(kind)); + if (len < 2) { break; } + if (kind == CV_SymKind_LOCAL || kind == CV_SymKind_REGREL32) { return 1; } + o += len + 2; + } + } + return 0; +} + +// /OPT:ICF folded-function debug-info slimming. Without this pass a folded function's associated +// .debug$S stays collected in full (REF marked it live with its then-live parent; the ICF fold +// removes only the .text follower), so the module stream receives every folded body's WHOLE record +// tree (S_GPROC/locals + Lines) bound to the leader RVA -- link.exe-parity content at ~+30% module +// bytes. The C13 Lines subsections are ~1% of that and are all a source breakpoint needs to bind. +// So mark each folded follower's associated .debug$S LnkRemove (drops it from full collection); +// the reloc patcher still patches it and the C13 pass merges back ONLY its Lines -- their +// SECREL/SECTION relocs target the folded function symbol, which resolves to the leader RVA +// through the redirected symlink/sect_map, so the lines land on the surviving body. +internal +THREAD_POOL_TASK_FUNC(lnk_icf_mark_folded_lines_task) +{ + Temp scratch = scratch_begin(&arena, 1); + + LNK_ICFMarkFoldedLinesTask *task = raw_task; + LNK_Obj *obj = task->objs[task_id]; + if (obj->icf_fold == 0) { scratch_end(scratch); return; } + + String8 follower_chksms = str8_zero(); + B32 follower_chksms_init = 0; + LNK_Obj *cached_leader = 0; + String8 leader_chksms = str8_zero(); + + for (U32 section_number = 1; section_number <= obj->coff.sections.count_no_null; section_number += 1) { + LNK_ICFFold fold = obj->icf_fold[section_number]; + if (!fold.set) { continue; } + if (~obj->coff.sections.headers[section_number].flags & COFF_SectionFlag_LnkRemove) { continue; } // follower kept live -> its own records emit + LNK_Obj *leader_obj = task->objs[fold.leader_obj_idx]; + if (leader_obj->coff.sections.headers[fold.leader_sn].flags & COFF_SectionFlag_LnkRemove) { continue; } // whole class dead-stripped + + // Lines-only by default. Escalate to the FULL record tree (link.exe parity for this one + // fold) when the follower comes from a DIFFERENT source location than the leader (else the + // trees are textually identical -- template twins) AND it has locals to show. Measured on + // the FN editor DLL: ~6.5% of folds differ in source, most of those are empty virtuals, so + // the escalation set is small. + U8 mark = 1; + { + Temp fold_temp = temp_begin(scratch.arena); + U32 child_sn = lnk_icf_debug_s_child_from_section(obj, section_number); + if (child_sn != 0) { + // the returned String8s slice the objs' section data (only the parse's list nodes live + // on the temp arena), so caching them across fold_temp scopes is safe + if (!follower_chksms_init) { + follower_chksms = lnk_icf_obj_file_chksms(fold_temp.arena, obj); + follower_chksms_init = 1; + } + if (leader_obj != cached_leader) { + cached_leader = leader_obj; + leader_chksms = lnk_icf_obj_file_chksms(fold_temp.arena, leader_obj); + } + LNK_ICFSrcKey fk = lnk_icf_src_key_from_fn(fold_temp.arena, obj, section_number, follower_chksms); + LNK_ICFSrcKey lk = lnk_icf_src_key_from_fn(fold_temp.arena, leader_obj, fold.leader_sn, leader_chksms); + B32 same_src = fk.valid && lk.valid && + fk.line == lk.line && fk.chksum_kind == lk.chksum_kind && + str8_match(fk.chksum, lk.chksum, 0); + if (fk.valid && lk.valid && !same_src && lnk_icf_debug_s_has_locals(fold_temp.arena, obj, child_sn)) { + mark = 2; + } + } + temp_end(fold_temp); + } + + for EachNode(assoc_n, U32Node, obj->coff.sections.associated_section_numbers[section_number]) { + U32 assoc_sn = assoc_n->data; + if (assoc_sn == 0 || assoc_sn > obj->coff.sections.count_no_null) { continue; } + if (~obj->coff.sections.headers[assoc_sn].flags & LNK_SECTION_FLAG_DEBUG) { continue; } + // exclude the follower's .debug$S from full module collection (it would otherwise merge + // its whole record tree at the leader RVA, link.exe-parity size); the consumers below + // merge back just its Lines (mark 1) or, rarely, the whole tree (mark 2) + obj->coff.sections.headers[assoc_sn].flags |= COFF_SectionFlag_LnkRemove; + if (obj->icf_lines_only == 0) { + obj->icf_lines_only = push_array(arena, B8, obj->coff.sections.count_no_null + 1); + } + obj->icf_lines_only[assoc_sn] = mark; + } + } + + scratch_end(scratch); +} + +internal void +lnk_icf_mark_folded_lines(TP_Context *tp, TP_Arena *arena, LNK_Obj **objs, U64 objs_count) +{ + ProfBeginFunction(); + LNK_ICFMarkFoldedLinesTask task = { .objs = objs }; + tp_for_parallel(tp, arena, objs_count, lnk_icf_mark_folded_lines_task, &task); // arena: per-obj icf_lines_only bitmaps + ProfEnd(); +} + internal THREAD_POOL_TASK_FUNC(lnk_gather_sections_task) { @@ -4742,7 +4951,11 @@ THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) COFF_SectionFlags section_flags = *it.v.flags; if (section_flags & COFF_SectionFlag_LnkInfo) { continue; } - if (section_flags & COFF_SectionFlag_LnkRemove) { continue; } + if (section_flags & COFF_SectionFlag_LnkRemove) { + // exception: ICF-folded functions' .debug$S stays dead but its Lines are merged into the + // module bound to the leader RVA -- patch it so those Lines carry real addresses + if (!(obj->icf_lines_only && obj->icf_lines_only[it.v.section_number])) { continue; } + } if (section_flags & COFF_SectionFlag_CntUninitializedData) { continue; } COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(obj, section_header); diff --git a/src/linker/lnk.h b/src/linker/lnk.h index 556693f5f..95eb56614 100644 --- a/src/linker/lnk.h +++ b/src/linker/lnk.h @@ -396,7 +396,9 @@ internal LNK_LinkResult lnk_link_image (TP_Context *tp, TP_Arena *arena, LNK_Con // --- Optimizations ----------------------------------------------------------- internal void lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count); -internal void lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count); +internal void lnk_opt_icf(TP_Context *tp, Arena *perm, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count); +internal U32 lnk_icf_debug_s_child_from_section(LNK_Obj *obj, U32 fn_sn); +internal void lnk_icf_mark_folded_lines(TP_Context *tp, TP_Arena *arena, LNK_Obj **objs, U64 objs_count); // --- Win32 Image ------------------------------------------------------------- diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 3f07829f7..af79374fb 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -206,6 +206,31 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_s_task) lnk_error_obj(LNK_Warning_IllData, task->obj_arr[obj_idx], ".debug$S has %u file checksum sub-sections defined, picking first sub-section", checksum_data_list.node_count); } } + + // ICF-folded functions' associated .debug$S (dead-stripped, excluded from the list above): + // merge ONLY their Lines subsections. The reloc patcher patched them to the fold leader's RVA, + // so source breakpoints on folded bodies bind; symbol records stay dropped (that is the bulk + // of link.exe's size cost for the same feature). File ids in these Lines index the obj-wide + // FILECHKSMS merged above, so they stay consistent within this module. Mark 2 (fold joins a + // different source location and has locals) keeps the WHOLE record tree instead, so the watch + // window labels a folded frame with that source's own variable names. + { + LNK_Obj *obj = task->obj_arr[obj_idx]; + if (obj->icf_lines_only != 0) { + for (U32 section_number = 1; section_number <= obj->coff.sections.count_no_null; section_number += 1) { + if (!obj->icf_lines_only[section_number]) { continue; } + String8 raw_data = lnk_obj_section_data_from_number(obj, section_number); + CV_DebugS ds = cv_debug_s_from_data(arena, raw_data); + if (obj->icf_lines_only[section_number] == 2) { + cv_debug_s_concat_in_place(debug_s, &ds); + } else { + String8List lines = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Lines); + String8List *dst = cv_sub_section_ptr_from_debug_s(debug_s, CV_C13SubSectionKind_Lines); + str8_list_concat_in_place(dst, &lines); + } + } + } + } } internal int @@ -1677,19 +1702,16 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, for EachNode(n, String8Node, input.debug_s_list_arr[obj_idx].first) { if (n->string.size) { ranges[range_count++] = rng_1u64((U64)n->string.str, (U64)n->string.str + n->string.size); } } - if (obj->debug_t_sect_idx < obj->header.section_count_no_null) { - LNK_ObjSection sect = lnk_obj_section_from_sect_idx(obj, obj->debug_t_sect_idx); - String8 data = lnk_obj_get_sect_data(obj, obj->debug_t_sect_idx, sect.frange); + if (obj->coff.debug_t_section_number) { + String8 data = lnk_obj_section_data_from_number(obj, obj->coff.debug_t_section_number); if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } } - if (obj->debug_p_sect_idx < obj->header.section_count_no_null) { - LNK_ObjSection sect = lnk_obj_section_from_sect_idx(obj, obj->debug_p_sect_idx); - String8 data = lnk_obj_get_sect_data(obj, obj->debug_p_sect_idx, sect.frange); + if (obj->coff.debug_p_section_number) { + String8 data = lnk_obj_section_data_from_number(obj, obj->coff.debug_p_section_number); if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } } - if (config->ghash && obj->debug_h_sect_idx < obj->header.section_count_no_null) { - LNK_ObjSection sect = lnk_obj_section_from_sect_idx(obj, obj->debug_h_sect_idx); - String8 data = lnk_obj_get_sect_data(obj, obj->debug_h_sect_idx, sect.frange); + if (config->ghash && obj->coff.debug_h_section_number) { + String8 data = lnk_obj_section_data_from_number(obj, obj->coff.debug_h_section_number); if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } } } diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index 8d9519e42..34d46eb76 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -41,6 +41,17 @@ typedef struct LNK_ObjCoff B8 hotpatch; } LNK_ObjCoff; +// /OPT:ICF fold record (one per section number; slot zero is null), filled at fold-apply. +// Distinguishes ICF folds from same-name COMDAT selection and /OPT:REF removal (all three end +// up LnkRemove'd with a redirected symlink, but only ICF folds join DIFFERENT-named sections, +// which is what the debug-info aliasing below needs to know). set==0 means not ICF-folded. +typedef struct LNK_ICFFold +{ + U32 leader_obj_idx; // input_idx of the leader's obj + U32 leader_sn; // leader section number + B8 set; +} LNK_ICFFold; + typedef struct LNK_Obj { String8 path; @@ -53,7 +64,15 @@ typedef struct LNK_Obj U32 input_idx; // link state - LNK_ObjSymbolRef *symlinks; // indexed by COFF section number; slot zero is null + LNK_ObjSymbolRef *symlinks; // indexed by COFF section number; slot zero is null + LNK_ICFFold *icf_fold; // /OPT:ICF fold map (per section number); 0 if ICF off + B8 *icf_lines_only; // .debug$S sections associated to an ICF-folded function: stay + // LnkRemove'd, but merge into the module remapped to the leader RVA + // (section-number indexed; slot zero is null; 0 array ptr when + // ICF is off or no folds exist). + // 1 = C13 Lines only (source breakpoints bind); 2 = full record + // tree (fold joins a DIFFERENT source location and has locals -- + // watch-window labels come from the right source) // link struct LNK_LibMemberRef *link_member; From 5aae2d5b677c7a124ad27cde97004551f1e0858f Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Sun, 19 Jul 2026 13:23:56 -0700 Subject: [PATCH 039/121] linker: memoize ICF FILECHKSMS lookups Folded-debug source matching compares per-function line records against the object-wide FILECHKSMS subsection. Scan that subsection directly and memoize its immutable slice per object, so shared leaders are not reparsed for every follower object. The cache uses an idempotent atomic publication and does not change ICF selection or output bytes. Co-Authored-By: Claude Fable 5 --- src/linker/lnk.c | 53 ++++++++++++++++++++++++++------------------ src/linker/lnk_obj.h | 2 ++ 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index f5803c864..6819613c5 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -4058,23 +4058,47 @@ lnk_icf_debug_s_child_from_section(LNK_Obj *obj, U32 fn_sn) // FILECHKSMS of the obj-wide (non-COMDAT) .debug$S -- the table every per-function -// Lines fragment's file_off indexes into +// Lines fragment's file_off indexes into. Direct header walk with early-out instead of +// cv_debug_s_from_data: the obj-wide .debug$S is megabytes of subsections and the full +// parse pushes a list node per subsection; here we only need one slice. internal String8 -lnk_icf_obj_file_chksms(Arena *scratch, LNK_Obj *obj) +lnk_icf_obj_file_chksms_scan(LNK_Obj *obj) { for LNK_EachCoffSection(it, obj) { COFF_SectionFlags flags = *it.v.flags; if (~flags & LNK_SECTION_FLAG_DEBUG) { continue; } if ( flags & COFF_SectionFlag_LnkCOMDAT) { continue; } if (!str8_match(lnk_obj_section_name_from_section_number(obj, it.v.section_number), str8_lit(".debug$S"), 0)) { continue; } - String8 raw = lnk_obj_section_data_from_number(obj, it.v.section_number); - CV_DebugS ds = cv_debug_s_from_data(scratch, raw); - String8List ck = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_FileChksms); - if (ck.node_count) { return ck.first->string; } + String8 raw = lnk_obj_section_data_from_number(obj, it.v.section_number); + if (raw.size < sizeof(CV_Signature) || cv_signature_from_debug_s(raw) != CV_Signature_C13) { continue; } + for (U64 cursor = sizeof(CV_Signature); cursor + sizeof(CV_C13SubSectionHeader) <= raw.size; ) { + CV_C13SubSectionHeader header = {0}; + cursor += str8_deserial_read_struct(raw, cursor, &header); + if (header.kind == CV_C13SubSectionKind_FileChksms) { + return str8_substr(raw, r1u64(cursor, cursor + header.size)); + } + cursor += header.size; + cursor = AlignPow2(cursor, CV_C13SubSectionAlign); + } } return str8_zero(); } +// Memoized per obj: leaders are shared across many follower objs, so without the memo the +// scan reruns once per (follower obj x leader switch). The result slices the immutable +// obj->data mapping, so the racy fill is idempotent (every worker writes identical bytes); +// the init flag is published last. +internal String8 +lnk_icf_obj_file_chksms(LNK_Obj *obj) +{ + if (!ins_atomic_u32_eval((U32 *)&obj->icf_file_chksms_init)) { + String8 chksms = lnk_icf_obj_file_chksms_scan(obj); + obj->icf_file_chksms = chksms; + ins_atomic_u32_eval_assign((U32 *)&obj->icf_file_chksms_init, 1); + } + return obj->icf_file_chksms; +} + // source identity of a function: (checksum of its file, first line). Two ICF fold members // with equal keys are the same source text (template twins) -- their locals/labels are // identical and the leader's record tree serves both. @@ -4149,11 +4173,6 @@ THREAD_POOL_TASK_FUNC(lnk_icf_mark_folded_lines_task) LNK_Obj *obj = task->objs[task_id]; if (obj->icf_fold == 0) { scratch_end(scratch); return; } - String8 follower_chksms = str8_zero(); - B32 follower_chksms_init = 0; - LNK_Obj *cached_leader = 0; - String8 leader_chksms = str8_zero(); - for (U32 section_number = 1; section_number <= obj->coff.sections.count_no_null; section_number += 1) { LNK_ICFFold fold = obj->icf_fold[section_number]; if (!fold.set) { continue; } @@ -4171,16 +4190,8 @@ THREAD_POOL_TASK_FUNC(lnk_icf_mark_folded_lines_task) Temp fold_temp = temp_begin(scratch.arena); U32 child_sn = lnk_icf_debug_s_child_from_section(obj, section_number); if (child_sn != 0) { - // the returned String8s slice the objs' section data (only the parse's list nodes live - // on the temp arena), so caching them across fold_temp scopes is safe - if (!follower_chksms_init) { - follower_chksms = lnk_icf_obj_file_chksms(fold_temp.arena, obj); - follower_chksms_init = 1; - } - if (leader_obj != cached_leader) { - cached_leader = leader_obj; - leader_chksms = lnk_icf_obj_file_chksms(fold_temp.arena, leader_obj); - } + String8 follower_chksms = lnk_icf_obj_file_chksms(obj); // per-obj memo -- leaders + String8 leader_chksms = lnk_icf_obj_file_chksms(leader_obj); // shared across follower objs LNK_ICFSrcKey fk = lnk_icf_src_key_from_fn(fold_temp.arena, obj, section_number, follower_chksms); LNK_ICFSrcKey lk = lnk_icf_src_key_from_fn(fold_temp.arena, leader_obj, fold.leader_sn, leader_chksms); B32 same_src = fk.valid && lk.valid && diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index 34d46eb76..8c9f1f06d 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -66,6 +66,8 @@ typedef struct LNK_Obj // link state LNK_ObjSymbolRef *symlinks; // indexed by COFF section number; slot zero is null LNK_ICFFold *icf_fold; // /OPT:ICF fold map (per section number); 0 if ICF off + String8 icf_file_chksms; // memoized obj-wide FILECHKSMS slice (see lnk_icf_obj_file_chksms); + B32 icf_file_chksms_init; // idempotent racy fill, flag published last B8 *icf_lines_only; // .debug$S sections associated to an ICF-folded function: stay // LnkRemove'd, but merge into the module remapped to the leader RVA // (section-number indexed; slot zero is null; 0 array ptr when From d5bba7c4da4d9e3b0bf17de545f1f6e3ab809627 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Sun, 19 Jul 2026 18:26:25 -0700 Subject: [PATCH 040/121] build: PGO training also links with /OPT:REF /OPT:ICF lyra.rsp trains with /OPT:NOREF /OPT:NOICF, so the profile had zero coverage of the /OPT:REF walk and the ICF refinement -- the largest CPU consumers of a real editor link -- and -fprofile-use laid them out as cold. Run a second training link with REF+ICF appended (later switches win), store per-process profraws, and merge both. Co-Authored-By: Claude Fable 5 --- build.bat | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/build.bat b/build.bat index 6a692d18f..8d60e837e 100644 --- a/build.bat +++ b/build.bat @@ -53,13 +53,15 @@ if "%pgo%"=="1" ( where llvm-profdata /q || echo llvm-profdata is not in the PATH || exit /b 1 if "%clang%"=="1" ( if "%pgo_run%" == "1" ( - call llvm-profdata merge %LLVM_PROFILE_FILE% -output=%~dp0build\build.profdata || exit /b 1 + call llvm-profdata merge %~dp0build\pgo_raw\*.profraw -output=%~dp0build\build.profdata || exit /b 1 set auto_compile_flags=%auto_compile_flags% -fprofile-use=%~dp0build\build.profdata set pgo_run=0 ) else ( echo [pgo enabled] set auto_compile_flags=%auto_compile_flags% -fprofile-generate -mllvm -vp-counters-per-site=5 - set LLVM_PROFILE_FILE=%~dp0build\build.profraw + if not exist %~dp0build\pgo_raw mkdir %~dp0build\pgo_raw + del /q %~dp0build\pgo_raw\*.profraw 2>nul + set LLVM_PROFILE_FILE=%~dp0build\pgo_raw\build.%%p.profraw set pgo_run=1 ) ) else ( @@ -200,6 +202,10 @@ if "%pgo_run%"=="1" ( if "%radlink%"=="1" ( pushd local\lyra_pgo call %~dp0build\radlink @lyra.rsp || exit /b 1 + rem lyra.rsp trains with /OPT:NOREF /OPT:NOICF; run a second link with REF+ICF on + rem (later switches win) so the profile also covers the /OPT:REF walk and the ICF + rem refinement -- otherwise PGO marks them cold and the build regresses on real links + call %~dp0build\radlink @lyra.rsp /OPT:REF /OPT:ICF || exit /b 1 popd ) goto restart From 43bdaf053ee2d1e6f687284050b20a7fcd8e25b0 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 20 Jul 2026 09:24:58 -0700 Subject: [PATCH 041/121] linker: implement by-name-no-prefix and undecorate import members pe_make_import_dll_obj_static/_delayed left COFF_ImportBy_Undecorate and COFF_ImportBy_NameNoPrefix as NotImplemented, which is a no-op in release builds: iat_symbol stayed null, the jump-thunk emit pushed a reloc with a null symbol, and coff_obj_writer_serialize crashed dereferencing it (coff_obj_writer.c:207). Hit by GDK import libraries (Xbox FortniteClient), which use both name types; lld-link and link.exe accept the same inputs. Derive the import lookup name per the PE/COFF spec -- NO_PREFIX skips one leading '?', '@', or '_'; UNDECORATE additionally truncates at the first '@' -- and share the ImportBy_Name path. The __imp_ symbol and jump thunk keep the decorated name. Co-Authored-By: Claude Fable 5 --- src/coff/coff.c | 20 ++++++++++++++++++++ src/coff/coff.h | 1 + src/pe/pe_make_import_table.c | 14 ++++++++------ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/coff/coff.c b/src/coff/coff.c index 437b39de1..e7da807b7 100644 --- a/src/coff/coff.c +++ b/src/coff/coff.c @@ -293,6 +293,26 @@ coff_make_import_lookup(Arena *arena, U16 hint, String8 name) return result; } +internal String8 +coff_import_lookup_name_from_import_by(String8 name, COFF_ImportByType import_by) +{ + // IMPORT_OBJECT_NAME_NO_PREFIX: the public symbol name without a leading ?, @, or _. + // IMPORT_OBJECT_NAME_UNDECORATE: prefix skipped as above, then truncated at the first @. + String8 result = name; + if (import_by == COFF_ImportBy_NameNoPrefix || import_by == COFF_ImportBy_Undecorate) { + if (result.size > 0 && (result.str[0] == '?' || result.str[0] == '@' || result.str[0] == '_')) { + result = str8_skip(result, 1); + } + if (import_by == COFF_ImportBy_Undecorate) { + U64 at_pos = str8_find_needle(result, 0, str8_lit("@"), 0); + if (at_pos < result.size) { + result = str8_prefix(result, at_pos); + } + } + } + return result; +} + internal U32 coff_make_ordinal32(U16 hint) { diff --git a/src/coff/coff.h b/src/coff/coff.h index 5f379ee55..94b91f511 100644 --- a/src/coff/coff.h +++ b/src/coff/coff.h @@ -621,6 +621,7 @@ internal String8 coff_ordinal_data_from_hint(Arena *arena, COFF_MachineType mach internal String8 coff_make_lib_member_header(Arena *arena, String8 name, COFF_TimeStamp time_stamp, U16 user_id, U16 group_id, U16 mode, U32 size); internal String8 coff_make_import_lookup(Arena *arena, U16 hint, String8 name); +internal String8 coff_import_lookup_name_from_import_by(String8 name, COFF_ImportByType import_by); internal String8 coff_make_import_header(Arena *arena, COFF_MachineType machine, COFF_TimeStamp time_stamp, String8 dll_name, COFF_ImportByType import_by, String8 name, U16 hint_or_ordinal, COFF_ImportType type); //////////////////////////////// diff --git a/src/pe/pe_make_import_table.c b/src/pe/pe_make_import_table.c index eb20b0e45..0baf5390b 100644 --- a/src/pe/pe_make_import_table.c +++ b/src/pe/pe_make_import_table.c @@ -286,11 +286,14 @@ pe_make_import_dll_obj_static(Arena *arena, COFF_TimeStamp time_stamp, COFF_Mach str8_list_push(obj_writer->arena, &ilt_sect->data, ordinal_data); str8_list_push(obj_writer->arena, &iat_sect->data, ordinal_data); } break; + case COFF_ImportBy_NameNoPrefix: + case COFF_ImportBy_Undecorate: case COFF_ImportBy_Name: { COFF_ObjSection *int_sect = coff_obj_writer_push_section(obj_writer, str8_lit(".idata$6"), PE_IDATA_SECTION_FLAGS|COFF_SectionFlag_Align2Bytes|comdat_flags, str8_zero()); coff_obj_writer_push_symbol_associative(obj_writer, int_sect, iat_sect); COFF_ObjSymbol *int_symbol = coff_obj_writer_push_symbol_static(obj_writer, int_sect->name, 0, int_sect); - String8 int_data = coff_make_import_lookup(obj_writer->arena, import_header.hint_or_ordinal, import_header.func_name); + String8 lookup_name = coff_import_lookup_name_from_import_by(import_header.func_name, import_header.import_by); + String8 int_data = coff_make_import_lookup(obj_writer->arena, import_header.hint_or_ordinal, lookup_name); str8_list_push(obj_writer->arena, &int_sect->data, int_data); str8_list_push_aligner(obj_writer->arena, &int_sect->data, 0, 2); @@ -302,8 +305,6 @@ pe_make_import_dll_obj_static(Arena *arena, COFF_TimeStamp time_stamp, COFF_Mach str8_list_push(obj_writer->arena, &ilt_sect->data, str8_array(import_entry, import_size)); str8_list_push(obj_writer->arena, &iat_sect->data, str8_array(import_entry, import_size)); } break; - case COFF_ImportBy_Undecorate: { NotImplemented; } break; - case COFF_ImportBy_NameNoPrefix: { NotImplemented; } break; default: { InvalidPath; } break; } @@ -461,9 +462,12 @@ pe_make_import_dll_obj_delayed(Arena *arena, COFF_TimeStamp time_stamp, COFF_Mac coff_obj_writer_section_push_reloc_addr(obj_writer, uiat_sect, uiat_offset, load_thunk_symbol); } } break; + case COFF_ImportBy_NameNoPrefix: + case COFF_ImportBy_Undecorate: case COFF_ImportBy_Name: { // put together name look up entry - String8 int_data = coff_make_import_lookup(obj_writer->arena, import_header.hint_or_ordinal, import_header.func_name); + String8 lookup_name = coff_import_lookup_name_from_import_by(import_header.func_name, import_header.import_by); + String8 int_data = coff_make_import_lookup(obj_writer->arena, import_header.hint_or_ordinal, lookup_name); U64 int_data_offset = int_sect->data.total_size; str8_list_push(obj_writer->arena, &int_sect->data, int_data); @@ -503,8 +507,6 @@ pe_make_import_dll_obj_delayed(Arena *arena, COFF_TimeStamp time_stamp, COFF_Mac coff_obj_writer_section_push_reloc_addr(obj_writer, uiat_sect, uiat_data_offset, load_thunk_symbol); } } break; - case COFF_ImportBy_Undecorate: { NotImplemented; } break; - case COFF_ImportBy_NameNoPrefix: { NotImplemented; } break; } } From c885a84da5614333d7b52bb8a2ee39d56c99037b Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 20 Jul 2026 12:04:19 -0700 Subject: [PATCH 042/121] thread_pool: fair-share floor for barrier-pass cohorts tp_barrier_begin sized the cohort by a non-blocking sweep of free budget slots and pinned it for the whole bracket. A long barrier phase (ICF refinement: 60-130s CPU) that opened its bracket while sibling links held the machine ran the entire phase at width 1-2 and never grew -- even after the siblings exited. 4-concurrent convoy: one link's icf phase 2.5s -> 76s wall, grant_avg 2.5, park 6588s. If the sweep lands below this process's fair share (machine budget / attached processes, from the nproc counter), keep taking with bounded waits (200ms deadline) until the floor is reached. Slots flow back continuously as sibling path-A workers drain, so the floor normally fills within milliseconds; if every sibling is pinned in its own bracket the deadline bounds the wait -- never a deadlock, cohort >= 1 always. Convoy A/B (4x UnrealEditorFortnite-Engine.dll): worst-case unchanged when all four ICF brackets collide, but off-peak links drop 54-57s -> 37s. The full fix -- re-opening the bracket between refinement rounds so width tracks availability -- needs the ICF driver split into per-round passes; kept separate. Co-Authored-By: Claude Fable 5 --- src/linker/thread_pool/thread_pool.c | 30 ++++++++++++++++++++++++++++ src/linker/thread_pool/thread_pool.h | 2 ++ 2 files changed, 32 insertions(+) diff --git a/src/linker/thread_pool/thread_pool.c b/src/linker/thread_pool/thread_pool.c index 2482b109e..151edc5d3 100644 --- a/src/linker/thread_pool/thread_pool.c +++ b/src/linker/thread_pool/thread_pool.c @@ -351,6 +351,7 @@ tp_alloc(Arena *arena, U32 worker_count, U32 max_worker_count, String8 name) // kernel objects for the same pool name String8 budget_name = push_str8f(scratch.arena, "%S.budget." TP_SHARED_V, name); pool->budget_semaphore = semaphore_alloc(max_worker_count, max_worker_count, budget_name); + pool->max_worker_count = max_worker_count; // local wake/governor signalling. governor_semaphore is a 0/1 "at least one // pending pass" flag: main pings it with semaphore_drop_if_room (a redundant @@ -630,6 +631,35 @@ tp_barrier_begin(TP_Context *pool) } } + // FAIR-SHARE FLOOR: the cohort is pinned for the whole bracket, so a bracket + // opened at a bad instant (siblings momentarily holding the machine) would run + // a long phase at width 1-2 even after the machine empties. If the free-slot + // sweep landed below this process's fair share (machine budget / attached + // processes), keep taking with bounded waits until we reach it or the deadline + // expires. Slots flow back continuously as sibling path-A workers drain, so + // this normally fills within a few ms; if every sibling is pinned in its own + // long bracket the deadline bounds the wait and we proceed with what we hold -- + // never a deadlock, cohort >= 1 always. + if (extra < want) { + U32 procs = 0, procs_maxseen = 0; + tp_procs_snapshot(&procs, &procs_maxseen); + if (procs > 1) { + U32 fair = pool->max_worker_count / procs; + fair = Clamp(1, fair, want + 1); + if (1 + extra < fair) { + U64 deadline_us = now_time_us() + TP_BARRIER_FLOOR_WAIT_US; + for (; 1 + extra < fair; ) { + U64 now_us = now_time_us(); + if (now_us >= deadline_us) { break; } + U64 slice_us = Min(deadline_us - now_us, 5000); + if (semaphore_take(pool->budget_semaphore, now_us + slice_us)) { + extra += 1; + } + } + } + } + } + U32 cohort = 1 + extra; // main + grabbed workers pool->barrier_saved_workers = pool->worker_count; diff --git a/src/linker/thread_pool/thread_pool.h b/src/linker/thread_pool/thread_pool.h index 143d23d78..d7eb23fe1 100644 --- a/src/linker/thread_pool/thread_pool.h +++ b/src/linker/thread_pool/thread_pool.h @@ -47,6 +47,7 @@ typedef struct TP_Context volatile U32 pass_active; // 1 while a path-A (barrier-free) pass is in flight volatile U32 barrier_pass; // 1 while the current wake cohort is a path-B barrier pass volatile S64 granted; // budget slots currently held by woken path-A workers + U32 max_worker_count; // machine core budget (budget_semaphore init/max) // FAIR-SHARE barrier-pass cohort state (path B). A barrier pass runs at the // cohort the governor currently allows this process to hold: main + however @@ -148,6 +149,7 @@ internal void tp_stats_snapshot(F64 *grant_avg_out, F64 *park_seconds_out); #define TP_SHARED_V "v2" #define TP_NPROC_V "v3" #define TP_NPROC_MAX (1u << 20) // far above any plausible concurrent-link count +#define TP_BARRIER_FLOOR_WAIT_US 200000 // tp_barrier_begin: max wait to reach the fair-share cohort floor internal void tp_procs_snapshot(U32 *attached_out, U32 *maxseen_out); // 0/0 when no shared pool internal void tp_procs_detach(void); // give the permit back + close (idempotent) From 766b1dd8b2c8aee0f7ffbead37cb2ca6b7c8d562 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 20 Jul 2026 23:50:53 -0700 Subject: [PATCH 043/121] linker: fuse leaf TI-fixup into unbucket -- materialize unique leaves into private memory The leaf type-index fixup (lnk_cv_patcher_leaves_task) patched final TIs in place into the mapped .debug sections, copy-on-writing one page per touched page. On /Z7-style input where most objs carry their own type records (e.g. clang /DEBUG:GHASH builds) that is ~690K pages / ~2.7GB of CoW commit per editor link; PCH/type-server-heavy input mostly hides it because the leaves concentrate in few objs. Fuse the fixup into the unbucket pass: copy each unique raw leaf (in the existing sorted order) into one contiguous private buffer, apply the fixup to the copy, and point result.v at the copy. Identical bytes, identical order, clean input pages; the buffer lives exactly as long as the merged-types result it backs. Measured (UnrealEditorFortnite-Engine.dll, /DEBUG:GHASH inputs): CoW-promoted pages 694,335 -> 4,159, kernel CPU -50s. DLL and PDB byte-identical; determinism pair 0 bytes. Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.c | 77 +++++++++++++++++++++++++------------ src/linker/lnk_debug_info.h | 5 +++ 2 files changed, 58 insertions(+), 24 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index af79374fb..2d1ac721c 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -2842,19 +2842,49 @@ THREAD_POOL_TASK_FUNC(lnk_cv_patcher_inlines_task) } internal -THREAD_POOL_TASK_FUNC(lnk_cv_patcher_leaves_task) +THREAD_POOL_TASK_FUNC(lnk_count_unique_leaf_sizes_task) +{ + LNK_MergeTypes *task = raw_task; + Rng1U64 range = task->ranges[task_id]; + U64 size = 0; + for EachInRange(i, range) { + LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; + CV_DebugT *debug_t = &task->input->debug_t_arr[lnk_leaf_ref_obj_idx(leaf_ref)]; + size += cv_debug_t_get_raw_leaf(debug_t, lnk_leaf_ref_leaf_idx(leaf_ref)).size; + } + task->leaf_buffer_offsets[task_id] = size; // exclusive-scanned into offsets on the main thread +} + +// Materialize unique leaves: copy each unique raw leaf (existing sorted order) into one contiguous +// private buffer and apply the type-index fixup to the COPY. This fuses the old +// lnk_cv_patcher_leaves_task (which patched TIs in-place into the mapped input, dirtying one +// copy-on-write page per touched .debug$T page) with the old lnk_unbucket_raw_leaves_task (which +// pointed result.v into the input). result.v now points into the copy: identical bytes, identical +// order, clean input pages. +internal +THREAD_POOL_TASK_FUNC(lnk_materialize_unique_leaves_task) { ProfBeginFunction(); LNK_MergeTypes *task = raw_task; Rng1U64 range = task->ranges[task_id]; Arena *fixed_arena = task->fixed_arenas[task_id]; - for EachInRange(leaf_ref_idx, range) { - Temp temp = temp_begin(fixed_arena); - LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[leaf_ref_idx]; + U8 *cursor = task->leaf_buffer + task->leaf_buffer_offsets[task_id]; + for EachInRange(i, range) { + LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_idx); + + // copy raw leaf into the private buffer + MemoryCopy(cursor, raw_leaf.str, raw_leaf.size); + task->result.v[task->ti_source][i] = cursor; + cursor += raw_leaf.size; + + // fixup type indices on the copy (same math the in-place leaf patcher applied) + Temp temp = temp_begin(fixed_arena); + CV_Leaf leaf = {0}; + cv_read_leaf(str8(task->result.v[task->ti_source][i], raw_leaf.size), 0, 1, &leaf); CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); lnk_fixup_cv_type_indices(task, obj_idx, leaf.data, ti_offs); temp_end(temp); @@ -2862,19 +2892,6 @@ THREAD_POOL_TASK_FUNC(lnk_cv_patcher_leaves_task) ProfEnd(); } -internal -THREAD_POOL_TASK_FUNC(lnk_unbucket_raw_leaves_task) -{ - LNK_MergeTypes *task = raw_task; - Rng1U64 range = task->ranges[task_id]; - for EachInRange(i, range) { - LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; - CV_DebugT *debug_t = &task->input->debug_t_arr[lnk_leaf_ref_obj_idx(leaf_ref)]; - String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, lnk_leaf_ref_leaf_idx(leaf_ref)); - task->result.v[task->ti_source][i] = raw_leaf.str; - } -} - internal THREAD_POOL_TASK_FUNC(lnk_unbucket_hashes_task) { @@ -3309,11 +3326,9 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK tp_for_parallel_prof(tp, 0, input->count, lnk_cv_patcher_inlines_task, &task, "Fixup Inlines Type Indices"); } - for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { - task.ti_source = ti_source; - task.ranges = tp_divide_work(scratch.arena, task.unique_leaf_refs_arr[ti_source].count, tp->worker_count); - tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_cv_patcher_leaves_task, &task, "Fixup Types Type Indices"); - } + // NOTE: the leaf TI-fixup is fused into the unbucket/materialize pass below -- it copies each + // unique leaf into a private buffer and patches the copy, instead of patching the mapped input + // (which copy-on-writes one page per touched .debug$T page). } ProfEnd(); @@ -3340,7 +3355,21 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.result.count[ti_source] = unique_leaf_refs.count; task.result.v [ti_source] = push_array(tp_temp->v[0], U8 *, unique_leaf_refs.count); task.ranges = tp_divide_work(scratch.arena, unique_leaf_refs.count, tp->worker_count); - tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_unbucket_raw_leaves_task, &task, "Unbucket Leaves"); + + // per-lane byte totals for the materialize buffer, exclusive-scanned into offsets + task.leaf_buffer_offsets = push_array_no_zero(scratch.arena, U64, tp->worker_count + 1); + tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_count_unique_leaf_sizes_task, &task, "Count Leaf Sizes"); + { + U64 acc = 0; + for EachIndex(lane, tp->worker_count) { + U64 lane_size = task.leaf_buffer_offsets[lane]; + task.leaf_buffer_offsets[lane] = acc; + acc += lane_size; + } + task.leaf_buffer_offsets[tp->worker_count] = acc; + task.leaf_buffer = push_array_no_zero(tp_temp->v[0], U8, acc ? acc : 1); + } + tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_materialize_unique_leaves_task, &task, "Materialize + Fixup Leaves"); if (merge_flags & LNK_MergeTypeFlag_ExportHashes) { task.result.hashes[ti_source] = push_array_no_zero(tp_temp->v[0], U64, unique_leaf_refs.count); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 2d8c55fca..fb54ccb8f 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -223,6 +223,11 @@ typedef struct // iff the unique count exceeds cap, which is a function of the input alone. U32 leaf_ht_overflow; + // materialize unique leaves (unbucket + leaf TI-fixup fused, applied to a private copy so the + // fixup never dirties the copy-on-write input mapping) + U64 *leaf_buffer_offsets; // [worker_count+1] per-lane byte offsets into leaf_buffer + U8 *leaf_buffer; + LNK_MergedTypes result; } LNK_MergeTypes; From d6741470b04c65c687a90cd6ab72d235b6979b3f Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Tue, 21 Jul 2026 14:47:30 -0700 Subject: [PATCH 044/121] linker: escalate folded-debug-info to the full tree on function-scoped statics too lnk_icf_debug_s_has_locals gated the full-record-tree escalation for different-source ICF folds on S_LOCAL/S_REGREL32 only. A fold whose follower tree carries function-scoped static records (S_LDATA32 and friends) but no stack locals got Lines-only treatment, dropping the record that names the static -- the (folded) data itself survives in the image, but the debugger could no longer evaluate the follower's static by name. Reaching this requires the static's section to fold too (COMDAT const data), so the window is narrow, but the check should match the intent: escalate whenever the tree names anything the leader's tree cannot describe. Add S_LDATA32/S_GDATA32/S_LTHREAD32/S_GTHREAD32/S_FILESTATIC/S_CONSTANT to the escalation set. Cost at editor scale: +8KB PDB total; DLL and PDB otherwise byte-identical, determinism pair 0 bytes. Co-Authored-By: Claude Fable 5 --- src/linker/lnk.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 6819613c5..ad4edcf15 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -4148,7 +4148,21 @@ lnk_icf_debug_s_has_locals(Arena *scratch, LNK_Obj *obj, U32 child_sn) MemoryCopy(&len, s.str + o, sizeof(len)); MemoryCopy(&kind, s.str + o + 2, sizeof(kind)); if (len < 2) { break; } - if (kind == CV_SymKind_LOCAL || kind == CV_SymKind_REGREL32) { return 1; } + switch (kind) { + // stack locals + case CV_SymKind_LOCAL: + case CV_SymKind_REGREL32: + // function-scoped statics (S_LDATA32 and friends): the record naming the static lives in + // this tree; if it were dropped the debugger could no longer evaluate the follower's + // static by name, even though the (folded) data itself survives in the image + case CV_SymKind_LDATA32: + case CV_SymKind_GDATA32: + case CV_SymKind_LTHREAD32: + case CV_SymKind_GTHREAD32: + case CV_SymKind_FILESTATIC: + case CV_SymKind_CONSTANT: + return 1; + } o += len + 2; } } From c7884ee4b935d38cec70a8990c13f6507813ba0c Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 13 Jul 2026 15:00:52 -0700 Subject: [PATCH 045/121] linker: accept /OPT:ICFSTATIC as an alias for /OPT:ICF The fork gated internal-linkage COMDAT folding behind /OPT:ICFSTATIC; this implementation always considers them. Build systems (UnrealBuildTool) still pass the switch -- treat it as ICF instead of failing the link with 'unknown option'. Co-Authored-By: Claude Fable 5 --- src/linker/lnk_config.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index 4533d67ce..01c3fdc2b 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -1734,6 +1734,10 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val } } config->opt_icf = LNK_SwitchState_Yes; + } else if (str8_match_lit("icfstatic", param, StringMatchFlag_CaseInsensitive)) { + // compatibility: build systems that drove the fork's /OPT:ICFSTATIC. Internal-linkage + // COMDATs are always fold candidates here, so this is plain ICF. + config->opt_icf = LNK_SwitchState_Yes; } else if (str8_match_lit("noicf", param, StringMatchFlag_CaseInsensitive)) { config->opt_icf = LNK_SwitchState_No; } else if (str8_match_lit("lbr", param, StringMatchFlag_CaseInsensitive)) { From 43e3cbb7ebc13cd2cee5338bb129939bcd8c359f Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 10 Aug 2026 17:05:01 -0700 Subject: [PATCH 046/121] linker: cap fault-bound debug-input stages at /RAD_DEBUG_WORKERS workers (default 12) -- the kernel working-set-insert path ceilings near 3M pages/s, so lanes past ~12 only convert free cores into page-fault spin; measured on the FN editor DLL: link kernel CPU 802s -> 483s, wall unchanged, output byte-identical Co-Authored-By: Claude Fable 5 --- src/linker/lnk_config.c | 14 +++++++++ src/linker/lnk_config.h | 2 ++ src/linker/lnk_debug_info.c | 59 ++++++++++++++++++++++++++++++------- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index 01c3fdc2b..cc4e8524a 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -109,6 +109,7 @@ global read_only LNK_CmdSwitch g_cmd_switch_map[] = { LNK_CmdSwitch_Rad_UnresolvedSymbolRefLimit, 0, LNK_CmdValueKind_Scalar, "RAD_UNRESOLVED_SYMBOL_REF_LIMIT", ":#", "Limit number of unresolved symbol references linker reports." }, { LNK_CmdSwitch_Rad_Version, 0, LNK_CmdValueKind_Null, "RAD_VERSION", "", "Print version and exit." }, { LNK_CmdSwitch_Rad_Workers, 0, LNK_CmdValueKind_Scalar, "RAD_WORKERS", ":#", "Set number of workers created in the pool. Number is capped at 1024. When /RAD_SHARED_THREAD_POOL is specified this number cant exceed /RAD_SHARED_THREAD_POOL_MAX_WORKERS." }, + { LNK_CmdSwitch_Rad_DebugWorkers, 0, LNK_CmdValueKind_Scalar, "RAD_DEBUG_WORKERS", ":#", "Cap concurrent workers in page-fault-bound debug-input stages (parse/prefetch). Default 12; 0 = uncapped. Output is identical either way; the cap only trades idle spinning in the kernel page-fault path for free cores." }, { LNK_CmdSwitch_Rad_WorkDir, 0, LNK_CmdValueKind_Scalar, "RAD_WORK_DIR", ":PATH", "Working directory used for stable debug paths." }, { LNK_CmdSwitch_RadTypeServer, 0, LNK_CmdValueKind_Scalar, "RAD_TYPE_SERVER", ":FILENAME", "Merge types and store them in the specified file. The filename must have the .rrt extension." }, @@ -2329,6 +2330,14 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val } } break; + case LNK_CmdSwitch_Rad_DebugWorkers: { + U64 cap; + if (lnk_cmd_switch_parse_u64(obj, cmd_switch, value_strings, &cap, 0)) { + config->debug_worker_cap = cap; + } + } break; + + case LNK_CmdSwitch_Rad_WorkDir: { lnk_cmd_switch_parse_string_copy(config->arena, obj, cmd_switch, value, &config->work_dir); } break; @@ -2976,6 +2985,11 @@ lnk_config_init(U64 argc, char **argv) config->work_dir = get_current_path(arena); config->force = lnk_cmd_line_has_switch(cmd_line, LNK_CmdSwitch_Force); + // fault-bound debug-input stages spin on the kernel page-fault path past ~12 + // concurrent workers (throughput ceiling); default cap trades that spin for + // free cores, /RAD_DEBUG_WORKERS:0 restores full width + config->debug_worker_cap = 12; + // apply command line switches for EachNode(cmd, LNK_CmdOption, cmd_line.first_option) { lnk_apply_cmd_option_to_config(config, cmd->string, cmd->value, 0); diff --git a/src/linker/lnk_config.h b/src/linker/lnk_config.h index 8f9052afc..4e082b958 100644 --- a/src/linker/lnk_config.h +++ b/src/linker/lnk_config.h @@ -138,6 +138,7 @@ typedef enum LNK_CmdSwitch_Rad_SortImports, LNK_CmdSwitch_Rad_TimeStamp, LNK_CmdSwitch_Rad_DebugTypeHash, + LNK_CmdSwitch_Rad_DebugWorkers, LNK_CmdSwitch_Rad_UnresolvedSymbolLimit, LNK_CmdSwitch_Rad_UnresolvedSymbolRefLimit, LNK_CmdSwitch_Rad_Version, @@ -347,6 +348,7 @@ typedef struct LNK_Config U64 pdb_page_size; U64 worker_count; U64 max_worker_count; + U64 debug_worker_cap; String8 shared_thread_pool_name; LNK_SwitchState do_function_pad_min; B32 infer_function_pad_min; diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 2d1ac721c..930ab0604 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -99,8 +99,47 @@ THREAD_POOL_TASK_FUNC(lnk_prefetch_task) } #endif +// Run a per-item parallel-for on at most `cap` workers. The debug-input stages +// this wraps are page-fault-bound: the kernel working-set-insert path tops out +// near ~3M pages/s regardless of thread count, so lanes past ~12 only convert +// free cores into spin inside the fault handler (measured on the FN editor +// DLL: 12 lanes beat 64 on wall AND cut stage kernel CPU 357s -> 41s). Items +// are pulled from a shared cursor, so per-item outputs land in the same +// item-indexed slots as the uncapped path -- output is byte-identical. +typedef struct +{ + TP_TaskFunc *func; + void *data; + U64 item_count; + U64 cursor; +} LNK_CappedForTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_capped_for_task) +{ + LNK_CappedForTask *wrap = raw_task; + for (;;) { + U64 item_idx = ins_atomic_u64_inc_eval(&wrap->cursor) - 1; + if (item_idx >= wrap->item_count) { break; } + wrap->func(arena, worker_id, item_idx, wrap->data, tp); + } +} + +internal void +lnk_tp_for_parallel_capped(TP_Context *tp, TP_Arena *task_arena, U64 cap, U64 item_count, TP_TaskFunc *func, void *data) +{ + if (cap == 0 || cap >= item_count) { + tp_for_parallel(tp, task_arena, item_count, func, data); + } else { + LNK_CappedForTask wrap = { .func = func, .data = data, .item_count = item_count }; + tp_for_parallel(tp, task_arena, cap, lnk_capped_for_task, &wrap); + } +} + +#define lnk_tp_for_parallel_capped_prof(pool, arena, cap, item_count, task_func, task_data, zone_name) ProfBegin(zone_name); lnk_tp_for_parallel_capped(pool, arena, cap, item_count, task_func, task_data); ProfEnd(); + internal void -lnk_prefetch_ranges(TP_Context *tp, U64 range_count, Rng1U64 *ranges) +lnk_prefetch_ranges(TP_Context *tp, U64 worker_cap, U64 range_count, Rng1U64 *ranges) { #if OS_WINDOWS // resolve once (Win8+; on older OS fall through silently). Only called from @@ -154,7 +193,7 @@ lnk_prefetch_ranges(TP_Context *tp, U64 range_count, Rng1U64 *ranges) LNK_PrefetchTask task = { .proc = prefetch_proc, .entry_count = entry_count, .entries = entries }; U64 batch_count = CeilIntegerDiv(entry_count, LNK_PREFETCH_BATCH_SIZE); if (tp != 0 && batch_count > 1) { - tp_for_parallel(tp, 0, batch_count, lnk_prefetch_task, &task); + lnk_tp_for_parallel_capped(tp, 0, worker_cap, batch_count, lnk_prefetch_task, &task); } else { for EachIndex(batch_idx, batch_count) { lnk_prefetch_task(0, 0, batch_idx, &task, 0); } } @@ -1719,7 +1758,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 prefetch_begin_us = now_time_us(); U64 prefetch_bytes = 0; for EachIndex(range_idx, range_count) { prefetch_bytes += dim_1u64(ranges[range_idx]); } - lnk_prefetch_ranges(tp, range_count, ranges); + lnk_prefetch_ranges(tp, config->debug_worker_cap, range_count, ranges); lnk_log(LNK_Log_Timers, "[mcvi] prefetched %llu debug section ranges (%llu MiB) in %.2f ms", range_count, prefetch_bytes / MB(1), (F64)(now_time_us() - prefetch_begin_us) / 1000.0); @@ -1776,7 +1815,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, { // parse .debug$S input.debug_s_arr = push_array(tp_arena->v[0], CV_DebugS, input.obj_count); - tp_for_parallel_prof(tp, tp_arena, obj_count, lnk_parse_debug_s_task, &input, "Parse .debug$S"); + lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, obj_count, lnk_parse_debug_s_task, &input, "Parse .debug$S"); // collect .debug$P and .debug$T String8Array *raw_debug_p_arr = push_array(scratch.arena, String8Array, obj_count); @@ -1803,20 +1842,20 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, debug_p_arr = push_array(tp_arena->v[0], CV_DebugT, obj_count); parse_types.raw_types = raw_debug_p_arr; parse_types.out_types = debug_p_arr; - tp_for_parallel_prof(tp, 0, obj_count, lnk_strip_debug_t_sig_task, &parse_types, "Strip .debug$P"); - tp_for_parallel_prof(tp, tp_arena, obj_count, lnk_parse_debug_t_task, &parse_types, "Parse .debug$P"); + lnk_tp_for_parallel_capped_prof(tp, 0, config->debug_worker_cap, obj_count, lnk_strip_debug_t_sig_task, &parse_types, "Strip .debug$P"); + lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, obj_count, lnk_parse_debug_t_task, &parse_types, "Parse .debug$P"); // parse .debug$T input.debug_t_arr = push_array(tp_arena->v[0], CV_DebugT, obj_count); parse_types.raw_types = raw_debug_t_arr; parse_types.out_types = input.debug_t_arr; - tp_for_parallel_prof(tp, 0, obj_count, lnk_strip_debug_t_sig_task, &parse_types, "Strip .debug$T"); - tp_for_parallel_prof(tp, tp_arena, obj_count, lnk_parse_debug_t_task, &parse_types, "Parse .debug$T"); + lnk_tp_for_parallel_capped_prof(tp, 0, config->debug_worker_cap, obj_count, lnk_strip_debug_t_sig_task, &parse_types, "Strip .debug$T"); + lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, obj_count, lnk_parse_debug_t_task, &parse_types, "Parse .debug$T"); // parse .debug$H input.debug_h_arr = push_array(tp_arena->v[0], CV_DebugH, input.obj_count); if (config->ghash) { - tp_for_parallel_prof(tp, tp_arena, obj_count, lnk_parse_debug_h_task, &input, "Parse .debug$H"); + lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, obj_count, lnk_parse_debug_h_task, &input, "Parse .debug$H"); } } ProfEnd(); @@ -3063,7 +3102,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK U64 prefetch_begin_us = now_time_us(); U64 prefetch_bytes = 0; for EachIndex(range_idx, range_count) { prefetch_bytes += dim_1u64(ranges[range_idx]); } - lnk_prefetch_ranges(tp, range_count, ranges); + lnk_prefetch_ranges(tp, input->config->debug_worker_cap, range_count, ranges); lnk_log(LNK_Log_Timers, "[merge] prefetched %llu type data ranges (%llu MiB) in %.2f ms", range_count, prefetch_bytes / MB(1), (F64)(now_time_us() - prefetch_begin_us) / 1000.0); From 47b494ffe5a1940f9cc151d902a905656f03b83b Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 10 Aug 2026 22:41:14 -0700 Subject: [PATCH 047/121] linker: open background-written output files on the writer thread -- CreateFile that replaces a previous multi-GB PDB pays the NTFS truncate/dealloc of the old allocation (measured 1.45s serial inside lnk_build_pdb on an 8.7GB editor PDB); jobs on one file are queue-ordered so every write stays behind the open, and an open failure is reported from the EndFile job. Output byte-identical. Co-Authored-By: Claude Fable 5 --- src/linker/lnk_io.c | 92 +++++++++++++++++++++++++++++++-------------- 1 file changed, 63 insertions(+), 29 deletions(-) diff --git a/src/linker/lnk_io.c b/src/linker/lnk_io.c index f16b984d0..216c24528 100644 --- a/src/linker/lnk_io.c +++ b/src/linker/lnk_io.c @@ -500,10 +500,12 @@ struct LNK_BackgroundFile B32 is_finished; B32 is_complete; B32 write_failed; + B32 open_failed; }; typedef enum { + LNK_BackgroundFileJobKind_OpenFile, LNK_BackgroundFileJobKind_Write, LNK_BackgroundFileJobKind_EndFile, LNK_BackgroundFileJobKind_EndWriter, @@ -518,9 +520,47 @@ typedef struct String8 data; } LNK_BackgroundFileWriteJob; +// Opening the output file can cost seconds on the caller's thread when it +// replaces a previous multi-GB artifact (NTFS truncate/dealloc of the old +// allocation happens inside CreateFile). Jobs on one file are queue-ordered, +// so deferring the open to the writer thread keeps every write behind it +// while the caller returns immediately. +internal void +lnk_background_file_writer_open_file_on_thread(LNK_BackgroundFile *file) +{ + if (file->open_with_rename) { + file->file = lnk_file_open_with_rename_permissions(file->temp_path); + file->open_path = file->temp_path; + } else { + lnk_open_file_write((char *)file->path.str, file->path.size, &file->file, sizeof(file->file)); + file->open_path = file->path; + } + + file->is_open = !file_match(file->file, file_zero()); + if (!file->is_open) { + file->open_failed = 1; + file->write_failed = 1; + return; + } + + if (file->open_with_rename && !lnk_file_set_delete_on_close(file->file, 1)) { + lnk_error(LNK_Error_IO, "failed to update file disposition on %S", file->open_path); + lnk_close_file(&file->file); + file->is_open = 0; + file->open_failed = 1; + file->write_failed = 1; + } +} + internal void lnk_background_file_writer_end_file_on_thread(LNK_BackgroundFile *file, U64 expected_byte_count) { + if (file->open_failed) { + lnk_error(LNK_Error_NoAccess, "don't have access to write to %S", file->path); + file->is_complete = 0; + return; + } + B32 is_complete = !file->write_failed && file->bytes_written == expected_byte_count; if (is_complete && file->open_with_rename) { if (!lnk_file_set_delete_on_close(file->file, 0)) { @@ -559,12 +599,16 @@ lnk_background_file_writer_thread(void *raw_writer) if (job.kind == LNK_BackgroundFileJobKind_EndWriter) { break; } - if (job.kind == LNK_BackgroundFileJobKind_Write) { - U64 write_size = lnk_write_file(&job.file->file, job.file_off, job.data.str, job.data.size); - if (write_size != job.data.size) { - job.file->write_failed = 1; + if (job.kind == LNK_BackgroundFileJobKind_OpenFile) { + lnk_background_file_writer_open_file_on_thread(job.file); + } else if (job.kind == LNK_BackgroundFileJobKind_Write) { + if (job.file->is_open) { + U64 write_size = lnk_write_file(&job.file->file, job.file_off, job.data.str, job.data.size); + if (write_size != job.data.size) { + job.file->write_failed = 1; + } + job.file->bytes_written += write_size; } - job.file->bytes_written += write_size; } else if (job.kind == LNK_BackgroundFileJobKind_EndFile) { lnk_background_file_writer_end_file_on_thread(job.file, job.expected_byte_count); } @@ -603,32 +647,22 @@ lnk_background_file_writer_begin_file(LNK_BackgroundFileWriter *writer, String8 file->temp_path = temp_path; file->open_with_rename = (temp_path.size > 0); - if (file->open_with_rename) { - file->file = lnk_file_open_with_rename_permissions(temp_path); - file->open_path = temp_path; - } else { - lnk_open_file_write((char *)path.str, path.size, &file->file, sizeof(file->file)); - file->open_path = path; - } - - file->is_open = !file_match(file->file, file_zero()); - if (!file->is_open) { - lnk_error(LNK_Error_NoAccess, "don't have access to write to %S", path); - goto exit; - } + SLLQueuePush(writer->file_first, writer->file_last, file); - if (file->open_with_rename && !lnk_file_set_delete_on_close(file->file, 1)) { - lnk_error(LNK_Error_IO, "failed to update file disposition on %S", file->open_path); - lnk_close_file(&file->file); - file->is_open = 0; - goto exit; + // the open runs on the writer thread (multi-GB truncate of a previous + // artifact is seconds of caller-thread stall otherwise); jobs on one file + // are queue-ordered, so every write lands behind the open. An open failure + // flips write_failed and is reported from the EndFile job. + { + LNK_BackgroundFileWriteJob job = { .kind = LNK_BackgroundFileJobKind_OpenFile, .file = file }; + RingGuard guard = guarded_ring_open(writer->queue); + B32 is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64); + guarded_ring_close(&guard); + Assert(is_written); } - SLLQueuePush(writer->file_first, writer->file_last, file); - - exit:; ProfEnd(); - return file->is_open ? file : 0; + return file; } internal void @@ -637,7 +671,7 @@ lnk_background_file_writer_enqueue(LNK_BackgroundFileWriter *writer, LNK_Backgro ProfBegin("Background File Writer Enqueue"); Assert(writer->is_running); - Assert(file->is_open && !file->is_finished); + Assert(!file->is_finished); // is_open is owned by the writer thread (open is a queued job) if (data.size > 0) { LNK_BackgroundFileWriteJob job = { @@ -661,7 +695,7 @@ lnk_background_file_writer_end_file(LNK_BackgroundFileWriter *writer, LNK_Backgr ProfBegin("Background File Writer End File"); Assert(writer->is_running); - Assert(file->is_open && !file->is_finished); + Assert(!file->is_finished); // is_open is owned by the writer thread (open is a queued job) file->is_finished = 1; LNK_BackgroundFileWriteJob job = { From 92569e65fbf8d985e0221928b5a2250f2d858f57 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 10 Aug 2026 23:13:28 -0700 Subject: [PATCH 048/121] linker: cap the input-view unmap pass at the debug worker cap -- UnmapViewOfFile serializes on the process address-space lock, so the full 64-worker pass burned ~257s of kernel CPU spinning for a ~4.1s wall pass on the FN editor DLL; 12 lanes do the same wall for a fraction of the spin. Output byte-identical. Co-Authored-By: Claude Fable 5 --- src/linker/lnk.c | 11 ++++++++--- src/linker/lnk.h | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index ad4edcf15..771c5bfff 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -1061,7 +1061,7 @@ THREAD_POOL_TASK_FUNC(lnk_release_file_map_task) } internal void -lnk_inputer_release_file_maps(TP_Context *tp, LNK_Inputer *inputer) +lnk_inputer_release_file_maps(TP_Context *tp, U64 worker_cap, LNK_Inputer *inputer) { Temp scratch = scratch_begin(0, 0); @@ -1087,7 +1087,12 @@ lnk_inputer_release_file_maps(TP_Context *tp, LNK_Inputer *inputer) } Assert(mapped_input_idx == mapped_input_count); - tp_for_parallel(tp, 0, mapped_input_count, lnk_release_file_map_task, mapped_inputs); + // UnmapViewOfFile serializes on the process address-space lock: with the + // full pool every worker camps in the kernel for the whole pass (measured + // ~257s of kernel CPU for a ~4.1s wall pass at 64 workers on the FN editor + // DLL). Cap the lanes like the other fault/VAD-bound stages -- same wall, + // a fraction of the spin. + lnk_tp_for_parallel_capped(tp, 0, worker_cap, mapped_input_count, lnk_release_file_map_task, mapped_inputs); scratch_end(scratch); } @@ -7680,7 +7685,7 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) #if OS_WINDOWS // for unexplained reasons, file mappings on Windows cause slow process exit times ProfBegin("Release Input File Maps"); - lnk_inputer_release_file_maps(tp, inputer); + lnk_inputer_release_file_maps(tp, config->debug_worker_cap, inputer); ProfEnd(); #endif diff --git a/src/linker/lnk.h b/src/linker/lnk.h index 95eb56614..d566d9bba 100644 --- a/src/linker/lnk.h +++ b/src/linker/lnk.h @@ -378,7 +378,7 @@ internal LNK_Input * lnk_inputer_push_lib_thin(LNK_Inputer *inputer, LNK_Config internal B32 lnk_inputer_has_items(LNK_Inputer *inputer); internal LNK_InputPtrArray lnk_inputer_flush(Arena *arena, TP_Context *tp, LNK_Inputer *inputer, LNK_IO_Flags io_flags, LNK_InputList *all_inputs, LNK_InputList *new_inputs); -internal void lnk_inputer_release_file_maps(TP_Context *tp, LNK_Inputer *inputer); +internal void lnk_inputer_release_file_maps(TP_Context *tp, U64 worker_cap, LNK_Inputer *inputer); // --- Link Context ------------------------------------------------------------ From dd1adf19f6c247d19ba5919522fa3c76c7057ab4 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 10 Aug 2026 23:13:29 -0700 Subject: [PATCH 049/121] msf: direct stream-number -> node table -- msf_find_stream_node walked the sectab list from the head per lookup (O(streams) each, O(streams^2) across the module passes); a fixed 64K-entry table (512KiB) makes it an indexed load. Stream numbers, list order, and output bytes unchanged. Co-Authored-By: Claude Fable 5 --- src/linker/pdb_ext/msf_builder.c | 12 ++++++------ src/linker/pdb_ext/msf_builder.h | 4 ++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/linker/pdb_ext/msf_builder.c b/src/linker/pdb_ext/msf_builder.c index b19560af3..9574977e8 100644 --- a/src/linker/pdb_ext/msf_builder.c +++ b/src/linker/pdb_ext/msf_builder.c @@ -800,6 +800,7 @@ msf_stream_alloc_ex(MSF_Context *msf, MSF_UInt size) { MSF_StreamNode *node = msf_stream_alloc_(msf->arena, &msf->sectab); MSF_Stream *stream = &node->data; + msf->stream_from_sn[stream->sn] = node; msf_stream_resize_ex(msf, stream, size); return stream->sn; } @@ -854,6 +855,7 @@ msf_stream_free(MSF_Context *msf, MSF_StreamNumber sn) B32 is_free_ok = 0; MSF_StreamNode *stream_node = msf_find_stream_node(msf, sn); if (stream_node) { + msf->stream_from_sn[sn] = 0; msf_stream_list_remove(&msf->sectab, stream_node); msf_stream_resize_ex(msf, &stream_node->data, 0); stream_node->data.size = MSF_DELETED_STREAM_STAMP; @@ -1406,6 +1408,7 @@ msf_alloc__(Arena *arena, MSF_UInt page_size, MSF_PageNumber active_fpm) msf->arena = arena; msf->page_size = page_size; msf->active_fpm = active_fpm; + msf->stream_from_sn = push_array(arena, MSF_StreamNode *, MSF_STREAM_NUMBER_MAX); ProfEnd(); return msf; @@ -1438,13 +1441,10 @@ msf_alloc(MSF_UInt page_size, MSF_UInt active_fpm) internal MSF_StreamNode * msf_find_stream_node(MSF_Context *msf, MSF_StreamNumber sn) { - MSF_StreamNode *node; - for (node = msf->sectab.first; node != 0; node = node->next) { - if (node->data.sn == sn) { - break; - } + if (sn < MSF_STREAM_NUMBER_MAX) { + return msf->stream_from_sn[sn]; } - return node; + return 0; } internal MSF_Stream * diff --git a/src/linker/pdb_ext/msf_builder.h b/src/linker/pdb_ext/msf_builder.h index c07b582a8..4bbb36422 100644 --- a/src/linker/pdb_ext/msf_builder.h +++ b/src/linker/pdb_ext/msf_builder.h @@ -83,6 +83,10 @@ typedef struct MSF_Context MSF_PageList st_page_list; MSF_PageList page_pool; MSF_StreamList sectab; + // direct stream-number -> node table (sn is allocated sequentially, so the + // linear sectab walk in msf_find_stream_node is O(streams) per lookup and + // O(streams^2) across the module passes); fixed 64K-entry table = 512KiB + MSF_StreamNode **stream_from_sn; } MSF_Context; typedef enum MSF_Error From 1b1a4635d7f5d159c656dfcbbf55616764d7094a Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 10 Aug 2026 23:13:29 -0700 Subject: [PATCH 050/121] linker: fix O(T^2) dummy type-server obj allocation -- the loop pushed a ts_arr.count-sized LNK_Obj array per iteration and used only its first element; allocate the array once. (Codex finding.) Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 930ab0604..046923d62 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -1993,10 +1993,12 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, input.ts_obj_range = r1u64(prev.count, input.count); // alloc dummy objs with for each loaded type server + // (one obj per type server; this used to push a ts_arr.count-sized array + // per iteration and use only its first element -- O(T^2) arena growth) + LNK_Obj *ts_objs = push_array(tp_arena->v[0], LNK_Obj, ts_arr.count); for EachIndex(i, ts_arr.count) { - LNK_Obj *ts_obj = push_array(tp_arena->v[0], LNK_Obj, ts_arr.count); - ts_obj->path = ts_arr.v[i].ts_path; - input.obj_arr[prev.count + i] = ts_obj; + ts_objs[i].path = ts_arr.v[i].ts_path; + input.obj_arr[prev.count + i] = &ts_objs[i]; } // make type server indices From 33d79ab5fecb44e7bc905f734886703407d149b8 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 10 Aug 2026 23:57:15 -0700 Subject: [PATCH 051/121] linker: parallelize giant .debug$T parsing inside the object -- a SharedPCH-scale .debug$T is a serial pointer-chase (each leaf offset depends on the previous leaf's size) and a handful of such objs tail-bound the capped parse stage (measured 6.5s for one obj while the pool sat parked). One cheap serial hop per giant records true-chain checkpoints every 64K leaves plus the exact leaf count, then the full pool re-walks intervals from those true boundaries storing offsets and classifying kinds. Speculative mid-stream resync is unsound for CodeView (arbitrary payload bytes chain validly), so checkpoints are the only safe boundaries; offsets come from the true chain and per-source counts reduce in interval order, so output is bit-identical to the serial parse (gated). Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.c | 171 +++++++++++++++++++++++++++++++++++- 1 file changed, 170 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 046923d62..654c9e4e6 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -376,7 +376,9 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_t_task) { ProfBeginFunction(); LNK_ParseCvTypes *task = raw_task; - if (task->raw_types[task_id].count > 0) { + if (task->out_types[task_id].offsets != 0) { + // giant obj, already parsed by lnk_parse_giant_debug_t + } else if (task->raw_types[task_id].count > 0) { task->out_types[task_id] = cv_debug_t_from_data(arena, task->raw_types[task_id].v[0], CV_LeafAlign); } else { MemoryZeroStruct(&task->out_types[task_id]); @@ -384,6 +386,172 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_t_task) ProfEnd(); } +// Giant .debug$T parse: a single SharedPCH-scale .debug$T is a multi-second +// SERIAL pointer-chase (each leaf's offset depends on the previous leaf's +// size), and a handful of such objs bound the whole capped parse stage +// (measured: one 6.5 s obj on the FN editor DLL while the rest of the pool +// sat parked). Speculative mid-stream resynchronization is unsound for +// CodeView (arbitrary payload bytes chain "validly", so a wrong guess is not +// locally detectable), so instead: one cheap serial hop per giant walks the +// true chain recording a checkpoint offset every LNK_GIANT_DEBUG_T_INTERVAL +// leaves, then the full pool re-walks the intervals from those true +// boundaries, storing leaf offsets and classifying kinds. Offsets come from +// the true chain and per-source counts are reduced in interval order, so the +// result is bit-identical to the serial parse. +#define LNK_GIANT_DEBUG_T_SIZE MB(16) +#define LNK_GIANT_DEBUG_T_INTERVAL (64*1024) + +typedef U64 LNK_GiantSourceCounts[CV_TypeIndexSource_COUNT]; + +typedef struct +{ + U64 obj_idx; + String8 data; + U64 leaf_count; + U64 interval_count; + U32 *checkpoints; // [interval_count] leaf offset at each interval start + LNK_GiantSourceCounts *interval_counts; +} LNK_GiantDebugT; + +typedef struct +{ + LNK_ParseCvTypes *parse; + LNK_GiantDebugT *giants; + U32 *interval_giant; // flat interval index -> giant index + U32 *interval_local; // flat interval index -> interval within giant +} LNK_GiantDebugTTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_giant_debug_t_hop_task) +{ + ProfBeginFunction(); + LNK_GiantDebugTTask *task = raw_task; + LNK_GiantDebugT *g = &task->giants[task_id]; + String8 data = g->data; + + // bare chain walk; bounds replicate cv_read_leaf exactly (incl. its + // total-size quirks) so the leaf set matches the serial parse bit for bit. + // CV_LeafAlign == 1 => stride is sizeof(CV_LeafHeader) + (size - sizeof(CV_LeafKind)). + U64 checkpoint_cap = data.size / (sizeof(CV_LeafHeader) * LNK_GIANT_DEBUG_T_INTERVAL) + 2; + g->checkpoints = push_array_no_zero(arena, U32, checkpoint_cap); + + U64 leaf_count = 0; + if (data.size >= sizeof(CV_LeafHeader)) { + for (U64 cursor = 0; cursor < data.size; ) { + CV_LeafHeader header = { .v = memory_read32(data.str + cursor) }; + if (header.size < sizeof(CV_LeafKind)) { break; } + if (sizeof(CV_LeafSize) + (U64)header.size > data.size) { break; } + U64 stride = AlignPow2(sizeof(CV_LeafHeader) + (U64)(header.size - sizeof(CV_LeafKind)), CV_LeafAlign); + if (stride > data.size) { break; } + if ((leaf_count % LNK_GIANT_DEBUG_T_INTERVAL) == 0) { + Assert(leaf_count / LNK_GIANT_DEBUG_T_INTERVAL < checkpoint_cap); + g->checkpoints[leaf_count / LNK_GIANT_DEBUG_T_INTERVAL] = (U32)cursor; + } + leaf_count += 1; + cursor += stride; + } + } + + g->leaf_count = leaf_count; + g->interval_count = CeilIntegerDiv(leaf_count, LNK_GIANT_DEBUG_T_INTERVAL); + g->interval_counts = push_array(arena, LNK_GiantSourceCounts, g->interval_count); + + CV_DebugT *out = &task->parse->out_types[g->obj_idx]; + MemoryZeroStruct(out); + out->data = data; + out->count = leaf_count; + out->offsets = push_array_no_zero(arena, U32, leaf_count); + ProfEnd(); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_giant_debug_t_interval_task) +{ + ProfBeginFunction(); + LNK_GiantDebugTTask *task = raw_task; + LNK_GiantDebugT *g = &task->giants[task->interval_giant[task_id]]; + U64 local_idx = task->interval_local[task_id]; + String8 data = g->data; + CV_DebugT *out = &task->parse->out_types[g->obj_idx]; + + U64 leaf_lo = local_idx * LNK_GIANT_DEBUG_T_INTERVAL; + U64 leaf_hi = Min(leaf_lo + LNK_GIANT_DEBUG_T_INTERVAL, g->leaf_count); + U64 cursor = g->checkpoints[local_idx]; + U64 *counts = g->interval_counts[local_idx]; + + for (U64 leaf_idx = leaf_lo; leaf_idx < leaf_hi; leaf_idx += 1) { + CV_LeafHeader header = { .v = memory_read32(data.str + cursor) }; + out->offsets[leaf_idx] = (U32)cursor; + counts[cv_type_index_source_from_leaf_kind(header.kind)] += 1; + cursor += AlignPow2(sizeof(CV_LeafHeader) + (U64)(header.size - sizeof(CV_LeafKind)), CV_LeafAlign); + } + ProfEnd(); +} + +internal void +lnk_parse_giant_debug_t(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 obj_count, LNK_ParseCvTypes *parse) +{ + ProfBeginFunction(); + Temp scratch = scratch_begin(0,0); + + // gather giants + U64 giant_count = 0; + LNK_GiantDebugT *giants = push_array(scratch.arena, LNK_GiantDebugT, obj_count); + for EachIndex(obj_idx, obj_count) { + if (parse->raw_types[obj_idx].count > 0 && parse->raw_types[obj_idx].v[0].size >= LNK_GIANT_DEBUG_T_SIZE) { + giants[giant_count].obj_idx = obj_idx; + giants[giant_count].data = parse->raw_types[obj_idx].v[0]; + giant_count += 1; + } + } + + if (giant_count > 0) { + LNK_GiantDebugTTask task = { .parse = parse, .giants = giants }; + + // phase 1: serial chain hop per giant (capped: the hop is the fault-bound + // dependent-load chase, extra lanes only spin) + lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, giant_count, lnk_giant_debug_t_hop_task, &task, "Giant .debug$T Hop"); + + // flatten intervals + U64 interval_total = 0; + for EachIndex(i, giant_count) { interval_total += giants[i].interval_count; } + task.interval_giant = push_array_no_zero(scratch.arena, U32, interval_total); + task.interval_local = push_array_no_zero(scratch.arena, U32, interval_total); + for (U64 i = 0, flat = 0; i < giant_count; i += 1) { + for EachIndex(k, giants[i].interval_count) { + task.interval_giant[flat] = (U32)i; + task.interval_local[flat] = (U32)k; + flat += 1; + } + } + + // phase 2: offsets + kind classification from true checkpoints; pages are + // hot from the hop, so this is CPU-bound -- full pool width + tp_for_parallel_prof(tp, 0, interval_total, lnk_giant_debug_t_interval_task, &task, "Giant .debug$T Intervals"); + + // phase 3: deterministic interval-order reduce + the serial-parse tail + for EachIndex(i, giant_count) { + LNK_GiantDebugT *g = &giants[i]; + CV_DebugT *out = &parse->out_types[g->obj_idx]; + for EachIndex(k, g->interval_count) { + for EachElement(s, out->source_counts) { out->source_counts[s] += g->interval_counts[k][s]; } + } +#if BUILD_DEBUG + { U64 total = 0; for EachElement(s, out->source_counts) { total += out->source_counts[s]; } Assert(total == g->leaf_count); } +#endif + for EachElement(s, out->ti_ranges) { out->ti_ranges[s] = r1u64(CV_MinComplexTypeIndex, CV_MinComplexTypeIndex + out->count); } + CV_Leaf leaf = cv_debug_t_get_leaf(out, 0); + if (leaf.kind == CV_LeafKind_PRECOMP) { + CV_PrecompInfo precomp_info = cv_precomp_info_from_leaf(leaf); + for EachElement(s, out->ti_ranges) { out->ti_ranges[s].max += precomp_info.leaf_count; } + } + } + } + + scratch_end(scratch); + ProfEnd(); +} + internal THREAD_POOL_TASK_FUNC(lnk_read_type_servers_task) { @@ -1850,6 +2018,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, parse_types.raw_types = raw_debug_t_arr; parse_types.out_types = input.debug_t_arr; lnk_tp_for_parallel_capped_prof(tp, 0, config->debug_worker_cap, obj_count, lnk_strip_debug_t_sig_task, &parse_types, "Strip .debug$T"); + lnk_parse_giant_debug_t(tp, tp_arena, config, obj_count, &parse_types); lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, obj_count, lnk_parse_debug_t_task, &parse_types, "Parse .debug$T"); // parse .debug$H From 1160b1a3b8132482e94a65be773f6a7d45816a57 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Tue, 11 Aug 2026 15:11:11 -0700 Subject: [PATCH 052/121] linker: PDB commit diet, first two steps -- (1) the background file writer decommits sealed MSF stream runs once they are on disk (write-once, immutable, never re-read; off under /RAD_DEBUG which re-reads PDB page memory for RDI). The final enqueue_remaining runs are exempt: they carry MSF metadata (FPM/header/stream table) that msf_get_save_size/msf_build still read -- decommitting those raced the writer and crashed msf_find_max_pn_ on small links. (2) merged-leaf materialize buffers move to a standalone allocation released right after pdb_build_types copies them into MSF pages (an arena push could not be handed back). FN editor DLL: peak commit 53.5 -> 47.3GB, output byte-identical, deterministic; lyra 9/9 clean on the previously-crashing path. Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.c | 38 +++++++++++++++++++++++++++++++------ src/linker/lnk_debug_info.h | 4 ++++ src/linker/lnk_io.c | 20 ++++++++++++++----- src/linker/lnk_io.h | 2 +- 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 654c9e4e6..8e6af345c 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3577,7 +3577,19 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK acc += lane_size; } task.leaf_buffer_offsets[tp->worker_count] = acc; - task.leaf_buffer = push_array_no_zero(tp_temp->v[0], U8, acc ? acc : 1); + // standalone allocation (not an arena push): pdb_build_types copies these bytes + // into MSF pages and nothing reads cv_types.v afterwards, so the linker path + // releases the buffer right after -- multi-GB on AutoRTFM-scale inputs. An arena + // push could not be handed back without poisoning later pushes into the range. + if (acc > 0) { + task.leaf_buffer = reserve_memory(acc); + if (task.leaf_buffer == 0 || !commit_memory(task.leaf_buffer, acc)) { + lnk_error(LNK_Error_Boot, "failed to allocate %M for merged type leaves", acc); + } + task.result.leaf_buffers[ti_source] = str8(task.leaf_buffer, acc); + } else { + task.leaf_buffer = push_array_no_zero(tp_temp->v[0], U8, 1); + } } tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_materialize_unique_leaves_task, &task, "Materialize + Fixup Leaves"); @@ -4547,6 +4559,7 @@ typedef struct MSF_StreamNumber *sealed_streams; U64 sealed_stream_count; U64 sealed_stream_cap; + B32 decommit_flushed; // off when /RAD_DEBUG re-reads the PDB page memory for RDI conversion } LNK_PdbOutput; typedef struct LNK_MsfPageCursor @@ -4598,7 +4611,7 @@ lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_Strea run_size += msf->page_size; } else { if (run_data != 0) { - lnk_background_file_writer_enqueue(output->writer, output->file, (U64)run_first_pn * msf->page_size, str8(run_data, run_size)); + lnk_background_file_writer_enqueue(output->writer, output->file, (U64)run_first_pn * msf->page_size, str8(run_data, run_size), output->decommit_flushed); } run_first_pn = run_last_pn = page->pn; run_data = page_data; @@ -4606,7 +4619,7 @@ lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_Strea } } if (run_data != 0) { - lnk_background_file_writer_enqueue(output->writer, output->file, (U64)run_first_pn * msf->page_size, str8(run_data, run_size)); + lnk_background_file_writer_enqueue(output->writer, output->file, (U64)run_first_pn * msf->page_size, str8(run_data, run_size), output->decommit_flushed); } } @@ -4648,20 +4661,20 @@ lnk_pdb_output_enqueue_remaining(LNK_PdbOutput *output, MSF_Context *msf) run_size += page_size; } else { if (run_data != 0) { - lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size)); + lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size), /*decommit*/ 0); // MSF metadata (FPM/header/stream table) is read after this enqueue } run_first_pn = pn; run_data = page_data; run_size = page_size; } } else if (run_data != 0) { - lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size)); + lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size), /*decommit*/ 0); // MSF metadata (FPM/header/stream table) is read after this enqueue run_data = 0; run_size = 0; } } if (run_data != 0) { - lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size)); + lnk_background_file_writer_enqueue(output->writer, output->file, run_first_pn * msf->page_size, str8(run_data, run_size), /*decommit*/ 0); // MSF metadata (FPM/header/stream table) is read after this enqueue } scratch_end(scratch); } @@ -5085,6 +5098,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config LNK_PdbOutput *output_ptr = 0; if (writer.output_path.size > 0) { output.writer = writer.file_writer; + output.decommit_flushed = (config->rad_debug != LNK_SwitchState_Yes); output.file = lnk_background_file_writer_begin_file(output.writer, writer.output_path, writer.temp_output_path); if (output.file != 0) { output.sealed_stream_cap = cv->obj_count + 128; @@ -5135,6 +5149,18 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config ProfEnd(); pdb_build_types(tp, task.pdb, &build_hooks); + // merged leaf bytes are now in MSF pages (and on their way to disk); no consumer of + // cv_types.v remains on this path (RDI converts from the PDB artifact, the RRT export + // is a separate boot mode), so hand the multi-GB materialize buffers back to the OS + for EachElement(ti_source, cv_types.leaf_buffers) { + if (cv_types.leaf_buffers[ti_source].size > 0) { + release_memory(cv_types.leaf_buffers[ti_source].str, cv_types.leaf_buffers[ti_source].size); + cv_types.leaf_buffers[ti_source] = str8_zero(); + cv_types.v[ti_source] = 0; // poison dangling leaf pointers + cv_types.count[ti_source] = 0; + } + } + if (builder_flags & LNK_PDB_BuilderFlag_Modules) { ProfScope ("Alloc Modules") for EachIndex(obj_idx, cv->obj_count) { diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index fb54ccb8f..98c8e6fc3 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -181,6 +181,10 @@ typedef struct // @type_server CV_TypeIndex **obj_ti_maps; U64 *hashes[CV_TypeIndexSource_COUNT]; + + // standalone backing for the merged leaf bytes v[] points into (one per source); + // owned by the caller, released after pdb_build_types on the linker path + String8 leaf_buffers[CV_TypeIndexSource_COUNT]; } LNK_MergedTypes; typedef struct diff --git a/src/linker/lnk_io.c b/src/linker/lnk_io.c index 216c24528..cdc5f96fc 100644 --- a/src/linker/lnk_io.c +++ b/src/linker/lnk_io.c @@ -518,6 +518,7 @@ typedef struct U64 file_off; U64 expected_byte_count; String8 data; + B32 decommit_after_write; // hand the buffer's pages back to the OS once written } LNK_BackgroundFileWriteJob; // Opening the output file can cost seconds on the caller's thread when it @@ -609,6 +610,14 @@ lnk_background_file_writer_thread(void *raw_writer) } job.file->bytes_written += write_size; } + if (job.decommit_after_write) { + // the buffer is immutable and write-once (a sealed MSF stream run); once it is + // on disk its pages are dead weight in the commit charge. Round INWARD to whole + // OS pages so neighbors sharing the boundary pages are untouched. + U64 lo = AlignPow2((U64)job.data.str, KB(4)); + U64 hi = ((U64)job.data.str + job.data.size) & ~(U64)(KB(4) - 1); + if (lo < hi) { decommit_memory((void *)lo, hi - lo); } + } } else if (job.kind == LNK_BackgroundFileJobKind_EndFile) { lnk_background_file_writer_end_file_on_thread(job.file, job.expected_byte_count); } @@ -666,7 +675,7 @@ lnk_background_file_writer_begin_file(LNK_BackgroundFileWriter *writer, String8 } internal void -lnk_background_file_writer_enqueue(LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 file_off, String8 data) +lnk_background_file_writer_enqueue(LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 file_off, String8 data, B32 decommit_after_write) { ProfBegin("Background File Writer Enqueue"); @@ -675,10 +684,11 @@ lnk_background_file_writer_enqueue(LNK_BackgroundFileWriter *writer, LNK_Backgro if (data.size > 0) { LNK_BackgroundFileWriteJob job = { - .kind = LNK_BackgroundFileJobKind_Write, - .file = file, - .file_off = file_off, - .data = data, + .kind = LNK_BackgroundFileJobKind_Write, + .file = file, + .file_off = file_off, + .data = data, + .decommit_after_write = decommit_after_write, }; RingGuard guard = guarded_ring_open(writer->queue); B32 is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64); diff --git a/src/linker/lnk_io.h b/src/linker/lnk_io.h index 442456e46..76621da98 100644 --- a/src/linker/lnk_io.h +++ b/src/linker/lnk_io.h @@ -62,6 +62,6 @@ internal String8 lnk_data_from_file_artifact(Arena *arena, LNK_FileArtifact *art internal void lnk_background_file_writer_begin (LNK_BackgroundFileWriter *writer); internal LNK_BackgroundFile *lnk_background_file_writer_begin_file(LNK_BackgroundFileWriter *writer, String8 path, String8 temp_path); -internal void lnk_background_file_writer_enqueue (LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 file_off, String8 data); +internal void lnk_background_file_writer_enqueue (LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 file_off, String8 data, B32 decommit_after_write); internal void lnk_background_file_writer_end_file (LNK_BackgroundFileWriter *writer, LNK_BackgroundFile *file, U64 expected_byte_count); internal void lnk_background_file_writer_end (LNK_BackgroundFileWriter *writer); From 83ca569b3e66af37320388c930889285d89368b2 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Tue, 11 Aug 2026 18:48:43 -0700 Subject: [PATCH 053/121] linker: stream GSI/PSI symbol payloads through a reused window + enqueue module streams per obj -- gsi_build_ex serialized the entire symbol payload (GB-class at FN scale) into a staging buffer before the MSF copy; buckets now serialize into a 256MB window streamed straight into the symbols stream (absolute offsets unchanged, bytes identical). Module streams enqueue to the background writer as each module finalizes (atomic sealed-slot; file writes are positional so completion order cannot affect output) instead of all-at-once after the pass. Both are commit-structural: measured peak on the FN editor DLL is noise-neutral on a warm box (the old staging landed on recycled free-list scratch; the writer drains at disk speed), but worst-case scratch demand drops by the full symrec payload and module pages start flushing during the write pass. Output byte-identical, deterministic, lyra 9/9. Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.c | 39 +++++++++++++------ src/linker/lnk_debug_info.h | 1 + src/linker/pdb_ext/pdb_builder.c | 66 +++++++++++++++++++++----------- src/linker/pdb_ext/pdb_builder.h | 8 ++-- 4 files changed, 76 insertions(+), 38 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 8e6af345c..5d173d54e 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4332,6 +4332,9 @@ lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8No return mod_cursor; } +typedef struct LNK_PdbOutput LNK_PdbOutput; +internal void lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_StreamNumber sn); + internal THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) { @@ -4478,6 +4481,15 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) str8_list_concat_in_place(&mod->source_file_list, &source_file_list); temp_end(temp); + + // the module stream is final (symbols + C13 written, checksum name + // offsets patched): hand it to the background writer NOW so its pages + // flush + decommit while the remaining modules are still being written + // -- enqueueing all modules after the pass kept the entire module + // payload (GB-class) committed through the globals phase + if (task->output != 0) { + lnk_pdb_output_enqueue_stream(task->output, task->pdb->msf, mod->sn); + } } barrier_wait(tp->barrier); } @@ -4552,7 +4564,7 @@ THREAD_POOL_TASK_FUNC(lnk_push_dbi_sec_contrib_task) } } -typedef struct +struct LNK_PdbOutput { LNK_BackgroundFileWriter *writer; LNK_BackgroundFile *file; @@ -4560,7 +4572,7 @@ typedef struct U64 sealed_stream_count; U64 sealed_stream_cap; B32 decommit_flushed; // off when /RAD_DEBUG re-reads the PDB page memory for RDI conversion -} LNK_PdbOutput; +}; typedef struct LNK_MsfPageCursor { @@ -4589,8 +4601,13 @@ internal void lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_StreamNumber sn) { if (sn == MSF_INVALID_STREAM_NUMBER) { return; } - Assert(output->sealed_stream_count < output->sealed_stream_cap); - output->sealed_streams[output->sealed_stream_count++] = sn; + // atomic slot: module streams enqueue from pool workers as each module + // finishes; order in sealed_streams is irrelevant (set semantics for the + // remaining-pages bitmap), file writes are positional, so output bytes are + // unaffected by completion order + U64 slot = ins_atomic_u64_add_eval(&output->sealed_stream_count, 1) - 1; + Assert(slot < output->sealed_stream_cap); + output->sealed_streams[slot] = sn; MSF_Stream *stream = msf_find_stream(msf, sn); Assert(stream != 0); @@ -5107,6 +5124,8 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config } } + task.output = output_ptr; + PDB_BuildHooks build_hooks = {0}; if (output_ptr != 0) { build_hooks.stream_finalize = lnk_pdb_output_finalize_stream; @@ -5167,7 +5186,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config task.mod_arr[obj_idx] = dbi_push_module(task.pdb->dbi, cv->obj_arr[obj_idx]->path, lnk_obj_get_lib_path(cv->obj_arr[obj_idx])); } - ProfScope("Write Modules") +ProfScope("Write Modules") { lnk_summary_phase_begin(LNK_SummaryPhase_PdbGsi); U64 phase_begin_us = now_time_us(); @@ -5189,12 +5208,8 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config lnk_log(LNK_Log_Timers, "[pdb] write modules in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); lnk_summary_phase_end(LNK_SummaryPhase_PdbMod); } - if (output_ptr != 0) { - for EachIndex(obj_idx, cv->obj_count) { - lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.mod_arr[obj_idx]->sn); - } - } - ProfScope("Move Global Symbols") + // module streams were enqueued per-obj inside lnk_write_pdb_modules +ProfScope("Move Global Symbols") { U64 phase_begin_us = now_time_us(); U32 C = tp_barrier_begin(tp); @@ -5207,7 +5222,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config tp_barrier_end(tp); lnk_log(LNK_Log_Timers, "[pdb] move global symbols in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); } - ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); +ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); if (output_ptr != 0) { lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->publics_sn); lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->globals_sn); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 98c8e6fc3..ad17a8747 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -269,6 +269,7 @@ typedef struct Rng1U64Array image_section_file_ranges; U64 *image_section_file_section_numbers; PDB_DbiSCArray *sc_arrays; // [obj_count] + struct LNK_PdbOutput *output; // when non-null, module streams enqueue to the background writer as they complete } LNK_BuildPdb; typedef struct diff --git a/src/linker/pdb_ext/pdb_builder.c b/src/linker/pdb_ext/pdb_builder.c index 71898a54d..6acbf50e9 100644 --- a/src/linker/pdb_ext/pdb_builder.c +++ b/src/linker/pdb_ext/pdb_builder.c @@ -1725,10 +1725,6 @@ gsi_write_build_result(TP_Context *tp, msf_stream_reserve(msf, gsi_sn, gsi_size); ProfEnd(); - ProfBeginV("Reserve %M for symbols", build.symbol_data.size); - msf_stream_reserve(msf, symbols_sn, build.symbol_data.size); - ProfEnd(); - ProfBegin("Write GSI header"); msf_stream_write_struct(msf, gsi_sn, &build.header); ProfEnd(); @@ -1745,9 +1741,8 @@ gsi_write_build_result(TP_Context *tp, msf_stream_write(msf, gsi_sn, &build.compressed_bucket_arr[0], compressed_bucket_arr_size); ProfEnd(); - ProfBegin("Write symbols [%M]", build.symbol_data.size); - msf_stream_write_string_parallel(tp, msf, symbols_sn, build.symbol_data); - ProfEnd(); + // NOTE: the symbol payload itself was streamed into symbols_sn inside + // gsi_build_ex, window by window ProfEnd(); } @@ -2077,8 +2072,8 @@ psi_addr_map_from_gsi_records(TP_Context *tp, Arena *arena, PDB_GsiSortRecord *g internal THREAD_POOL_TASK_FUNC(gsi_size_buckets_task) { - U64 bucket_idx = task_id; - PDB_GsiSerializeSymbolsTask *task = raw_task; + PDB_GsiSerializeSymbolsTask *task = raw_task; + U64 bucket_idx = task->bucket_base + task_id; CV_SymbolList *bucket_list = &task->bucket_arr[bucket_idx]; for (CV_SymbolNode *node = bucket_list->first; node != 0; node = node->next) { task->bucket_size_arr[bucket_idx] += cv_size_from_symbol(&node->data, task->symbol_align); @@ -2150,8 +2145,8 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_pub32) { Temp scratch = scratch_begin(&arena, 1); - U64 bucket_idx = task_id; - PDB_GsiSerializeSymbolsTask *task = raw_task; + PDB_GsiSerializeSymbolsTask *task = raw_task; + U64 bucket_idx = task->bucket_base + task_id; CV_SymbolList bucket = task->bucket_arr[bucket_idx]; @@ -2165,7 +2160,7 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_pub32) PDB_GsiSortRecord *sort_record_arr = task->sort_record_arr_arr[bucket_idx]; U64 buffer_size = task->bucket_size_arr[bucket_idx]; U64 buffer_base = task->bucket_off_arr[bucket_idx]; - U8 *buffer = task->buffer + buffer_base; + U8 *buffer = task->buffer + (buffer_base - task->window_base); U64 sort_idx = 0; U64 buffer_cursor = 0; @@ -2201,8 +2196,8 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_symbols_task) { Temp scratch = scratch_begin(&arena, 1); - U64 bucket_idx = task_id; - PDB_GsiSerializeSymbolsTask *task = raw_task; + PDB_GsiSerializeSymbolsTask *task = raw_task; + U64 bucket_idx = task->bucket_base + task_id; CV_SymbolList bucket = task->bucket_arr[bucket_idx]; CV_Symbol **symbol_arr = push_array(scratch.arena, CV_Symbol *, bucket.count); @@ -2219,7 +2214,7 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_symbols_task) PDB_GsiSortRecord *sort_record_arr = task->sort_record_arr_arr[bucket_idx]; U64 buffer_size = task->bucket_size_arr[bucket_idx]; U64 buffer_base = task->bucket_off_arr[bucket_idx]; - U8 *buffer = task->buffer + buffer_base; + U8 *buffer = task->buffer + (buffer_base - task->window_base); U64 sort_idx = 0; U64 buffer_cursor = 0; @@ -2243,7 +2238,7 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_symbols_task) } internal PDB_GsiBuildResult -gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_base, B32 is_pub32, U64 msf_page_size) +gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumber symbols_sn, U64 symbol_data_base, B32 is_pub32, U64 msf_page_size) { ProfBeginFunction(); Temp scratch = scratch_begin(&arena,1); @@ -2259,9 +2254,17 @@ gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_ // estimate each bucket size tp_for_parallel(tp, 0, gsi->bucket_count, gsi_size_buckets_task, &serial_task); - // prepare serial buffer + // prepare serial buffer -- a REUSED WINDOW, not a full staging copy: the + // whole symbol payload is GB-class at FN scale and used to sit committed + // alongside the payload copies it was serialized from (the link's commit + // peak). Buckets are serialized in groups that fit the window and streamed + // into the symbols stream immediately; absolute offsets are unchanged, so + // the emitted bytes are identical to the single-buffer path. U64 buffer_size = sum_array_u64(gsi->bucket_count, serial_task.bucket_size_arr); - serial_task.buffer = push_array_no_zero(arena, U8, buffer_size); + U64 max_bucket_size = 0; + for EachIndex(bucket_idx, gsi->bucket_count) { max_bucket_size = Max(max_bucket_size, serial_task.bucket_size_arr[bucket_idx]); } + U64 window_cap = Max(MB(256), max_bucket_size); + serial_task.buffer = push_array_no_zero(scratch.arena, U8, Min(window_cap, buffer_size ? buffer_size : 1)); serial_task.bucket_off_arr = push_array_copy_u64(scratch.arena, serial_task.bucket_size_arr, gsi->bucket_count); u64_array_counts_to_offsets(gsi->bucket_count, serial_task.bucket_off_arr); @@ -2273,9 +2276,26 @@ gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_ cursor += gsi->bucket_arr[bucket_idx].count; } - // fill out sort records & serialize symbols + // fill out sort records & serialize symbols, window by window + msf_stream_reserve(msf, symbols_sn, buffer_size); TP_TaskFunc *serial_func = is_pub32 ? gsi_serialize_pub32 : gsi_serialize_symbols_task; - tp_for_parallel(tp, 0, gsi->bucket_count, serial_func, &serial_task); + for (U64 window_lo = 0; window_lo < gsi->bucket_count;) { + U64 window_base = serial_task.bucket_off_arr[window_lo]; + U64 window_hi = window_lo; + while (window_hi < gsi->bucket_count && + (serial_task.bucket_off_arr[window_hi] + serial_task.bucket_size_arr[window_hi]) - window_base <= window_cap) { + window_hi += 1; + } + Assert(window_hi > window_lo); // window_cap >= max_bucket_size guarantees progress + serial_task.bucket_base = window_lo; + serial_task.window_base = window_base; + tp_for_parallel(tp, 0, window_hi - window_lo, serial_func, &serial_task); + U64 window_end = (window_hi < gsi->bucket_count) ? serial_task.bucket_off_arr[window_hi] : buffer_size; + if (window_end > window_base) { + msf_stream_write_parallel(tp, msf, symbols_sn, serial_task.buffer, window_end - window_base); + } + window_lo = window_hi; + } ProfEnd(); @@ -2326,7 +2346,7 @@ gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_ result.compressed_bucket_count = compressed_offset_count; result.compressed_bucket_arr = compressed_offset_arr; result.total_hash_size = sizeof(header) + header.hash_record_arr_size + header.bucket_data_size; - result.symbol_data = str8(serial_task.buffer, buffer_size); + result.symbol_data = str8(0, buffer_size); // already streamed into symbols_sn window by window; size kept for logging scratch_end(scratch); ProfEnd(); @@ -2340,7 +2360,7 @@ gsi_build(TP_Context *tp, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumbe Temp scratch = scratch_begin(0,0); U64 symbol_data_base = msf_stream_get_pos(msf, symbols_sn); - PDB_GsiBuildResult build = gsi_build_ex(tp, scratch.arena, gsi, symbol_data_base, /* is_pub32: */ 0, msf->page_size); + PDB_GsiBuildResult build = gsi_build_ex(tp, scratch.arena, gsi, msf, symbols_sn, symbol_data_base, /* is_pub32: */ 0, msf->page_size); gsi_write_build_result(tp, build, msf, sn, symbols_sn); scratch_end(scratch); @@ -2473,7 +2493,7 @@ psi_build(TP_Context *tp, PDB_PsiContext *psi, MSF_Context *msf, MSF_StreamNumbe Temp scratch = scratch_begin(0,0); U64 symbol_data_base = msf_stream_get_pos(msf, symbols_sn); - PDB_GsiBuildResult gsi_build = gsi_build_ex(tp, scratch.arena, psi->gsi, symbol_data_base, /* is_pub32: */ 1, msf->page_size); + PDB_GsiBuildResult gsi_build = gsi_build_ex(tp, scratch.arena, psi->gsi, msf, symbols_sn, symbol_data_base, /* is_pub32: */ 1, msf->page_size); ProfBegin("Address Map"); diff --git a/src/linker/pdb_ext/pdb_builder.h b/src/linker/pdb_ext/pdb_builder.h index 016e4aeb2..667e658d6 100644 --- a/src/linker/pdb_ext/pdb_builder.h +++ b/src/linker/pdb_ext/pdb_builder.h @@ -213,8 +213,10 @@ typedef struct PDB_GsiSerializeSymbolsTask U64 symbol_align; CV_SymbolList *bucket_arr; U64 *bucket_size_arr; - U64 *bucket_off_arr; - U8 *buffer; + U64 *bucket_off_arr; // absolute offsets within the full symbol payload + U8 *buffer; // window backing: holds buckets [bucket_base, bucket_base+N) + U64 bucket_base; // first bucket of the current window + U64 window_base; // absolute offset the window starts at PDB_GsiSortRecord **sort_record_arr_arr; PDB_GsiSortRecord *sort_record_arr; } PDB_GsiSerializeSymbolsTask; @@ -365,7 +367,7 @@ internal PDB_GsiContext * gsi_alloc(void); internal void gsi_build(TP_Context *tp, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumber gsi_sn, MSF_StreamNumber symbols_sn); internal void gsi_release(PDB_GsiContext *gsi); internal void gsi_write_build_result(TP_Context *tp, PDB_GsiBuildResult build, MSF_Context *msf, MSF_StreamNumber sn, MSF_StreamNumber symbols_sn); -internal PDB_GsiBuildResult gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, U64 symbol_data_base, B32 export_symbol_ptr_arr, U64 msf_page_size); +internal PDB_GsiBuildResult gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumber symbols_sn, U64 symbol_data_base, B32 is_pub32, U64 msf_page_size); internal U32 gsi_hash(PDB_GsiContext *gsi, String8 input); internal CV_SymbolNode * gsi_push(PDB_GsiContext *gsi, CV_Symbol *symbol); internal void gsi_push_many_arr(TP_Context *tp, PDB_GsiContext *gsi, U64 count, CV_SymbolNode **symbol_arr); From 953f0527064dd64f6c09f43175ad52bbe97d90e1 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Tue, 11 Aug 2026 21:41:59 -0700 Subject: [PATCH 054/121] linker: GSI/PSI buckets hold CV_Symbol values in per-bucket arrays instead of CV_SymbolNode chains -- the three 48B-per-symbol node arrays (globals, proc-refs, publics) are gone; inserts stay sharded single-owner-per-bucket in global walk order after a per-wave count+reserve pass, the content-hash sort tie-break and windowed serialize are unchanged, and the serialize tasks sort pointers into the bucket arrays instead of flattening chains. Output byte-identical (DLL hash-equal across the restructure, det pair 0/0), lyra 9/9. Peak commit measured noise-neutral on a warm box (~48GB) -- the peak is dominated by in-flight MSF pages and payload copies, not node overhead; this lands as structure and as the prerequisite for emitting publics without their staged payload copy. Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.c | 152 +++++++++++++++++++------------ src/linker/pdb_ext/pdb_builder.c | 90 +++++++++--------- src/linker/pdb_ext/pdb_builder.h | 20 +++- 3 files changed, 156 insertions(+), 106 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 5d173d54e..ef43ba846 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3954,25 +3954,31 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } barrier_wait(tp->barrier); + // size buckets up front so each one reallocs at most once for this wave (arena pushes are + // single-threaded on task 0; sizing has no determinism impact) + if (task_id == 0) { + U64 *bucket_adds = push_array(scratch.arena, U64, gsi->bucket_count); + for EachIndex(i, symbol_count) { bucket_adds[symbol_hashes[i] % gsi->bucket_count] += 1; } + for EachIndex(bucket_idx, gsi->bucket_count) { + if (bucket_adds[bucket_idx]) { gsi_reserve(gsi, bucket_idx, bucket_adds[bucket_idx]); } + } + } + barrier_wait(tp->barrier); + // push global symbols, sharded by bucket range: worker i owns buckets [i*B/W, (i+1)*B/W) and // walks the FULL symbol sequence in global order, inserting only symbols whose bucket lands in // its range. each bucket has a single owner and receives its inserts in global sequence order, - // so per-chain order (which is serialized into the PDB) is byte-identical to a serial loop, for - // any worker count -- no locks, no atomics. - CV_SymbolNode *global_nodes = 0; - if (task_id == 0) { - global_nodes = push_array_no_zero(gsi->arena, CV_SymbolNode, symbol_count); - } - tp_broadcast(&global_nodes); + // so per-bucket order (which is serialized into the PDB) is byte-identical to a serial loop, + // for any worker count -- no locks, no atomics. { U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; for EachIndex(i, symbol_count) { U64 bucket_idx = symbol_hashes[i] % gsi->bucket_count; if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } - CV_SymbolNode *n = &global_nodes[i]; - n->prev = n->next = 0; - n->data = cv_symbol_from_ptr(symbol_arr[i]); + PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; + CV_Symbol *dst = &bucket->v[bucket->count]; + *dst = cv_symbol_from_ptr(symbol_arr[i]); // deterministic same-name tie-break for the per-bucket sort in gsi_serialize_symbols_task // (gsi_symbol_is_before compares name -> offset -> kind -> data bytes): key on the content // hash of the full raw record instead of the compacted deduper slot index. Slot order is @@ -3982,8 +3988,8 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) // symrec bytes run-to-run. The hash is schedule-independent; on collision the comparator's // kind/data-bytes fallback stays content-deterministic, and byte-identical records cannot // reach the sort (the deduper folds them), so the pointer tiebreaker stays unreachable. - n->data.offset = u64_hash_from_str8(cv_raw_from_symbol(symbol_arr[i])); - cv_symbol_list_push_node(&gsi->bucket_arr[bucket_idx], n); + dst->offset = u64_hash_from_str8(cv_raw_from_symbol(symbol_arr[i])); + bucket->count += 1; } } barrier_wait(tp->barrier); @@ -4031,17 +4037,17 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) U64 *proc_ref_hashes = 0; U64 *proc_ref_indices = 0; Arena **proc_ref_arenas = 0; - CV_SymbolNode *proc_ref_nodes = 0; + CV_Symbol *proc_ref_symbols = 0; if (task_id == 0) { proc_ref_hashes = push_array(scratch.arena, U64, total_proc_ref_count); proc_ref_indices = offsets_from_counts_array_u64(scratch.arena, proc_ref_counts, tp->worker_count); proc_ref_arenas = alloc_arena_many(gsi->arena, tp->worker_count, proc_ref_sizes); - proc_ref_nodes = push_array(gsi->arena, CV_SymbolNode, total_proc_ref_count); + proc_ref_symbols = push_array(scratch.arena, CV_Symbol, total_proc_ref_count); } tp_broadcast(&proc_ref_hashes); tp_broadcast(&proc_ref_indices); tp_broadcast(&proc_ref_arenas); - tp_broadcast(&proc_ref_nodes); + tp_broadcast(&proc_ref_symbols); Arena *proc_ref_arena = proc_ref_arenas[task_id]; U64 proc_ref_idx = proc_ref_indices[task_id]; @@ -4067,9 +4073,9 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32) { String8 name = cv_name_from_symbol(symbol.kind, symbol.data); - proc_ref_nodes [proc_ref_idx].data = cv_make_proc_ref(proc_ref_arena, imod, symbol_cursor, name, cv_is_lproc(symbol)); - proc_ref_nodes [proc_ref_idx].data.offset = symbol_cursor; - proc_ref_hashes[proc_ref_idx] = gsi_hash(gsi, name); + proc_ref_symbols[proc_ref_idx] = cv_make_proc_ref(proc_ref_arena, imod, symbol_cursor, name, cv_is_lproc(symbol)); + proc_ref_symbols[proc_ref_idx].offset = symbol_cursor; + proc_ref_hashes [proc_ref_idx] = gsi_hash(gsi, name); proc_ref_idx += 1; } @@ -4079,15 +4085,28 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } barrier_wait(tp->barrier); - // push proc refs, sharded by bucket range (single owner per bucket, inserts in global node - // order -> per-chain order identical to a serial loop for any worker count) + // size buckets up front so each one reallocs at most once for this wave (arena pushes are + // single-threaded on task 0; sizing has no determinism impact) + if (task_id == 0) { + U64 *bucket_adds = push_array(scratch.arena, U64, gsi->bucket_count); + for EachIndex(i, total_proc_ref_count) { bucket_adds[(U32)proc_ref_hashes[i] % gsi->bucket_count] += 1; } + for EachIndex(bucket_idx, gsi->bucket_count) { + if (bucket_adds[bucket_idx]) { gsi_reserve(gsi, bucket_idx, bucket_adds[bucket_idx]); } + } + } + barrier_wait(tp->barrier); + + // push proc refs, sharded by bucket range (single owner per bucket, inserts in global + // order -> per-bucket order identical to a serial loop for any worker count) { U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; for EachIndex(i, total_proc_ref_count) { U64 bucket_idx = (U32)proc_ref_hashes[i] % gsi->bucket_count; if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } - cv_symbol_list_push_node(&gsi->bucket_arr[bucket_idx], &proc_ref_nodes[i]); + PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; + bucket->v[bucket->count] = proc_ref_symbols[i]; + bucket->count += 1; } } barrier_wait(tp->barrier); @@ -4140,27 +4159,28 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) barrier_wait(tp->barrier); Arena **public_symbol_arenas = 0; - Arena **public_symbol_node_arenas = 0; - CV_SymbolList *public_symbols = 0; // [fixed_lane_count] - U32 **public_symbol_hashes = 0; // [fixed_lane_count][public_symbol.count] + CV_Symbol **public_symbol_vals = 0; // [fixed_lane_count][public_symbol_lane_counts[lane]] + U64 *public_symbol_lane_counts = 0; // [fixed_lane_count] + U32 **public_symbol_hashes = 0; // [fixed_lane_count][public_symbol_lane_counts[lane]] if (task_id == 0) { - U64 public_symbol_total_count = sum_array_u64(fixed_lane_count, public_symbol_node_counts); public_symbol_arenas = alloc_arena_many(psi->gsi->arena, fixed_lane_count, public_symbol_sizes); - public_symbol_node_arenas = alloc_arena_array(psi->gsi->arena, fixed_lane_count, public_symbol_node_counts, CV_SymbolNode); - public_symbols = push_array(scratch.arena, CV_SymbolList, fixed_lane_count); + public_symbol_vals = push_array(scratch.arena, CV_Symbol *, fixed_lane_count); + public_symbol_lane_counts = push_array(scratch.arena, U64, fixed_lane_count); public_symbol_hashes = push_array(scratch.arena, U32 *, fixed_lane_count); } tp_broadcast(&public_symbol_arenas); - tp_broadcast(&public_symbol_node_arenas); - tp_broadcast(&public_symbols); + tp_broadcast(&public_symbol_vals); + tp_broadcast(&public_symbol_lane_counts); tp_broadcast(&public_symbol_hashes); - // make CV public symbols + // make CV public symbols (per-lane CV_Symbol value arrays on scratch; payload bytes stay on + // public_symbol_arenas). lane arrays live on the owning worker's scratch and are only read by + // the same lane->worker striding below, then copied into the flat array. for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { - LNK_SymbolHashTrieChunkList symbol_chunks = task->symtab->chunks[lane]; - Arena *public_symbol_arena = public_symbol_arenas [lane]; - Arena *public_symbol_node_arena = public_symbol_node_arenas[lane]; - CV_SymbolList *public_symbol_list = &public_symbols [lane]; + LNK_SymbolHashTrieChunkList symbol_chunks = task->symtab->chunks[lane]; + Arena *public_symbol_arena = public_symbol_arenas[lane]; + CV_Symbol *vals = push_array_no_zero(scratch.arena, CV_Symbol, public_symbol_node_counts[lane]); + U64 val_count = 0; for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { for EachIndex(i, chunk->count) { LNK_Symbol *symbol = chunk->v[i].symbol; @@ -4172,59 +4192,73 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } - CV_Pub32Flags flags = COFF_SymbolType_IsFunc(symbol_parsed.type) ? CV_Pub32Flag_Function : 0; - ISectOff sc = lnk_sc_from_symbol(symbol); - CV_Symbol pub_symbol = cv_make_pub32(public_symbol_arena, flags, safe_cast_u32(sc.off), safe_cast_u16(sc.isect), symbol->name); - cv_symbol_list_push(public_symbol_node_arena, public_symbol_list, pub_symbol); + CV_Pub32Flags flags = COFF_SymbolType_IsFunc(symbol_parsed.type) ? CV_Pub32Flag_Function : 0; + ISectOff sc = lnk_sc_from_symbol(symbol); + Assert(val_count < public_symbol_node_counts[lane]); + vals[val_count++] = cv_make_pub32(public_symbol_arena, flags, safe_cast_u32(sc.off), safe_cast_u16(sc.isect), symbol->name); } } + public_symbol_vals [lane] = vals; + public_symbol_lane_counts[lane] = val_count; } barrier_wait(tp->barrier); // hash public symbols for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { - U64 hash_idx = 0; - U32 *hashes = push_array(scratch.arena, U32, public_symbols[lane].count); - for EachNode(n, CV_SymbolNode, public_symbols[lane].first) { - String8 name = cv_name_from_symbol(n->data.kind, n->data.data); - hashes[hash_idx++] = gsi_hash(gsi, name); + U64 lane_count = public_symbol_lane_counts[lane]; + CV_Symbol *vals = public_symbol_vals[lane]; + U32 *hashes = push_array(scratch.arena, U32, lane_count); + for EachIndex(k, lane_count) { + String8 name = cv_name_from_symbol(vals[k].kind, vals[k].data); + hashes[k] = gsi_hash(gsi, name); } public_symbol_hashes[lane] = hashes; } barrier_wait(tp->barrier); - // flatten the per-worker symbol lists (in worker order, matching the old serial walk) into one - // global-order node/hash array, so the sharded insert below can walk it without racing on the - // list links - U64 public_symbol_total_count = 0; - U64 *public_symbol_offsets = 0; // [fixed_lane_count] - CV_SymbolNode **public_symbol_flat_nodes = 0; // [public_symbol_total_count] - U32 *public_symbol_flat_hashes = 0; // [public_symbol_total_count] + // flatten the per-worker symbol arrays (in worker order, matching the old serial walk) into + // one global-order value/hash array, so the sharded insert below can walk it + U64 public_symbol_total_count = 0; + U64 *public_symbol_offsets = 0; // [fixed_lane_count] + CV_Symbol *public_symbol_flat_vals = 0; // [public_symbol_total_count] + U32 *public_symbol_flat_hashes = 0; // [public_symbol_total_count] if (task_id == 0) { U64 *list_counts = push_array_no_zero(scratch.arena, U64, fixed_lane_count); - for EachIndex(i, fixed_lane_count) { list_counts[i] = public_symbols[i].count; } + for EachIndex(i, fixed_lane_count) { list_counts[i] = public_symbol_lane_counts[i]; } public_symbol_offsets = offsets_from_counts_array_u64(scratch.arena, list_counts, fixed_lane_count); public_symbol_total_count = sum_array_u64(fixed_lane_count, list_counts); - public_symbol_flat_nodes = push_array_no_zero(scratch.arena, CV_SymbolNode *, public_symbol_total_count); + public_symbol_flat_vals = push_array_no_zero(scratch.arena, CV_Symbol, public_symbol_total_count); public_symbol_flat_hashes = push_array_no_zero(scratch.arena, U32, public_symbol_total_count); } tp_broadcast(&public_symbol_total_count); tp_broadcast(&public_symbol_offsets); - tp_broadcast(&public_symbol_flat_nodes); + tp_broadcast(&public_symbol_flat_vals); tp_broadcast(&public_symbol_flat_hashes); for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { U64 cursor = public_symbol_offsets[lane]; - U64 k = 0; - for (CV_SymbolNode *curr = public_symbols[lane].first; curr != 0; curr = curr->next, k += 1) { - public_symbol_flat_nodes [cursor] = curr; + U64 lane_count = public_symbol_lane_counts[lane]; + for EachIndex(k, lane_count) { + public_symbol_flat_vals [cursor] = public_symbol_vals [lane][k]; public_symbol_flat_hashes[cursor] = public_symbol_hashes[lane][k]; cursor += 1; } } barrier_wait(tp->barrier); + // size buckets up front so each one reallocs at most once for this wave (arena pushes are + // single-threaded on task 0; sizing has no determinism impact) + if (task_id == 0) { + PDB_GsiContext *pub_gsi = psi->gsi; + U64 *bucket_adds = push_array(scratch.arena, U64, pub_gsi->bucket_count); + for EachIndex(i, public_symbol_total_count) { bucket_adds[public_symbol_flat_hashes[i] % pub_gsi->bucket_count] += 1; } + for EachIndex(bucket_idx, pub_gsi->bucket_count) { + if (bucket_adds[bucket_idx]) { gsi_reserve(pub_gsi, bucket_idx, bucket_adds[bucket_idx]); } + } + } + barrier_wait(tp->barrier); + // insert public symbols into PSI, sharded by bucket range (single owner per bucket, inserts in - // global order -> per-chain order identical to a serial loop for any worker count) + // global order -> per-bucket order identical to a serial loop for any worker count) { PDB_GsiContext *pub_gsi = psi->gsi; U64 shard_min = (task_id * pub_gsi->bucket_count) / tp->worker_count; @@ -4232,7 +4266,9 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) for EachIndex(i, public_symbol_total_count) { U64 bucket_idx = public_symbol_flat_hashes[i] % pub_gsi->bucket_count; if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } - cv_symbol_list_push_node(&pub_gsi->bucket_arr[bucket_idx], public_symbol_flat_nodes[i]); + PDB_GsiSymbolBucket *bucket = &pub_gsi->bucket_arr[bucket_idx]; + bucket->v[bucket->count] = public_symbol_flat_vals[i]; + bucket->count += 1; } } barrier_wait(tp->barrier); diff --git a/src/linker/pdb_ext/pdb_builder.c b/src/linker/pdb_ext/pdb_builder.c index 6acbf50e9..b4f594126 100644 --- a/src/linker/pdb_ext/pdb_builder.c +++ b/src/linker/pdb_ext/pdb_builder.c @@ -1694,7 +1694,7 @@ gsi_alloc(void) gsi->word_size = PDB_GSI_V70_WORD_SIZE; gsi->symbol_align = PDB_GSI_V70_SYMBOL_ALIGN; gsi->bucket_count = PDB_GSI_V70_BUCKET_COUNT; - gsi->bucket_arr = push_array(arena, CV_SymbolList, gsi->bucket_count); + gsi->bucket_arr = push_array(arena, PDB_GsiSymbolBucket, gsi->bucket_count); ProfEnd(); return gsi; } @@ -2074,9 +2074,9 @@ THREAD_POOL_TASK_FUNC(gsi_size_buckets_task) { PDB_GsiSerializeSymbolsTask *task = raw_task; U64 bucket_idx = task->bucket_base + task_id; - CV_SymbolList *bucket_list = &task->bucket_arr[bucket_idx]; - for (CV_SymbolNode *node = bucket_list->first; node != 0; node = node->next) { - task->bucket_size_arr[bucket_idx] += cv_size_from_symbol(&node->data, task->symbol_align); + PDB_GsiSymbolBucket *bucket = &task->bucket_arr[bucket_idx]; + for EachIndex(i, bucket->count) { + task->bucket_size_arr[bucket_idx] += cv_size_from_symbol(&bucket->v[i], task->symbol_align); } } @@ -2148,11 +2148,10 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_pub32) PDB_GsiSerializeSymbolsTask *task = raw_task; U64 bucket_idx = task->bucket_base + task_id; - CV_SymbolList bucket = task->bucket_arr[bucket_idx]; + PDB_GsiSymbolBucket bucket = task->bucket_arr[bucket_idx]; - CV_Symbol **symbol_arr = push_array(scratch.arena, CV_Symbol *, bucket.count); - U64 symbol_arr_count = 0; - for EachNode(n, CV_SymbolNode, bucket.first) { symbol_arr[symbol_arr_count++] = &n->data; } + CV_Symbol **symbol_arr = push_array(scratch.arena, CV_Symbol *, bucket.count); + for EachIndex(i, bucket.count) { symbol_arr[i] = &bucket.v[i]; } // sort symbols within bucket radsort(symbol_arr, bucket.count, gsi_pub_symbol_is_before); @@ -2198,13 +2197,10 @@ THREAD_POOL_TASK_FUNC(gsi_serialize_symbols_task) PDB_GsiSerializeSymbolsTask *task = raw_task; U64 bucket_idx = task->bucket_base + task_id; - CV_SymbolList bucket = task->bucket_arr[bucket_idx]; + PDB_GsiSymbolBucket bucket = task->bucket_arr[bucket_idx]; - CV_Symbol **symbol_arr = push_array(scratch.arena, CV_Symbol *, bucket.count); - { - U64 i = 0; - for EachNode(n, CV_SymbolNode, bucket.first) { symbol_arr[i++] = &n->data; } - } + CV_Symbol **symbol_arr = push_array(scratch.arena, CV_Symbol *, bucket.count); + for EachIndex(i, bucket.count) { symbol_arr[i] = &bucket.v[i]; } // sort symbols within bucket radsort(symbol_arr, bucket.count, gsi_symbol_is_before); @@ -2309,7 +2305,7 @@ gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, MSF_Context *msf ProfBegin("Write Bitmap & Record Offsets"); for (U64 bucket_idx = 0, hash_idx = 0; bucket_idx < gsi->bucket_count; bucket_idx += 1) { // set bit for each occupied bucket - CV_SymbolList bucket_list = gsi->bucket_arr[bucket_idx]; + PDB_GsiSymbolBucket bucket_list = gsi->bucket_arr[bucket_idx]; if (bucket_list.count) { U64 word_idx = bucket_idx / gsi->word_size; Assert(word_idx < bitmap_count); @@ -2375,28 +2371,40 @@ gsi_hash(PDB_GsiContext *gsi, String8 input) } internal void -gsi_push_(PDB_GsiContext *gsi, U32 hash, CV_SymbolNode *node) +gsi_reserve(PDB_GsiContext *gsi, U64 bucket_idx, U64 additional) +{ + PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; + if (bucket->count + additional > bucket->cap) { + U64 new_cap = bucket->count + additional; + CV_Symbol *new_v = push_array_no_zero(gsi->arena, CV_Symbol, new_cap); + MemoryCopy(new_v, bucket->v, sizeof(bucket->v[0]) * bucket->count); + bucket->v = new_v; + bucket->cap = new_cap; + } +} + +internal CV_Symbol * +gsi_push_(PDB_GsiContext *gsi, U32 hash, CV_Symbol *symbol) { - U64 bucket_idx = hash % gsi->bucket_count; - CV_SymbolList *list = &gsi->bucket_arr[bucket_idx]; - cv_symbol_list_push_node(list, node); + U64 bucket_idx = hash % gsi->bucket_count; + PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; + if (bucket->count == bucket->cap) { + // rare path: bulk inserters reserve up front via gsi_reserve + gsi_reserve(gsi, bucket_idx, Max(bucket->cap, 8)); + } + CV_Symbol *dst = &bucket->v[bucket->count]; + *dst = *symbol; + bucket->count += 1; gsi->symbol_count += 1; + return dst; } -internal CV_SymbolNode * +internal CV_Symbol * gsi_push(PDB_GsiContext *gsi, CV_Symbol *symbol) { String8 name = cv_name_from_symbol(symbol->kind, symbol->data); U32 hash = gsi_hash(gsi, name); - - CV_SymbolNode *node = push_array_no_zero(gsi->arena, CV_SymbolNode, 1); - node->next = 0; - node->prev = 0; - node->data = *symbol; - - gsi_push_(gsi, hash, node); - - return node; + return gsi_push_(gsi, hash, symbol); } internal @@ -2429,7 +2437,7 @@ gsi_push_many_arr(TP_Context *tp, PDB_GsiContext *gsi, U64 count, CV_SymbolNode ProfEnd(); for (U64 i = 0; i < count; ++i) { - gsi_push_(gsi, task.hashes[i], symbols[i]); + gsi_push_(gsi, task.hashes[i], &symbols[i]->data); } scratch_end(scratch); @@ -2444,28 +2452,24 @@ gsi_push_many_list(PDB_GsiContext *gsi, U64 count, U32 *hash_arr, CV_SymbolList U64 hash_idx = 0; for (CV_SymbolNode *curr = list->first, *next = 0; curr != 0; curr = next, ++hash_idx) { next = curr->next; - - curr->prev = 0; - curr->next = 0; - - gsi_push_(gsi, hash_arr[hash_idx], curr); + gsi_push_(gsi, hash_arr[hash_idx], &curr->data); } MemoryZeroStruct(list); } -internal CV_SymbolNode * +internal CV_Symbol * gsi_search(PDB_GsiContext *gsi, CV_Symbol *symbol) { String8 name = cv_name_from_symbol(symbol->kind, symbol->data); U32 hash = gsi_hash(gsi, name); U64 ibucket = hash % gsi->bucket_count; - CV_SymbolList bucket_list = gsi->bucket_arr[ibucket]; - for (CV_SymbolNode *node = bucket_list.first; node != 0; node = node->next) { - String8 that_name = cv_name_from_symbol(node->data.kind, node->data.data); + PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[ibucket]; + for EachIndex(i, bucket->count) { + String8 that_name = cv_name_from_symbol(bucket->v[i].kind, bucket->v[i].data); if (str8_match(name, that_name, 0)) { - return node; + return &bucket->v[i]; } } @@ -2532,12 +2536,12 @@ psi_release(PDB_PsiContext *psi) ProfEnd(); } -internal CV_SymbolNode * +internal CV_Symbol * psi_push(PDB_PsiContext *psi, CV_Pub32Flags flags, U32 offset, U16 isect, String8 name) { CV_Symbol pub = cv_make_pub32(psi->arena, flags, offset, isect, name); - CV_SymbolNode *node = gsi_push(psi->gsi, &pub); - return node; + CV_Symbol *symbol = gsi_push(psi->gsi, &pub); + return symbol; } //////////////////////////////// diff --git a/src/linker/pdb_ext/pdb_builder.h b/src/linker/pdb_ext/pdb_builder.h index 667e658d6..4816c22fc 100644 --- a/src/linker/pdb_ext/pdb_builder.h +++ b/src/linker/pdb_ext/pdb_builder.h @@ -176,6 +176,15 @@ typedef enum #define PDB_GSI_V70_BITMAP_COUNT ((PDB_GSI_V70_BUCKET_COUNT / PDB_GSI_V70_WORD_SIZE) + 1) #define PDB_GSI_V70_BITMAP_SIZE (PDB_GSI_V70_BITMAP_COUNT * sizeof(U32)) +// per-bucket array of CV_Symbol values (replaces linked lists of 48-byte +// CV_SymbolNode; cuts PDB-phase commit and pointer-chase on serialize) +typedef struct PDB_GsiSymbolBucket +{ + U64 count; + U64 cap; + CV_Symbol *v; +} PDB_GsiSymbolBucket; + typedef struct PDB_GsiContext { Arena *arena; @@ -183,7 +192,7 @@ typedef struct PDB_GsiContext U64 symbol_align; U64 bucket_count; U64 symbol_count; - CV_SymbolList *bucket_arr; + PDB_GsiSymbolBucket *bucket_arr; } PDB_GsiContext; typedef struct PDB_GsiSortRecord @@ -211,7 +220,7 @@ typedef struct PDB_GsiSerializeSymbolsTask { U64 symbol_data_base; U64 symbol_align; - CV_SymbolList *bucket_arr; + PDB_GsiSymbolBucket *bucket_arr; U64 *bucket_size_arr; U64 *bucket_off_arr; // absolute offsets within the full symbol payload U8 *buffer; // window backing: holds buckets [bucket_base, bucket_base+N) @@ -369,10 +378,11 @@ internal void gsi_release(PDB_GsiContext *gsi); internal void gsi_write_build_result(TP_Context *tp, PDB_GsiBuildResult build, MSF_Context *msf, MSF_StreamNumber sn, MSF_StreamNumber symbols_sn); internal PDB_GsiBuildResult gsi_build_ex(TP_Context *tp, Arena *arena, PDB_GsiContext *gsi, MSF_Context *msf, MSF_StreamNumber symbols_sn, U64 symbol_data_base, B32 is_pub32, U64 msf_page_size); internal U32 gsi_hash(PDB_GsiContext *gsi, String8 input); -internal CV_SymbolNode * gsi_push(PDB_GsiContext *gsi, CV_Symbol *symbol); +internal void gsi_reserve(PDB_GsiContext *gsi, U64 bucket_idx, U64 additional); +internal CV_Symbol * gsi_push(PDB_GsiContext *gsi, CV_Symbol *symbol); internal void gsi_push_many_arr(TP_Context *tp, PDB_GsiContext *gsi, U64 count, CV_SymbolNode **symbol_arr); internal void gsi_push_many_list(PDB_GsiContext *gsi, U64 count, U32 *hash_arr, CV_SymbolList *list); -internal CV_SymbolNode * gsi_search(PDB_GsiContext *gsi, CV_Symbol *symbol); +internal CV_Symbol * gsi_search(PDB_GsiContext *gsi, CV_Symbol *symbol); //////////////////////////////// // PSI @@ -380,7 +390,7 @@ internal CV_SymbolNode * gsi_search(PDB_GsiContext *gsi, CV_Symbol *symbol); internal PDB_PsiContext * psi_alloc(void); internal void psi_build(TP_Context *tp, PDB_PsiContext *psi, MSF_Context *msf, MSF_StreamNumber sn, MSF_StreamNumber symbols_sn); internal void psi_release(PDB_PsiContext *psi); -internal CV_SymbolNode * psi_push(PDB_PsiContext *psi, CV_Pub32Flags flags, U32 offset, U16 isect, String8 name); +internal CV_Symbol * psi_push(PDB_PsiContext *psi, CV_Pub32Flags flags, U32 offset, U16 isect, String8 name); // TODO: //internal CV_Symbol psi_neareset_symbol(PDB_PsiContext *psi, U16 isect, U32 off); From af4aea48fc518a40c1f0d1c9db4abdf010d8d541 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 12 Aug 2026 12:07:32 -0700 Subject: [PATCH 055/121] linker: release patched debug-section copies before the GSI/PSI build -- patch-on-copy privates (~11.6GB on the FN editor DLL) stayed committed to process exit, sitting under the link's commit peak in the globals/GSI window. Copies now live on dedicated per-worker arenas (free-list block reuse keeps the patch pass at zero fresh-page cost); global symbol records materialize their payload at bucket fill (proc-refs and publics already did); the /names string-hash winners rehome into a blob and module source-file lists repoint at it, so no PDB consumer aliases the copies; then the copy arenas release wholesale right before Build GSI and PSI. Gated off under /PDBSTRIPPED, whose second build re-reads the copies. FN editor DLL (frozen-input rig): peak commit 53.4 -> 48.6GB, wall unchanged, output DLL+PDB byte-identical, deterministic. Co-Authored-By: Claude Fable 5 --- src/linker/lnk.c | 14 +++-- src/linker/lnk.h | 7 +++ src/linker/lnk_debug_info.c | 102 ++++++++++++++++++++++++++++++++++-- src/linker/lnk_debug_info.h | 8 +++ src/linker/lnk_obj.c | 15 ++++++ src/linker/lnk_obj.h | 1 + 6 files changed, 141 insertions(+), 6 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 771c5bfff..ee31804a3 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -4091,7 +4091,7 @@ lnk_icf_obj_file_chksms_scan(LNK_Obj *obj) // Memoized per obj: leaders are shared across many follower objs, so without the memo the // scan reruns once per (follower obj x leader switch). The result slices the immutable -// obj->data mapping, so the racy fill is idempotent (every worker writes identical bytes); +// obj->coff.data mapping, so the racy fill is idempotent (every worker writes identical bytes); // the init flag is published last. internal String8 lnk_icf_obj_file_chksms(LNK_Obj *obj) @@ -5004,7 +5004,7 @@ THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) obj->section_data_copies = push_array(arena, String8, obj->coff.sections.count_no_null + 1); } String8 src = str8_substr(obj->coff.data, section_frange); - U8 *copy = push_array_no_zero(arena, U8, src.size); + U8 *copy = push_array_no_zero(g_sect_copy_arenas[worker_id], U8, src.size); MemoryCopy(copy, src.str, src.size); obj->section_data_copies[it.v.section_number] = str8(copy, src.size); section_data = obj->section_data_copies[it.v.section_number]; @@ -6541,8 +6541,16 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT // patch relocs { + // dedicated per-worker arenas for the patched debug-section copies: free-list + // block reuse keeps the pages warm (a raw reserve+commit per copy paid ~11.6GB of + // fresh zero-page faults per link at FN scale), and lnk_build_pdb hands the whole + // set back with arena_release after the last $S reader + g_sect_copy_arena_count = tp->worker_count; + g_sect_copy_arenas = push_array(arena->v[0], Arena *, g_sect_copy_arena_count); + for EachIndex(i, g_sect_copy_arena_count) { g_sect_copy_arenas[i] = arena_alloc(.name = "SECT_DATA_COPIES"); } + LNK_ObjRelocPatcher task = { .image_data = image_data, .objs = objs, .image_base = pe.image_base, .image_section_table = image_section_table }; - tp_for_parallel_prof(tp, arena, objs_count, lnk_obj_reloc_patcher, &task, "Patch Relocs"); + tp_for_parallel_prof(tp, arena, objs_count, lnk_obj_reloc_patcher, &task, "Patch Relocs"); // arena: the per-obj section_data_copies String8 tables } // patch load config diff --git a/src/linker/lnk.h b/src/linker/lnk.h index d566d9bba..9728e5339 100644 --- a/src/linker/lnk.h +++ b/src/linker/lnk.h @@ -233,6 +233,13 @@ typedef struct COFF_SectionHeader **image_section_table; } LNK_ObjRelocPatcher; +// Per-worker arenas backing the patched debug-section copies (lnk_obj_reloc_patcher +// pushes on g_sect_copy_arenas[worker_id]; arena_alloc recycles free-list blocks so the +// pages are warm). Released wholesale in lnk_build_pdb after the last $S reader +// (module write + global-record materialize), gated off when /PDBSTRIPPED follows. +global Arena **g_sect_copy_arenas = 0; +global U64 g_sect_copy_arena_count = 0; + typedef struct { U64 page_size; diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index ef43ba846..5a8d48c18 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3954,6 +3954,34 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } barrier_wait(tp->barrier); + // size per-lane payload arenas for materializing the global records (mirrors the + // proc-ref pattern below): bucket CV_Symbol values must not point into the objs' + // patched debug-section copies -- those are dropped right after this pass in + // lnk_build_pdb, and GSI serialization reads the record bytes after that. Each lane owns the + // bucket shard [shard_min, shard_max) and copies exactly the records it inserts. + U64 *global_payload_sizes = 0; // [worker_count] + Arena **global_payload_arenas = 0; // [worker_count] + if (task_id == 0) { + global_payload_sizes = push_array(scratch.arena, U64, tp->worker_count); + } + tp_broadcast(&global_payload_sizes); + { + U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; + U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; + U64 payload_size = 0; + for EachIndex(i, symbol_count) { + U64 bucket_idx = symbol_hashes[i] % gsi->bucket_count; + if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } + payload_size += AlignPow2(cv_raw_from_symbol(symbol_arr[i]).size, sizeof(void *)); + } + global_payload_sizes[task_id] = payload_size; + } + barrier_wait(tp->barrier); + if (task_id == 0) { + global_payload_arenas = alloc_arena_many(gsi->arena, tp->worker_count, global_payload_sizes); + } + tp_broadcast(&global_payload_arenas); + // size buckets up front so each one reallocs at most once for this wave (arena pushes are // single-threaded on task 0; sizing has no determinism impact) if (task_id == 0) { @@ -3971,14 +3999,21 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) // so per-bucket order (which is serialized into the PDB) is byte-identical to a serial loop, // for any worker count -- no locks, no atomics. { - U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; - U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; + U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; + U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; + Arena *payload_arena = global_payload_arenas[task_id]; for EachIndex(i, symbol_count) { U64 bucket_idx = symbol_hashes[i] % gsi->bucket_count; if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; CV_Symbol *dst = &bucket->v[bucket->count]; - *dst = cv_symbol_from_ptr(symbol_arr[i]); + // materialize: copy the raw record out of the obj's debug-section copy so the + // bucket value survives the copy release right after this pass in lnk_build_pdb + // (byte-identical copy; insert order and all sort keys unchanged) + String8 raw = cv_raw_from_symbol(symbol_arr[i]); + U8 *copy = push_array_no_zero(payload_arena, U8, raw.size); + MemoryCopy(copy, raw.str, raw.size); + *dst = cv_symbol_from_ptr(copy); // deterministic same-name tie-break for the per-bucket sort in gsi_serialize_symbols_task // (gsi_symbol_is_before compares name -> offset -> kind -> data bytes): key on the content // hash of the full raw record instead of the compacted deduper slot index. Slot order is @@ -4114,6 +4149,24 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } ProfEnd(); + // the proc-ref build above was the LAST $S reader (publics come from the symbol + // table, GSI/PSI serialize reads the materialized payload copies): drop the patched + // debug-section copies here, before the publics/GSI staging stacks on top of them -- + // releasing after the whole phase kept ~11.6GB overlapping this window on the FN + // editor DLL. Same /PDBSTRIPPED gate as elsewhere (its second build re-reads $S). + barrier_wait(tp->barrier); + if (task_id == 0 && task->free_sect_copies) { + ProfScope("Release Sect Data Copies") { + for EachIndex(obj_idx, task->cv->obj_count) { + lnk_obj_drop_section_data_copies(task->cv->obj_arr[obj_idx]); + } + for EachIndex(i, g_sect_copy_arena_count) { + if (g_sect_copy_arenas[i] != 0) { arena_release(g_sect_copy_arenas[i]); g_sect_copy_arenas[i] = 0; } + } + } + } + barrier_wait(tp->barrier); + ProfBegin("Public Symbols"); { // FAIR-SHARE: task->symtab->chunks is a FIXED [symtab->arena->count] partition built at @@ -4279,6 +4332,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) scratch_end(scratch); } + internal U64 lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8Node *buf, U64 *buf_pos) { @@ -4514,6 +4568,18 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) // collect mod source files String8List source_file_list = str8_split_by_string_chars(string_arenas[task_id], string_table, str8_lit("\0"), 0); + if (task->free_sect_copies) { + // the split nodes alias the string table inside the obj's debug-section copy + // (str8_split does not copy) and DBI file-info hashes these bytes after this + // pass releases the copies. Every piece is present in string_ht -- the dedup + // task split the very same table -- and its bucket bytes were rehomed to a + // surviving blob before the strtab add, so repoint at the bucket's copy. + for EachNode(n, String8Node, source_file_list.first) { + CV_StringBucket *bucket = cv_string_hash_table_lookup(task->string_ht, n->string); + Assert(bucket != 0); + if (bucket != 0) { n->string = bucket->string; } + } + } str8_list_concat_in_place(&mod->source_file_list, &source_file_list); temp_end(temp); @@ -5162,6 +5228,13 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config task.output = output_ptr; + // patched debug-section copies (obj->section_data_copies, GB-class at FN scale) can be + // handed back per-obj during the module-write pass -- UNLESS a /PDBSTRIPPED build + // follows: its pre-build stripping loop re-walks cv->debug_s_arr Symbols (which alias + // the copies) after this build returns. The RDI converter is safe (it reads the PDB + // artifact pages, not obj debug sections). + task.free_sect_copies = (config->pdb_stripped_name.size == 0); + PDB_BuildHooks build_hooks = {0}; if (output_ptr != 0) { build_hooks.stream_finalize = lnk_pdb_output_finalize_stream; @@ -5195,6 +5268,28 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config ProfBegin("Merge String Tables"); task.string_ht = cv_dedup_string_tables(tp_arena, tp, cv->obj_count, cv->debug_s_arr); cv_string_hash_table_assign_buffer_offsets(tp, task.string_ht); + + // the deduped buckets alias the objs' patched debug-section copies (the dedup task + // splits each raw string table in place), and both /names (pdb_strtab_build memcpys + // bucket bytes at serialize time in pdb_build_dbi_info) and DBI file-info hash these + // bytes AFTER the module-write pass releases the copies -- rehome the winning buckets + // into their own blob first (total = the deduped /names payload, tiny next to the + // copies being released). Offsets are already assigned; bytes are identical, so the + // /names stream and every recorded offset are unchanged. + if (task.free_sect_copies && task.string_ht.total_string_size > 0) { + ProfBegin("Materialize String Table Bytes"); + U8 *blob = push_array_no_zero(tp_arena->v[0], U8, task.string_ht.total_string_size); + U64 cursor = 0; + for EachIndex(bucket_idx, task.string_ht.bucket_cap) { + CV_StringBucket *bucket = task.string_ht.buckets[bucket_idx]; + if (bucket == 0) { continue; } + Assert(cursor + bucket->string.size <= task.string_ht.total_string_size); + MemoryCopy(blob + cursor, bucket->string.str, bucket->string.size); + bucket->string.str = blob + cursor; + cursor += bucket->string.size; + } + ProfEnd(); + } ProfEnd(); lnk_summary_phase_end(LNK_SummaryPhase_PdbStr); @@ -5258,6 +5353,7 @@ ProfScope("Move Global Symbols") tp_barrier_end(tp); lnk_log(LNK_Log_Timers, "[pdb] move global symbols in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); } + ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); if (output_ptr != 0) { lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->publics_sn); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index ad17a8747..dce9f8004 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -270,6 +270,14 @@ typedef struct U64 *image_section_file_section_numbers; PDB_DbiSCArray *sc_arrays; // [obj_count] struct LNK_PdbOutput *output; // when non-null, module streams enqueue to the background writer as they complete + + // when set, lnk_build_pdb drops every obj's patched debug-section copies + // (LNK_Obj.section_data_copies) and releases the SECT_DATA_COPIES arenas right after + // "Move Global Symbols" (the last $S reader); also gates the /names bucket rehome and + // the mod->source_file_list repoint that decouple the string consumers from the + // copies. Must be 0 when a /PDBSTRIPPED build follows -- it re-walks cv->debug_s_arr + // after lnk_build_pdb. + B32 free_sect_copies; } LNK_BuildPdb; typedef struct diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index aa743be3b..07f1fe2d1 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -854,6 +854,21 @@ lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx) return result; } +// Drop the obj's patched debug-section copies: zero each entry so +// lnk_obj_section_data_from_number +// falls back to the (still-mapped) input view. The copy BYTES live on the shared +// per-worker SECT_DATA_COPIES arenas (see lnk_obj_reloc_patcher) and are handed back +// wholesale via arena_release at the caller; only call once every reader of the patched +// bytes is done. Idempotent. +internal void +lnk_obj_drop_section_data_copies(LNK_Obj *obj) +{ + if (obj->section_data_copies == 0) { return; } + for (U32 section_number = 1; section_number <= obj->coff.sections.count_no_null; section_number += 1) { + obj->section_data_copies[section_number] = str8_zero(); + } +} + internal THREAD_POOL_TASK_FUNC(lnk_collect_obj_chunks_task) { diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index 8c9f1f06d..99e11436c 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -285,6 +285,7 @@ internal COFF_SectionHeader * lnk_coff_section_header_from_section_number(LNK_Ob internal String8 lnk_obj_section_data_from_number(LNK_Obj *obj, U64 section_number); internal U64 lnk_obj_foff_from_section_data_ptr(LNK_Obj *obj, void *ptr); internal String8 lnk_obj_section_name_from_section_number(LNK_Obj *obj, U64 section_number); +internal void lnk_obj_drop_section_data_copies(LNK_Obj *obj); internal LNK_ObjSection lnk_obj_section_from_section_number(LNK_Obj *obj, U64 section_number); internal COFF_RelocArray lnk_coff_relocs_from_section_header(LNK_Obj *obj, COFF_SectionHeader *section_header); internal String8 lnk_coff_string_table_from_obj(LNK_Obj *obj); From f6b181af0b8238be540339f14e8b9d08d8ef8766 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 12 Aug 2026 14:14:13 -0700 Subject: [PATCH 056/121] linker: ring P1.1 -- canonicalize .debug$S subsection access + dormant provenance -- every data_list read outside codeview_ext now goes through cv_ accessors (new: cv_total_sub_section_size_from_debug_s, cv_sub_section_ptr_from_debug_s, cv_debug_s_push_synthetic_sub_section, cv_debug_s_concat_sub_section_in_place), and cv_debug_s_from_data records each subsection node's (section, offset, size) provenance in a parallel list (asserted consistent in debug builds, nothing reads it yet). Groundwork for streaming debug sections through a reused window: the representation swap now has one owner. Output byte-identical, deterministic, lyra 9/9. Co-Authored-By: Claude Fable 5 --- src/linker/codeview_ext/codeview.c | 111 ++++++++++++++++++++++- src/linker/codeview_ext/codeview.h | 39 +++++++- src/linker/lnk.c | 8 +- src/linker/lnk_debug_helper.c | 4 +- src/linker/lnk_debug_info.c | 24 +++-- src/linker/lnk_debug_info.h | 1 + src/linker/lnk_obj.c | 81 ++++++++++------- src/linker/lnk_obj.h | 3 +- src/torture/tests/torture_linker_tests.c | 4 +- 9 files changed, 222 insertions(+), 53 deletions(-) diff --git a/src/linker/codeview_ext/codeview.c b/src/linker/codeview_ext/codeview.c index 11592f7ac..afcba22b5 100644 --- a/src/linker/codeview_ext/codeview.c +++ b/src/linker/codeview_ext/codeview.c @@ -559,7 +559,42 @@ cv_obj_info_from_symbol(CV_Symbol symbol) //~ .debug$S helpers internal void -cv_debug_s_from_data_c13_(Arena *arena, CV_DebugS *debug_s, String8 raw_debug_s) +cv_debug_s_prov_list_push_node(CV_DebugSProvList *list, CV_DebugSProvNode *node) +{ + SLLQueuePush(list->first, list->last, node); + list->count += 1; +} + +internal void +cv_debug_s_prov_list_push(Arena *arena, CV_DebugSProvList *list, U64 off, U64 size, U32 sect_idx, B32 is_synthetic) +{ + CV_DebugSProvNode *node = push_array_no_zero(arena, CV_DebugSProvNode, 1); + node->next = 0; + node->off = off; + node->size = size; + node->sect_idx = sect_idx; + node->is_synthetic = is_synthetic; + cv_debug_s_prov_list_push_node(list, node); +} + +internal void +cv_debug_s_prov_list_concat_in_place(CV_DebugSProvList *dst, CV_DebugSProvList *src) +{ + if (src->first != 0) { + if (dst->last != 0) { + dst->last->next = src->first; + dst->last = src->last; + } else { + dst->first = src->first; + dst->last = src->last; + } + dst->count += src->count; + MemoryZeroStruct(src); + } +} + +internal void +cv_debug_s_from_data_c13_(Arena *arena, CV_DebugS *debug_s, String8 raw_debug_s, U64 prov_base_off) { for (U64 cursor = 0; cursor + sizeof(CV_C13SubSectionHeader) <= raw_debug_s.size; ) { // read header @@ -575,6 +610,12 @@ cv_debug_s_from_data_c13_(Arena *arena, CV_DebugS *debug_s, String8 raw_debug_s) Rng1U64 sub_sect_range = r1u64(cursor, cursor + header.size); String8 sub_sect_data = str8_substr(raw_debug_s, sub_sect_range); str8_list_push(arena, sub_sect_list, sub_sect_data); + + // record provenance: slice position within the raw input (sub_sect_data.size, not + // header.size -- str8_substr clamps a header that overruns the section, and provenance + // must describe the bytes the node actually holds) + cv_debug_s_prov_list_push(arena, &debug_s->prov_list[sub_sect_idx], prov_base_off + cursor, sub_sect_data.size, CV_DebugSProvSect_Nil, 0); + Assert(debug_s->prov_list[sub_sect_idx].last->size == sub_sect_data.size); } // advance @@ -595,7 +636,7 @@ internal CV_DebugS cv_debug_s_from_data_c13(Arena *arena, String8 raw_debug_s) { CV_DebugS debug_s = {0}; - cv_debug_s_from_data_c13_(arena, &debug_s, raw_debug_s); + cv_debug_s_from_data_c13_(arena, &debug_s, raw_debug_s, 0); return debug_s; } @@ -608,7 +649,9 @@ cv_debug_s_from_data(Arena *arena, String8 raw_debug_s) switch (sig) { case CV_Signature_C13: { String8 raw_debug_s_past_sig = str8_substr(raw_debug_s, r1u64(sizeof(sig), raw_debug_s.size)); - result = cv_debug_s_from_data_c13(arena, raw_debug_s_past_sig); + // prov base = sizeof(sig): provenance offsets are relative to the raw input handed to + // THIS function, which still carries the signature + cv_debug_s_from_data_c13_(arena, &result, raw_debug_s_past_sig, sizeof(sig)); } break; case CV_Signature_C6: { Assert(!"TODO: handle C6"); @@ -630,6 +673,7 @@ cv_debug_s_concat_in_place(CV_DebugS *dst, CV_DebugS *src) { for (U64 sub_sect_idx = 0; sub_sect_idx < ArrayCount(dst->data_list); sub_sect_idx += 1) { str8_list_concat_in_place(&dst->data_list[sub_sect_idx], &src->data_list[sub_sect_idx]); + cv_debug_s_prov_list_concat_in_place(&dst->prov_list[sub_sect_idx], &src->prov_list[sub_sect_idx]); } } @@ -749,6 +793,67 @@ cv_file_chksms_from_debug_s(CV_DebugS debug_s) return file_chksms; } +internal U64 +cv_total_sub_section_size_from_debug_s(CV_DebugS *debug_s) +{ + U64 total = 0; + for EachElement(i, debug_s->data_list) { total += debug_s->data_list[i].total_size; } + return total; +} + +internal CV_DebugSProvList * +cv_sub_section_prov_ptr_from_debug_s(CV_DebugS *debug_s, CV_C13SubSectionKind kind) +{ + CV_C13SubSectionIdxKind idx = cv_c13_sub_section_idx_from_kind(kind); + return &debug_s->prov_list[idx]; +} + +internal void +cv_debug_s_push_synthetic_sub_section(Arena *arena, CV_DebugS *debug_s, CV_C13SubSectionKind kind, String8 data) +{ + String8List *data_list = cv_sub_section_ptr_from_debug_s(debug_s, kind); + CV_DebugSProvList *prov_list = cv_sub_section_prov_ptr_from_debug_s(debug_s, kind); + Assert(prov_list->count == data_list->node_count); // keep tracked lists tracked + str8_list_push(arena, data_list, data); + cv_debug_s_prov_list_push(arena, prov_list, 0, data.size, CV_DebugSProvSect_Nil, 1); +} + +internal void +cv_debug_s_concat_sub_section_in_place(CV_DebugS *dst, CV_DebugS *src, CV_C13SubSectionKind kind) +{ + CV_C13SubSectionIdxKind idx = cv_c13_sub_section_idx_from_kind(kind); + str8_list_concat_in_place(&dst->data_list[idx], &src->data_list[idx]); + cv_debug_s_prov_list_concat_in_place(&dst->prov_list[idx], &src->prov_list[idx]); +} + +internal void +cv_debug_s_tag_prov_sect(CV_DebugS *debug_s, U32 sect_idx) +{ + for EachElement(i, debug_s->prov_list) { + for EachNode(n, CV_DebugSProvNode, debug_s->prov_list[i].first) { n->sect_idx = sect_idx; } + } +} + +// debug-only invariant walk: per subsection, provenance is either absent (untracked +// synthetic construction) or one node per data node with matching sizes +internal void +cv_debug_s_validate_prov(CV_DebugS *debug_s) +{ +#if BUILD_DEBUG + for EachElement(i, debug_s->data_list) { + if (debug_s->prov_list[i].count == 0) { Assert(debug_s->prov_list[i].first == 0); continue; } + Assert(debug_s->prov_list[i].count == debug_s->data_list[i].node_count); + CV_DebugSProvNode *prov_n = debug_s->prov_list[i].first; + for EachNode(data_n, String8Node, debug_s->data_list[i].first) { + Assert(prov_n != 0); + if (!prov_n->is_synthetic) { Assert(prov_n->size == data_n->string.size); } + prov_n = prov_n->next; + } + Assert(prov_n == 0); + } +#endif +} + //////////////////////////////// //~ String Table Deduper diff --git a/src/linker/codeview_ext/codeview.h b/src/linker/codeview_ext/codeview.h index f3e5ae9fd..75991b58f 100644 --- a/src/linker/codeview_ext/codeview.h +++ b/src/linker/codeview_ext/codeview.h @@ -216,9 +216,39 @@ typedef struct CV_C13SubSectionList //////////////////////////////// +// Provenance for parsed .debug$S subsection nodes (streaming-ring P1 groundwork). +// One node per data_list node, in lockstep order; records where the slice came from so a +// later representation swap (String8 slices -> offset records) has the origin on hand. +// DORMANT: nothing reads this yet; it must merely stay correct. +// - off/size: slice position within the raw input String8 handed to cv_debug_s_from_data +// (offsets include the leading CV_Signature when the input carried one) +// - sect_idx: 0-based COFF section index the raw input was sliced from; +// CV_DebugSProvSect_Nil until the caller tags it (cv_debug_s_tag_prov_sect) +// - is_synthetic: bytes were synthesized post-parse (not sliced from any input section); +// off/size/sect_idx are meaningless +// A CV_DebugS built without the parser (wholesale list assignment) may leave prov_list +// empty: prov_list[i].count is either 0 (untracked) or data_list[i].node_count (tracked). +#define CV_DebugSProvSect_Nil max_U32 +typedef struct CV_DebugSProvNode +{ + struct CV_DebugSProvNode *next; + U64 off; + U64 size; + U32 sect_idx; + B32 is_synthetic; +} CV_DebugSProvNode; + +typedef struct CV_DebugSProvList +{ + CV_DebugSProvNode *first; + CV_DebugSProvNode *last; + U64 count; +} CV_DebugSProvList; + typedef struct CV_DebugS { - String8List data_list[CV_C13SubSectionIdxKind_COUNT]; + String8List data_list[CV_C13SubSectionIdxKind_COUNT]; + CV_DebugSProvList prov_list[CV_C13SubSectionIdxKind_COUNT]; // parallel to data_list (see above) } CV_DebugS; typedef struct CV_DebugH @@ -390,6 +420,13 @@ internal String8List * cv_sub_section_ptr_from_debug_s(CV_DebugS *debug_s, CV_C1 internal String8List cv_sub_section_from_debug_s(CV_DebugS debug_s, CV_C13SubSectionKind kind); internal String8 cv_string_table_from_debug_s(CV_DebugS debug_s); internal String8 cv_file_chksms_from_debug_s(CV_DebugS debug_s); +internal U64 cv_total_sub_section_size_from_debug_s(CV_DebugS *debug_s); + +internal CV_DebugSProvList * cv_sub_section_prov_ptr_from_debug_s(CV_DebugS *debug_s, CV_C13SubSectionKind kind); +internal void cv_debug_s_push_synthetic_sub_section(Arena *arena, CV_DebugS *debug_s, CV_C13SubSectionKind kind, String8 data); +internal void cv_debug_s_concat_sub_section_in_place(CV_DebugS *dst, CV_DebugS *src, CV_C13SubSectionKind kind); +internal void cv_debug_s_tag_prov_sect(CV_DebugS *debug_s, U32 sect_idx); +internal void cv_debug_s_validate_prov(CV_DebugS *debug_s); //////////////////////////////// //~ .debug$T helpers diff --git a/src/linker/lnk.c b/src/linker/lnk.c index ee31804a3..81185ea5e 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -4123,6 +4123,7 @@ lnk_icf_src_key_from_fn(Arena *scratch, LNK_Obj *obj, U32 fn_sn, String8 chksms) if (child_sn == 0 || chksms.size == 0) { return key; } String8 raw = lnk_obj_section_data_from_number(obj, child_sn); CV_DebugS ds = cv_debug_s_from_data(scratch, raw); + cv_debug_s_tag_prov_sect(&ds, child_sn-1); String8List lines = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Lines); if (lines.node_count == 0) { return key; } String8 frag = lines.first->string; @@ -4145,6 +4146,7 @@ lnk_icf_debug_s_has_locals(Arena *scratch, LNK_Obj *obj, U32 child_sn) { String8 raw = lnk_obj_section_data_from_number(obj, child_sn); CV_DebugS ds = cv_debug_s_from_data(scratch, raw); + cv_debug_s_tag_prov_sect(&ds, child_sn-1); String8List syms = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Symbols); for EachNode(n, String8Node, syms.first) { String8 s = n->string; @@ -7622,8 +7624,7 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) CV_DebugS *debug_s_dst = &debug_s_arr[obj_idx]; CV_DebugS *debug_s_src = &cv.debug_s_arr[obj_idx]; - String8List *dst = &debug_s_dst->data_list[CV_C13SubSectionIdxKind_Symbols]; - String8List *src = &debug_s_src->data_list[CV_C13SubSectionIdxKind_Symbols]; + String8List *src = cv_sub_section_ptr_from_debug_s(debug_s_src, CV_C13SubSectionKind_Symbols); U64 proc_count = 0; U64 proc_size = 0; @@ -7667,7 +7668,8 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) } Assert(buffer_cursor == buffer_size); - str8_list_push(scratch.arena, dst, str8(buffer, buffer_size)); + // synthesized bytes (TI-stripped copies): provenance marked synthetic + cv_debug_s_push_synthetic_sub_section(scratch.arena, debug_s_dst, CV_C13SubSectionKind_Symbols, str8(buffer, buffer_size)); } } diff --git a/src/linker/lnk_debug_helper.c b/src/linker/lnk_debug_helper.c index fbf9bb360..c7fb5694d 100644 --- a/src/linker/lnk_debug_helper.c +++ b/src/linker/lnk_debug_helper.c @@ -26,7 +26,9 @@ lnk_make_debug_s(Arena *arena, String8List symbols) cv_patch_symbol_tree_offsets(symbols, sizeof(CV_Signature), CV_SymbolAlign); - CV_DebugS debug_s = { .data_list[CV_C13SubSectionIdxKind_Symbols] = symbols }; + // synthetic construction (linker-made symbols): provenance untracked by design + CV_DebugS debug_s = {0}; + *cv_sub_section_ptr_from_debug_s(&debug_s, CV_C13SubSectionKind_Symbols) = symbols; String8List debug_s_data_list = cv_data_from_debug_s_c13(scratch.arena, &debug_s, 1); String8 debug_s_data = str8_list_join(arena, &debug_s_data_list, 0); diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 5a8d48c18..42424e7f3 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -208,13 +208,15 @@ lnk_discard_cv_debug_info(LNK_CodeViewInput *input, U64 obj_idx) // discard types MemoryZeroStruct(&input->debug_t_arr[obj_idx]); - // discard symbols + // discard symbols (provenance zeroed in lockstep with the data list) String8List *symbols_ptr = cv_sub_section_ptr_from_debug_s(&input->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); MemoryZeroStruct(symbols_ptr); + MemoryZeroStruct(cv_sub_section_prov_ptr_from_debug_s(&input->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols)); // discard inline sites String8List *inlineelines_ptr = cv_sub_section_ptr_from_debug_s(&input->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); MemoryZeroStruct(inlineelines_ptr); + MemoryZeroStruct(cv_sub_section_prov_ptr_from_debug_s(&input->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines)); } internal @@ -226,9 +228,14 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_s_task) String8List sect_list = task->debug_s_list_arr[obj_idx]; CV_DebugS *debug_s = &task->debug_s_arr [obj_idx]; - for EachNode(n, String8Node, sect_list.first) { + U32Array sect_indices = task->debug_s_sect_idx_arr[obj_idx]; + Assert(sect_indices.count == sect_list.node_count); + + U64 input_ordinal = 0; + for (String8Node *n = sect_list.first; n != 0; n = n->next, input_ordinal += 1) { // parse & merge sub sections CV_DebugS ds = cv_debug_s_from_data(arena, n->string); + cv_debug_s_tag_prov_sect(&ds, sect_indices.v[input_ordinal]); cv_debug_s_concat_in_place(debug_s, &ds); // make sure there is one string table @@ -260,16 +267,17 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_s_task) if (!obj->icf_lines_only[section_number]) { continue; } String8 raw_data = lnk_obj_section_data_from_number(obj, section_number); CV_DebugS ds = cv_debug_s_from_data(arena, raw_data); + cv_debug_s_tag_prov_sect(&ds, section_number - 1); // merged from a different child section if (obj->icf_lines_only[section_number] == 2) { cv_debug_s_concat_in_place(debug_s, &ds); } else { - String8List lines = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Lines); - String8List *dst = cv_sub_section_ptr_from_debug_s(debug_s, CV_C13SubSectionKind_Lines); - str8_list_concat_in_place(dst, &lines); + cv_debug_s_concat_sub_section_in_place(debug_s, &ds, CV_C13SubSectionKind_Lines); } } } } + + cv_debug_s_validate_prov(debug_s); } internal int @@ -1889,7 +1897,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, ProfEnd(); ProfBegin("Collect CodeView"); - input.debug_s_list_arr = lnk_collect_obj_sections(tp, tp_arena, obj_count, obj_arr, str8_lit(".debug$S"), 0); + input.debug_s_list_arr = lnk_collect_obj_sections(tp, tp_arena, obj_count, obj_arr, str8_lit(".debug$S"), 0, &input.debug_s_sect_idx_arr); ProfEnd(); // batch-populate the mapped .debug$S/$T/$P/$H input ranges before the parse @@ -5329,9 +5337,7 @@ ProfScope("Write Modules") // subsection of the obj (symbols + lines + checksums + ...) U64 *weights = push_array_no_zero(scratch.arena, U64, cv->obj_count); for EachIndex(obj_idx, cv->obj_count) { - U64 total = 0; - for EachIndex(k, CV_C13SubSectionIdxKind_COUNT) { total += cv->debug_s_arr[obj_idx].data_list[k].total_size; } - weights[obj_idx] = total; + weights[obj_idx] = cv_total_sub_section_size_from_debug_s(&cv->debug_s_arr[obj_idx]); } lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C, weights); tp_for_parallel_reserve(tp, 0, C, lnk_write_pdb_modules, &task); // BARRIER pass (path B): barrier_wait/tp_broadcast diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index dce9f8004..fc36d92ba 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -92,6 +92,7 @@ typedef struct U64 *obj_to_ts; String8List *debug_s_list_arr; + U32Array *debug_s_sect_idx_arr; // per obj: 0-based sect_idx of each debug_s_list_arr node (provenance tagging) U32Array int_obj_indices; U32Array ext_obj_indices; diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index 07f1fe2d1..97710220c 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -361,7 +361,9 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) Temp temp = temp_begin(scratch.arena); String8 debug_s_data = str8_substr(input->data, rng_1u64(sect_header->foff, sect_header->foff+sect_header->fsize)); CV_DebugS debug_s = cv_debug_s_from_data(temp.arena, debug_s_data); - for EachNode(symbols_n, String8Node, debug_s.data_list[CV_C13SubSectionIdxKind_Symbols].first) { + cv_debug_s_tag_prov_sect(&debug_s, (U32)sect_idx); + String8List symbols_list = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); + for EachNode(symbols_n, String8Node, symbols_list.first) { for (U64 cursor = 0, count = 0; cursor < symbols_n->string.size && count < 2; count += 1) { CV_SymbolHeader symbol_header; TryReadBreak(str8_deserial_read_struct(symbols_n->string, cursor, &symbol_header), cursor); @@ -875,6 +877,17 @@ THREAD_POOL_TASK_FUNC(lnk_collect_obj_chunks_task) LNK_SectionCollector *task = raw_task; LNK_Obj *obj = task->objs[task_id]; + // Optional 0-based section-index sidecar, kept in the same order as out_lists. + if (task->out_sect_indices != 0) { + U64 match_count = 0; + for LNK_EachCoffSection(count_it, obj) { + if (*count_it.v.flags & COFF_SectionFlag_LnkRemove && !task->collect_discarded) { continue; } + if (str8_match(lnk_obj_section_name_from_section_number(obj, count_it.v.section_number), task->name, 0)) { match_count += 1; } + } + task->out_sect_indices[task_id].count = 0; + task->out_sect_indices[task_id].v = push_array_no_zero(arena, U32, match_count ? match_count : 1); + } + for LNK_EachCoffSection(it, obj) { LNK_ObjSection section = it.v; @@ -886,19 +899,29 @@ THREAD_POOL_TASK_FUNC(lnk_collect_obj_chunks_task) if (str8_match(section_name, task->name, 0)) { String8 section_data = lnk_obj_section_data_from_number(obj, section.section_number); str8_list_push(arena, &task->out_lists[task_id], section_data); + if (task->out_sect_indices != 0) { + U32Array *indices = &task->out_sect_indices[task_id]; + indices->v[indices->count++] = safe_cast_u32(section.section_number - 1); + } } } } internal String8List * -lnk_collect_obj_sections(TP_Context *tp, TP_Arena *arena, U64 objs_count, LNK_Obj **objs, String8 name, B32 collect_discarded) +lnk_collect_obj_sections(TP_Context *tp, TP_Arena *arena, U64 objs_count, LNK_Obj **objs, String8 name, B32 collect_discarded, U32Array **sect_indices_out) { LNK_SectionCollector task = {0}; task.objs = objs; task.name = name; task.collect_discarded = collect_discarded; task.out_lists = push_array(arena->v[0], String8List, objs_count); + if (sect_indices_out != 0) { + task.out_sect_indices = push_array(arena->v[0], U32Array, objs_count); + } tp_for_parallel(tp, arena, objs_count, lnk_collect_obj_chunks_task, &task); + if (sect_indices_out != 0) { + *sect_indices_out = task.out_sect_indices; + } return task.out_lists; } @@ -1063,42 +1086,34 @@ lnk_symbol_from_section_offset(LNK_ObjSymbolMap *map, U32 section_number, U32 of internal CV_DebugS lnk_debug_s_from_obj(Arena *arena, LNK_Obj *obj) { - Temp scratch = scratch_begin(&arena, 1); - - String8List raw_debug_s = {0}; - { - for LNK_EachCoffSection(it, obj) { - LNK_ObjSection section = it.v; - String8 section_name = lnk_obj_section_name_from_section_number(obj, section.section_number); - if (str8_match(section_name, str8_lit(".debug$S"), 0)) { - String8 debug_s = lnk_obj_section_data_from_number(obj, section.section_number); - str8_list_push(scratch.arena, &raw_debug_s, debug_s); - } - } - } - + // (single loop so each parse can tag provenance with its section index; the old + // collect-then-parse split had no side effects between the loops) CV_DebugS debug_s = {0}; - { - for (String8Node *node = raw_debug_s.first; node != 0; node = node->next) { - // parse & merge sub sections - CV_DebugS ds = cv_debug_s_from_data(arena, node->string); - cv_debug_s_concat_in_place(&debug_s, &ds); - - // make sure there is one string table - String8List string_data_list = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_StringTable); - if (string_data_list.node_count > 1) { - break; - } + for LNK_EachCoffSection(it, obj) { + LNK_ObjSection section = it.v; + String8 section_name = lnk_obj_section_name_from_section_number(obj, section.section_number); + if (!str8_match(section_name, str8_lit(".debug$S"), 0)) { continue; } + + // parse & merge sub sections + String8 raw_debug_s = lnk_obj_section_data_from_number(obj, section.section_number); + CV_DebugS ds = cv_debug_s_from_data(arena, raw_debug_s); + cv_debug_s_tag_prov_sect(&ds, safe_cast_u32(section.section_number - 1)); + cv_debug_s_concat_in_place(&debug_s, &ds); + + // make sure there is one string table + String8List string_data_list = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_StringTable); + if (string_data_list.node_count > 1) { + break; + } - // make sure there is one file checksum table - String8List checksum_data_list = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_FileChksms); - if (checksum_data_list.node_count > 1) { - continue; - } + // make sure there is one file checksum table + String8List checksum_data_list = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_FileChksms); + if (checksum_data_list.node_count > 1) { + continue; } } - scratch_end(scratch); + cv_debug_s_validate_prov(&debug_s); return debug_s; } diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index 99e11436c..a8dc91c76 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -251,6 +251,7 @@ typedef struct String8 name; B32 collect_discarded; String8List *out_lists; + U32Array *out_sect_indices; // optional; per obj, 0-based sect_idx of each collected node (parallel to out_lists[obj]) } LNK_SectionCollector; // --- Error ------------------------------------------------------------------- @@ -297,7 +298,7 @@ internal force_inline B32 lnk_obj_symbol_iter_next(LNK_Obj *obj, LNK_ObjSymbolIt // --- Helpers ----------------------------------------------------------------- -internal String8List * lnk_collect_obj_sections(TP_Context *tp, TP_Arena *arena, U64 objs_count, LNK_Obj **objs, String8 name, B32 collect_discarded); +internal String8List * lnk_collect_obj_sections(TP_Context *tp, TP_Arena *arena, U64 objs_count, LNK_Obj **objs, String8 name, B32 collect_discarded, U32Array **sect_indices_out); internal B32 lnk_obj_is_before(void *raw_a, void *raw_b); // --- Directive Parser -------------------------------------------------------- diff --git a/src/torture/tests/torture_linker_tests.c b/src/torture/tests/torture_linker_tests.c index fac1f1eee..30578c1b5 100644 --- a/src/torture/tests/torture_linker_tests.c +++ b/src/torture/tests/torture_linker_tests.c @@ -199,7 +199,7 @@ TEST(expect_pdb_indexes_linker_output) cv_make_data32(arena, (CV_SymData32){.off = 1, .sec = 1}, str8_lit("indexed_data"))); CV_Symbol symbol = cv_symbol_from_ptr(raw_symbol.str); CV_DebugS debug_s = {0}; - str8_list_push(arena, &debug_s.data_list[CV_C13SubSectionIdxKind_Symbols], cv_data_from_symbol(arena, &symbol, CV_SymbolAlign)); + str8_list_push(arena, cv_sub_section_ptr_from_debug_s(&debug_s, CV_C13SubSectionKind_Symbols), cv_data_from_symbol(arena, &symbol, CV_SymbolAlign)); String8List debug_s_data = cv_data_from_debug_s_c13(arena, &debug_s, 1); T_Ok(t_write_def_obj("indexed.obj", (T_COFF_DefObj){ .machine = T_COFF_DefSetMachine(X64), @@ -250,7 +250,7 @@ TEST(pdbstripped_coff_artifact_parity) CV_DebugS debug_s = {0}; for EachElement(i, raw_symbols) { CV_Symbol symbol = cv_symbol_from_ptr(raw_symbols[i].str); - str8_list_push(arena, &debug_s.data_list[CV_C13SubSectionIdxKind_Symbols], cv_data_from_symbol(arena, &symbol, CV_SymbolAlign)); + str8_list_push(arena, cv_sub_section_ptr_from_debug_s(&debug_s, CV_C13SubSectionKind_Symbols), cv_data_from_symbol(arena, &symbol, CV_SymbolAlign)); } String8List raw_debug_s_list = cv_data_from_debug_s_c13(arena, &debug_s, 1); String8 raw_debug_s = str8_list_join(arena, &raw_debug_s_list, 0); From d5c2cd41e0d3de6a39d42ae66066dd6cadb30689 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 12 Aug 2026 14:26:24 -0700 Subject: [PATCH 057/121] linker: ring P1.2 -- provenance resolver + debug-build parity walks -- lnk_resolve_debug_s_node maps a subsection provenance record back to bytes via lnk_obj_get_sect_data; debug builds content-assert every tracked node against its slice at module write (both passes) and the globals collect walk. Proves the provenance layer is authoritative for the two biggest .debug$S consumers before any representation swap; release builds unchanged. Debug lyra links clean with asserts live. Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.c | 59 +++++++++++++++++++++++++++++++++++++ src/linker/lnk_obj.c | 17 +++++++++++ src/linker/lnk_obj.h | 1 + 3 files changed, 77 insertions(+) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 42424e7f3..3a27b4d6d 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3833,6 +3833,35 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) ProfBegin("Global Symbols"); { +#if BUILD_DEBUG + // Streaming-ring P1.2 parity (debug builds only): each symbol input's raw_symbols is a + // slice of one Symbols subsection node of its obj; find that node by pointer identity, + // resolve its provenance through lnk_resolve_debug_s_node, and assert the CONTENT still + // matches at the very start of the globals collect walk. This runs BEFORE the sect-data + // copies release below (barrier at "Release Sect Data Copies"), so the resolver still + // returns exactly what the parse consumed. Untracked lists and synthetic nodes skip. + for (U64 lane = task_id; lane < task->cv->symbol_input_range_count; lane += tp->worker_count) { + for EachInRange(i, task->cv->symbol_input_ranges[lane]) { + LNK_SymbolInput *in = &task->cv->symbol_inputs[i]; + LNK_Obj *obj = task->cv->obj_arr[in->obj_idx]; + CV_DebugS *debug_s = &task->cv->debug_s_arr[in->obj_idx]; + String8List *data_list = cv_sub_section_ptr_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); + CV_DebugSProvList *prov_list = cv_sub_section_prov_ptr_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); + if (prov_list->count == 0) { continue; } // untracked construction + Assert(prov_list->count == data_list->node_count); + CV_DebugSProvNode *prov = prov_list->first; + String8Node *data_n = data_list->first; + for (; data_n != 0; data_n = data_n->next, prov = prov->next) { + if (data_n->string.str == in->raw_symbols.str && data_n->string.size == in->raw_symbols.size) { break; } + } + Assert(data_n != 0); // every symbol input must originate from a Symbols node + if (prov->is_synthetic) { continue; } + String8 resolved = lnk_resolve_debug_s_node(obj, prov); + Assert(resolved.size == in->raw_symbols.size); + Assert(str8_match(resolved, in->raw_symbols, 0)); + } + } +#endif // FAIR-SHARE: symbol_input_ranges is a FIXED [symbol_input_range_count] partition built at // full pool width, but this barrier pass runs at the pinned cohort C == tp->worker_count // (C <= fixed). Walk the fixed lanes strided by the cohort so every fixed lane is processed @@ -4341,6 +4370,34 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } +// Streaming-ring P1.2 parity walk (debug builds only -- it doubles the reads): prove the +// dormant provenance recorded at parse time is authoritative by re-resolving every tracked +// subsection node through lnk_resolve_debug_s_node and comparing CONTENT against the node's +// String8. For reloc-PATCHED sections both the node slice and the resolver point into the +// same sect_data_copies bytes (pointers may even be equal), so the assert is on content, +// which also holds across the in-place $S TI/kind fixups (they mutate the shared bytes). +// Skips untracked lists (prov count == 0: wholesale synthetic constructions) and synthetic +// nodes (no backing section). Valid only while lnk_obj_section_data_from_number still returns what the +// parse consumed, i.e. before the sect-data copies release in lnk_move_global_symbols_to_gsi. +internal void +lnk_assert_debug_s_prov_parity(LNK_Obj *obj, CV_DebugS *debug_s) +{ +#if BUILD_DEBUG + for EachElement(k, debug_s->data_list) { + if (debug_s->prov_list[k].count == 0) { continue; } // untracked construction + Assert(debug_s->prov_list[k].count == debug_s->data_list[k].node_count); + CV_DebugSProvNode *prov = debug_s->prov_list[k].first; + for (String8Node *data_n = debug_s->data_list[k].first; data_n != 0; data_n = data_n->next, prov = prov->next) { + String8 resolved = lnk_resolve_debug_s_node(obj, prov); + if (prov->is_synthetic) { Assert(resolved.size == 0); continue; } + Assert(resolved.size == data_n->string.size); + Assert(str8_match(resolved, data_n->string, 0)); + } + Assert(prov == 0); + } +#endif +} + internal U64 lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8Node *buf, U64 *buf_pos) { @@ -4444,6 +4501,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) // compute sizes for module streams for EachIndex(i, obj_indices.count) { U64 obj_idx = obj_indices.v[i]; + lnk_assert_debug_s_prov_parity(task->cv->obj_arr[obj_idx], &task->cv->debug_s_arr[obj_idx]); // debug-only P1.2 parity lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0); } barrier_wait(tp->barrier); @@ -4473,6 +4531,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) String8List mod_data = msf_data_from_sn(temp.arena, task->pdb->msf, mod->sn); if (mod_data.node_count) { + lnk_assert_debug_s_prov_parity(task->cv->obj_arr[obj_idx], &task->cv->debug_s_arr[obj_idx]); // debug-only P1.2 parity String8Node buf = *mod_data.first; U64 pos = 0; lnk_write_debug_s_to_pdb_module(mod, debug_s, &buf, &pos); diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index 97710220c..ddae9d767 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -871,6 +871,23 @@ lnk_obj_drop_section_data_copies(LNK_Obj *obj) } } +// Resolve a .debug$S subsection provenance record (streaming-ring P1) back to its bytes. +// Returns exactly what the parse consumed: lnk_obj_section_data_from_number prefers the reloc-patched +// private copy when one exists (the parsed node slices point into that same copy), otherwise +// the raw mapped input. Synthetic provenance (linker-made bytes, is_synthetic) and untagged +// records (sect_idx == CV_DebugSProvSect_Nil) have no backing section to resolve through: +// returns str8_zero and the caller must fall back to the node's String8. +internal String8 +lnk_resolve_debug_s_node(LNK_Obj *obj, CV_DebugSProvNode *prov) +{ + if (prov == 0 || prov->is_synthetic || prov->sect_idx == CV_DebugSProvSect_Nil) { + return str8_zero(); + } + U64 section_number = (U64)prov->sect_idx + 1; + String8 sect_data = lnk_obj_section_data_from_number(obj, section_number); + return str8_substr(sect_data, rng_1u64(prov->off, prov->off + prov->size)); +} + internal THREAD_POOL_TASK_FUNC(lnk_collect_obj_chunks_task) { diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index a8dc91c76..24ee4c8c5 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -287,6 +287,7 @@ internal String8 lnk_obj_section_data_from_number(LNK_Obj *obj, U64 internal U64 lnk_obj_foff_from_section_data_ptr(LNK_Obj *obj, void *ptr); internal String8 lnk_obj_section_name_from_section_number(LNK_Obj *obj, U64 section_number); internal void lnk_obj_drop_section_data_copies(LNK_Obj *obj); +internal String8 lnk_resolve_debug_s_node(LNK_Obj *obj, CV_DebugSProvNode *prov); internal LNK_ObjSection lnk_obj_section_from_section_number(LNK_Obj *obj, U64 section_number); internal COFF_RelocArray lnk_coff_relocs_from_section_header(LNK_Obj *obj, COFF_SectionHeader *section_header); internal String8 lnk_coff_string_table_from_obj(LNK_Obj *obj); From e313cc14c09ca4ca30913c996bd5a0f52bcef420 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 12 Aug 2026 15:43:45 -0700 Subject: [PATCH 058/121] linker: ring P2a -- .debug$S TI fixups journal at merge, replay at module write -- the three all-obj barrier passes (symbol TI patch, inlinee-lines TI patch, *_ID kind/itype rewrite) are gone; lnk_merge_types journals per-node patch runs (8B entries, {off:31|width:1, value}, wide-run fallback for >2GiB nodes) on dedicated per-worker arenas while merge state is alive, and lnk_write_pdb_modules replays each obj's run at the top of its sizing visit, immediately before the bytes are consumed. Journal arenas release right after Write Modules (background reaper); /OPT:GCTYPES and PDBSTRIPPED-only builds consume the journal eagerly. Also fixes an OOB: injected type-server/.ifc blob pseudo-objs live at [obj_count, count) and the inline journal arrays were sized obj_count. FN editor DLL: peak commit 47.5 -> 47.0GB, wall par, byte-identical (incl /OPT:GCTYPES A/B), deterministic; debug builds carry replay-exactness and provenance-parity asserts, debug lyra links clean. Co-Authored-By: Claude Fable 5 --- src/linker/lnk.c | 13 + src/linker/lnk_debug_info.c | 476 ++++++++++++++++++++++++++++++------ src/linker/lnk_debug_info.h | 45 ++++ 3 files changed, 459 insertions(+), 75 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 81185ea5e..8fa7a6ab8 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -7546,6 +7546,19 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) LNK_MergedTypes cv_types = lnk_merge_types(tp, arena, &cv, 0); lnk_summary_phase_end(LNK_SummaryPhase_DbgMerge); + // Streaming-ring P2 slice A: $S TI/kind fixups are journaled in lnk_merge_types and + // normally replayed per obj at the start of the module-write visit (lnk_write_pdb_modules). + // Two configs consume fixed-up $S bytes before/without that pass and need the journal + // applied eagerly instead: + // - /OPT:GCTYPES reads final type indices out of $S (mark roots) and rewrites them + // (compaction remap) right below; + // - a /PDBSTRIPPED-only build (no full PDB / RDI) re-walks $S Symbols without ever + // running the module-write pass. + if (config->opt_gc_types == LNK_SwitchState_Yes || + !(config->debug_mode == LNK_DebugMode_Full || config->rad_debug == LNK_SwitchState_Yes)) { + lnk_apply_debug_s_fixups_eager(tp, &cv); + } + // prune merged types not reachable from any surviving symbol (PDB-size win). OFF by default: // it removes types that a debugger can still legitimately cast to in the watch window // (reachable-from-symbols is a subset of castable-types). Opt in with /OPT:GCTYPES. diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 3a27b4d6d..cc0e95c24 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -52,6 +52,32 @@ lnk_arena_release_thread(void *raw_arena) ProfEnd(); } +// Reaper entry point for a per-worker arena array (TP_Arena). Same ownership rule as +// lnk_arena_release_thread: caller hands over EXCLUSIVE ownership. The TP_Arena header and its +// v[] array live inside v[0] (tp_arena_alloc layout), which tp_arena_release frees last, so the +// walk below and the release order are safe. +internal void +lnk_tp_arena_release_thread(void *raw_arena) +{ + ProfBeginFunction(); + U64 begin_us = now_time_us(); + TP_Arena *tp_arena = raw_arena; + + U64 committed_size = 0; + for EachIndex(i, tp_arena->count) { + for (Arena *n = tp_arena->v[i]->current; n != 0; n = n->prev) { committed_size += n->cmt; } +#if ARENA_FREE_LIST + for (Arena *n = tp_arena->v[i]->free_last; n != 0; n = n->prev) { committed_size += n->cmt; } +#endif + } + + tp_arena_release(&tp_arena); + + lnk_log(LNK_Log_Timers, "[teardown] background release of %llu MiB worker arenas took %.2f ms (off main thread)", + committed_size / MB(1), (F64)(now_time_us() - begin_us) / 1000.0); + ProfEnd(); +} + //////////////////////////////////////////////////////////////////////////////// //~ Fault-storm mitigation: batched PrefetchVirtualMemory over mapped input // ranges. The .debug$S/$T parse and type-merge loops first-touch tens of GB of @@ -3023,39 +3049,326 @@ lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_TiO } } +// Streaming-ring P2 slice A: upper-bound count of journal entries the ti_offs walk emits. +// Exact for the TI-fixup part -- an entry is emitted iff the raw TI is >= the source's min +// (the "skip basic types" test), which needs no merge-state lookups. +internal U64 +lnk_count_cv_type_index_fixups(LNK_MergeTypes *ctx, String8 data, CV_TiOffsets ti_offs) +{ + U64 count = 0; + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(data, n.offset, sizeof(*ti_ptr)); + if (memory_read32(ti_ptr) >= ctx->input->min_type_indices[n.source]) { count += 1; } + } + return count; +} + +// 8B/16B entry emit + decode helpers. `off` is relative to the run's node base; narrow runs +// are chosen up front (node size < 2GiB guarantees every off fits off:31), so the narrow +// branch never truncates. +force_inline void +lnk_debug_s_patch_emit(LNK_DebugSPatchArray *journal, U64 off, U32 value, U32 size) +{ + if (journal->is_wide) { + ((LNK_DebugSPatchWide *)journal->v)[journal->count++] = (LNK_DebugSPatchWide){ .off = off, .value = value, .size = size }; + } else { + Assert(off < (1ull << 31)); + ((LNK_DebugSPatch *)journal->v)[journal->count++] = (LNK_DebugSPatch){ .off_w = (U32)((off << 1) | (size == 4 ? 1 : 0)), .value = value }; + } +} + +force_inline U64 +lnk_debug_s_patch_off_at(LNK_DebugSPatchArray *journal, U64 k) +{ + return journal->is_wide ? ((LNK_DebugSPatchWide *)journal->v)[k].off + : (U64)(((LNK_DebugSPatch *)journal->v)[k].off_w >> 1); +} + +force_inline U32 +lnk_debug_s_patch_value_at(LNK_DebugSPatchArray *journal, U64 k) +{ + return journal->is_wide ? ((LNK_DebugSPatchWide *)journal->v)[k].value + : ((LNK_DebugSPatch *)journal->v)[k].value; +} + +// journal-emitting twin of lnk_fixup_cv_type_indices: identical TI resolution (assigned-TI +// hash search over the merge result), but the write is RECORDED instead of applied -- the $S +// bytes stay pre-fixup until the per-obj replay at module write. `base` = the run's node base +// (entries store node-relative offsets). +internal void +lnk_journal_cv_type_index_fixups(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_TiOffsets ti_offs, LNK_DebugSPatchArray *journal, U8 *base) +{ + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(data, n.offset, sizeof(*ti_ptr)); + CV_TypeIndex ti = memory_read32(ti_ptr); + + // skip basic types + if (ti < ctx->input->min_type_indices[n.source]) { continue; } + + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n.source, ti); + CV_TypeIndex final_ti = lnk_assigned_ti_hash_search(&ctx->assigned_ti_arr[n.source], ctx->input, leaf_ref); + lnk_debug_s_patch_emit(journal, (U64)((U8 *)ti_ptr - base), final_ti, 4); + +#if LNK_PARANOID + if (final_ti == 0) { + lnk_error_obj(LNK_Error_InvalidTypeIndex, ctx->input->obj_arr[obj_idx], "no itype 0x%x", ti); + } +#endif + } +} + +// Fuses the old lnk_cv_patcher_symbols_task (symbol-record TI fixup) and lnk_fixup_symbols_task +// (*_ID kind rewrite + itype -> FUNC_ID/MFUNC_ID itype resolve) into one journal-building walk +// per symbol input. Runs AFTER the materialize pass so the itype resolve reads the fixed-up +// merged IPI leaf copies -- exactly what the old standalone pass consumed. The old second pass +// read proc32->itype back from memory AFTER the first pass patched it; here that value is the +// resolved TI recorded for the record's itype slot (raw bytes when the slot wasn't journaled: +// basic types, MIPS/IA64 kinds cv_symbol_ti_offsets has no entry for). Emitting BOTH itype +// entries in order (resolved IPI TI, then the FUNC_ID itype) makes the sequential replay land +// on the same end-state bytes for every branch, including the early-out edge cases. internal -THREAD_POOL_TASK_FUNC(lnk_cv_patcher_symbols_task) +THREAD_POOL_TASK_FUNC(lnk_journal_symbol_fixups_task) { ProfBeginFunction(); LNK_MergeTypes *task = raw_task; + + Arena *journal_arena = task->journal_arena->v[worker_id]; + U64 leaf_count_ipi = task->result.count [CV_TypeIndexSource_IPI]; + U8 **leaf_arr_ipi = task->result.v [CV_TypeIndexSource_IPI]; + CV_TypeIndex min_ti_ipi = task->min_type_indices[CV_TypeIndexSource_IPI]; + Rng1U64 range = task->input->symbol_patch_task[task_id].input_range; for EachInRange(i, range) { LNK_SymbolInput symbols = task->input->symbol_inputs[i]; + + // upper-bound pass: TI entries (exact) + kind rewrite / itype slots for *_ID records + U64 cap = 0; for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { CV_Symbol symbol = {0}; TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); + cap += lnk_count_cv_type_index_fixups(task, symbol.data, cv_symbol_ti_offsets(symbol.kind, symbol.data)); + switch (symbol.kind) { + case CV_SymKind_PROC_ID_END: cap += 1; break; + case CV_SymKind_LPROC32_ID: + case CV_SymKind_GPROC32_ID: + case CV_SymKind_LPROC32_DPC_ID: + case CV_SymKind_LPROCMIPS_ID: + case CV_SymKind_GPROCMIPS_ID: + case CV_SymKind_LPROCIA64_ID: + case CV_SymKind_GPROCIA64_ID: cap += 2; break; + default: break; + } + } + + U8 *base = symbols.raw_symbols.str; + + LNK_DebugSPatchArray *journal = &task->input->debug_s_sym_fixups[i]; + journal->is_wide = (symbols.raw_symbols.size >> 31) != 0; // narrow off:31 covers the whole node otherwise + journal->v = journal->is_wide ? (void *)push_array_no_zero(journal_arena, LNK_DebugSPatchWide, cap) + : (void *)push_array_no_zero(journal_arena, LNK_DebugSPatch, cap); + journal->count = 0; + + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); + + CV_TiOffsets ti_offs = cv_symbol_ti_offsets(symbol.kind, symbol.data); + U64 rec_journal_start = journal->count; + lnk_journal_cv_type_index_fixups(task, symbols.obj_idx, symbol.data, ti_offs, journal, base); + + // convert symbol to final type + CV_SymKind *sym_kind_ptr = cv_kind_ptr_from_symbol(symbol); + CV_SymKind new_kind = CV_SymKind_END; + switch (symbol.kind) { + case CV_SymKind_PROC_ID_END: { + lnk_debug_s_patch_emit(journal, (U64)((U8 *)sym_kind_ptr - base), CV_SymKind_END, 2); + } break; + + case CV_SymKind_LPROC32_ID: new_kind = CV_SymKind_LPROC32; goto fixup_id; + case CV_SymKind_GPROC32_ID: new_kind = CV_SymKind_GPROC32; goto fixup_id; + case CV_SymKind_LPROC32_DPC_ID: new_kind = CV_SymKind_LPROC32_DPC; goto fixup_id; + case CV_SymKind_LPROCMIPS_ID: new_kind = CV_SymKind_LPROCMIPS; goto fixup_id; + case CV_SymKind_GPROCMIPS_ID: new_kind = CV_SymKind_GPROCMIPS; goto fixup_id; + case CV_SymKind_LPROCIA64_ID: new_kind = CV_SymKind_LPROCIA64; goto fixup_id; + case CV_SymKind_GPROCIA64_ID: new_kind = CV_SymKind_GPROCIA64; goto fixup_id; + fixup_id:; { + lnk_debug_s_patch_emit(journal, (U64)((U8 *)sym_kind_ptr - base), new_kind, 2); + + CV_SymProc32 *proc32 = str8_deserial_get_raw_ptr(symbol.data, 0, sizeof(*proc32)); + + // effective post-TI-fixup itype (what the old pass read back from patched memory) + U64 itype_off = (U64)((U8 *)&proc32->itype - base); + CV_TypeIndex itype = proc32->itype; + for (U64 k = rec_journal_start; k < journal->count; k += 1) { + if (lnk_debug_s_patch_off_at(journal, k) == itype_off) { itype = lnk_debug_s_patch_value_at(journal, k); break; } + } + + if (itype < min_ti_ipi) { + // TODO: in some cases destructors don't have a type, need a repro + break; + } + + if ((itype - min_ti_ipi) > leaf_count_ipi) { + Assert(0 && "TODO: error handle corrupted type index"); + break; + } + + U64 leaf_idx = itype - min_ti_ipi; + String8 leaf_data = str8(leaf_arr_ipi[leaf_idx], max_U64); + + CV_Leaf leaf; + if (cv_read_leaf(leaf_data, 0, 1, &leaf) == 0) { InvalidPath; } + + U64 min_leaf_size = cv_header_struct_size_from_leaf_kind(leaf.kind); + if (min_leaf_size > leaf.data.size) { Assert(!"TODO: error handle corrupt leaf"); break; } + + if (leaf.kind == CV_LeafKind_FUNC_ID) { + CV_LeafFuncId *func_id = str8_deserial_get_raw_ptr(leaf.data, 0, sizeof(*func_id)); + lnk_debug_s_patch_emit(journal, itype_off, func_id->itype, 4); + } else if (leaf.kind == CV_LeafKind_MFUNC_ID) { + CV_LeafMFuncId *mfunc_id = str8_deserial_get_raw_ptr(leaf.data, 0, sizeof(*mfunc_id)); + lnk_debug_s_patch_emit(journal, itype_off, mfunc_id->itype, 4); + } else { + Assert(!"TODO: erorr handle unexpected leaf type"); + break; + } + } break; - CV_TiOffsets ti_offs = cv_symbol_ti_offsets(symbol.kind, symbol.data); - lnk_fixup_cv_type_indices(task, symbols.obj_idx, symbol.data, ti_offs); + default: break; + } } + Assert(journal->count <= cap); } ProfEnd(); } +// journal-building replacement for the old lnk_cv_patcher_inlines_task (per obj) internal -THREAD_POOL_TASK_FUNC(lnk_cv_patcher_inlines_task) +THREAD_POOL_TASK_FUNC(lnk_journal_inline_fixups_task) { ProfBeginFunction(); LNK_MergeTypes *task = raw_task; U64 obj_idx = task_id; String8List inlinee_lines = cv_sub_section_from_debug_s(task->input->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); Arena *fixed_arena = task->fixed_arenas[worker_id]; + Arena *journal_arena = task->journal_arena->v[worker_id]; + + // exact count; a node >= 2GiB forces the obj's whole run set to wide entries + U64 cap = 0; + B32 is_wide = 0; + for EachNode(inline_data_n, String8Node, inlinee_lines.first) { + Temp temp = temp_begin(fixed_arena); + CV_TiOffsets ti_offs = cv_inlinee_ti_offsets(temp.arena, inline_data_n->string); + cap += lnk_count_cv_type_index_fixups(task, inline_data_n->string, ti_offs); + is_wide |= (inline_data_n->string.size >> 31) != 0; + temp_end(temp); + } + + LNK_DebugSInlineJournal *journal = &task->input->debug_s_inline_fixups[obj_idx]; + journal->patches.is_wide = is_wide; + journal->patches.v = is_wide ? (void *)push_array_no_zero(journal_arena, LNK_DebugSPatchWide, cap) + : (void *)push_array_no_zero(journal_arena, LNK_DebugSPatch, cap); + journal->patches.count = 0; + journal->node_counts = push_array_no_zero(journal_arena, U32, inlinee_lines.node_count ? inlinee_lines.node_count : 1); + + U64 node_idx = 0; for EachNode(inline_data_n, String8Node, inlinee_lines.first) { Temp temp = temp_begin(fixed_arena); + U64 run_start = journal->patches.count; CV_TiOffsets ti_offs = cv_inlinee_ti_offsets(temp.arena, inline_data_n->string); - lnk_fixup_cv_type_indices(task, obj_idx, inline_data_n->string, ti_offs); + lnk_journal_cv_type_index_fixups(task, obj_idx, inline_data_n->string, ti_offs, &journal->patches, inline_data_n->string.str); + journal->node_counts[node_idx++] = (U32)(journal->patches.count - run_start); temp_end(temp); } + Assert(journal->patches.count == cap); + ProfEnd(); +} + +internal void +lnk_apply_debug_s_patch_run(U8 *base, LNK_DebugSPatchArray *arr, U64 lo, U64 opl) +{ + if (arr->is_wide) { + LNK_DebugSPatchWide *v = arr->v; + for (U64 k = lo; k < opl; k += 1) { + if (v[k].size == 4) { memory_write32(base + v[k].off, v[k].value); } + else { memory_write16(base + v[k].off, (U16)v[k].value); } + } + } else { + LNK_DebugSPatch *v = arr->v; + for (U64 k = lo; k < opl; k += 1) { + U8 *ptr = base + (v[k].off_w >> 1); + if (v[k].off_w & 1) { memory_write32(ptr, v[k].value); } + else { memory_write16(ptr, (U16)v[k].value); } + } + } +} + +// Replays the obj's deferred $S TI/kind fixups (journaled in lnk_merge_types). Entries write +// only into the obj's own $S backing bytes (its patched section copies / raw-mapped sections; +// $S is never shared across objs, unlike $T PCH refs), so this is safe inside any per-obj +// parallel loop and produces the same bytes regardless of schedule. Write order == journal +// emission order: symbol inputs in index order (= Symbols data_list node order), then the +// InlineeLines node runs in data_list order. +internal void +lnk_apply_debug_s_fixups_for_obj(LNK_CodeViewInput *cv, U64 obj_idx) +{ + if (!cv->has_debug_s_fixup_journal) { return; } + + for (U64 i = cv->debug_s_sym_fixup_offsets[obj_idx], opl = cv->debug_s_sym_fixup_offsets[obj_idx+1]; i < opl; i += 1) { + Assert(cv->symbol_inputs[i].obj_idx == obj_idx); + lnk_apply_debug_s_patch_run(cv->symbol_inputs[i].raw_symbols.str, &cv->debug_s_sym_fixups[i], 0, cv->debug_s_sym_fixups[i].count); + } + + LNK_DebugSInlineJournal *inline_journal = &cv->debug_s_inline_fixups[obj_idx]; + if (inline_journal->patches.count > 0) { + String8List inlinee_lines = cv_sub_section_from_debug_s(cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); + U64 node_idx = 0, cursor = 0; + for EachNode(inline_data_n, String8Node, inlinee_lines.first) { + U64 run = inline_journal->node_counts[node_idx++]; + lnk_apply_debug_s_patch_run(inline_data_n->string.str, &inline_journal->patches, cursor, cursor + run); + cursor += run; + } + Assert(cursor == inline_journal->patches.count); + } +} + +internal +THREAD_POOL_TASK_FUNC(lnk_apply_debug_s_fixups_task) +{ + LNK_CodeViewInput *cv = raw_task; + lnk_apply_debug_s_fixups_for_obj(cv, task_id); +} + +// Eager whole-input replay for configs that consume fixed-up $S bytes before (or without) +// the module-write pass: /OPT:GCTYPES reads + rewrites $S type indices right after the merge, +// and a /PDBSTRIPPED-only build re-walks $S without ever writing modules. Consumes the +// journal -- the module-write replay is skipped afterwards. +// Consumes + releases the journal: drops every reference (entry arrays, per-input/per-obj +// headers, offsets table all live inside the DEBUG_S_FIXUP_JOURNAL arenas), then hands the +// arena set to the background reaper. Idempotent; no-op when the journal was never built +// (SkipSymbolTypeFixup / stripped cv) or already consumed. +internal void +lnk_release_debug_s_fixup_journal(LNK_CodeViewInput *cv) +{ + if (cv->debug_s_fixup_journal_arenas == 0) { return; } + cv->debug_s_sym_fixups = 0; + cv->debug_s_inline_fixups = 0; + cv->debug_s_sym_fixup_offsets = 0; + cv->has_debug_s_fixup_journal = 0; + if (g_arena_reaper_thread.u64[0] != 0) { thread_join(g_arena_reaper_thread, max_U64); } + g_arena_reaper_thread = thread_launch(lnk_tp_arena_release_thread, cv->debug_s_fixup_journal_arenas); + cv->debug_s_fixup_journal_arenas = 0; +} + +internal void +lnk_apply_debug_s_fixups_eager(TP_Context *tp, LNK_CodeViewInput *cv) +{ + if (!cv->has_debug_s_fixup_journal) { return; } + ProfBegin("Apply $S Fixups (eager)"); + tp_for_parallel(tp, 0, cv->obj_count, lnk_apply_debug_s_fixups_task, cv); + lnk_release_debug_s_fixup_journal(cv); // consumed -- module write skips replay AND release ProfEnd(); } @@ -3125,68 +3438,6 @@ THREAD_POOL_TASK_FUNC(lnk_unbucket_hashes_task) } } -internal -THREAD_POOL_TASK_FUNC(lnk_fixup_symbols_task) -{ - LNK_MergeTypes *task = raw_task; - - LNK_SymbolInput symbols = task->input->symbol_inputs[task_id]; - U64 leaf_count_ipi = task->result.count [CV_TypeIndexSource_IPI]; - U8 **leaf_arr_ipi = task->result.v [CV_TypeIndexSource_IPI]; - CV_TypeIndex min_ti_ipi = task->min_type_indices[CV_TypeIndexSource_IPI]; - - for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); - - // convert symbol to final type - CV_SymKind *sym_kind_ptr = cv_kind_ptr_from_symbol(symbol); - switch (*sym_kind_ptr) { - case CV_SymKind_PROC_ID_END: *sym_kind_ptr = CV_SymKind_END; break; - - case CV_SymKind_LPROC32_ID: *sym_kind_ptr = CV_SymKind_LPROC32; goto fixup_id; - case CV_SymKind_GPROC32_ID: *sym_kind_ptr = CV_SymKind_GPROC32; goto fixup_id; - case CV_SymKind_LPROC32_DPC_ID: *sym_kind_ptr = CV_SymKind_LPROC32_DPC; goto fixup_id; - case CV_SymKind_LPROCMIPS_ID: *sym_kind_ptr = CV_SymKind_LPROCMIPS; goto fixup_id; - case CV_SymKind_GPROCMIPS_ID: *sym_kind_ptr = CV_SymKind_GPROCMIPS; goto fixup_id; - case CV_SymKind_LPROCIA64_ID: *sym_kind_ptr = CV_SymKind_LPROCIA64; goto fixup_id; - case CV_SymKind_GPROCIA64_ID: *sym_kind_ptr = CV_SymKind_GPROCIA64; goto fixup_id; - fixup_id:; { - CV_SymProc32 *proc32 = str8_deserial_get_raw_ptr(symbol.data, 0, sizeof(*proc32)); - if (proc32->itype < min_ti_ipi) { - // TODO: in some cases destructors don't have a type, need a repro - break; - } - - if ((proc32->itype - min_ti_ipi) > leaf_count_ipi) { - Assert(0 && "TODO: error handle corrupted type index"); - break; - } - - U64 leaf_idx = proc32->itype - min_ti_ipi; - String8 leaf_data = str8(leaf_arr_ipi[leaf_idx], max_U64); - - CV_Leaf leaf; - if (cv_read_leaf(leaf_data, 0, 1, &leaf) == 0) { InvalidPath; } - - U64 min_leaf_size = cv_header_struct_size_from_leaf_kind(leaf.kind); - if (min_leaf_size > leaf.data.size) { Assert(!"TODO: error handle corrupt leaf"); break; } - - if (leaf.kind == CV_LeafKind_FUNC_ID) { - CV_LeafFuncId *func_id = str8_deserial_get_raw_ptr(leaf.data, 0, sizeof(*func_id)); - proc32->itype = func_id->itype; - } else if (leaf.kind == CV_LeafKind_MFUNC_ID) { - CV_LeafMFuncId *mfunc_id = str8_deserial_get_raw_ptr(leaf.data, 0, sizeof(*mfunc_id)); - proc32->itype = mfunc_id->itype; - } else { - Assert(!"TODO: erorr handle unexpected leaf type"); - break; - } - } break; - } - } -} - internal THREAD_POOL_TASK_FUNC(lnk_build_obj_ti_map) { @@ -3536,13 +3787,8 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } ProfEnd(); - if (~merge_flags & LNK_MergeTypeFlag_SkipSymbolTypeFixup) { - tp_for_parallel_prof(tp, 0, input->symbol_patch_task_count, lnk_cv_patcher_symbols_task, &task, "Fixup Symbol Type Indices"); - - task.ranges = 0; - task.debug_s_arr = input->debug_s_arr; - tp_for_parallel_prof(tp, 0, input->count, lnk_cv_patcher_inlines_task, &task, "Fixup Inlines Type Indices"); - } + // NOTE: the $S symbol/inlinee TI fixups are journaled below (after the materialize pass); + // the bytes are patched per obj at the start of the module-write visit. // NOTE: the leaf TI-fixup is fused into the unbucket/materialize pass below -- it copies each // unique leaf into a private buffer and patches the copy, instead of patching the mapped input @@ -3607,8 +3853,71 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } } + // Streaming-ring P2 slice A: the $S TI/kind fixups no longer patch bytes here. While the + // merge state they consume is still alive -- the assigned-TI hash tables (merge scratch, + // dead at temp_end below) and the materialized IPI leaf copies (released after + // pdb_build_types, i.e. BEFORE the module-write pass) -- record every write into per-input / + // per-obj journals. lnk_write_pdb_modules replays an obj's journal at the start of its + // module-write visit; configs that read fixed-up $S earlier (or never write modules) run + // lnk_apply_debug_s_fixups_eager instead (see lnk.c). if (~merge_flags & LNK_MergeTypeFlag_SkipSymbolTypeFixup) { - tp_for_parallel_prof(tp, 0, input->symbol_input_count, lnk_fixup_symbols_task, &task, "Fixup ID Symbols"); + ProfBegin("Journal $S Fixups"); + // The journal is dead after the last per-obj replay (module write / eager apply) but is + // GB-class at FN scale -- park it on dedicated per-worker arenas (TYPE_MERGE_SCRATCH + // pattern: TP_Arena header + v[] live inside v[0], released last by tp_arena_release) so + // lnk_release_debug_s_fixup_journal can hand the whole set to the background reaper + // instead of the commit riding the link-lifetime TP arena to process exit. + { + Temp temp = temp_begin(scratch.arena); + Arena **arr = push_array(temp.arena, Arena *, tp->worker_count); + for EachIndex(i, tp->worker_count) { arr[i] = arena_alloc(.commit_size = MB(2), .name = "DEBUG_S_FIXUP_JOURNAL"); } + TP_Arena *journal_arenas = push_array(arr[0], TP_Arena, 1); + journal_arenas->count = tp->worker_count; + journal_arenas->v = push_array(arr[0], Arena *, tp->worker_count); + MemoryCopyTyped(journal_arenas->v, arr, tp->worker_count); + input->debug_s_fixup_journal_arenas = journal_arenas; + temp_end(temp); + } + + // NOTE: all per-obj journal arrays span input->count, NOT input->obj_count -- injected + // type-server / .ifc blob pseudo objs live at indices [obj_count, count) in the parallel + // arrays (their $S is empty, so their journals stay empty, but the inline task dispatches + // over input->count and must have a slot to write). + task.journal_arena = input->debug_s_fixup_journal_arenas; + task.debug_s_arr = input->debug_s_arr; + input->debug_s_sym_fixups = push_array(task.journal_arena->v[0], LNK_DebugSPatchArray, input->symbol_input_count ? input->symbol_input_count : 1); + input->debug_s_inline_fixups = push_array(task.journal_arena->v[0], LNK_DebugSInlineJournal, input->count ? input->count : 1); + input->debug_s_sym_fixup_offsets = push_array(task.journal_arena->v[0], U64, input->count + 1); + { + // symbol_inputs are filled per obj at prefix-sum offsets => obj-contiguous, ascending + U64 *counts = push_array(scratch.arena, U64, input->count ? input->count : 1); + for EachIndex(i, input->symbol_input_count) { counts[input->symbol_inputs[i].obj_idx] += 1; } + U64 acc = 0; + for EachIndex(obj_idx, input->count) { input->debug_s_sym_fixup_offsets[obj_idx] = acc; acc += counts[obj_idx]; } + input->debug_s_sym_fixup_offsets[input->count] = acc; + Assert(acc == input->symbol_input_count); + } + tp_for_parallel_prof(tp, 0, input->symbol_patch_task_count, lnk_journal_symbol_fixups_task, &task, "Journal Symbol Fixups"); + tp_for_parallel_prof(tp, 0, input->count, lnk_journal_inline_fixups_task, &task, "Journal Inline Fixups"); + input->has_debug_s_fixup_journal = 1; + + if (lnk_get_log_status(LNK_Log_Debug)) { + U64 entry_count = 0, entry_bytes = 0, wide_runs = 0; + for EachIndex(i, input->symbol_input_count) { + LNK_DebugSPatchArray *a = &input->debug_s_sym_fixups[i]; + entry_count += a->count; entry_bytes += a->count * (a->is_wide ? sizeof(LNK_DebugSPatchWide) : sizeof(LNK_DebugSPatch)); wide_runs += !!a->is_wide; + } + for EachIndex(i, input->count) { + LNK_DebugSPatchArray *a = &input->debug_s_inline_fixups[i].patches; + entry_count += a->count; entry_bytes += a->count * (a->is_wide ? sizeof(LNK_DebugSPatchWide) : sizeof(LNK_DebugSPatch)); wide_runs += !!a->is_wide; + } + U64 assigned_bytes = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { assigned_bytes += task.assigned_ti_arr[s].cap * (sizeof(CV_TypeIndex) + sizeof(U64)); } + U64 leaf_buffer_bytes = task.result.leaf_buffers[CV_TypeIndexSource_TPI].size + task.result.leaf_buffers[CV_TypeIndexSource_IPI].size; + lnk_log(LNK_Log_Debug, "$S fixup journal: %llu entries / %llu bytes (%llu wide runs) (state it decouples from module write: assigned-TI tables %llu bytes, merged leaf buffers %llu bytes)", + entry_count, entry_bytes, wide_runs, assigned_bytes, leaf_buffer_bytes); + } + ProfEnd(); } MemoryCopyTyped(task.result.min_type_indices, input->min_type_indices, CV_TypeIndexSource_COUNT); @@ -4501,6 +4810,15 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) // compute sizes for module streams for EachIndex(i, obj_indices.count) { U64 obj_idx = obj_indices.v[i]; + // Streaming-ring P2 slice A: replay the obj's deferred $S TI/kind fixups (journaled at + // merge time) before ANY consumer touches these bytes -- the sizing walk here, the write + // pass below, the globals collect after this barrier pass, and the /PDBSTRIPPED re-walk + // after lnk_build_pdb returns. The sizing loop visits every obj in the lane partition -- + // including objs that end up with mod->sn == MSF_INVALID_STREAM_NUMBER, whose $S is still + // consumed by globals -- so no obj misses its fixups. The parity walk below is fixup- + // agnostic (node bytes and resolver alias the same backing), but replaying first keeps + // both sides post-fixup. + lnk_apply_debug_s_fixups_for_obj(task->cv, obj_idx); lnk_assert_debug_s_prov_parity(task->cv->obj_arr[obj_idx], &task->cv->debug_s_arr[obj_idx]); // debug-only P1.2 parity lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0); } @@ -5404,6 +5722,14 @@ ProfScope("Write Modules") lnk_log(LNK_Log_Timers, "[pdb] write modules in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); lnk_summary_phase_end(LNK_SummaryPhase_PdbMod); } + + // the sizing loop above ran the last per-obj $S fixup replay; every later consumer + // (globals collect, /PDBSTRIPPED re-walk) reads the already-patched backing bytes, not + // the journal -- hand the GB-class journal arenas to the background reaper now, before + // the GSI/PSI commit peak. No-op when the eager path already consumed it (or for the + // stripped/SkipSymbolTypeFixup cv, which never built one). + lnk_release_debug_s_fixup_journal(cv); + // module streams were enqueued per-obj inside lnk_write_pdb_modules ProfScope("Move Global Symbols") { diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index fc36d92ba..6e42d7bd6 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -76,6 +76,32 @@ typedef struct LNK_SymbolInputTask U64 weight; } LNK_SymbolInputTask; +// Streaming-ring P2 slice A: deferred .debug$S TI/kind fixup journal. Built inside +// lnk_merge_types while the merge state it needs is still alive (assigned-TI hash tables on +// merge scratch, materialized IPI leaf copies released after pdb_build_types); replayed per +// obj at the START of the module-write visit (lnk_write_pdb_modules sizing loop), before any +// consumer reads the obj's $S bytes. Entries alias the obj's own $S backing (patched copies / +// raw maps); replay applies entries in build order, so the byte end-state is identical to the +// old in-place lnk_cv_patcher_symbols / lnk_cv_patcher_inlines / lnk_fixup_symbols passes. +// Entries are 8B, keyed per NODE run: the target is a byte offset relative to the run's +// subsection-node base (symbols: the LNK_SymbolInput node; inlinees: each InlineeLines +// data_list node in list order), so no pointer is stored. off:31 | width:1 (set = 32-bit +// type-index write, clear = 16-bit kind rewrite). A node >= 2GiB cannot encode -> that run +// falls back to 16B wide entries (per-run is_wide flag), decided up front from the node size. +typedef struct LNK_DebugSPatch { U32 off_w; U32 value; } LNK_DebugSPatch; +typedef struct LNK_DebugSPatchWide { U64 off; U32 value; U32 size; } LNK_DebugSPatchWide; +typedef struct LNK_DebugSPatchArray +{ + void *v; // LNK_DebugSPatch[] or LNK_DebugSPatchWide[] (is_wide) + U64 count; + B32 is_wide; +} LNK_DebugSPatchArray; +typedef struct LNK_DebugSInlineJournal +{ + LNK_DebugSPatchArray patches; // all InlineeLines entries of the obj, node runs in data_list order + U32 *node_counts; // [InlineeLines node_count] entries per node run +} LNK_DebugSInlineJournal; + typedef struct { LNK_Config *config; @@ -114,6 +140,18 @@ typedef struct // [task_id, symbol_input_range_count) strided by the cohort, NOT index by task_id alone. U64 symbol_input_range_count; + // deferred $S TI/kind fixup journal (see LNK_DebugSPatch). Null / flag == 0 when the merge + // ran with LNK_MergeTypeFlag_SkipSymbolTypeFixup, or after the journal was consumed + // (lnk_release_debug_s_fixup_journal: eager path, or end of the module-write pass). + // All journal storage (entry arrays + the three tables below) lives inside dedicated + // per-worker DEBUG_S_FIXUP_JOURNAL arenas, handed to the background reaper at consume + // time -- GB-class at FN scale, dead after the last per-obj replay. + B32 has_debug_s_fixup_journal; + TP_Arena *debug_s_fixup_journal_arenas; + LNK_DebugSPatchArray *debug_s_sym_fixups; // [symbol_input_count] one node run per symbol input + LNK_DebugSInlineJournal *debug_s_inline_fixups; // [count] + U64 *debug_s_sym_fixup_offsets; // [count+1] obj -> symbol_inputs range (inputs are obj-contiguous) + // IFC (header-unit debug-record) resolution: // redirects a consuming obj's local LF_IFC_RECORD placeholder TI to a leaf in // an injected .ifc debug-records blob "obj". Consulted first in lnk_leaf_ref_from_ti. @@ -233,6 +271,10 @@ typedef struct U64 *leaf_buffer_offsets; // [worker_count+1] per-lane byte offsets into leaf_buffer U8 *leaf_buffer; + // $S fixup journal build: per-worker arenas the LNK_DebugSPatch entry arrays land on + // (must outlive the merge -- replay happens at module write) + TP_Arena *journal_arena; + LNK_MergedTypes result; } LNK_MergeTypes; @@ -316,6 +358,9 @@ internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs); internal CV_TypeIndex lnk_assigned_ti_hash_search (LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); internal LNK_MergedTypes lnk_merge_types (TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK_MergeTypeFlags merge_flags); +internal void lnk_apply_debug_s_fixups_for_obj (LNK_CodeViewInput *cv, U64 obj_idx); +internal void lnk_apply_debug_s_fixups_eager (TP_Context *tp, LNK_CodeViewInput *cv); +internal void lnk_release_debug_s_fixup_journal (LNK_CodeViewInput *cv); internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP_Arena *arena, U64 leaf_count, U8 **leaf_arr, LNK_TypeNameHashMode mode, U64 hash_length, String8 map_name); //////////////////////////////// From c47373f3f0da37d77ba8c2cd333376149d810529 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 12 Aug 2026 16:41:55 -0700 Subject: [PATCH 059/121] linker: ring P2b -- global-symbol + proc-ref extraction fused into the module-write visit -- the collect and proc-ref $S walks leave lnk_move_global_symbols_to_gsi (two fewer all-obj barrier passes) and run per obj in the module-write sizing visit, right after journal replay: candidates as 16B {ref,hash} in obj-index positional layout, proc-ref payloads synthesized into their surviving arenas. Dedup/compaction/winner-materialize-at-bucket-fill and the copies release keep their exact pre-P2b positions and data flow (refs point into the live patched backing until fill completes). Module write is now the only phase that walks $S -- the shape the P3 ring window drops into. Cost: ref tables live through the module window, peak 47.2 -> 48.5GB (recovered in P3); output byte-identical (incl /OPT:GCTYPES), deterministic, lyra 9/9, debug asserts live. Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.c | 463 +++++++++++++++++++++--------------- src/linker/lnk_debug_info.h | 42 +++- 2 files changed, 309 insertions(+), 196 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index cc0e95c24..e58014507 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4135,113 +4135,74 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) { Temp scratch = scratch_begin(&arena, 1); - LNK_BuildPdb *task = raw_task; - PDB_GsiContext *gsi = task->pdb->gsi; - PDB_PsiContext *psi = task->pdb->psi; - U32Array obj_indices = task->obj_indices[task_id]; + LNK_BuildPdb *task = raw_task; + PDB_GsiContext *gsi = task->pdb->gsi; + PDB_PsiContext *psi = task->pdb->psi; ProfBegin("Global Symbols"); { -#if BUILD_DEBUG - // Streaming-ring P1.2 parity (debug builds only): each symbol input's raw_symbols is a - // slice of one Symbols subsection node of its obj; find that node by pointer identity, - // resolve its provenance through lnk_resolve_debug_s_node, and assert the CONTENT still - // matches at the very start of the globals collect walk. This runs BEFORE the sect-data - // copies release below (barrier at "Release Sect Data Copies"), so the resolver still - // returns exactly what the parse consumed. Untracked lists and synthetic nodes skip. - for (U64 lane = task_id; lane < task->cv->symbol_input_range_count; lane += tp->worker_count) { - for EachInRange(i, task->cv->symbol_input_ranges[lane]) { - LNK_SymbolInput *in = &task->cv->symbol_inputs[i]; - LNK_Obj *obj = task->cv->obj_arr[in->obj_idx]; - CV_DebugS *debug_s = &task->cv->debug_s_arr[in->obj_idx]; - String8List *data_list = cv_sub_section_ptr_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); - CV_DebugSProvList *prov_list = cv_sub_section_prov_ptr_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); - if (prov_list->count == 0) { continue; } // untracked construction - Assert(prov_list->count == data_list->node_count); - CV_DebugSProvNode *prov = prov_list->first; - String8Node *data_n = data_list->first; - for (; data_n != 0; data_n = data_n->next, prov = prov->next) { - if (data_n->string.str == in->raw_symbols.str && data_n->string.size == in->raw_symbols.size) { break; } - } - Assert(data_n != 0); // every symbol input must originate from a Symbols node - if (prov->is_synthetic) { continue; } - String8 resolved = lnk_resolve_debug_s_node(obj, prov); - Assert(resolved.size == in->raw_symbols.size); - Assert(str8_match(resolved, in->raw_symbols, 0)); - } - } -#endif - // FAIR-SHARE: symbol_input_ranges is a FIXED [symbol_input_range_count] partition built at - // full pool width, but this barrier pass runs at the pinned cohort C == tp->worker_count - // (C <= fixed). Walk the fixed lanes strided by the cohort so every fixed lane is processed - // exactly once for any C; at C == fixed this degenerates to lane == task_id (one lane each, - // identical to the old direct indexing). - VoidList global_symbols = {0}; - for (U64 lane = task_id; lane < task->cv->symbol_input_range_count; lane += tp->worker_count) { - for EachInRange(i, task->cv->symbol_input_ranges[lane]) { - LNK_SymbolInput symbols = task->cv->symbol_inputs[i]; - for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); - - if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { - void *ptr = cv_ptr_from_symbol(symbol); - void_list_push(scratch.arena, &global_symbols, ptr); - } - - if (cv_is_scope_symbol(symbol.kind)) { - depth += 1; - } else if (cv_is_end_symbol(symbol.kind)) { - if (depth == 0) { Assert(0 && "malformed symbol stream"); break; } - depth -= 1; - } - } - } - } - - // collect global data and global typedefs - U64 global_symbol_count = tp_sum_u64(tp, task_id, global_symbols.count); - - U64 bucket_cap; - void **buckets; - U64 *collect_counts; // [worker_count] - void **flat_symbols; // [global_symbol_count] + // P2b: candidate refs were pre-extracted per obj inside the module-write visit -- this + // pass does no $S decode walks; its only $S touches are byte reads through the refs + // (dedup compare + winner materialize at bucket fill), so the patched section copies stay + // alive until the release below. Flatten the per-obj segments in ascending OBJ-INDEX order: + // deterministic and cohort-independent. The old order was the symbol-input lane partition; + // the order change is output-invariant because (a) the deduper is content-CAS (any arrival + // order folds to the same content set), (b) compaction is bucket-slot order feeding a + // content-hash tie-break (dst->offset below), and (c) every GSI bucket chain is + // content-sorted at serialization (gsi_symbol_is_before: name -> offset -> kind -> data + // bytes -- a total order over distinct records; byte-identical records were folded by the + // deduper). Nothing downstream is arrival-order-dependent. + U64 global_symbol_count = 0; + U64 *cand_offsets = 0; // [obj_count+1] prefix sums in obj-index order + void **flat_symbols = 0; // [global_symbol_count] + U64 *flat_hashes = 0; // [global_symbol_count] precomputed at copy time + U64 bucket_cap = 0; + void **buckets = 0; if (task_id == 0) { - bucket_cap = global_symbol_count * 13 / 10; - buckets = push_array(scratch.arena, void *, bucket_cap); - collect_counts = push_array(scratch.arena, U64, tp->worker_count); - flat_symbols = push_array_no_zero(scratch.arena, void *, global_symbol_count ? global_symbol_count : 1); - } + cand_offsets = push_array_no_zero(scratch.arena, U64, task->cv->obj_count + 1); + U64 acc = 0; + for EachIndex(obj_idx, task->cv->obj_count) { + cand_offsets[obj_idx] = acc; + acc += task->preext[obj_idx].cand_count; + } + cand_offsets[task->cv->obj_count] = acc; + global_symbol_count = acc; + flat_symbols = push_array_no_zero(scratch.arena, void *, global_symbol_count ? global_symbol_count : 1); + flat_hashes = push_array_no_zero(scratch.arena, U64, global_symbol_count ? global_symbol_count : 1); + bucket_cap = global_symbol_count * 13 / 10; + buckets = push_array(scratch.arena, void *, bucket_cap); + } + tp_broadcast(&global_symbol_count); + tp_broadcast(&cand_offsets); + tp_broadcast(&flat_symbols); + tp_broadcast(&flat_hashes); tp_broadcast(&bucket_cap); tp_broadcast(&buckets); - tp_broadcast(&collect_counts); - tp_broadcast(&flat_symbols); - // BALANCE: per-lane global-symbol density varies a lot (the collect partition above is - // weighted by raw symbol bytes, not by global-symbol count), which skewed the insert phase - // by ~2x. Flatten the per-worker lists (in worker order) into one array and re-divide the - // INSERTS evenly. The deduper is CAS-based and content-keyed: any insert partition yields - // the same deduped content set, and downstream order is already schedule-independent (the - // per-chain sort keys on the content hash written into n->data.offset below), so this only - // changes who performs an insert, never the output. ProfBegin("Insert Global Symbols"); - collect_counts[task_id] = global_symbols.count; - barrier_wait(tp->barrier); - { - U64 flat_off = 0; - for (U64 w = 0; w < task_id; w += 1) { flat_off += collect_counts[w]; } - for EachNode(n, VoidNode, global_symbols.first) { flat_symbols[flat_off++] = n->v; } + // positional flatten: each obj's segment lands at its prefix-sum offset, so the flat order + // is obj-index order for ANY cohort width or schedule + for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { + LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; + if (pre->cand_count == 0) { continue; } + MemoryCopyTyped(&flat_symbols[cand_offsets[obj_idx]], pre->cand_ptrs, pre->cand_count); + MemoryCopyTyped(&flat_hashes [cand_offsets[obj_idx]], pre->cand_hashes, pre->cand_count); } barrier_wait(tp->barrier); - // insert symbols into hash table (even split of the flattened array) + // insert symbols into hash table (even split of the flattened array). The deduper + // content-compares (cv_symbol_match) -- the key is the record BYTES, never the address -- + // and the hash was precomputed at extraction with the same expression the old code used + // here (u64_hash_from_str8 over cv_raw_from_symbol on the same $S bytes): literally the + // old data flow, only the collect location changed. { U64 ins_lo = (task_id * global_symbol_count) / tp->worker_count; U64 ins_hi = ((task_id + 1) * global_symbol_count) / tp->worker_count; for (U64 i = ins_lo; i < ins_hi; i += 1) { - String8 raw = cv_raw_from_symbol(flat_symbols[i]); - U64 hash = u64_hash_from_str8(raw); - cv_symbol_deduper_insert_or_update(buckets, bucket_cap, hash, flat_symbols[i]); +#if BUILD_DEBUG + Assert(flat_hashes[i] == u64_hash_from_str8(cv_raw_from_symbol(flat_symbols[i]))); +#endif + cv_symbol_deduper_insert_or_update(buckets, bucket_cap, flat_hashes[i], flat_symbols[i]); } } barrier_wait(tp->barrier); @@ -4380,89 +4341,38 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) ProfBegin("Proc Refs"); { - U64 *proc_ref_sizes = 0; - U64 *proc_ref_counts = 0; + // P2b: proc-refs were pre-built per obj at module write (payloads on the surviving + // procref_payload_arenas); flatten the per-obj segments POSITIONALLY in ascending + // obj-index order -- deterministic for any cohort width or schedule, never completion + // order. (The old flat order was the obj_indices lane concatenation, which was already + // cohort-dependent and relied on the content sort at serialization; obj-index order is + // strictly more deterministic.) Zero $S reads. + U64 total_proc_ref_count = 0; + U64 *procref_offsets = 0; // [obj_count+1] prefix sums in obj-index order + U64 *proc_ref_hashes = 0; // [total_proc_ref_count] + CV_Symbol *proc_ref_symbols = 0; // [total_proc_ref_count] if (task_id == 0) { - proc_ref_sizes = push_array(scratch.arena, U64, tp->worker_count); - proc_ref_counts = push_array(scratch.arena, U64, tp->worker_count); - } - tp_broadcast(&proc_ref_sizes); - tp_broadcast(&proc_ref_counts); - - U64 proc_ref_size = 0; - U64 proc_ref_count = 0; - for EachIndex(i, obj_indices.count) { - U64 obj_idx = obj_indices.v[i]; - CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; - String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); - for EachNode(n, String8Node, symbols.first) { - for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); - - if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32) { - String8 name = cv_name_from_symbol(symbol.kind, symbol.data); - proc_ref_size += AlignPow2(sizeof(CV_SymRef2) + name.size + 1, sizeof(void *)); - proc_ref_count += 1; - } - } + procref_offsets = push_array_no_zero(scratch.arena, U64, task->cv->obj_count + 1); + U64 acc = 0; + for EachIndex(obj_idx, task->cv->obj_count) { + procref_offsets[obj_idx] = acc; + acc += task->preext[obj_idx].procref_count; } + procref_offsets[task->cv->obj_count] = acc; + total_proc_ref_count = acc; + proc_ref_hashes = push_array_no_zero(scratch.arena, U64, total_proc_ref_count ? total_proc_ref_count : 1); + proc_ref_symbols = push_array_no_zero(scratch.arena, CV_Symbol, total_proc_ref_count ? total_proc_ref_count : 1); } - proc_ref_sizes[task_id] = proc_ref_size; - proc_ref_counts[task_id] = proc_ref_count; - barrier_wait(tp->barrier); - - U64 total_proc_ref_size = tp_sum_u64(tp, task_id, proc_ref_size); - U64 total_proc_ref_count = tp_sum_u64(tp, task_id, proc_ref_count); - - U64 *proc_ref_hashes = 0; - U64 *proc_ref_indices = 0; - Arena **proc_ref_arenas = 0; - CV_Symbol *proc_ref_symbols = 0; - if (task_id == 0) { - proc_ref_hashes = push_array(scratch.arena, U64, total_proc_ref_count); - proc_ref_indices = offsets_from_counts_array_u64(scratch.arena, proc_ref_counts, tp->worker_count); - proc_ref_arenas = alloc_arena_many(gsi->arena, tp->worker_count, proc_ref_sizes); - proc_ref_symbols = push_array(scratch.arena, CV_Symbol, total_proc_ref_count); - } + tp_broadcast(&total_proc_ref_count); + tp_broadcast(&procref_offsets); tp_broadcast(&proc_ref_hashes); - tp_broadcast(&proc_ref_indices); - tp_broadcast(&proc_ref_arenas); tp_broadcast(&proc_ref_symbols); - Arena *proc_ref_arena = proc_ref_arenas[task_id]; - U64 proc_ref_idx = proc_ref_indices[task_id]; - for EachIndex(i, obj_indices.count) { - U64 obj_idx = obj_indices.v[i]; - CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; - String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); - CV_ModIndex imod = task->mod_arr[obj_idx]->imod; - U64 symbol_cursor = sizeof(CV_Signature); - U64 scope_depth = 0; - for EachNode(n, String8Node, symbols.first) { - for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); - - if (symbol.kind == CV_SymKind_SKIP) { continue; } - else if (cv_is_global_symbol(symbol.kind)) { continue; } - else if (cv_is_typedef(symbol.kind) && scope_depth == 0) { continue; } - else if (symbol.kind == 0x1176) { continue; } - - if (cv_is_scope_symbol(symbol.kind)) { scope_depth += 1; } - else if (cv_is_end_symbol(symbol.kind)) { scope_depth -= 1; } - - if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32) { - String8 name = cv_name_from_symbol(symbol.kind, symbol.data); - proc_ref_symbols[proc_ref_idx] = cv_make_proc_ref(proc_ref_arena, imod, symbol_cursor, name, cv_is_lproc(symbol)); - proc_ref_symbols[proc_ref_idx].offset = symbol_cursor; - proc_ref_hashes [proc_ref_idx] = gsi_hash(gsi, name); - proc_ref_idx += 1; - } - - symbol_cursor += cv_write_symbol_buf(0, 0, &symbol, PDB_SYMBOL_ALIGN); - } - } + for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { + LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; + if (pre->procref_count == 0) { continue; } + MemoryCopyTyped(&proc_ref_symbols[procref_offsets[obj_idx]], pre->procref_syms, pre->procref_count); + MemoryCopyTyped(&proc_ref_hashes [procref_offsets[obj_idx]], pre->procref_hashes, pre->procref_count); } barrier_wait(tp->barrier); @@ -4495,20 +4405,30 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } ProfEnd(); - // the proc-ref build above was the LAST $S reader (publics come from the symbol - // table, GSI/PSI serialize reads the materialized payload copies): drop the patched - // debug-section copies here, before the publics/GSI staging stacks on top of them -- - // releasing after the whole phase kept ~11.6GB overlapping this window on the FN - // editor DLL. Same /PDBSTRIPPED gate as elsewhere (its second build re-reads $S). + // the bucket fill above was the LAST $S reader (it materialized the dedup winners' bytes out + // of the patched copies through the candidate refs; publics come from the symbol table, + // GSI/PSI serialize reads the materialized payloads): drop the patched debug-section copies + // here, before the publics/GSI staging stacks on top of them -- releasing after the whole + // phase kept ~11.6GB overlapping this window on the FN editor DLL. Same /PDBSTRIPPED gate as + // elsewhere (its second build re-reads $S). The P2b candidate/segment ref arenas die with the + // copies (UNCONDITIONALLY -- they are ours): both flattens and the fill consumed them above. barrier_wait(tp->barrier); - if (task_id == 0 && task->free_sect_copies) { - ProfScope("Release Sect Data Copies") { - for EachIndex(obj_idx, task->cv->obj_count) { - lnk_obj_drop_section_data_copies(task->cv->obj_arr[obj_idx]); + if (task_id == 0) { + if (task->free_sect_copies) { + ProfScope("Release Sect Data Copies") { + for EachIndex(obj_idx, task->cv->obj_count) { + lnk_obj_drop_section_data_copies(task->cv->obj_arr[obj_idx]); + } + for EachIndex(i, g_sect_copy_arena_count) { + if (g_sect_copy_arenas[i] != 0) { arena_release(g_sect_copy_arenas[i]); g_sect_copy_arenas[i] = 0; } + } } - for EachIndex(i, g_sect_copy_arena_count) { - if (g_sect_copy_arenas[i] != 0) { arena_release(g_sect_copy_arenas[i]); g_sect_copy_arenas[i] = 0; } + } + ProfScope("Release GSI Candidate Arenas") { + for EachIndex(i, task->preext_arena_count) { + if (task->cand_arenas[i] != 0) { arena_release(task->cand_arenas[i]); task->cand_arenas[i] = 0; } } + MemoryZeroTyped(task->preext, task->cv->obj_count); // poison dangling ref/segment pointers } } barrier_wait(tp->barrier); @@ -4683,7 +4603,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) // dormant provenance recorded at parse time is authoritative by re-resolving every tracked // subsection node through lnk_resolve_debug_s_node and comparing CONTENT against the node's // String8. For reloc-PATCHED sections both the node slice and the resolver point into the -// same sect_data_copies bytes (pointers may even be equal), so the assert is on content, +// same section_data_copies bytes (pointers may even be equal), so the assert is on content, // which also holds across the in-place $S TI/kind fixups (they mutate the shared bytes). // Skips untracked lists (prov count == 0: wholesale synthetic constructions) and synthetic // nodes (no backing section). Valid only while lnk_obj_section_data_from_number still returns what the @@ -4799,6 +4719,139 @@ lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8No typedef struct LNK_PdbOutput LNK_PdbOutput; internal void lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_StreamNumber sn); +// Streaming-ring P2b: extract the obj's GSI inputs -- global-record candidate refs and +// proc-refs -- inside the module-write per-obj visit, so lnk_move_global_symbols_to_gsi does +// no $S record-decode walks (its only remaining $S touch is the bucket-fill materialize of +// the dedup winners' bytes through the refs). Runs right after the obj's deferred $S fixup +// replay, so records are post-fixup (identical bytes to what the old post-modules collect +// walk saw). +// +// Faithful to the two walks it replaces: +// - candidate walk: per-NODE scope depth reset + malformed-end break (the old collect ran per +// symbol input, and symbol inputs are exactly the Symbols data_list nodes); +// - proc-ref walk: obj-continuous scope depth + module-stream cursor (starts at +// sizeof(CV_Signature); records the module write drops -- SKIP, globals, top-level typedefs, +// 0x1176 -- do not advance it). +// Per-obj ref/hash + segment arrays land on the transient per-worker cand_arenas that release +// with the section copies mid-globals. Proc-ref payloads go on the surviving +// procref_payload_arenas (referenced until GSI serialization), same lifetime as the old +// proc_ref_arenas. +internal void +lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id) +{ + CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; + String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); + if (symbols.total_size == 0) { return; } + + LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; + PDB_GsiContext *gsi = task->pdb->gsi; + Arena *cand_arena = task->cand_arenas[task_id]; + + // candidate count pass (mirrors the old collect loop in lnk_move_global_symbols_to_gsi) + U64 cand_count = 0; + for EachNode(n, String8Node, symbols.first) { + for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); + + if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { + cand_count += 1; + } + + if (cv_is_scope_symbol(symbol.kind)) { + depth += 1; + } else if (cv_is_end_symbol(symbol.kind)) { + if (depth == 0) { Assert(0 && "malformed symbol stream"); break; } + depth -= 1; + } + } + } + + // candidate fill pass: store a REF into the live $S backing (exactly the pointer the old + // collect walk pushed -- cv_ptr_from_symbol) + the dedup hash precomputed over those bytes + // (same expression the old insert used: u64_hash_from_str8 over cv_raw_from_symbol). NO + // payload copy -- pre-dedup candidate payloads at FN scale (~6GB) stacked on top of the + // still-alive section copies in the module window. The refs stay valid until the copies + // release mid-globals; the GSI bucket fill materializes only the dedup winners' bytes. + if (cand_count) { + pre->cand_ptrs = push_array_no_zero(cand_arena, void *, cand_count); + pre->cand_hashes = push_array_no_zero(cand_arena, U64, cand_count); + U64 k = 0; + for EachNode(n, String8Node, symbols.first) { + for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); + + if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { + void *ptr = cv_ptr_from_symbol(symbol); + pre->cand_ptrs [k] = ptr; + pre->cand_hashes[k] = u64_hash_from_str8(cv_raw_from_symbol(ptr)); + k += 1; + } + + if (cv_is_scope_symbol(symbol.kind)) { + depth += 1; + } else if (cv_is_end_symbol(symbol.kind)) { + if (depth == 0) { Assert(0 && "malformed symbol stream"); break; } + depth -= 1; + } + } + } + Assert(k == cand_count); + pre->cand_count = cand_count; + } + + // proc-ref count pass (mirrors the old proc-ref sizing walk) + U64 procref_count = 0; + for EachNode(n, String8Node, symbols.first) { + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); + if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32) { + procref_count += 1; + } + } + } + + // proc-ref fill pass (mirrors the old build walk: identical layout math + skip set) + if (procref_count) { + Arena *payload_arena = task->procref_payload_arenas[task_id]; + pre->procref_syms = push_array_no_zero(cand_arena, CV_Symbol, procref_count); + pre->procref_hashes = push_array_no_zero(cand_arena, U64, procref_count); + + CV_ModIndex imod = task->mod_arr[obj_idx]->imod; + U64 symbol_cursor = sizeof(CV_Signature); + U64 scope_depth = 0; + U64 k = 0; + for EachNode(n, String8Node, symbols.first) { + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); + + if (symbol.kind == CV_SymKind_SKIP) { continue; } + else if (cv_is_global_symbol(symbol.kind)) { continue; } + else if (cv_is_typedef(symbol.kind) && scope_depth == 0) { continue; } + else if (symbol.kind == 0x1176) { continue; } + + if (cv_is_scope_symbol(symbol.kind)) { scope_depth += 1; } + else if (cv_is_end_symbol(symbol.kind)) { scope_depth -= 1; } + + if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32) { + String8 name = cv_name_from_symbol(symbol.kind, symbol.data); + pre->procref_syms[k] = cv_make_proc_ref(payload_arena, imod, symbol_cursor, name, cv_is_lproc(symbol)); + pre->procref_syms[k].offset = symbol_cursor; + pre->procref_hashes[k] = gsi_hash(gsi, name); + k += 1; + } + + symbol_cursor += cv_write_symbol_buf(0, 0, &symbol, PDB_SYMBOL_ALIGN); + } + } + Assert(k == procref_count); + pre->procref_count = procref_count; + } +} + internal THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) { @@ -4820,6 +4873,9 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) // both sides post-fixup. lnk_apply_debug_s_fixups_for_obj(task->cv, obj_idx); lnk_assert_debug_s_prov_parity(task->cv->obj_arr[obj_idx], &task->cv->debug_s_arr[obj_idx]); // debug-only P1.2 parity + // P2b: fuse the globals-candidate + proc-ref extraction into this visit (post-fixup bytes, + // same as the old post-modules walks) -- the globals pass then does no $S decode walks + lnk_extract_gsi_inputs_for_obj(task, obj_idx, task_id); lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0); } barrier_wait(tp->barrier); @@ -4981,6 +5037,10 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) barrier_wait(tp->barrier); } + // NOTE (P2b): the section copies do NOT release here -- the candidate refs extracted above + // point into them, and the GSI bucket fill still materializes the dedup winners' bytes out + // of them. Release stays at its pre-P2b position (mid-globals, after the proc-ref insert). + scratch_end(scratch); } @@ -5631,11 +5691,12 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config // per-worker obj indices are (re)distributed per barrier pass to the cohort // that pass actually runs at (FAIR-SHARE: tp->worker_count is pinned to the - // cohort C inside each tp_barrier_begin/end bracket, and the lnk_move_global_ - // symbols_to_gsi / lnk_write_pdb_modules tasks read task.obj_indices[task_id] - // for lanes [0,C)). Distributing to the full worker_count up front would leave - // objs in buckets [C,worker_count) unprocessed when Cdebug_s_arr[obj_idx]); } lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C, weights); + + // P2b pre-extraction state: per-obj GSI input tables (candidate REFS into the live $S + // backing + synthesized proc-refs) filled inside the module-write visit, consumed + // walk-free by "Move Global Symbols". cand_arenas hold only the 16B-class ref/segment + // arrays and are transient (released with the section copies after the proc-ref + // insert); the proc-ref payload arenas survive until GSI serialization (same lifetime + // as the old proc_ref_arenas). + task.preext = push_array(scratch.arena, LNK_GsiPreExtractObj, cv->obj_count); + task.preext_arena_count = C; + task.cand_arenas = push_array(scratch.arena, Arena *, C); + task.procref_payload_arenas = push_array(scratch.arena, Arena *, C); + for EachIndex(i, C) { + task.cand_arenas[i] = arena_alloc(.commit_size = MB(2), .name = "GSI_CANDIDATES"); + task.procref_payload_arenas[i] = arena_alloc(.commit_size = MB(2), .name = "GSI_PROC_REFS"); + } +#if BUILD_DEBUG + // the fused extraction walks objs [0, obj_count) only; the old globals collect walked + // every symbol input, which spans [0, cv->count) INCLUDING injected type-server/.ifc + // blob pseudo-objs -- prove those never carry a Symbols subsection so nothing is missed + for (U64 pseudo_idx = cv->obj_count; pseudo_idx < cv->count; pseudo_idx += 1) { + Assert(cv_sub_section_from_debug_s(cv->debug_s_arr[pseudo_idx], CV_C13SubSectionKind_Symbols).total_size == 0); + } +#endif tp_for_parallel_reserve(tp, 0, C, lnk_write_pdb_modules, &task); // BARRIER pass (path B): barrier_wait/tp_broadcast tp_barrier_end(tp); lnk_log(LNK_Log_Timers, "[pdb] write modules in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); @@ -5735,11 +5819,8 @@ ProfScope("Move Global Symbols") { U64 phase_begin_us = now_time_us(); U32 C = tp_barrier_begin(tp); - // weight = per-obj symbols-subsection byte size: both proc-refs passes walk - // exactly these bytes per obj, and String8List.total_size makes it O(1) - U64 *weights = push_array_no_zero(scratch.arena, U64, cv->obj_count); - for EachIndex(obj_idx, cv->obj_count) { weights[obj_idx] = cv_sub_section_from_debug_s(cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols).total_size; } - lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C, weights); + // P2b: no obj_indices distribution -- the pass reads no $S bytes; it flattens the + // pre-extracted per-obj segments with obj-index striding and even-split inserts tp_for_parallel_reserve(tp, 0, C, lnk_move_global_symbols_to_gsi, &task); // BARRIER pass (path B): tp_sum_u64/tp_broadcast/barrier_wait tp_barrier_end(tp); lnk_log(LNK_Log_Timers, "[pdb] move global symbols in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 6e42d7bd6..c246205e6 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -291,6 +291,27 @@ typedef enum LNK_PDB_BuilderFlag_NATVIS = (1<<5), } LNK_PDB_BuilderFlags; +// Streaming-ring P2b: per-obj global-symbol candidate REFS + proc-refs, extracted inside the +// module-write per-obj visit (fused with the sizing walk, right after the obj's $S fixup +// replay). Candidates are {pointer into the live $S backing, content hash} -- 16B/record, NO +// payload copies (copying every pre-dedup candidate stacked ~6GB on top of the still-alive +// section copies in the module window at FN scale). The $S backing (patched section copies) +// therefore MUST stay alive until the GSI bucket fill materializes the dedup winners' bytes -- +// the copies release keeps its pre-P2b position (mid-globals, after the proc-ref insert). +// Proc-ref payloads are synthesized by cv_make_proc_ref into their own surviving arenas at +// extraction time. lnk_move_global_symbols_to_gsi consumes these tables walk-free (no $S +// record decode), flattening per-obj segments in ascending obj-index order (deterministic and +// cohort-independent). +typedef struct +{ + U64 cand_count; // global records (cv_is_global_symbol + top-level typedefs) + void **cand_ptrs; // [cand_count] record starts INSIDE the obj's $S backing (alive until copies release) + U64 *cand_hashes; // [cand_count] u64_hash_from_str8(raw) computed at extraction + U64 procref_count; // GPROC32/LPROC32 records + CV_Symbol *procref_syms; // [procref_count] cv_make_proc_ref results (payload on survive arenas) + U64 *procref_hashes; // [procref_count] gsi_hash(name) +} LNK_GsiPreExtractObj; + typedef struct { String8 image_data; @@ -315,12 +336,23 @@ typedef struct struct LNK_PdbOutput *output; // when non-null, module streams enqueue to the background writer as they complete // when set, lnk_build_pdb drops every obj's patched debug-section copies - // (LNK_Obj.section_data_copies) and releases the SECT_DATA_COPIES arenas right after - // "Move Global Symbols" (the last $S reader); also gates the /names bucket rehome and - // the mod->source_file_list repoint that decouple the string consumers from the - // copies. Must be 0 when a /PDBSTRIPPED build follows -- it re-walks cv->debug_s_arr - // after lnk_build_pdb. + // (LNK_Obj.section_data_copies) and releases the SECT_DATA_COPIES arenas right after the + // proc-ref insert in "Move Global Symbols" -- the GSI bucket fill is the last $S reader + // (it materializes dedup-winner bytes out of the copies via the P2b candidate refs); also + // gates the /names bucket rehome and the mod->source_file_list repoint that decouple the + // string consumers from the copies. Must be 0 when a /PDBSTRIPPED build follows -- it + // re-walks cv->debug_s_arr after lnk_build_pdb. B32 free_sect_copies; + + // P2b pre-extraction state (see LNK_GsiPreExtractObj). cand_arenas hold ONLY the per-obj + // ref/hash + proc-ref segment arrays (16B-per-record class, no payloads) and release with + // the section copies mid-globals. procref_payload_arenas hold cv_make_proc_ref payloads + // and must survive until GSI serialization (like the old proc_ref_arenas, they live to + // process exit). + LNK_GsiPreExtractObj *preext; // [cv->obj_count] + Arena **cand_arenas; // [preext_arena_count] + Arena **procref_payload_arenas; // [preext_arena_count] + U64 preext_arena_count; // Write Modules cohort width } LNK_BuildPdb; typedef struct From bd046428bd7d7ca8e5428649bf45b69f2fb4ef7b Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 12 Aug 2026 17:13:29 -0700 Subject: [PATCH 060/121] linker: ring P3.1+P3.2 -- unconditional string-table decoupling + dedup/winner-materialize fused into the module-write epilogue -- /names bucket rehome and source_file_list repoint now run regardless of copy state and cover raw-mapped string tables (required before input views can die); global-symbol dedup (content-CAS), slot compaction, and winner payload materialization run as the module-write epilogue while backing is provably live, so the patched-copy release moves to end of Write Modules and nothing downstream reads $S. Transitional peak cost (47.2 -> 49.1GB): the epilogue stacks with in-flight module MSF pages -- deleted wholesale by the P3.3 ring window (copies stop existing). Output byte-identical (incl /OPT:GCTYPES), deterministic, lyra 9/9, debug asserts live. Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.c | 469 ++++++++++++++++++++---------------- src/linker/lnk_debug_info.h | 46 ++-- 2 files changed, 288 insertions(+), 227 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index e58014507..405ca4140 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4141,118 +4141,28 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) ProfBegin("Global Symbols"); { - // P2b: candidate refs were pre-extracted per obj inside the module-write visit -- this - // pass does no $S decode walks; its only $S touches are byte reads through the refs - // (dedup compare + winner materialize at bucket fill), so the patched section copies stay - // alive until the release below. Flatten the per-obj segments in ascending OBJ-INDEX order: - // deterministic and cohort-independent. The old order was the symbol-input lane partition; - // the order change is output-invariant because (a) the deduper is content-CAS (any arrival - // order folds to the same content set), (b) compaction is bucket-slot order feeding a - // content-hash tie-break (dst->offset below), and (c) every GSI bucket chain is - // content-sorted at serialization (gsi_symbol_is_before: name -> offset -> kind -> data - // bytes -- a total order over distinct records; byte-identical records were folded by the - // deduper). Nothing downstream is arrival-order-dependent. - U64 global_symbol_count = 0; - U64 *cand_offsets = 0; // [obj_count+1] prefix sums in obj-index order - void **flat_symbols = 0; // [global_symbol_count] - U64 *flat_hashes = 0; // [global_symbol_count] precomputed at copy time - U64 bucket_cap = 0; - void **buckets = 0; + // P3.2: dedup + compaction + winner materialize ran inside the module-write phase + // (lnk_write_pdb_modules epilogue) while the patched $S backing was alive -- the backing + // and the candidate ref tables are already RELEASED by the time this pass runs. Consume + // only the materialized winner records (task->gsi_winner_ptrs, compacted-slot order, + // payload on gsi->arena): hash names, size buckets, sharded fill. Zero $S touches + // anywhere in this pass. Determinism unchanged: the winner array is compacted-slot order + // (cohort-invariant), the fill below walks it in global order per bucket shard, and every + // GSI bucket chain is content-sorted at serialization (gsi_symbol_is_before). + U64 symbol_count = task->gsi_winner_count; + void **symbol_arr = task->gsi_winner_ptrs; // [symbol_count] materialized copies + + Rng1U64 *symbol_ranges = 0; // [worker_count] + U32 *symbol_hashes = 0; // [symbol_count] if (task_id == 0) { - cand_offsets = push_array_no_zero(scratch.arena, U64, task->cv->obj_count + 1); - U64 acc = 0; - for EachIndex(obj_idx, task->cv->obj_count) { - cand_offsets[obj_idx] = acc; - acc += task->preext[obj_idx].cand_count; - } - cand_offsets[task->cv->obj_count] = acc; - global_symbol_count = acc; - flat_symbols = push_array_no_zero(scratch.arena, void *, global_symbol_count ? global_symbol_count : 1); - flat_hashes = push_array_no_zero(scratch.arena, U64, global_symbol_count ? global_symbol_count : 1); - bucket_cap = global_symbol_count * 13 / 10; - buckets = push_array(scratch.arena, void *, bucket_cap); - } - tp_broadcast(&global_symbol_count); - tp_broadcast(&cand_offsets); - tp_broadcast(&flat_symbols); - tp_broadcast(&flat_hashes); - tp_broadcast(&bucket_cap); - tp_broadcast(&buckets); - - ProfBegin("Insert Global Symbols"); - // positional flatten: each obj's segment lands at its prefix-sum offset, so the flat order - // is obj-index order for ANY cohort width or schedule - for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { - LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; - if (pre->cand_count == 0) { continue; } - MemoryCopyTyped(&flat_symbols[cand_offsets[obj_idx]], pre->cand_ptrs, pre->cand_count); - MemoryCopyTyped(&flat_hashes [cand_offsets[obj_idx]], pre->cand_hashes, pre->cand_count); - } - barrier_wait(tp->barrier); - - // insert symbols into hash table (even split of the flattened array). The deduper - // content-compares (cv_symbol_match) -- the key is the record BYTES, never the address -- - // and the hash was precomputed at extraction with the same expression the old code used - // here (u64_hash_from_str8 over cv_raw_from_symbol on the same $S bytes): literally the - // old data flow, only the collect location changed. - { - U64 ins_lo = (task_id * global_symbol_count) / tp->worker_count; - U64 ins_hi = ((task_id + 1) * global_symbol_count) / tp->worker_count; - for (U64 i = ins_lo; i < ins_hi; i += 1) { -#if BUILD_DEBUG - Assert(flat_hashes[i] == u64_hash_from_str8(cv_raw_from_symbol(flat_symbols[i]))); -#endif - cv_symbol_deduper_insert_or_update(buckets, bucket_cap, flat_hashes[i], flat_symbols[i]); - } - } - barrier_wait(tp->barrier); - ProfEnd(); - - // compact buckets in parallel: each worker owns a contiguous slot range, counts its - // occupied slots, then copies them to its prefix-sum offset. Concatenated ranges preserve - // ascending slot order, so symbol_arr is byte-identical to the old task-0-serial compaction - // (which stalled the other workers at the next barrier for the whole bucket_cap sweep). - U64 symbol_count = 0; - void **symbol_arr = 0; // [symbol_count] - Rng1U64 *symbol_ranges = 0; // [worker_count] - U32 *symbol_hashes = 0; // [symbol_count] - U64 *compact_counts = 0; // [worker_count] - if (task_id == 0) { - compact_counts = push_array(scratch.arena, U64, tp->worker_count); - } - tp_broadcast(&compact_counts); - - ProfBeginV("Compact Buckets [bucket_cap %llu]", bucket_cap); - U64 slot_lo = (task_id * bucket_cap) / tp->worker_count; - U64 slot_hi = ((task_id + 1) * bucket_cap) / tp->worker_count; - { - U64 c = 0; - for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { c += buckets[slot] != 0; } - compact_counts[task_id] = c; - } - barrier_wait(tp->barrier); - - for EachIndex(w, tp->worker_count) { symbol_count += compact_counts[w]; } // same value on every worker - if (task_id == 0) { - symbol_arr = push_array_no_zero(scratch.arena, void *, symbol_count ? symbol_count : 1); symbol_ranges = tp_divide_work(scratch.arena, symbol_count, tp->worker_count); - symbol_hashes = push_array_no_zero(scratch.arena, U32, symbol_count); + symbol_hashes = push_array_no_zero(scratch.arena, U32, symbol_count ? symbol_count : 1); } - tp_broadcast(&symbol_arr); tp_broadcast(&symbol_ranges); tp_broadcast(&symbol_hashes); - { - U64 dst = 0; - for (U64 w = 0; w < task_id; w += 1) { dst += compact_counts[w]; } - for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { - if (buckets[slot]) { symbol_arr[dst++] = buckets[slot]; } - } - } - barrier_wait(tp->barrier); - ProfEnd(); - - // hash symbols + // hash symbols (reads the materialized winner bytes -- byte-identical to the $S records + // they were copied from, so every hash value matches the pre-P3.2 flow) Rng1U64 symbol_range = symbol_ranges[task_id]; for EachInRange(i, symbol_range) { CV_Symbol symbol = cv_symbol_from_ptr(symbol_arr[i]); @@ -4261,34 +4171,6 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } barrier_wait(tp->barrier); - // size per-lane payload arenas for materializing the global records (mirrors the - // proc-ref pattern below): bucket CV_Symbol values must not point into the objs' - // patched debug-section copies -- those are dropped right after this pass in - // lnk_build_pdb, and GSI serialization reads the record bytes after that. Each lane owns the - // bucket shard [shard_min, shard_max) and copies exactly the records it inserts. - U64 *global_payload_sizes = 0; // [worker_count] - Arena **global_payload_arenas = 0; // [worker_count] - if (task_id == 0) { - global_payload_sizes = push_array(scratch.arena, U64, tp->worker_count); - } - tp_broadcast(&global_payload_sizes); - { - U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; - U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; - U64 payload_size = 0; - for EachIndex(i, symbol_count) { - U64 bucket_idx = symbol_hashes[i] % gsi->bucket_count; - if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } - payload_size += AlignPow2(cv_raw_from_symbol(symbol_arr[i]).size, sizeof(void *)); - } - global_payload_sizes[task_id] = payload_size; - } - barrier_wait(tp->barrier); - if (task_id == 0) { - global_payload_arenas = alloc_arena_many(gsi->arena, tp->worker_count, global_payload_sizes); - } - tp_broadcast(&global_payload_arenas); - // size buckets up front so each one reallocs at most once for this wave (arena pushes are // single-threaded on task 0; sizing has no determinism impact) if (task_id == 0) { @@ -4306,21 +4188,17 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) // so per-bucket order (which is serialized into the PDB) is byte-identical to a serial loop, // for any worker count -- no locks, no atomics. { - U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; - U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; - Arena *payload_arena = global_payload_arenas[task_id]; + U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; + U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; for EachIndex(i, symbol_count) { U64 bucket_idx = symbol_hashes[i] % gsi->bucket_count; if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; - CV_Symbol *dst = &bucket->v[bucket->count]; - // materialize: copy the raw record out of the obj's debug-section copy so the - // bucket value survives the copy release right after this pass in lnk_build_pdb - // (byte-identical copy; insert order and all sort keys unchanged) - String8 raw = cv_raw_from_symbol(symbol_arr[i]); - U8 *copy = push_array_no_zero(payload_arena, U8, raw.size); - MemoryCopy(copy, raw.str, raw.size); - *dst = cv_symbol_from_ptr(copy); + CV_Symbol *dst = &bucket->v[bucket->count]; + // the bucket value points at the winner's MATERIALIZED copy (made at the end of Write + // Modules, byte-identical to the original record, alive through GSI serialization) -- + // no copy and no $S read here + *dst = cv_symbol_from_ptr(symbol_arr[i]); // deterministic same-name tie-break for the per-bucket sort in gsi_serialize_symbols_task // (gsi_symbol_is_before compares name -> offset -> kind -> data bytes): key on the content // hash of the full raw record instead of the compacted deduper slot index. Slot order is @@ -4330,6 +4208,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) // symrec bytes run-to-run. The hash is schedule-independent; on collision the comparator's // kind/data-bytes fallback stays content-deterministic, and byte-identical records cannot // reach the sort (the deduper folds them), so the pointer tiebreaker stays unreachable. + // Hashing the materialized copy yields the exact pre-P3.2 value (bytes identical). dst->offset = u64_hash_from_str8(cv_raw_from_symbol(symbol_arr[i])); bucket->count += 1; } @@ -4405,33 +4284,10 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } ProfEnd(); - // the bucket fill above was the LAST $S reader (it materialized the dedup winners' bytes out - // of the patched copies through the candidate refs; publics come from the symbol table, - // GSI/PSI serialize reads the materialized payloads): drop the patched debug-section copies - // here, before the publics/GSI staging stacks on top of them -- releasing after the whole - // phase kept ~11.6GB overlapping this window on the FN editor DLL. Same /PDBSTRIPPED gate as - // elsewhere (its second build re-reads $S). The P2b candidate/segment ref arenas die with the - // copies (UNCONDITIONALLY -- they are ours): both flattens and the fill consumed them above. - barrier_wait(tp->barrier); - if (task_id == 0) { - if (task->free_sect_copies) { - ProfScope("Release Sect Data Copies") { - for EachIndex(obj_idx, task->cv->obj_count) { - lnk_obj_drop_section_data_copies(task->cv->obj_arr[obj_idx]); - } - for EachIndex(i, g_sect_copy_arena_count) { - if (g_sect_copy_arenas[i] != 0) { arena_release(g_sect_copy_arenas[i]); g_sect_copy_arenas[i] = 0; } - } - } - } - ProfScope("Release GSI Candidate Arenas") { - for EachIndex(i, task->preext_arena_count) { - if (task->cand_arenas[i] != 0) { arena_release(task->cand_arenas[i]); task->cand_arenas[i] = 0; } - } - MemoryZeroTyped(task->preext, task->cv->obj_count); // poison dangling ref/segment pointers - } - } - barrier_wait(tp->barrier); + // P3.2: the patched debug-section copies + candidate ref arenas were released at the END of + // the module-write phase (lnk_write_pdb_modules epilogue) -- nothing here reads $S. The + // procref value/hash arrays + payloads consumed above live on the surviving + // procref_payload_arenas (through GSI serialization). ProfBegin("Public Symbols"); { @@ -4732,10 +4588,11 @@ internal void lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context * // - proc-ref walk: obj-continuous scope depth + module-stream cursor (starts at // sizeof(CV_Signature); records the module write drops -- SKIP, globals, top-level typedefs, // 0x1176 -- do not advance it). -// Per-obj ref/hash + segment arrays land on the transient per-worker cand_arenas that release -// with the section copies mid-globals. Proc-ref payloads go on the surviving -// procref_payload_arenas (referenced until GSI serialization), same lifetime as the old -// proc_ref_arenas. +// Per-obj candidate ref/hash arrays land on the transient per-worker cand_arenas that release +// with the section copies at the END of the module-write phase (P3.2). Proc-ref value/hash +// arrays AND payloads go on the surviving procref_payload_arenas (referenced until GSI +// serialization -- the globals pass flattens them AFTER the copies are gone), same lifetime +// as the old proc_ref_arenas. internal void lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id) { @@ -4771,8 +4628,9 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id) // collect walk pushed -- cv_ptr_from_symbol) + the dedup hash precomputed over those bytes // (same expression the old insert used: u64_hash_from_str8 over cv_raw_from_symbol). NO // payload copy -- pre-dedup candidate payloads at FN scale (~6GB) stacked on top of the - // still-alive section copies in the module window. The refs stay valid until the copies - // release mid-globals; the GSI bucket fill materializes only the dedup winners' bytes. + // still-alive section copies in the module window. The refs are consumed by the fused + // dedup + winner materialize at the END of this module-write phase (P3.2) and die there + // with the copies; only the materialized winner bytes survive. if (cand_count) { pre->cand_ptrs = push_array_no_zero(cand_arena, void *, cand_count); pre->cand_hashes = push_array_no_zero(cand_arena, U64, cand_count); @@ -4813,11 +4671,13 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id) } } - // proc-ref fill pass (mirrors the old build walk: identical layout math + skip set) + // proc-ref fill pass (mirrors the old build walk: identical layout math + skip set). + // P3.2: the value/hash arrays live on the SURVIVING payload arena (not cand_arena) -- the + // globals pass flattens them after cand_arenas die with the section copies. if (procref_count) { Arena *payload_arena = task->procref_payload_arenas[task_id]; - pre->procref_syms = push_array_no_zero(cand_arena, CV_Symbol, procref_count); - pre->procref_hashes = push_array_no_zero(cand_arena, U64, procref_count); + pre->procref_syms = push_array_no_zero(payload_arena, CV_Symbol, procref_count); + pre->procref_hashes = push_array_no_zero(payload_arena, U64, procref_count); CV_ModIndex imod = task->mod_arr[obj_idx]->imod; U64 symbol_cursor = sizeof(CV_Signature); @@ -5009,12 +4869,14 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) // collect mod source files String8List source_file_list = str8_split_by_string_chars(string_arenas[task_id], string_table, str8_lit("\0"), 0); - if (task->free_sect_copies) { - // the split nodes alias the string table inside the obj's debug-section copy - // (str8_split does not copy) and DBI file-info hashes these bytes after this - // pass releases the copies. Every piece is present in string_ht -- the dedup - // task split the very same table -- and its bucket bytes were rehomed to a - // surviving blob before the strtab add, so repoint at the bucket's copy. + { + // the split nodes alias the string table inside the obj's $S backing (str8_split + // does not copy) and DBI file-info hashes these bytes after the backing dies -- + // whether that backing is a debug-section COPY (released at the end of this phase) + // or a reloc-free RAW-MAPPED view (alive today, dies in P5). P3.1: repoint + // UNCONDITIONALLY. Every piece is present in string_ht -- the dedup task split the + // very same table -- and its bucket bytes were rehomed to a surviving blob before + // the strtab add, so repoint at the bucket's copy (byte-identical). for EachNode(n, String8Node, source_file_list.first) { CV_StringBucket *bucket = cv_string_hash_table_lookup(task->string_ht, n->string); Assert(bucket != 0); @@ -5037,9 +4899,195 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) barrier_wait(tp->barrier); } - // NOTE (P2b): the section copies do NOT release here -- the candidate refs extracted above - // point into them, and the GSI bucket fill still materializes the dedup winners' bytes out - // of them. Release stays at its pre-P2b position (mid-globals, after the proc-ref insert). + // ===== Streaming-ring P3.2: fused global-symbol dedup + winner materialize ===== + // The candidate refs extracted above point into the objs' patched $S backing, and dedup must + // byte-compare records through them (cv_symbol_match runs on every hash hit -- each duplicate + // fold is a confirm compare of BOTH sides, tens of millions of resolutions at FN's dup + // factor) -- so dedup cannot run after the backing dies. Run the whole pipeline here, as the + // epilogue of the module-write phase, while every obj's backing is still resident; then + // materialize the winner set into surviving arenas; THEN release the copies. + // lnk_move_global_symbols_to_gsi consumes only the materialized records: P2b's + // refs-held-through-globals window is gone, and the copies stop overlapping the + // globals/publics staging entirely. + ProfBegin("Dedup Global Symbols"); + { + PDB_GsiContext *gsi = task->pdb->gsi; + + // positional flatten in ascending obj-index order (identical to the P2b flatten that + // lived in lnk_move_global_symbols_to_gsi: deterministic for any cohort width/schedule) + U64 global_symbol_count = 0; + U64 *cand_offsets = 0; // [obj_count+1] prefix sums in obj-index order + void **flat_symbols = 0; // [global_symbol_count] + U64 *flat_hashes = 0; // [global_symbol_count] precomputed at extraction + U64 bucket_cap = 0; + void **buckets = 0; + if (task_id == 0) { + cand_offsets = push_array_no_zero(scratch.arena, U64, task->cv->obj_count + 1); + U64 acc = 0; + for EachIndex(obj_idx, task->cv->obj_count) { + cand_offsets[obj_idx] = acc; + acc += task->preext[obj_idx].cand_count; + } + cand_offsets[task->cv->obj_count] = acc; + global_symbol_count = acc; + flat_symbols = push_array_no_zero(scratch.arena, void *, global_symbol_count ? global_symbol_count : 1); + flat_hashes = push_array_no_zero(scratch.arena, U64, global_symbol_count ? global_symbol_count : 1); + bucket_cap = global_symbol_count * 13 / 10; + buckets = push_array(scratch.arena, void *, bucket_cap ? bucket_cap : 1); + } + tp_broadcast(&global_symbol_count); + tp_broadcast(&cand_offsets); + tp_broadcast(&flat_symbols); + tp_broadcast(&flat_hashes); + tp_broadcast(&bucket_cap); + tp_broadcast(&buckets); + + for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { + LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; + if (pre->cand_count == 0) { continue; } + MemoryCopyTyped(&flat_symbols[cand_offsets[obj_idx]], pre->cand_ptrs, pre->cand_count); + MemoryCopyTyped(&flat_hashes [cand_offsets[obj_idx]], pre->cand_hashes, pre->cand_count); + } + barrier_wait(tp->barrier); + + // insert into the content-CAS deduper (even split of the flattened array). The deduper + // content-compares (cv_symbol_match) through the refs into the LIVE backing -- the key is + // the record BYTES, never the address -- and any arrival order folds to the same content + // set; the winner among byte-identical duplicates is byte-invisible. + ProfBegin("Insert Global Symbols"); + { + U64 ins_lo = (task_id * global_symbol_count) / tp->worker_count; + U64 ins_hi = ((task_id + 1) * global_symbol_count) / tp->worker_count; + for (U64 i = ins_lo; i < ins_hi; i += 1) { +#if BUILD_DEBUG + // hash-reverify (P3.2: moved here from the globals insert -- the last point where the + // $S backing behind the ref is guaranteed live) + Assert(flat_hashes[i] == u64_hash_from_str8(cv_raw_from_symbol(flat_symbols[i]))); +#endif + cv_symbol_deduper_insert_or_update(buckets, bucket_cap, flat_hashes[i], flat_symbols[i]); + } + } + barrier_wait(tp->barrier); + ProfEnd(); + + // compact buckets in parallel: each worker owns a contiguous slot range, counts its + // occupied slots, then copies them to its prefix-sum offset. Concatenated ranges preserve + // ascending slot order, so symbol_arr is byte-identical to a serial compaction for any + // worker count. + U64 symbol_count = 0; + void **symbol_arr = 0; // [symbol_count] compacted-slot order + U64 *compact_counts = 0; // [worker_count] + if (task_id == 0) { + compact_counts = push_array(scratch.arena, U64, tp->worker_count); + } + tp_broadcast(&compact_counts); + + ProfBeginV("Compact Buckets [bucket_cap %llu]", bucket_cap); + U64 slot_lo = (task_id * bucket_cap) / tp->worker_count; + U64 slot_hi = ((task_id + 1) * bucket_cap) / tp->worker_count; + { + U64 c = 0; + for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { c += buckets[slot] != 0; } + compact_counts[task_id] = c; + } + barrier_wait(tp->barrier); + + for EachIndex(w, tp->worker_count) { symbol_count += compact_counts[w]; } // same value on every worker + if (task_id == 0) { + symbol_arr = push_array_no_zero(scratch.arena, void *, symbol_count ? symbol_count : 1); + } + tp_broadcast(&symbol_arr); + + { + U64 dst = 0; + for (U64 w = 0; w < task_id; w += 1) { dst += compact_counts[w]; } + for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { + if (buckets[slot]) { symbol_arr[dst++] = buckets[slot]; } + } + } + barrier_wait(tp->barrier); + ProfEnd(); + + // winner materialize: copy each winner's raw record out of the still-live backing into + // per-lane payload arenas on gsi->arena (alive through GSI serialization), in compacted + // slot order. The ptr ARRAY order is slot order for any worker count -- only the arena a + // copy lands in varies with the cohort, and every consumer reads through the ordered + // array, never the arenas -- so downstream bytes are cohort-invariant. + ProfBegin("Materialize Global Winners"); + Rng1U64 *mat_ranges = 0; // [worker_count] + U64 *mat_sizes = 0; // [worker_count] + Arena **mat_arenas = 0; // [worker_count] + void **winner_ptrs = 0; // [symbol_count] on gsi->arena + if (task_id == 0) { + mat_ranges = tp_divide_work(scratch.arena, symbol_count, tp->worker_count); + mat_sizes = push_array(scratch.arena, U64, tp->worker_count); + winner_ptrs = push_array_no_zero(gsi->arena, void *, symbol_count ? symbol_count : 1); + } + tp_broadcast(&mat_ranges); + tp_broadcast(&mat_sizes); + tp_broadcast(&winner_ptrs); + + { + Rng1U64 mat_range = mat_ranges[task_id]; + for EachInRange(i, mat_range) { + mat_sizes[task_id] += AlignPow2(cv_raw_from_symbol(symbol_arr[i]).size, sizeof(void *)); + } + } + barrier_wait(tp->barrier); + if (task_id == 0) { + mat_arenas = alloc_arena_many(gsi->arena, tp->worker_count, mat_sizes); + } + tp_broadcast(&mat_arenas); + { + Rng1U64 mat_range = mat_ranges[task_id]; + Arena *dst_arena = mat_arenas[task_id]; + for EachInRange(i, mat_range) { + String8 raw = cv_raw_from_symbol(symbol_arr[i]); + U8 *copy = push_array_no_zero(dst_arena, U8, raw.size); + MemoryCopy(copy, raw.str, raw.size); + winner_ptrs[i] = copy; + } + } + barrier_wait(tp->barrier); + if (task_id == 0) { + task->gsi_winner_count = symbol_count; + task->gsi_winner_ptrs = winner_ptrs; + } + ProfEnd(); + + // copies release: moved here from mid-globals (P3.2). The winner materialize above was + // the LAST $S reader on this path -- release the patched debug-section copies before the + // globals/publics staging begins instead of overlapping it. Same /PDBSTRIPPED gate as + // before (its second build re-walks cv->debug_s_arr Symbols after lnk_build_pdb returns; + // the materialize still ran -- winner ptrs never dangle either way). cand_arenas hold + // only the candidate ref/hash arrays (proc-ref arrays live on the surviving payload + // arenas) and die with the copies UNCONDITIONALLY -- they are ours; poison the dangling + // cand fields, keep the procref fields for the globals pass. The barrier after the copy + // loop above guarantees every worker is done reading before task 0 releases. + if (task_id == 0) { + if (task->free_sect_copies) { + ProfScope("Release Sect Data Copies") { + for EachIndex(obj_idx, task->cv->obj_count) { + lnk_obj_drop_section_data_copies(task->cv->obj_arr[obj_idx]); + } + for EachIndex(i, g_sect_copy_arena_count) { + if (g_sect_copy_arenas[i] != 0) { arena_release(g_sect_copy_arenas[i]); g_sect_copy_arenas[i] = 0; } + } + } + } + ProfScope("Release GSI Candidate Arenas") { + for EachIndex(i, task->preext_arena_count) { + if (task->cand_arenas[i] != 0) { arena_release(task->cand_arenas[i]); task->cand_arenas[i] = 0; } + } + for EachIndex(obj_idx, task->cv->obj_count) { + task->preext[obj_idx].cand_count = 0; + task->preext[obj_idx].cand_ptrs = 0; // poison dangling refs + task->preext[obj_idx].cand_hashes = 0; + } + } + } + } + ProfEnd(); scratch_end(scratch); } @@ -5673,11 +5721,11 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config task.output = output_ptr; - // patched debug-section copies (obj->section_data_copies, GB-class at FN scale) can be - // handed back per-obj during the module-write pass -- UNLESS a /PDBSTRIPPED build - // follows: its pre-build stripping loop re-walks cv->debug_s_arr Symbols (which alias - // the copies) after this build returns. The RDI converter is safe (it reads the PDB - // artifact pages, not obj debug sections). + // patched debug-section copies (obj->section_data_copies, GB-class at FN scale) release at + // the END of the module-write phase, right after the fused dedup + winner materialize + // (P3.2) -- UNLESS a /PDBSTRIPPED build follows: its pre-build stripping loop re-walks + // cv->debug_s_arr Symbols (which alias the copies) after this build returns. The RDI + // converter is safe (it reads the PDB artifact pages, not obj debug sections). task.free_sect_copies = (config->pdb_stripped_name.size == 0); PDB_BuildHooks build_hooks = {0}; @@ -5715,14 +5763,18 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config task.string_ht = cv_dedup_string_tables(tp_arena, tp, cv->obj_count, cv->debug_s_arr); cv_string_hash_table_assign_buffer_offsets(tp, task.string_ht); - // the deduped buckets alias the objs' patched debug-section copies (the dedup task - // splits each raw string table in place), and both /names (pdb_strtab_build memcpys - // bucket bytes at serialize time in pdb_build_dbi_info) and DBI file-info hash these - // bytes AFTER the module-write pass releases the copies -- rehome the winning buckets - // into their own blob first (total = the deduped /names payload, tiny next to the - // copies being released). Offsets are already assigned; bytes are identical, so the - // /names stream and every recorded offset are unchanged. - if (task.free_sect_copies && task.string_ht.total_string_size > 0) { + // the deduped buckets alias the objs' $S string tables in place -- patched debug-section + // COPIES for reloc-carrying $S, RAW-MAPPED view bytes for reloc-free $S -- and both /names + // (pdb_strtab_build memcpys bucket bytes at serialize time in pdb_build_dbi_info) and DBI + // file-info hash these bytes AFTER the backing dies (copies release at the end of Write + // Modules; views die in P5) -- rehome the winning buckets into their own blob first + // (total = the deduped /names payload, tiny next to the backing being released). P3.1: + // UNCONDITIONAL, and the bucket walk below is backing-agnostic (every non-null winning + // bucket is copied, whatever its bytes alias), so raw-mapped tables are covered too. + // Offsets are already assigned; bytes are identical, so the /names stream and every + // recorded offset are unchanged -- /PDBSTRIPPED is unaffected (its second build makes its + // own string_ht from its own debug_s_arr; strtab serialize reads content-identical bytes). + if (task.string_ht.total_string_size > 0) { ProfBegin("Materialize String Table Bytes"); U8 *blob = push_array_no_zero(tp_arena->v[0], U8, task.string_ht.total_string_size); U64 cursor = 0; @@ -5780,11 +5832,12 @@ ProfScope("Write Modules") lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C, weights); // P2b pre-extraction state: per-obj GSI input tables (candidate REFS into the live $S - // backing + synthesized proc-refs) filled inside the module-write visit, consumed - // walk-free by "Move Global Symbols". cand_arenas hold only the 16B-class ref/segment - // arrays and are transient (released with the section copies after the proc-ref - // insert); the proc-ref payload arenas survive until GSI serialization (same lifetime - // as the old proc_ref_arenas). + // backing + synthesized proc-refs) filled inside the module-write visit. P3.2: the + // candidate refs are consumed by the fused dedup + winner materialize in the + // lnk_write_pdb_modules epilogue and die there with the section copies (cand_arenas + // hold only the 16B-class ref arrays); "Move Global Symbols" consumes the materialized + // winners + the proc-ref tables, which live on the surviving payload arenas until GSI + // serialization (same lifetime as the old proc_ref_arenas). task.preext = push_array(scratch.arena, LNK_GsiPreExtractObj, cv->obj_count); task.preext_arena_count = C; task.cand_arenas = push_array(scratch.arena, Arena *, C); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index c246205e6..db5116250 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -295,20 +295,21 @@ typedef enum // module-write per-obj visit (fused with the sizing walk, right after the obj's $S fixup // replay). Candidates are {pointer into the live $S backing, content hash} -- 16B/record, NO // payload copies (copying every pre-dedup candidate stacked ~6GB on top of the still-alive -// section copies in the module window at FN scale). The $S backing (patched section copies) -// therefore MUST stay alive until the GSI bucket fill materializes the dedup winners' bytes -- -// the copies release keeps its pre-P2b position (mid-globals, after the proc-ref insert). -// Proc-ref payloads are synthesized by cv_make_proc_ref into their own surviving arenas at -// extraction time. lnk_move_global_symbols_to_gsi consumes these tables walk-free (no $S -// record decode), flattening per-obj segments in ascending obj-index order (deterministic and -// cohort-independent). +// section copies in the module window at FN scale). P3.2: dedup byte-compares resolve through +// the refs into the live backing, so the whole dedup pipeline (insert + compact + winner +// materialize) runs as the EPILOGUE of the module-write phase while every obj's backing is +// resident; the copies AND the candidate ref tables release at the end of that phase. +// Proc-ref payloads AND the per-obj procref value/hash arrays live on the surviving +// procref_payload_arenas (consumed post-release by lnk_move_global_symbols_to_gsi, which +// flattens per-obj segments in ascending obj-index order -- deterministic and +// cohort-independent -- and otherwise reads only the materialized winner records). typedef struct { U64 cand_count; // global records (cv_is_global_symbol + top-level typedefs) - void **cand_ptrs; // [cand_count] record starts INSIDE the obj's $S backing (alive until copies release) + void **cand_ptrs; // [cand_count] record starts INSIDE the obj's $S backing (dies at end of Write Modules) U64 *cand_hashes; // [cand_count] u64_hash_from_str8(raw) computed at extraction U64 procref_count; // GPROC32/LPROC32 records - CV_Symbol *procref_syms; // [procref_count] cv_make_proc_ref results (payload on survive arenas) + CV_Symbol *procref_syms; // [procref_count] cv_make_proc_ref results (arrays + payload on survive arenas) U64 *procref_hashes; // [procref_count] gsi_hash(name) } LNK_GsiPreExtractObj; @@ -336,23 +337,30 @@ typedef struct struct LNK_PdbOutput *output; // when non-null, module streams enqueue to the background writer as they complete // when set, lnk_build_pdb drops every obj's patched debug-section copies - // (LNK_Obj.section_data_copies) and releases the SECT_DATA_COPIES arenas right after the - // proc-ref insert in "Move Global Symbols" -- the GSI bucket fill is the last $S reader - // (it materializes dedup-winner bytes out of the copies via the P2b candidate refs); also - // gates the /names bucket rehome and the mod->source_file_list repoint that decouple the - // string consumers from the copies. Must be 0 when a /PDBSTRIPPED build follows -- it - // re-walks cv->debug_s_arr after lnk_build_pdb. + // (LNK_Obj.section_data_copies) and releases the SECT_DATA_COPIES arenas at the END of the + // module-write phase, right after the fused dedup + winner materialize (P3.2: the last $S + // readers on this path). Must be 0 when a /PDBSTRIPPED build follows -- it re-walks + // cv->debug_s_arr after lnk_build_pdb. P3.1: the /names bucket rehome and the + // mod->source_file_list repoint are UNCONDITIONAL (they must also cover string tables in + // reloc-free RAW-MAPPED $S sections, which never had copies to begin with). B32 free_sect_copies; // P2b pre-extraction state (see LNK_GsiPreExtractObj). cand_arenas hold ONLY the per-obj - // ref/hash + proc-ref segment arrays (16B-per-record class, no payloads) and release with - // the section copies mid-globals. procref_payload_arenas hold cv_make_proc_ref payloads - // and must survive until GSI serialization (like the old proc_ref_arenas, they live to - // process exit). + // candidate ref/hash arrays (16B-per-record class, no payloads) and release with the + // section copies at the end of Write Modules. procref_payload_arenas hold the procref + // value/hash arrays + cv_make_proc_ref payloads and must survive until GSI serialization + // (like the old proc_ref_arenas, they live to process exit). LNK_GsiPreExtractObj *preext; // [cv->obj_count] Arena **cand_arenas; // [preext_arena_count] Arena **procref_payload_arenas; // [preext_arena_count] U64 preext_arena_count; // Write Modules cohort width + + // P3.2 fused-dedup output: the global-record winner set, materialized at the end of the + // module-write phase in compacted-slot order (ptr array + payload bytes on gsi->arena, + // alive through GSI serialization). The ONLY global-symbol input + // lnk_move_global_symbols_to_gsi reads. + U64 gsi_winner_count; + void **gsi_winner_ptrs; // [gsi_winner_count] } LNK_BuildPdb; typedef struct From e5cfc1e2ae15037cf1274483b10a863587094987 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 12 Aug 2026 17:59:00 -0700 Subject: [PATCH 061/121] linker: ring P3.3 -- .debug$S streams through a reused per-worker window; patched copies eliminated -- lnk_obj_reloc_patcher no longer copies debug sections; lnk_obj_window_debug_s (the ring primitive) re-reads a section from the immutable input view, applies its relocs (lnk_obj_apply_relocs_to_buffer, factored from the patcher) and replays the obj's fixup-journal runs into the window, idempotently; module-write visits consume the window, and the dedup epilogue re-reads candidates by position records {hash, node, off, size} with hash-confirmed grouping (64-bit collision fallback byte-compares; winner set provably equal to the content-CAS deduper). String tables are asserted reloc-free and stay on raw views; /PDBSTRIPPED routes its pre-pass through the window primitive; /OPT:GCTYPES keeps the legacy copy path behind g_debug_s_window. FN editor DLL: peak commit 47.2 -> 43.7GB, working set 119.5 -> 113.0GB, wall +1.3s (dual windowed reads), byte-identical x3 (incl /OPT:GCTYPES), deterministic, lyra 9/9, debug builds carry window-vs-raw and re-read-fidelity asserts. Co-Authored-By: Claude Fable 5 --- src/linker/lnk.c | 176 +++------- src/linker/lnk_debug_info.c | 673 ++++++++++++++++++++++++++++-------- src/linker/lnk_debug_info.h | 26 +- src/linker/lnk_obj.c | 124 +++++++ src/linker/lnk_obj.h | 10 + 5 files changed, 736 insertions(+), 273 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 8fa7a6ab8..7d0f508ca 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -4943,40 +4943,15 @@ lnk_compute_win32_image_header_size(LNK_Config *config, U64 sect_count) return image_header_size; } -// reloc patch ordering: sort each section's relocs by apply_off so the RMW -// write stream into the image is monotone-forward (HW-prefetchable) instead of -// scattered in on-disk table order. apply_off is the primary key; orig_idx is a -// tiebreak so the total order is deterministic and the final bytes are identical -// to the unsorted patch order (each reloc writes its own disjoint field; only a -// pathological same-apply_off overlap could depend on order, and the orig_idx -// tiebreak preserves the original sequence there too). -typedef struct LNK_RelocSortKey -{ - COFF_Reloc reloc; - U32 orig_idx; -} LNK_RelocSortKey; - -internal int -lnk_reloc_sort_key_is_before(void *raw_a, void *raw_b) -{ - LNK_RelocSortKey *a = raw_a, *b = raw_b; - if (a->reloc.apply_off != b->reloc.apply_off) { - return a->reloc.apply_off < b->reloc.apply_off; - } - return a->orig_idx < b->orig_idx; -} - internal THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) { ProfBeginFunction(); - Temp scratch = scratch_begin(0, 0); LNK_ObjRelocPatcher *task = raw_task; LNK_Obj *obj = task->objs[task_id]; - COFF_FileHeaderInfo obj_header = obj->coff.header; - String8 string_table = lnk_coff_string_table_from_obj(obj); + String8 string_table = lnk_coff_string_table_from_obj(obj); for LNK_EachCoffSection(it, obj) { COFF_SectionHeader *section_header = it.v.header; @@ -5002,6 +4977,10 @@ THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) // A relocation-free debug section can stay on the clean input view. if (relocs.count == 0) { continue; } + // With the default streaming window, .debug$S is reconstructed on demand during + // module writing, so no persistent patched copy is needed. + if (g_debug_s_window && str8_match(coff_name_from_section_header(string_table, section_header), str8_lit(".debug$S"), 0)) { continue; } + if (obj->section_data_copies == 0) { obj->section_data_copies = push_array(arena, String8, obj->coff.sections.count_no_null + 1); } @@ -5014,92 +4993,10 @@ THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) section_data = str8_substr(task->image_data, section_frange); } - // apply relocs (sorted by apply_off for monotone-forward image writes) - Temp reloc_temp = temp_begin(scratch.arena); - LNK_RelocSortKey *sorted_relocs = push_array_no_zero(reloc_temp.arena, LNK_RelocSortKey, relocs.count); - for EachIndex(reloc_idx, relocs.count) { - sorted_relocs[reloc_idx].reloc = relocs.v[reloc_idx]; - sorted_relocs[reloc_idx].orig_idx = (U32)reloc_idx; - } - radsort(sorted_relocs, relocs.count, lnk_reloc_sort_key_is_before); - for EachIndex(reloc_idx, relocs.count) { - COFF_Reloc *reloc = &sorted_relocs[reloc_idx].reloc; - - // error check relocation - if (obj->coff.header.machine == COFF_MachineType_X64) { - if (reloc->type > COFF_Reloc_X64_Last) { - lnk_error_obj(LNK_Error_IllegalRelocation, obj, "unknown relocation type 0x%x", reloc->type); - } - } else if (obj->coff.header.machine != COFF_MachineType_Unknown) { - lnk_not_implemented("relocation patching is not implemented for %S", coff_string_from_machine_type(obj->coff.header.machine)); - continue; - } - - // compute virtual offsets - U64 reloc_voff = section_header->voff + reloc->apply_off; - - // compute symbol location values - U32 symbol_secnum = 0; - U32 symbol_secoff = 0; - S64 symbol_voff = 0; - { - COFF_ParsedSymbol symbol = lnk_parsed_symbol_from_coff_symbol_idx_no_name(obj, reloc->isymbol); - COFF_SymbolValueInterpType interp = coff_interp_from_parsed_symbol(symbol); - if (interp == COFF_SymbolValueInterp_Regular) { - if (symbol.section_number == lnk_obj_get_removed_section_number(obj)) { - if (~section_flags & LNK_SECTION_FLAG_DEBUG) { - String8 sect_name = coff_name_from_section_header(string_table, section_header); - String8 symbol_name = lnk_symbol_name_from_coff_symbol_idx(obj, reloc->isymbol); - lnk_error_obj(LNK_Error_RelocationAgainstRemovedSection, obj, "relocating against symbol that is in a removed section (symbol: %S, reloc-section: %S 0x%llx, reloc-index: 0x%llx)", symbol_name, sect_name, it.v.section_number, reloc_idx); - } - continue; - } - symbol_secnum = symbol.section_number; - symbol_secoff = symbol.value; - symbol_voff = safe_cast_u32((U64)task->image_section_table[symbol.section_number]->voff + (U64)symbol_secoff); - } else if (interp == COFF_SymbolValueInterp_Abs) { - // There aren't enough bits in COFF symbol to store full image base address, - // so we special case __ImageBase. A better solution would be to add - // a 64-bit symbol format to COFF. - if (str8_match(lnk_symbol_name_from_coff_symbol_idx(obj, reloc->isymbol), str8_lit("__ImageBase"), 0)) { - symbol.value = task->image_base; - } - symbol_secnum = 0; - symbol_secoff = 0; - symbol_voff = (S64)symbol.value - (S64)task->image_base; - } else if (interp == COFF_SymbolValueInterp_Weak) { - // unresolved weak - } else if (interp == COFF_SymbolValueInterp_Undefined) { - // unresolved undefined - } else { - InvalidPath; - } - } - - // pick reloc value - COFF_RelocValue reloc_value = {0}; - switch (obj_header.machine) { - case COFF_MachineType_Unknown: {} break; - case COFF_MachineType_X64: { reloc_value = coff_pick_reloc_value_x64(reloc->type, task->image_base, reloc_voff, symbol_secnum, symbol_secoff, symbol_voff); } break; - default: { NotImplemented; } break; - } - - // read addend - Assert(reloc_value.size <= section_data.size); - U64 raw_addend = 0; - str8_deserial_read(section_data, reloc->apply_off, &raw_addend, reloc_value.size, 1); - - // compute new reloc value - S64 addend = extend_sign64(raw_addend, reloc_value.size); - U64 reloc_result = reloc_value.value + addend; - - // commit new reloc value - MemoryCopy(section_data.str + reloc->apply_off, &reloc_result, reloc_value.size); - } - temp_end(reloc_temp); + // apply relocs (factored: shared with the P3.3 module-write window fill) + lnk_obj_apply_relocs_to_buffer(obj, it.v.section_number, section_header, section_data, task->image_base, task->image_section_table); } - scratch_end(scratch); ProfEnd(); } @@ -6543,6 +6440,13 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT // patch relocs { + // Streaming-ring P3.3: default to windowed $S consumption (patcher skips $S copies; + // the module-write visit re-reads + patches into a reused per-worker window). + // /OPT:GCTYPES needs the persistent patched+fixed-up $S backing (it reads and rewrites + // type indices in place between the merge and the PDB build) -- keep the old copy-based + // path wholesale there. + g_debug_s_window = (config->opt_gc_types != LNK_SwitchState_Yes); + // dedicated per-worker arenas for the patched debug-section copies: free-list // block reuse keeps the pages warm (a raw reserve+commit per copy paid ~11.6GB of // fresh zero-page faults per link at FN scale), and lnk_build_pdb hands the whole @@ -7547,15 +7451,14 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) lnk_summary_phase_end(LNK_SummaryPhase_DbgMerge); // Streaming-ring P2 slice A: $S TI/kind fixups are journaled in lnk_merge_types and - // normally replayed per obj at the start of the module-write visit (lnk_write_pdb_modules). - // Two configs consume fixed-up $S bytes before/without that pass and need the journal - // applied eagerly instead: - // - /OPT:GCTYPES reads final type indices out of $S (mark roots) and rewrites them - // (compaction remap) right below; - // - a /PDBSTRIPPED-only build (no full PDB / RDI) re-walks $S Symbols without ever - // running the module-write pass. - if (config->opt_gc_types == LNK_SwitchState_Yes || - !(config->debug_mode == LNK_DebugMode_Full || config->rad_debug == LNK_SwitchState_Yes)) { + // normally replayed per obj into the window at the module-write visit + // (lnk_write_pdb_modules). /OPT:GCTYPES consumes fixed-up $S bytes in place right below + // (mark roots + compaction rewrite) and runs with the window disabled (persistent patched + // copies), so it still needs the eager whole-input replay. P3.3: a /PDBSTRIPPED-only + // build no longer takes this path -- its pre-build stripping loop re-reads each Symbols + // node through lnk_obj_window_debug_s, which applies relocs AND replays the journal per + // node, so nothing is written into the raw mapped views. + if (config->opt_gc_types == LNK_SwitchState_Yes) { lnk_apply_debug_s_fixups_eager(tp, &cv); } @@ -7632,12 +7535,36 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // stripped PDB // if (config->pdb_stripped_name.size != 0) { + // P3.3 fold: with the $S window enabled there are no patched copies and no persisted + // fixup replay -- the raw mapped Symbols bytes are pre-reloc and pre-TI-fixup. Re-read + // each obj's Symbols nodes through lnk_obj_window_debug_s (raw view -> window copy + + // relocs + journal replay: exactly the bytes the module-write pass consumed); the + // journal was kept alive across lnk_build_pdb for this (released below). The strip + // loop already copies every surviving record out, so the window is transient per obj. + // With the window disabled (/OPT:GCTYPES) the old flow is intact: free_sect_copies==0 + // kept the patched+eagerly-fixed copies alive and the nodes are read in place. + PE_BinInfo stripped_pe = {0}; + COFF_SectionHeader **stripped_image_sectab = 0; + Temp wscratch = scratch_begin(&scratch.arena, 1); + if (g_debug_s_window) { + stripped_pe = pe_bin_info_from_data(scratch.arena, image_ctx.image_data); + stripped_image_sectab = coff_section_table_from_data(scratch.arena, image_ctx.image_data, stripped_pe.section_table_range); + } + CV_DebugS *debug_s_arr = push_array(scratch.arena, CV_DebugS, cv.obj_count); for EachIndex(obj_idx, cv.obj_count) { + Temp wtemp = temp_begin(wscratch.arena); + CV_DebugS *debug_s_dst = &debug_s_arr[obj_idx]; - CV_DebugS *debug_s_src = &cv.debug_s_arr[obj_idx]; - String8List *src = cv_sub_section_ptr_from_debug_s(debug_s_src, CV_C13SubSectionKind_Symbols); + CV_DebugS debug_s_win = {0}; + String8List *src; + if (g_debug_s_window) { + debug_s_win = lnk_obj_window_debug_s(wtemp.arena, &cv, obj_idx, stripped_pe.image_base, stripped_image_sectab, 1 /* symbols_only */); + src = cv_sub_section_ptr_from_debug_s(&debug_s_win, CV_C13SubSectionKind_Symbols); + } else { + src = cv_sub_section_ptr_from_debug_s(&cv.debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); + } U64 proc_count = 0; U64 proc_size = 0; @@ -7684,7 +7611,14 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // synthesized bytes (TI-stripped copies): provenance marked synthetic cv_debug_s_push_synthetic_sub_section(scratch.arena, debug_s_dst, CV_C13SubSectionKind_Symbols, str8(buffer, buffer_size)); } + + temp_end(wtemp); // window bytes are consumed (records copied into `buffer`) } + scratch_end(wscratch); + + // last $S-journal reader on the stripped path is done (lnk_build_pdb kept the journal + // alive when a /PDBSTRIPPED build follows); no-op when already consumed/never built + lnk_release_debug_s_fixup_journal(&cv); LNK_CodeViewInput stripped_cv = {0}; stripped_cv.config = config; diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 405ca4140..cb8590ed4 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3372,6 +3372,150 @@ lnk_apply_debug_s_fixups_eager(TP_Context *tp, LNK_CodeViewInput *cv) ProfEnd(); } +// high-water mark of a single obj's windowed $S bytes (telemetry: bounds the per-worker +// window arena growth; reported under /RAD_LOG:Debug at the end of Write Modules) +global U64 g_debug_s_window_hwm = 0; + +// ===== Streaming-ring P3.3: the window ========================================================== +// Materializes an obj's parsed .debug$S subsections as CONSUMABLE bytes in `arena` (a per-worker +// scratch reset per obj -- the "window"), without ever writing to the raw mapped input views and +// without any persistent patched copy: +// - every source section a provenance record references is either aliased RAW (no relocs, no +// in-window mutation needed) or COPIED into the window and reloc-patched there via +// lnk_obj_apply_relocs_to_buffer (the exact routine the image-build patcher uses -- relocs, +// symbol tables and the image section table are immutable by module-write time, so the bytes +// are identical to the retired image-time patch-on-copy); +// - Symbols/InlineeLines sections are always windowed while the $S fixup journal is alive +// (replay writes into the window, never a mapped view), and the obj's journal runs replay +// here in build order -- entries are absolute writes, so every re-fill of the window replays +// to identical bytes (relocs are RMW but always start from the immutable raw addend); +// - synthetic / untracked nodes alias their existing linker-made bytes. +// The returned CV_DebugS carries data_list only (no provenance -- parity walks run against the +// original struct). `symbols_only` limits the fill to the Symbols subsection (epilogue dedup +// re-reads, /PDBSTRIPPED pre-pass). Callers own the arena lifetime; nothing in the result may +// outlive it except aliased raw/synthetic node bytes. +internal CV_DebugS +lnk_obj_window_debug_s(Arena *arena, LNK_CodeViewInput *cv, U64 obj_idx, U64 image_base, COFF_SectionHeader **image_section_table, B32 symbols_only) +{ + LNK_Obj *obj = cv->obj_arr[obj_idx]; + CV_DebugS *src = &cv->debug_s_arr[obj_idx]; + CV_DebugS out = {0}; + + U64 idx_symbols = cv_c13_sub_section_idx_from_kind(CV_C13SubSectionKind_Symbols); + U64 idx_inlinees = cv_c13_sub_section_idx_from_kind(CV_C13SubSectionKind_InlineeLines); + + // decide per-section representation from the sections the prov records reference + // (0 = unused, 1 = alias raw view, 2 = window copy + relocs) + U64 sect_count = obj->coff.sections.count_no_null; + U8 *sect_state = push_array(arena, U8, sect_count); + U8 **sect_base = push_array(arena, U8 *, sect_count); + for EachElement(k, src->data_list) { + if (symbols_only && k != idx_symbols) { continue; } + if (src->prov_list[k].count == 0) { continue; } // untracked: nodes alias as-is below + B32 is_journal_kind = (k == idx_symbols || k == idx_inlinees) && cv->has_debug_s_fixup_journal; + for (CV_DebugSProvNode *prov = src->prov_list[k].first; prov != 0; prov = prov->next) { + if (prov->is_synthetic || prov->sect_idx == CV_DebugSProvSect_Nil) { continue; } + U8 state = 1; + if (is_journal_kind) { + state = 2; + } else { + U64 section_number = (U64)prov->sect_idx + 1; + COFF_SectionHeader *hdr = &obj->coff.sections.headers[section_number]; + if (lnk_coff_relocs_from_section_header(obj, hdr).count > 0) { state = 2; } + } + if (state > sect_state[prov->sect_idx]) { sect_state[prov->sect_idx] = state; } + } + } + + // materialize window sections: raw memcpy + reloc application + COFF_SectionHeader *raw_section_table = (COFF_SectionHeader *)str8_substr(obj->coff.data, obj->coff.header.section_table_range).str; + U64 window_size = 0; + for EachIndex(sect_idx, sect_count) { + if (sect_state[sect_idx] == 0) { continue; } + U64 section_number = sect_idx + 1; + LNK_ObjSection section = lnk_obj_section_from_section_number(obj, section_number); + COFF_SectionHeader *raw_hdr = &raw_section_table[sect_idx]; + String8 raw = str8_substr(obj->coff.data, r1u64s(raw_hdr->foff, raw_hdr->fsize)); // always start from the immutable raw view + if (sect_state[sect_idx] == 1) { + sect_base[sect_idx] = raw.str; + } else { + U8 *copy = push_array_no_zero(arena, U8, raw.size); + MemoryCopy(copy, raw.str, raw.size); + lnk_obj_apply_relocs_to_buffer(obj, section_number, section.header, str8(copy, raw.size), image_base, image_section_table); + sect_base[sect_idx] = copy; + window_size += raw.size; + } + } + if (window_size > 0) { + for (U64 hwm = g_debug_s_window_hwm; window_size > hwm; hwm = g_debug_s_window_hwm) { + ins_atomic_u64_eval_cond_assign(&g_debug_s_window_hwm, window_size, hwm); + } + } + + // build the remapped CV_DebugS: tracked nodes -> section base + prov offset, + // synthetic/untracked nodes alias their existing bytes + for EachElement(k, src->data_list) { + if (symbols_only && k != idx_symbols) { continue; } + CV_DebugSProvNode *prov = src->prov_list[k].count ? src->prov_list[k].first : 0; + for (String8Node *n = src->data_list[k].first; n != 0; n = n->next) { + String8 s = n->string; + if (prov != 0 && !prov->is_synthetic && prov->sect_idx != CV_DebugSProvSect_Nil) { + Assert(prov->size == n->string.size); + s = str8(sect_base[prov->sect_idx] + prov->off, prov->size); + } + str8_list_push(arena, &out.data_list[k], s); + if (prov != 0) { prov = prov->next; } + } + } + +#if BUILD_DEBUG + // the string table is consumed from the RAW view by cv_dedup_string_tables and the + // module-write checksum/source-file pass -- prove relocs never alter it (journal entries + // cannot: they only target Symbols/InlineeLines runs by construction) + if (!symbols_only) { + U64 idx_strtab = cv_c13_sub_section_idx_from_kind(CV_C13SubSectionKind_StringTable); + CV_DebugSProvNode *prov = src->prov_list[idx_strtab].count ? src->prov_list[idx_strtab].first : 0; + for (String8Node *n = src->data_list[idx_strtab].first; n != 0 && prov != 0; n = n->next, prov = prov->next) { + if (prov->is_synthetic || prov->sect_idx == CV_DebugSProvSect_Nil) { continue; } + if (sect_state[prov->sect_idx] == 2) { + COFF_SectionHeader *raw_hdr = &raw_section_table[prov->sect_idx]; + String8 raw = str8_substr(obj->coff.data, r1u64s(raw_hdr->foff, raw_hdr->fsize)); + Assert(MemoryMatch(sect_base[prov->sect_idx] + prov->off, raw.str + prov->off, prov->size)); + } + } + } +#endif + + // replay the obj's deferred TI/kind fixup journal into the window: same runs, same order as + // lnk_apply_debug_s_fixups_for_obj -- only the destination base differs (the i-th symbol + // input IS the i-th Symbols data_list node, asserted below) + if (cv->has_debug_s_fixup_journal) { + { + String8Node *n = out.data_list[idx_symbols].first; + for (U64 i = cv->debug_s_sym_fixup_offsets[obj_idx], opl = cv->debug_s_sym_fixup_offsets[obj_idx+1]; i < opl; i += 1, n = n->next) { + AssertAlways(n != 0); + Assert(cv->symbol_inputs[i].obj_idx == obj_idx); + Assert(cv->symbol_inputs[i].raw_symbols.size == n->string.size); + lnk_apply_debug_s_patch_run(n->string.str, &cv->debug_s_sym_fixups[i], 0, cv->debug_s_sym_fixups[i].count); + } + } + if (!symbols_only) { + LNK_DebugSInlineJournal *inline_journal = &cv->debug_s_inline_fixups[obj_idx]; + if (inline_journal->patches.count > 0) { + U64 node_idx = 0, cursor = 0; + for (String8Node *n = out.data_list[idx_inlinees].first; n != 0; n = n->next) { + U64 run = inline_journal->node_counts[node_idx++]; + lnk_apply_debug_s_patch_run(n->string.str, &inline_journal->patches, cursor, cursor + run); + cursor += run; + } + Assert(cursor == inline_journal->patches.count); + } + } + } + + return out; +} + internal THREAD_POOL_TASK_FUNC(lnk_count_unique_leaf_sizes_task) { @@ -4201,14 +4345,13 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) *dst = cv_symbol_from_ptr(symbol_arr[i]); // deterministic same-name tie-break for the per-bucket sort in gsi_serialize_symbols_task // (gsi_symbol_is_before compares name -> offset -> kind -> data bytes): key on the content - // hash of the full raw record instead of the compacted deduper slot index. Slot order is - // CAS-arrival order in cv_symbol_deduper_insert_or_update and can flip between same-name - // different-content records (e.g. duplicate S_UDTs with distinct type indices) whenever the - // lane->worker schedule changes (shared-pool cohorts) or probe chains contend, permuting - // symrec bytes run-to-run. The hash is schedule-independent; on collision the comparator's - // kind/data-bytes fallback stays content-deterministic, and byte-identical records cannot - // reach the sort (the deduper folds them), so the pointer tiebreaker stays unreachable. - // Hashing the materialized copy yields the exact pre-P3.2 value (bytes identical). + // hash of the full raw record, never a slot/array position. The winner ARRAY order is a + // dedup-implementation detail (P3.3: hash-group order; P3.2: compacted CAS slots) and + // per-bucket insert order follows it -- the content-keyed sort is what makes the + // serialized bytes invariant to it. On collision the comparator's kind/data-bytes + // fallback stays content-deterministic, and byte-identical records cannot reach the + // sort (dedup folds them), so the pointer tiebreaker stays unreachable. Hashing the + // materialized copy yields the exact pre-dedup-restructure value (bytes identical). dst->offset = u64_hash_from_str8(cv_raw_from_symbol(symbol_arr[i])); bucket->count += 1; } @@ -4588,15 +4731,16 @@ internal void lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context * // - proc-ref walk: obj-continuous scope depth + module-stream cursor (starts at // sizeof(CV_Signature); records the module write drops -- SKIP, globals, top-level typedefs, // 0x1176 -- do not advance it). -// Per-obj candidate ref/hash arrays land on the transient per-worker cand_arenas that release -// with the section copies at the END of the module-write phase (P3.2). Proc-ref value/hash -// arrays AND payloads go on the surviving procref_payload_arenas (referenced until GSI -// serialization -- the globals pass flattens them AFTER the copies are gone), same lifetime -// as the old proc_ref_arenas. +// Per-obj candidate position/hash arrays (P3.3: {hash, node, off, size} -- no pointers into +// the reused window) land on the transient per-worker cand_arenas that release at the END of +// the module-write phase, after the epilogue dedup consumed them. Proc-ref value/hash arrays +// AND payloads go on the surviving procref_payload_arenas (referenced until GSI serialization +// -- the globals pass flattens them AFTER the window phase), same lifetime as the old +// proc_ref_arenas. internal void -lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id) +lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_DebugS *debug_s_ptr) { - CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; + CV_DebugS debug_s = *debug_s_ptr; // window copy (g_debug_s_window) or the patched backing String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); if (symbols.total_size == 0) { return; } @@ -4624,26 +4768,31 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id) } } - // candidate fill pass: store a REF into the live $S backing (exactly the pointer the old - // collect walk pushed -- cv_ptr_from_symbol) + the dedup hash precomputed over those bytes - // (same expression the old insert used: u64_hash_from_str8 over cv_raw_from_symbol). NO - // payload copy -- pre-dedup candidate payloads at FN scale (~6GB) stacked on top of the - // still-alive section copies in the module window. The refs are consumed by the fused - // dedup + winner materialize at the END of this module-write phase (P3.2) and die there - // with the copies; only the materialized winner bytes survive. + // candidate fill pass (P3.3): store a POSITION record -- {content hash, Symbols node + // ordinal, offset within node, record size} -- NOT a pointer: the window these bytes live + // in is reused for the next obj. The hash is the same expression the old insert used + // (u64_hash_from_str8 over cv_raw_from_symbol) computed over the post-reloc post-fixup + // bytes, so every value matches the pre-P3.3 flow. The epilogue dedup groups by hash and + // re-reads only the bytes it needs through a fresh window fill (deterministic re-fill). if (cand_count) { - pre->cand_ptrs = push_array_no_zero(cand_arena, void *, cand_count); - pre->cand_hashes = push_array_no_zero(cand_arena, U64, cand_count); + pre->cand_hashes = push_array_no_zero(cand_arena, U64, cand_count); + pre->cand_offs = push_array_no_zero(cand_arena, U64, cand_count); + pre->cand_nodes = push_array_no_zero(cand_arena, U32, cand_count); + pre->cand_sizes = push_array_no_zero(cand_arena, U32, cand_count); U64 k = 0; - for EachNode(n, String8Node, symbols.first) { + U64 node_idx = 0; + for (String8Node *n = symbols.first; n != 0; n = n->next, node_idx += 1) { for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { CV_Symbol symbol = {0}; TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { - void *ptr = cv_ptr_from_symbol(symbol); - pre->cand_ptrs [k] = ptr; - pre->cand_hashes[k] = u64_hash_from_str8(cv_raw_from_symbol(ptr)); + U8 *ptr = cv_ptr_from_symbol(symbol); + String8 raw = cv_raw_from_symbol(ptr); + pre->cand_hashes[k] = u64_hash_from_str8(raw); + pre->cand_offs [k] = (U64)(ptr - n->string.str); + pre->cand_nodes [k] = (U32)node_idx; + pre->cand_sizes [k] = (U32)raw.size; k += 1; } @@ -4712,6 +4861,31 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id) } } +// P3.3 epilogue-dedup working types (see the "Dedup Global Symbols" block in +// lnk_write_pdb_modules): candidates sort by (content hash, flat idx) -- flat idx is the +// positional obj-major index, so the tiebreak (and thus each group's LEADER) is +// schedule-independent. +typedef struct { U64 hash; U64 flat_idx; } LNK_GsiSortEnt; +typedef struct { U64 lo; U64 count; } LNK_GsiGroup; +typedef struct LNK_GsiMismatch { struct LNK_GsiMismatch *next; U64 grp; U64 flat_idx; String8 bytes; } LNK_GsiMismatch; +#define LNK_GSI_LEADER_BIT (1u << 31) + +internal int +lnk_gsi_sort_ent_is_before(void *raw_a, void *raw_b) +{ + LNK_GsiSortEnt *a = raw_a, *b = raw_b; + if (a->hash != b->hash) { return a->hash < b->hash; } + return a->flat_idx < b->flat_idx; +} + +internal int +lnk_gsi_mismatch_is_before(void *raw_a, void *raw_b) +{ + LNK_GsiMismatch *a = *(LNK_GsiMismatch **)raw_a, *b = *(LNK_GsiMismatch **)raw_b; + if (a->grp != b->grp) { return a->grp < b->grp; } + return a->flat_idx < b->flat_idx; +} + internal THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) { @@ -4731,12 +4905,25 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) // consumed by globals -- so no obj misses its fixups. The parity walk below is fixup- // agnostic (node bytes and resolver alias the same backing), but replaying first keeps // both sides post-fixup. - lnk_apply_debug_s_fixups_for_obj(task->cv, obj_idx); - lnk_assert_debug_s_prov_parity(task->cv->obj_arr[obj_idx], &task->cv->debug_s_arr[obj_idx]); // debug-only P1.2 parity - // P2b: fuse the globals-candidate + proc-ref extraction into this visit (post-fixup bytes, - // same as the old post-modules walks) -- the globals pass then does no $S decode walks - lnk_extract_gsi_inputs_for_obj(task, obj_idx, task_id); - lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0); + if (g_debug_s_window) { + // P3.3: consume through the per-worker window -- raw view bytes re-read + reloc-patched + // + journal-replayed into a temp that dies with this iteration (the arena keeps its + // high-water pages committed, so the window is reused across objs). The backing views + // are never written; the persistent patched-copy set never exists on this config. + Temp wtemp = temp_begin(scratch.arena); + CV_DebugS win = lnk_obj_window_debug_s(wtemp.arena, task->cv, obj_idx, task->pe.image_base, task->image_section_table, 0); + lnk_assert_debug_s_prov_parity(task->cv->obj_arr[obj_idx], &task->cv->debug_s_arr[obj_idx]); // debug-only P1.2 parity (raw-view provenance) + // P2b: fuse the globals-candidate + proc-ref extraction into this visit (post-fixup bytes, + // same as the old post-modules walks) -- the globals pass then does no $S decode walks + lnk_extract_gsi_inputs_for_obj(task, obj_idx, task_id, &win); + lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], win, 0, 0); + temp_end(wtemp); + } else { + lnk_apply_debug_s_fixups_for_obj(task->cv, obj_idx); + lnk_assert_debug_s_prov_parity(task->cv->obj_arr[obj_idx], &task->cv->debug_s_arr[obj_idx]); // debug-only P1.2 parity + lnk_extract_gsi_inputs_for_obj(task, obj_idx, task_id, &task->cv->debug_s_arr[obj_idx]); + lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0); + } } barrier_wait(tp->barrier); @@ -4761,7 +4948,11 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) if (mod->sn == MSF_INVALID_STREAM_NUMBER) { continue; } - CV_DebugS debug_s = task->cv->debug_s_arr[obj_idx]; + // P3.3: re-fill the window for the stream write (second read of the obj's $S bytes -- + // deterministic re-fill: raw + relocs + journal replays to the exact sizing-pass bytes) + CV_DebugS debug_s = g_debug_s_window + ? lnk_obj_window_debug_s(temp.arena, task->cv, obj_idx, task->pe.image_base, task->image_section_table, 0) + : task->cv->debug_s_arr[obj_idx]; String8List mod_data = msf_data_from_sn(temp.arena, task->pdb->msf, mod->sn); if (mod_data.node_count) { @@ -4899,28 +5090,35 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) barrier_wait(tp->barrier); } - // ===== Streaming-ring P3.2: fused global-symbol dedup + winner materialize ===== - // The candidate refs extracted above point into the objs' patched $S backing, and dedup must - // byte-compare records through them (cv_symbol_match runs on every hash hit -- each duplicate - // fold is a confirm compare of BOTH sides, tens of millions of resolutions at FN's dup - // factor) -- so dedup cannot run after the backing dies. Run the whole pipeline here, as the - // epilogue of the module-write phase, while every obj's backing is still resident; then - // materialize the winner set into surviving arenas; THEN release the copies. - // lnk_move_global_symbols_to_gsi consumes only the materialized records: P2b's - // refs-held-through-globals window is gone, and the copies stop overlapping the - // globals/publics staging entirely. + // ===== Streaming-ring P3.3: two-pass global-symbol dedup + winner materialize ===== + // Candidates are POSITION records ({hash, node, off, size} -- see LNK_GsiPreExtractObj): + // nothing aliases the reused window, so dedup cannot byte-compare through live pointers the + // way the P3.2 content-CAS did. Instead: sort the candidate set by (hash, flat idx), take + // each equal-hash run as a group, then RE-READ only the bytes the outcome needs through a + // fresh symbols-only window fill (deterministic re-fill = the visit bytes): each group's + // LEADER (first member in flat order) materializes as the winner payload; every other + // member re-reads once to byte-confirm equality against its leader. A confirmed mismatch is + // a genuine 64-bit hash collision (expected ~never at any scale we link) and materializes + // as its own additional winner in first-occurrence order -- so the winner CONTENT SET is + // exactly the old deduper's (distinct record bytes), only the array ORDER differs (group + // order instead of compacted-slot order). That order is downstream-invisible: + // lnk_move_global_symbols_to_gsi buckets winners by name hash and every GSI bucket is + // content-sorted at serialization (gsi_symbol_is_before keyed on the content hash written + // into dst->offset), and byte-identical records cannot reach the sort. ProfBegin("Dedup Global Symbols"); { PDB_GsiContext *gsi = task->pdb->gsi; - // positional flatten in ascending obj-index order (identical to the P2b flatten that - // lived in lnk_move_global_symbols_to_gsi: deterministic for any cohort width/schedule) - U64 global_symbol_count = 0; - U64 *cand_offsets = 0; // [obj_count+1] prefix sums in obj-index order - void **flat_symbols = 0; // [global_symbol_count] - U64 *flat_hashes = 0; // [global_symbol_count] precomputed at extraction - U64 bucket_cap = 0; - void **buckets = 0; + // positional flatten in ascending obj-index order (deterministic for any cohort + // width/schedule) -- {hash, flat} sort entries + parallel position arrays + U64 global_symbol_count = 0; + U64 *cand_offsets = 0; // [obj_count+1] prefix sums in obj-index order + LNK_GsiSortEnt *sort_ents = 0; // [N] (hash, flat) -- sorted below + U64 *flat_hashes = 0; // [N] + U64 *flat_offs = 0; // [N] + U32 *flat_nodes = 0; // [N] + U32 *flat_sizes = 0; // [N] + U32 *grp_of_flat = 0; // [N] group id | LNK_GSI_LEADER_BIT if (task_id == 0) { cand_offsets = push_array_no_zero(scratch.arena, U64, task->cv->obj_count + 1); U64 acc = 0; @@ -4930,107 +5128,160 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) } cand_offsets[task->cv->obj_count] = acc; global_symbol_count = acc; - flat_symbols = push_array_no_zero(scratch.arena, void *, global_symbol_count ? global_symbol_count : 1); - flat_hashes = push_array_no_zero(scratch.arena, U64, global_symbol_count ? global_symbol_count : 1); - bucket_cap = global_symbol_count * 13 / 10; - buckets = push_array(scratch.arena, void *, bucket_cap ? bucket_cap : 1); + U64 n = global_symbol_count ? global_symbol_count : 1; + sort_ents = push_array_no_zero(scratch.arena, LNK_GsiSortEnt, n); + flat_hashes = push_array_no_zero(scratch.arena, U64, n); + flat_offs = push_array_no_zero(scratch.arena, U64, n); + flat_nodes = push_array_no_zero(scratch.arena, U32, n); + flat_sizes = push_array_no_zero(scratch.arena, U32, n); + grp_of_flat = push_array_no_zero(scratch.arena, U32, n); } tp_broadcast(&global_symbol_count); tp_broadcast(&cand_offsets); - tp_broadcast(&flat_symbols); + tp_broadcast(&sort_ents); tp_broadcast(&flat_hashes); - tp_broadcast(&bucket_cap); - tp_broadcast(&buckets); + tp_broadcast(&flat_offs); + tp_broadcast(&flat_nodes); + tp_broadcast(&flat_sizes); + tp_broadcast(&grp_of_flat); for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; if (pre->cand_count == 0) { continue; } - MemoryCopyTyped(&flat_symbols[cand_offsets[obj_idx]], pre->cand_ptrs, pre->cand_count); - MemoryCopyTyped(&flat_hashes [cand_offsets[obj_idx]], pre->cand_hashes, pre->cand_count); + U64 base = cand_offsets[obj_idx]; + MemoryCopyTyped(&flat_hashes[base], pre->cand_hashes, pre->cand_count); + MemoryCopyTyped(&flat_offs [base], pre->cand_offs, pre->cand_count); + MemoryCopyTyped(&flat_nodes [base], pre->cand_nodes, pre->cand_count); + MemoryCopyTyped(&flat_sizes [base], pre->cand_sizes, pre->cand_count); + for EachIndex(k, pre->cand_count) { + sort_ents[base + k].hash = pre->cand_hashes[k]; + sort_ents[base + k].flat_idx = base + k; + } } barrier_wait(tp->barrier); - // insert into the content-CAS deduper (even split of the flattened array). The deduper - // content-compares (cv_symbol_match) through the refs into the LIVE backing -- the key is - // the record BYTES, never the address -- and any arrival order folds to the same content - // set; the winner among byte-identical duplicates is byte-invisible. - ProfBegin("Insert Global Symbols"); - { - U64 ins_lo = (task_id * global_symbol_count) / tp->worker_count; - U64 ins_hi = ((task_id + 1) * global_symbol_count) / tp->worker_count; - for (U64 i = ins_lo; i < ins_hi; i += 1) { -#if BUILD_DEBUG - // hash-reverify (P3.2: moved here from the globals insert -- the last point where the - // $S backing behind the ref is guaranteed live) - Assert(flat_hashes[i] == u64_hash_from_str8(cv_raw_from_symbol(flat_symbols[i]))); -#endif - cv_symbol_deduper_insert_or_update(buckets, bucket_cap, flat_hashes[i], flat_symbols[i]); - } + // sort by (hash, flat idx): 256-way MSB scatter (deterministic positional layout: bucket + // base + per-(bucket,worker) prefix over ascending flat slices), then a full per-bucket + // sort -- the final array is a pure function of the candidate set for any cohort + ProfBegin("Group Global Symbols"); + U64 bucket_count = 256; + U64 *scatter_counts = 0; // [C][256] + U64 *scatter_cursor = 0; // [C][256] + U64 *bucket_base = 0; // [256+1] + LNK_GsiSortEnt *sorted = 0; // [N] + Rng1U64 *flat_ranges = 0; // [C] even split of [0,N) + if (task_id == 0) { + scatter_counts = push_array(scratch.arena, U64, tp->worker_count * bucket_count); + scatter_cursor = push_array(scratch.arena, U64, tp->worker_count * bucket_count); + bucket_base = push_array(scratch.arena, U64, bucket_count + 1); + sorted = push_array_no_zero(scratch.arena, LNK_GsiSortEnt, global_symbol_count ? global_symbol_count : 1); + flat_ranges = tp_divide_work(scratch.arena, global_symbol_count, tp->worker_count); + } + tp_broadcast(&scatter_counts); + tp_broadcast(&scatter_cursor); + tp_broadcast(&bucket_base); + tp_broadcast(&sorted); + tp_broadcast(&flat_ranges); + + for EachInRange(i, flat_ranges[task_id]) { + scatter_counts[task_id * bucket_count + (sort_ents[i].hash >> 56)] += 1; } barrier_wait(tp->barrier); - ProfEnd(); - // compact buckets in parallel: each worker owns a contiguous slot range, counts its - // occupied slots, then copies them to its prefix-sum offset. Concatenated ranges preserve - // ascending slot order, so symbol_arr is byte-identical to a serial compaction for any - // worker count. - U64 symbol_count = 0; - void **symbol_arr = 0; // [symbol_count] compacted-slot order - U64 *compact_counts = 0; // [worker_count] if (task_id == 0) { - compact_counts = push_array(scratch.arena, U64, tp->worker_count); + U64 acc = 0; + for EachIndex(b, bucket_count) { + bucket_base[b] = acc; + for EachIndex(w, tp->worker_count) { + scatter_cursor[w * bucket_count + b] = acc; + acc += scatter_counts[w * bucket_count + b]; + } + } + bucket_base[bucket_count] = acc; + Assert(acc == global_symbol_count); } - tp_broadcast(&compact_counts); + barrier_wait(tp->barrier); - ProfBeginV("Compact Buckets [bucket_cap %llu]", bucket_cap); - U64 slot_lo = (task_id * bucket_cap) / tp->worker_count; - U64 slot_hi = ((task_id + 1) * bucket_cap) / tp->worker_count; - { - U64 c = 0; - for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { c += buckets[slot] != 0; } - compact_counts[task_id] = c; + for EachInRange(i, flat_ranges[task_id]) { + U64 b = sort_ents[i].hash >> 56; + sorted[scatter_cursor[task_id * bucket_count + b]++] = sort_ents[i]; + } + barrier_wait(tp->barrier); + + for (U64 b = task_id; b < bucket_count; b += tp->worker_count) { + radsort(sorted + bucket_base[b], bucket_base[b+1] - bucket_base[b], lnk_gsi_sort_ent_is_before); } barrier_wait(tp->barrier); - for EachIndex(w, tp->worker_count) { symbol_count += compact_counts[w]; } // same value on every worker + // groups = equal-hash runs; group ids are bucket-major run ordinals (deterministic) + U64 *bucket_group_counts = 0; // [256] + U64 *group_base = 0; // [256+1] + U64 group_count = 0; + LNK_GsiGroup *groups = 0; // [group_count] if (task_id == 0) { - symbol_arr = push_array_no_zero(scratch.arena, void *, symbol_count ? symbol_count : 1); + bucket_group_counts = push_array(scratch.arena, U64, bucket_count); + group_base = push_array(scratch.arena, U64, bucket_count + 1); } - tp_broadcast(&symbol_arr); + tp_broadcast(&bucket_group_counts); + tp_broadcast(&group_base); - { - U64 dst = 0; - for (U64 w = 0; w < task_id; w += 1) { dst += compact_counts[w]; } - for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { - if (buckets[slot]) { symbol_arr[dst++] = buckets[slot]; } + for (U64 b = task_id; b < bucket_count; b += tp->worker_count) { + U64 c = 0; + for (U64 k = bucket_base[b], opl = bucket_base[b+1]; k < opl; ) { + U64 e = k + 1; + while (e < opl && sorted[e].hash == sorted[k].hash) { e += 1; } + c += 1; k = e; + } + bucket_group_counts[b] = c; + } + barrier_wait(tp->barrier); + + if (task_id == 0) { + U64 acc = 0; + for EachIndex(b, bucket_count) { group_base[b] = acc; acc += bucket_group_counts[b]; } + group_base[bucket_count] = acc; + groups = push_array_no_zero(scratch.arena, LNK_GsiGroup, acc ? acc : 1); + } + group_count = 0; + tp_broadcast(&groups); + if (task_id == 0) { group_count = group_base[bucket_count]; } + tp_broadcast(&group_count); + + for (U64 b = task_id; b < bucket_count; b += tp->worker_count) { + U64 gid = group_base[b]; + for (U64 k = bucket_base[b], opl = bucket_base[b+1]; k < opl; ) { + U64 e = k + 1; + while (e < opl && sorted[e].hash == sorted[k].hash) { e += 1; } + groups[gid].lo = k; + groups[gid].count = e - k; + grp_of_flat[sorted[k].flat_idx] = (U32)gid | LNK_GSI_LEADER_BIT; // min flat idx of the run (flat is the sort tiebreak) + for (U64 m = k + 1; m < e; m += 1) { grp_of_flat[sorted[m].flat_idx] = (U32)gid; } + gid += 1; k = e; } + Assert(gid == group_base[b+1]); } barrier_wait(tp->barrier); ProfEnd(); - // winner materialize: copy each winner's raw record out of the still-live backing into - // per-lane payload arenas on gsi->arena (alive through GSI serialization), in compacted - // slot order. The ptr ARRAY order is slot order for any worker count -- only the arena a - // copy lands in varies with the cohort, and every consumer reads through the ordered - // array, never the arenas -- so downstream bytes are cohort-invariant. + // leader materialize (pass A): per obj, re-read the leader records' bytes (symbols-only + // window fill under g_debug_s_window; the live patched backing otherwise) and copy them + // into per-lane payload arenas on gsi->arena (alive through GSI serialization), keyed by + // group id -- exactly one writer per group. Arena packing order varies with the cohort; + // every consumer reads through winner_ptrs (group order), never the arenas. ProfBegin("Materialize Global Winners"); - Rng1U64 *mat_ranges = 0; // [worker_count] - U64 *mat_sizes = 0; // [worker_count] - Arena **mat_arenas = 0; // [worker_count] - void **winner_ptrs = 0; // [symbol_count] on gsi->arena + U64 *mat_sizes = 0; // [C] leader payload bytes per lane + Arena **mat_arenas = 0; // [C] + void **winner_ptrs = 0; // [group_count] on gsi->arena if (task_id == 0) { - mat_ranges = tp_divide_work(scratch.arena, symbol_count, tp->worker_count); mat_sizes = push_array(scratch.arena, U64, tp->worker_count); - winner_ptrs = push_array_no_zero(gsi->arena, void *, symbol_count ? symbol_count : 1); + winner_ptrs = push_array_no_zero(gsi->arena, void *, group_count ? group_count : 1); } - tp_broadcast(&mat_ranges); tp_broadcast(&mat_sizes); tp_broadcast(&winner_ptrs); - { - Rng1U64 mat_range = mat_ranges[task_id]; - for EachInRange(i, mat_range) { - mat_sizes[task_id] += AlignPow2(cv_raw_from_symbol(symbol_arr[i]).size, sizeof(void *)); + for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { + for (U64 k = cand_offsets[obj_idx], opl = cand_offsets[obj_idx+1]; k < opl; k += 1) { + if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { mat_sizes[task_id] += AlignPow2(flat_sizes[k], sizeof(void *)); } } } barrier_wait(tp->barrier); @@ -5038,21 +5289,147 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) mat_arenas = alloc_arena_many(gsi->arena, tp->worker_count, mat_sizes); } tp_broadcast(&mat_arenas); - { - Rng1U64 mat_range = mat_ranges[task_id]; - Arena *dst_arena = mat_arenas[task_id]; - for EachInRange(i, mat_range) { - String8 raw = cv_raw_from_symbol(symbol_arr[i]); - U8 *copy = push_array_no_zero(dst_arena, U8, raw.size); - MemoryCopy(copy, raw.str, raw.size); - winner_ptrs[i] = copy; + + for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { + U64 lo = cand_offsets[obj_idx], opl = cand_offsets[obj_idx+1]; + + B32 any_leader = 0; + for (U64 k = lo; k < opl; k += 1) { if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { any_leader = 1; break; } } + if (!any_leader) { continue; } + + Temp wtemp = temp_begin(scratch.arena); + CV_DebugS resolved = g_debug_s_window + ? lnk_obj_window_debug_s(wtemp.arena, task->cv, obj_idx, task->pe.image_base, task->image_section_table, 1) + : task->cv->debug_s_arr[obj_idx]; + String8List symbols = cv_sub_section_from_debug_s(resolved, CV_C13SubSectionKind_Symbols); + U8 **node_base = push_array_no_zero(wtemp.arena, U8 *, symbols.node_count ? symbols.node_count : 1); + { U64 ni = 0; for EachNode(n, String8Node, symbols.first) { node_base[ni++] = n->string.str; } } + + Arena *dst_arena = mat_arenas[task_id]; + for (U64 k = lo; k < opl; k += 1) { + if (!(grp_of_flat[k] & LNK_GSI_LEADER_BIT)) { continue; } + U8 *src = node_base[flat_nodes[k]] + flat_offs[k]; + U8 *copy = push_array_no_zero(dst_arena, U8, flat_sizes[k]); + MemoryCopy(copy, src, flat_sizes[k]); +#if BUILD_DEBUG + // re-read fidelity: the fresh window fill must reproduce the visit-time bytes exactly + Assert(flat_hashes[k] == u64_hash_from_str8(str8(copy, flat_sizes[k]))); +#endif + winner_ptrs[grp_of_flat[k] & ~LNK_GSI_LEADER_BIT] = copy; } + temp_end(wtemp); } barrier_wait(tp->barrier); + + // byte-confirm (pass B): every non-leader member re-reads once and memcmps against its + // leader's materialized copy. A mismatch (same 64-bit hash, different bytes) is recorded + // -- bytes copied onto the transient per-lane cand arena -- and resolved serially below. + LNK_GsiMismatch **mismatch_lists = 0; // [C] + U64 *mismatch_counts = 0; // [C] if (task_id == 0) { - task->gsi_winner_count = symbol_count; - task->gsi_winner_ptrs = winner_ptrs; + mismatch_lists = push_array(scratch.arena, LNK_GsiMismatch *, tp->worker_count); + mismatch_counts = push_array(scratch.arena, U64, tp->worker_count); } + tp_broadcast(&mismatch_lists); + tp_broadcast(&mismatch_counts); + + for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { + U64 lo = cand_offsets[obj_idx], opl = cand_offsets[obj_idx+1]; + + B32 any_member = 0; + for (U64 k = lo; k < opl; k += 1) { if (!(grp_of_flat[k] & LNK_GSI_LEADER_BIT)) { any_member = 1; break; } } + if (!any_member) { continue; } + + Temp wtemp = temp_begin(scratch.arena); + CV_DebugS resolved = g_debug_s_window + ? lnk_obj_window_debug_s(wtemp.arena, task->cv, obj_idx, task->pe.image_base, task->image_section_table, 1) + : task->cv->debug_s_arr[obj_idx]; + String8List symbols = cv_sub_section_from_debug_s(resolved, CV_C13SubSectionKind_Symbols); + U8 **node_base = push_array_no_zero(wtemp.arena, U8 *, symbols.node_count ? symbols.node_count : 1); + { U64 ni = 0; for EachNode(n, String8Node, symbols.first) { node_base[ni++] = n->string.str; } } + + for (U64 k = lo; k < opl; k += 1) { + if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { continue; } + U64 grp = grp_of_flat[k]; + U8 *src = node_base[flat_nodes[k]] + flat_offs[k]; + String8 leader = cv_raw_from_symbol(winner_ptrs[grp]); +#if BUILD_DEBUG + Assert(flat_hashes[k] == u64_hash_from_str8(str8(src, flat_sizes[k]))); +#endif + if (leader.size == flat_sizes[k] && MemoryMatch(leader.str, src, flat_sizes[k])) { continue; } + // genuine hash collision: keep the bytes (transient cand arena, dies with the phase) + LNK_GsiMismatch *m = push_array_no_zero(task->cand_arenas[task_id], LNK_GsiMismatch, 1); + U8 *bytes_copy = push_array_no_zero(task->cand_arenas[task_id], U8, flat_sizes[k]); + MemoryCopy(bytes_copy, src, flat_sizes[k]); + m->grp = grp; + m->flat_idx = k; + m->bytes = str8(bytes_copy, flat_sizes[k]); + m->next = mismatch_lists[task_id]; + mismatch_lists[task_id] = m; + mismatch_counts[task_id] += 1; + } + temp_end(wtemp); + } + barrier_wait(tp->barrier); + + // serial collision resolution + final winner array (task 0). Mismatches are expected to + // be ZERO on real inputs; the path exists so a 64-bit collision degrades to correct + // output instead of silently folding distinct records. Distinct extra contents + // materialize in (group, first-occurrence flat) order -- deterministic. + if (task_id == 0) { + U64 total_mismatches = 0; + for EachIndex(w, tp->worker_count) { total_mismatches += mismatch_counts[w]; } + + U64 winner_total = group_count; + + if (total_mismatches == 0) { + task->gsi_winner_count = winner_total; + task->gsi_winner_ptrs = winner_ptrs; + } else { + Temp mtemp = temp_begin(scratch.arena); + LNK_GsiMismatch **flat_mismatches = push_array_no_zero(mtemp.arena, LNK_GsiMismatch *, total_mismatches); + { + U64 c = 0; + for EachIndex(w, tp->worker_count) { + for (LNK_GsiMismatch *m = mismatch_lists[w]; m != 0; m = m->next) { flat_mismatches[c++] = m; } + } + Assert(c == total_mismatches); + } + radsort(flat_mismatches, total_mismatches, lnk_gsi_mismatch_is_before); + + void **extras = push_array_no_zero(mtemp.arena, void *, total_mismatches); + U64 extra_count = 0; + for (U64 a = 0; a < total_mismatches; ) { + U64 b = a; + while (b < total_mismatches && flat_mismatches[b]->grp == flat_mismatches[a]->grp) { b += 1; } + // distinct contents beyond the leader, first-occurrence order within the group + U64 group_extra_lo = extra_count; + for (U64 k = a; k < b; k += 1) { + String8 bytes = flat_mismatches[k]->bytes; + B32 is_distinct = 1; + for (U64 e = group_extra_lo; e < extra_count && is_distinct; e += 1) { + String8 seen = cv_raw_from_symbol(extras[e]); + if (seen.size == bytes.size && MemoryMatch(seen.str, bytes.str, bytes.size)) { is_distinct = 0; } + } + if (is_distinct) { + U8 *copy = push_array_no_zero(gsi->arena, U8, bytes.size); + MemoryCopy(copy, bytes.str, bytes.size); + extras[extra_count++] = copy; + } + } + a = b; + } + + void **final_ptrs = push_array_no_zero(gsi->arena, void *, winner_total + extra_count); + MemoryCopyTyped(final_ptrs, winner_ptrs, winner_total); + MemoryCopyTyped(final_ptrs + winner_total, extras, extra_count); + temp_end(mtemp); + + task->gsi_winner_count = winner_total + extra_count; + task->gsi_winner_ptrs = final_ptrs; + } + } + barrier_wait(tp->barrier); ProfEnd(); // copies release: moved here from mid-globals (P3.2). The winner materialize above was @@ -5081,8 +5458,10 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) } for EachIndex(obj_idx, task->cv->obj_count) { task->preext[obj_idx].cand_count = 0; - task->preext[obj_idx].cand_ptrs = 0; // poison dangling refs - task->preext[obj_idx].cand_hashes = 0; + task->preext[obj_idx].cand_hashes = 0; // poison dangling refs (incl. the pass-B mismatch bytes on cand_arenas) + task->preext[obj_idx].cand_offs = 0; + task->preext[obj_idx].cand_nodes = 0; + task->preext[obj_idx].cand_sizes = 0; } } } @@ -5721,11 +6100,13 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config task.output = output_ptr; - // patched debug-section copies (obj->section_data_copies, GB-class at FN scale) release at - // the END of the module-write phase, right after the fused dedup + winner materialize - // (P3.2) -- UNLESS a /PDBSTRIPPED build follows: its pre-build stripping loop re-walks - // cv->debug_s_arr Symbols (which alias the copies) after this build returns. The RDI - // converter is safe (it reads the PDB artifact pages, not obj debug sections). + // patched debug-section copies (obj->section_data_copies) release at the END of the + // module-write phase. P3.3: with g_debug_s_window set (default) $S never has copies -- + // only the (essentially nonexistent) reloc-bearing non-$S debug sections do -- and the + // /PDBSTRIPPED pre-pass re-reads through lnk_obj_window_debug_s, so the gate below only + // still matters for the /OPT:GCTYPES copy path, where the stripping loop re-walks + // cv->debug_s_arr Symbols aliasing the copies after this build returns. The RDI converter + // is safe (it reads the PDB artifact pages, not obj debug sections). task.free_sect_copies = (config->pdb_stripped_name.size == 0); PDB_BuildHooks build_hooks = {0}; @@ -5857,15 +6238,23 @@ ProfScope("Write Modules") tp_for_parallel_reserve(tp, 0, C, lnk_write_pdb_modules, &task); // BARRIER pass (path B): barrier_wait/tp_broadcast tp_barrier_end(tp); lnk_log(LNK_Log_Timers, "[pdb] write modules in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); + if (g_debug_s_window && lnk_get_log_status(LNK_Log_Debug)) { + // bounds the per-worker window arena growth: worst case commit = cohort x this value + lnk_log(LNK_Log_Debug, "[pdb] $S window high-water: %llu bytes (largest single obj window)", g_debug_s_window_hwm); + } lnk_summary_phase_end(LNK_SummaryPhase_PdbMod); } - // the sizing loop above ran the last per-obj $S fixup replay; every later consumer - // (globals collect, /PDBSTRIPPED re-walk) reads the already-patched backing bytes, not - // the journal -- hand the GB-class journal arenas to the background reaper now, before - // the GSI/PSI commit peak. No-op when the eager path already consumed it (or for the - // stripped/SkipSymbolTypeFixup cv, which never built one). - lnk_release_debug_s_fixup_journal(cv); + // the module-write phase ran the last $S fixup replay on this path -- hand the GB-class + // journal arenas to the background reaper now, before the GSI/PSI commit peak. No-op when + // the eager path already consumed it (or for the stripped/SkipSymbolTypeFixup cv, which + // never built one). P3.3: when a /PDBSTRIPPED build follows, the journal must SURVIVE -- + // the stripping pre-pass re-reads Symbols nodes through lnk_obj_window_debug_s, whose fill + // replays the journal (nothing persists into the raw views anymore); the pre-pass releases + // it when done. + if (config->pdb_stripped_name.size == 0) { + lnk_release_debug_s_fixup_journal(cv); + } // module streams were enqueued per-obj inside lnk_write_pdb_modules ProfScope("Move Global Symbols") diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index db5116250..ab23b7f1a 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -291,14 +291,17 @@ typedef enum LNK_PDB_BuilderFlag_NATVIS = (1<<5), } LNK_PDB_BuilderFlags; -// Streaming-ring P2b: per-obj global-symbol candidate REFS + proc-refs, extracted inside the -// module-write per-obj visit (fused with the sizing walk, right after the obj's $S fixup -// replay). Candidates are {pointer into the live $S backing, content hash} -- 16B/record, NO -// payload copies (copying every pre-dedup candidate stacked ~6GB on top of the still-alive -// section copies in the module window at FN scale). P3.2: dedup byte-compares resolve through -// the refs into the live backing, so the whole dedup pipeline (insert + compact + winner -// materialize) runs as the EPILOGUE of the module-write phase while every obj's backing is -// resident; the copies AND the candidate ref tables release at the end of that phase. +// Streaming-ring P2b/P3.3: per-obj global-symbol candidate REFS + proc-refs, extracted inside +// the module-write per-obj visit (fused with the sizing walk, over the obj's post-fixup $S +// bytes -- the window copy when g_debug_s_window, the patched backing otherwise). P3.3 +// candidates are POSITION records, not pointers: {content hash, Symbols node ordinal, offset +// within node, record size} -- 24B/record, NO payload copies and NO live-backing aliasing, so +// they survive the window being reused for the next obj. The module-write epilogue dedups by +// hash grouping (sort by (hash, flat idx)), then RE-READS just the bytes it needs through the +// same window fill: group leaders to materialize the winner payload, other members to +// byte-confirm equality against their leader (a same-hash different-bytes member is a genuine +// 64-bit collision -- expected ~never -- and materializes as its own winner, preserving the +// old deduper's exact content set). // Proc-ref payloads AND the per-obj procref value/hash arrays live on the surviving // procref_payload_arenas (consumed post-release by lnk_move_global_symbols_to_gsi, which // flattens per-obj segments in ascending obj-index order -- deterministic and @@ -306,8 +309,10 @@ typedef enum typedef struct { U64 cand_count; // global records (cv_is_global_symbol + top-level typedefs) - void **cand_ptrs; // [cand_count] record starts INSIDE the obj's $S backing (dies at end of Write Modules) - U64 *cand_hashes; // [cand_count] u64_hash_from_str8(raw) computed at extraction + U64 *cand_hashes; // [cand_count] u64_hash_from_str8(raw) computed at extraction (post-reloc post-fixup bytes) + U64 *cand_offs; // [cand_count] record start offset within its Symbols data_list node + U32 *cand_nodes; // [cand_count] Symbols data_list node ordinal + U32 *cand_sizes; // [cand_count] full record size (kind + length prefix + data) U64 procref_count; // GPROC32/LPROC32 records CV_Symbol *procref_syms; // [procref_count] cv_make_proc_ref results (arrays + payload on survive arenas) U64 *procref_hashes; // [procref_count] gsi_hash(name) @@ -400,6 +405,7 @@ internal CV_TypeIndex lnk_assigned_ti_hash_search (LNK_AssignedTiHas internal LNK_MergedTypes lnk_merge_types (TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK_MergeTypeFlags merge_flags); internal void lnk_apply_debug_s_fixups_for_obj (LNK_CodeViewInput *cv, U64 obj_idx); internal void lnk_apply_debug_s_fixups_eager (TP_Context *tp, LNK_CodeViewInput *cv); +internal CV_DebugS lnk_obj_window_debug_s (Arena *arena, LNK_CodeViewInput *cv, U64 obj_idx, U64 image_base, COFF_SectionHeader **image_section_table, B32 symbols_only); internal void lnk_release_debug_s_fixup_journal (LNK_CodeViewInput *cv); internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP_Arena *arena, U64 leaf_count, U8 **leaf_arr, LNK_TypeNameHashMode mode, U64 hash_length, String8 map_name); diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index ddae9d767..f66e09684 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -888,6 +888,130 @@ lnk_resolve_debug_s_node(LNK_Obj *obj, CV_DebugSProvNode *prov) return str8_substr(sect_data, rng_1u64(prov->off, prov->off + prov->size)); } +// reloc patch ordering: sort each section's relocs by apply_off so the RMW write stream into +// the destination buffer is monotone-forward (HW-prefetchable) instead of scattered in on-disk +// table order. apply_off is the primary key; orig_idx is a tiebreak so the total order is +// deterministic and the final bytes are identical to the unsorted patch order (each reloc +// writes its own disjoint field; only a pathological same-apply_off overlap could depend on +// order, and the orig_idx tiebreak preserves the original sequence there too). +typedef struct LNK_RelocSortKey +{ + COFF_Reloc reloc; + U32 orig_idx; +} LNK_RelocSortKey; + +internal int +lnk_reloc_sort_key_is_before(void *raw_a, void *raw_b) +{ + LNK_RelocSortKey *a = raw_a, *b = raw_b; + if (a->reloc.apply_off != b->reloc.apply_off) { + return a->reloc.apply_off < b->reloc.apply_off; + } + return a->orig_idx < b->orig_idx; +} + +// Applies one section's relocations into `section_data` -- a writable buffer holding that +// section's bytes (the section's slice of the image at image-build time, a private patched +// copy for non-$S debug sections, or the streaming-ring window copy at module-write time). +// Factored out of lnk_obj_reloc_patcher so the image patch pass and the P3.3 window fill +// share one definition: same reloc order, same symbol resolution, same skip semantics +// (debug-section relocs against removed sections are silently dropped), byte-identical +// application at either call time (relocs + symbol tables + image section table are all +// immutable after the image build). +internal void +lnk_obj_apply_relocs_to_buffer(LNK_Obj *obj, U64 section_number, COFF_SectionHeader *section_header, String8 section_data, U64 image_base, COFF_SectionHeader **image_section_table) +{ + Assert(1 <= section_number && section_number <= obj->coff.sections.count_no_null); + COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(obj, section_header); + if (relocs.count == 0) { return; } + + Temp scratch = scratch_begin(0, 0); + COFF_SectionFlags section_flags = obj->coff.sections.headers[section_number].flags; + + // apply relocs (sorted by apply_off for monotone-forward writes) + LNK_RelocSortKey *sorted_relocs = push_array_no_zero(scratch.arena, LNK_RelocSortKey, relocs.count); + for EachIndex(reloc_idx, relocs.count) { + sorted_relocs[reloc_idx].reloc = relocs.v[reloc_idx]; + sorted_relocs[reloc_idx].orig_idx = (U32)reloc_idx; + } + radsort(sorted_relocs, relocs.count, lnk_reloc_sort_key_is_before); + for EachIndex(reloc_idx, relocs.count) { + COFF_Reloc *reloc = &sorted_relocs[reloc_idx].reloc; + + // error check relocation + if (obj->coff.header.machine == COFF_MachineType_X64) { + if (reloc->type > COFF_Reloc_X64_Last) { + lnk_error_obj(LNK_Error_IllegalRelocation, obj, "unknown relocation type 0x%x", reloc->type); + } + } else if (obj->coff.header.machine != COFF_MachineType_Unknown) { + lnk_not_implemented("relocation patching is not implemented for %S", coff_string_from_machine_type(obj->coff.header.machine)); + continue; + } + + // compute virtual offsets + U64 reloc_voff = section_header->voff + reloc->apply_off; + + // compute symbol location values + U32 symbol_secnum = 0; + U32 symbol_secoff = 0; + S64 symbol_voff = 0; + { + COFF_ParsedSymbol symbol = lnk_parsed_symbol_from_coff_symbol_idx_no_name(obj, reloc->isymbol); + COFF_SymbolValueInterpType interp = coff_interp_from_parsed_symbol(symbol); + if (interp == COFF_SymbolValueInterp_Regular) { + if (symbol.section_number == lnk_obj_get_removed_section_number(obj)) { + if (~section_flags & LNK_SECTION_FLAG_DEBUG) { + String8 sect_name = lnk_obj_section_name_from_section_number(obj, section_number); + String8 symbol_name = lnk_symbol_name_from_coff_symbol_idx(obj, reloc->isymbol); + lnk_error_obj(LNK_Error_RelocationAgainstRemovedSection, obj, "relocating against symbol that is in a removed section (symbol: %S, reloc-section: %S 0x%llx, reloc-index: 0x%llx)", symbol_name, sect_name, section_number, reloc_idx); + } + continue; + } + symbol_secnum = symbol.section_number; + symbol_secoff = symbol.value; + symbol_voff = safe_cast_u32((U64)image_section_table[symbol.section_number]->voff + (U64)symbol_secoff); + } else if (interp == COFF_SymbolValueInterp_Abs) { + // There aren't enough bits in COFF symbol to store full image base address, + // so we special case __ImageBase. A better solution would be to add + // a 64-bit symbol format to COFF. + if (str8_match(lnk_symbol_name_from_coff_symbol_idx(obj, reloc->isymbol), str8_lit("__ImageBase"), 0)) { + symbol.value = image_base; + } + symbol_secnum = 0; + symbol_secoff = 0; + symbol_voff = (S64)symbol.value - (S64)image_base; + } else if (interp == COFF_SymbolValueInterp_Weak) { + // unresolved weak + } else if (interp == COFF_SymbolValueInterp_Undefined) { + // unresolved undefined + } else { + InvalidPath; + } + } + + // pick reloc value + COFF_RelocValue reloc_value = {0}; + switch (obj->coff.header.machine) { + case COFF_MachineType_Unknown: {} break; + case COFF_MachineType_X64: { reloc_value = coff_pick_reloc_value_x64(reloc->type, image_base, reloc_voff, symbol_secnum, symbol_secoff, symbol_voff); } break; + default: { NotImplemented; } break; + } + + // read addend + Assert(reloc_value.size <= section_data.size); + U64 raw_addend = 0; + str8_deserial_read(section_data, reloc->apply_off, &raw_addend, reloc_value.size, 1); + + // compute new reloc value + S64 addend = extend_sign64(raw_addend, reloc_value.size); + U64 reloc_result = reloc_value.value + addend; + + // commit new reloc value + MemoryCopy(section_data.str + reloc->apply_off, &reloc_result, reloc_value.size); + } + scratch_end(scratch); +} + internal THREAD_POOL_TASK_FUNC(lnk_collect_obj_chunks_task) { diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index 24ee4c8c5..5563b2155 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -288,6 +288,16 @@ internal U64 lnk_obj_foff_from_section_data_ptr(LNK_Obj *obj, v internal String8 lnk_obj_section_name_from_section_number(LNK_Obj *obj, U64 section_number); internal void lnk_obj_drop_section_data_copies(LNK_Obj *obj); internal String8 lnk_resolve_debug_s_node(LNK_Obj *obj, CV_DebugSProvNode *prov); +internal void lnk_obj_apply_relocs_to_buffer(LNK_Obj *obj, U64 section_number, COFF_SectionHeader *section_header, String8 section_data, U64 image_base, COFF_SectionHeader **image_section_table); + +// Streaming-ring P3.3: when set (default), .debug$S sections are NOT copied+patched at +// image-build time (lnk_obj_reloc_patcher skips them); the PDB module-write visit re-reads +// each obj's raw mapped bytes into a small per-worker window and applies relocs + the +// journaled TI/kind fixups there (lnk_obj_window_debug_s), so the GB-class patched-copy set +// never exists. Cleared under /OPT:GCTYPES, which reads AND rewrites $S type indices in +// place after an eager journal apply and therefore needs the persistent patched copies +// (the old path, kept intact). +global B32 g_debug_s_window = 1; internal LNK_ObjSection lnk_obj_section_from_section_number(LNK_Obj *obj, U64 section_number); internal COFF_RelocArray lnk_coff_relocs_from_section_header(LNK_Obj *obj, COFF_SectionHeader *section_header); internal String8 lnk_coff_string_table_from_obj(LNK_Obj *obj); From 58d859d6a49ceb9d72bdff7595913eeee81d3169 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 12 Aug 2026 18:17:29 -0700 Subject: [PATCH 062/121] linker: ring P3.4 -- one window fill per obj -- module sizing no longer fills a window: c13/lines/globrefs sizes come from subsection metadata and the Symbols term walks record headers on the raw view (record size/kind fields are never reloc targets, and journal kind rewrites are classification-invariant for the sizing skip set -- verified against cv_is_scope_symbol/cv_is_end_symbol, asserted in debug builds by re-accumulating sizes from the write-pass window). Journal replay, parity, extraction and stream write ride the write loop's single fill. Also fixes a temp leak on the sn==INVALID continue. Byte-identical (incl /OPT:GCTYPES), deterministic x3, lyra 9/9, peak 43.8GB steady. Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.c | 82 +++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index cb8590ed4..e922444c9 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4897,33 +4897,19 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) // compute sizes for module streams for EachIndex(i, obj_indices.count) { U64 obj_idx = obj_indices.v[i]; - // Streaming-ring P2 slice A: replay the obj's deferred $S TI/kind fixups (journaled at - // merge time) before ANY consumer touches these bytes -- the sizing walk here, the write - // pass below, the globals collect after this barrier pass, and the /PDBSTRIPPED re-walk - // after lnk_build_pdb returns. The sizing loop visits every obj in the lane partition -- - // including objs that end up with mod->sn == MSF_INVALID_STREAM_NUMBER, whose $S is still - // consumed by globals -- so no obj misses its fixups. The parity walk below is fixup- - // agnostic (node bytes and resolver alias the same backing), but replaying first keeps - // both sides post-fixup. - if (g_debug_s_window) { - // P3.3: consume through the per-worker window -- raw view bytes re-read + reloc-patched - // + journal-replayed into a temp that dies with this iteration (the arena keeps its - // high-water pages committed, so the window is reused across objs). The backing views - // are never written; the persistent patched-copy set never exists on this config. - Temp wtemp = temp_begin(scratch.arena); - CV_DebugS win = lnk_obj_window_debug_s(wtemp.arena, task->cv, obj_idx, task->pe.image_base, task->image_section_table, 0); - lnk_assert_debug_s_prov_parity(task->cv->obj_arr[obj_idx], &task->cv->debug_s_arr[obj_idx]); // debug-only P1.2 parity (raw-view provenance) - // P2b: fuse the globals-candidate + proc-ref extraction into this visit (post-fixup bytes, - // same as the old post-modules walks) -- the globals pass then does no $S decode walks - lnk_extract_gsi_inputs_for_obj(task, obj_idx, task_id, &win); - lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], win, 0, 0); - temp_end(wtemp); - } else { - lnk_apply_debug_s_fixups_for_obj(task->cv, obj_idx); - lnk_assert_debug_s_prov_parity(task->cv->obj_arr[obj_idx], &task->cv->debug_s_arr[obj_idx]); // debug-only P1.2 parity - lnk_extract_gsi_inputs_for_obj(task, obj_idx, task_id, &task->cv->debug_s_arr[obj_idx]); - lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0); - } + // P3.4: sizing needs NO window fill. Every term of the module stream size is either pure + // subsection metadata (C13 layout kinds come from the data_list INDEX, sizes/alignment + // from node sizes) or -- for sym_data_size -- a read-only walk of the Symbols records' + // size prefixes + kinds. Both are invariant under the window transforms: relocs and TI + // fixups never touch record size/kind fields, and the journal's kind REWRITES are + // classification-invariant for every predicate the sizing walk uses + // (cv_is_scope_symbol treats GPROC32_ID/LPROC32_ID like their non-ID rewrites, + // cv_is_end_symbol treats PROC_ID_END like END, and no rewrite produces or removes a + // global/typedef/SKIP/0x1176 kind). So size straight off the raw mapped view under + // g_debug_s_window (the patched backing otherwise) -- the write pass below asserts the + // re-accumulated sizes match. Fixup replay, parity, and the GSI extraction all move to + // the write pass's single fill. + lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0); } barrier_wait(tp->barrier); @@ -4946,20 +4932,46 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) U64 obj_idx = obj_indices.v[i]; PDB_DbiModule *mod = task->mod_arr[obj_idx]; - if (mod->sn == MSF_INVALID_STREAM_NUMBER) { continue; } + if (mod->sn == MSF_INVALID_STREAM_NUMBER) { +#if BUILD_DEBUG + // an obj skipped here must have NO Symbols payload -- otherwise sym_data_size >= sig + // would have allocated a stream -- so skipping the extraction below loses nothing + Assert(cv_sub_section_from_debug_s(task->cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols).total_size == 0); +#endif + temp_end(temp); + continue; + } + + // P3.4: THE single window fill per obj in module write (sizing above reads raw metadata/ + // headers only). Copy mode ( /OPT:GCTYPES ) consumes the patched backing; its journal was + // consumed eagerly, so the replay call is a structural no-op. + CV_DebugS debug_s; + if (g_debug_s_window) { + debug_s = lnk_obj_window_debug_s(temp.arena, task->cv, obj_idx, task->pe.image_base, task->image_section_table, 0); + } else { + lnk_apply_debug_s_fixups_for_obj(task->cv, obj_idx); + debug_s = task->cv->debug_s_arr[obj_idx]; + } + lnk_assert_debug_s_prov_parity(task->cv->obj_arr[obj_idx], &task->cv->debug_s_arr[obj_idx]); // debug-only P1.2 parity + // P2b: globals-candidate + proc-ref extraction fused into this (now only) fill + lnk_extract_gsi_inputs_for_obj(task, obj_idx, task_id, &debug_s); - // P3.3: re-fill the window for the stream write (second read of the obj's $S bytes -- - // deterministic re-fill: raw + relocs + journal replays to the exact sizing-pass bytes) - CV_DebugS debug_s = g_debug_s_window - ? lnk_obj_window_debug_s(temp.arena, task->cv, obj_idx, task->pe.image_base, task->image_section_table, 0) - : task->cv->debug_s_arr[obj_idx]; - String8List mod_data = msf_data_from_sn(temp.arena, task->pdb->msf, mod->sn); + String8List mod_data = msf_data_from_sn(temp.arena, task->pdb->msf, mod->sn); if (mod_data.node_count) { - lnk_assert_debug_s_prov_parity(task->cv->obj_arr[obj_idx], &task->cv->debug_s_arr[obj_idx]); // debug-only P1.2 parity +#if BUILD_DEBUG + // raw-header sizing vs post-transform write must agree byte-for-byte + U64 size_check[4] = { mod->sym_data_size, mod->c11_data_size, mod->c13_data_size, mod->globrefs_size }; +#endif String8Node buf = *mod_data.first; U64 pos = 0; lnk_write_debug_s_to_pdb_module(mod, debug_s, &buf, &pos); +#if BUILD_DEBUG + Assert(mod->sym_data_size == size_check[0]); + Assert(mod->c11_data_size == size_check[1]); + Assert(mod->c13_data_size == size_check[2]); + Assert(mod->globrefs_size == size_check[3]); +#endif // sub range symbol data pages and patch symbol tree offsets if (mod->sym_data_size) { From f4d5da261d8c1e1823b7195d1096834016b4d7af Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 12 Aug 2026 21:23:32 -0700 Subject: [PATCH 063/121] linker: ring P4 -- .debug$T/$P views become read-only; discards journal instead of mutating -- LF_ENDPRECOMP strips, invalid-TI/cyclic hash discards, and IFC 0x1522 rewrites record into a per-obj sorted NOTYPE journal (single writer per obj by construction: hash runs one task per obj, ENDPRECOMP is serial, IFC preallocates serially) with a lazily-built per-obj bitmap; readers take the original raw-byte inner loops behind a null-bitmap test and reconstruct only journaled leaves. ts_info.name copies out of the raw leaf. Mapped type views are never written, so they stay CoW-clean and trimmable -- the prerequisite for releasing input views after materialize. Byte-identical (incl /OPT:GCTYPES), deterministic 6/6 torture rounds on the frozen rig, lyra 9/9, interleaved wall medians par-or-better. Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.c | 296 ++++++++++++++++++++++++++++++------ src/linker/lnk_debug_info.h | 38 ++++- 2 files changed, 286 insertions(+), 48 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index e922444c9..5ca5fd2ff 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -1394,15 +1394,15 @@ THREAD_POOL_TASK_FUNC(lnk_ifc_resolve_task) LNK_IfcRawRec *recs = task->recs[obj_idx]; U64 n = task->counts[obj_idx]; if (n == 0) { task->res_counts[obj_idx] = 0; return; } - CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; U64 res_count = 0; U64 k_first = 0, k_last = 0; for EachIndex(t, n) { - LNK_IfcRawRec *r = &recs[t]; - CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, r->leaf_idx); - // exclude the placeholder leaf from output regardless: rewrite to NOTYPE - memory_write16(MemberFromPtr(CV_LeafHeader, hdr, kind), (U16)CV_LeafKind_NOTYPE); + LNK_IfcRawRec *r = &recs[t]; + // exclude the placeholder leaf from output regardless: rewrite to NOTYPE. P4: journaled + // (KIND_ONLY -- the old write changed only the kind, size/payload stay) instead of + // CoW-dirtying the mapped view; capacity is pre-reserved serially (arena == 0 here) + lnk_notype_journal_push(0, &input->notype_journal[obj_idx], (U32)r->leaf_idx, 1, 0 /* bitmap pre-allocated */); if (r->entry == 0 || r->entry->blob_slot == 0) { continue; } U64 blob_i = r->entry->blob_slot - 1; IFC_File *f = &task->ifc_files[blob_i]; @@ -1675,6 +1675,25 @@ lnk_apply_ifc_debug_records(TP_Context *tp, TP_Arena *tp_arena, LNK_CodeViewInpu resolve.res_counts = push_array(scratch.arena, U64, input->obj_count); resolve.k_first = push_array(scratch.arena, U64, input->obj_count); resolve.k_last = push_array(scratch.arena, U64, input->obj_count); + // P4: pre-reserve NOTYPE journal capacity + bitmap serially (one entry per 0x1522 record); + // the parallel resolve task pushes with arena == 0 and must never allocate + for EachIndex(obj_idx, input->obj_count) { + U64 n = scan.out_counts[obj_idx]; + if (n == 0) { continue; } + LNK_NotypeJournal *journal = &input->notype_journal[obj_idx]; + U32 need = journal->count + (U32)n; + if (need > journal->cap) { + U32 *new_v = push_array_no_zero(arena, U32, need); + MemoryCopyTyped(new_v, journal->v, journal->count); + journal->v = new_v; + journal->cap = need; + } + if (journal->bitmap == 0) { + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + journal->bit_cap = debug_t->count; + journal->bitmap = push_array(arena, U64, (journal->bit_cap + 63) / 64); + } + } tp_for_parallel(tp, 0, input->obj_count, lnk_ifc_resolve_task, &resolve); lnk_log(LNK_Log_Timers, "[IFC] parallel resolve in %.2f ms", (F64)(now_time_us() - resolve_begin_us) / 1000.0); @@ -1861,6 +1880,10 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, LNK_CodeViewInput input = { .config = config, .obj_count = obj_count, .count = obj_count, .obj_arr = obj_arr, .rrt_input = rrt_input, .ts_obj_range = r1u64(0,0) }; + // $T streaming (ring P4): per-real-obj NOTYPE journals (see LNK_NotypeJournal). Real objs + // only ([0, obj_count)); pseudo objs appended later mutate their arena-backed $T in place. + input.notype_journal = push_array(tp_arena->v[0], LNK_NotypeJournal, obj_count ? obj_count : 1); + HashMap rrt_hm = {0}; ProfScope("Make obj path -> RRT hash map") { @@ -2131,6 +2154,10 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, 0); ts_kind = LNK_TypeServerKind_PDB; ts_info = cv_type_server_info_from_leaf(leaf); + // P4: ts_info.name points into the raw $T leaf 0 bytes of the mapped obj view -- copy it + // out so nothing downstream retains a raw-view pointer (ts_info is stored in ts_arr and + // read during/after the merge) + ts_info.name = push_str8_copy(tp_arena->v[0], ts_info.name); ts_path = lnk_find_first_file(scratch.arena, config->lib_dir_list, ts_info.name); } @@ -2337,7 +2364,9 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, debug_t->offsets += 1; } - // remove LF_ENDPRECOMP from .debug$P + // remove LF_ENDPRECOMP from .debug$P -- P4: journal the NOTYPE rewrite instead of dirtying + // the mapped $P view (the backward header scan to FIND it still reads the raw view; those + // pages are hot from the parse) for EachIndex(i, input.debug_p_indices.count) { U64 debug_p_idx = input.debug_p_indices.v[i]; CV_DebugT *debug_p = &input.debug_t_arr[debug_p_idx]; @@ -2345,8 +2374,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 lf_idx = debug_p->count - (i + 1); CV_LeafHeader *lf = cv_debug_t_get_leaf_header(debug_p, lf_idx); if (lf->kind == CV_LeafKind_ENDPRECOMP) { - memory_write16(&lf->size, sizeof(lf->kind)); - memory_write16(&lf->kind, CV_LeafKind_NOTYPE); + lnk_notype_journal_push(tp_arena->v[0], &input.notype_journal[debug_p_idx], (U32)lf_idx, 0, debug_p->count); break; } } @@ -2471,6 +2499,101 @@ lnk_leaf_ref_leaf_idx(LNK_LeafRef ref) return (U32)ref; } +// $T streaming (ring P4): NOTYPE journal ops. Pushes are single-threaded per obj (input-phase +// pushes are serial or per-obj tasks; hash-phase pushes come from the obj's own hash task), so +// no atomics. Sorted insert keeps lookup a bsearch; the common shapes are 0 entries (fast path) +// or an append at the tail (input-phase entries land before hash-phase entries only for $P objs +// that are also IFC consumers -- the linear tail walk handles the interleave). +internal void +lnk_notype_journal_push(Arena *arena, LNK_NotypeJournal *journal, U32 leaf_idx, B32 kind_only, U64 bit_cap) +{ + if (journal->count == journal->cap) { + AssertAlways(arena != 0); // pseudo objs never journal; real-obj pushes always have an arena + U32 new_cap = journal->cap ? journal->cap * 2 : 8; + U32 *new_v = push_array_no_zero(arena, U32, new_cap); + MemoryCopyTyped(new_v, journal->v, journal->count); + journal->v = new_v; + journal->cap = new_cap; + } + if (journal->bitmap == 0) { + AssertAlways(arena != 0); + journal->bitmap = push_array(arena, U64, (bit_cap + 63) / 64); + journal->bit_cap = bit_cap; + } + // out-of-span indices (the pre-existing `curr_ti - min` quirk on invalid-TI error paths can + // exceed the leaf count) get a journal entry but no bit; readers only query leaf_idx < count, + // so the entry is unreachable either way (the old in-place write was equally out-of-bounds) + if (leaf_idx < journal->bit_cap) { journal->bitmap[leaf_idx >> 6] |= (1ull << (leaf_idx & 63)); } + U32 entry = leaf_idx | (kind_only ? LNK_NOTYPE_JOURNAL_KIND_ONLY : 0); + U32 i = journal->count; + for (; i > 0 && (journal->v[i-1] & ~LNK_NOTYPE_JOURNAL_KIND_ONLY) > leaf_idx; i -= 1) { + journal->v[i] = journal->v[i-1]; + } + Assert(i == 0 || (journal->v[i-1] & ~LNK_NOTYPE_JOURNAL_KIND_ONLY) != leaf_idx); + journal->v[i] = entry; + journal->count += 1; +} + +// O(1) hot-path test: bitmap == 0 for every obj without journal entries (the common case) +internal B32 +lnk_notype_journal_test(LNK_NotypeJournal *journal, U64 leaf_idx) +{ + return journal->bitmap != 0 && leaf_idx < journal->bit_cap && ((journal->bitmap[leaf_idx >> 6] >> (leaf_idx & 63)) & 1); +} + +internal B32 +lnk_notype_journal_find(LNK_NotypeJournal *journal, U32 leaf_idx, B32 *kind_only_out) +{ + if (journal->count == 0) { return 0; } + U32 lo = 0, hi = journal->count; + while (lo < hi) { + U32 mid = lo + (hi - lo) / 2; + U32 key = journal->v[mid] & ~LNK_NOTYPE_JOURNAL_KIND_ONLY; + if (key < leaf_idx) { lo = mid + 1; } + else if (key > leaf_idx) { hi = mid; } + else { + if (kind_only_out) { *kind_only_out = !!(journal->v[mid] & LNK_NOTYPE_JOURNAL_KIND_ONLY); } + return 1; + } + } + return 0; +} + +// Journal-aware leaf read: raw views hold PRE-rewrite bytes for real objs, so present the +// journaled view of the leaf to the hashers (identical to what the old in-place writes produced: +// full rewrite => { kind=LF_NOTYPE, empty payload }; KIND_ONLY => kind=LF_NOTYPE, payload kept). +internal CV_Leaf +lnk_cv_leaf_from_leaf_ref(LNK_CodeViewInput *input, U32 obj_idx, U32 leaf_idx) +{ + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + // hot path: one bitmap test; journals are empty for ~all objs + if (obj_idx < input->obj_count && lnk_notype_journal_test(&input->notype_journal[obj_idx], leaf_idx)) { + B32 kind_only = 0; + lnk_notype_journal_find(&input->notype_journal[obj_idx], leaf_idx, &kind_only); + leaf.kind = CV_LeafKind_NOTYPE; + if (!kind_only) { leaf.data.size = 0; } + } + return leaf; +} + +// Journal-aware raw leaf size (materialize buffer sizing + copy). Hot path = the ORIGINAL +// header read (one bitmap test on top); only a full NOTYPE rewrite changes the answer (the +// leaf shrank to its 4-byte header; KIND_ONLY keeps the size field). +internal U64 +lnk_leaf_ref_raw_size(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) +{ + U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + if (obj_idx < input->obj_count && lnk_notype_journal_test(&input->notype_journal[obj_idx], leaf_idx)) { + B32 kind_only = 0; + lnk_notype_journal_find(&input->notype_journal[obj_idx], leaf_idx, &kind_only); + if (!kind_only) { return sizeof(CV_LeafHeader); } + } + return cv_debug_t_get_raw_leaf(debug_t, leaf_idx).size; +} + internal LNK_LeafRef lnk_leaf_ref_from_ti(LNK_CodeViewInput *input, U32 obj_idx, CV_TypeIndexSource source, CV_TypeIndex ti) { @@ -2487,7 +2610,7 @@ lnk_leaf_ref_from_ti(LNK_CodeViewInput *input, U32 obj_idx, CV_TypeIndexSource s if (bits[rel >> 6] & (1ull << (rel & 63))) { U64 *packed = hash_map_search_u64_u64(&input->ifc_redirect_hm, Compose64Bit(obj_idx, ti)); if (packed) { - return (LNK_LeafRef){ (U32)(*packed >> 32), (U32)(*packed & max_U32) }; + return lnk_leaf_ref_make((U32)(*packed >> 32), (U32)(*packed & max_U32)); } } } @@ -2556,13 +2679,15 @@ lnk_match_leaf_ref(LNK_CodeViewInput *input, LNK_LeafRef a, LNK_LeafRef b) return a_hash == b_hash; } +// P4: `leaf` is the caller's (journal-aware) read of the leaf -- this function no longer +// re-reads it from the raw view. `journal_arena` backs NOTYPE journal growth for real objs +// (pseudo objs keep in-place rewrites and may pass 0). internal U64 -lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs, B32 discard_cycles) +lnk_hash_cv_leaf(LNK_CodeViewInput *input, Arena *journal_arena, LNK_LeafRef leaf_ref, CV_Leaf leaf, CV_TiOffsets ti_offs, B32 discard_cycles) { U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); CV_TypeIndexSource curr_ti_source = cv_type_index_source_from_leaf_kind(leaf.kind); CV_TypeIndex curr_ti = cv_ti_from_leaf_idx(debug_t, curr_ti_source, leaf_idx); U64 ti_count = cv_ti_offsets_count(&ti_offs); @@ -2588,6 +2713,10 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti lnk_hasher_update(&hasher, bytes, size); } + // P4: set when a discard below rewrote THIS leaf's header (the final header mix-in must + // then hash the NOTYPE header, exactly like the old post-write pointer read did) + B32 self_discarded = 0; + // mix-in sub leaf hashes for (U64 ti_idx = 0; ti_idx < ti_count; ti_idx += 1) { CV_TiOff sub_ti_n = cv_ti_offset_at(&ti_offs, ti_idx); @@ -2601,11 +2730,17 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti } if (sub_ti >= debug_t->ti_ranges[sub_ti_n.source].max) { - // discard type - U32 leaf_idx = curr_ti - debug_t->ti_ranges[curr_ti_source].min; - U8 *leaf_header = debug_t->data.str + debug_t->offsets[leaf_idx]; - memory_write16(leaf_header + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); - memory_write16(leaf_header + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); + // discard type: journal the NOTYPE rewrite for view-backed real objs (raw input pages + // stay clean); pseudo objs keep the in-place write on their arena-backed copy + U32 leaf_idx = curr_ti - debug_t->ti_ranges[curr_ti_source].min; + if (obj_idx < input->obj_count) { + lnk_notype_journal_push(journal_arena, &input->notype_journal[obj_idx], leaf_idx, 0, debug_t->count); + } else { + U8 *leaf_header = debug_t->data.str + debug_t->offsets[leaf_idx]; + memory_write16(leaf_header + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); + memory_write16(leaf_header + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); + } + if (leaf_idx == lnk_leaf_ref_leaf_idx(leaf_ref)) { self_discarded = 1; } // reset hasher lnk_hasher_init(&hasher, input->config->debug_types_hash); @@ -2623,11 +2758,16 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti // discard type with a cyclic-ref B32 is_type_graph_cyclic = discard_cycles && sub_ti > 0 && sub_ti > curr_ti; if (is_type_graph_cyclic) { - // discard type - U32 leaf_idx = curr_ti - debug_t->ti_ranges[curr_ti_source].min; - U8 *leaf_header = debug_t->data.str + debug_t->offsets[leaf_idx]; - memory_write16(leaf_header + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); - memory_write16(leaf_header + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); + // discard type (journal for real objs, in-place for pseudo -- see the invalid-TI branch) + U32 leaf_idx = curr_ti - debug_t->ti_ranges[curr_ti_source].min; + if (obj_idx < input->obj_count) { + lnk_notype_journal_push(journal_arena, &input->notype_journal[obj_idx], leaf_idx, 0, debug_t->count); + } else { + U8 *leaf_header = debug_t->data.str + debug_t->offsets[leaf_idx]; + memory_write16(leaf_header + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); + memory_write16(leaf_header + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); + } + if (leaf_idx == lnk_leaf_ref_leaf_idx(leaf_ref)) { self_discarded = 1; } // reset hasher lnk_hasher_init(&hasher, input->config->debug_types_hash); @@ -2650,9 +2790,20 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti lnk_hasher_update_struct(&hasher, &sub_hash); } - // hash leaf header - CV_LeafHeader *leaf_header_ptr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); - lnk_hasher_update_struct(&hasher, leaf_header_ptr); + // hash leaf header. Hot path = the ORIGINAL raw pointer read; only journaled leaves (whose + // raw header is unpatched) and self-discards (whose rewrite went to the journal for real + // objs) reconstruct the header ({ size = data.size + sizeof(kind), kind } is byte-identical + // to what the old post-in-place-write read produced) + if (self_discarded) { + CV_LeafHeader leaf_header = { .size = sizeof(CV_LeafKind), .kind = CV_LeafKind_NOTYPE }; + lnk_hasher_update_struct(&hasher, &leaf_header); + } else if (obj_idx < input->obj_count && + lnk_notype_journal_test(&input->notype_journal[obj_idx], leaf_idx)) { + CV_LeafHeader leaf_header = { .size = (CV_LeafSize)(leaf.data.size + sizeof(CV_LeafKind)), .kind = leaf.kind }; + lnk_hasher_update_struct(&hasher, &leaf_header); + } else { + lnk_hasher_update_struct(&hasher, cv_debug_t_get_leaf_header(debug_t, leaf_idx)); + } // finalize the type hash U64 hash = lnk_hasher_digest64(&hasher); @@ -2692,7 +2843,7 @@ lnk_hash_cv_leaf_deep(Arena *arena, root_frame->ti_offs = root_ti_offs; root_frame->ti_next = 0; root_frame->ti_count = cv_ti_offsets_count(&root_ti_offs); - root_frame->leaf = cv_debug_t_get_leaf(root_debug_t, lnk_leaf_ref_leaf_idx(root_leaf_ref)); + root_frame->leaf = lnk_cv_leaf_from_leaf_ref(input, lnk_leaf_ref_obj_idx(root_leaf_ref), lnk_leaf_ref_leaf_idx(root_leaf_ref)); root_frame->ti_source = cv_type_index_source_from_leaf_kind(root_frame->leaf.kind); root_frame->ti = cv_ti_from_leaf_idx(root_debug_t, root_frame->ti_source, lnk_leaf_ref_leaf_idx(root_leaf_ref)); @@ -2721,7 +2872,7 @@ lnk_hash_cv_leaf_deep(Arena *arena, // recurse down to sub types HashStack *frame = push_array(temp.arena, HashStack, 1); frame->leaf_ref = leaf_ref; - frame->leaf = cv_debug_t_get_leaf(&input->debug_t_arr[obj_idx], leaf_idx); + frame->leaf = lnk_cv_leaf_from_leaf_ref(input, obj_idx, leaf_idx); frame->ti_offs = cv_leaf_ti_offsets(temp.arena, frame->leaf.kind, frame->leaf.data); frame->ti_next = 0; frame->ti_count = cv_ti_offsets_count(&frame->ti_offs); @@ -2733,7 +2884,9 @@ lnk_hash_cv_leaf_deep(Arena *arena, // no more type indices, pop frame if (stack->ti_next >= stack->ti_count) { - lnk_hash_cv_leaf(input, stack->leaf_ref, stack->ti_offs, 0); + // deep hashing only runs on pseudo objs (type servers / .ifc blobs) and never leaves the + // root obj, so no journal arena is needed (in-place rewrite path) + lnk_hash_cv_leaf(input, 0, stack->leaf_ref, stack->leaf, stack->ti_offs, 0); SLLStackPop(stack); } } @@ -2767,9 +2920,9 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_task) CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; for EachIndex(leaf_idx, debug_t->count) { Temp temp = temp_begin(task->fixed_arenas[worker_id]); - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + CV_Leaf leaf = lnk_cv_leaf_from_leaf_ref(task->input, obj_idx, leaf_idx); CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); - lnk_hash_cv_leaf(task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_offs, 1); + lnk_hash_cv_leaf(task->input, arena, lnk_leaf_ref_make(obj_idx, leaf_idx), leaf, ti_offs, 1); temp_end(temp); } ProfEnd(); @@ -2785,9 +2938,9 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_deep_task) B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); for EachIndex(leaf_idx, debug_t->count) { if (task->input->debug_h_arr[obj_idx].v[leaf_idx] != 0) { continue; } - if (is_ifc_blob && cv_debug_t_get_leaf_header(debug_t, leaf_idx)->kind == CV_LeafKind_NOTYPE) { continue; } + if (is_ifc_blob && cv_debug_t_get_leaf_header(debug_t, leaf_idx)->kind == CV_LeafKind_NOTYPE) { continue; } // blob $T is arena-backed + mutated in place -- raw read is post-rewrite Temp temp = temp_begin(task->fixed_arenas[worker_id]); - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + CV_Leaf leaf = lnk_cv_leaf_from_leaf_ref(task->input, obj_idx, leaf_idx); CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); lnk_hash_cv_leaf_deep(temp.arena, task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_offs); temp_end(temp); @@ -2825,9 +2978,18 @@ THREAD_POOL_TASK_FUNC(lnk_estimate_unique_leaves_task) // same prune rule as lnk_leaf_dedup_task: NOTYPE'd IFC blob leaves were never hashed and are // never inserted, so they must not contribute to the estimate either B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); + // P4: raw views hold pre-rewrite bytes for journaled real-obj leaves -- overlay LF_NOTYPE + // via the per-obj journal bitmap (0 for ~all objs; one register test per leaf) + U64 *notype_bm = 0; + U64 notype_cap = 0; + if (obj_idx < task->input->obj_count) { + notype_bm = task->input->notype_journal[obj_idx].bitmap; + notype_cap = task->input->notype_journal[obj_idx].bit_cap; + } for (U64 leaf_idx = 0; leaf_idx < debug_t->count; leaf_idx += LNK_ESTIMATE_SAMPLE_STRIDE) { CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); + if (notype_bm && leaf_idx < notype_cap && ((notype_bm[leaf_idx >> 6] >> (leaf_idx & 63)) & 1)) { kind = CV_LeafKind_NOTYPE; } if (is_ifc_blob && kind == CV_LeafKind_NOTYPE) { continue; } CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); U64 bit_idx = debug_h->v[leaf_idx] & (task->estimate_bitmap_bits[leaf_source] - 1); @@ -2856,6 +3018,9 @@ THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); B32 is_inserted_or_updated = 1; + + // pop obj is a type-server pseudo obj: arena-backed, mutated in place, never journaled -- + // the raw header read is the post-rewrite kind (original code path) CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); // leaf index -> leaf header CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); // leaf header -> leaf kind CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); // leaf kind -> type stream @@ -2898,6 +3063,16 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) ProfBeginDynamic("dedup in obj 0x%llx (%.*s) leaf count %llu", obj_idx, str8_varg(task->input->obj_arr[obj_idx]->path), debug_t->count); + // P4: raw views hold pre-rewrite bytes for journaled real-obj leaves -- overlay LF_NOTYPE via + // the per-obj journal bitmap (0 for ~all objs). Blob objs are pseudo (in-place rewrites), so + // their skip below keeps the plain raw read. + U64 *notype_bm = 0; + U64 notype_cap = 0; + if (obj_idx < task->input->obj_count) { + notype_bm = task->input->notype_journal[obj_idx].bitmap; + notype_cap = task->input->notype_journal[obj_idx].bit_cap; + } + for EachIndex(leaf_idx, debug_t->count) { // another worker overflowed an estimate-sized table -- the whole dedup result is discarded // and retried with the total-based caps, so bail out early @@ -2909,6 +3084,7 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); // leaf index -> leaf header CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); // leaf header -> leaf kind + if (notype_bm && leaf_idx < notype_cap && ((notype_bm[leaf_idx >> 6] >> (leaf_idx & 63)) & 1)) { kind = CV_LeafKind_NOTYPE; } CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); // leaf kind -> type stream LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[leaf_source]; // type stream -> hash table U64 best_idx = debug_h->v[leaf_idx] % leaf_ht->cap; // leaf ref -> hash -> bucket index @@ -3523,9 +3699,8 @@ THREAD_POOL_TASK_FUNC(lnk_count_unique_leaf_sizes_task) Rng1U64 range = task->ranges[task_id]; U64 size = 0; for EachInRange(i, range) { - LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; - CV_DebugT *debug_t = &task->input->debug_t_arr[lnk_leaf_ref_obj_idx(leaf_ref)]; - size += cv_debug_t_get_raw_leaf(debug_t, lnk_leaf_ref_leaf_idx(leaf_ref)).size; + // P4: journal-aware offsets-chain size -- no leaf-header reads through the mapped views + size += lnk_leaf_ref_raw_size(task->input, task->unique_leaf_refs_arr[task->ti_source].v[i]); } task->leaf_buffer_offsets[task_id] = size; // exclusive-scanned into offsets on the main thread } @@ -3549,17 +3724,38 @@ THREAD_POOL_TASK_FUNC(lnk_materialize_unique_leaves_task) U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; - String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_idx); - // copy raw leaf into the private buffer - MemoryCopy(cursor, raw_leaf.str, raw_leaf.size); + // copy raw leaf into the private buffer. Hot path = the ORIGINAL read+memcpy (one bitmap + // test on top); journaled leaves replay the NOTYPE rewrite into the COPY (full rewrite => + // bare { size=sizeof(CV_LeafKind), kind=LF_NOTYPE } header; KIND_ONLY (0x1522) => copy then + // patch the kind). Byte-identical to the old post-in-place-write copy. + U64 raw_size; + if (obj_idx < task->input->obj_count && + lnk_notype_journal_test(&task->input->notype_journal[obj_idx], leaf_idx)) { + B32 kind_only = 0; + lnk_notype_journal_find(&task->input->notype_journal[obj_idx], leaf_idx, &kind_only); + if (kind_only) { + String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_idx); + raw_size = raw_leaf.size; + MemoryCopy(cursor, raw_leaf.str, raw_size); + memory_write16(cursor + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); + } else { + raw_size = sizeof(CV_LeafHeader); + memory_write16(cursor + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); + memory_write16(cursor + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); + } + } else { + String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_idx); + raw_size = raw_leaf.size; + MemoryCopy(cursor, raw_leaf.str, raw_size); + } task->result.v[task->ti_source][i] = cursor; - cursor += raw_leaf.size; + cursor += raw_size; // fixup type indices on the copy (same math the in-place leaf patcher applied) Temp temp = temp_begin(fixed_arena); CV_Leaf leaf = {0}; - cv_read_leaf(str8(task->result.v[task->ti_source][i], raw_leaf.size), 0, 1, &leaf); + cv_read_leaf(str8(task->result.v[task->ti_source][i], raw_size), 0, 1, &leaf); CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); lnk_fixup_cv_type_indices(task, obj_idx, leaf.data, ti_offs); temp_end(temp); @@ -3573,11 +3769,9 @@ THREAD_POOL_TASK_FUNC(lnk_unbucket_hashes_task) LNK_MergeTypes *task = raw_task; Rng1U64 range = task->ranges[task_id]; for EachInRange(i, range) { - LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; - U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); - U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); - CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; - String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_idx); + LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; + U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); task->result.hashes[task->ti_source][i] = task->input->debug_h_arr[obj_idx].v[leaf_idx]; } } @@ -3592,9 +3786,15 @@ THREAD_POOL_TASK_FUNC(lnk_build_obj_ti_map) CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; CV_TypeIndex *obj_ti_map = task->obj_ti_batch + task->obj_ti_map_offsets[obj_idx]; + // P4: journal bitmap overlays LF_NOTYPE on journaled real-obj leaves (raw bytes are pre-rewrite) + U64 *notype_bm = input->notype_journal[obj_idx].bitmap; + U64 notype_cap = input->notype_journal[obj_idx].bit_cap; + for EachIndex(leaf_idx, debug_t->count) { - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); - CV_TypeIndexSource source = cv_type_index_source_from_leaf_kind(leaf.kind); + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + CV_LeafKind kind = leaf.kind; + if (notype_bm && leaf_idx < notype_cap && ((notype_bm[leaf_idx >> 6] >> (leaf_idx & 63)) & 1)) { kind = CV_LeafKind_NOTYPE; } + CV_TypeIndexSource source = cv_type_index_source_from_leaf_kind(kind); LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); LNK_AssignedTiHash *assigned = &task->assigned_ti_arr[source]; obj_ti_map[leaf_idx] = lnk_assigned_ti_hash_search(assigned, input, leaf_ref); @@ -3686,7 +3886,9 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK for EachElement(i, hash_targets) { task.indices = hash_targets[i].hash_indices; ProfBegin("Hash [Count: %.*s]", str8_varg(str8_from_count(scratch.arena, task.indices.count))); - tp_for_parallel(tp, 0, task.indices.count, hash_targets[i].hasher_task, &task); + // P4: pass real worker arenas -- the shallow hasher's invalid-TI/cyclic discards push + // NOTYPE journal entries (rare error paths, bytes are negligible on tp_temp) + tp_for_parallel(tp, tp_temp, task.indices.count, hash_targets[i].hasher_task, &task); ProfEnd(); } diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index ab23b7f1a..a1732d2cd 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -102,6 +102,33 @@ typedef struct LNK_DebugSInlineJournal U32 *node_counts; // [InlineeLines node_count] entries per node run } LNK_DebugSInlineJournal; +// $T streaming (ring P4): per-real-obj journal of LF_NOTYPE kind rewrites. Raw mapped +// .debug$T/$P views are never written anymore (writes used to CoW-dirty one input page per +// discard); instead the rewrite is journaled here and mirrored into CV_DebugT.tags, and the +// downstream byte readers (hash current-leaf reads, unique-leaf sizing, materialize) replay it. +// Entry = leaf_idx | optional KIND_ONLY flag: +// full rewrite (default): header becomes { size=sizeof(CV_LeafKind), kind=LF_NOTYPE } -- the +// ENDPRECOMP strip and the invalid-TI/cyclic discards in lnk_hash_cv_leaf; +// KIND_ONLY: only the kind becomes LF_NOTYPE, size/payload preserved -- the LF_IFC_RECORD +// (0x1522) placeholder rewrite in lnk_ifc_resolve_task. +// Entries are kept sorted by leaf_idx (sorted insert; pushes are single-threaded per obj). +// Pseudo objs (type servers, .ifc blobs; obj_idx >= obj_count) keep their in-place writes: +// their $T is arena-backed, not a mapped input view. +// +// HOT-PATH CONTRACT: readers stay on the ORIGINAL raw-byte code path unless the per-obj +// bitmap says the leaf is journaled. `bitmap == 0` (the overwhelmingly common case: journals +// exist only for $P objs, IFC consumers, and invalid-input error paths) costs one load+branch; +// only journaled leaves pay the bsearch + reconstruct. +#define LNK_NOTYPE_JOURNAL_KIND_ONLY (1u << 31) +typedef struct LNK_NotypeJournal +{ + U32 *v; // (leaf_idx | LNK_NOTYPE_JOURNAL_KIND_ONLY?) ascending by leaf_idx + U32 count; + U32 cap; + U64 *bitmap; // 1 bit / parse-time leaf slot; 0 until the first push + U64 bit_cap; // bits in bitmap +} LNK_NotypeJournal; + typedef struct { LNK_Config *config; @@ -166,6 +193,10 @@ typedef struct // original map is searched unchanged, so results are bit-identical to always searching. U64 **ifc_redirect_bits; // [count]; null == obj has no redirect keys Rng1U64 *ifc_redirect_ti_rng; // [count]; [min,max) local-TI span covered by the obj's bitset + + // $T streaming (ring P4): [obj_count] NOTYPE-rewrite journals, real objs only (see + // LNK_NotypeJournal). Pseudo objs at [obj_count, count) mutate their arena-backed $T in place. + LNK_NotypeJournal *notype_journal; } LNK_CodeViewInput; typedef struct @@ -399,7 +430,12 @@ internal LNK_CodeViewInput lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp internal int lnk_leaf_ref_compare (LNK_LeafRef a, LNK_LeafRef b); internal B32 lnk_match_leaf_ref (LNK_CodeViewInput *input, LNK_LeafRef a, LNK_LeafRef b); -internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs, B32 discard_cycles); +internal void lnk_notype_journal_push (Arena *arena, LNK_NotypeJournal *journal, U32 leaf_idx, B32 kind_only, U64 bit_cap); +internal B32 lnk_notype_journal_test (LNK_NotypeJournal *journal, U64 leaf_idx); +internal B32 lnk_notype_journal_find (LNK_NotypeJournal *journal, U32 leaf_idx, B32 *kind_only_out); +internal CV_Leaf lnk_cv_leaf_from_leaf_ref (LNK_CodeViewInput *input, U32 obj_idx, U32 leaf_idx); +internal U64 lnk_leaf_ref_raw_size (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); +internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, Arena *journal_arena, LNK_LeafRef leaf_ref, CV_Leaf leaf, CV_TiOffsets ti_offs, B32 discard_cycles); internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs); internal CV_TypeIndex lnk_assigned_ti_hash_search (LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); internal LNK_MergedTypes lnk_merge_types (TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK_MergeTypeFlags merge_flags); From 76a371172922002522c8e59cc7e4c36de0672484 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 12 Aug 2026 22:30:26 -0700 Subject: [PATCH 064/121] linker: ring P5 -- input views release at their last reader, not process exit -- the exit path's capped parallel unmap sweep relocates to right after the SC phase (the final view reader: publics alias COFF string tables, SC reads section headers from views), releasing ~89GB of mapped input residency before the NatVis/DBI/MSF/serialize tail and PDB write drain instead of holding it to end-of-link; exit sweeps become idempotent no-ops. Gated off under /PDBSTRIPPED (second build re-walks Symbols) and /OPT:GCTYPES (copy mode aliases views). A first cut with a single background unmapper thread cost +6.5s FN wall (serialized ~60s of VAD-locked unmap CPU the pool sweep lands in ~1s) and was rejected by the interleaved gate. Final: FN interleaved wall medians 23.2 vs 23.7 (faster), kernel -30-40s, byte-identical x4, deterministic, lyra 9/9. Co-Authored-By: Claude Fable 5 --- src/linker/lnk.c | 6 ++++-- src/linker/lnk_debug_info.c | 34 +++++++++++++++++++++++++++++++++- src/linker/lnk_debug_info.h | 2 +- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 7d0f508ca..96d6c035d 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -7512,7 +7512,7 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) pdb_writer.output_path = config->debug_mode == LNK_DebugMode_Full ? config->pdb_name : str8_zero(); pdb_writer.temp_output_path = config->debug_mode == LNK_DebugMode_Full ? config->temp_pdb_name : str8_zero(); lnk_summary_phase_begin(LNK_SummaryPhase_PdbWr); - pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &cv, cv_types, pdb_writer, LNK_PDB_BuilderFlag_All); + pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &cv, cv_types, pdb_writer, LNK_PDB_BuilderFlag_All, inputer); lnk_summary_phase_end(LNK_SummaryPhase_PdbWr); lnk_timer_end(LNK_Timer_Pdb); @@ -7629,7 +7629,9 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) stripped_cv.debug_s_arr = debug_s_arr; stripped_cv.symbol_input_ranges = push_array(scratch.arena, Rng1U64, tp->worker_count); - LNK_FileArtifact pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &stripped_cv, (LNK_MergedTypes){0}, (LNK_PdbWriter){0}, LNK_PDB_BuilderFlag_All); + // inputer==0: never early-release from the stripped build (and the first build was + // already gated off by pdb_stripped_name) -- its window fills read the raw views + LNK_FileArtifact pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &stripped_cv, (LNK_MergedTypes){0}, (LNK_PdbWriter){0}, LNK_PDB_BuilderFlag_All, 0); lnk_summary_phase_begin(LNK_SummaryPhase_PdbWr); lnk_write_data_list_to_file_path(config->pdb_stripped_name, str8f(scratch.arena, "%S.tmp", config->pdb_stripped_name), pdb_artifact.data); lnk_summary_phase_end(LNK_SummaryPhase_PdbWr); diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 5ca5fd2ff..fbf0ccf05 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -6270,7 +6270,7 @@ lnk_build_pdb_distribute_obj_indices(Arena *arena, LNK_BuildPdb *task, U64 obj_c } internal LNK_FileArtifact -lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags) +lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags, struct LNK_Inputer *inputer) { ProfBeginFunction(); Temp scratch = scratch_begin(tp_arena->v, tp_arena->count); @@ -6536,6 +6536,38 @@ ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); lnk_summary_phase_end(LNK_SummaryPhase_PdbSc); } + // Streaming-ring P5: the SC pass above was the LAST reader of the memory-mapped input + // views on this path (audited: pass B of the module-write epilogue = last .debug$S read, + // publics in "Move Global Symbols" = last COFF symbol-table + string-table read [long + // public names alias the view's COFF string table, symbol records fall back to the view + // when no symbol_table_copy exists], SC = last section-header/name read; NatVis below + // reads its own files, /names serializes the P3.1 rehomed blob, DBI file-info hashes the + // repointed bucket copies, MSF build/serialize reads PDB pages only). Release the views + // NOW with the exit path's own capped parallel sweep, relocated: the pool is idle between + // the SC pass and the serial NatVis/DBI/MSF tail, so the sweep lands at parallel-unmap + // wall (~1s FN-scale; a single background thread here serialized ~60s of unmap CPU and + // showed up as +6.5s FN wall) and mapped input residency is gone before the tail + PDB + // write drain instead of held to the end of the link. The sweep zeroes data/owns_file_map + // per input, so the exit-time calls (lnk_inputer_release_file_maps, + // lnk_release_input_views) turn into no-ops -- idempotent. Gated OFF when a /PDBSTRIPPED + // build follows (its pre-build strip loop re-walks Symbols through lnk_obj_window_debug_s + // = raw view reads after this function returns; the caller also passes inputer==0 for the + // stripped build itself) and under /OPT:GCTYPES (copy mode: reloc-free $S slices in + // debug_s_arr alias the raw views in place -- keep the exit-time release). Only for the + // CoW read-only mapping mode, mirroring lnk_release_input_views (read-write-shared unmap + // flushes dirty pages back to the input files). + if (inputer != 0 && + config->pdb_stripped_name.size == 0 && + config->opt_gc_types != LNK_SwitchState_Yes && + (config->io_flags & LNK_IO_Flags_MemoryMapFilesReadOnly) && + !(config->io_flags & LNK_IO_Flags_MemoryMapFilesReadWrite)) { + ProfBegin("Release Input Views Early"); + U64 unmap_begin_us = now_time_us(); + lnk_inputer_release_file_maps(tp, config->debug_worker_cap, inputer); + lnk_log(LNK_Log_Timers, "[pdb] early input-view release in %.2f ms", (F64)(now_time_us() - unmap_begin_us) / 1000.0); + ProfEnd(); + } + if (builder_flags & LNK_PDB_BuilderFlag_NATVIS) { ProfBegin("Build NatVis"); { diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index a1732d2cd..5274139dd 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -449,4 +449,4 @@ internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP // PDB internal void lnk_gc_types (TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedTypes *types); -internal LNK_FileArtifact lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags); +internal LNK_FileArtifact lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags, struct LNK_Inputer *inputer); From e5a612b38a2309ca20611d037537599d2949fc10 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 12 Aug 2026 22:31:47 -0700 Subject: [PATCH 065/121] linker: forward-declare LNK_Inputer at file scope -- struct declared inside a prototype is scope-local in clang, breaking the PGO build Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 5274139dd..c6e116145 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -449,4 +449,5 @@ internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP // PDB internal void lnk_gc_types (TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedTypes *types); +struct LNK_Inputer; internal LNK_FileArtifact lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags, struct LNK_Inputer *inputer); From 891d70338f4f43fa805b15496b05c603e46149b4 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 00:52:37 -0700 Subject: [PATCH 066/121] linker: coalesce background PDB writes -- the progressive writer issued ~8K tiny QD-1 positional WriteFile calls (per-module page runs, ~194-650KB), collapsing to 85-350MB/s during pool-busy phases and exposing an 8.4s drain after the last enqueue; the writer thread now batch-drains its queue, sorts write jobs by file offset, and merges jobs contiguous in both file and memory (MSF page number maps linearly to page data) into single writes: 7934 jobs -> 40 (~215MB avg), ~2GB/s sustained through the build, PDB on disk before DBI info, end drain 8360ms -> 28ms. Jobs are disjoint pages so reordering is output-neutral; control jobs remain ordering barriers. Byte-identical (incl /OPT:GCTYPES), deterministic x3, lyra 9/9, FN interleaved spread 4.9s -> 1.25s (median par, worst-case mode eliminated). Co-Authored-By: Claude Fable 5 --- src/linker/lnk_debug_info.c | 18 +++++ src/linker/lnk_io.c | 151 +++++++++++++++++++++++++++++------- src/linker/lnk_io.h | 6 ++ 3 files changed, 149 insertions(+), 26 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index fbf0ccf05..cce23afa2 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -5834,6 +5834,17 @@ lnk_pdb_output_finalize_stream(void *user_data, MSF_Context *msf, MSF_StreamNumb lnk_pdb_output_enqueue_stream(user_data, msf, sn); } +// Timers telemetry: cumulative enqueued/completed bytes at a named point in the build +internal void +lnk_pdb_output_log_mark(LNK_PdbOutput *output, char *tag) +{ + if (output == 0) { return; } + lnk_log(LNK_Log_Timers, "[pdbw] t=%.3fs %s: enq=%.2f GiB done=%.2f GiB", + (F64)(now_time_us() - output->writer->begin_time_us) / 1e6, tag, + (F64)ins_atomic_u64_eval(&output->writer->bytes_enqueued) / GB(1), + (F64)ins_atomic_u64_eval(&output->writer->bytes_completed) / GB(1)); +} + internal void lnk_pdb_output_enqueue_remaining(LNK_PdbOutput *output, MSF_Context *msf) { @@ -6391,6 +6402,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config pdb_strtab_add_cv_string_hash_table(&task.pdb->info->strtab, task.string_ht); ProfEnd(); pdb_build_types(tp, task.pdb, &build_hooks); + lnk_pdb_output_log_mark(output_ptr, "after pdb_build_types (TPI/IPI sealed)"); // merged leaf bytes are now in MSF pages (and on their way to disk); no consumer of // cv_types.v remains on this path (RDI converts from the PDB artifact, the RRT export @@ -6452,6 +6464,7 @@ ProfScope("Write Modules") tp_for_parallel_reserve(tp, 0, C, lnk_write_pdb_modules, &task); // BARRIER pass (path B): barrier_wait/tp_broadcast tp_barrier_end(tp); lnk_log(LNK_Log_Timers, "[pdb] write modules in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); + lnk_pdb_output_log_mark(output_ptr, "after write modules"); if (g_debug_s_window && lnk_get_log_status(LNK_Log_Debug)) { // bounds the per-worker window arena growth: worst case commit = cohort x this value lnk_log(LNK_Log_Debug, "[pdb] $S window high-water: %llu bytes (largest single obj window)", g_debug_s_window_hwm); @@ -6487,6 +6500,7 @@ ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->publics_sn); lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->globals_sn); lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->symbols_sn); + lnk_pdb_output_log_mark(output_ptr, "after GSI/PSI streams sealed"); } } @@ -6602,7 +6616,9 @@ ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); } lnk_summary_phase_begin(LNK_SummaryPhase_PdbMsf); + lnk_pdb_output_log_mark(output_ptr, "before pdb_build_dbi_info"); pdb_build_dbi_info(tp, task.pdb, task.string_ht, 0, cv->is_stripped, &build_hooks); + lnk_pdb_output_log_mark(output_ptr, "after pdb_build_dbi_info"); MSF_Error msf_err = msf_build(task.pdb->msf); if (msf_err != MSF_Error_OK) { @@ -6610,7 +6626,9 @@ ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); } if (output_ptr != 0) { + lnk_pdb_output_log_mark(output_ptr, "after msf_build"); lnk_pdb_output_enqueue_remaining(output_ptr, task.pdb->msf); + lnk_pdb_output_log_mark(output_ptr, "after enqueue_remaining"); } ProfBegin("Get Page Nodes"); diff --git a/src/linker/lnk_io.c b/src/linker/lnk_io.c index cdc5f96fc..e4657a2d6 100644 --- a/src/linker/lnk_io.c +++ b/src/linker/lnk_io.c @@ -584,6 +584,70 @@ lnk_background_file_writer_end_file_on_thread(LNK_BackgroundFile *file, U64 expe } } +internal int +lnk_background_file_write_job_is_before(const void *raw_a, const void *raw_b) +{ + const LNK_BackgroundFileWriteJob *a = raw_a, *b = raw_b; + if (a->file != b->file) { return a->file < b->file ? -1 : +1; } + if (a->file_off != b->file_off) { return a->file_off < b->file_off ? -1 : +1; } + return 0; +} + +// Write jobs on one file cover disjoint file ranges (each MSF page is enqueued exactly +// once: sealed streams are disjoint page sets, and the final remainder pass skips pages +// already enqueued), so reordering Write jobs among themselves cannot change the output. +// Control jobs (OpenFile/EndFile/EndWriter) are ordering barriers: the batch drain stops +// at the first control job and the batched writes are issued BEFORE it runs, preserving +// the queue-order guarantees (every write lands behind its file's open and before its +// file's end). +internal void +lnk_background_file_writer_issue_write_batch(LNK_BackgroundFileWriter *writer, LNK_BackgroundFileWriteJob *batch, U64 count) +{ + qsort(batch, count, sizeof(batch[0]), lnk_background_file_write_job_is_before); + for (U64 i = 0; i < count; ) { + // coalesce jobs that are contiguous in BOTH file space and memory into one write; + // MSF page numbers map linearly to page-data-node memory, so page runs from + // interleaved stream enqueues merge back into large sequential writes here + U64 j = i; + U64 end_off = batch[i].file_off + batch[i].data.size; + U8 *end_ptr = batch[i].data.str + batch[i].data.size; + for (; j + 1 < count; j += 1) { + LNK_BackgroundFileWriteJob *next = &batch[j+1]; + if (next->file != batch[i].file || + next->decommit_after_write != batch[i].decommit_after_write || + next->file_off != end_off || + next->data.str != end_ptr) { + break; + } + end_off += next->data.size; + end_ptr += next->data.size; + } + U64 merged_size = end_off - batch[i].file_off; + if (batch[i].file->is_open) { + U64 write_size = lnk_write_file(&batch[i].file->file, batch[i].file_off, batch[i].data.str, merged_size); + if (write_size != merged_size) { + batch[i].file->write_failed = 1; + } + batch[i].file->bytes_written += write_size; + U64 done_now = ins_atomic_u64_add_eval(&writer->bytes_completed, write_size); + writer->jobs_completed += (j + 1 - i); + writer->writes_issued += 1; + if ((done_now >> 29) != ((done_now - write_size) >> 29)) { // every 512MiB + lnk_log(LNK_Log_Timers, "[pdbw] t=%.3fs written=%.2f GiB enq=%.2f GiB jobs=%llu/%llu writes=%llu avg=%llu KiB", (F64)(now_time_us() - writer->begin_time_us) / 1e6, (F64)done_now / GB(1), (F64)ins_atomic_u64_eval(&writer->bytes_enqueued) / GB(1), writer->jobs_completed, ins_atomic_u64_eval(&writer->jobs_enqueued), writer->writes_issued, (done_now / writer->writes_issued) / KB(1)); + } + } + if (batch[i].decommit_after_write) { + // the buffer is immutable and write-once (a sealed MSF stream run); once it is + // on disk its pages are dead weight in the commit charge. Round INWARD to whole + // OS pages so neighbors sharing the boundary pages are untouched. + U64 lo = AlignPow2((U64)batch[i].data.str, KB(4)); + U64 hi = ((U64)end_ptr) & ~(U64)(KB(4) - 1); + if (lo < hi) { decommit_memory((void *)lo, hi - lo); } + } + i = j + 1; + } +} + internal void lnk_background_file_writer_thread(void *raw_writer) { @@ -591,37 +655,64 @@ lnk_background_file_writer_thread(void *raw_writer) set_thread_namef("Background File Writer"); LNK_BackgroundFileWriter *writer = raw_writer; - for (;;) { - LNK_BackgroundFileWriteJob job = {0}; - RingGuard guard = guarded_ring_open(writer->queue); - B32 is_read = guarded_ring_read_struct_or_wait(&guard, &job, max_U64); - guarded_ring_close(&guard); - Assert(is_read); - if (job.kind == LNK_BackgroundFileJobKind_EndWriter) { break; } + U64 batch_cap = 8192; + LNK_BackgroundFileWriteJob *batch = (LNK_BackgroundFileWriteJob *)reserve_memory(batch_cap * sizeof(batch[0])); + commit_memory(batch, batch_cap * sizeof(batch[0])); - if (job.kind == LNK_BackgroundFileJobKind_OpenFile) { - lnk_background_file_writer_open_file_on_thread(job.file); - } else if (job.kind == LNK_BackgroundFileJobKind_Write) { - if (job.file->is_open) { - U64 write_size = lnk_write_file(&job.file->file, job.file_off, job.data.str, job.data.size); - if (write_size != job.data.size) { - job.file->write_failed = 1; + for (B32 quit = 0; !quit; ) { + LNK_BackgroundFileWriteJob job = {0}; + { + RingGuard guard = guarded_ring_open(writer->queue); + B32 is_read = guarded_ring_read_struct_or_wait(&guard, &job, max_U64); + guarded_ring_close(&guard); + Assert(is_read); + } + + // batch up every immediately-available Write job (stop at the first control job -- + // it is an ordering barrier) so scattered small page runs coalesce into few large + // sequential writes instead of thousands of QD-1 16KB-class WriteFile calls + U64 batch_count = 0; + B32 have_control = 0; + LNK_BackgroundFileWriteJob control = {0}; + if (job.kind == LNK_BackgroundFileJobKind_Write) { + batch[batch_count++] = job; + while (batch_count < batch_cap) { + LNK_BackgroundFileWriteJob more = {0}; + RingGuard guard = guarded_ring_open(writer->queue); + B32 is_read = guarded_ring_try_read(&guard, sizeof(more), &more); + guarded_ring_close(&guard); + if (!is_read) { break; } + if (more.kind == LNK_BackgroundFileJobKind_Write) { + batch[batch_count++] = more; + } else { + control = more; + have_control = 1; + break; } - job.file->bytes_written += write_size; } - if (job.decommit_after_write) { - // the buffer is immutable and write-once (a sealed MSF stream run); once it is - // on disk its pages are dead weight in the commit charge. Round INWARD to whole - // OS pages so neighbors sharing the boundary pages are untouched. - U64 lo = AlignPow2((U64)job.data.str, KB(4)); - U64 hi = ((U64)job.data.str + job.data.size) & ~(U64)(KB(4) - 1); - if (lo < hi) { decommit_memory((void *)lo, hi - lo); } + } else { + control = job; + have_control = 1; + } + + if (batch_count > 0) { + lnk_background_file_writer_issue_write_batch(writer, batch, batch_count); + } + + if (have_control) { + if (control.kind == LNK_BackgroundFileJobKind_EndWriter) { + quit = 1; + } else if (control.kind == LNK_BackgroundFileJobKind_OpenFile) { + lnk_background_file_writer_open_file_on_thread(control.file); + lnk_log(LNK_Log_Timers, "[pdbw] t=%.3fs file open done (%S)", (F64)(now_time_us() - writer->begin_time_us) / 1e6, control.file->open_path); + } else if (control.kind == LNK_BackgroundFileJobKind_EndFile) { + lnk_background_file_writer_end_file_on_thread(control.file, control.expected_byte_count); } - } else if (job.kind == LNK_BackgroundFileJobKind_EndFile) { - lnk_background_file_writer_end_file_on_thread(job.file, job.expected_byte_count); } } + + release_memory(batch, batch_cap * sizeof(batch[0])); ProfEnd(); } @@ -630,8 +721,9 @@ lnk_background_file_writer_begin(LNK_BackgroundFileWriter *writer) { ProfBegin("Background File Writer Begin"); - writer->queue_arena = arena_alloc(.reserve_size = MB(2), .commit_size = MB(2), .name = "BACKGROUND_FILE_WRITE_QUEUE"); - writer->queue = guarded_ring_alloc(writer->queue_arena, MB(1)); + writer->queue_arena = arena_alloc(.reserve_size = MB(2), .commit_size = MB(2), .name = "BACKGROUND_FILE_WRITE_QUEUE"); + writer->queue = guarded_ring_alloc(writer->queue_arena, MB(1)); + writer->begin_time_us = now_time_us(); ProfEnd(); } @@ -683,6 +775,8 @@ lnk_background_file_writer_enqueue(LNK_BackgroundFileWriter *writer, LNK_Backgro Assert(!file->is_finished); // is_open is owned by the writer thread (open is a queued job) if (data.size > 0) { + ins_atomic_u64_add_eval(&writer->bytes_enqueued, data.size); + ins_atomic_u64_add_eval(&writer->jobs_enqueued, 1); LNK_BackgroundFileWriteJob job = { .kind = LNK_BackgroundFileJobKind_Write, .file = file, @@ -727,12 +821,17 @@ lnk_background_file_writer_end(LNK_BackgroundFileWriter *writer) ProfBegin("Background File Writer End"); if (writer->is_running) { + U64 wait_begin_us = now_time_us(); LNK_BackgroundFileWriteJob job = { .kind = LNK_BackgroundFileJobKind_EndWriter }; RingGuard guard = guarded_ring_open(writer->queue); B32 is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64); guarded_ring_close(&guard); Assert(is_written); thread_join(writer->thread, -1); + lnk_log(LNK_Log_Timers, "[pdbw] t=%.3fs writer end: drain wait %.2f ms (enq=%.2f GiB done=%.2f GiB)", + (F64)(now_time_us() - writer->begin_time_us) / 1e6, + (F64)(now_time_us() - wait_begin_us) / 1000.0, + (F64)writer->bytes_enqueued / GB(1), (F64)writer->bytes_completed / GB(1)); } for EachNode(file, LNK_BackgroundFile, writer->file_first) { diff --git a/src/linker/lnk_io.h b/src/linker/lnk_io.h index 76621da98..0238dcc3b 100644 --- a/src/linker/lnk_io.h +++ b/src/linker/lnk_io.h @@ -34,6 +34,12 @@ typedef struct LNK_BackgroundFile *file_first; LNK_BackgroundFile *file_last; B32 is_running; + U64 begin_time_us; // Timers telemetry anchor + U64 bytes_enqueued; // atomic; producer side + U64 bytes_completed; // atomic; writer thread + U64 jobs_enqueued; // atomic; producer side + U64 jobs_completed; // writer thread + U64 writes_issued; // writer thread; post-coalesce WriteFile count } LNK_BackgroundFileWriter; // --- Shared File API --------------------------------------------------------- From 6f723f20fa4928a29509809aa3125836876655a7 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 09:57:53 -0700 Subject: [PATCH 067/121] linker: skip debug-input prefetch for lone links PrefetchVirtualMemory was introduced to reduce the machine-wide fault storm when many shared-pool linker processes compete under farm pressure. On a lone warm link it adds a second walk over 80.7 GiB of mapped CodeView input, while the parse still pays the working-set insertion cost. Use the existing advisory shared-pool process counter to run both debug-input prefetch passes only when more than one linker is attached. Concurrent links retain the farm mitigation; a two-process Lyra test confirmed both passes still execute. FN frozen-input PGO, 6 interleaved runs: wall median 22.20 -> 20.95 s; CodeView median 7.40 -> 5.63 s; kernel CPU median 270.5 -> 257.6 s. DLL/PDB byte-identical and deterministic across 3 exact-PGO rounds; Lyra 9/9; debug default + GCTYPES pass. --- src/linker/lnk_debug_info.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index cce23afa2..fa1b0d1ff 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -87,9 +87,10 @@ lnk_tp_arena_release_thread(void *raw_arena) // in mcvi alone). PrefetchVirtualMemory populates the ranges in bulk (large MM // batches, no per-page trap), so issue it over each phase's input ranges right // before the parse walk. Purely a paging hint: no output byte depends on it, -// failure is silently ignored (pre-Win8 OS / memory pressure), and prefetching -// an already-resident page is a cheap no-op -- so no residency tracking, -// blanket prefetch per phase. +// and failure is silently ignored (pre-Win8 OS / memory pressure). A lone +// link skips the hint because walking its already-cached 80+ GiB input set is +// measurable overhead; concurrent shared-pool links retain it to reduce the +// machine-wide fault storm. #if OS_WINDOWS // declared locally so we do not depend on the SDK's _WIN32_WINNT gate for @@ -228,6 +229,20 @@ lnk_prefetch_ranges(TP_Context *tp, U64 worker_cap, U64 range_count, Rng1U64 *ra #endif } +// PrefetchVirtualMemory was added to reduce the machine-wide page-fault storm +// when many shared-pool linker processes run concurrently. For a lone link it +// only populates pages that the parsing walks immediately touch again, adding a +// full extra pass over tens of GiB. The shared-pool process counter is advisory, +// which is exactly the precision this paging hint needs. +internal B32 +lnk_should_prefetch_mapped_input(void) +{ + U32 attached_process_count = 0; + U32 max_process_count = 0; + tp_procs_snapshot(&attached_process_count, &max_process_count); + return attached_process_count > 1; +} + internal void lnk_discard_cv_debug_info(LNK_CodeViewInput *input, U64 obj_idx) { @@ -1951,7 +1966,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, // batch-populate the mapped .debug$S/$T/$P/$H input ranges before the parse // loops below first-touch them page by page (see lnk_prefetch_ranges) - ProfScope("Prefetch CodeView") + if (lnk_should_prefetch_mapped_input()) ProfScope("Prefetch CodeView") { Temp temp = temp_begin(scratch.arena); @@ -3858,7 +3873,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK // batch-populate the .debug$T/$P leaf data the hashers below walk leaf by // leaf; under farm-wide memory pressure these mapped pages were trimmed // since the parse phase touched them (see lnk_prefetch_ranges) - ProfScope("Prefetch Type Data") + if (lnk_should_prefetch_mapped_input()) ProfScope("Prefetch Type Data") { Temp temp = temp_begin(scratch.arena); From 41f555d9215642a8363ff639724446a1b5a91948 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 11:59:49 -0700 Subject: [PATCH 068/121] radlink: avoid duplicate GSI candidate columns --- src/linker/lnk_debug_info.c | 53 +++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index fa1b0d1ff..3b21d35bf 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -5339,14 +5339,13 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) PDB_GsiContext *gsi = task->pdb->gsi; // positional flatten in ascending obj-index order (deterministic for any cohort - // width/schedule) -- {hash, flat} sort entries + parallel position arrays + // width/schedule). Keep only {hash, flat} sort entries here: the per-obj candidate + // arrays remain live through this phase and are already indexed by flat-base + local, + // so copying their hash/offset/node/size columns into scratch would duplicate 24 bytes + // per candidate at the link's peak-memory phase. U64 global_symbol_count = 0; U64 *cand_offsets = 0; // [obj_count+1] prefix sums in obj-index order LNK_GsiSortEnt *sort_ents = 0; // [N] (hash, flat) -- sorted below - U64 *flat_hashes = 0; // [N] - U64 *flat_offs = 0; // [N] - U32 *flat_nodes = 0; // [N] - U32 *flat_sizes = 0; // [N] U32 *grp_of_flat = 0; // [N] group id | LNK_GSI_LEADER_BIT if (task_id == 0) { cand_offsets = push_array_no_zero(scratch.arena, U64, task->cv->obj_count + 1); @@ -5359,29 +5358,17 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) global_symbol_count = acc; U64 n = global_symbol_count ? global_symbol_count : 1; sort_ents = push_array_no_zero(scratch.arena, LNK_GsiSortEnt, n); - flat_hashes = push_array_no_zero(scratch.arena, U64, n); - flat_offs = push_array_no_zero(scratch.arena, U64, n); - flat_nodes = push_array_no_zero(scratch.arena, U32, n); - flat_sizes = push_array_no_zero(scratch.arena, U32, n); grp_of_flat = push_array_no_zero(scratch.arena, U32, n); } tp_broadcast(&global_symbol_count); tp_broadcast(&cand_offsets); tp_broadcast(&sort_ents); - tp_broadcast(&flat_hashes); - tp_broadcast(&flat_offs); - tp_broadcast(&flat_nodes); - tp_broadcast(&flat_sizes); tp_broadcast(&grp_of_flat); for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; if (pre->cand_count == 0) { continue; } U64 base = cand_offsets[obj_idx]; - MemoryCopyTyped(&flat_hashes[base], pre->cand_hashes, pre->cand_count); - MemoryCopyTyped(&flat_offs [base], pre->cand_offs, pre->cand_count); - MemoryCopyTyped(&flat_nodes [base], pre->cand_nodes, pre->cand_count); - MemoryCopyTyped(&flat_sizes [base], pre->cand_sizes, pre->cand_count); for EachIndex(k, pre->cand_count) { sort_ents[base + k].hash = pre->cand_hashes[k]; sort_ents[base + k].flat_idx = base + k; @@ -5509,8 +5496,10 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) tp_broadcast(&winner_ptrs); for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { - for (U64 k = cand_offsets[obj_idx], opl = cand_offsets[obj_idx+1]; k < opl; k += 1) { - if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { mat_sizes[task_id] += AlignPow2(flat_sizes[k], sizeof(void *)); } + LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; + U64 lo = cand_offsets[obj_idx], opl = cand_offsets[obj_idx+1]; + for (U64 k = lo; k < opl; k += 1) { + if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { mat_sizes[task_id] += AlignPow2(pre->cand_sizes[k - lo], sizeof(void *)); } } } barrier_wait(tp->barrier); @@ -5521,6 +5510,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { U64 lo = cand_offsets[obj_idx], opl = cand_offsets[obj_idx+1]; + LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; B32 any_leader = 0; for (U64 k = lo; k < opl; k += 1) { if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { any_leader = 1; break; } } @@ -5537,12 +5527,14 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) Arena *dst_arena = mat_arenas[task_id]; for (U64 k = lo; k < opl; k += 1) { if (!(grp_of_flat[k] & LNK_GSI_LEADER_BIT)) { continue; } - U8 *src = node_base[flat_nodes[k]] + flat_offs[k]; - U8 *copy = push_array_no_zero(dst_arena, U8, flat_sizes[k]); - MemoryCopy(copy, src, flat_sizes[k]); + U64 local_idx = k - lo; + U32 size = pre->cand_sizes[local_idx]; + U8 *src = node_base[pre->cand_nodes[local_idx]] + pre->cand_offs[local_idx]; + U8 *copy = push_array_no_zero(dst_arena, U8, size); + MemoryCopy(copy, src, size); #if BUILD_DEBUG // re-read fidelity: the fresh window fill must reproduce the visit-time bytes exactly - Assert(flat_hashes[k] == u64_hash_from_str8(str8(copy, flat_sizes[k]))); + Assert(pre->cand_hashes[local_idx] == u64_hash_from_str8(str8(copy, size))); #endif winner_ptrs[grp_of_flat[k] & ~LNK_GSI_LEADER_BIT] = copy; } @@ -5564,6 +5556,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { U64 lo = cand_offsets[obj_idx], opl = cand_offsets[obj_idx+1]; + LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; B32 any_member = 0; for (U64 k = lo; k < opl; k += 1) { if (!(grp_of_flat[k] & LNK_GSI_LEADER_BIT)) { any_member = 1; break; } } @@ -5579,20 +5572,22 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) for (U64 k = lo; k < opl; k += 1) { if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { continue; } + U64 local_idx = k - lo; U64 grp = grp_of_flat[k]; - U8 *src = node_base[flat_nodes[k]] + flat_offs[k]; + U32 size = pre->cand_sizes[local_idx]; + U8 *src = node_base[pre->cand_nodes[local_idx]] + pre->cand_offs[local_idx]; String8 leader = cv_raw_from_symbol(winner_ptrs[grp]); #if BUILD_DEBUG - Assert(flat_hashes[k] == u64_hash_from_str8(str8(src, flat_sizes[k]))); + Assert(pre->cand_hashes[local_idx] == u64_hash_from_str8(str8(src, size))); #endif - if (leader.size == flat_sizes[k] && MemoryMatch(leader.str, src, flat_sizes[k])) { continue; } + if (leader.size == size && MemoryMatch(leader.str, src, size)) { continue; } // genuine hash collision: keep the bytes (transient cand arena, dies with the phase) LNK_GsiMismatch *m = push_array_no_zero(task->cand_arenas[task_id], LNK_GsiMismatch, 1); - U8 *bytes_copy = push_array_no_zero(task->cand_arenas[task_id], U8, flat_sizes[k]); - MemoryCopy(bytes_copy, src, flat_sizes[k]); + U8 *bytes_copy = push_array_no_zero(task->cand_arenas[task_id], U8, size); + MemoryCopy(bytes_copy, src, size); m->grp = grp; m->flat_idx = k; - m->bytes = str8(bytes_copy, flat_sizes[k]); + m->bytes = str8(bytes_copy, size); m->next = mismatch_lists[task_id]; mismatch_lists[task_id] = m; mismatch_counts[task_id] += 1; From aec88c5fb761a23447e62179afa98d12df82c5d8 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 12:13:39 -0700 Subject: [PATCH 069/121] radlink: scatter GSI sort entries directly --- src/linker/lnk_debug_info.c | 46 +++++++++++++++---------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 3b21d35bf..188432b8b 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -5339,13 +5339,12 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) PDB_GsiContext *gsi = task->pdb->gsi; // positional flatten in ascending obj-index order (deterministic for any cohort - // width/schedule). Keep only {hash, flat} sort entries here: the per-obj candidate - // arrays remain live through this phase and are already indexed by flat-base + local, - // so copying their hash/offset/node/size columns into scratch would duplicate 24 bytes - // per candidate at the link's peak-memory phase. + // width/schedule). The per-obj candidate arrays remain live through this phase and are + // already indexed by flat-base + local. Count radix buckets directly from those arrays, + // then scatter {hash, flat} into the single sorted array below; a separate flat-order + // sort-entry array would duplicate another 16 bytes per candidate at peak memory. U64 global_symbol_count = 0; U64 *cand_offsets = 0; // [obj_count+1] prefix sums in obj-index order - LNK_GsiSortEnt *sort_ents = 0; // [N] (hash, flat) -- sorted below U32 *grp_of_flat = 0; // [N] group id | LNK_GSI_LEADER_BIT if (task_id == 0) { cand_offsets = push_array_no_zero(scratch.arena, U64, task->cv->obj_count + 1); @@ -5357,27 +5356,14 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) cand_offsets[task->cv->obj_count] = acc; global_symbol_count = acc; U64 n = global_symbol_count ? global_symbol_count : 1; - sort_ents = push_array_no_zero(scratch.arena, LNK_GsiSortEnt, n); grp_of_flat = push_array_no_zero(scratch.arena, U32, n); } tp_broadcast(&global_symbol_count); tp_broadcast(&cand_offsets); - tp_broadcast(&sort_ents); tp_broadcast(&grp_of_flat); - for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { - LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; - if (pre->cand_count == 0) { continue; } - U64 base = cand_offsets[obj_idx]; - for EachIndex(k, pre->cand_count) { - sort_ents[base + k].hash = pre->cand_hashes[k]; - sort_ents[base + k].flat_idx = base + k; - } - } - barrier_wait(tp->barrier); - // sort by (hash, flat idx): 256-way MSB scatter (deterministic positional layout: bucket - // base + per-(bucket,worker) prefix over ascending flat slices), then a full per-bucket + // base + per-(bucket,worker) prefix), then a full per-bucket // sort -- the final array is a pure function of the candidate set for any cohort ProfBegin("Group Global Symbols"); U64 bucket_count = 256; @@ -5385,22 +5371,22 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) U64 *scatter_cursor = 0; // [C][256] U64 *bucket_base = 0; // [256+1] LNK_GsiSortEnt *sorted = 0; // [N] - Rng1U64 *flat_ranges = 0; // [C] even split of [0,N) if (task_id == 0) { scatter_counts = push_array(scratch.arena, U64, tp->worker_count * bucket_count); scatter_cursor = push_array(scratch.arena, U64, tp->worker_count * bucket_count); bucket_base = push_array(scratch.arena, U64, bucket_count + 1); sorted = push_array_no_zero(scratch.arena, LNK_GsiSortEnt, global_symbol_count ? global_symbol_count : 1); - flat_ranges = tp_divide_work(scratch.arena, global_symbol_count, tp->worker_count); } tp_broadcast(&scatter_counts); tp_broadcast(&scatter_cursor); tp_broadcast(&bucket_base); tp_broadcast(&sorted); - tp_broadcast(&flat_ranges); - for EachInRange(i, flat_ranges[task_id]) { - scatter_counts[task_id * bucket_count + (sort_ents[i].hash >> 56)] += 1; + for EachIndex(i, obj_indices.count) { + LNK_GsiPreExtractObj *pre = &task->preext[obj_indices.v[i]]; + for EachIndex(k, pre->cand_count) { + scatter_counts[task_id * bucket_count + (pre->cand_hashes[k] >> 56)] += 1; + } } barrier_wait(tp->barrier); @@ -5418,9 +5404,15 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) } barrier_wait(tp->barrier); - for EachInRange(i, flat_ranges[task_id]) { - U64 b = sort_ents[i].hash >> 56; - sorted[scatter_cursor[task_id * bucket_count + b]++] = sort_ents[i]; + for EachIndex(i, obj_indices.count) { + U64 obj_idx = obj_indices.v[i]; + LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; + U64 flat_base = cand_offsets[obj_idx]; + for EachIndex(k, pre->cand_count) { + U64 hash = pre->cand_hashes[k]; + U64 b = hash >> 56; + sorted[scatter_cursor[task_id * bucket_count + b]++] = (LNK_GsiSortEnt){hash, flat_base + k}; + } } barrier_wait(tp->barrier); From 5f9402d2cf529d217b2fb6930fbe018446e650a9 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 12:29:34 -0700 Subject: [PATCH 070/121] radlink: compact GSI grouping scratch --- src/linker/lnk_debug_info.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 188432b8b..373726de4 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -5082,8 +5082,10 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_ // lnk_write_pdb_modules): candidates sort by (content hash, flat idx) -- flat idx is the // positional obj-major index, so the tiebreak (and thus each group's LEADER) is // schedule-independent. -typedef struct { U64 hash; U64 flat_idx; } LNK_GsiSortEnt; -typedef struct { U64 lo; U64 count; } LNK_GsiGroup; +#pragma pack(push, 4) +typedef struct { U64 hash; U32 flat_idx; } LNK_GsiSortEnt; +#pragma pack(pop) +StaticAssert(sizeof(LNK_GsiSortEnt) == 12, lnk_gsi_sort_ent_size_check); typedef struct LNK_GsiMismatch { struct LNK_GsiMismatch *next; U64 grp; U64 flat_idx; String8 bytes; } LNK_GsiMismatch; #define LNK_GSI_LEADER_BIT (1u << 31) @@ -5411,7 +5413,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) for EachIndex(k, pre->cand_count) { U64 hash = pre->cand_hashes[k]; U64 b = hash >> 56; - sorted[scatter_cursor[task_id * bucket_count + b]++] = (LNK_GsiSortEnt){hash, flat_base + k}; + sorted[scatter_cursor[task_id * bucket_count + b]++] = (LNK_GsiSortEnt){hash, safe_cast_u32(flat_base + k)}; } } barrier_wait(tp->barrier); @@ -5425,7 +5427,6 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) U64 *bucket_group_counts = 0; // [256] U64 *group_base = 0; // [256+1] U64 group_count = 0; - LNK_GsiGroup *groups = 0; // [group_count] if (task_id == 0) { bucket_group_counts = push_array(scratch.arena, U64, bucket_count); group_base = push_array(scratch.arena, U64, bucket_count + 1); @@ -5448,10 +5449,8 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) U64 acc = 0; for EachIndex(b, bucket_count) { group_base[b] = acc; acc += bucket_group_counts[b]; } group_base[bucket_count] = acc; - groups = push_array_no_zero(scratch.arena, LNK_GsiGroup, acc ? acc : 1); } group_count = 0; - tp_broadcast(&groups); if (task_id == 0) { group_count = group_base[bucket_count]; } tp_broadcast(&group_count); @@ -5460,8 +5459,6 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) for (U64 k = bucket_base[b], opl = bucket_base[b+1]; k < opl; ) { U64 e = k + 1; while (e < opl && sorted[e].hash == sorted[k].hash) { e += 1; } - groups[gid].lo = k; - groups[gid].count = e - k; grp_of_flat[sorted[k].flat_idx] = (U32)gid | LNK_GSI_LEADER_BIT; // min flat idx of the run (flat is the sort tiebreak) for (U64 m = k + 1; m < e; m += 1) { grp_of_flat[sorted[m].flat_idx] = (U32)gid; } gid += 1; k = e; From 3b55e2f146dcbd92a7f18fa6738f88143838fd70 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 12:33:06 -0700 Subject: [PATCH 071/121] radlink: narrow GSI candidate offsets --- src/linker/lnk_debug_info.c | 4 ++-- src/linker/lnk_debug_info.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 373726de4..d05b1576e 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4993,7 +4993,7 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_ // re-reads only the bytes it needs through a fresh window fill (deterministic re-fill). if (cand_count) { pre->cand_hashes = push_array_no_zero(cand_arena, U64, cand_count); - pre->cand_offs = push_array_no_zero(cand_arena, U64, cand_count); + pre->cand_offs = push_array_no_zero(cand_arena, U32, cand_count); pre->cand_nodes = push_array_no_zero(cand_arena, U32, cand_count); pre->cand_sizes = push_array_no_zero(cand_arena, U32, cand_count); U64 k = 0; @@ -5007,7 +5007,7 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_ U8 *ptr = cv_ptr_from_symbol(symbol); String8 raw = cv_raw_from_symbol(ptr); pre->cand_hashes[k] = u64_hash_from_str8(raw); - pre->cand_offs [k] = (U64)(ptr - n->string.str); + pre->cand_offs [k] = safe_cast_u32(ptr - n->string.str); pre->cand_nodes [k] = (U32)node_idx; pre->cand_sizes [k] = (U32)raw.size; k += 1; diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index c6e116145..5b7751697 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -326,7 +326,7 @@ typedef enum // the module-write per-obj visit (fused with the sizing walk, over the obj's post-fixup $S // bytes -- the window copy when g_debug_s_window, the patched backing otherwise). P3.3 // candidates are POSITION records, not pointers: {content hash, Symbols node ordinal, offset -// within node, record size} -- 24B/record, NO payload copies and NO live-backing aliasing, so +// within node, record size} -- 20B/record, NO payload copies and NO live-backing aliasing, so // they survive the window being reused for the next obj. The module-write epilogue dedups by // hash grouping (sort by (hash, flat idx)), then RE-READS just the bytes it needs through the // same window fill: group leaders to materialize the winner payload, other members to @@ -341,7 +341,7 @@ typedef struct { U64 cand_count; // global records (cv_is_global_symbol + top-level typedefs) U64 *cand_hashes; // [cand_count] u64_hash_from_str8(raw) computed at extraction (post-reloc post-fixup bytes) - U64 *cand_offs; // [cand_count] record start offset within its Symbols data_list node + U32 *cand_offs; // [cand_count] record start offset within its Symbols data_list node (COFF section-sized) U32 *cand_nodes; // [cand_count] Symbols data_list node ordinal U32 *cand_sizes; // [cand_count] full record size (kind + length prefix + data) U64 procref_count; // GPROC32/LPROC32 records From dd95ceb36ee0065361bd2e7c4da67358a1f9dc07 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 12:45:25 -0700 Subject: [PATCH 072/121] radlink: use native widths for GSI metadata --- src/linker/lnk_debug_info.c | 22 +++++++++++----------- src/linker/lnk_debug_info.h | 8 ++++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index d05b1576e..3523e18a9 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4588,7 +4588,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) // strictly more deterministic.) Zero $S reads. U64 total_proc_ref_count = 0; U64 *procref_offsets = 0; // [obj_count+1] prefix sums in obj-index order - U64 *proc_ref_hashes = 0; // [total_proc_ref_count] + U32 *proc_ref_hashes = 0; // [total_proc_ref_count] CV_Symbol *proc_ref_symbols = 0; // [total_proc_ref_count] if (task_id == 0) { procref_offsets = push_array_no_zero(scratch.arena, U64, task->cv->obj_count + 1); @@ -4599,7 +4599,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } procref_offsets[task->cv->obj_count] = acc; total_proc_ref_count = acc; - proc_ref_hashes = push_array_no_zero(scratch.arena, U64, total_proc_ref_count ? total_proc_ref_count : 1); + proc_ref_hashes = push_array_no_zero(scratch.arena, U32, total_proc_ref_count ? total_proc_ref_count : 1); proc_ref_symbols = push_array_no_zero(scratch.arena, CV_Symbol, total_proc_ref_count ? total_proc_ref_count : 1); } tp_broadcast(&total_proc_ref_count); @@ -4619,7 +4619,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) // single-threaded on task 0; sizing has no determinism impact) if (task_id == 0) { U64 *bucket_adds = push_array(scratch.arena, U64, gsi->bucket_count); - for EachIndex(i, total_proc_ref_count) { bucket_adds[(U32)proc_ref_hashes[i] % gsi->bucket_count] += 1; } + for EachIndex(i, total_proc_ref_count) { bucket_adds[proc_ref_hashes[i] % gsi->bucket_count] += 1; } for EachIndex(bucket_idx, gsi->bucket_count) { if (bucket_adds[bucket_idx]) { gsi_reserve(gsi, bucket_idx, bucket_adds[bucket_idx]); } } @@ -4632,7 +4632,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; for EachIndex(i, total_proc_ref_count) { - U64 bucket_idx = (U32)proc_ref_hashes[i] % gsi->bucket_count; + U64 bucket_idx = proc_ref_hashes[i] % gsi->bucket_count; if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; bucket->v[bucket->count] = proc_ref_symbols[i]; @@ -4995,7 +4995,7 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_ pre->cand_hashes = push_array_no_zero(cand_arena, U64, cand_count); pre->cand_offs = push_array_no_zero(cand_arena, U32, cand_count); pre->cand_nodes = push_array_no_zero(cand_arena, U32, cand_count); - pre->cand_sizes = push_array_no_zero(cand_arena, U32, cand_count); + pre->cand_lens = push_array_no_zero(cand_arena, U16, cand_count); U64 k = 0; U64 node_idx = 0; for (String8Node *n = symbols.first; n != 0; n = n->next, node_idx += 1) { @@ -5009,7 +5009,7 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_ pre->cand_hashes[k] = u64_hash_from_str8(raw); pre->cand_offs [k] = safe_cast_u32(ptr - n->string.str); pre->cand_nodes [k] = (U32)node_idx; - pre->cand_sizes [k] = (U32)raw.size; + pre->cand_lens [k] = ((CV_SymbolHeader *)ptr)->size; k += 1; } @@ -5043,7 +5043,7 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_ if (procref_count) { Arena *payload_arena = task->procref_payload_arenas[task_id]; pre->procref_syms = push_array_no_zero(payload_arena, CV_Symbol, procref_count); - pre->procref_hashes = push_array_no_zero(payload_arena, U64, procref_count); + pre->procref_hashes = push_array_no_zero(payload_arena, U32, procref_count); CV_ModIndex imod = task->mod_arr[obj_idx]->imod; U64 symbol_cursor = sizeof(CV_Signature); @@ -5488,7 +5488,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; U64 lo = cand_offsets[obj_idx], opl = cand_offsets[obj_idx+1]; for (U64 k = lo; k < opl; k += 1) { - if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { mat_sizes[task_id] += AlignPow2(pre->cand_sizes[k - lo], sizeof(void *)); } + if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { mat_sizes[task_id] += AlignPow2(pre->cand_lens[k - lo] + sizeof(CV_SymSize), sizeof(void *)); } } } barrier_wait(tp->barrier); @@ -5517,7 +5517,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) for (U64 k = lo; k < opl; k += 1) { if (!(grp_of_flat[k] & LNK_GSI_LEADER_BIT)) { continue; } U64 local_idx = k - lo; - U32 size = pre->cand_sizes[local_idx]; + U32 size = pre->cand_lens[local_idx] + sizeof(CV_SymSize); U8 *src = node_base[pre->cand_nodes[local_idx]] + pre->cand_offs[local_idx]; U8 *copy = push_array_no_zero(dst_arena, U8, size); MemoryCopy(copy, src, size); @@ -5563,7 +5563,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { continue; } U64 local_idx = k - lo; U64 grp = grp_of_flat[k]; - U32 size = pre->cand_sizes[local_idx]; + U32 size = pre->cand_lens[local_idx] + sizeof(CV_SymSize); U8 *src = node_base[pre->cand_nodes[local_idx]] + pre->cand_offs[local_idx]; String8 leader = cv_raw_from_symbol(winner_ptrs[grp]); #if BUILD_DEBUG @@ -5674,7 +5674,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) task->preext[obj_idx].cand_hashes = 0; // poison dangling refs (incl. the pass-B mismatch bytes on cand_arenas) task->preext[obj_idx].cand_offs = 0; task->preext[obj_idx].cand_nodes = 0; - task->preext[obj_idx].cand_sizes = 0; + task->preext[obj_idx].cand_lens = 0; } } } diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 5b7751697..e3eba9853 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -326,7 +326,7 @@ typedef enum // the module-write per-obj visit (fused with the sizing walk, over the obj's post-fixup $S // bytes -- the window copy when g_debug_s_window, the patched backing otherwise). P3.3 // candidates are POSITION records, not pointers: {content hash, Symbols node ordinal, offset -// within node, record size} -- 20B/record, NO payload copies and NO live-backing aliasing, so +// within node, record size} -- 18B/record, NO payload copies and NO live-backing aliasing, so // they survive the window being reused for the next obj. The module-write epilogue dedups by // hash grouping (sort by (hash, flat idx)), then RE-READS just the bytes it needs through the // same window fill: group leaders to materialize the winner payload, other members to @@ -343,10 +343,10 @@ typedef struct U64 *cand_hashes; // [cand_count] u64_hash_from_str8(raw) computed at extraction (post-reloc post-fixup bytes) U32 *cand_offs; // [cand_count] record start offset within its Symbols data_list node (COFF section-sized) U32 *cand_nodes; // [cand_count] Symbols data_list node ordinal - U32 *cand_sizes; // [cand_count] full record size (kind + length prefix + data) + U16 *cand_lens; // [cand_count] native CV_SymSize; full record size adds sizeof(CV_SymSize) U64 procref_count; // GPROC32/LPROC32 records CV_Symbol *procref_syms; // [procref_count] cv_make_proc_ref results (arrays + payload on survive arenas) - U64 *procref_hashes; // [procref_count] gsi_hash(name) + U32 *procref_hashes; // [procref_count] gsi_hash(name) } LNK_GsiPreExtractObj; typedef struct @@ -382,7 +382,7 @@ typedef struct B32 free_sect_copies; // P2b pre-extraction state (see LNK_GsiPreExtractObj). cand_arenas hold ONLY the per-obj - // candidate ref/hash arrays (16B-per-record class, no payloads) and release with the + // candidate position/hash arrays (18B per record, no payloads) and release with the // section copies at the end of Write Modules. procref_payload_arenas hold the procref // value/hash arrays + cv_make_proc_ref payloads and must survive until GSI serialization // (like the old proc_ref_arenas, they live to process exit). From 659f6f4e1dd19bd9fa4e05bf1530fb0cf51c0727 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 17:23:24 -0700 Subject: [PATCH 073/121] linker: right-size per-worker OPT:REF resolve cache Reduce the lossy resolve memo from 1M to 32K slots per worker. This bounds compulsory initialization at 48 MiB across 64 workers instead of 1.5 GiB while preserving the measured FN resolve phase (~0.38 s). Under kernel pressure, OPT:REF fell from 4.18 s to 0.37-0.39 s and total page faults dropped by ~300K. Frozen FN DLL/PDB output is byte-identical to baseline and deterministic across three candidate runs; Lyra hammer passes 9/9. --- src/linker/lnk.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 96d6c035d..791db85c7 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -2784,8 +2784,10 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) // millions of times. Open-addressing and lossy (a collision past the probe window evicts); // a miss only costs the recompute -- the cached value is a pure function of the key because // the symbol table and parsed_symbols are read-only during /OPT:REF. + // Keep the table small enough that initializing every worker does not become a page-fault + // amplifier (32K slots is 768KiB per worker, versus 24MiB at 1M slots). typedef struct { U64 key; LNK_Obj *obj; U64 symbol_idx; } LNK_RefResolveSlot; - U64 resolve_cache_mask = (1ull << 20) - 1; + U64 resolve_cache_mask = (1ull << 15) - 1; LNK_RefResolveSlot *resolve_cache = push_array_no_zero(scratch.arena, LNK_RefResolveSlot, resolve_cache_mask + 1); MemorySet(resolve_cache, 0xff, sizeof(resolve_cache[0]) * (resolve_cache_mask + 1)); From 9a0228a3ce92e828072d904f0d1630142d3cad5e Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 18:05:46 -0700 Subject: [PATCH 074/121] linker: compact OPT:ICF refinement records Use 32-bit bounded contribution/table indices and generations, and reorder the in-place records to shrink Contrib from 80 to 64 bytes, ColorHashSlot from 56 to 48 bytes, and OldColorHashSlot from 24 to 16 bytes. FN PGO A/B is wall-neutral at a 21.9 s median, reduces ICF page faults from 914K to 816K (~392 MiB), and lowers ICF kernel CPU from ~11.3 to ~9.8 core-seconds. Exact PGO DLL/PDB output is byte-identical to baseline and deterministic across three candidate runs; debug Lyra and 9/9 hammer pass. --- src/linker/lnk.c | 100 +++++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 791db85c7..8913791c6 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -3206,11 +3206,11 @@ lnk_icf_scope_from_section_number(LNK_Obj *obj, U32 section_number) } internal void -lnk_icf_atomic_min_u64(U64 *dst, U64 value) +lnk_icf_atomic_min_u32(U32 *dst, U32 value) { // preserve stable leaders despite concurrent insertion - for (U64 old_value = ins_atomic_u64_eval(dst); value < old_value;) { - U64 observed = ins_atomic_u64_eval_cond_assign(dst, value, old_value); + for (U32 old_value = ins_atomic_u32_eval(dst); value < old_value;) { + U32 observed = ins_atomic_u32_eval_cond_assign(dst, value, old_value); if (observed == old_value) { break; } old_value = observed; } @@ -3238,32 +3238,34 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) COFF_SymbolValueInterpType interp; } RelocTarget; + // Contribution and table indices are bounded to U32 below. Keeping the hot contribution + // record at 64 bytes cuts both its footprint and the demand-zero work during ICF. typedef struct { ColorKey key; U128 static_hash; RelocTarget **reloc_targets; - U64 reloc_count; - U64 color_slot_idx; + U32 color_slot_idx; + U32 reloc_count; U32 obj_idx; U32 section_number; - LNK_ICF_Scope scope; } Contrib; - // reuse both tables without clearing them between refinement rounds + // Reuse both tables without clearing them between refinement rounds. U32 generations and + // indices keep these records at 48 and 16 bytes respectively (down from 56 and 24 bytes). typedef struct { - U64 state; // generation << 2 | (0 = empty, 1 = initializing, 2 = ready) ColorKey key; - U64 first_contrib_idx; - U64 old_color_slot_idx; U64 color; + U32 state; // generation << 2 | (0 = empty, 1 = initializing, 2 = ready) + U32 first_contrib_idx; + U32 old_color_slot_idx; } ColorHashSlot; typedef struct { ColorHashSlot *slots; U64 slots_count; } ColorHashTable; typedef struct { - U64 state; // generation << 2 | (0 = empty, 1 = initializing, 2 = ready) U64 old_color; - U64 first_contrib_idx; + U32 state; // generation << 2 | (0 = empty, 1 = initializing, 2 = ready) + U32 first_contrib_idx; } OldColorHashSlot; typedef struct { OldColorHashSlot *slots; U64 slots_count; } OldColorHashTable; @@ -3386,6 +3388,7 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) } shared = { .contrib_count = sum_array_u64(task->objs_count, contrib_counts) }; + Assert(shared.contrib_count <= max_U32); if (task_id == 0) { ProfBegin("Init"); @@ -3438,7 +3441,6 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) *contrib = (Contrib){ .obj_idx = safe_cast_u32(obj->input_idx), .section_number = safe_cast_u32(it.v.section_number), - .scope = scope, }; Temp temp = temp_begin(scratch.arena); @@ -3446,11 +3448,13 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) U32List associated_sections = lnk_obj_collect_associated_section_numbers(temp.arena, obj, it.v.section_number, associated_filter); u32_list_push(temp.arena, &associated_sections, it.v.section_number); + U64 reloc_count = 0; for EachNode(associated_n, U32Node, associated_sections.first) { COFF_SectionHeader *associated_header = lnk_coff_section_header_from_section_number(obj, associated_n->data); COFF_RelocArray associated_relocs = lnk_coff_relocs_from_section_header(obj, associated_header); - contrib->reloc_count += associated_relocs.count; + reloc_count += associated_relocs.count; } + contrib->reloc_count = safe_cast_u32(reloc_count); if (contrib->reloc_count) { contrib->reloc_targets = push_array(scratch2.arena, RelocTarget *, contrib->reloc_count); } @@ -3559,14 +3563,15 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) ColorHashTable color_table = {0}; OldColorHashTable old_color_table = {0}; - U64 *table_generation = 0; + U32 *table_generation = 0; if (task_id == 0) { ProfBegin("Alloc hash tables"); color_table.slots_count = u64_up_to_pow2(Max(2, shared.contrib_count*2)); + Assert(color_table.slots_count <= (U64)max_U32 + 1); color_table.slots = push_array(scratch.arena, ColorHashSlot, color_table.slots_count); old_color_table.slots_count = color_table.slots_count; old_color_table.slots = push_array(scratch.arena, OldColorHashSlot, old_color_table.slots_count); - table_generation = push_array_no_zero(scratch.arena, U64, 1); + table_generation = push_array_no_zero(scratch.arena, U32, 1); *table_generation = 0; ProfEnd(); } @@ -3589,15 +3594,15 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) *shared.is_part_stable = 1; // update hash tables generations - Assert(*table_generation < (max_U64 >> 2)); + Assert(*table_generation < (max_U32 >> 2)); *table_generation += 1; } barrier_wait(tp->barrier); // unpack the table generation - U64 table_generation_value = *table_generation; - U64 initializing_state = (table_generation_value << 2) | 1; - U64 ready_state = (table_generation_value << 2) | 2; + U32 table_generation_value = *table_generation; + U32 initializing_state = (table_generation_value << 2) | 1; + U32 ready_state = (table_generation_value << 2) | 2; ProfBegin("Compute colored hashes"); for EachInRange(contrib_idx, shared.contrib_ranges[task_id]) { @@ -3623,29 +3628,29 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) U64 color_slot_idx = table_hash & (color_table.slots_count - 1); for (;;) { ColorHashSlot *color_slot = &color_table.slots[color_slot_idx]; - U64 state = ins_atomic_u64_eval(&color_slot->state); + U32 state = ins_atomic_u32_eval(&color_slot->state); if ((state >> 2) != table_generation_value) { - if (ins_atomic_u64_eval_cond_assign(&color_slot->state, initializing_state, state) == state) { + if (ins_atomic_u32_eval_cond_assign(&color_slot->state, initializing_state, state) == state) { color_slot->key = contrib->key; - color_slot->first_contrib_idx = contrib_idx; - contrib->color_slot_idx = color_slot_idx; - ins_atomic_u64_eval_assign(&color_slot->state, ready_state); + color_slot->first_contrib_idx = safe_cast_u32(contrib_idx); + contrib->color_slot_idx = safe_cast_u32(color_slot_idx); + ins_atomic_u32_eval_assign(&color_slot->state, ready_state); break; } continue; } if (state == initializing_state) { - do { state = ins_atomic_u64_eval(&color_slot->state); } while (state == initializing_state); + do { state = ins_atomic_u32_eval(&color_slot->state); } while (state == initializing_state); continue; } Assert(state == ready_state); if (color_slot->key.old_color == contrib->key.old_color && u128_match(color_slot->key.hash, contrib->key.hash)) { - lnk_icf_atomic_min_u64(&color_slot->first_contrib_idx, contrib_idx); - contrib->color_slot_idx = color_slot_idx; + lnk_icf_atomic_min_u32(&color_slot->first_contrib_idx, safe_cast_u32(contrib_idx)); + contrib->color_slot_idx = safe_cast_u32(color_slot_idx); break; } @@ -3661,40 +3666,40 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) for EachInRange(contrib_idx, contrib_range) { Contrib *contrib = &shared.contribs[contrib_idx]; ColorHashSlot *slot = &color_table.slots[contrib->color_slot_idx]; - if (ins_atomic_u64_eval(&slot->first_contrib_idx) == contrib_idx) { + if (ins_atomic_u32_eval(&slot->first_contrib_idx) == contrib_idx) { Assert(old_color_table.slots_count > 0 && (old_color_table.slots_count & (old_color_table.slots_count - 1)) == 0); U64 old_color = slot->key.old_color; U64 table_hash = hash_map_hasher(str8_struct(&old_color)); U64 old_slot_idx = table_hash & (old_color_table.slots_count - 1); for (;;) { OldColorHashSlot *old_color_slot = &old_color_table.slots[old_slot_idx]; - U64 state = ins_atomic_u64_eval(&old_color_slot->state); + U32 state = ins_atomic_u32_eval(&old_color_slot->state); if ((state >> 2) != table_generation_value) { - if (ins_atomic_u64_eval_cond_assign(&old_color_slot->state, initializing_state, state) == state) { + if (ins_atomic_u32_eval_cond_assign(&old_color_slot->state, initializing_state, state) == state) { old_color_slot->old_color = old_color; - old_color_slot->first_contrib_idx = contrib_idx; - ins_atomic_u64_eval_assign(&old_color_slot->state, ready_state); + old_color_slot->first_contrib_idx = safe_cast_u32(contrib_idx); + ins_atomic_u32_eval_assign(&old_color_slot->state, ready_state); break; } continue; } if (state == initializing_state) { - do { state = ins_atomic_u64_eval(&old_color_slot->state); } while (state == initializing_state); + do { state = ins_atomic_u32_eval(&old_color_slot->state); } while (state == initializing_state); continue; } Assert(state == ready_state); if (old_color_slot->old_color == old_color) { - lnk_icf_atomic_min_u64(&old_color_slot->first_contrib_idx, contrib_idx); + lnk_icf_atomic_min_u32(&old_color_slot->first_contrib_idx, safe_cast_u32(contrib_idx)); break; } old_slot_idx = (old_slot_idx + 1) & (old_color_table.slots_count - 1); } - slot->old_color_slot_idx = old_slot_idx; + slot->old_color_slot_idx = safe_cast_u32(old_slot_idx); } } ProfEnd(); @@ -3706,10 +3711,10 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) for EachInRange(contrib_idx, contrib_range) { Contrib *contrib = &shared.contribs[contrib_idx]; ColorHashSlot *color_slot = &color_table.slots[contrib->color_slot_idx]; - if (ins_atomic_u64_eval(&color_slot->first_contrib_idx) != contrib_idx) { continue; } + if (ins_atomic_u32_eval(&color_slot->first_contrib_idx) != contrib_idx) { continue; } OldColorHashSlot *old_color_slot = &old_color_table.slots[color_slot->old_color_slot_idx]; - if (ins_atomic_u64_eval(&old_color_slot->first_contrib_idx) != contrib_idx) { + if (ins_atomic_u32_eval(&old_color_slot->first_contrib_idx) != contrib_idx) { split_count += 1; } } @@ -3744,10 +3749,10 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) for EachInRange(contrib_idx, contrib_range) { Contrib *contrib = &shared.contribs[contrib_idx]; ColorHashSlot *color_slot = &color_table.slots[contrib->color_slot_idx]; - if (ins_atomic_u64_eval(&color_slot->first_contrib_idx) != contrib_idx) { continue; } + if (ins_atomic_u32_eval(&color_slot->first_contrib_idx) != contrib_idx) { continue; } OldColorHashSlot *old_color_slot = &old_color_table.slots[color_slot->old_color_slot_idx]; - if (ins_atomic_u64_eval(&old_color_slot->first_contrib_idx) == contrib_idx) { + if (ins_atomic_u32_eval(&old_color_slot->first_contrib_idx) == contrib_idx) { color_slot->color = color_slot->key.old_color; } else { color_slot->color = ++next_split_color; @@ -3791,20 +3796,21 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) for EachInRange(contrib_idx, shared.contrib_ranges[task_id]) { Contrib *contrib = &shared.contribs[contrib_idx]; ColorHashSlot *color_slot = &color_table.slots[contrib->color_slot_idx]; - Contrib *leader = &shared.contribs[ins_atomic_u64_eval(&color_slot->first_contrib_idx)]; + Contrib *leader = &shared.contribs[ins_atomic_u32_eval(&color_slot->first_contrib_idx)]; LNK_Obj *contrib_obj = objs[contrib->obj_idx]; LNK_Obj *leader_obj = objs[leader->obj_idx]; if (fold_stats) { - FoldStats *st = fold_stats + (task_id * LNK_ICF_Scope_COUNT); - U64 fsize = lnk_coff_section_header_from_section_number(contrib_obj, contrib->section_number)->fsize; + FoldStats *st = fold_stats + (task_id * LNK_ICF_Scope_COUNT); + U64 fsize = lnk_coff_section_header_from_section_number(contrib_obj, contrib->section_number)->fsize; + LNK_ICF_Scope scope = lnk_icf_scope_from_section_number(contrib_obj, contrib->section_number); if (leader == contrib) { - st[leader->scope].live_count += 1; - st[leader->scope].live_size += fsize; + st[scope].live_count += 1; + st[scope].live_size += fsize; } else { - st[contrib->scope].count += 1; - st[contrib->scope].size += fsize; + st[scope].count += 1; + st[scope].size += fsize; } } From 62fdf9a46d4facb15fd125a9029b0688f4adaf46 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 18:43:57 -0700 Subject: [PATCH 075/121] linker: compact associative section graph --- src/linker/lnk.c | 20 ++++++---- src/linker/lnk_obj.c | 75 +++++++++++++++++++++-------------- src/linker/lnk_obj.h | 4 +- src/linker/lnk_symbol_table.c | 7 ++-- 4 files changed, 64 insertions(+), 42 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 8913791c6..7319afafb 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -2926,8 +2926,9 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) } // push associated section - for EachNode(associated_n, U32Node, walk_obj->coff.sections.associated_section_numbers[section_number]) { - U32 assoc_sn = associated_n->data; + U32Array associated_sections = lnk_obj_associated_sections_from_section_number(walk_obj, section_number); + for EachIndex(associated_idx, associated_sections.count) { + U32 assoc_sn = associated_sections.v[associated_idx]; { @@ -3505,8 +3506,9 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) } U64 child_pos = 0; - for EachNode(child_n, U32Node, target_obj->coff.sections.associated_section_numbers[parent_sect]) { - if (child_n->data == target_sect) { break; } + U32Array associated_sections = lnk_obj_associated_sections_from_section_number(target_obj, parent_sect); + for EachIndex(child_idx, associated_sections.count) { + if (associated_sections.v[child_idx] == target_sect) { break; } child_pos += 1; } @@ -4060,8 +4062,9 @@ internal U32 lnk_icf_debug_s_child_from_section(LNK_Obj *obj, U32 fn_sn) { if (fn_sn == 0 || fn_sn > obj->coff.sections.count_no_null) { return 0; } - for EachNode(assoc_n, U32Node, obj->coff.sections.associated_section_numbers[fn_sn]) { - U32 sn = assoc_n->data; + U32Array associated_sections = lnk_obj_associated_sections_from_section_number(obj, fn_sn); + for EachIndex(assoc_idx, associated_sections.count) { + U32 sn = associated_sections.v[assoc_idx]; if (sn == 0 || sn > obj->coff.sections.count_no_null) { continue; } if (~obj->coff.sections.headers[sn].flags & LNK_SECTION_FLAG_DEBUG) { continue; } if (str8_match(lnk_obj_section_name_from_section_number(obj, sn), str8_lit(".debug$S"), 0)) { return sn; } @@ -4233,8 +4236,9 @@ THREAD_POOL_TASK_FUNC(lnk_icf_mark_folded_lines_task) temp_end(fold_temp); } - for EachNode(assoc_n, U32Node, obj->coff.sections.associated_section_numbers[section_number]) { - U32 assoc_sn = assoc_n->data; + U32Array associated_sections = lnk_obj_associated_sections_from_section_number(obj, section_number); + for EachIndex(assoc_idx, associated_sections.count) { + U32 assoc_sn = associated_sections.v[assoc_idx]; if (assoc_sn == 0 || assoc_sn > obj->coff.sections.count_no_null) { continue; } if (~obj->coff.sections.headers[assoc_sn].flags & LNK_SECTION_FLAG_DEBUG) { continue; } // exclude the follower's .debug$S from full module collection (it would otherwise merge diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index f66e09684..9a30e8045 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -169,6 +169,7 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) } } + U32 *associated_section_offsets = push_array(arena, U32, header.section_count_no_null + 2); U64 symbol_block_count = CeilIntegerDiv(header.symbol_count, 64); LNK_ObjSymbolArray symbols = { .count = primary_symbol_count, @@ -220,6 +221,8 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) if (select == COFF_ComdatSelect_Associative) { if (section_number == 0 || section_number > header.section_count_no_null) { lnk_error_input_obj(LNK_Error_IllData, input, "section definition symbol %S (No. 0x%x) associates with an out of bounds section 0x%x", symbol.name, symbol_idx, section_number); + } else if (symbol.section_number > 0 && symbol.section_number <= header.section_count_no_null) { + associated_section_offsets[section_number + 1] += 1; } } } @@ -231,6 +234,14 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) Assert(primary_idx == primary_symbol_count); } + // Convert per-parent association counts to CSR offsets. The offsets double as fill cursors + // below; the backwards pass restores them and the old SLLStackPush sibling order. + for (U64 section_number = 1; section_number <= header.section_count_no_null; section_number += 1) { + associated_section_offsets[section_number + 1] += associated_section_offsets[section_number]; + } + U32 associated_section_count = associated_section_offsets[header.section_count_no_null + 1]; + U32 *associated_section_numbers = push_array_no_zero(arena, U32, associated_section_count); + // // create symbol links to COMDAT sections // @@ -248,10 +259,16 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) if (symbol.storage_class == COFF_SymStorageClass_Static) { if (symbol.section_number > 0 && symbol.section_number <= header.section_count_no_null) { COFF_SectionHeader *sect_header = &coff_section_table[symbol.section_number-1]; - if (section_headers[symbol.section_number].flags & COFF_SectionFlag_LnkCOMDAT) { - if (symbol.aux_symbol_count) { - U32 section_length = 0; - coff_parse_secdef(symbol, header.is_big_obj, 0, 0, §ion_length, 0); + if (symbol.aux_symbol_count) { + COFF_ComdatSelectType selection = COFF_ComdatSelect_Null; + U32 section_number = 0; + U32 section_length = 0; + coff_parse_secdef(symbol, header.is_big_obj, &selection, §ion_number, §ion_length, 0); + if (selection == COFF_ComdatSelect_Associative && section_number > 0 && section_number <= header.section_count_no_null) { + U32 cursor = associated_section_offsets[section_number]++; + associated_section_numbers[cursor] = symbol.section_number; + } + if (section_headers[symbol.section_number].flags & COFF_SectionFlag_LnkCOMDAT) { if (sect_header->fsize == section_length) { if (comdats[symbol.section_number] == ~0) { comdats[symbol.section_number] = symbol_idx; @@ -270,6 +287,14 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) } } } + for (U64 section_number = header.section_count_no_null; section_number > 0; section_number -= 1) { + associated_section_offsets[section_number] = associated_section_offsets[section_number - 1]; + U32 min = associated_section_offsets[section_number]; + U32 max = associated_section_offsets[section_number + 1]; + for (U32 child_idx = 0; child_idx < (max - min) / 2; child_idx += 1) { + Swap(U32, associated_section_numbers[min + child_idx], associated_section_numbers[max - child_idx - 1]); + } + } // // COMDAT loop checker @@ -320,28 +345,6 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) scratch_end(scratch); } - // - // collect sections associations - // - U32Node **associated_sections = push_array(arena, U32Node *, header.section_count_no_null + 1); - { - COFF_ParsedSymbol symbol; - for (U32 symbol_idx = 0; symbol_idx < header.symbol_count; symbol_idx += (1 + symbol.aux_symbol_count)) { - symbol = coff_parse_symbol_no_name(header, raw_coff_symbol_table, symbol_idx); - COFF_SymbolValueInterpType interp = coff_interp_from_parsed_symbol(symbol); - if (interp == COFF_SymbolValueInterp_Regular && symbol.storage_class == COFF_SymStorageClass_Static && symbol.aux_symbol_count > 0) { - COFF_ComdatSelectType selection = COFF_ComdatSelect_Null; - U32 section_number = 0; - coff_parse_secdef(symbol, header.is_big_obj, &selection, §ion_number, 0, 0); - if (selection != COFF_ComdatSelect_Associative) { continue; } - - U32Node *associated_node = push_array(arena, U32Node, 1); - associated_node->data = symbol.section_number; - SLLStackPush(associated_sections[section_number], associated_node); - } - } - } - B8 hotpatch = 0; if (header.machine == COFF_MachineType_X64) { hotpatch = 1; @@ -398,7 +401,8 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) .count_no_null = header.section_count_no_null, .headers = section_headers, .comdats = comdats, - .associated_section_numbers = associated_sections, + .associated_section_offsets = associated_section_offsets, + .associated_section_numbers = associated_section_numbers, }, .symbols = symbols, .debug_t_section_number = debug_t_section_number, @@ -571,8 +575,9 @@ lnk_obj_collect_associated_section_numbers(Arena *arena, LNK_Obj *obj, U32 root_ // walk the complete descendant chain because associated COMDATs can nest for EachNode(parent_n, U32Node, queue.first) { - for EachNode(associated_n, U32Node, obj->coff.sections.associated_section_numbers[parent_n->data]) { - U32 child_section_number = associated_n->data; + U32Array associated_sections = lnk_obj_associated_sections_from_section_number(obj, parent_n->data); + for EachIndex(associated_idx, associated_sections.count) { + U32 child_section_number = associated_sections.v[associated_idx]; if (child_section_number == 0) { continue; } if (hash_map_search_u64_u64(&seen_hm, child_section_number)) { continue; } @@ -675,6 +680,18 @@ lnk_obj_get_comdat_symlink_from_section_number(LNK_Obj *obj, U64 section_number, return is_valid; } +internal U32Array +lnk_obj_associated_sections_from_section_number(LNK_Obj *obj, U32 section_number) +{ + Assert(section_number <= obj->coff.sections.count_no_null); + U32 min = obj->coff.sections.associated_section_offsets[section_number]; + U32 max = obj->coff.sections.associated_section_offsets[section_number + 1]; + U32Array result = { .count = max - min }; + if (result.count) { + result.v = obj->coff.sections.associated_section_numbers + min; + } + return result; +} internal String8 lnk_obj_section_data_from_number(LNK_Obj *obj, U64 section_number) { diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index 5563b2155..c35c254f1 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -25,7 +25,8 @@ typedef struct LNK_ObjSectionArray U64 count_no_null; COFF_SectionHeader *headers; U32 *comdats; - U32Node **associated_section_numbers; + U32 *associated_section_offsets; + U32 *associated_section_numbers; } LNK_ObjSectionArray; typedef struct LNK_ObjCoff @@ -273,6 +274,7 @@ internal struct LNK_Lib * lnk_obj_get_lib(LNK_Obj *obj); internal String8 lnk_obj_get_lib_path(LNK_Obj *obj); internal U32 lnk_obj_get_removed_section_number(LNK_Obj *obj); internal B32 lnk_obj_get_comdat_symlink_from_section_number(LNK_Obj *obj, U64 section_number, LNK_ObjSymbolRef *symlink_out); +internal U32Array lnk_obj_associated_sections_from_section_number(LNK_Obj *obj, U32 section_number); internal U32List lnk_obj_collect_associated_section_numbers(Arena *arena, LNK_Obj *obj, U32 root_section_number, COFF_SectionFlags skip_flags); // --- Symbol & Section Helpers ------------------------------------------------ diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index aea705b97..a5adab703 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -396,10 +396,9 @@ lnk_on_symbol_replace(LNK_Symbol *dst, LNK_Symbol *src) *dst_section.flags |= COFF_SectionFlag_LnkRemove; // remove associated sections from the output - for (U32Node *associated_section = dst_ref.obj->coff.sections.associated_section_numbers[dst_parsed.section_number]; - associated_section != 0; - associated_section = associated_section->next) { - LNK_ObjSection section = lnk_obj_section_from_section_number(dst_ref.obj, associated_section->data); + U32Array associated_sections = lnk_obj_associated_sections_from_section_number(dst_ref.obj, dst_parsed.section_number); + for EachIndex(associated_idx, associated_sections.count) { + LNK_ObjSection section = lnk_obj_section_from_section_number(dst_ref.obj, associated_sections.v[associated_idx]); *section.flags |= COFF_SectionFlag_LnkRemove; } } From 943479c39eb8f80ac176cd5431755459f4a99422 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 19:56:30 -0700 Subject: [PATCH 076/121] linker: deduplicate GSI records during module write Deduplicate exact global-symbol records directly from the existing transformed module window. Only unique records are copied to surviving per-worker storage; remove the per-candidate position metadata, radix grouping, and two subsequent debug-section reread passes. On the frozen FN link this cuts the module phase from roughly 1.3s to 0.7s, saves 20-24 user CPU seconds and about 1GB working set, with flat peak commit. DLL and PDB remain byte-identical and deterministic. --- src/linker/lnk_debug_info.c | 550 ++++++++---------------------------- src/linker/lnk_debug_info.h | 45 ++- 2 files changed, 133 insertions(+), 462 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 3523e18a9..8feda8a71 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4502,14 +4502,12 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) ProfBegin("Global Symbols"); { - // P3.2: dedup + compaction + winner materialize ran inside the module-write phase - // (lnk_write_pdb_modules epilogue) while the patched $S backing was alive -- the backing - // and the candidate ref tables are already RELEASED by the time this pass runs. Consume - // only the materialized winner records (task->gsi_winner_ptrs, compacted-slot order, - // payload on gsi->arena): hash names, size buckets, sharded fill. Zero $S touches - // anywhere in this pass. Determinism unchanged: the winner array is compacted-slot order - // (cohort-invariant), the fill below walks it in global order per bucket shard, and every - // GSI bucket chain is content-sorted at serialization (gsi_symbol_is_before). + // Exact dedup + compaction ran inside the module-write phase while each transformed $S + // window was alive; section backing is already released by the time this pass runs. + // Consume only materialized winner records (pointer array in hash-table slot order, + // payload on surviving per-worker arenas): hash names, size buckets, sharded fill. This + // pass never touches $S. Winner-array order is schedule-dependent, but every GSI bucket + // chain is content-sorted at serialization (gsi_symbol_is_before), so output is stable. U64 symbol_count = task->gsi_winner_count; void **symbol_arr = task->gsi_winner_ptrs; // [symbol_count] materialized copies @@ -4523,7 +4521,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) tp_broadcast(&symbol_hashes); // hash symbols (reads the materialized winner bytes -- byte-identical to the $S records - // they were copied from, so every hash value matches the pre-P3.2 flow) + // they were copied from, so every hash value matches the pre-direct-dedup flow) Rng1U64 symbol_range = symbol_ranges[task_id]; for EachInRange(i, symbol_range) { CV_Symbol symbol = cv_symbol_from_ptr(symbol_arr[i]); @@ -4563,7 +4561,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) // deterministic same-name tie-break for the per-bucket sort in gsi_serialize_symbols_task // (gsi_symbol_is_before compares name -> offset -> kind -> data bytes): key on the content // hash of the full raw record, never a slot/array position. The winner ARRAY order is a - // dedup-implementation detail (P3.3: hash-group order; P3.2: compacted CAS slots) and + // dedup-implementation detail (currently compacted open-addressing slot order) and // per-bucket insert order follows it -- the content-keyed sort is what makes the // serialized bytes invariant to it. On collision the comparator's kind/data-bytes // fallback stays content-deterministic, and byte-identical records cannot reach the @@ -4644,10 +4642,9 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } ProfEnd(); - // P3.2: the patched debug-section copies + candidate ref arenas were released at the END of - // the module-write phase (lnk_write_pdb_modules epilogue) -- nothing here reads $S. The - // procref value/hash arrays + payloads consumed above live on the surviving - // procref_payload_arenas (through GSI serialization). + // Patched debug-section copies were released at the end of the module-write phase; nothing + // here reads $S. Global winner bytes and the procref value/hash arrays + payloads consumed + // above live on the surviving procref_payload_arenas through GSI serialization. ProfBegin("Public Symbols"); { @@ -4935,7 +4932,60 @@ lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8No typedef struct LNK_PdbOutput LNK_PdbOutput; internal void lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context *msf, MSF_StreamNumber sn); -// Streaming-ring P2b: extract the obj's GSI inputs -- global-record candidate refs and +#define LNK_GSI_DEDUP_RESERVED ((void *)(U64)1) + +internal U64 +lnk_count_gsi_candidates(CV_DebugS debug_s) +{ + U64 count = 0; + String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); + for EachNode(n, String8Node, symbols.first) { + for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); + if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { count += 1; } + if (cv_is_scope_symbol(symbol.kind)) { + depth += 1; + } else if (cv_is_end_symbol(symbol.kind)) { + if (depth == 0) { Assert(0 && "malformed symbol stream"); break; } + depth -= 1; + } + } + } + return count; +} + +// Exact concurrent content set. Claiming an empty slot with a sentinel before allocating is +// important: only the thread that adds a distinct record copies bytes, so the common duplicate +// case consumes neither transient metadata nor discarded arena space. Published pointers own +// immutable bytes on a per-worker surviving arena and are safe to compare immediately. +internal void +lnk_gsi_deduper_insert_copy(void **buckets, U64 bucket_cap, U64 hash, void *symbol_ptr, Arena *dst_arena) +{ + String8 raw = cv_raw_from_symbol(symbol_ptr); + U64 best_idx = hash % bucket_cap; + U64 idx = best_idx; + for (;;) { + void *curr = ins_atomic_ptr_eval(&buckets[idx]); + if (curr == 0) { + void *cmp = ins_atomic_ptr_eval_cond_assign(&buckets[idx], LNK_GSI_DEDUP_RESERVED, 0); + if (cmp == 0) { + U8 *copy = push_array_no_zero(dst_arena, U8, raw.size); + MemoryCopy(copy, raw.str, raw.size); + ins_atomic_ptr_eval_assign(&buckets[idx], copy); + return; + } + curr = cmp; + } + while (curr == LNK_GSI_DEDUP_RESERVED) { curr = ins_atomic_ptr_eval(&buckets[idx]); } + String8 seen = cv_raw_from_symbol(curr); + if (seen.size == raw.size && MemoryMatch(seen.str, raw.str, raw.size)) { return; } + idx = (idx + 1 == bucket_cap) ? 0 : idx + 1; + Assert(idx != best_idx); + } +} + +// Streaming-ring P2b: extract the obj's GSI inputs -- direct global-record winners and // proc-refs -- inside the module-write per-obj visit, so lnk_move_global_symbols_to_gsi does // no $S record-decode walks (its only remaining $S touch is the bucket-fill materialize of // the dedup winners' bytes through the refs). Runs right after the obj's deferred $S fixup @@ -4948,11 +4998,8 @@ internal void lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context * // - proc-ref walk: obj-continuous scope depth + module-stream cursor (starts at // sizeof(CV_Signature); records the module write drops -- SKIP, globals, top-level typedefs, // 0x1176 -- do not advance it). -// Per-obj candidate position/hash arrays (P3.3: {hash, node, off, size} -- no pointers into -// the reused window) land on the transient per-worker cand_arenas that release at the END of -// the module-write phase, after the epilogue dedup consumed them. Proc-ref value/hash arrays -// AND payloads go on the surviving procref_payload_arenas (referenced until GSI serialization -// -- the globals pass flattens them AFTER the window phase), same lifetime as the old +// Exact global winners plus proc-ref value/hash arrays and payloads go on the surviving +// procref_payload_arenas (referenced until GSI serialization), same lifetime as the old // proc_ref_arenas. internal void lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_DebugS *debug_s_ptr) @@ -4963,19 +5010,21 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_ LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; PDB_GsiContext *gsi = task->pdb->gsi; - Arena *cand_arena = task->cand_arenas[task_id]; + Arena *payload_arena = task->procref_payload_arenas[task_id]; - // candidate count pass (mirrors the old collect loop in lnk_move_global_symbols_to_gsi) + // Dedup directly from the one post-reloc/post-fixup window. The size-only pass already + // counted these records from invariant headers so the shared table is pre-sized. U64 cand_count = 0; for EachNode(n, String8Node, symbols.first) { for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { CV_Symbol symbol = {0}; TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); - if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { + U8 *ptr = cv_ptr_from_symbol(symbol); + lnk_gsi_deduper_insert_copy(task->gsi_dedup_buckets, task->gsi_dedup_bucket_cap, + u64_hash_from_str8(cv_raw_from_symbol(ptr)), ptr, payload_arena); cand_count += 1; } - if (cv_is_scope_symbol(symbol.kind)) { depth += 1; } else if (cv_is_end_symbol(symbol.kind)) { @@ -4984,46 +5033,7 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_ } } } - - // candidate fill pass (P3.3): store a POSITION record -- {content hash, Symbols node - // ordinal, offset within node, record size} -- NOT a pointer: the window these bytes live - // in is reused for the next obj. The hash is the same expression the old insert used - // (u64_hash_from_str8 over cv_raw_from_symbol) computed over the post-reloc post-fixup - // bytes, so every value matches the pre-P3.3 flow. The epilogue dedup groups by hash and - // re-reads only the bytes it needs through a fresh window fill (deterministic re-fill). - if (cand_count) { - pre->cand_hashes = push_array_no_zero(cand_arena, U64, cand_count); - pre->cand_offs = push_array_no_zero(cand_arena, U32, cand_count); - pre->cand_nodes = push_array_no_zero(cand_arena, U32, cand_count); - pre->cand_lens = push_array_no_zero(cand_arena, U16, cand_count); - U64 k = 0; - U64 node_idx = 0; - for (String8Node *n = symbols.first; n != 0; n = n->next, node_idx += 1) { - for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); - - if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { - U8 *ptr = cv_ptr_from_symbol(symbol); - String8 raw = cv_raw_from_symbol(ptr); - pre->cand_hashes[k] = u64_hash_from_str8(raw); - pre->cand_offs [k] = safe_cast_u32(ptr - n->string.str); - pre->cand_nodes [k] = (U32)node_idx; - pre->cand_lens [k] = ((CV_SymbolHeader *)ptr)->size; - k += 1; - } - - if (cv_is_scope_symbol(symbol.kind)) { - depth += 1; - } else if (cv_is_end_symbol(symbol.kind)) { - if (depth == 0) { Assert(0 && "malformed symbol stream"); break; } - depth -= 1; - } - } - } - Assert(k == cand_count); - pre->cand_count = cand_count; - } + Assert(cand_count == pre->cand_count); // proc-ref count pass (mirrors the old proc-ref sizing walk) U64 procref_count = 0; @@ -5038,10 +5048,8 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_ } // proc-ref fill pass (mirrors the old build walk: identical layout math + skip set). - // P3.2: the value/hash arrays live on the SURVIVING payload arena (not cand_arena) -- the - // globals pass flattens them after cand_arenas die with the section copies. + // The value/hash arrays live on the surviving payload arena. if (procref_count) { - Arena *payload_arena = task->procref_payload_arenas[task_id]; pre->procref_syms = push_array_no_zero(payload_arena, CV_Symbol, procref_count); pre->procref_hashes = push_array_no_zero(payload_arena, U32, procref_count); @@ -5078,33 +5086,6 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_ } } -// P3.3 epilogue-dedup working types (see the "Dedup Global Symbols" block in -// lnk_write_pdb_modules): candidates sort by (content hash, flat idx) -- flat idx is the -// positional obj-major index, so the tiebreak (and thus each group's LEADER) is -// schedule-independent. -#pragma pack(push, 4) -typedef struct { U64 hash; U32 flat_idx; } LNK_GsiSortEnt; -#pragma pack(pop) -StaticAssert(sizeof(LNK_GsiSortEnt) == 12, lnk_gsi_sort_ent_size_check); -typedef struct LNK_GsiMismatch { struct LNK_GsiMismatch *next; U64 grp; U64 flat_idx; String8 bytes; } LNK_GsiMismatch; -#define LNK_GSI_LEADER_BIT (1u << 31) - -internal int -lnk_gsi_sort_ent_is_before(void *raw_a, void *raw_b) -{ - LNK_GsiSortEnt *a = raw_a, *b = raw_b; - if (a->hash != b->hash) { return a->hash < b->hash; } - return a->flat_idx < b->flat_idx; -} - -internal int -lnk_gsi_mismatch_is_before(void *raw_a, void *raw_b) -{ - LNK_GsiMismatch *a = *(LNK_GsiMismatch **)raw_a, *b = *(LNK_GsiMismatch **)raw_b; - if (a->grp != b->grp) { return a->grp < b->grp; } - return a->flat_idx < b->flat_idx; -} - internal THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) { @@ -5129,11 +5110,17 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) // re-accumulated sizes match. Fixup replay, parity, and the GSI extraction all move to // the write pass's single fill. lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0); + task->preext[obj_idx].cand_count = lnk_count_gsi_candidates(task->cv->debug_s_arr[obj_idx]); } barrier_wait(tp->barrier); - // alloc module streams + // Allocate module streams and the exact global-symbol content set. Candidate count depends + // only on record headers, so this table is ready before the one transformed window visit. if (task_id == 0) { + U64 cand_count = 0; + for EachIndex(obj_idx, task->cv->obj_count) { cand_count += task->preext[obj_idx].cand_count; } + task->gsi_dedup_bucket_cap = Max(cand_count * 13 / 10, 1); + task->gsi_dedup_buckets = push_array(scratch.arena, void *, task->gsi_dedup_bucket_cap); for EachIndex(obj_idx, task->cv->obj_count) { PDB_DbiModule *mod = task->mod_arr[obj_idx]; U64 mod_size = mod->sym_data_size + mod->c11_data_size + mod->c13_data_size + mod->globrefs_size; @@ -5321,339 +5308,52 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) barrier_wait(tp->barrier); } - // ===== Streaming-ring P3.3: two-pass global-symbol dedup + winner materialize ===== - // Candidates are POSITION records ({hash, node, off, size} -- see LNK_GsiPreExtractObj): - // nothing aliases the reused window, so dedup cannot byte-compare through live pointers the - // way the P3.2 content-CAS did. Instead: sort the candidate set by (hash, flat idx), take - // each equal-hash run as a group, then RE-READ only the bytes the outcome needs through a - // fresh symbols-only window fill (deterministic re-fill = the visit bytes): each group's - // LEADER (first member in flat order) materializes as the winner payload; every other - // member re-reads once to byte-confirm equality against its leader. A confirmed mismatch is - // a genuine 64-bit hash collision (expected ~never at any scale we link) and materializes - // as its own additional winner in first-occurrence order -- so the winner CONTENT SET is - // exactly the old deduper's (distinct record bytes), only the array ORDER differs (group - // order instead of compacted-slot order). That order is downstream-invisible: - // lnk_move_global_symbols_to_gsi buckets winners by name hash and every GSI bucket is - // content-sorted at serialization (gsi_symbol_is_before keyed on the content hash written - // into dst->offset), and byte-identical records cannot reach the sort. - ProfBegin("Dedup Global Symbols"); + // Compact the exact winners published during the one module-write window visit. Bucket + // order is deterministic for a fixed content set; downstream GSI buckets content-sort + // records before serialization, so the pointer-array order is byte-invisible. + ProfBegin("Compact Global Symbol Winners"); { - PDB_GsiContext *gsi = task->pdb->gsi; - - // positional flatten in ascending obj-index order (deterministic for any cohort - // width/schedule). The per-obj candidate arrays remain live through this phase and are - // already indexed by flat-base + local. Count radix buckets directly from those arrays, - // then scatter {hash, flat} into the single sorted array below; a separate flat-order - // sort-entry array would duplicate another 16 bytes per candidate at peak memory. - U64 global_symbol_count = 0; - U64 *cand_offsets = 0; // [obj_count+1] prefix sums in obj-index order - U32 *grp_of_flat = 0; // [N] group id | LNK_GSI_LEADER_BIT - if (task_id == 0) { - cand_offsets = push_array_no_zero(scratch.arena, U64, task->cv->obj_count + 1); - U64 acc = 0; - for EachIndex(obj_idx, task->cv->obj_count) { - cand_offsets[obj_idx] = acc; - acc += task->preext[obj_idx].cand_count; - } - cand_offsets[task->cv->obj_count] = acc; - global_symbol_count = acc; - U64 n = global_symbol_count ? global_symbol_count : 1; - grp_of_flat = push_array_no_zero(scratch.arena, U32, n); - } - tp_broadcast(&global_symbol_count); - tp_broadcast(&cand_offsets); - tp_broadcast(&grp_of_flat); - - // sort by (hash, flat idx): 256-way MSB scatter (deterministic positional layout: bucket - // base + per-(bucket,worker) prefix), then a full per-bucket - // sort -- the final array is a pure function of the candidate set for any cohort - ProfBegin("Group Global Symbols"); - U64 bucket_count = 256; - U64 *scatter_counts = 0; // [C][256] - U64 *scatter_cursor = 0; // [C][256] - U64 *bucket_base = 0; // [256+1] - LNK_GsiSortEnt *sorted = 0; // [N] - if (task_id == 0) { - scatter_counts = push_array(scratch.arena, U64, tp->worker_count * bucket_count); - scatter_cursor = push_array(scratch.arena, U64, tp->worker_count * bucket_count); - bucket_base = push_array(scratch.arena, U64, bucket_count + 1); - sorted = push_array_no_zero(scratch.arena, LNK_GsiSortEnt, global_symbol_count ? global_symbol_count : 1); - } - tp_broadcast(&scatter_counts); - tp_broadcast(&scatter_cursor); - tp_broadcast(&bucket_base); - tp_broadcast(&sorted); - - for EachIndex(i, obj_indices.count) { - LNK_GsiPreExtractObj *pre = &task->preext[obj_indices.v[i]]; - for EachIndex(k, pre->cand_count) { - scatter_counts[task_id * bucket_count + (pre->cand_hashes[k] >> 56)] += 1; - } + U64 *compact_counts = 0; + U64 winner_count = 0; + void **winner_ptrs = 0; + if (task_id == 0) { compact_counts = push_array(scratch.arena, U64, tp->worker_count); } + tp_broadcast(&compact_counts); + + U64 slot_lo = (task_id * task->gsi_dedup_bucket_cap) / tp->worker_count; + U64 slot_hi = ((task_id + 1) * task->gsi_dedup_bucket_cap) / tp->worker_count; + for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { + void *p = task->gsi_dedup_buckets[slot]; + Assert(p != LNK_GSI_DEDUP_RESERVED); + compact_counts[task_id] += (p != 0); } barrier_wait(tp->barrier); + for EachIndex(w, tp->worker_count) { winner_count += compact_counts[w]; } if (task_id == 0) { - U64 acc = 0; - for EachIndex(b, bucket_count) { - bucket_base[b] = acc; - for EachIndex(w, tp->worker_count) { - scatter_cursor[w * bucket_count + b] = acc; - acc += scatter_counts[w * bucket_count + b]; - } - } - bucket_base[bucket_count] = acc; - Assert(acc == global_symbol_count); + winner_ptrs = push_array_no_zero(task->pdb->gsi->arena, void *, winner_count ? winner_count : 1); } - barrier_wait(tp->barrier); - - for EachIndex(i, obj_indices.count) { - U64 obj_idx = obj_indices.v[i]; - LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; - U64 flat_base = cand_offsets[obj_idx]; - for EachIndex(k, pre->cand_count) { - U64 hash = pre->cand_hashes[k]; - U64 b = hash >> 56; - sorted[scatter_cursor[task_id * bucket_count + b]++] = (LNK_GsiSortEnt){hash, safe_cast_u32(flat_base + k)}; - } - } - barrier_wait(tp->barrier); - - for (U64 b = task_id; b < bucket_count; b += tp->worker_count) { - radsort(sorted + bucket_base[b], bucket_base[b+1] - bucket_base[b], lnk_gsi_sort_ent_is_before); - } - barrier_wait(tp->barrier); - - // groups = equal-hash runs; group ids are bucket-major run ordinals (deterministic) - U64 *bucket_group_counts = 0; // [256] - U64 *group_base = 0; // [256+1] - U64 group_count = 0; - if (task_id == 0) { - bucket_group_counts = push_array(scratch.arena, U64, bucket_count); - group_base = push_array(scratch.arena, U64, bucket_count + 1); - } - tp_broadcast(&bucket_group_counts); - tp_broadcast(&group_base); - - for (U64 b = task_id; b < bucket_count; b += tp->worker_count) { - U64 c = 0; - for (U64 k = bucket_base[b], opl = bucket_base[b+1]; k < opl; ) { - U64 e = k + 1; - while (e < opl && sorted[e].hash == sorted[k].hash) { e += 1; } - c += 1; k = e; - } - bucket_group_counts[b] = c; - } - barrier_wait(tp->barrier); - - if (task_id == 0) { - U64 acc = 0; - for EachIndex(b, bucket_count) { group_base[b] = acc; acc += bucket_group_counts[b]; } - group_base[bucket_count] = acc; - } - group_count = 0; - if (task_id == 0) { group_count = group_base[bucket_count]; } - tp_broadcast(&group_count); - - for (U64 b = task_id; b < bucket_count; b += tp->worker_count) { - U64 gid = group_base[b]; - for (U64 k = bucket_base[b], opl = bucket_base[b+1]; k < opl; ) { - U64 e = k + 1; - while (e < opl && sorted[e].hash == sorted[k].hash) { e += 1; } - grp_of_flat[sorted[k].flat_idx] = (U32)gid | LNK_GSI_LEADER_BIT; // min flat idx of the run (flat is the sort tiebreak) - for (U64 m = k + 1; m < e; m += 1) { grp_of_flat[sorted[m].flat_idx] = (U32)gid; } - gid += 1; k = e; - } - Assert(gid == group_base[b+1]); - } - barrier_wait(tp->barrier); - ProfEnd(); - - // leader materialize (pass A): per obj, re-read the leader records' bytes (symbols-only - // window fill under g_debug_s_window; the live patched backing otherwise) and copy them - // into per-lane payload arenas on gsi->arena (alive through GSI serialization), keyed by - // group id -- exactly one writer per group. Arena packing order varies with the cohort; - // every consumer reads through winner_ptrs (group order), never the arenas. - ProfBegin("Materialize Global Winners"); - U64 *mat_sizes = 0; // [C] leader payload bytes per lane - Arena **mat_arenas = 0; // [C] - void **winner_ptrs = 0; // [group_count] on gsi->arena - if (task_id == 0) { - mat_sizes = push_array(scratch.arena, U64, tp->worker_count); - winner_ptrs = push_array_no_zero(gsi->arena, void *, group_count ? group_count : 1); - } - tp_broadcast(&mat_sizes); tp_broadcast(&winner_ptrs); - for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { - LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; - U64 lo = cand_offsets[obj_idx], opl = cand_offsets[obj_idx+1]; - for (U64 k = lo; k < opl; k += 1) { - if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { mat_sizes[task_id] += AlignPow2(pre->cand_lens[k - lo] + sizeof(CV_SymSize), sizeof(void *)); } - } - } - barrier_wait(tp->barrier); - if (task_id == 0) { - mat_arenas = alloc_arena_many(gsi->arena, tp->worker_count, mat_sizes); - } - tp_broadcast(&mat_arenas); - - for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { - U64 lo = cand_offsets[obj_idx], opl = cand_offsets[obj_idx+1]; - LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; - - B32 any_leader = 0; - for (U64 k = lo; k < opl; k += 1) { if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { any_leader = 1; break; } } - if (!any_leader) { continue; } - - Temp wtemp = temp_begin(scratch.arena); - CV_DebugS resolved = g_debug_s_window - ? lnk_obj_window_debug_s(wtemp.arena, task->cv, obj_idx, task->pe.image_base, task->image_section_table, 1) - : task->cv->debug_s_arr[obj_idx]; - String8List symbols = cv_sub_section_from_debug_s(resolved, CV_C13SubSectionKind_Symbols); - U8 **node_base = push_array_no_zero(wtemp.arena, U8 *, symbols.node_count ? symbols.node_count : 1); - { U64 ni = 0; for EachNode(n, String8Node, symbols.first) { node_base[ni++] = n->string.str; } } - - Arena *dst_arena = mat_arenas[task_id]; - for (U64 k = lo; k < opl; k += 1) { - if (!(grp_of_flat[k] & LNK_GSI_LEADER_BIT)) { continue; } - U64 local_idx = k - lo; - U32 size = pre->cand_lens[local_idx] + sizeof(CV_SymSize); - U8 *src = node_base[pre->cand_nodes[local_idx]] + pre->cand_offs[local_idx]; - U8 *copy = push_array_no_zero(dst_arena, U8, size); - MemoryCopy(copy, src, size); -#if BUILD_DEBUG - // re-read fidelity: the fresh window fill must reproduce the visit-time bytes exactly - Assert(pre->cand_hashes[local_idx] == u64_hash_from_str8(str8(copy, size))); -#endif - winner_ptrs[grp_of_flat[k] & ~LNK_GSI_LEADER_BIT] = copy; - } - temp_end(wtemp); - } - barrier_wait(tp->barrier); - - // byte-confirm (pass B): every non-leader member re-reads once and memcmps against its - // leader's materialized copy. A mismatch (same 64-bit hash, different bytes) is recorded - // -- bytes copied onto the transient per-lane cand arena -- and resolved serially below. - LNK_GsiMismatch **mismatch_lists = 0; // [C] - U64 *mismatch_counts = 0; // [C] - if (task_id == 0) { - mismatch_lists = push_array(scratch.arena, LNK_GsiMismatch *, tp->worker_count); - mismatch_counts = push_array(scratch.arena, U64, tp->worker_count); - } - tp_broadcast(&mismatch_lists); - tp_broadcast(&mismatch_counts); - - for (U64 obj_idx = task_id; obj_idx < task->cv->obj_count; obj_idx += tp->worker_count) { - U64 lo = cand_offsets[obj_idx], opl = cand_offsets[obj_idx+1]; - LNK_GsiPreExtractObj *pre = &task->preext[obj_idx]; - - B32 any_member = 0; - for (U64 k = lo; k < opl; k += 1) { if (!(grp_of_flat[k] & LNK_GSI_LEADER_BIT)) { any_member = 1; break; } } - if (!any_member) { continue; } - - Temp wtemp = temp_begin(scratch.arena); - CV_DebugS resolved = g_debug_s_window - ? lnk_obj_window_debug_s(wtemp.arena, task->cv, obj_idx, task->pe.image_base, task->image_section_table, 1) - : task->cv->debug_s_arr[obj_idx]; - String8List symbols = cv_sub_section_from_debug_s(resolved, CV_C13SubSectionKind_Symbols); - U8 **node_base = push_array_no_zero(wtemp.arena, U8 *, symbols.node_count ? symbols.node_count : 1); - { U64 ni = 0; for EachNode(n, String8Node, symbols.first) { node_base[ni++] = n->string.str; } } - - for (U64 k = lo; k < opl; k += 1) { - if (grp_of_flat[k] & LNK_GSI_LEADER_BIT) { continue; } - U64 local_idx = k - lo; - U64 grp = grp_of_flat[k]; - U32 size = pre->cand_lens[local_idx] + sizeof(CV_SymSize); - U8 *src = node_base[pre->cand_nodes[local_idx]] + pre->cand_offs[local_idx]; - String8 leader = cv_raw_from_symbol(winner_ptrs[grp]); -#if BUILD_DEBUG - Assert(pre->cand_hashes[local_idx] == u64_hash_from_str8(str8(src, size))); -#endif - if (leader.size == size && MemoryMatch(leader.str, src, size)) { continue; } - // genuine hash collision: keep the bytes (transient cand arena, dies with the phase) - LNK_GsiMismatch *m = push_array_no_zero(task->cand_arenas[task_id], LNK_GsiMismatch, 1); - U8 *bytes_copy = push_array_no_zero(task->cand_arenas[task_id], U8, size); - MemoryCopy(bytes_copy, src, size); - m->grp = grp; - m->flat_idx = k; - m->bytes = str8(bytes_copy, size); - m->next = mismatch_lists[task_id]; - mismatch_lists[task_id] = m; - mismatch_counts[task_id] += 1; - } - temp_end(wtemp); + U64 dst = 0; + for (U64 w = 0; w < task_id; w += 1) { dst += compact_counts[w]; } + for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { + void *p = task->gsi_dedup_buckets[slot]; + if (p != 0) { winner_ptrs[dst++] = p; } } barrier_wait(tp->barrier); - - // serial collision resolution + final winner array (task 0). Mismatches are expected to - // be ZERO on real inputs; the path exists so a 64-bit collision degrades to correct - // output instead of silently folding distinct records. Distinct extra contents - // materialize in (group, first-occurrence flat) order -- deterministic. if (task_id == 0) { - U64 total_mismatches = 0; - for EachIndex(w, tp->worker_count) { total_mismatches += mismatch_counts[w]; } - - U64 winner_total = group_count; - - if (total_mismatches == 0) { - task->gsi_winner_count = winner_total; - task->gsi_winner_ptrs = winner_ptrs; - } else { - Temp mtemp = temp_begin(scratch.arena); - LNK_GsiMismatch **flat_mismatches = push_array_no_zero(mtemp.arena, LNK_GsiMismatch *, total_mismatches); - { - U64 c = 0; - for EachIndex(w, tp->worker_count) { - for (LNK_GsiMismatch *m = mismatch_lists[w]; m != 0; m = m->next) { flat_mismatches[c++] = m; } - } - Assert(c == total_mismatches); - } - radsort(flat_mismatches, total_mismatches, lnk_gsi_mismatch_is_before); - - void **extras = push_array_no_zero(mtemp.arena, void *, total_mismatches); - U64 extra_count = 0; - for (U64 a = 0; a < total_mismatches; ) { - U64 b = a; - while (b < total_mismatches && flat_mismatches[b]->grp == flat_mismatches[a]->grp) { b += 1; } - // distinct contents beyond the leader, first-occurrence order within the group - U64 group_extra_lo = extra_count; - for (U64 k = a; k < b; k += 1) { - String8 bytes = flat_mismatches[k]->bytes; - B32 is_distinct = 1; - for (U64 e = group_extra_lo; e < extra_count && is_distinct; e += 1) { - String8 seen = cv_raw_from_symbol(extras[e]); - if (seen.size == bytes.size && MemoryMatch(seen.str, bytes.str, bytes.size)) { is_distinct = 0; } - } - if (is_distinct) { - U8 *copy = push_array_no_zero(gsi->arena, U8, bytes.size); - MemoryCopy(copy, bytes.str, bytes.size); - extras[extra_count++] = copy; - } - } - a = b; - } - - void **final_ptrs = push_array_no_zero(gsi->arena, void *, winner_total + extra_count); - MemoryCopyTyped(final_ptrs, winner_ptrs, winner_total); - MemoryCopyTyped(final_ptrs + winner_total, extras, extra_count); - temp_end(mtemp); - - task->gsi_winner_count = winner_total + extra_count; - task->gsi_winner_ptrs = final_ptrs; - } + task->gsi_winner_count = winner_count; + task->gsi_winner_ptrs = winner_ptrs; } - barrier_wait(tp->barrier); - ProfEnd(); + } + ProfEnd(); - // copies release: moved here from mid-globals (P3.2). The winner materialize above was + // Copies release: moved here from mid-globals. Direct winner materialization above was // the LAST $S reader on this path -- release the patched debug-section copies before the // globals/publics staging begins instead of overlapping it. Same /PDBSTRIPPED gate as // before (its second build re-walks cv->debug_s_arr Symbols after lnk_build_pdb returns; - // the materialize still ran -- winner ptrs never dangle either way). cand_arenas hold - // only the candidate ref/hash arrays (proc-ref arrays live on the surviving payload - // arenas) and die with the copies UNCONDITIONALLY -- they are ours; poison the dangling - // cand fields, keep the procref fields for the globals pass. The barrier after the copy - // loop above guarantees every worker is done reading before task 0 releases. + // the materialize still ran -- winner ptrs never dangle either way). The barrier above + // guarantees every worker is done reading before task 0 releases section copies. if (task_id == 0) { if (task->free_sect_copies) { ProfScope("Release Sect Data Copies") { @@ -5665,21 +5365,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) } } } - ProfScope("Release GSI Candidate Arenas") { - for EachIndex(i, task->preext_arena_count) { - if (task->cand_arenas[i] != 0) { arena_release(task->cand_arenas[i]); task->cand_arenas[i] = 0; } - } - for EachIndex(obj_idx, task->cv->obj_count) { - task->preext[obj_idx].cand_count = 0; - task->preext[obj_idx].cand_hashes = 0; // poison dangling refs (incl. the pass-B mismatch bytes on cand_arenas) - task->preext[obj_idx].cand_offs = 0; - task->preext[obj_idx].cand_nodes = 0; - task->preext[obj_idx].cand_lens = 0; - } - } } - } - ProfEnd(); scratch_end(scratch); } @@ -6437,19 +6123,13 @@ ProfScope("Write Modules") } lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C, weights); - // P2b pre-extraction state: per-obj GSI input tables (candidate REFS into the live $S - // backing + synthesized proc-refs) filled inside the module-write visit. P3.2: the - // candidate refs are consumed by the fused dedup + winner materialize in the - // lnk_write_pdb_modules epilogue and die there with the section copies (cand_arenas - // hold only the 16B-class ref arrays); "Move Global Symbols" consumes the materialized - // winners + the proc-ref tables, which live on the surviving payload arenas until GSI - // serialization (same lifetime as the old proc_ref_arenas). + // Per-obj counts/proc-refs plus per-lane surviving payload arenas. Exact global-symbol + // winners are copied into these arenas directly from the one transformed $S window; + // Move Global Symbols later consumes the compacted winner pointers and proc-ref tables. task.preext = push_array(scratch.arena, LNK_GsiPreExtractObj, cv->obj_count); task.preext_arena_count = C; - task.cand_arenas = push_array(scratch.arena, Arena *, C); task.procref_payload_arenas = push_array(scratch.arena, Arena *, C); for EachIndex(i, C) { - task.cand_arenas[i] = arena_alloc(.commit_size = MB(2), .name = "GSI_CANDIDATES"); task.procref_payload_arenas[i] = arena_alloc(.commit_size = MB(2), .name = "GSI_PROC_REFS"); } #if BUILD_DEBUG diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index e3eba9853..19ac55b1f 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -322,28 +322,16 @@ typedef enum LNK_PDB_BuilderFlag_NATVIS = (1<<5), } LNK_PDB_BuilderFlags; -// Streaming-ring P2b/P3.3: per-obj global-symbol candidate REFS + proc-refs, extracted inside -// the module-write per-obj visit (fused with the sizing walk, over the obj's post-fixup $S -// bytes -- the window copy when g_debug_s_window, the patched backing otherwise). P3.3 -// candidates are POSITION records, not pointers: {content hash, Symbols node ordinal, offset -// within node, record size} -- 18B/record, NO payload copies and NO live-backing aliasing, so -// they survive the window being reused for the next obj. The module-write epilogue dedups by -// hash grouping (sort by (hash, flat idx)), then RE-READS just the bytes it needs through the -// same window fill: group leaders to materialize the winner payload, other members to -// byte-confirm equality against their leader (a same-hash different-bytes member is a genuine -// 64-bit collision -- expected ~never -- and materializes as its own winner, preserving the -// old deduper's exact content set). -// Proc-ref payloads AND the per-obj procref value/hash arrays live on the surviving +// Streaming-ring P2b: per-obj global-symbol counts + proc-refs. Global-symbol records are +// content-deduped directly from the one module-write window and only exact winners are copied +// to surviving storage; no candidate metadata or payload survives the window. Proc-ref +// payloads AND the per-obj procref value/hash arrays live on the surviving // procref_payload_arenas (consumed post-release by lnk_move_global_symbols_to_gsi, which // flattens per-obj segments in ascending obj-index order -- deterministic and // cohort-independent -- and otherwise reads only the materialized winner records). typedef struct { U64 cand_count; // global records (cv_is_global_symbol + top-level typedefs) - U64 *cand_hashes; // [cand_count] u64_hash_from_str8(raw) computed at extraction (post-reloc post-fixup bytes) - U32 *cand_offs; // [cand_count] record start offset within its Symbols data_list node (COFF section-sized) - U32 *cand_nodes; // [cand_count] Symbols data_list node ordinal - U16 *cand_lens; // [cand_count] native CV_SymSize; full record size adds sizeof(CV_SymSize) U64 procref_count; // GPROC32/LPROC32 records CV_Symbol *procref_syms; // [procref_count] cv_make_proc_ref results (arrays + payload on survive arenas) U32 *procref_hashes; // [procref_count] gsi_hash(name) @@ -374,27 +362,30 @@ typedef struct // when set, lnk_build_pdb drops every obj's patched debug-section copies // (LNK_Obj.section_data_copies) and releases the SECT_DATA_COPIES arenas at the END of the - // module-write phase, right after the fused dedup + winner materialize (P3.2: the last $S - // readers on this path). Must be 0 when a /PDBSTRIPPED build follows -- it re-walks + // module-write phase, right after direct winner dedup/materialization (the last $S reader + // on this path). Must be 0 when a /PDBSTRIPPED build follows -- it re-walks // cv->debug_s_arr after lnk_build_pdb. P3.1: the /names bucket rehome and the // mod->source_file_list repoint are UNCONDITIONAL (they must also cover string tables in // reloc-free RAW-MAPPED $S sections, which never had copies to begin with). B32 free_sect_copies; - // P2b pre-extraction state (see LNK_GsiPreExtractObj). cand_arenas hold ONLY the per-obj - // candidate position/hash arrays (18B per record, no payloads) and release with the - // section copies at the end of Write Modules. procref_payload_arenas hold the procref - // value/hash arrays + cv_make_proc_ref payloads and must survive until GSI serialization + // P2b pre-extraction state (see LNK_GsiPreExtractObj). procref_payload_arenas hold exact + // global-symbol winners plus the procref value/hash arrays and cv_make_proc_ref payloads. + // They must survive until GSI serialization // (like the old proc_ref_arenas, they live to process exit). LNK_GsiPreExtractObj *preext; // [cv->obj_count] - Arena **cand_arenas; // [preext_arena_count] Arena **procref_payload_arenas; // [preext_arena_count] U64 preext_arena_count; // Write Modules cohort width - // P3.2 fused-dedup output: the global-record winner set, materialized at the end of the - // module-write phase in compacted-slot order (ptr array + payload bytes on gsi->arena, - // alive through GSI serialization). The ONLY global-symbol input - // lnk_move_global_symbols_to_gsi reads. + // Transient exact-content set used only inside Write Modules. Empty slots are claimed with + // a reservation sentinel before a winner is copied, so duplicates allocate no payload. + U64 gsi_dedup_bucket_cap; + void **gsi_dedup_buckets; + + // Direct exact-dedup output: the winner pointer array is compacted from hash-table slot + // order onto gsi->arena; payload bytes remain on the per-worker surviving arenas above. + // Both stay alive through GSI serialization. This is the only global-symbol input read by + // lnk_move_global_symbols_to_gsi. U64 gsi_winner_count; void **gsi_winner_ptrs; // [gsi_winner_count] } LNK_BuildPdb; From b8142b92bf42bb635a5fec22fb09b7c90e2392e6 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 21:53:59 -0700 Subject: [PATCH 077/121] linker: overlap PDB page decommit with writes --- src/linker/lnk_io.c | 104 ++++++++++++++++++++++++++++++++++++-------- src/linker/lnk_io.h | 5 +++ 2 files changed, 92 insertions(+), 17 deletions(-) diff --git a/src/linker/lnk_io.c b/src/linker/lnk_io.c index e4657a2d6..3b0049a55 100644 --- a/src/linker/lnk_io.c +++ b/src/linker/lnk_io.c @@ -521,6 +521,33 @@ typedef struct B32 decommit_after_write; // hand the buffer's pages back to the OS once written } LNK_BackgroundFileWriteJob; +typedef struct +{ + void *ptr; + U64 size; // zero marks the end of the queue +} LNK_BackgroundFileDecommitJob; + +internal void +lnk_background_file_decommit_thread(void *raw_writer) +{ + ProfBeginFunction(); + set_thread_namef("PDB Page Decommit"); + + LNK_BackgroundFileWriter *writer = raw_writer; + for (;;) { + LNK_BackgroundFileDecommitJob job = {0}; + RingGuard guard = guarded_ring_open(writer->decommit_queue); + B32 is_read = guarded_ring_read_struct_or_wait(&guard, &job, max_U64); + guarded_ring_close(&guard); + Assert(is_read); + if (job.size == 0) { break; } + decommit_memory(job.ptr, job.size); + ins_atomic_u64_add_eval(&writer->decommit_bytes_completed, job.size); + } + + ProfEnd(); +} + // Opening the output file can cost seconds on the caller's thread when it // replaces a previous multi-GB artifact (NTFS truncate/dealloc of the old // allocation happens inside CreateFile). Jobs on one file are queue-ordered, @@ -623,26 +650,55 @@ lnk_background_file_writer_issue_write_batch(LNK_BackgroundFileWriter *writer, L end_ptr += next->data.size; } U64 merged_size = end_off - batch[i].file_off; - if (batch[i].file->is_open) { - U64 write_size = lnk_write_file(&batch[i].file->file, batch[i].file_off, batch[i].data.str, merged_size); - if (write_size != merged_size) { - batch[i].file->write_failed = 1; + + // Keep at most one MSF data node of dead-but-still-committed page data in + // flight. Besides bounding the memory lag versus synchronous decommit, + // this prevents one multi-node VirtualFree from delaying all reclamation. + U64 decommit_pipeline_cap = MB(128); + U64 part_cap = writer->is_decommit_running && batch[i].decommit_after_write ? decommit_pipeline_cap : max_U64; + for (U64 part_off = 0; part_off < merged_size; ) { + U64 part_size = Min(merged_size - part_off, part_cap); + U8 *part_ptr = batch[i].data.str + part_off; + if (batch[i].file->is_open) { + U64 write_size = lnk_write_file(&batch[i].file->file, batch[i].file_off + part_off, part_ptr, part_size); + if (write_size != part_size) { + batch[i].file->write_failed = 1; + } + batch[i].file->bytes_written += write_size; + U64 done_now = ins_atomic_u64_add_eval(&writer->bytes_completed, write_size); + writer->writes_issued += 1; + if ((done_now >> 29) != ((done_now - write_size) >> 29)) { // every 512MiB + lnk_log(LNK_Log_Timers, "[pdbw] t=%.3fs written=%.2f GiB enq=%.2f GiB jobs=%llu/%llu writes=%llu avg=%llu KiB", (F64)(now_time_us() - writer->begin_time_us) / 1e6, (F64)done_now / GB(1), (F64)ins_atomic_u64_eval(&writer->bytes_enqueued) / GB(1), writer->jobs_completed, ins_atomic_u64_eval(&writer->jobs_enqueued), writer->writes_issued, (done_now / writer->writes_issued) / KB(1)); + } } - batch[i].file->bytes_written += write_size; - U64 done_now = ins_atomic_u64_add_eval(&writer->bytes_completed, write_size); - writer->jobs_completed += (j + 1 - i); - writer->writes_issued += 1; - if ((done_now >> 29) != ((done_now - write_size) >> 29)) { // every 512MiB - lnk_log(LNK_Log_Timers, "[pdbw] t=%.3fs written=%.2f GiB enq=%.2f GiB jobs=%llu/%llu writes=%llu avg=%llu KiB", (F64)(now_time_us() - writer->begin_time_us) / 1e6, (F64)done_now / GB(1), (F64)ins_atomic_u64_eval(&writer->bytes_enqueued) / GB(1), writer->jobs_completed, ins_atomic_u64_eval(&writer->jobs_enqueued), writer->writes_issued, (done_now / writer->writes_issued) / KB(1)); + if (batch[i].decommit_after_write) { + // The buffer is immutable and write-once (a sealed MSF stream run); once it is + // on disk its pages are dead weight in the commit charge. Round INWARD to whole + // OS pages so neighbors sharing the boundary pages are untouched. + U64 lo = AlignPow2((U64)part_ptr, KB(4)); + U64 hi = ((U64)part_ptr + part_size) & ~(U64)(KB(4) - 1); + if (lo < hi) { + U64 decommit_size = hi - lo; + if (writer->is_decommit_running) { + while (ins_atomic_u64_eval(&writer->decommit_bytes_enqueued) - + ins_atomic_u64_eval(&writer->decommit_bytes_completed) + decommit_size > decommit_pipeline_cap) { + sleep_ms(0); + } + ins_atomic_u64_add_eval(&writer->decommit_bytes_enqueued, decommit_size); + LNK_BackgroundFileDecommitJob job = { .ptr = (void *)lo, .size = decommit_size }; + RingGuard guard = guarded_ring_open(writer->decommit_queue); + B32 is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64); + guarded_ring_close(&guard); + Assert(is_written); + } else { + decommit_memory((void *)lo, decommit_size); + } + } } + part_off += part_size; } - if (batch[i].decommit_after_write) { - // the buffer is immutable and write-once (a sealed MSF stream run); once it is - // on disk its pages are dead weight in the commit charge. Round INWARD to whole - // OS pages so neighbors sharing the boundary pages are untouched. - U64 lo = AlignPow2((U64)batch[i].data.str, KB(4)); - U64 hi = ((U64)end_ptr) & ~(U64)(KB(4) - 1); - if (lo < hi) { decommit_memory((void *)lo, hi - lo); } + if (batch[i].file->is_open) { + writer->jobs_completed += (j + 1 - i); } i = j + 1; } @@ -656,6 +712,9 @@ lnk_background_file_writer_thread(void *raw_writer) LNK_BackgroundFileWriter *writer = raw_writer; + writer->decommit_thread = thread_launch(lnk_background_file_decommit_thread, writer); + writer->is_decommit_running = writer->decommit_thread.u64[0] != 0; + U64 batch_cap = 8192; LNK_BackgroundFileWriteJob *batch = (LNK_BackgroundFileWriteJob *)reserve_memory(batch_cap * sizeof(batch[0])); commit_memory(batch, batch_cap * sizeof(batch[0])); @@ -712,6 +771,16 @@ lnk_background_file_writer_thread(void *raw_writer) } } + if (writer->is_decommit_running) { + LNK_BackgroundFileDecommitJob job = {0}; + RingGuard guard = guarded_ring_open(writer->decommit_queue); + B32 is_written = guarded_ring_write_struct_or_wait(&guard, &job, max_U64); + guarded_ring_close(&guard); + Assert(is_written); + thread_join(writer->decommit_thread, max_U64); + writer->is_decommit_running = 0; + } + release_memory(batch, batch_cap * sizeof(batch[0])); ProfEnd(); } @@ -723,6 +792,7 @@ lnk_background_file_writer_begin(LNK_BackgroundFileWriter *writer) writer->queue_arena = arena_alloc(.reserve_size = MB(2), .commit_size = MB(2), .name = "BACKGROUND_FILE_WRITE_QUEUE"); writer->queue = guarded_ring_alloc(writer->queue_arena, MB(1)); + writer->decommit_queue = guarded_ring_alloc(writer->queue_arena, KB(64)); writer->begin_time_us = now_time_us(); ProfEnd(); } diff --git a/src/linker/lnk_io.h b/src/linker/lnk_io.h index 0238dcc3b..dfc7414bb 100644 --- a/src/linker/lnk_io.h +++ b/src/linker/lnk_io.h @@ -30,13 +30,18 @@ typedef struct { Arena *queue_arena; GuardedRing *queue; + GuardedRing *decommit_queue; Thread thread; + Thread decommit_thread; LNK_BackgroundFile *file_first; LNK_BackgroundFile *file_last; B32 is_running; + B32 is_decommit_running; U64 begin_time_us; // Timers telemetry anchor U64 bytes_enqueued; // atomic; producer side U64 bytes_completed; // atomic; writer thread + U64 decommit_bytes_enqueued; // atomic; writer thread + U64 decommit_bytes_completed; // atomic; decommit thread U64 jobs_enqueued; // atomic; producer side U64 jobs_completed; // writer thread U64 writes_issued; // writer thread; post-coalesce WriteFile count From e5027dfba825b81337a70d4b385285ce5c894192 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 13 Aug 2026 23:58:52 -0700 Subject: [PATCH 078/121] linker: accelerate type-leaf hash probes Use power-of-two masking in the dense leaf-dedup probe loops and tag bucket pointers with a three-bit hash fingerprint to avoid most representative-hash reloads. Winner ordering and output bytes remain unchanged. Frozen FN PGO A/B: merge wall 3066 -> 2975 ms average; total user CPU 278.2 -> 268.6 s; peak commit effectively flat (28.5 vs 28.6 GB median). DLL/PDB byte-identical and deterministic; Lyra 9/9; debug default and GCTYPES pass. --- src/linker/lnk_debug_info.c | 60 ++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 8feda8a71..7365bc75a 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -2497,7 +2497,10 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, internal force_inline LNK_LeafRef lnk_leaf_ref_make(U64 obj_idx, U64 leaf_idx) { - LNK_LeafRef result = Compose64Bit(safe_cast_u32(obj_idx), safe_cast_u32(leaf_idx)); + // Type indices consume the low 32 bits. Realistic object counts fit comfortably in the + // remaining 29 payload bits; the top three bits stay free for the dedup-table hash tag. + Assert(obj_idx < (1ull << 29) - 1); + LNK_LeafRef result = (obj_idx << 32) | safe_cast_u32(leaf_idx); Assert(result != LNK_LEAF_REF_NULL); return result; } @@ -2505,7 +2508,7 @@ lnk_leaf_ref_make(U64 obj_idx, U64 leaf_idx) internal force_inline U32 lnk_leaf_ref_obj_idx(LNK_LeafRef ref) { - return (U32)(ref >> 32); + return (U32)((ref >> 32) & ((1ull << 29) - 1)); } internal force_inline U32 @@ -2694,6 +2697,23 @@ lnk_match_leaf_ref(LNK_CodeViewInput *input, LNK_LeafRef a, LNK_LeafRef b) return a_hash == b_hash; } +#define LNK_LEAF_BUCKET_TAG_MASK (7ull << 61) + +internal force_inline LNK_LeafRef +lnk_leaf_bucket_tag(LNK_LeafRef ref, U64 hash) +{ + Assert((ref & LNK_LEAF_BUCKET_TAG_MASK) == 0); + LNK_LeafRef result = ref | (hash & LNK_LEAF_BUCKET_TAG_MASK); + Assert(result != LNK_LEAF_REF_NULL); + return result; +} + +internal force_inline LNK_LeafRef +lnk_leaf_bucket_untag(LNK_LeafRef ref) +{ + return ref & ~LNK_LEAF_BUCKET_TAG_MASK; +} + // P4: `leaf` is the caller's (journal-aware) read of the leaf -- this function no longer // re-reads it from the raw view. `journal_arena` backs NOTYPE journal growth for real objs // (pseudo objs keep in-place rewrites and may pass 0). @@ -3040,20 +3060,22 @@ THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); // leaf header -> leaf kind CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); // leaf kind -> type stream LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[leaf_source]; // type stream -> hash table - U64 best_idx = debug_h->v[leaf_idx] % leaf_ht->cap; // leaf ref -> hash -> bucket index + U64 hash = debug_h->v[leaf_idx]; // leaf ref -> hash + U64 best_idx = hash & (leaf_ht->cap - 1); // hash -> bucket index + LNK_LeafRef tagged = lnk_leaf_bucket_tag(leaf_ref, hash); U64 idx = best_idx; do { - LNK_LeafRef curr = ins_atomic_u64_eval(&leaf_ht->bucket_arr[idx]); - if (curr == LNK_LEAF_REF_NULL) { - LNK_LeafRef cmp = ins_atomic_u64_eval_cond_assign(&leaf_ht->bucket_arr[idx], leaf_ref, curr); - if (cmp == curr) { + LNK_LeafRef curr_tagged = ins_atomic_u64_eval(&leaf_ht->bucket_arr[idx]); + if (curr_tagged == LNK_LEAF_REF_NULL) { + LNK_LeafRef cmp = ins_atomic_u64_eval_cond_assign(&leaf_ht->bucket_arr[idx], tagged, curr_tagged); + if (cmp == curr_tagged) { goto exit; } } // advance to next bucket - idx = ((idx + 1) == leaf_ht->cap ? 0 : (idx + 1)); + idx = (idx + 1) & (leaf_ht->cap - 1); } while (idx != best_idx); is_inserted_or_updated = 0; exit:; @@ -3093,7 +3115,6 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) // and retried with the total-based caps, so bail out early if (ins_atomic_u32_eval(&task->leaf_ht_overflow) != 0) { break; } if (is_ifc_blob && cv_debug_t_get_leaf_header(debug_t, leaf_idx)->kind == CV_LeafKind_NOTYPE) { continue; } - B32 is_inserted_or_updated = 1; LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); @@ -3102,31 +3123,36 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) if (notype_bm && leaf_idx < notype_cap && ((notype_bm[leaf_idx >> 6] >> (leaf_idx & 63)) & 1)) { kind = CV_LeafKind_NOTYPE; } CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); // leaf kind -> type stream LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[leaf_source]; // type stream -> hash table - U64 best_idx = debug_h->v[leaf_idx] % leaf_ht->cap; // leaf ref -> hash -> bucket index + U64 hash = debug_h->v[leaf_idx]; // leaf ref -> hash + U64 best_idx = hash & (leaf_ht->cap - 1); // hash -> bucket index + LNK_LeafRef tagged = lnk_leaf_bucket_tag(leaf_ref, hash); U64 idx = best_idx; do { // load leaf ref - LNK_LeafRef curr = ins_atomic_u64_eval(&leaf_ht->bucket_arr[idx]); + LNK_LeafRef curr_tagged = ins_atomic_u64_eval(&leaf_ht->bucket_arr[idx]); - while (curr == LNK_LEAF_REF_NULL || lnk_match_leaf_ref(task->input, curr, leaf_ref)) { + while (curr_tagged == LNK_LEAF_REF_NULL || + ((curr_tagged & LNK_LEAF_BUCKET_TAG_MASK) == (tagged & LNK_LEAF_BUCKET_TAG_MASK) && + lnk_hash_from_leaf_ref(task->input, lnk_leaf_bucket_untag(curr_tagged)) == hash)) { + LNK_LeafRef curr = curr_tagged == LNK_LEAF_REF_NULL ? LNK_LEAF_REF_NULL : lnk_leaf_bucket_untag(curr_tagged); // exit if leaf ref is not recent if (curr != LNK_LEAF_REF_NULL && lnk_leaf_ref_compare(leaf_ref, curr) >= 0) { goto exit; } // try to update the bucket - LNK_LeafRef cmp = ins_atomic_u64_eval_cond_assign(&leaf_ht->bucket_arr[idx], leaf_ref, curr); - if (cmp == curr) { + LNK_LeafRef cmp = ins_atomic_u64_eval_cond_assign(&leaf_ht->bucket_arr[idx], tagged, curr_tagged); + if (cmp == curr_tagged) { goto exit; } // another thread updated the bucket -- retry - curr = cmp; + curr_tagged = cmp; } // advance to next bucket - idx = ((idx + 1) == leaf_ht->cap ? 0 : (idx + 1)); + idx = (idx + 1) & (leaf_ht->cap - 1); } while (idx != best_idx); is_inserted_or_updated = 0; @@ -3173,7 +3199,7 @@ THREAD_POOL_TASK_FUNC(lnk_get_present_buckets_task) for EachInRange(bucket_idx, task->ranges[task_id]) { if (ht->bucket_arr[bucket_idx] != LNK_LEAF_REF_NULL) { - unique_leaf_refs.v[cursor++] = ht->bucket_arr[bucket_idx]; + unique_leaf_refs.v[cursor++] = lnk_leaf_bucket_untag(ht->bucket_arr[bucket_idx]); } } From 9577c824314a22518a774dc0dbfab4b97c336874 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 00:40:06 -0700 Subject: [PATCH 079/121] linker: skip identity radix passes The type-leaf radix sort already detected all-zero digits but still performed the full stable scatter. Skip those identity passes; on current editor links the zero-width low object-index digit no longer streams all unique-leaf pointers through an unnecessary pass. Frozen FN PGO A/B: merge wall 2983 -> 2920 ms average, merge user CPU 89.9 -> 88.0 s, median total wall 18.5 -> 18.2 s; peak commit and WS unchanged. DLL/PDB byte-identical and deterministic; Lyra 9/9; debug default and GCTYPES pass. --- src/linker/base_ext/base_radix_sort.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/linker/base_ext/base_radix_sort.c b/src/linker/base_ext/base_radix_sort.c index f4c07fab7..e5739ff85 100644 --- a/src/linker/base_ext/base_radix_sort.c +++ b/src/linker/base_ext/base_radix_sort.c @@ -67,16 +67,22 @@ u64_array_sort_radix_parallel(TP_Context *tp, U64 count, U64 *v) tp_for_parallel(tp, 0, task_count, u64_radix_hist_task, &task); U64 cursor = 0; + U64 populated_digit_count = 0; for EachIndex(digit, U64_RADIX_SIZE) { + U64 digit_begin = cursor; for EachIndex(task_idx, task_count) { U64 *slot = &offsets[task_idx * U64_RADIX_SIZE + digit]; U64 slot_count = *slot; *slot = cursor; cursor += slot_count; } + populated_digit_count += (cursor != digit_begin); } Assert(cursor == count); + // A stable radix pass with a single populated digit is the identity transform. + if (populated_digit_count == 1) { continue; } + tp_for_parallel(tp, 0, task_count, u64_radix_scatter_task, &task); Swap(U64 *, task.src, task.dst); } From 42f88b99751005cfe8ea920517d79887339c239a Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 01:00:04 -0700 Subject: [PATCH 080/121] linker: remove division from assigned-type probes Map 64-bit type hashes into the existing assigned-TI table with multiply-high instead of runtime remainder. The mapping remains uniform and deterministic, table capacity and memory are unchanged, and the final sorted type order remains byte-identical. Frozen FN PGO A/B: merge wall 2978 -> 2911 ms average, merge user CPU 89.9 -> 87.0 s; total wall neutral and peak commit/WS unchanged. DLL/PDB byte-identical and deterministic; Lyra 9/9; debug default and GCTYPES pass. --- src/linker/lnk_debug_info.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 7365bc75a..0dfbce4cd 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -2929,12 +2929,36 @@ lnk_hash_cv_leaf_deep(Arena *arena, temp_end(temp); } +// Map a uniformly distributed 64-bit hash into [0, cap) with one multiply-high. Unlike `% cap`, +// this does not require a runtime integer division; hash tables only require a stable uniform +// mapping, not the remainder specifically. +force_inline U64 +lnk_hash_range(U64 hash, U64 cap) +{ +#if COMPILER_MSVC && ARCH_X64 + U64 high; + _umul128(hash, cap, &high); + return high; +#elif (COMPILER_CLANG || COMPILER_GCC) && ARCH_64BIT + return (U64)(((__uint128_t)hash * (__uint128_t)cap) >> 64); +#else + U64 hash_lo = (U32)hash, hash_hi = hash >> 32; + U64 cap_lo = (U32)cap, cap_hi = cap >> 32; + U64 p00 = hash_lo * cap_lo; + U64 p01 = hash_lo * cap_hi; + U64 p10 = hash_hi * cap_lo; + U64 p11 = hash_hi * cap_hi; + U64 carry = ((p00 >> 32) + (U32)p01 + (U32)p10) >> 32; + return p11 + (p01 >> 32) + (p10 >> 32) + carry; +#endif +} + internal CV_TypeIndex lnk_assigned_ti_hash_search(LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) { CV_DebugH *debug_h = &input->debug_h_arr[lnk_leaf_ref_obj_idx(leaf_ref)]; U64 hash = debug_h->v[lnk_leaf_ref_leaf_idx(leaf_ref)]; - U64 best_idx = hash % ht->cap; + U64 best_idx = lnk_hash_range(hash, ht->cap); U64 idx = best_idx; do { CV_TypeIndex ti = ht->ti_arr[idx]; @@ -3222,7 +3246,7 @@ THREAD_POOL_TASK_FUNC(lnk_assign_type_indices_task) CV_TypeIndex type_index = min_type_index + i; U64 hash = debug_h_arr[lnk_leaf_ref_obj_idx(leaf_ref)].v[lnk_leaf_ref_leaf_idx(leaf_ref)]; - U64 best_idx = hash % assigned->cap; + U64 best_idx = lnk_hash_range(hash, assigned->cap); U64 idx = best_idx; B32 is_inserted = 0; From d1041dcf4e5140951404fa734a0a433b9c435f04 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 01:24:18 -0700 Subject: [PATCH 081/121] linker: raise debug-input worker cap to 20 Giant-input jobification moved the page-fault throughput knee above the old 12-worker default. Use 20 workers for parse, prefetch, and overlapped unmap stages; /RAD_DEBUG_WORKERS remains available for machine-specific overrides and output is unchanged. Frozen FN PGO cap sweep: 20 vs 16 averaged 17.8 vs 18.65 s total wall and 4.11 vs 4.61 s mcvi, for about +5 s kernel CPU and +1.4 s user CPU; 24/32 crossed the contention cliff. Peak commit and WS unchanged. Lyra 9/9. --- src/linker/lnk_config.c | 8 ++++---- src/linker/lnk_debug_info.c | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index cc4e8524a..acf7959e5 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -109,7 +109,7 @@ global read_only LNK_CmdSwitch g_cmd_switch_map[] = { LNK_CmdSwitch_Rad_UnresolvedSymbolRefLimit, 0, LNK_CmdValueKind_Scalar, "RAD_UNRESOLVED_SYMBOL_REF_LIMIT", ":#", "Limit number of unresolved symbol references linker reports." }, { LNK_CmdSwitch_Rad_Version, 0, LNK_CmdValueKind_Null, "RAD_VERSION", "", "Print version and exit." }, { LNK_CmdSwitch_Rad_Workers, 0, LNK_CmdValueKind_Scalar, "RAD_WORKERS", ":#", "Set number of workers created in the pool. Number is capped at 1024. When /RAD_SHARED_THREAD_POOL is specified this number cant exceed /RAD_SHARED_THREAD_POOL_MAX_WORKERS." }, - { LNK_CmdSwitch_Rad_DebugWorkers, 0, LNK_CmdValueKind_Scalar, "RAD_DEBUG_WORKERS", ":#", "Cap concurrent workers in page-fault-bound debug-input stages (parse/prefetch). Default 12; 0 = uncapped. Output is identical either way; the cap only trades idle spinning in the kernel page-fault path for free cores." }, + { LNK_CmdSwitch_Rad_DebugWorkers, 0, LNK_CmdValueKind_Scalar, "RAD_DEBUG_WORKERS", ":#", "Cap concurrent workers in page-fault-bound debug-input stages (parse/prefetch). Default 20; 0 = uncapped. Output is identical either way; the cap only trades idle spinning in the kernel page-fault path for free cores." }, { LNK_CmdSwitch_Rad_WorkDir, 0, LNK_CmdValueKind_Scalar, "RAD_WORK_DIR", ":PATH", "Working directory used for stable debug paths." }, { LNK_CmdSwitch_RadTypeServer, 0, LNK_CmdValueKind_Scalar, "RAD_TYPE_SERVER", ":FILENAME", "Merge types and store them in the specified file. The filename must have the .rrt extension." }, @@ -2985,10 +2985,10 @@ lnk_config_init(U64 argc, char **argv) config->work_dir = get_current_path(arena); config->force = lnk_cmd_line_has_switch(cmd_line, LNK_CmdSwitch_Force); - // fault-bound debug-input stages spin on the kernel page-fault path past ~12 - // concurrent workers (throughput ceiling); default cap trades that spin for + // fault-bound debug-input stages spin on the kernel page-fault path past ~20 + // concurrent workers (current throughput knee); default cap trades that spin for // free cores, /RAD_DEBUG_WORKERS:0 restores full width - config->debug_worker_cap = 12; + config->debug_worker_cap = 20; // apply command line switches for EachNode(cmd, LNK_CmdOption, cmd_line.first_option) { diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 0dfbce4cd..915b7b9ac 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -128,9 +128,9 @@ THREAD_POOL_TASK_FUNC(lnk_prefetch_task) // Run a per-item parallel-for on at most `cap` workers. The debug-input stages // this wraps are page-fault-bound: the kernel working-set-insert path tops out -// near ~3M pages/s regardless of thread count, so lanes past ~12 only convert -// free cores into spin inside the fault handler (measured on the FN editor -// DLL: 12 lanes beat 64 on wall AND cut stage kernel CPU 357s -> 41s). Items +// near ~3M pages/s regardless of thread count, so lanes past ~20 only convert +// free cores into spin inside the fault handler (the knee moved up after giant-input +// jobification; 24+ lanes still regress wall and sharply increase kernel CPU). Items // are pulled from a shared cursor, so per-item outputs land in the same // item-indexed slots as the uncapped path -- output is byte-identical. typedef struct From 07b17532dd8ce996e2d1d88588c52556909245aa Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 01:47:51 -0700 Subject: [PATCH 082/121] linker: remove division from GSI dedup probes --- src/linker/lnk_debug_info.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 915b7b9ac..2caabb1ad 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -5013,7 +5013,7 @@ internal void lnk_gsi_deduper_insert_copy(void **buckets, U64 bucket_cap, U64 hash, void *symbol_ptr, Arena *dst_arena) { String8 raw = cv_raw_from_symbol(symbol_ptr); - U64 best_idx = hash % bucket_cap; + U64 best_idx = lnk_hash_range(hash, bucket_cap); U64 idx = best_idx; for (;;) { void *curr = ins_atomic_ptr_eval(&buckets[idx]); From 1a4ca6a804ec95956c45c8c721a01f4952c29ee3 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 02:00:25 -0700 Subject: [PATCH 083/121] linker: mask fixed-size GSI bucket probes --- src/linker/lnk_debug_info.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 2caabb1ad..4e13e12c4 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4584,7 +4584,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) // single-threaded on task 0; sizing has no determinism impact) if (task_id == 0) { U64 *bucket_adds = push_array(scratch.arena, U64, gsi->bucket_count); - for EachIndex(i, symbol_count) { bucket_adds[symbol_hashes[i] % gsi->bucket_count] += 1; } + for EachIndex(i, symbol_count) { bucket_adds[symbol_hashes[i] & (PDB_GSI_V70_BUCKET_COUNT - 1)] += 1; } for EachIndex(bucket_idx, gsi->bucket_count) { if (bucket_adds[bucket_idx]) { gsi_reserve(gsi, bucket_idx, bucket_adds[bucket_idx]); } } @@ -4600,7 +4600,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; for EachIndex(i, symbol_count) { - U64 bucket_idx = symbol_hashes[i] % gsi->bucket_count; + U64 bucket_idx = symbol_hashes[i] & (PDB_GSI_V70_BUCKET_COUNT - 1); if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; CV_Symbol *dst = &bucket->v[bucket->count]; @@ -4667,7 +4667,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) // single-threaded on task 0; sizing has no determinism impact) if (task_id == 0) { U64 *bucket_adds = push_array(scratch.arena, U64, gsi->bucket_count); - for EachIndex(i, total_proc_ref_count) { bucket_adds[proc_ref_hashes[i] % gsi->bucket_count] += 1; } + for EachIndex(i, total_proc_ref_count) { bucket_adds[proc_ref_hashes[i] & (PDB_GSI_V70_BUCKET_COUNT - 1)] += 1; } for EachIndex(bucket_idx, gsi->bucket_count) { if (bucket_adds[bucket_idx]) { gsi_reserve(gsi, bucket_idx, bucket_adds[bucket_idx]); } } @@ -4680,7 +4680,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; for EachIndex(i, total_proc_ref_count) { - U64 bucket_idx = proc_ref_hashes[i] % gsi->bucket_count; + U64 bucket_idx = proc_ref_hashes[i] & (PDB_GSI_V70_BUCKET_COUNT - 1); if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; bucket->v[bucket->count] = proc_ref_symbols[i]; @@ -4832,7 +4832,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) if (task_id == 0) { PDB_GsiContext *pub_gsi = psi->gsi; U64 *bucket_adds = push_array(scratch.arena, U64, pub_gsi->bucket_count); - for EachIndex(i, public_symbol_total_count) { bucket_adds[public_symbol_flat_hashes[i] % pub_gsi->bucket_count] += 1; } + for EachIndex(i, public_symbol_total_count) { bucket_adds[public_symbol_flat_hashes[i] & (PDB_GSI_V70_BUCKET_COUNT - 1)] += 1; } for EachIndex(bucket_idx, pub_gsi->bucket_count) { if (bucket_adds[bucket_idx]) { gsi_reserve(pub_gsi, bucket_idx, bucket_adds[bucket_idx]); } } @@ -4846,7 +4846,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) U64 shard_min = (task_id * pub_gsi->bucket_count) / tp->worker_count; U64 shard_max = ((task_id + 1) * pub_gsi->bucket_count) / tp->worker_count; for EachIndex(i, public_symbol_total_count) { - U64 bucket_idx = public_symbol_flat_hashes[i] % pub_gsi->bucket_count; + U64 bucket_idx = public_symbol_flat_hashes[i] & (PDB_GSI_V70_BUCKET_COUNT - 1); if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } PDB_GsiSymbolBucket *bucket = &pub_gsi->bucket_arr[bucket_idx]; bucket->v[bucket->count] = public_symbol_flat_vals[i]; From debab827d6b546f8c38fda7aaa5b9d6992a319d3 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 03:04:23 -0700 Subject: [PATCH 084/121] linker: specialize fixed PDB bucket probes --- src/linker/pdb_ext/pdb_builder.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/linker/pdb_ext/pdb_builder.c b/src/linker/pdb_ext/pdb_builder.c index b4f594126..f519d8944 100644 --- a/src/linker/pdb_ext/pdb_builder.c +++ b/src/linker/pdb_ext/pdb_builder.c @@ -1133,7 +1133,8 @@ pdb_type_server_push_udt_arr(PDB_TypeServer *ts, U64 count, U32 *hash_arr, Strin bucket->raw_leaf = raw_leaf; bucket->type_index = ts->ti_lo + ts->leaf_list.node_count + leaf_idx; - U32 bucket_idx = hash % ts->bucket_cap; + Assert(ts->bucket_cap == PDB_TYPE_SERVER_HASH_BUCKET_COUNT_CURRENT); + U32 bucket_idx = hash % PDB_TYPE_SERVER_HASH_BUCKET_COUNT_CURRENT; SLLStackPush(ts->buckets[bucket_idx], bucket); } @@ -1204,7 +1205,8 @@ THREAD_POOL_TASK_FUNC(pdb_push_udt_leaf_task) if (~udt_info.props & CV_TypeProp_FwdRef) { // hash udt and compute bucket index U32 hash = pdb_hash_udt(udt_info, leaf.data); - U32 bucket_idx = hash % type_ht_cap; + Assert(type_ht_cap == PDB_TYPE_SERVER_HASH_BUCKET_COUNT_CURRENT); + U32 bucket_idx = hash % PDB_TYPE_SERVER_HASH_BUCKET_COUNT_CURRENT; // fill out & insert bucket PDB_TypeBucket *bucket = &new_buckets[bucket_cursor++]; @@ -2386,7 +2388,7 @@ gsi_reserve(PDB_GsiContext *gsi, U64 bucket_idx, U64 additional) internal CV_Symbol * gsi_push_(PDB_GsiContext *gsi, U32 hash, CV_Symbol *symbol) { - U64 bucket_idx = hash % gsi->bucket_count; + U64 bucket_idx = hash & (PDB_GSI_V70_BUCKET_COUNT - 1); PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; if (bucket->count == bucket->cap) { // rare path: bulk inserters reserve up front via gsi_reserve @@ -2463,7 +2465,7 @@ gsi_search(PDB_GsiContext *gsi, CV_Symbol *symbol) { String8 name = cv_name_from_symbol(symbol->kind, symbol->data); U32 hash = gsi_hash(gsi, name); - U64 ibucket = hash % gsi->bucket_count; + U64 ibucket = hash & (PDB_GSI_V70_BUCKET_COUNT - 1); PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[ibucket]; for EachIndex(i, bucket->count) { From 830a7b62fc2b849998be60a8b52abfc610de2dab Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 04:00:04 -0700 Subject: [PATCH 085/121] linker: improve type-fixup locality without extra memory Reuse the merged-leaf pointer array for sizing metadata so materialization avoids a second journal/header lookup. Store assigned-TI hash and index in adjacent 12-byte entries, preserving the former 8+4 byte footprint while reducing random cache-line loads during fixups. FN PGO A/B: leaf metadata wall median 17.35 -> 17.10 s with memory flat; packed assigned table user CPU 265.7 -> 262.5 s and merge user CPU 88.5 -> 86.5 s with wall/memory neutral. DLL/PDB byte identity and repeat determinism pass. --- src/linker/lnk_debug_info.c | 83 ++++++++++++++++++++----------------- src/linker/lnk_debug_info.h | 11 +++-- 2 files changed, 53 insertions(+), 41 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 4e13e12c4..78d2f0299 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -2598,18 +2598,28 @@ lnk_cv_leaf_from_leaf_ref(LNK_CodeViewInput *input, U32 obj_idx, U32 leaf_idx) // Journal-aware raw leaf size (materialize buffer sizing + copy). Hot path = the ORIGINAL // header read (one bitmap test on top); only a full NOTYPE rewrite changes the answer (the // leaf shrank to its 4-byte header; KIND_ONLY keeps the size field). +#define LNK_LEAF_MATERIALIZE_FULL_NOTYPE 1 +#define LNK_LEAF_MATERIALIZE_KIND_NOTYPE 2 + internal U64 -lnk_leaf_ref_raw_size(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) +lnk_leaf_ref_materialize_meta(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) { U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + U64 raw_size = cv_debug_t_get_raw_leaf(debug_t, leaf_idx).size; + U64 rewrite = 0; if (obj_idx < input->obj_count && lnk_notype_journal_test(&input->notype_journal[obj_idx], leaf_idx)) { B32 kind_only = 0; lnk_notype_journal_find(&input->notype_journal[obj_idx], leaf_idx, &kind_only); - if (!kind_only) { return sizeof(CV_LeafHeader); } + if (kind_only) { + rewrite = LNK_LEAF_MATERIALIZE_KIND_NOTYPE; + } else { + raw_size = sizeof(CV_LeafHeader); + rewrite = LNK_LEAF_MATERIALIZE_FULL_NOTYPE; + } } - return cv_debug_t_get_raw_leaf(debug_t, leaf_idx).size; + return (raw_size << 2) | rewrite; } internal LNK_LeafRef @@ -2961,9 +2971,10 @@ lnk_assigned_ti_hash_search(LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LN U64 best_idx = lnk_hash_range(hash, ht->cap); U64 idx = best_idx; do { - CV_TypeIndex ti = ht->ti_arr[idx]; + U8 *entry = ht->v + idx * LNK_ASSIGNED_TI_ENTRY_SIZE; + CV_TypeIndex ti = memory_read32(entry + LNK_ASSIGNED_TI_TI_OFF); if (ti == 0) { break; } - if (ht->hash_arr[idx] == hash) { return ti; } + if (memory_read64(entry + LNK_ASSIGNED_TI_HASH_OFF) == hash) { return ti; } idx = (idx + 1) == ht->cap ? 0 : (idx + 1); } while (idx != best_idx); @@ -3251,11 +3262,13 @@ THREAD_POOL_TASK_FUNC(lnk_assign_type_indices_task) B32 is_inserted = 0; do { - CV_TypeIndex curr_type_index = assigned->ti_arr[idx]; + U8 *entry = assigned->v + idx * LNK_ASSIGNED_TI_ENTRY_SIZE; + CV_TypeIndex *ti_ptr = (CV_TypeIndex *)(entry + LNK_ASSIGNED_TI_TI_OFF); + CV_TypeIndex curr_type_index = *ti_ptr; if (curr_type_index == 0) { - CV_TypeIndex cmp_type_index = ins_atomic_u32_eval_cond_assign(&assigned->ti_arr[idx], type_index, curr_type_index); + CV_TypeIndex cmp_type_index = ins_atomic_u32_eval_cond_assign(ti_ptr, type_index, curr_type_index); if (cmp_type_index == curr_type_index) { - assigned->hash_arr[idx] = hash; + memory_write64(entry + LNK_ASSIGNED_TI_HASH_OFF, hash); is_inserted = 1; break; } @@ -3764,8 +3777,12 @@ THREAD_POOL_TASK_FUNC(lnk_count_unique_leaf_sizes_task) Rng1U64 range = task->ranges[task_id]; U64 size = 0; for EachInRange(i, range) { - // P4: journal-aware offsets-chain size -- no leaf-header reads through the mapped views - size += lnk_leaf_ref_raw_size(task->input, task->unique_leaf_refs_arr[task->ti_source].v[i]); + // Cache size + NOTYPE disposition in result.v's otherwise-unused slots. Materialization + // consumes and replaces each value with the final pointer, avoiding a second journal lookup + // and raw-leaf header read without allocating a side array. + U64 meta = lnk_leaf_ref_materialize_meta(task->input, task->unique_leaf_refs_arr[task->ti_source].v[i]); + task->result.v[task->ti_source][i] = (U8 *)meta; + size += meta >> 2; } task->leaf_buffer_offsets[task_id] = size; // exclusive-scanned into offsets on the main thread } @@ -3789,38 +3806,33 @@ THREAD_POOL_TASK_FUNC(lnk_materialize_unique_leaves_task) U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + U64 meta = (U64)task->result.v[task->ti_source][i]; + U64 raw_size = meta >> 2; + U64 rewrite = meta & 3; // copy raw leaf into the private buffer. Hot path = the ORIGINAL read+memcpy (one bitmap // test on top); journaled leaves replay the NOTYPE rewrite into the COPY (full rewrite => // bare { size=sizeof(CV_LeafKind), kind=LF_NOTYPE } header; KIND_ONLY (0x1522) => copy then // patch the kind). Byte-identical to the old post-in-place-write copy. - U64 raw_size; - if (obj_idx < task->input->obj_count && - lnk_notype_journal_test(&task->input->notype_journal[obj_idx], leaf_idx)) { - B32 kind_only = 0; - lnk_notype_journal_find(&task->input->notype_journal[obj_idx], leaf_idx, &kind_only); - if (kind_only) { - String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_idx); - raw_size = raw_leaf.size; - MemoryCopy(cursor, raw_leaf.str, raw_size); - memory_write16(cursor + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); - } else { - raw_size = sizeof(CV_LeafHeader); - memory_write16(cursor + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); + if (rewrite == LNK_LEAF_MATERIALIZE_FULL_NOTYPE) { + memory_write16(cursor + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); + memory_write16(cursor + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); + } else { + U8 *raw_leaf = debug_t->data.str + debug_t->offsets[leaf_idx]; + MemoryCopy(cursor, raw_leaf, raw_size); + if (rewrite == LNK_LEAF_MATERIALIZE_KIND_NOTYPE) { memory_write16(cursor + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); } - } else { - String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_idx); - raw_size = raw_leaf.size; - MemoryCopy(cursor, raw_leaf.str, raw_size); } task->result.v[task->ti_source][i] = cursor; cursor += raw_size; // fixup type indices on the copy (same math the in-place leaf patcher applied) Temp temp = temp_begin(fixed_arena); - CV_Leaf leaf = {0}; - cv_read_leaf(str8(task->result.v[task->ti_source][i], raw_size), 0, 1, &leaf); + CV_Leaf leaf = { + .kind = memory_read16(task->result.v[task->ti_source][i] + OffsetOf(CV_LeafHeader, kind)), + .data = str8(task->result.v[task->ti_source][i] + sizeof(CV_LeafHeader), raw_size - sizeof(CV_LeafHeader)), + }; CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); lnk_fixup_cv_type_indices(task, obj_idx, leaf.data, ti_offs); temp_end(temp); @@ -4184,13 +4196,10 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { task.ti_source = ti_source; task.assigned_ti_arr[ti_source].cap = ((task.unique_leaf_refs_arr[ti_source].count * 13) / 10); - task.assigned_ti_arr[ti_source].ti_arr = push_array(scratch.arena, CV_TypeIndex, task.assigned_ti_arr[ti_source].cap); - - // bucket_arr used to back hash_arr here to avoid a fresh allocation, but the bucket - // arena is released right after the extract loop (above) so the probe tables' multi-GB - // working set is gone before the merge-types commit peak; allocate hash_arr fresh -- - // it is sized by the (much smaller) unique count, not the total-based bucket cap - task.assigned_ti_arr[ti_source].hash_arr = push_array(scratch.arena, U64, task.assigned_ti_arr[ti_source].cap); + // Keep hash + assigned TI adjacent: the fixup passes probe this table hundreds of + // millions of times and usually need both fields. Packed 12-byte entries preserve the + // exact former 8+4 byte footprint while normally requiring one cache line instead of two. + task.assigned_ti_arr[ti_source].v = push_array(scratch.arena, U8, task.assigned_ti_arr[ti_source].cap * LNK_ASSIGNED_TI_ENTRY_SIZE); task.min_type_indices[ti_source] = CV_MinComplexTypeIndex; task.ranges = tp_divide_work(scratch.arena, task.unique_leaf_refs_arr[ti_source].count, tp->worker_count); @@ -4323,7 +4332,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK entry_count += a->count; entry_bytes += a->count * (a->is_wide ? sizeof(LNK_DebugSPatchWide) : sizeof(LNK_DebugSPatch)); wide_runs += !!a->is_wide; } U64 assigned_bytes = 0; - for EachIndex(s, CV_TypeIndexSource_COUNT) { assigned_bytes += task.assigned_ti_arr[s].cap * (sizeof(CV_TypeIndex) + sizeof(U64)); } + for EachIndex(s, CV_TypeIndexSource_COUNT) { assigned_bytes += task.assigned_ti_arr[s].cap * LNK_ASSIGNED_TI_ENTRY_SIZE; } U64 leaf_buffer_bytes = task.result.leaf_buffers[CV_TypeIndexSource_TPI].size + task.result.leaf_buffers[CV_TypeIndexSource_IPI].size; lnk_log(LNK_Log_Debug, "$S fixup journal: %llu entries / %llu bytes (%llu wide runs) (state it decouples from module write: assigned-TI tables %llu bytes, merged leaf buffers %llu bytes)", entry_count, entry_bytes, wide_runs, assigned_bytes, leaf_buffer_bytes); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 19ac55b1f..8c3e89393 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -220,11 +220,14 @@ typedef struct LNK_LeafRef *bucket_arr; } LNK_LeafHashTable; +#define LNK_ASSIGNED_TI_ENTRY_SIZE 12 +#define LNK_ASSIGNED_TI_HASH_OFF 0 +#define LNK_ASSIGNED_TI_TI_OFF 8 + typedef struct { - U64 cap; - CV_TypeIndex *ti_arr; - U64 *hash_arr; + U64 cap; + U8 *v; } LNK_AssignedTiHash; typedef struct LNK_LeafRange @@ -425,7 +428,7 @@ internal void lnk_notype_journal_push (Arena *arena, LNK_ internal B32 lnk_notype_journal_test (LNK_NotypeJournal *journal, U64 leaf_idx); internal B32 lnk_notype_journal_find (LNK_NotypeJournal *journal, U32 leaf_idx, B32 *kind_only_out); internal CV_Leaf lnk_cv_leaf_from_leaf_ref (LNK_CodeViewInput *input, U32 obj_idx, U32 leaf_idx); -internal U64 lnk_leaf_ref_raw_size (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); +internal U64 lnk_leaf_ref_materialize_meta (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, Arena *journal_arena, LNK_LeafRef leaf_ref, CV_Leaf leaf, CV_TiOffsets ti_offs, B32 discard_cycles); internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs); internal CV_TypeIndex lnk_assigned_ti_hash_search (LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); From a5c10e185368e3d7b95e50fd3d3b84aa8dac5c0d Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 04:19:26 -0700 Subject: [PATCH 086/121] linker: reuse ICF key storage for output colors The color hash key's old_color is dead after group indexing and split counting. Reuse it for the assigned output color instead of retaining a separate U64 in every refinement slot. This shrinks the 16.8M-slot FN table from 48 to 40 bytes per slot (~128 MiB). Four PGO pairs: ICF wall 1.019 -> 0.990 s, user CPU 40.85 -> 38.76 s, kernel CPU 9.45 -> 8.25 s. DLL/PDB byte identity and repeat determinism pass. --- src/linker/lnk.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 7319afafb..54eaef159 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -3252,10 +3252,11 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) } Contrib; // Reuse both tables without clearing them between refinement rounds. U32 generations and - // indices keep these records at 48 and 16 bytes respectively (down from 56 and 24 bytes). + // indices keep these records at 40 and 16 bytes respectively. The old-color key is dead + // after group indexing/counting, so the assignment phase overwrites it with the output color + // instead of carrying a second U64 through the 16M-slot table. typedef struct { ColorKey key; - U64 color; U32 state; // generation << 2 | (0 = empty, 1 = initializing, 2 = ready) U32 first_contrib_idx; U32 old_color_slot_idx; @@ -3755,9 +3756,9 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) OldColorHashSlot *old_color_slot = &old_color_table.slots[color_slot->old_color_slot_idx]; if (ins_atomic_u32_eval(&old_color_slot->first_contrib_idx) == contrib_idx) { - color_slot->color = color_slot->key.old_color; + // key.old_color has completed its lookup lifetime; reuse it as the output color } else { - color_slot->color = ++next_split_color; + color_slot->key.old_color = ++next_split_color; } } ProfEnd(); @@ -3770,7 +3771,7 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) Rng1U64 obj_contrib_range = r1u64(shared.contrib_offsets[obj_idx], shared.contrib_offsets[obj_idx] + contrib_counts[obj_idx]); for EachInRange(contrib_idx, obj_contrib_range) { Contrib *contrib = &shared.contribs[contrib_idx]; - shared.color_map[contrib->obj_idx][contrib->section_number] = color_table.slots[contrib->color_slot_idx].color; + shared.color_map[contrib->obj_idx][contrib->section_number] = color_table.slots[contrib->color_slot_idx].key.old_color; } } ProfEnd(); From 6e102ef7adf110ccf6279a7fc0a96048a31c28da Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 04:28:13 -0700 Subject: [PATCH 087/121] linker: reuse dead ICF hash bytes for group indices After colored-hash insertion completes, the 128-bit key hash is dead. Store the old-color table slot index in those bytes instead of retaining a dedicated U32 field. Together with output-color reuse this shrinks FN's 16.8M color slots from 40 to 32 bytes (~128 MiB more). Four PGO pairs vs the 40-byte version: ICF wall 1.003 -> 0.966 s, user CPU 39.99 -> 38.64 s, kernel CPU 8.21 -> 7.10 s. DLL/PDB byte identity and repeat determinism pass. --- src/linker/lnk.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 54eaef159..c4f579fc9 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -3252,14 +3252,14 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) } Contrib; // Reuse both tables without clearing them between refinement rounds. U32 generations and - // indices keep these records at 40 and 16 bytes respectively. The old-color key is dead - // after group indexing/counting, so the assignment phase overwrites it with the output color - // instead of carrying a second U64 through the 16M-slot table. + // indices keep these records at 32 and 16 bytes respectively. The 128-bit group hash is dead + // before old-color indexing, so that phase stores its U32 slot index in the hash bytes. The + // old-color key is then dead after split counting, so assignment overwrites it with the output + // color. These lifetime reuses avoid carrying either result through the 16M-slot table. typedef struct { ColorKey key; U32 state; // generation << 2 | (0 = empty, 1 = initializing, 2 = ready) U32 first_contrib_idx; - U32 old_color_slot_idx; } ColorHashSlot; typedef struct { ColorHashSlot *slots; U64 slots_count; } ColorHashTable; @@ -3702,7 +3702,7 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) old_slot_idx = (old_slot_idx + 1) & (old_color_table.slots_count - 1); } - slot->old_color_slot_idx = safe_cast_u32(old_slot_idx); + memory_write32(&slot->key.hash, safe_cast_u32(old_slot_idx)); } } ProfEnd(); @@ -3716,7 +3716,8 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) ColorHashSlot *color_slot = &color_table.slots[contrib->color_slot_idx]; if (ins_atomic_u32_eval(&color_slot->first_contrib_idx) != contrib_idx) { continue; } - OldColorHashSlot *old_color_slot = &old_color_table.slots[color_slot->old_color_slot_idx]; + U32 old_color_slot_idx = memory_read32(&color_slot->key.hash); + OldColorHashSlot *old_color_slot = &old_color_table.slots[old_color_slot_idx]; if (ins_atomic_u32_eval(&old_color_slot->first_contrib_idx) != contrib_idx) { split_count += 1; } @@ -3754,7 +3755,8 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) ColorHashSlot *color_slot = &color_table.slots[contrib->color_slot_idx]; if (ins_atomic_u32_eval(&color_slot->first_contrib_idx) != contrib_idx) { continue; } - OldColorHashSlot *old_color_slot = &old_color_table.slots[color_slot->old_color_slot_idx]; + U32 old_color_slot_idx = memory_read32(&color_slot->key.hash); + OldColorHashSlot *old_color_slot = &old_color_table.slots[old_color_slot_idx]; if (ins_atomic_u32_eval(&old_color_slot->first_contrib_idx) == contrib_idx) { // key.old_color has completed its lookup lifetime; reuse it as the output color } else { From f6b438664ba39152d7bf923a288e00905e09286e Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 04:45:00 -0700 Subject: [PATCH 088/121] linker: compact cached ICF relocation targets A cached relocation target uses either a live color pointer (Regular symbols) or a static ID, never both. Union those fields and use the existing interpretation tag to select the active member, shrinking targets from 24 to 16 bytes. Four PGO pairs: ICF wall 0.959 -> 0.948 s, user CPU 38.67 -> 37.30 s, and faults 750.5K -> 732K pages. DLL/PDB identity and repeat determinism passed for the union layout. --- src/linker/lnk.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index c4f579fc9..5788ca04c 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -3232,8 +3232,10 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) // only target colors vary between rounds, so cache non-recursive relocation data // and rehash target colors each round typedef struct { - U64 *color; - U64 static_id; + union { + U64 *color; + U64 static_id; + }; U64 association_id; U32 value; COFF_SymbolValueInterpType interp; @@ -3617,7 +3619,7 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) lnk_hasher_update_struct(&hasher, &contrib->static_hash); for EachIndex(reloc_idx, contrib->reloc_count) { RelocTarget *target = contrib->reloc_targets[reloc_idx]; - U64 target_id = target->color ? *target->color : target->static_id; + U64 target_id = target->interp == COFF_SymbolValueInterp_Regular ? *target->color : target->static_id; lnk_hasher_update_struct(&hasher, &target_id); lnk_hasher_update_struct(&hasher, &target->association_id); } From ac6bd5a82cf688f0aced32a3d2e7acc8f7e77191 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 04:54:21 -0700 Subject: [PATCH 089/121] linker: pack OPT:REF resolve cache values Store resolved object input index and U32 symbol index in one U64 and recover the object through the existing objs_by_idx table. This reduces each worker's 32K-slot memo from 768 to 512 KiB (~16 MiB total). Four PGO pairs: REF wall 360.5 vs 361.5 ms and CPU neutral within noise, with ~4K fewer faults. DLL/PDB byte identity and repeat determinism pass. --- src/linker/lnk.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 5788ca04c..0725b1994 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -2785,8 +2785,10 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) // a miss only costs the recompute -- the cached value is a pure function of the key because // the symbol table and parsed_symbols are read-only during /OPT:REF. // Keep the table small enough that initializing every worker does not become a page-fault - // amplifier (32K slots is 768KiB per worker, versus 24MiB at 1M slots). - typedef struct { U64 key; LNK_Obj *obj; U64 symbol_idx; } LNK_RefResolveSlot; + // amplifier. Pack the resolved object input index + symbol index into one U64 and recover the + // object through objs_by_idx: 32K slots is 512KiB per worker (versus 768KiB with a pointer and + // widened symbol index, or 24MiB at the old 1M-slot size). + typedef struct { U64 key; U64 ref; } LNK_RefResolveSlot; U64 resolve_cache_mask = (1ull << 15) - 1; LNK_RefResolveSlot *resolve_cache = push_array_no_zero(scratch.arena, LNK_RefResolveSlot, resolve_cache_mask + 1); MemorySet(resolve_cache, 0xff, sizeof(resolve_cache[0]) * (resolve_cache_mask + 1)); @@ -2818,7 +2820,9 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) for (U64 probe_idx = 0; probe_idx < 8; probe_idx += 1) { U64 slot = (cache_hash + probe_idx) & resolve_cache_mask; if (resolve_cache[slot].key == cache_key) { - ref_symbol = (LNK_ObjSymbolRef){ .obj = resolve_cache[slot].obj, .symbol_idx = (U32)resolve_cache[slot].symbol_idx }; + U64 packed_ref = resolve_cache[slot].ref; + ref_symbol = packed_ref != max_U64 ? (LNK_ObjSymbolRef){ .obj = objs_by_idx[packed_ref >> 32], .symbol_idx = (U32)packed_ref } + : (LNK_ObjSymbolRef){0}; cache_hit = 1; break; } @@ -2877,7 +2881,8 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) // memoize (skip the cyclic-warning path so the warning replays per reloc as before) if (!was_cyclic) { if (cache_slot == max_U64) { cache_slot = cache_hash & resolve_cache_mask; } - resolve_cache[cache_slot] = (LNK_RefResolveSlot){ .key = cache_key, .obj = ref_symbol.obj, .symbol_idx = ref_symbol.symbol_idx }; + U64 packed_ref = ref_symbol.obj ? Compose64Bit(ref_symbol.obj->input_idx, ref_symbol.symbol_idx) : max_U64; + resolve_cache[cache_slot] = (LNK_RefResolveSlot){ .key = cache_key, .ref = packed_ref }; } } } From a66d188d3ea778c9ca3c62ac1b91fd55c585a270 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 10:12:47 -0700 Subject: [PATCH 090/121] linker: fuse PDB global and proc-ref extraction --- src/linker/lnk_debug_info.c | 102 +++++++++++++++++------------------- 1 file changed, 48 insertions(+), 54 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 78d2f0299..c6a00d530 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4994,15 +4994,22 @@ internal void lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context * #define LNK_GSI_DEDUP_RESERVED ((void *)(U64)1) internal U64 -lnk_count_gsi_candidates(CV_DebugS debug_s) +lnk_count_gsi_candidates(CV_DebugS debug_s, U64 *procref_count_out) { - U64 count = 0; + U64 count = 0; + U64 procref_count = 0; String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); for EachNode(n, String8Node, symbols.first) { for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { CV_Symbol symbol = {0}; TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { count += 1; } + // ID procs are rewritten to their non-ID forms by the deferred fixup replay before + // extraction. Count both spellings here while reading the raw invariant headers. + if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32 || + symbol.kind == CV_SymKind_GPROC32_ID || symbol.kind == CV_SymKind_LPROC32_ID) { + procref_count += 1; + } if (cv_is_scope_symbol(symbol.kind)) { depth += 1; } else if (cv_is_end_symbol(symbol.kind)) { @@ -5011,6 +5018,7 @@ lnk_count_gsi_candidates(CV_DebugS debug_s) } } } + *procref_count_out = procref_count; return count; } @@ -5071,78 +5079,64 @@ lnk_extract_gsi_inputs_for_obj(LNK_BuildPdb *task, U64 obj_idx, U64 task_id, CV_ PDB_GsiContext *gsi = task->pdb->gsi; Arena *payload_arena = task->procref_payload_arenas[task_id]; - // Dedup directly from the one post-reloc/post-fixup window. The size-only pass already - // counted these records from invariant headers so the shared table is pre-sized. - U64 cand_count = 0; + // Dedup globals and build proc refs in one decode walk over the post-reloc/post-fixup + // window. The size-only pass counted both record sets from invariant raw headers, so all + // arrays and the shared table are exact-sized before this visit. + U64 procref_count = pre->procref_count; + if (procref_count) { + pre->procref_syms = push_array_no_zero(payload_arena, CV_Symbol, procref_count); + pre->procref_hashes = push_array_no_zero(payload_arena, U32, procref_count); + } + + CV_ModIndex imod = task->mod_arr[obj_idx]->imod; + U64 symbol_cursor = sizeof(CV_Signature); + U64 scope_depth = 0; + U64 cand_count = 0; + U64 procref_idx = 0; for EachNode(n, String8Node, symbols.first) { - for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { + U64 cand_depth = 0; + B32 cand_active = 1; + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { CV_Symbol symbol = {0}; TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); - if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { + + if (cand_active && (cv_is_global_symbol(symbol.kind) || (cand_depth == 0 && cv_is_typedef(symbol.kind)))) { U8 *ptr = cv_ptr_from_symbol(symbol); lnk_gsi_deduper_insert_copy(task->gsi_dedup_buckets, task->gsi_dedup_bucket_cap, u64_hash_from_str8(cv_raw_from_symbol(ptr)), ptr, payload_arena); cand_count += 1; } - if (cv_is_scope_symbol(symbol.kind)) { - depth += 1; - } else if (cv_is_end_symbol(symbol.kind)) { - if (depth == 0) { Assert(0 && "malformed symbol stream"); break; } - depth -= 1; - } - } - } - Assert(cand_count == pre->cand_count); - - // proc-ref count pass (mirrors the old proc-ref sizing walk) - U64 procref_count = 0; - for EachNode(n, String8Node, symbols.first) { - for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); - if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32) { - procref_count += 1; + if (cand_active) { + if (cv_is_scope_symbol(symbol.kind)) { + cand_depth += 1; + } else if (cv_is_end_symbol(symbol.kind)) { + if (cand_depth == 0) { Assert(0 && "malformed symbol stream"); cand_active = 0; } + else { cand_depth -= 1; } + } } - } - } - - // proc-ref fill pass (mirrors the old build walk: identical layout math + skip set). - // The value/hash arrays live on the surviving payload arena. - if (procref_count) { - pre->procref_syms = push_array_no_zero(payload_arena, CV_Symbol, procref_count); - pre->procref_hashes = push_array_no_zero(payload_arena, U32, procref_count); - - CV_ModIndex imod = task->mod_arr[obj_idx]->imod; - U64 symbol_cursor = sizeof(CV_Signature); - U64 scope_depth = 0; - U64 k = 0; - for EachNode(n, String8Node, symbols.first) { - for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); - - if (symbol.kind == CV_SymKind_SKIP) { continue; } - else if (cv_is_global_symbol(symbol.kind)) { continue; } - else if (cv_is_typedef(symbol.kind) && scope_depth == 0) { continue; } - else if (symbol.kind == 0x1176) { continue; } + B32 is_module_symbol = (symbol.kind != CV_SymKind_SKIP && + !cv_is_global_symbol(symbol.kind) && + !(cv_is_typedef(symbol.kind) && scope_depth == 0) && + symbol.kind != 0x1176); + if (is_module_symbol) { if (cv_is_scope_symbol(symbol.kind)) { scope_depth += 1; } else if (cv_is_end_symbol(symbol.kind)) { scope_depth -= 1; } if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32) { String8 name = cv_name_from_symbol(symbol.kind, symbol.data); - pre->procref_syms[k] = cv_make_proc_ref(payload_arena, imod, symbol_cursor, name, cv_is_lproc(symbol)); - pre->procref_syms[k].offset = symbol_cursor; - pre->procref_hashes[k] = gsi_hash(gsi, name); - k += 1; + pre->procref_syms[procref_idx] = cv_make_proc_ref(payload_arena, imod, symbol_cursor, name, cv_is_lproc(symbol)); + pre->procref_syms[procref_idx].offset = symbol_cursor; + pre->procref_hashes[procref_idx] = gsi_hash(gsi, name); + procref_idx += 1; } symbol_cursor += cv_write_symbol_buf(0, 0, &symbol, PDB_SYMBOL_ALIGN); } } - Assert(k == procref_count); - pre->procref_count = procref_count; } + Assert(cand_count == pre->cand_count); + Assert(procref_idx == procref_count); } internal @@ -5169,7 +5163,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) // re-accumulated sizes match. Fixup replay, parity, and the GSI extraction all move to // the write pass's single fill. lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0); - task->preext[obj_idx].cand_count = lnk_count_gsi_candidates(task->cv->debug_s_arr[obj_idx]); + task->preext[obj_idx].cand_count = lnk_count_gsi_candidates(task->cv->debug_s_arr[obj_idx], &task->preext[obj_idx].procref_count); } barrier_wait(tp->barrier); From 6eb94a4807fdd042a56964b8e7d68349ab1b7b3a Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 10:27:52 -0700 Subject: [PATCH 091/121] linker: count PDB symbol inputs during module sizing --- src/linker/lnk_debug_info.c | 56 +++++++++++++++---------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index c6a00d530..3405096c7 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4900,7 +4900,7 @@ lnk_assert_debug_s_prov_parity(LNK_Obj *obj, CV_DebugS *debug_s) } internal U64 -lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8Node *buf, U64 *buf_pos) +lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8Node *buf, U64 *buf_pos, LNK_GsiPreExtractObj *pre) { U64 mod_cursor = 0; @@ -4920,10 +4920,30 @@ lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8No // write symbols U64 scope_depth = 0; for EachNode(n, String8Node, symbols.first) { + U64 cand_depth = 0; + B32 cand_active = 1; for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { CV_Symbol symbol = {0}; TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); + if (pre != 0) { + if (cand_active && (cv_is_global_symbol(symbol.kind) || (cand_depth == 0 && cv_is_typedef(symbol.kind)))) { + pre->cand_count += 1; + } + if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32 || + symbol.kind == CV_SymKind_GPROC32_ID || symbol.kind == CV_SymKind_LPROC32_ID) { + pre->procref_count += 1; + } + if (cand_active) { + if (cv_is_scope_symbol(symbol.kind)) { + cand_depth += 1; + } else if (cv_is_end_symbol(symbol.kind)) { + if (cand_depth == 0) { Assert(0 && "malformed symbol stream"); cand_active = 0; } + else { cand_depth -= 1; } + } + } + } + if (symbol.kind == CV_SymKind_SKIP) { continue; } else if (cv_is_global_symbol(symbol.kind)) { continue; } else if (cv_is_typedef(symbol.kind) && scope_depth == 0) { continue; } @@ -4993,35 +5013,6 @@ internal void lnk_pdb_output_enqueue_stream(LNK_PdbOutput *output, MSF_Context * #define LNK_GSI_DEDUP_RESERVED ((void *)(U64)1) -internal U64 -lnk_count_gsi_candidates(CV_DebugS debug_s, U64 *procref_count_out) -{ - U64 count = 0; - U64 procref_count = 0; - String8List symbols = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); - for EachNode(n, String8Node, symbols.first) { - for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); - if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { count += 1; } - // ID procs are rewritten to their non-ID forms by the deferred fixup replay before - // extraction. Count both spellings here while reading the raw invariant headers. - if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32 || - symbol.kind == CV_SymKind_GPROC32_ID || symbol.kind == CV_SymKind_LPROC32_ID) { - procref_count += 1; - } - if (cv_is_scope_symbol(symbol.kind)) { - depth += 1; - } else if (cv_is_end_symbol(symbol.kind)) { - if (depth == 0) { Assert(0 && "malformed symbol stream"); break; } - depth -= 1; - } - } - } - *procref_count_out = procref_count; - return count; -} - // Exact concurrent content set. Claiming an empty slot with a sentinel before allocating is // important: only the thread that adds a distinct record copies bytes, so the common duplicate // case consumes neither transient metadata nor discarded arena space. Published pointers own @@ -5162,8 +5153,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) // g_debug_s_window (the patched backing otherwise) -- the write pass below asserts the // re-accumulated sizes match. Fixup replay, parity, and the GSI extraction all move to // the write pass's single fill. - lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0); - task->preext[obj_idx].cand_count = lnk_count_gsi_candidates(task->cv->debug_s_arr[obj_idx], &task->preext[obj_idx].procref_count); + lnk_write_debug_s_to_pdb_module(task->mod_arr[obj_idx], task->cv->debug_s_arr[obj_idx], 0, 0, &task->preext[obj_idx]); } barrier_wait(tp->barrier); @@ -5224,7 +5214,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) #endif String8Node buf = *mod_data.first; U64 pos = 0; - lnk_write_debug_s_to_pdb_module(mod, debug_s, &buf, &pos); + lnk_write_debug_s_to_pdb_module(mod, debug_s, &buf, &pos, 0); #if BUILD_DEBUG Assert(mod->sym_data_size == size_check[0]); Assert(mod->c11_data_size == size_check[1]); From 05921903bf42358cef3384426591ed75017ddcd0 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 10:43:52 -0700 Subject: [PATCH 092/121] linker: copy aligned PDB module symbols directly --- src/linker/lnk_debug_info.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 3405096c7..bec35ed59 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4952,7 +4952,9 @@ lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8No if (cv_is_scope_symbol(symbol.kind)) { scope_depth += 1; } else if (cv_is_end_symbol(symbol.kind)) { scope_depth -= 1; } - U64 symbol_size = cv_write_symbol_buf(buf, buf_pos, &symbol, PDB_SYMBOL_ALIGN); + String8 raw_symbol = cv_raw_from_symbol(cv_ptr_from_symbol(symbol)); + U64 symbol_size = (raw_symbol.size & (PDB_SYMBOL_ALIGN - 1)) == 0 ? str8_buffer_write(buf, buf_pos, raw_symbol) + : cv_write_symbol_buf(buf, buf_pos, &symbol, PDB_SYMBOL_ALIGN); mod_cursor += symbol_size; mod->sym_data_size += symbol_size; } From c104782d420a7fcd933a692811aa0611558aa1be Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 14 Aug 2026 12:56:36 -0700 Subject: [PATCH 093/121] linker: rescan symbols when resolving libraries --- src/linker/lnk.c | 40 ++++------------------------------------ src/linker/lnk.h | 1 - src/linker/lnk_lib.h | 4 ---- 3 files changed, 4 insertions(+), 41 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 0725b1994..7823a8b66 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -1545,15 +1545,8 @@ THREAD_POOL_TASK_FUNC(lnk_search_lib_task) LNK_LibMemberInfo *lib_member_infos = task->lib_member_infos; LNK_LibMemberRefList *member_ref_list = &task->member_ref_lists[task_id]; - LNK_SymbolHashTrieChunk *start_chunk = task->reset_search_cursor ? 0 : lib->search_cursor_chunks[task_id]; - U64 start_idx = task->reset_search_cursor ? 0 : lib->search_cursor_indices[task_id]; - LNK_SymbolHashTrieChunk *end_chunk = symtab->search_chunks[task_id].last; - U64 end_count = end_chunk ? end_chunk->count : 0; - - for EachNode(c, LNK_SymbolHashTrieChunk, start_chunk ? start_chunk : symtab->search_chunks[task_id].first) { - U64 i_begin = (c == start_chunk) ? start_idx : 0; - U64 i_end = (c == end_chunk) ? end_count : c->count; - for (U64 i = i_begin; i < i_end; i += 1) { + for EachNode(c, LNK_SymbolHashTrieChunk, symtab->search_chunks[task_id].first) { + for EachIndex(i, c->count) { LNK_Symbol *symbol = c->v[i].symbol; LNK_SymbolSearchType search_type = lnk_search_type_from_symbol(symbol); @@ -1578,27 +1571,15 @@ THREAD_POOL_TASK_FUNC(lnk_search_lib_task) } } } - - // cache search cursors - lib->search_cursor_chunks[task_id] = end_chunk; - lib->search_cursor_indices[task_id] = end_count; } internal U64 lnk_search_lib_task_work_count(LNK_SearchLibTask *task, U64 task_id) { - LNK_Lib *lib = task->lib; LNK_SymbolTable *symtab = task->symtab; - LNK_SymbolHashTrieChunk *start_chunk = task->reset_search_cursor ? 0 : lib->search_cursor_chunks[task_id]; - U64 start_idx = task->reset_search_cursor ? 0 : lib->search_cursor_indices[task_id]; - LNK_SymbolHashTrieChunk *end_chunk = symtab->search_chunks[task_id].last; - U64 end_count = end_chunk ? end_chunk->count : 0; - U64 work_count = 0; - for EachNode(c, LNK_SymbolHashTrieChunk, start_chunk ? start_chunk : symtab->search_chunks[task_id].first) { - U64 i_begin = (c == start_chunk) ? start_idx : 0; - U64 i_end = (c == end_chunk) ? end_count : c->count; - work_count += i_end - i_begin; + for EachNode(c, LNK_SymbolHashTrieChunk, symtab->search_chunks[task_id].first) { + work_count += c->count; } return work_count; } @@ -1749,18 +1730,8 @@ lnk_link_inputs(TP_Context *tp, } else { // search symbols in lib MemoryZeroTyped(member_ref_lists, tp->worker_count); - // anti-dep mode changes which weak symbols can resolve from this lib - B32 reset_search_cursor = lib->search_cursor_chunks != 0 && lib->searched_anti_deps != search_anti_deps; - - // lazy alloc cursors for tracking searched symbols - if (lib->search_cursor_chunks == 0) { - lib->search_cursor_chunks = push_array(link->arena, LNK_SymbolHashTrieChunk *, tp->worker_count); - lib->search_cursor_indices = push_array(link->arena, U64, tp->worker_count); - } - LNK_SearchLibTask search_task = { .search_anti_deps = search_anti_deps, - .reset_search_cursor = reset_search_cursor, .link = link, .imports_hm = &imports_hm, .lib = lib, @@ -1785,9 +1756,6 @@ lnk_link_inputs(TP_Context *tp, } else { tp_for_parallel(tp, arena, tp->worker_count, lnk_search_lib_task, &search_task); } - - // cache last search mode, if the mode changes then skipped weak anti-dependency must be searched again - lib->searched_anti_deps = search_anti_deps; } LNK_LibMemberRefList queued_members = {0}; diff --git a/src/linker/lnk.h b/src/linker/lnk.h index 9728e5339..64b1f30a3 100644 --- a/src/linker/lnk.h +++ b/src/linker/lnk.h @@ -206,7 +206,6 @@ typedef struct LNK_BaseRelocPageArray typedef struct { B32 search_anti_deps; - B32 reset_search_cursor; LNK_Link *link; HashMap *imports_hm; LNK_SymbolTable *symtab; diff --git a/src/linker/lnk_lib.h b/src/linker/lnk_lib.h index bce3b9290..589fb898a 100644 --- a/src/linker/lnk_lib.h +++ b/src/linker/lnk_lib.h @@ -23,10 +23,6 @@ typedef struct LNK_Lib U64 *symbol_discs; String8 long_names; U64 input_idx; - - struct LNK_SymbolHashTrieChunk **search_cursor_chunks; - U64 *search_cursor_indices; - B32 searched_anti_deps; } LNK_Lib; typedef struct LNK_LibNode From 88b2c45472e35c1b17e00fbd279743f4af51417f Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 17 Aug 2026 11:15:38 -0700 Subject: [PATCH 094/121] linker: test library search symbol-state rescan --- .../lib_search_rescans_changed_symbol.tst | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/linker/tests/lib_search_rescans_changed_symbol.tst diff --git a/src/linker/tests/lib_search_rescans_changed_symbol.tst b/src/linker/tests/lib_search_rescans_changed_symbol.tst new file mode 100644 index 000000000..de3c6b798 --- /dev/null +++ b/src/linker/tests/lib_search_rescans_changed_symbol.tst @@ -0,0 +1,71 @@ +test: +{ + // provider.lib is searched before target exists. seed.lib introduces a + // no-library weak target; promote.lib then replaces it in-place with an + // earlier search-library weak. No search-chunk entry is appended, so the + // existing entry must be revisited when provider.lib is searched again. + artifacts: + { + provider_lib: + { + file_name: "provider.lib" + coff: { library: { second_linker_member: true, members: { provider: { + path: "provider.obj" + object: { machine: x64, + sections: { target: { + name: ".target", permissions: (read), content: initialized_data, + alignment: 1, data: { text: "provider" } + } } + symbols: { target: { kind: external, name: "target", section: target, value: 0 } } + } + } } } } + } + promote_lib: + { + file_name: "promote.lib" + coff: { library: { second_linker_member: true, members: { promote: { + path: "promote.obj" + object: { machine: x64, symbols: { + load_promote: { kind: absolute, name: "load_promote", value: 1, storage: external } + fallback: { kind: absolute, name: "promote_fallback", value: 2, storage: external } + target: { kind: weak, name: "target", fallback: fallback, search: search_library } + } } + } } } } + } + seed_lib: + { + file_name: "seed.lib" + coff: { library: { second_linker_member: true, members: { seed: { + path: "seed.obj" + object: { machine: x64, symbols: { + load_seed: { kind: absolute, name: "load_seed", value: 1, storage: external } + load_promote: { kind: undefined, name: "load_promote" } + fallback: { kind: absolute, name: "seed_fallback", value: 3, storage: external } + target: { kind: weak, name: "target", fallback: fallback, search: no_library } + } } + } } } } + } + entry_obj: + { + file_name: "entry.obj" + coff: { object: { machine: x64, + sections: { text: { + name: ".text", permissions: (read, execute), content: code, + alignment: 1, data: { hex: "c3" } + } } + symbols: { + entry: { kind: external_function, name: "entry", section: text, value: 0 } + load_seed: { kind: undefined, name: "load_seed" } + } + } } + } + image: { file_name: "a.exe", pe: {} } + } + build: { link: { + args: "/nodefaultlib /subsystem:console /entry:entry /out:a.exe provider.lib promote.lib seed.lib entry.obj" + artifact: image + } } + steps: { expect_pe: { artifact: image, expected: { pe: { sections: { + ".target": { data: 70726f7669646572 } + } } } } } +} From afe68dac879a96724f8f8be31104ca7d00628134 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 19 Aug 2026 09:08:39 -0700 Subject: [PATCH 095/121] linker: adapt PR options to typed command-line API --- src/linker/codeview_ext/ifc.c | 7 ++++++- src/linker/lnk_config.c | 10 ++++++---- src/linker/lnk_debug_info.c | 5 +++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/linker/codeview_ext/ifc.c b/src/linker/codeview_ext/ifc.c index e5693740f..b18cc4b5b 100644 --- a/src/linker/codeview_ext/ifc.c +++ b/src/linker/codeview_ext/ifc.c @@ -10,8 +10,13 @@ ifc_file_read(Arena *arena, String8 path, String8 *error_out) IFC_File ifc = {0}; ifc.path = push_str8_copy(arena, path); - String8 data = lnk_read_data_from_file_path(arena, 0, path); + B8 was_read = 0; + String8 data = lnk_read_data_from_file_path(arena, 0, path, &was_read); ifc.data = data; + if ( ! was_read) { + *error_out = push_str8f(arena, "unable to read IFC '%S'", path); + return ifc; + } if (data.size < 4) { *error_out = push_str8f(arena, "IFC '%S' is too small (%llu bytes)", path, data.size); return ifc; diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index acf7959e5..fc099b05b 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -2332,7 +2332,7 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val case LNK_CmdSwitch_Rad_DebugWorkers: { U64 cap; - if (lnk_cmd_switch_parse_u64(obj, cmd_switch, value_strings, &cap, 0)) { + if (lnk_cmd_switch_parse_u64(obj, cmd_switch, value, &cap, 0)) { config->debug_worker_cap = cap; } } break; @@ -2367,12 +2367,14 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val } break; case LNK_CmdSwitch_IfcMap: { // collect .toml paths (header-unit -> .ifc); parsed lazily during debug-info build - String8List copy = str8_list_copy(config->arena, &value_strings); - str8_list_concat_in_place(&config->ifc_map_list, ©); + String8 path = {0}; + if (lnk_cmd_switch_parse_string(obj, cmd_switch, value, &path)) { + str8_list_push(config->arena, &config->ifc_map_list, push_str8_copy(config->arena, path)); + } } break; case LNK_CmdSwitch_IfcDebugRecords: { LNK_SwitchState state = LNK_SwitchState_Null; - if (lnk_cmd_switch_parse_flag(obj, cmd_switch, value_strings, &state)) { + if (lnk_cmd_switch_parse_flag(obj, cmd_switch, value, &state)) { config->ifc_debug_records = state; } } break; diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index bec35ed59..b59b0bf14 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -1154,8 +1154,9 @@ lnk_build_ifc_map(Arena *arena, LNK_Config *config) HashMap hm = {0}; Temp scratch = scratch_begin(&arena, 1); for EachNode(n, String8Node, config->ifc_map_list.first) { - String8 toml = lnk_read_data_from_file_path(scratch.arena, 0, n->string); - if (toml.size == 0) { + B8 was_read = 0; + String8 toml = lnk_read_data_from_file_path(scratch.arena, 0, n->string, &was_read); + if ( ! was_read || toml.size == 0) { lnk_error(LNK_Error_Cmdl, "/ifcMap: unable to read TOML '%S'", n->string); continue; } From 92cb76ddf764074d970f8eccc56126de8ab7ffec Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 19 Aug 2026 09:18:52 -0700 Subject: [PATCH 096/121] linker: restore the active radix buffer after skipped passes Identity radix passes do not swap source and destination. When an odd number of non-identity passes runs, the final sorted values therefore remain in task.src. Copying task.dst back into the caller array restored stale partial ordering, which could leave type records unmaterialized and later crash PDB construction.\n\nCopy from the active source buffer and enable the existing object/leaf ordering check in debug builds. --- src/linker/lnk_debug_info.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index b59b0bf14..aa2f56f7c 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -4171,6 +4171,12 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK // sort output leaves based on { location index, leaf index } to guarantee determinism ProfScope("Radix Sort") { u64_array_sort_radix_parallel(tp, task.unique_leaf_refs_arr[ti_source].count, task.unique_leaf_refs_arr[ti_source].v); +#if BUILD_DEBUG + LNK_LeafRefArray arr = task.unique_leaf_refs_arr[ti_source]; + for (U64 i = 1; i < arr.count; ++i) { + AssertAlways(lnk_leaf_ref_compare(arr.v[i-1], arr.v[i]) <= 0); + } +#endif } } From 4353099da1ae5b08ca4ad36f8bd9f58ce4385f67 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 2 Sep 2026 23:05:58 -0700 Subject: [PATCH 097/121] thread_pool: fail fast on stranded grants without blocking diagnostics --- src/linker/thread_pool/thread_pool.c | 9 +++++++++ src/win32/base/win32_base.c | 16 +++++++--------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/linker/thread_pool/thread_pool.c b/src/linker/thread_pool/thread_pool.c index 151edc5d3..7ecb38895 100644 --- a/src/linker/thread_pool/thread_pool.c +++ b/src/linker/thread_pool/thread_pool.c @@ -64,6 +64,8 @@ tp_worker_main(void *raw_worker) //~ SHARED-mode governor stats (summary line). Zero cost when the pool is off: // every call site below is on a shared-mode-only path. +#define TP_STALL_ABORT_US 30000000ull + global TP_SharedStats g_tp_shared_stats; internal void @@ -576,8 +578,15 @@ tp_for_parallel(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskF // worker will consume, drain immediately (task_left<0), and account for. So // `granted` monotonically drains to 0 once the governor has stopped; spin // until it does. This is brief (workers see task_left<0 and exit at once). + U64 stall_begin_us = now_time_us(); for (; ins_atomic_u64_eval((U64 *)&pool->granted) != 0; ) { sleep_ms(0); + U64 elapsed_us = now_time_us() - stall_begin_us; + if (elapsed_us >= TP_STALL_ABORT_US) { + // Never perform stderr I/O here. Under UBA that enters the WriteFile detour and can block, + // turning the diagnostic path itself into a permanent hang before the fail-fast executes. + AssertAlways(0); + } } } diff --git a/src/win32/base/win32_base.c b/src/win32/base/win32_base.c index e68b51bb0..09d59e010 100644 --- a/src/win32/base/win32_base.c +++ b/src/win32/base/win32_base.c @@ -695,7 +695,10 @@ semaphore_take(Semaphore semaphore, U64 endt_us) internal void semaphore_drop_count(Semaphore semaphore, U64 drop_count) { - ReleaseSemaphore((HANDLE)*semaphore.u64, drop_count, 0); + BOOL ok = ReleaseSemaphore((HANDLE)*semaphore.u64, drop_count, 0); + // Do not log here: stderr may itself be a detoured pipe, and blocking before the assertion + // would convert a synchronization failure into an undiagnosable process hang. + AssertAlways(ok); } internal B32 @@ -729,14 +732,9 @@ semaphore_drop_n(Semaphore semaphore, U32 count) HANDLE handle = (HANDLE)semaphore.u64[0]; BOOL ok = ReleaseSemaphore(handle, count, 0); if (!ok) { - // The non-shared thread pool intentionally batches a wake of up to - // worker_count permits onto a semaphore that may still hold un-retaken - // permits from a prior pass; the surplus clamps at the max and the OS - // returns ERROR_TOO_MANY_POSTS. That is a benign over-wake (workers are - // already runnable), so tolerate it -- but ONLY it. Any other failure - // (e.g. a bad handle) is a real bug and must not be swallowed. - DWORD err = GetLastError(); - AssertAlways(err == ERROR_TOO_MANY_POSTS); + // ReleaseSemaphore is all-or-nothing: ERROR_TOO_MANY_POSTS does not clamp + // the count. Ignoring any failure can therefore strand a thread-pool grant. + AssertAlways(ok); } } } From 9f9a097d405e0e6c0faa344563d5c3efac7485b7 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 20 Aug 2026 15:49:20 -0700 Subject: [PATCH 098/121] linker: report library state when entry inference fails On entry-point inference errors, print object and command-line default libraries plus the libraries successfully loaded. This distinguishes a lost .drectve/defaultlib from a failed archive open or symbol-index lookup in remote Wine builds. --- src/linker/lnk.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 7823a8b66..0563a68f8 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -2238,6 +2238,15 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer for EachIndex(i, entry_points.count) { str8_list_push(scratch.arena, &list, entry_points.v[i]); } String8 default_entries = str8_list_join(scratch.arena, &list, &(StringJoin){.sep = str8_lit(", ")}); + StringJoin comma_join = {.sep = str8_lit(", ")}; + String8 obj_default_libs = str8_list_join(scratch.arena, &config->input_obj_lib_list, &comma_join); + String8 cmd_default_libs = str8_list_join(scratch.arena, &config->input_default_lib_list, &comma_join); + String8List loaded_lib_list = {0}; + for EachNode(lib_n, LNK_LibNode, link->libs.first) { + str8_list_push(scratch.arena, &loaded_lib_list, str8_skip_last_slash(lib_n->data.path)); + } + String8 loaded_libs = str8_list_join(scratch.arena, &loaded_lib_list, &comma_join); + lnk_error(LNK_Error_EntryPoint, "failed to infer entry point symbol from the inputs\n" " Machine: %S\n" @@ -2248,7 +2257,10 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer " Default Entries: %S\n" " User Entry: \"%S\"\n" " Input Obj Count: %S\n" - " Input Lib Count: %S", + " Input Lib Count: %S\n" + " Obj Default Libs: %S\n" + " Cmd Default Libs: %S\n" + " Loaded Libs: %S", machine_str.size ? machine_str : str8_lit("Unknown"), // Machine subsystem_str.size ? subsystem_str : str8_lit("Unknown"), // Version config->subsystem_ver.major, config->subsystem_ver.minor, // Subsystem @@ -2257,7 +2269,10 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer default_entries.size ? default_entries : str8_lit("None"), // Default Entry Points config->entry_point_name, // /ENTRY str8_from_count(scratch.arena, config->input_list[LNK_Input_Obj].node_count), // Input Objects Count - str8_from_count(scratch.arena, config->input_list[LNK_Input_Lib].node_count) // Input Libs Count + str8_from_count(scratch.arena, config->input_list[LNK_Input_Lib].node_count), // Input Libs Count + obj_default_libs.size ? obj_default_libs : str8_lit("None"), + cmd_default_libs.size ? cmd_default_libs : str8_lit("None"), + loaded_libs.size ? loaded_libs : str8_lit("None") ); } From c7f33b3e19d4f22d9361b9f1b4d63cab6cb48082 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 20 Aug 2026 15:49:21 -0700 Subject: [PATCH 099/121] linker: fix invalid-library result indexing Parallel library parsing stored invalid results one slot past the array and reported later-batch paths with a global index into a batch-local input array. Correct both indices so a malformed or transiently unreadable archive produces a deterministic error instead of memory corruption. --- src/linker/lnk_lib.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/linker/lnk_lib.c b/src/linker/lnk_lib.c index 506cc4468..76173acfe 100644 --- a/src/linker/lnk_lib.c +++ b/src/linker/lnk_lib.c @@ -185,7 +185,7 @@ THREAD_POOL_TASK_FUNC(lnk_lib_initer) U64 valid_lib_idx = ins_atomic_u64_inc_eval(&task->valid_libs_count)-1; task->valid_libs[valid_lib_idx] = lib_node; } else { - U64 invalid_lib_idx = ins_atomic_u64_inc_eval(&task->invalid_libs_count); + U64 invalid_lib_idx = ins_atomic_u64_inc_eval(&task->invalid_libs_count)-1; task->invalid_libs[invalid_lib_idx] = lib_node; } } @@ -228,7 +228,7 @@ lnk_lib_list_push_parallel(TP_Context *tp, TP_Arena *arena, LNK_LibList *list, U radsort(task.invalid_libs, task.invalid_libs_count, lnk_lib_node_ptr_is_before); for EachIndex(i, task.invalid_libs_count) { U64 input_idx = task.invalid_libs[i]->data.input_idx; - lnk_error(LNK_Error_InvalidLib, "%S: failed to parse library", inputs[input_idx]->path); + lnk_error(LNK_Error_InvalidLib, "%S: failed to parse library", inputs[input_idx - lib_id_base]->path); } // push parsed libs From 345877804944dd76a0823b399f5243689c8f1abe Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 2 Sep 2026 13:33:34 -0700 Subject: [PATCH 100/121] linker: restore missing DLL-import dependency diagnostics Recover the #892 diagnostic omitted from the earlier clean-stack refresh and adapt it to upstream's unresolved-symbol reporting. Extend link_undef.tst with a failing missing-import case and stderr expectations. --- src/linker/lnk.c | 4 ++++ src/linker/tests/link_undef.tst | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 0563a68f8..51fbde4e8 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -2435,6 +2435,10 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer next_undefined_symbol:; lnk_error(LNK_Error_UnresolvedSymbol, "unresolved symbol '%S'", symbol->name); + if (str8_match(str8_prefix(symbol->name, 6), str8_lit("__imp_"), 0)) { + lnk_supplement_error("this is a DLL import, but no linked object or import library defines it"); + lnk_supplement_error("verify that the response file includes the import library for this module dependency"); + } lnk_supplement_error_list(ref_messages); } diff --git a/src/linker/tests/link_undef.tst b/src/linker/tests/link_undef.tst index 7363a9781..adbfb9597 100644 --- a/src/linker/tests/link_undef.tst +++ b/src/linker/tests/link_undef.tst @@ -3,6 +3,7 @@ test: artifacts: { undef_obj: { file_name: "undef.obj", coff: { object: { machine: x64, symbols: { undef: { kind: undefined, name: "undef" } } } } } + import_obj: { file_name: "import.obj", coff: { object: { machine: x64, symbols: { missing: { kind: undefined, name: "__imp_missing_dependency" } } } } } entry_obj: { file_name: "entry.obj" @@ -19,6 +20,11 @@ test: { // try linking unresolved symbol and see if linker picks up on that link: { args: "/subsystem:console /entry:entry /out:a.exe entry.obj undef.obj", expect_exit: 47 } + link: { + args: "/subsystem:console /entry:entry /out:missing_import.exe entry.obj import.obj" + expect_exit: 47 + stderr_matches: "*unresolved symbol '__imp_missing_dependency'*this is a DLL import*response file includes the import library for this module dependency*" + } } steps: {} } From f555a9dd0180c1cddda80c1ac7d781f0ed5aaa19 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 2 Sep 2026 14:17:13 -0700 Subject: [PATCH 101/121] coff: emit BigObj for oversized generated import objects --- src/coff/coff_obj_writer.c | 95 ++++++++++++++++-------- src/torture/tests/torture_coff_tests.c | 55 ++++++++++++++ src/torture/tests/torture_linker_tests.c | 41 ++++++++++ 3 files changed, 158 insertions(+), 33 deletions(-) diff --git a/src/coff/coff_obj_writer.c b/src/coff/coff_obj_writer.c index 01b9cbfbd..bceaf9646 100644 --- a/src/coff/coff_obj_writer.c +++ b/src/coff/coff_obj_writer.c @@ -32,12 +32,13 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) str8_list_push(scratch.arena, &string_table, str8_struct(string_table_size)); // - // assing section numbers + // assign section numbers // U64 obj_sections_count; COFF_ObjSection **obj_sections; { obj_sections_count = obj_writer->sect_count; + AssertAlways(obj_sections_count < COFF_Symbol_DebugSection32); obj_sections = push_array(scratch.arena, COFF_ObjSection *, obj_writer->sect_count); U64 sect_idx = 0; for (COFF_ObjSectionNode *sect_n = obj_writer->sect_first; sect_n != 0; sect_n = sect_n->next, sect_idx += 1) { @@ -47,7 +48,10 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) } } - AssertAlways(obj_sections_count <= max_U16); + // Standard COFF reserves section numbers 0xff00 and above. BigObj also + // widens every symbol/auxiliary record from 18 to 20 bytes. + B32 is_big_obj = obj_sections_count > 0xfeff; + U64 symbol_size = is_big_obj ? sizeof(COFF_Symbol32) : sizeof(COFF_Symbol16); // // serialize symbol table @@ -69,9 +73,6 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) for (COFF_ObjSymbolNode *symbol_n = obj_writer->symbol_first; symbol_n != 0; symbol_n = symbol_n->next) { COFF_ObjSymbol *s = &symbol_n->v; - COFF_Symbol16 *d = push_array(scratch.arena, COFF_Symbol16, 1); - str8_list_push(scratch.arena, &symbol_table, str8_struct(d)); - COFF_SymbolName name = {0}; // long name if (s->name.size > sizeof(name.short_name)) { @@ -90,28 +91,43 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) // symbol header AssertAlways(s->aux_symbols.node_count <= max_U8); - d->name = name; - d->value = s->value; + U32 section_number = 0; switch (s->loc.type) { case COFF_SymbolLocation_Null: break; - case COFF_SymbolLocation_Section: d->section_number = safe_cast_u16(s->loc.u.section->section_number); break; - case COFF_SymbolLocation_Abs: d->section_number = COFF_Symbol_AbsSection16; break; - case COFF_SymbolLocation_Undef: d->section_number = COFF_Symbol_UndefinedSection; break; - case COFF_SymbolLocation_Common: d->section_number = COFF_Symbol_UndefinedSection; break; + case COFF_SymbolLocation_Section: section_number = s->loc.u.section->section_number; break; + case COFF_SymbolLocation_Abs: section_number = is_big_obj ? COFF_Symbol_AbsSection32 : COFF_Symbol_AbsSection16; break; + case COFF_SymbolLocation_Undef: section_number = COFF_Symbol_UndefinedSection; break; + case COFF_SymbolLocation_Common: section_number = COFF_Symbol_UndefinedSection; break; + } + if (is_big_obj) { + COFF_Symbol32 *d = push_array(scratch.arena, COFF_Symbol32, 1); + d->name = name; + d->value = s->value; + d->section_number = section_number; + d->type = s->type; + d->storage_class = s->storage_class; + d->aux_symbol_count = (U8)s->aux_symbols.node_count; + str8_list_push(scratch.arena, &symbol_table, str8_struct(d)); + } else { + COFF_Symbol16 *d = push_array(scratch.arena, COFF_Symbol16, 1); + d->name = name; + d->value = s->value; + d->section_number = safe_cast_u16(section_number); + d->type = s->type; + d->storage_class = s->storage_class; + d->aux_symbol_count = (U8)s->aux_symbols.node_count; + str8_list_push(scratch.arena, &symbol_table, str8_struct(d)); } - d->type = s->type; - d->storage_class = s->storage_class; - d->aux_symbol_count = 0; U64 start_symbol_idx = symbol_idx; if (s->storage_class == COFF_SymStorageClass_WeakExternal) { if (s->aux_symbols.node_count > 0) { COFF_ObjSymbolWeak *s_weak = (COFF_ObjSymbolWeak *)s->aux_symbols.first->string.str; - COFF_SymbolWeakExt *d_weak = push_array(scratch.arena, COFF_SymbolWeakExt, 1); + COFF_SymbolWeakExt *d_weak = (COFF_SymbolWeakExt *)push_array(scratch.arena, U8, symbol_size); d_weak->tag_index = s_weak->tag ? s_weak->tag->idx : max_U32; d_weak->characteristics = s_weak->characteristics; - str8_list_push(scratch.arena, &symbol_table, str8_struct(d_weak)); + str8_list_push(scratch.arena, &symbol_table, str8((U8 *)d_weak, symbol_size)); symbol_idx += 1; } } else if (s->storage_class == COFF_SymStorageClass_Static) { @@ -120,15 +136,17 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) COFF_ObjSection *sect = s->loc.u.section; COFF_ObjSymbolSecDef *s_sd = (COFF_ObjSymbolSecDef *)s->aux_symbols.first->string.str; - COFF_SymbolSecDef *d_sd = push_array(scratch.arena, COFF_SymbolSecDef, 1); + COFF_SymbolSecDef *d_sd = (COFF_SymbolSecDef *)push_array(scratch.arena, U8, symbol_size); d_sd->length = safe_cast_u32(sect->data.total_size); d_sd->number_of_relocations = (U16)sect->reloc_count; d_sd->check_sum = 0; - d_sd->number_lo = s_sd->selection == COFF_ComdatSelect_Associative ? safe_cast_u16(s_sd->associate->section_number) : 0; + U32 associate_number = s_sd->selection == COFF_ComdatSelect_Associative ? s_sd->associate->section_number : 0; + d_sd->number_lo = (U16)associate_number; + d_sd->number_hi = (U16)(associate_number >> 16); d_sd->selection = s_sd->selection; - str8_list_push(scratch.arena, &symbol_table, str8_struct(d_sd)); + str8_list_push(scratch.arena, &symbol_table, str8((U8 *)d_sd, symbol_size)); symbol_idx += 1; } } @@ -136,26 +154,37 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) U8 processed_aux_symbol_count = (U8)(symbol_idx - start_symbol_idx); for (U64 aux_idx = processed_aux_symbol_count; aux_idx < s->aux_symbols.node_count; aux_idx += 1) { - COFF_Symbol16 *a = push_array(scratch.arena, COFF_Symbol16, 1); - str8_list_push(scratch.arena, &symbol_table, str8_struct(a)); + U8 *a = push_array(scratch.arena, U8, symbol_size); + str8_list_push(scratch.arena, &symbol_table, str8(a, symbol_size)); } - - d->aux_symbol_count = (U8)s->aux_symbols.node_count; } } // // file header // - COFF_FileHeader *file_header = push_array(scratch.arena, COFF_FileHeader, 1); - file_header->machine = obj_writer->machine; - file_header->section_count = obj_sections_count; - file_header->time_stamp = obj_writer->time_stamp; - file_header->symbol_table_foff = 0; - file_header->symbol_count = safe_cast_u32(symbol_table.node_count); - file_header->optional_header_size = 0; - file_header->flags = 0; - str8_list_push(scratch.arena, &srl, str8_struct(file_header)); + U32 *symbol_table_foff; + if (is_big_obj) { + COFF_BigObjHeader *header = push_array(scratch.arena, COFF_BigObjHeader, 1); + header->sig1 = COFF_MachineType_Unknown; + header->sig2 = max_U16; + header->version = 2; + header->machine = obj_writer->machine; + header->time_stamp = obj_writer->time_stamp; + MemoryCopy(header->magic, g_coff_big_header_magic, sizeof(header->magic)); + header->section_count = safe_cast_u32(obj_sections_count); + header->symbol_count = safe_cast_u32(symbol_table.node_count); + symbol_table_foff = &header->symbol_table_foff; + str8_list_push(scratch.arena, &srl, str8_struct(header)); + } else { + COFF_FileHeader *header = push_array(scratch.arena, COFF_FileHeader, 1); + header->machine = obj_writer->machine; + header->section_count = safe_cast_u16(obj_sections_count); + header->time_stamp = obj_writer->time_stamp; + header->symbol_count = safe_cast_u32(symbol_table.node_count); + symbol_table_foff = &header->symbol_table_foff; + str8_list_push(scratch.arena, &srl, str8_struct(header)); + } // // section table @@ -230,7 +259,7 @@ coff_obj_writer_serialize(Arena *arena, COFF_ObjWriter *obj_writer) // symbol table // if (symbol_table.total_size || string_table.total_size > sizeof(*string_table_size)) { - file_header->symbol_table_foff = srl.total_size; + *symbol_table_foff = safe_cast_u32(srl.total_size); str8_list_concat_in_place(&srl, &symbol_table); } diff --git a/src/torture/tests/torture_coff_tests.c b/src/torture/tests/torture_coff_tests.c index c40504944..8e2c2f006 100644 --- a/src/torture/tests/torture_coff_tests.c +++ b/src/torture/tests/torture_coff_tests.c @@ -13,6 +13,61 @@ t_coff_test_encode(Arena *arena, TestCtx *ctx, String8 file_name, String8 source return value != 0 ? value->data : str8_zero(); } +TEST(coff_writer_bigobj) +{ + // Exercise the standard-COFF boundary and an associative parent above 16 bits. + U32 section_counts[] = {3, 0xfeff, 0xff00, 0x10002}; + for EachElement(case_idx, section_counts) { + U32 section_count = section_counts[case_idx]; + COFF_ObjWriter *writer = coff_obj_writer_alloc(0x12345678, COFF_MachineType_X64); + for (U32 i = 0; i < section_count - 2; i += 1) { + coff_obj_writer_push_section(writer, str8_lit(".empty"), COFF_SectionFlag_LnkRemove, str8_zero()); + } + COFF_SectionFlags flags = COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_LnkCOMDAT; + COFF_ObjSection *head = coff_obj_writer_push_section(writer, str8_lit(".long_head_section"), flags, str8_lit("head")); + COFF_ObjSection *assoc = coff_obj_writer_push_section(writer, str8_lit(".assoc"), flags, str8(push_array(arena, U8, 8), 8)); + coff_obj_writer_push_symbol_secdef(writer, head, COFF_ComdatSelect_Any); + COFF_ObjSymbol *target = coff_obj_writer_push_symbol_extern(writer, str8_lit("long_target_symbol"), 0, head); + COFF_ObjSymbol *weak = coff_obj_writer_push_symbol_weak(writer, str8_lit("weak"), COFF_WeakExt_SearchAlias, target); + COFF_ObjSymbol *absolute = coff_obj_writer_push_symbol_abs(writer, str8_lit("absolute"), 17, COFF_SymStorageClass_External); + COFF_ObjSymbol *assoc_def = coff_obj_writer_push_symbol_associative(writer, assoc, head); + COFF_ObjSymbol *undef = coff_obj_writer_push_symbol_undef(writer, str8_lit("undef")); + COFF_ObjSymbol *common = coff_obj_writer_push_symbol_common(writer, str8_lit("common"), 32); + coff_obj_writer_section_push_reloc_addr(writer, assoc, 0, weak); + + String8 data = coff_obj_writer_serialize(arena, writer); + COFF_FileHeaderInfo info = coff_file_header_info_from_data(data); + T_Ok(info.is_big_obj == (section_count > 0xfeff)); + T_Ok(info.section_count_no_null == section_count); + T_Ok(info.symbol_size == (info.is_big_obj ? sizeof(COFF_Symbol32) : sizeof(COFF_Symbol16))); + T_Ok(info.symbol_count == 10); + String8 symbols = str8_substr(data, info.symbol_table_range); + String8 strings = str8_substr(data, info.string_table_range); + COFF_ParsedSymbol parsed_target = coff_parse_symbol(info, strings, symbols, target->idx); + T_Ok(parsed_target.section_number == section_count - 1); + T_Ok(str8_match(parsed_target.name, target->name, 0)); + COFF_ParsedSymbol parsed_assoc = coff_parse_symbol(info, strings, symbols, assoc_def->idx); + U32 parent = 0; + COFF_ComdatSelectType selection = 0; + coff_parse_secdef(parsed_assoc, info.is_big_obj, &selection, &parent, 0, 0); + T_Ok(parsed_assoc.section_number == section_count); + T_Ok(selection == COFF_ComdatSelect_Associative && parent == section_count - 1); + COFF_ParsedSymbol parsed_weak = coff_parse_symbol(info, strings, symbols, weak->idx); + COFF_SymbolWeakExt *weak_aux = coff_parse_weak_tag(parsed_weak, info.is_big_obj); + T_Ok(weak_aux->tag_index == target->idx && weak_aux->characteristics == COFF_WeakExt_SearchAlias); + COFF_ParsedSymbol parsed_absolute = coff_parse_symbol(info, strings, symbols, absolute->idx); + T_Ok(parsed_absolute.section_number == COFF_Symbol_AbsSection32 && parsed_absolute.value == 17); + T_Ok(coff_parse_symbol(info, strings, symbols, undef->idx).section_number == COFF_Symbol_UndefinedSection); + COFF_ParsedSymbol parsed_common = coff_parse_symbol(info, strings, symbols, common->idx); + T_Ok(parsed_common.section_number == COFF_Symbol_UndefinedSection && parsed_common.value == 32); + COFF_SectionHeader *sections = (COFF_SectionHeader *)(data.str + info.section_table_range.min); + COFF_Reloc *reloc = (COFF_Reloc *)(data.str + sections[section_count - 1].relocs_foff); + T_Ok(sections[section_count - 1].reloc_count == 1 && reloc->isymbol == weak->idx); + T_Ok(t_write_file(str8f(arena, "writer_%u.obj", section_count), data)); + coff_obj_writer_release(&writer); + } +} + TEST(coff_codec_object_parity) { String8 source = str8_lit( diff --git a/src/torture/tests/torture_linker_tests.c b/src/torture/tests/torture_linker_tests.c index 30578c1b5..bedd0fb94 100644 --- a/src/torture/tests/torture_linker_tests.c +++ b/src/torture/tests/torture_linker_tests.c @@ -224,6 +224,47 @@ TEST(expect_pdb_indexes_linker_output) T_Ok(t_result_is_ok(result)); } +TEST(link_large_import_object) +{ + // Each named import creates three sections in the synthesized DLL object. + // Add a direct function reference as well to exercise its jump thunk. + U32 import_count = 22000; + COFF_LibWriter *lib = coff_lib_writer_alloc(); + COFF_ObjWriter *obj = coff_obj_writer_alloc(0, COFF_MachineType_X64); + COFF_ObjSection *refs = coff_obj_writer_push_section(obj, str8_lit(".data"), + COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemWrite|COFF_SectionFlag_Align8Bytes, + str8(push_array(arena, U8, (import_count + 1)*8), (import_count + 1)*8)); + for (U32 i = 0; i < import_count; i += 1) { + String8 name = str8f(arena, "import_%05u", i); + coff_lib_writer_push_import(lib, COFF_MachineType_X64, 0, str8_lit("large.dll"), COFF_ImportBy_Name, name, 0, COFF_ImportHeader_Code); + COFF_ObjSymbol *symbol = coff_obj_writer_push_symbol_undef(obj, str8f(arena, "__imp_%S", name)); + coff_obj_writer_section_push_reloc_addr(obj, refs, i*8, symbol); + } + COFF_ObjSymbol *func = coff_obj_writer_push_symbol_undef_func(obj, str8f(arena, "import_%05u", import_count - 1)); + coff_obj_writer_section_push_reloc_addr(obj, refs, import_count*8, func); + T_Ok(t_write_file(str8_lit("refs.obj"), coff_obj_writer_serialize(arena, obj))); + T_Ok(t_write_file(str8_lit("large.lib"), coff_lib_writer_serialize(arena, lib, 0, 0, 1))); + coff_obj_writer_release(&obj); + coff_lib_writer_release(&lib); + T_Ok(t_write_entry_obj()); + t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /opt:ref /out:large.exe entry.obj refs.obj large.lib"); + T_Ok(g_last_exit_code == 0); + String8 image = t_read_file(arena, str8_lit("large.exe")); + PE_BinInfo bin = pe_bin_info_from_data(arena, image); + T_Ok(bin.data_dir_count > PE_DataDirectoryIndex_IMPORT); + COFF_SectionHeader *sections = (COFF_SectionHeader *)(image.str + bin.section_table_range.min); + PE_ParsedStaticImportTable imports = pe_static_imports_from_data(arena, bin.is_pe32, bin.section_count, sections, + image, bin.data_dir_franges[PE_DataDirectoryIndex_IMPORT]); + T_Ok(imports.count == 1); + T_Ok(str8_match(imports.v[0].name, str8_lit("large.dll"), 0)); + T_Ok(imports.v[0].import_count == import_count); + for (U32 i = 0; i < import_count; i += 1) { + PE_ParsedImport *import = &imports.v[0].imports[i]; + T_Ok(import->type == PE_ParsedImport_Name); + T_Ok(str8_match(import->u.name.string, str8f(arena, "import_%05u", i), 0)); + } +} + TEST(pdbstripped_coff_artifact_parity) { String8 path = str8f(arena, "%S/linker/tests/pdbstripped.tst", t_src_path()); From 2aa63c012fc31f05c6dc694c4a32ce6646ca731b Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 2 Sep 2026 15:10:45 -0700 Subject: [PATCH 102/121] lnk: index function override aliases during directive parsing --- src/linker/lnk.c | 8 +++--- src/linker/lnk_config.c | 3 +++ src/linker/lnk_config.h | 1 + src/linker/tests/alt_name.tst | 50 +++++++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 51fbde4e8..6fcd75606 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -1322,11 +1322,11 @@ lnk_load_inputs(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer { B32 has_function_overrides = 0; - for EachNode(alt_name_n, LNK_AltNameNode, config->alt_name_list.first) { + // Ordinary alternate names can number in the millions. Classify overrides + // once when parsing directives, but revisit the override subset here: a + // later archive member may introduce a reference to an earlier directive. + for EachNode(alt_name_n, LNK_AltNameNode, config->function_override_list.first) { LNK_AltName alt_name = alt_name_n->v; - if ( ! str8_ends_with(alt_name.from, str8_lit("$fo$"), 0)) { - continue; - } LNK_Symbol *symbol = lnk_symbol_table_search(symtab, alt_name.from); if (symbol && lnk_interp_from_symbol(symbol) == COFF_SymbolValueInterp_Undefined) { diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index fc099b05b..0480309cd 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -1259,6 +1259,9 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val alt_name.to = push_str8_copy(config->arena, alt_name.to); lnk_alt_name_list_push(config->arena, &config->alt_name_list, alt_name); + if (str8_ends_with(alt_name.from, str8_lit("$fo$"), 0)) { + lnk_alt_name_list_push(config->arena, &config->function_override_list, alt_name); + } hash_map_push_string_string(config->arena, &config->alt_name_ht, alt_name.from, alt_name.to); } } diff --git a/src/linker/lnk_config.h b/src/linker/lnk_config.h index 4e082b958..6e606a3ae 100644 --- a/src/linker/lnk_config.h +++ b/src/linker/lnk_config.h @@ -399,6 +399,7 @@ typedef struct LNK_Config String8 rad_debug_alt_path; LNK_IncludeSymbolList include_symbol_list; LNK_AltNameList alt_name_list; + LNK_AltNameList function_override_list; LNK_MergeDirectiveList merge_list; LNK_SectionDirectiveList section_list; U64 data_dir_count; diff --git a/src/linker/tests/alt_name.tst b/src/linker/tests/alt_name.tst index bceb77fa7..70a51f5bb 100644 --- a/src/linker/tests/alt_name.tst +++ b/src/linker/tests/alt_name.tst @@ -2,6 +2,53 @@ test: { artifacts: { + override_entry_obj: + { + file_name: "override_entry.obj" + coff: { object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } } + symbols: { + entry: { kind: external_function, name: "entry", section: text, value: 0 } + early: { kind: undefined, name: "early$fo$" } + caller: { kind: undefined, name: "late_caller" } + ordinary: { kind: undefined, name: "foo" } + } + } } + } + override_caller_lib: + { + file_name: "override_caller.lib" + coff: { library: { second_linker_member: true, members: { caller: { + path: "late_caller.obj" + object: { machine: x64, + directives: { directive: "/alternatename:from_archive$fo$=test" } + sections: { text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } } + symbols: { + caller: { kind: external_function, name: "late_caller", section: text, value: 0 } + late: { kind: undefined, name: "late$fo$" } + from_archive: { kind: undefined, name: "from_archive$fo$" } + } + } + } } } } + } + override_trap_lib: + { + file_name: "override_trap.lib" + // These archive definitions must not be pulled in before the overrides. + // Pulling this member exposes an unresolved symbol and fails the link. + coff: { library: { second_linker_member: true, members: { trap: { + path: "override_trap.obj" + object: { machine: x64, + sections: { text: { name: ".text", permissions: (read, execute), content: code, data: { hex: "c3" } } } + symbols: { + early: { kind: external_function, name: "early$fo$", section: text, value: 0 } + late: { kind: external_function, name: "late$fo$", section: text, value: 0 } + from_archive: { kind: external_function, name: "from_archive$fo$", section: text, value: 0 } + missing: { kind: undefined, name: "override_archive_must_not_be_loaded" } + } + } + } } } } + } test_obj: { file_name: "test.obj" @@ -38,6 +85,9 @@ test: } build: { + // Cover command-line overrides, duplicate directives, references appearing + // in a later archive load, and new override directives from that archive. + link: { args: "/subsystem:console /entry:entry /out:overrides.exe /alternatename:early$fo$=test /alternatename:early$fo$=test /alternatename:late$fo$=test /alternatename:foo=test override_entry.obj test.obj override_caller.lib override_trap.lib" } // basic alternate name test link: { args: "/subsystem:console /entry:entry /out:alternate_basic.exe /alternatename:foo=test test.obj entry.obj" } // linker should not chase alt name links From a721072c7b2a5939cde4ffb2c1e2be85eb3424b5 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 2 Sep 2026 15:49:44 -0700 Subject: [PATCH 103/121] torture: match sort callback signature for Clang --- src/torture/torture.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/torture/torture.c b/src/torture/torture.c index 9a26ab0fa..d234e7cb7 100644 --- a/src/torture/torture.c +++ b/src/torture/torture.c @@ -850,8 +850,10 @@ t_invoke_linkerf(char *fmt, ...) } internal inline int -t_test_info_is_before(TestInfo **a, TestInfo **b) +t_test_info_is_before(void *raw_a, void *raw_b) { + TestInfo **a = raw_a; + TestInfo **b = raw_b; String8 layer_a = a[0]->layer; String8 layer_b = b[0]->layer; int cmp = str8_compar(layer_a, layer_b, 0); @@ -1122,7 +1124,7 @@ t_entry_point(CmdLine *cmdline) // Gather tests { for EachIndex(i, test_infos_count) { g_sorted_test_infos[i] = &test_infos[i]; } - radsort(g_sorted_test_infos, test_infos_count, (int (*)(void *, void *))t_test_info_is_before); + radsort(g_sorted_test_infos, test_infos_count, t_test_info_is_before); } // From c71da326ce13688191d29b8c6ebe0d0562b4bc6f Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 2 Sep 2026 21:43:36 -0700 Subject: [PATCH 104/121] Park shared-pool governor once pass demand is covered --- src/linker/thread_pool/thread_pool.c | 10 ++- src/torture/tests/torture_linker_tests.c | 78 ++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/src/linker/thread_pool/thread_pool.c b/src/linker/thread_pool/thread_pool.c index 7ecb38895..f355a2066 100644 --- a/src/linker/thread_pool/thread_pool.c +++ b/src/linker/thread_pool/thread_pool.c @@ -301,9 +301,13 @@ tp_governor_main(void *raw_pool) } } } else { - // No demand right now (work drained or fully covered). Briefly idle; the - // pass-end ping will release us promptly via the outer take when we loop. - sleep_ms(0); + // This pass needs no more grants. task_left only decreases, and a worker + // returns its grant only after draining the shared queue: demand cannot + // become uncovered again within this pass. Park on the outer semaphore; + // the next path-A dispatch (or shutdown) supplies a persistent wake. + // Sleep(0) here just yielded repeatedly, burning a core while long-running + // tasks finished, including the entire explicit input-unmap pass. + break; } } } diff --git a/src/torture/tests/torture_linker_tests.c b/src/torture/tests/torture_linker_tests.c index bedd0fb94..dc1b854af 100644 --- a/src/torture/tests/torture_linker_tests.c +++ b/src/torture/tests/torture_linker_tests.c @@ -1,6 +1,84 @@ // Copyright (c) Epic Games Tools // Licensed under the MIT license (https://opensource.org/license/mit/) +typedef struct T_SharedPoolTask +{ + U64 visits[64]; + U32 delay_ms; + B32 use_barrier; +} T_SharedPoolTask; + +internal THREAD_POOL_TASK_FUNC(t_shared_pool_task) +{ + T_SharedPoolTask *task = raw_task; + ins_atomic_u64_inc_eval(&task->visits[task_id]); + if (task->delay_ms) { sleep_ms(task->delay_ms); } + if (task->use_barrier) { barrier_wait(tp->barrier); } +} + +TEST(shared_pool_governor_repeated_passes) +{ +#if OS_WINDOWS + // An isolated budget must not borrow from (or disturb) running production links. + String8 name = str8f(arena, "radlink-torture-governor-%u-%llu", GetCurrentProcessId(), now_time_us()); + TP_Context *pool = tp_alloc(arena, 4, 4, name); + T_SharedPoolTask task = {0}; + tp_for_parallel(pool, 0, 64, t_shared_pool_task, &task); + + // Measure, but do not assert a timing threshold on a shared/busy test machine. + // The old governor burns a core while these already-assigned tasks sleep. + W32_Entity *governor = (W32_Entity *)PtrFromInt(pool->governor_handle.u64[0]); + FILETIME created, exited, kernel_before, user_before, kernel_after, user_after; + B32 before_ok = GetThreadTimes(governor->thread.handle, &created, &exited, &kernel_before, &user_before); + MemoryZeroStruct(&task); + task.delay_ms = 200; + tp_for_parallel(pool, 0, 3, t_shared_pool_task, &task); + B32 after_ok = GetThreadTimes(governor->thread.handle, &created, &exited, &kernel_after, &user_after); + for EachIndex(i, 3) { T_Ok(task.visits[i] == 1); } + T_Ok(before_ok && after_ok); + if (before_ok && after_ok) { + U64 cpu_before = (((U64)kernel_before.dwHighDateTime << 32) | kernel_before.dwLowDateTime) + + (((U64)user_before.dwHighDateTime << 32) | user_before.dwLowDateTime); + U64 cpu_after = (((U64)kernel_after.dwHighDateTime << 32) | kernel_after.dwLowDateTime) + + (((U64)user_after.dwHighDateTime << 32) | user_after.dwLowDateTime); + fprintf(stdout, "[shared governor] covered-pass CPU: %.3f ms\n", (F64)(cpu_after - cpu_before) / 10000.0); + } + + // Exercise wake coalescing, tiny/queued passes, and path-A/path-B transitions. + U64 counts[] = {1, 2, 3, 5, 31, 64}; + for EachIndex(pass, 96) { + MemoryZeroStruct(&task); + U64 count = counts[pass % ArrayCount(counts)]; + task.delay_ms = (pass % 12 == 0) ? 1 : 0; + tp_for_parallel(pool, 0, count, t_shared_pool_task, &task); + for EachIndex(i, count) { T_Ok(task.visits[i] == 1); } + T_Ok(ins_atomic_u64_eval(&pool->granted) == 0); + + MemoryZeroStruct(&task); + task.use_barrier = 1; + U64 cohort = tp_barrier_begin(pool); + tp_for_parallel_reserve(pool, 0, cohort, t_shared_pool_task, &task); + tp_barrier_end(pool); + for EachIndex(i, cohort) { T_Ok(task.visits[i] == 1); } + } + + // Join before releasing test-owned storage/synchronization. thread_join consumes + // each handle; clear it before the common release path detaches remaining handles. + pool->is_live = 0; + semaphore_drop_if_room(pool->governor_semaphore); + for (U64 i = 1; i < pool->worker_count; ++i) { semaphore_drop(pool->wake_semaphore); } + for (U64 i = 1; i < pool->worker_count; ++i) { + T_Ok(thread_join(pool->worker_arr[i].handle, max_U64)); + MemoryZeroStruct(&pool->worker_arr[i].handle); + } + T_Ok(thread_join(pool->governor_handle, max_U64)); + MemoryZeroStruct(&pool->governor_handle); + tp_release(pool); +#else + TestSkip(); +#endif +} + internal String8 t_codec_test_pdb_data(Arena *arena) { From 1076b06c91f288a54da84e4f3b67e039ad2e9211 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 2 Sep 2026 23:06:34 -0700 Subject: [PATCH 105/121] linker: correct PDB phase summary timers and test their bounds --- src/linker/lnk_debug_info.c | 8 ++++++-- src/torture/tests/torture_linker_tests.c | 15 ++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index aa2f56f7c..b466e3322 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -6161,7 +6161,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config ProfScope("Write Modules") { - lnk_summary_phase_begin(LNK_SummaryPhase_PdbGsi); + lnk_summary_phase_begin(LNK_SummaryPhase_PdbMod); U64 phase_begin_us = now_time_us(); // FAIR-SHARE: pin the cohort, distribute objs over exactly the cohort lanes, // then run the barrier pass at that cohort. tp_barrier_begin sets @@ -6217,6 +6217,7 @@ ProfScope("Write Modules") // module streams were enqueued per-obj inside lnk_write_pdb_modules ProfScope("Move Global Symbols") { + lnk_summary_phase_begin(LNK_SummaryPhase_PdbGsi); U64 phase_begin_us = now_time_us(); U32 C = tp_barrier_begin(tp); // P2b: no obj_indices distribution -- the pass reads no $S bytes; it flattens the @@ -6224,9 +6225,12 @@ ProfScope("Move Global Symbols") tp_for_parallel_reserve(tp, 0, C, lnk_move_global_symbols_to_gsi, &task); // BARRIER pass (path B): tp_sum_u64/tp_broadcast/barrier_wait tp_barrier_end(tp); lnk_log(LNK_Log_Timers, "[pdb] move global symbols in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); + lnk_summary_phase_end(LNK_SummaryPhase_PdbGsi); } -ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); + lnk_summary_phase_begin(LNK_SummaryPhase_PdbSym); + ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); + lnk_summary_phase_end(LNK_SummaryPhase_PdbSym); if (output_ptr != 0) { lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->publics_sn); lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->globals_sn); diff --git a/src/torture/tests/torture_linker_tests.c b/src/torture/tests/torture_linker_tests.c index dc1b854af..4fd16c769 100644 --- a/src/torture/tests/torture_linker_tests.c +++ b/src/torture/tests/torture_linker_tests.c @@ -325,8 +325,21 @@ TEST(link_large_import_object) coff_obj_writer_release(&obj); coff_lib_writer_release(&lib); T_Ok(t_write_entry_obj()); - t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /opt:ref /out:large.exe entry.obj refs.obj large.lib"); + U64 link_begin_us = now_time_us(); + t_invoke_linkerf("/subsystem:console /entry:entry /debug:full /opt:ref /rad_log:summary /out:large.exe entry.obj refs.obj large.lib"); + U64 link_elapsed_ms = (now_time_us() - link_begin_us) / 1000 + 1; T_Ok(g_last_exit_code == 0); + // A mismatched phase begin/end used to report system uptime for modules. + // Use the enclosing invocation, not a fixed performance threshold. + U64 mod_pos = str8_find_needle(g_output, 0, str8_lit(" mod="), 0); + T_Ok(mod_pos < g_output.size); + if (mod_pos < g_output.size) { + String8 mod = str8_skip(g_output, mod_pos + 5); + mod = str8_prefix(mod, str8_find_needle(mod, 0, str8_lit("/"), 0)); + U64 mod_ms = 0; + T_Ok(try_u64_from_str8_c_rules(mod, &mod_ms)); + T_Ok(mod_ms <= link_elapsed_ms); + } String8 image = t_read_file(arena, str8_lit("large.exe")); PE_BinInfo bin = pe_bin_info_from_data(arena, image); T_Ok(bin.data_dir_count > PE_DataDirectoryIndex_IMPORT); From bcbf9031e8f5c9642ac7ec1ca92cc9aa7d4bdcb0 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 2 Sep 2026 23:06:35 -0700 Subject: [PATCH 106/121] build: assemble BLAKE3 with LLVM for Clang builds --- build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.bat b/build.bat index 8d60e837e..660248983 100644 --- a/build.bat +++ b/build.bat @@ -101,7 +101,7 @@ if "%clang%"=="1" set rc=call llvm-rc if "%msvc%"=="1" set link_dll=/link /DLL if "%clang%"=="1" set link_dll=-Xlinker -DLL if "%msvc%"=="1" set asm=call ml64 -nologo -c -Zi -Fo -if "%clang%"=="1" set asm=call ml64 -nologo -c -Zi -Fo +if "%clang%"=="1" set asm=call llvm-ml -m64 /c /Fo if "%msvc%"=="1" set mklib=call lib -nologo if "%clang%"=="1" set mklib=call llvm-lib From 6fa35bcc78c762ba04bd5f85faeef8b1b99378bf Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 2 Sep 2026 23:06:39 -0700 Subject: [PATCH 107/121] linker: cover upstream dotted exports and real forwarders --- src/linker/tests/dotted_exports.tst | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/linker/tests/dotted_exports.tst diff --git a/src/linker/tests/dotted_exports.tst b/src/linker/tests/dotted_exports.tst new file mode 100644 index 000000000..70e690199 --- /dev/null +++ b/src/linker/tests/dotted_exports.tst @@ -0,0 +1,31 @@ +test: +{ + artifacts: + { + exports_obj: + { + file_name: "exports.obj" + coff: { object: { machine: x64, + sections: { text: { + name: ".text", permissions: (read, execute), content: code, + alignment: 1, data: { hex: "c3" } + } } + symbols: { + dotted: { kind: external_function, name: "__RTFM.autortfm_is_context_status", section: text, value: 0 } + } + } } + } + image: { file_name: "exports.dll", pe: {} } + } + build: { link: { + args: "/dll /entry:__RTFM.autortfm_is_context_status /nodefaultlib /out:exports.dll /export:__RTFM.autortfm_is_context_status /export:forwarded=other_dll.symbol exports.obj" + artifact: image + } } + steps: { expect_pe: { artifact: image, expected: { pe: { exports: { + count: 2 + entries: { + export_0: { name: "__RTFM.autortfm_is_context_status", forwarder: "" } + export_1: { name: "forwarded", forwarder: "other_dll.symbol" } + } + } } } } } +} From 22a3d6abb49d98f242f282f1ac0d1e2ffc4bc20a Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Sun, 16 Aug 2026 09:55:54 -0700 Subject: [PATCH 108/121] radlink: add portable segmented compressed OBJ reader --- src/linker/lnk_compressed_obj.c | 1531 ++++++++++++++++++++++++ src/linker/lnk_compressed_obj.h | 67 ++ src/linker/lnk_compressed_obj_format.h | 124 ++ 3 files changed, 1722 insertions(+) create mode 100644 src/linker/lnk_compressed_obj.c create mode 100644 src/linker/lnk_compressed_obj.h create mode 100644 src/linker/lnk_compressed_obj_format.h diff --git a/src/linker/lnk_compressed_obj.c b/src/linker/lnk_compressed_obj.c new file mode 100644 index 000000000..b141a9f73 --- /dev/null +++ b/src/linker/lnk_compressed_obj.c @@ -0,0 +1,1531 @@ +#if OS_WINDOWS && defined(LNK_OODLE) + +#include "lnk_compressed_obj_format.h" +#include + +//////////////////////////////// +// Runtime state + +#define LNK_COBJ_DEFAULT_CACHE_GIB 8u +#define LNK_COBJ_REGION_CAP (1u << 20) +#define LNK_COBJ_DEFAULT_WRITE_GROUP_SLOTS 64u +#define LNK_COBJ_FILE_MAPPING_GRANULARITY KB(64) +#define LNK_COBJ_KNOWN_HEADER_FLAGS (LNK_COBJ_FLAG_TYPE_INDEX | \ + LNK_COBJ_FLAG_UDT_HASH_INDEX | \ + LNK_COBJ_FLAG_BASE_RELOC_INDEX | \ + LNK_COBJ_FLAG_PORTABLE_RAW_MAP | \ + LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR | \ + LNK_COBJ_FLAG_PACKED_TYPE_OFFSETS_V2 | \ + LNK_COBJ_FLAG_DEBUG_S_INDEX | \ + LNK_COBJ_FLAG_DEBUG_S_SUMMARY | \ + LNK_COBJ_TYPE_INDEX_COUNT_MASK) + +// Every compressed input reserves a virtual range matching the original OBJ. Raw segments map +// directly from the portable file. Compressed segments remain placeholders until the vectored +// exception handler decodes them into a bounded pagefile-backed cache and maps the cache slot at +// the faulting logical address. The linker therefore keeps its ordinary pointer-based OBJ parser. + +typedef enum LNK_CObjSegState +{ + LNK_CObjSegState_Empty, + LNK_CObjSegState_Loading, + LNK_CObjSegState_Ready, + LNK_CObjSegState_Pinned, + LNK_CObjSegState_Evicting, +} LNK_CObjSegState; + +typedef struct LNK_CObjRuntimeSegment +{ + SRWLOCK lock; + volatile LONG state; + U32 slot_idx; + U32 decode_count; + B32 isolated_placeholder; + U8 *address; +} LNK_CObjRuntimeSegment; + +struct LNK_CompressedObj +{ + U8 *base; + U64 reserve_size; + U64 raw_size; + U32 segment_size; + U32 segment_count; + String8 mapped_file; + LNK_CObjHeader *header; + LNK_CObjSegment *directory; + LNK_CObjTypeIndex *type_indices; + LNK_CObjUdtHashIndex *udt_hash_indices; + LNK_CObjBaseRelocEntry *base_reloc_entries; + U32 base_reloc_entry_count; + B32 has_base_reloc_index; + LNK_CObjDebugSEntry *debug_s_entries; + LNK_CObjDebugSSummary *debug_s_summaries; + U32 debug_s_entry_count; + U32 type_index_count; + LNK_CObjRuntimeSegment *segments; + HANDLE portable_mapping; + SRWLOCK placeholder_lock; + volatile LONG active; +}; + +typedef struct LNK_CObjWriteGroup +{ + SRWLOCK lock; + U8 *view; + U32 completed_count; + U32 slot_count; + volatile LONG sealed; +} LNK_CObjWriteGroup; + +typedef struct LNK_CObjCache +{ + SRWLOCK lock; + HANDLE slot_mapping; + LNK_CObjWriteGroup *write_groups; + U64 write_group_count; + U32 write_group_slot_count; + U64 slot_count; + U64 active_slot_count; + U64 segment_size; + U64 resident_count; + U64 victim_cursor; + LNK_CObjRuntimeSegment **slot_segments; + LNK_CompressedObj **regions; + U64 region_count; + U64 region_cap; + PVOID (WINAPI *map_view_of_file_3)(HANDLE,HANDLE,PVOID,ULONG64,SIZE_T,ULONG,ULONG,MEM_EXTENDED_PARAMETER*,ULONG); + BOOL (WINAPI *unmap_view_of_file_2)(HANDLE,PVOID,ULONG); + PVOID (WINAPI *virtual_alloc_2)(HANDLE,PVOID,SIZE_T,ULONG,ULONG,MEM_EXTENDED_PARAMETER*,ULONG); + PVOID veh; + volatile LONG initialized; + volatile LONG failed; + U64 faults; + U64 decodes; + U64 redecodes; + U64 evictions; + U64 raw_segments; + U64 decoded_bytes; + U64 stored_bytes_read; + U64 occupancy_hwm; + U64 frozen_segments; + B32 skip_cleanup; +} LNK_CObjCache; + +global LNK_CObjCache g_lnk_cobj_cache; +global volatile LONG g_lnk_cobj_cache_init_state; + +global volatile LONG g_lnk_cobj_skip_cleanup_mode = -1; +global U64 g_lnk_cobj_window_decodes; +global U64 g_lnk_cobj_window_bytes; +global B32 g_lnk_cobj_cache_shrunk; +global U64 g_lnk_cobj_redecode_debug_s; +global U64 g_lnk_cobj_redecode_types; +global U64 g_lnk_cobj_redecode_segment_zero; + +//////////////////////////////// +// Configuration and diagnostics + +internal LNK_CompressedObj *lnk_cobj_region_from_address(U8 *address); + +internal int +lnk_cobj_address_ptr_compare(const void *a, const void *b) +{ + U8 *pa = *(U8 **)a; + U8 *pb = *(U8 **)b; + return pa < pb ? -1 : pa > pb; +} + +internal B32 +lnk_cobj_runtime_segment_overlaps_types(LNK_CObjRuntimeSegment *segment) +{ + LNK_CompressedObj *region = segment && segment->address ? + lnk_cobj_region_from_address(segment->address) : 0; + if (region == 0 || region->type_indices == 0) { return 0; } + U64 seg_idx = (U64)(segment->address - region->base) / region->segment_size; + Rng1U64 seg_range = r1u64(seg_idx * region->segment_size, + Min(region->raw_size, (seg_idx + 1) * region->segment_size)); + for EachIndex(type_idx, region->type_index_count) { + LNK_CObjTypeIndex *type = ®ion->type_indices[type_idx]; + Rng1U64 type_range = r1u64(type->raw_section_offset, + type->raw_section_offset + type->raw_section_size); + if (dim_1u64(intersect_1u64(seg_range, type_range)) != 0) { return 1; } + } + return 0; +} + +internal B32 +lnk_cobj_runtime_segment_overlaps_debug_s(LNK_CObjRuntimeSegment *segment) +{ + LNK_CompressedObj *region = segment && segment->address ? + lnk_cobj_region_from_address(segment->address) : 0; + if (region == 0 || region->debug_s_entries == 0) { return 0; } + U64 seg_idx = (U64)(segment->address - region->base) / region->segment_size; + Rng1U64 seg_range = r1u64(seg_idx * region->segment_size, + Min(region->raw_size, (seg_idx + 1) * region->segment_size)); + for EachIndex(entry_idx, region->debug_s_entry_count) { + LNK_CObjDebugSEntry *entry = ®ion->debug_s_entries[entry_idx]; + // Retain the complete span from the section signature through each indexed payload. Taking + // the union over a section's entries covers its C13 headers/padding as well as payload bytes. + Rng1U64 debug_s_range = r1u64(entry->raw_section_offset, + (U64)entry->raw_payload_offset + entry->raw_payload_size); + if (dim_1u64(intersect_1u64(seg_range, debug_s_range)) != 0) { return 1; } + } + return 0; +} + +internal void +lnk_compressed_obj_log_phase_stats(char *tag) +{ + if (g_lnk_cobj_cache.region_count == 0) { return; } + char *phase_stats_env = getenv("RAD_COBJ_PHASE_STATS"); + if (phase_stats_env != 0 && phase_stats_env[0] != 0 && phase_stats_env[0] != '0') { + lnk_log(LNK_Log_Timers, "[cobj phase] %s faults=%llu decodes=%llu redecodes=%llu evictions=%llu window=%llu", + tag, g_lnk_cobj_cache.faults, g_lnk_cobj_cache.decodes, g_lnk_cobj_cache.redecodes, + g_lnk_cobj_cache.evictions, g_lnk_cobj_window_decodes); + } +} + +internal B32 +lnk_cobj_skip_cleanup_enabled(void) +{ + if (g_lnk_cobj_skip_cleanup_mode < 0) { + char value[8]; + DWORD len = GetEnvironmentVariableA("RAD_COBJ_SKIP_CLEANUP", value, sizeof(value)); + LONG enabled = len > 0 && len < sizeof(value) && value[0] != '0'; + InterlockedCompareExchange(&g_lnk_cobj_skip_cleanup_mode, enabled, -1); + } + return g_lnk_cobj_skip_cleanup_mode != 0; +} + +//////////////////////////////// +// Indexed and explicit-decode access + +internal String8 +lnk_compressed_obj_direct_range(LNK_CompressedObj *obj, Rng1U64 range) +{ + String8 result = {0}; + if (!obj || range.min >= range.max || range.max > obj->raw_size || !obj->mapped_file.str) { return result; } + U32 first = (U32)(range.min / obj->segment_size); + U32 last = (U32)((range.max - 1) / obj->segment_size); + if (last >= obj->segment_count) { return result; } + + LNK_CObjSegment *first_entry = &obj->directory[first]; + if (!(first_entry->flags & LNK_COBJ_SEGMENT_RAW) || first_entry->stored_size != first_entry->raw_size) { return result; } + U64 expected_file_off = first_entry->file_offset + first_entry->stored_size; + for (U32 seg_idx = first + 1; seg_idx <= last; ++seg_idx) { + LNK_CObjSegment *entry = &obj->directory[seg_idx]; + if (!(entry->flags & LNK_COBJ_SEGMENT_RAW) || entry->stored_size != entry->raw_size || + entry->file_offset != expected_file_off) { + return result; + } + expected_file_off += entry->stored_size; + } + + // Raw runs are mapped directly into the reserved logical OBJ address range. + return str8(obj->base + range.min, dim_1u64(range)); +} + +internal B32 +lnk_compressed_obj_type_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjTypeIndexView *out) +{ + MemoryZeroStruct(out); + if (!obj || !obj->type_indices) { return 0; } + for (U32 i = 0; i < obj->type_index_count; ++i) { + LNK_CObjTypeIndex *idx = &obj->type_indices[i]; + if (idx->raw_section_offset == section_range.min && idx->raw_section_size == dim_1u64(section_range)) { + out->count = idx->leaf_count; + if (obj->header->flags & LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR) { + U64 group_count = ((U64)idx->leaf_count + + (((U64)1 << LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT) - 1)) >> + LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT; + U64 group_bytes = group_count * 2 * sizeof(U32); + out->packed_v2_offset_groups = (U32 *)(obj->mapped_file.str + idx->offsets_file_offset); + out->packed_v2_offset_payload = obj->mapped_file.str + idx->offsets_file_offset + + ((group_bytes + 7) & ~(U64)7); + out->offset_checkpoint_shift = LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT; + out->packed_kind_dictionary = (U16 *)(obj->mapped_file.str + idx->sizes_file_offset); + out->packed_kind_codes = obj->mapped_file.str + idx->kinds_file_offset; + out->packed_sidecar = 1; + } else { + out->offsets = (U32 *)(obj->mapped_file.str + idx->offsets_file_offset); + out->sizes = (U16 *)(obj->mapped_file.str + idx->sizes_file_offset); + out->kinds = (U16 *)(obj->mapped_file.str + idx->kinds_file_offset); + } + if (obj->udt_hash_indices) { + LNK_CObjUdtHashIndex *udt = &obj->udt_hash_indices[i]; + out->complete_udt_hashes = (U64 *)(obj->mapped_file.str + udt->hashes_file_offset); + out->complete_udt_hash_count = udt->hash_count; + } + return 1; + } + } + return 0; +} + +internal B32 +lnk_compressed_obj_base_reloc_index(LNK_CompressedObj *obj, LNK_CObjBaseRelocView *out) +{ + MemoryZeroStruct(out); + if (!obj || !obj->has_base_reloc_index) { return 0; } + out->count = obj->base_reloc_entry_count; + out->v = obj->base_reloc_entries; + return 1; +} + +internal B32 +lnk_compressed_obj_debug_s_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjDebugSView *out) +{ + MemoryZeroStruct(out); + if (!obj || !obj->debug_s_entries || section_range.min > max_U32) { return 0; } + U32 wanted = (U32)section_range.min; + U32 lo = 0, hi = obj->debug_s_entry_count; + while (lo < hi) { + U32 mid = lo + (hi - lo) / 2; + if (obj->debug_s_entries[mid].raw_section_offset < wanted) { lo = mid + 1; } + else { hi = mid; } + } + U32 first = lo; + while (lo < obj->debug_s_entry_count && obj->debug_s_entries[lo].raw_section_offset == wanted) { lo += 1; } + if (first == lo) { return 0; } + out->v = obj->debug_s_entries + first; + out->summaries = obj->debug_s_summaries ? obj->debug_s_summaries + first : 0; + out->count = lo - first; + return 1; +} + +internal B32 +lnk_compressed_obj_copy_range(LNK_CompressedObj *obj, Rng1U64 range, void *raw_dst, LNK_CObjDecodeWindow *window) +{ + if (!obj || range.min > range.max || range.max > obj->raw_size) { return 0; } + if (!window->buffer || window->buffer_cap < obj->segment_size) { + if (window->buffer) { VirtualFree(window->buffer, 0, MEM_RELEASE); } + window->buffer = VirtualAlloc(0, obj->segment_size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + window->buffer_cap = window->buffer ? obj->segment_size : 0; + window->seg_idx = max_U32; + if (!window->buffer) { return 0; } + } + if (window->obj != obj) { window->obj = obj; window->seg_idx = max_U32; } + + U8 *dst = raw_dst; + U64 cursor = range.min; + while (cursor < range.max) { + U32 seg_idx = (U32)(cursor / obj->segment_size); + LNK_CObjSegment *entry = &obj->directory[seg_idx]; + U64 in_seg = cursor - (U64)seg_idx * obj->segment_size; + U64 amount = Min(range.max - cursor, (U64)entry->raw_size - in_seg); + if (window->seg_idx != seg_idx) { + U8 *stored = obj->mapped_file.str + entry->file_offset; + B32 ok = 1; + B32 decode_direct = in_seg == 0 && amount == entry->raw_size; + U8 *decode_dst = decode_direct ? dst : window->buffer; + if (entry->flags & LNK_COBJ_SEGMENT_RAW) { + MemoryCopy(decode_dst, stored, entry->raw_size); + } else { + OO_SINTa got = OodleLZ_Decompress(stored, entry->stored_size, decode_dst, entry->raw_size, + OodleLZ_FuzzSafe_Yes, OodleLZ_CheckCRC_No, OodleLZ_Verbosity_None, + 0, 0, 0, 0, 0, 0, OodleLZ_Decode_Unthreaded); + ok = got == entry->raw_size; + } + if (!ok) { return 0; } + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_window_decodes); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_window_bytes, entry->stored_size); + if (decode_direct) { + // Large $S and symbol-fixup walks consume complete interior segments once. Decode those + // bytes into their final arena instead of filling the window and copying them again. + window->seg_idx = max_U32; + dst += amount; cursor += amount; + continue; + } + window->seg_idx = seg_idx; + window->valid_size = entry->raw_size; + } + MemoryCopy(dst, window->buffer + in_seg, amount); + dst += amount; cursor += amount; + } + return 1; +} + +internal B32 +lnk_compressed_obj_copy_string(LNK_CompressedObj *obj, String8 src, void *dst, LNK_CObjDecodeWindow *window) +{ + if (!obj) { return 0; } + U64 base = (U64)obj->base; + U64 ptr = (U64)src.str; + if (ptr < base || ptr - base > obj->raw_size || src.size > obj->raw_size - (ptr - base)) { return 0; } + return lnk_compressed_obj_copy_range(obj, r1u64(ptr - base, ptr - base + src.size), dst, window); +} + +internal void +lnk_compressed_obj_release_window(LNK_CObjDecodeWindow *window) +{ + if (window->buffer) { VirtualFree(window->buffer, 0, MEM_RELEASE); } + MemoryZeroStruct(window); +} + +internal U32 +lnk_compressed_obj_segment_count(LNK_CompressedObj *obj) +{ + return obj ? obj->segment_count : 0; +} + +internal U32 +lnk_compressed_obj_segment_size(LNK_CompressedObj *obj) +{ + return obj ? obj->segment_size : 0; +} + +internal Rng1U64 +lnk_compressed_obj_stored_segment_range(LNK_CompressedObj *obj, U32 segment_idx) +{ + Rng1U64 result = {0}; + if (obj && segment_idx < obj->segment_count) { + LNK_CObjSegment *segment = &obj->directory[segment_idx]; + U64 min = (U64)(obj->mapped_file.str + segment->file_offset); + result = r1u64(min, min + segment->stored_size); + } + return result; +} + +//////////////////////////////// +// Decoded-segment cache + + +internal LNK_CObjWriteGroup * +lnk_cobj_alloc_write_groups(U64 slot_count, U32 group_slot_count, U64 *group_count_out) +{ + U64 group_count = CeilIntegerDiv(slot_count, group_slot_count); + LNK_CObjWriteGroup *groups = HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, + group_count * sizeof(*groups)); + if (groups) { + for EachIndex(group_idx, group_count) { + U64 first_slot = group_idx * group_slot_count; + groups[group_idx].slot_count = (U32)Min((U64)group_slot_count, slot_count - first_slot); + } + *group_count_out = group_count; + } + return groups; +} + +internal void +lnk_cobj_release_write_groups(LNK_CObjWriteGroup *groups, U64 group_count) +{ + if (!groups) { return; } + for EachIndex(group_idx, group_count) { + if (groups[group_idx].view) { UnmapViewOfFile(groups[group_idx].view); } + } + HeapFree(GetProcessHeap(), 0, groups); +} + +internal U8 * +lnk_cobj_write_group_acquire(U32 slot_idx, U32 *group_idx_out) +{ + if (!g_lnk_cobj_cache.write_groups) { return 0; } + U32 group_idx = slot_idx / g_lnk_cobj_cache.write_group_slot_count; + LNK_CObjWriteGroup *group = &g_lnk_cobj_cache.write_groups[group_idx]; + AcquireSRWLockExclusive(&group->lock); + if (!group->sealed && !group->view) { + U64 group_off = (U64)group_idx * g_lnk_cobj_cache.write_group_slot_count * + g_lnk_cobj_cache.segment_size; + U64 group_size = (U64)group->slot_count * g_lnk_cobj_cache.segment_size; + group->view = MapViewOfFile(g_lnk_cobj_cache.slot_mapping, FILE_MAP_WRITE, + (DWORD)(group_off >> 32), (DWORD)group_off, group_size); + } + U8 *result = 0; + if (!group->sealed && group->view) { + U32 in_group_idx = slot_idx % g_lnk_cobj_cache.write_group_slot_count; + result = group->view + (U64)in_group_idx * g_lnk_cobj_cache.segment_size; + *group_idx_out = group_idx; + } + ReleaseSRWLockExclusive(&group->lock); + return result; +} + +internal void +lnk_cobj_write_group_complete(U32 group_idx) +{ + LNK_CObjWriteGroup *group = &g_lnk_cobj_cache.write_groups[group_idx]; + AcquireSRWLockExclusive(&group->lock); + group->completed_count += 1; + if (group->completed_count == group->slot_count) { + UnmapViewOfFile(group->view); + group->view = 0; + InterlockedExchange(&group->sealed, 1); + } + ReleaseSRWLockExclusive(&group->lock); +} + +internal void +lnk_compressed_obj_trim_working_set(void) +{ + char *trim_env = getenv("RAD_COBJ_TRIM_WS"); + if (trim_env == 0 || trim_env[0] == 0 || trim_env[0] == '0') { return; } + + U64 begin_us = now_time_us(); + if (trim_env[0] == '2') { + SetProcessWorkingSetSize(GetCurrentProcess(), (SIZE_T)-1, (SIZE_T)-1); + lnk_log(LNK_Log_Timers, "[cobj trim] whole-process elapsed=%.2f ms", + (F64)(now_time_us() - begin_us) / 1000.0); + return; + } + + // Drop only the decoded cache views from the process working set. Unlike the + // whole-process SetProcessWorkingSetSize path this preserves the linker's hot + // hash tables and arenas. The pagefile-section mappings stay intact, so a + // later pointer access can soft-fault the decoded bytes without running Oodle + // again; VirtualUnlock on an unlocked range intentionally returns + // ERROR_NOT_LOCKED after removing resident pages from the caller's working set. + U64 cache_view_count = 0; + U64 cache_view_bytes = 0; + U64 purged_count = 0; + U64 shrink_gib = 0; + { + char shrink_env[64]; + DWORD shrink_env_len = GetEnvironmentVariableA("RAD_COBJ_CACHE_SHRINK_GIB", shrink_env, sizeof(shrink_env)); + if (shrink_env_len > 0 && shrink_env_len < sizeof(shrink_env)) { shrink_gib = strtoull(shrink_env, 0, 10); } + } + if (shrink_gib > 0 && !g_lnk_cobj_cache_shrunk && g_lnk_cobj_cache.segment_size) { + U64 wanted_slots = Clamp(1, shrink_gib * GB(1) / g_lnk_cobj_cache.segment_size, + g_lnk_cobj_cache.slot_count); + char *freeze_env = getenv("RAD_COBJ_CACHE_FREEZE"); + B32 freeze_generation = freeze_env != 0 && freeze_env[0] != 0 && freeze_env[0] != '0'; + if (freeze_generation) { + // Preserve the complete first-generation mapping but never recycle one of its slots again. + // Existing logical OBJ pointers therefore remain valid. PDB misses use a fresh, independently + // evicting cache generation; this costs shrink_gib additional system commit but avoids the + // thousands of UnmapViewOfFile2 calls and forced re-decodes of a destructive phase reset. + U64 new_mapping_size = wanted_slots * g_lnk_cobj_cache.segment_size; + HANDLE new_mapping = CreateFileMappingW(INVALID_HANDLE_VALUE, 0, PAGE_READWRITE, + (DWORD)(new_mapping_size >> 32), + (DWORD)new_mapping_size, 0); + U64 new_write_group_count = 0; + LNK_CObjWriteGroup *new_write_groups = g_lnk_cobj_cache.write_groups ? + lnk_cobj_alloc_write_groups(wanted_slots, g_lnk_cobj_cache.write_group_slot_count, + &new_write_group_count) : 0; + LNK_CObjRuntimeSegment **new_slots = VirtualAlloc(0, wanted_slots * sizeof(void *), + MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + B32 can_freeze = new_mapping != 0 && new_slots != 0 && + (!g_lnk_cobj_cache.write_groups || new_write_groups != 0); + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + if (can_freeze) { + for EachIndex(slot_idx, g_lnk_cobj_cache.slot_count) { + LNK_CObjRuntimeSegment *segment = g_lnk_cobj_cache.slot_segments[slot_idx]; + if (segment != 0 && segment->state != LNK_CObjSegState_Ready && + segment->state != LNK_CObjSegState_Pinned) { + can_freeze = 0; + break; + } + } + } + if (can_freeze) { + HANDLE old_mapping = g_lnk_cobj_cache.slot_mapping; + LNK_CObjWriteGroup *old_write_groups = g_lnk_cobj_cache.write_groups; + U64 old_write_group_count = g_lnk_cobj_cache.write_group_count; + LNK_CObjRuntimeSegment **old_slots = g_lnk_cobj_cache.slot_segments; + U64 old_slot_count = g_lnk_cobj_cache.slot_count; + for EachIndex(slot_idx, old_slot_count) { + LNK_CObjRuntimeSegment *segment = old_slots[slot_idx]; + if (segment != 0) { + // max_U32 denotes a frozen slot: it is mapped and readable, but is not owned by the + // active generation's slot table and must never participate in victim selection. + segment->slot_idx = max_U32; + g_lnk_cobj_cache.frozen_segments += 1; + } + } + g_lnk_cobj_cache.slot_mapping = new_mapping; + g_lnk_cobj_cache.write_groups = new_write_groups; + g_lnk_cobj_cache.write_group_count = new_write_group_count; + g_lnk_cobj_cache.slot_segments = new_slots; + g_lnk_cobj_cache.slot_count = wanted_slots; + g_lnk_cobj_cache.active_slot_count = wanted_slots; + g_lnk_cobj_cache.resident_count = 0; + g_lnk_cobj_cache.victim_cursor = 0; + g_lnk_cobj_cache_shrunk = 1; + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + // Mapped views retain the old pagefile section object. Only its handle and now-unused + // reverse slot table are released here; ordinary close unmaps each frozen logical view. + lnk_cobj_release_write_groups(old_write_groups, old_write_group_count); + CloseHandle(old_mapping); + U64 trimmed_frozen = 0; + U64 trim_calls = 0; + char *freeze_trim_env = getenv("RAD_COBJ_CACHE_FREEZE_TRIM"); + if (freeze_trim_env != 0 && freeze_trim_env[0] != 0 && freeze_trim_env[0] != '0') { + // Keep every view valid but offer its resident pages back to the OS. Later consumers + // incur only soft page faults into the frozen pagefile section, never Oodle re-decodes. + // This is deliberately optional: it trades some kernel fault work for a lower PDB-phase + // working set, so deployments can choose the appropriate memory/speed point. + B32 types_only = _stricmp(freeze_trim_env, "types") == 0; + B32 except_debug_s = _stricmp(freeze_trim_env, "except_debug_s") == 0 || + _stricmp(freeze_trim_env, "except_debug_s_runs") == 0; + B32 runs_only = _stricmp(freeze_trim_env, "except_debug_s_runs") == 0; + U8 **trim_addresses = VirtualAlloc(0, old_slot_count * sizeof(*trim_addresses), + MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + for EachIndex(slot_idx, old_slot_count) { + LNK_CObjRuntimeSegment *segment = old_slots[slot_idx]; + B32 should_trim = segment != 0 && segment->address != 0; + if (should_trim && types_only) { + should_trim = lnk_cobj_runtime_segment_overlaps_types(segment); + } else if (should_trim && except_debug_s) { + should_trim = !lnk_cobj_runtime_segment_overlaps_debug_s(segment); + } + if (should_trim) { + if (trim_addresses != 0) { + trim_addresses[trimmed_frozen++] = segment->address; + } else { + VirtualUnlock(segment->address, g_lnk_cobj_cache.segment_size); + trimmed_frozen += 1; + trim_calls += 1; + } + } + } + if (trim_addresses != 0) { + qsort(trim_addresses, trimmed_frozen, sizeof(*trim_addresses), lnk_cobj_address_ptr_compare); + U64 selected_count = trimmed_frozen; + trimmed_frozen = 0; + for (U64 first = 0; first < selected_count;) { + U64 opl = first + 1; + while (opl < selected_count && + trim_addresses[opl] == trim_addresses[opl - 1] + g_lnk_cobj_cache.segment_size) { + opl += 1; + } + U64 run_count = opl - first; + if (!runs_only || run_count > 1) { + VirtualUnlock(trim_addresses[first], run_count * g_lnk_cobj_cache.segment_size); + trimmed_frozen += run_count; + trim_calls += 1; + } + first = opl; + } + VirtualFree(trim_addresses, 0, MEM_RELEASE); + } + } + VirtualFree(old_slots, 0, MEM_RELEASE); + lnk_log(LNK_Log_Timers, "[cobj freeze] retained=%llu trimmed=%llu trim-calls=%llu new-cache=%llu MiB elapsed=%.2f ms", + g_lnk_cobj_cache.frozen_segments, trimmed_frozen, trim_calls, new_mapping_size / MB(1), + (F64)(now_time_us() - begin_us) / 1000.0); + return; + } + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + lnk_cobj_release_write_groups(new_write_groups, new_write_group_count); + if (new_mapping) { CloseHandle(new_mapping); } + if (new_slots) { VirtualFree(new_slots, 0, MEM_RELEASE); } + } + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + // Reset the phase cache completely. Keeping arbitrary retained mappings across the + // limit transition is unsafe because later victim selection owns an index prefix. + for EachIndex(slot_idx, g_lnk_cobj_cache.slot_count) { + LNK_CObjRuntimeSegment *segment = g_lnk_cobj_cache.slot_segments[slot_idx]; + if (!segment || InterlockedCompareExchange(&segment->state, LNK_CObjSegState_Evicting, + LNK_CObjSegState_Ready) != LNK_CObjSegState_Ready) { + continue; + } + if (segment->address && + g_lnk_cobj_cache.unmap_view_of_file_2(GetCurrentProcess(), segment->address, + MEM_PRESERVE_PLACEHOLDER)) { + g_lnk_cobj_cache.slot_segments[slot_idx] = 0; + segment->slot_idx = max_U32; + InterlockedExchange(&segment->state, LNK_CObjSegState_Empty); + purged_count += 1; + } else { + InterlockedExchange(&segment->state, LNK_CObjSegState_Ready); + } + } + // Write-pinned segments (and the unlikely view that failed to unmap) still own their + // original slots. Count survivors in the active prefix instead of pretending the reset + // emptied it. If that prefix is completely occupied, extend it through the next free slot + // so a later cache miss cannot spin forever with no possible victim. + U64 active_slot_count = wanted_slots; + U64 retained_active_count = 0; + for EachIndex(slot_idx, active_slot_count) { + retained_active_count += g_lnk_cobj_cache.slot_segments[slot_idx] != 0; + } + if (retained_active_count == active_slot_count) { + for (U64 slot_idx = active_slot_count; slot_idx < g_lnk_cobj_cache.slot_count; ++slot_idx) { + retained_active_count += g_lnk_cobj_cache.slot_segments[slot_idx] != 0; + active_slot_count = slot_idx + 1; + if (g_lnk_cobj_cache.slot_segments[slot_idx] == 0) { break; } + } + } + g_lnk_cobj_cache.active_slot_count = active_slot_count; + g_lnk_cobj_cache.resident_count = retained_active_count; + g_lnk_cobj_cache.victim_cursor %= active_slot_count; + g_lnk_cobj_cache_shrunk = 1; + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + } + if (shrink_gib == 0) { + for EachIndex(region_idx, g_lnk_cobj_cache.region_count) { + LNK_CompressedObj *region = g_lnk_cobj_cache.regions[region_idx]; + if (!region || !region->active) { continue; } + for EachIndex(seg_idx, region->segment_count) { + LNK_CObjRuntimeSegment *segment = ®ion->segments[seg_idx]; + if ((segment->state == LNK_CObjSegState_Ready || segment->state == LNK_CObjSegState_Pinned) && + segment->address) { + VirtualUnlock(segment->address, region->segment_size); + cache_view_count += 1; + cache_view_bytes += region->segment_size; + } + } + } + } + lnk_log(LNK_Log_Timers, "[cobj trim] purged=%llu active=%llu MiB cache_views=%llu cache=%llu GiB elapsed=%.2f ms", + purged_count, (g_lnk_cobj_cache.active_slot_count * g_lnk_cobj_cache.segment_size) / MB(1), + cache_view_count, cache_view_bytes / GB(1), + (F64)(now_time_us() - begin_us) / 1000.0); +} + +internal int +lnk_cobj_region_ptr_compare(const void *a, const void *b) +{ + LNK_CompressedObj *ra = *(LNK_CompressedObj **)a; + LNK_CompressedObj *rb = *(LNK_CompressedObj **)b; + return ra->base < rb->base ? -1 : ra->base > rb->base; +} + +internal LNK_CompressedObj * +lnk_cobj_region_from_address(U8 *address) +{ + U64 min = 0, max = g_lnk_cobj_cache.region_count; + while (min < max) { + U64 mid = min + (max - min) / 2; + LNK_CompressedObj *region = g_lnk_cobj_cache.regions[mid]; + if (address < region->base) { + max = mid; + } else if (address >= region->base + region->reserve_size) { + min = mid + 1; + } else { + return region->active ? region : 0; + } + } + return 0; +} + +internal B32 +lnk_cobj_evict_one_locked(U32 *slot_idx_out) +{ + U64 victim_slot = max_U64; + LNK_CObjRuntimeSegment *victim = 0; + for (U64 probe = 0; probe < g_lnk_cobj_cache.active_slot_count; ++probe) { + U64 slot_idx = (g_lnk_cobj_cache.victim_cursor + probe) % g_lnk_cobj_cache.active_slot_count; + LNK_CObjRuntimeSegment *segment = g_lnk_cobj_cache.slot_segments[slot_idx]; + if (segment == 0) { + *slot_idx_out = (U32)slot_idx; + g_lnk_cobj_cache.victim_cursor = (slot_idx + 1) % g_lnk_cobj_cache.active_slot_count; + return 1; + } + B32 write_group_recyclable = 1; + if (g_lnk_cobj_cache.write_groups) { + U64 group_idx = slot_idx / g_lnk_cobj_cache.write_group_slot_count; + write_group_recyclable = g_lnk_cobj_cache.write_groups[group_idx].sealed != 0; + } + if (segment->state == LNK_CObjSegState_Ready && write_group_recyclable) { + victim_slot = slot_idx; + victim = segment; + break; + } + } + if (victim == 0 || InterlockedCompareExchange(&victim->state, LNK_CObjSegState_Evicting, LNK_CObjSegState_Ready) != LNK_CObjSegState_Ready) { + return 0; + } + + U8 *victim_address = victim->address; + if (victim_address == 0 || !g_lnk_cobj_cache.unmap_view_of_file_2(GetCurrentProcess(), victim_address, MEM_PRESERVE_PLACEHOLDER)) { + InterlockedExchange(&victim->state, LNK_CObjSegState_Ready); + return 0; + } + g_lnk_cobj_cache.slot_segments[victim_slot] = 0; + Assert(g_lnk_cobj_cache.resident_count > 0); + g_lnk_cobj_cache.resident_count -= 1; + victim->slot_idx = max_U32; + InterlockedExchange(&victim->state, LNK_CObjSegState_Empty); + g_lnk_cobj_cache.evictions += 1; + *slot_idx_out = (U32)victim_slot; + g_lnk_cobj_cache.victim_cursor = (victim_slot + 1) % g_lnk_cobj_cache.active_slot_count; + return 1; +} + +internal B32 +lnk_cobj_materialize(LNK_CompressedObj *region, U32 seg_idx) +{ + LNK_CObjRuntimeSegment *runtime = ®ion->segments[seg_idx]; + AcquireSRWLockExclusive(&runtime->lock); + if (runtime->state == LNK_CObjSegState_Ready || runtime->state == LNK_CObjSegState_Pinned) { + ReleaseSRWLockExclusive(&runtime->lock); + return 1; + } + while (runtime->state == LNK_CObjSegState_Evicting) { YieldProcessor(); } + InterlockedExchange(&runtime->state, LNK_CObjSegState_Loading); + + U32 slot_idx = max_U32; + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + while (!lnk_cobj_evict_one_locked(&slot_idx)) { + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + SwitchToThread(); + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + } + g_lnk_cobj_cache.slot_segments[slot_idx] = runtime; + runtime->slot_idx = slot_idx; + g_lnk_cobj_cache.resident_count += 1; + Assert(g_lnk_cobj_cache.resident_count <= g_lnk_cobj_cache.active_slot_count); + if (g_lnk_cobj_cache.resident_count > g_lnk_cobj_cache.occupancy_hwm) { + g_lnk_cobj_cache.occupancy_hwm = g_lnk_cobj_cache.resident_count; + } + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + + LNK_CObjSegment *entry = ®ion->directory[seg_idx]; + U64 slot_off = (U64)slot_idx * region->segment_size; + U32 write_group_idx = max_U32; + U8 *write_view = lnk_cobj_write_group_acquire(slot_idx, &write_group_idx); + B32 grouped_write_view = write_view != 0 && write_group_idx != max_U32; + B32 temporary_write_view = write_view == 0; + if (temporary_write_view) { + write_view = (U8 *)MapViewOfFile(g_lnk_cobj_cache.slot_mapping, FILE_MAP_WRITE, + (DWORD)(slot_off >> 32), (DWORD)slot_off, region->segment_size); + } + B32 ok = write_view != 0; + if (ok) { + U8 *stored = region->mapped_file.str + entry->file_offset; + if (entry->flags & LNK_COBJ_SEGMENT_RAW) { + MemoryCopy(write_view, stored, entry->raw_size); + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_cache.raw_segments); + } else { + OO_SINTa decoded = OodleLZ_Decompress(stored, entry->stored_size, write_view, entry->raw_size, + OodleLZ_FuzzSafe_Yes, OodleLZ_CheckCRC_No, OodleLZ_Verbosity_None, + 0, 0, 0, 0, 0, 0, OodleLZ_Decode_Unthreaded); + ok = decoded == entry->raw_size; + } + if (ok && entry->raw_size < region->segment_size) { + MemoryZero(write_view + entry->raw_size, region->segment_size - entry->raw_size); + } + if (temporary_write_view) { UnmapViewOfFile(write_view); } + } + U8 *target = region->base + (U64)seg_idx * region->segment_size; + if (ok) { ok = lnk_cobj_isolate_portable_segment(region, seg_idx); } + if (ok) { + void *mapped = g_lnk_cobj_cache.map_view_of_file_3(g_lnk_cobj_cache.slot_mapping, GetCurrentProcess(), target, + slot_off, region->segment_size, MEM_REPLACE_PLACEHOLDER, + PAGE_READONLY, 0, 0); + ok = mapped == target; + } + if (grouped_write_view) { lnk_cobj_write_group_complete(write_group_idx); } + if (!ok) { + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + if (g_lnk_cobj_cache.slot_segments[slot_idx] == runtime) { + g_lnk_cobj_cache.slot_segments[slot_idx] = 0; + Assert(g_lnk_cobj_cache.resident_count > 0); + g_lnk_cobj_cache.resident_count -= 1; + } + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + runtime->slot_idx = max_U32; + InterlockedExchange(&runtime->state, LNK_CObjSegState_Empty); + InterlockedExchange(&g_lnk_cobj_cache.failed, 1); + ReleaseSRWLockExclusive(&runtime->lock); + return 0; + } + + if (runtime->decode_count++ > 0) { + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_cache.redecodes); + if (lnk_cobj_runtime_segment_overlaps_debug_s(runtime)) { + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_redecode_debug_s); + } + if (lnk_cobj_runtime_segment_overlaps_types(runtime)) { + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_redecode_types); + } + if (seg_idx == 0) { InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_redecode_segment_zero); } + } + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_cache.decodes); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.decoded_bytes, entry->raw_size); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.stored_bytes_read, entry->stored_size); + InterlockedExchange(&runtime->state, LNK_CObjSegState_Ready); + ReleaseSRWLockExclusive(&runtime->lock); + return 1; +} + +// In-place COFF patches are rare, but they must retain FILE_MAP_COPY semantics. Raw-stored +// segments already map the portable file directly and can fall through to the generic COW VEH. +// Compressed segments first need a decoded mapping. Pin it so eviction can never discard a +// private modification, then promote the faulting page here instead of depending on the order of +// independently registered vectored exception handlers. +internal B32 +lnk_cobj_pin_compressed_segment_for_write(LNK_CompressedObj *region, U32 seg_idx) +{ + LNK_CObjRuntimeSegment *runtime = ®ion->segments[seg_idx]; + for (;;) { + LONG state = runtime->state; + if (state == LNK_CObjSegState_Pinned) { return 1; } + if (state == LNK_CObjSegState_Ready) { + if (InterlockedCompareExchange(&runtime->state, LNK_CObjSegState_Pinned, + LNK_CObjSegState_Ready) == LNK_CObjSegState_Ready) { + return 1; + } + continue; + } + if (!lnk_cobj_materialize(region, seg_idx)) { return 0; } + } +} + +internal LONG NTAPI +lnk_cobj_veh(EXCEPTION_POINTERS *info) +{ + EXCEPTION_RECORD *er = info->ExceptionRecord; + if (er->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && er->NumberParameters >= 2) { + U8 *address = (U8 *)er->ExceptionInformation[1]; + LNK_CompressedObj *region = lnk_cobj_region_from_address(address); + if (region != 0) { + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_cache.faults); + U32 seg_idx = (U32)((address - region->base) / region->segment_size); + U64 access_kind = er->ExceptionInformation[0]; + if (access_kind == 0 && seg_idx < region->segment_count && + lnk_cobj_materialize(region, seg_idx)) { + return EXCEPTION_CONTINUE_EXECUTION; + } + if (access_kind == 1 && seg_idx < region->segment_count && + !(region->directory[seg_idx].flags & LNK_COBJ_SEGMENT_RAW)) { + if (lnk_cobj_pin_compressed_segment_for_write(region, seg_idx)) { + // Complete the write-fault transition in this handler. Both this handler and the + // ordinary input COW handler are front-inserted, so relying on which one runs next + // would make a write-first access depend on their registration order. + void *page = (void *)((UINT_PTR)address & ~(UINT_PTR)(KB(4) - 1)); + DWORD old_protect = 0; + if (VirtualProtect(page, KB(4), PAGE_WRITECOPY, &old_protect)) { + return EXCEPTION_CONTINUE_EXECUTION; + } + } + } + } + } + return EXCEPTION_CONTINUE_SEARCH; +} + +internal B32 +lnk_cobj_cache_init(U32 segment_size) +{ + LONG init_state = InterlockedCompareExchange(&g_lnk_cobj_cache_init_state, 1, 0); + if (init_state != 0) { + while (g_lnk_cobj_cache_init_state == 1) { SwitchToThread(); } + return g_lnk_cobj_cache_init_state == 2 && g_lnk_cobj_cache.failed == 0 && + g_lnk_cobj_cache.segment_size == segment_size; + } + InitializeSRWLock(&g_lnk_cobj_cache.lock); + HMODULE kernelbase = GetModuleHandleA("KernelBase.dll"); + g_lnk_cobj_cache.map_view_of_file_3 = (void *)GetProcAddress(kernelbase, "MapViewOfFile3"); + g_lnk_cobj_cache.unmap_view_of_file_2 = (void *)GetProcAddress(kernelbase, "UnmapViewOfFile2"); + g_lnk_cobj_cache.virtual_alloc_2 = (void *)GetProcAddress(kernelbase, "VirtualAlloc2"); + if (!g_lnk_cobj_cache.map_view_of_file_3 || !g_lnk_cobj_cache.unmap_view_of_file_2 || !g_lnk_cobj_cache.virtual_alloc_2) { + g_lnk_cobj_cache.failed = 1; + g_lnk_cobj_cache.initialized = 1; + InterlockedExchange(&g_lnk_cobj_cache_init_state, 2); + return 0; + } + + U64 cache_gib = LNK_COBJ_DEFAULT_CACHE_GIB; + char cache_env[64]; + DWORD cache_env_len = GetEnvironmentVariableA("RAD_COBJ_CACHE_GIB", cache_env, sizeof(cache_env)); + if (cache_env_len > 0 && cache_env_len < sizeof(cache_env)) { + U64 parsed = strtoull(cache_env, 0, 10); + if (parsed > 0) { cache_gib = parsed; } + } + U64 cache_bytes = cache_gib * GB(1); + char cache_mib_env[64]; + DWORD cache_mib_env_len = GetEnvironmentVariableA("RAD_COBJ_CACHE_MIB", cache_mib_env, sizeof(cache_mib_env)); + if (cache_mib_env_len > 0 && cache_mib_env_len < sizeof(cache_mib_env)) { + U64 parsed = strtoull(cache_mib_env, 0, 10); + if (parsed > 0) { cache_bytes = parsed * MB(1); } + } + g_lnk_cobj_cache.skip_cleanup = lnk_cobj_skip_cleanup_enabled(); + g_lnk_cobj_cache.slot_count = Max(1, cache_bytes / segment_size); + g_lnk_cobj_cache.active_slot_count = g_lnk_cobj_cache.slot_count; + g_lnk_cobj_cache.segment_size = segment_size; + U64 mapping_size = g_lnk_cobj_cache.slot_count * segment_size; + g_lnk_cobj_cache.slot_mapping = CreateFileMappingW(INVALID_HANDLE_VALUE, 0, PAGE_READWRITE, + (DWORD)(mapping_size >> 32), (DWORD)mapping_size, 0); + if (g_lnk_cobj_cache.slot_mapping) { + g_lnk_cobj_cache.write_group_slot_count = LNK_COBJ_DEFAULT_WRITE_GROUP_SLOTS; + g_lnk_cobj_cache.write_groups = lnk_cobj_alloc_write_groups( + g_lnk_cobj_cache.slot_count, g_lnk_cobj_cache.write_group_slot_count, + &g_lnk_cobj_cache.write_group_count); + } + g_lnk_cobj_cache.slot_segments = VirtualAlloc(0, g_lnk_cobj_cache.slot_count * sizeof(void *), MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + // Keep the VEH lookup table fixed after publication, but make the ceiling high enough that + // it cannot constrain practical links. One million pointers cost only 8 MiB on x64. + g_lnk_cobj_cache.region_cap = LNK_COBJ_REGION_CAP; + g_lnk_cobj_cache.regions = VirtualAlloc(0, g_lnk_cobj_cache.region_cap * sizeof(void *), MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + if (!g_lnk_cobj_cache.slot_mapping || !g_lnk_cobj_cache.slot_segments || !g_lnk_cobj_cache.regions) { + g_lnk_cobj_cache.failed = 1; + } else { + g_lnk_cobj_cache.veh = AddVectoredExceptionHandler(1, lnk_cobj_veh); + if (!g_lnk_cobj_cache.veh) { g_lnk_cobj_cache.failed = 1; } + } + g_lnk_cobj_cache.initialized = 1; + InterlockedExchange(&g_lnk_cobj_cache_init_state, 2); + return g_lnk_cobj_cache.failed == 0; +} + +//////////////////////////////// +// Portable logical-address mapping + +internal B32 +lnk_cobj_segment_is_direct_raw(LNK_CObjSegment *segment) +{ + return (segment->flags & LNK_COBJ_SEGMENT_RAW) != 0; +} + +internal B32 +lnk_cobj_split_portable_raw_boundaries(LNK_CompressedObj *region) +{ + // Leave compressed runs intact. Raw runs still need exact placeholders because they are mapped + // directly from the portable file before any OBJ pointers are published. + for (U32 seg_idx = 1; seg_idx < region->segment_count; ++seg_idx) { + B32 prev_raw = lnk_cobj_segment_is_direct_raw(®ion->directory[seg_idx - 1]); + B32 curr_raw = lnk_cobj_segment_is_direct_raw(®ion->directory[seg_idx]); + if (prev_raw != curr_raw) { + U64 off = (U64)seg_idx * region->segment_size; + if (!VirtualFree(region->base + off, region->reserve_size - off, + MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER)) { + return 0; + } + } + } + LNK_CObjSegment *last = ®ion->directory[region->segment_count - 1]; + if (lnk_cobj_segment_is_direct_raw(last) && last->raw_size < region->segment_size) { + U64 map_end = (U64)(region->segment_count - 1) * region->segment_size + + AlignPow2(last->raw_size, LNK_COBJ_FILE_MAPPING_GRANULARITY); + if (map_end < region->reserve_size && + !VirtualFree(region->base + map_end, region->reserve_size - map_end, + MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER)) { + return 0; + } + } + return 1; +} + +internal B32 +lnk_cobj_isolate_portable_segment(LNK_CompressedObj *region, U32 seg_idx) +{ + LNK_CObjRuntimeSegment *runtime = ®ion->segments[seg_idx]; + AcquireSRWLockExclusive(®ion->placeholder_lock); + B32 ok = 1; + if (!runtime->isolated_placeholder) { + U32 first = seg_idx; + while (first > 0 && !region->segments[first - 1].isolated_placeholder && + !lnk_cobj_segment_is_direct_raw(®ion->directory[first - 1])) { + first -= 1; + } + U32 opl = seg_idx + 1; + while (opl < region->segment_count && !region->segments[opl].isolated_placeholder && + !lnk_cobj_segment_is_direct_raw(®ion->directory[opl])) { + opl += 1; + } + U8 *target = region->base + (U64)seg_idx * region->segment_size; + if (seg_idx > first) { + ok = VirtualFree(target, (U64)(opl - seg_idx) * region->segment_size, + MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER); + } + if (ok && seg_idx + 1 < opl) { + ok = VirtualFree(target + region->segment_size, + (U64)(opl - seg_idx - 1) * region->segment_size, + MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER); + } + if (ok) { runtime->isolated_placeholder = 1; } + } + ReleaseSRWLockExclusive(®ion->placeholder_lock); + return ok; +} + +internal void +lnk_cobj_unmap_portable_raw_runs(LNK_CompressedObj *region) +{ + for (U32 first = 0; first < region->segment_count;) { + LNK_CObjSegment *entry = ®ion->directory[first]; + if (!lnk_cobj_segment_is_direct_raw(entry)) { first += 1; continue; } + U32 opl = first + 1; + while (opl < region->segment_count && + lnk_cobj_segment_is_direct_raw(®ion->directory[opl]) && + region->directory[opl].file_offset == region->directory[opl - 1].file_offset + region->segment_size) { + opl += 1; + } + g_lnk_cobj_cache.unmap_view_of_file_2(GetCurrentProcess(), + region->base + (U64)first * region->segment_size, + MEM_PRESERVE_PLACEHOLDER); + first = opl; + } +} + +internal B32 +lnk_cobj_map_portable_raw_runs(LNK_CompressedObj *region) +{ + for (U32 first = 0; first < region->segment_count;) { + LNK_CObjSegment *entry = ®ion->directory[first]; + if (!lnk_cobj_segment_is_direct_raw(entry)) { first += 1; continue; } + U32 opl = first + 1; + while (opl < region->segment_count && + lnk_cobj_segment_is_direct_raw(®ion->directory[opl]) && + region->directory[opl].file_offset == region->directory[opl - 1].file_offset + region->segment_size) { + opl += 1; + } + U64 map_size = (U64)(opl - first) * region->segment_size; + if (opl == region->segment_count) { + LNK_CObjSegment *last = ®ion->directory[opl - 1]; + if (last->raw_size < region->segment_size) { + map_size -= region->segment_size - AlignPow2(last->raw_size, LNK_COBJ_FILE_MAPPING_GRANULARITY); + } + } + U64 file_off = entry->file_offset; + U8 *target = region->base + (U64)first * region->segment_size; + if ((file_off & (LNK_COBJ_FILE_MAPPING_GRANULARITY - 1)) != 0 || + file_off > region->mapped_file.size || + map_size > region->mapped_file.size - file_off || + g_lnk_cobj_cache.map_view_of_file_3(region->portable_mapping, GetCurrentProcess(), target, + file_off, map_size, MEM_REPLACE_PLACEHOLDER, + PAGE_READONLY, 0, 0) != target) { + lnk_cobj_unmap_portable_raw_runs(region); + return 0; + } + first = opl; + } + return 1; +} + +internal void +lnk_cobj_release_placeholder_reservation(LNK_CompressedObj *region) +{ + LNK_CObjSegment *last = ®ion->directory[region->segment_count - 1]; + B32 split_final_raw = lnk_cobj_segment_is_direct_raw(last) && + last->raw_size < region->segment_size; + if (region->segment_count > 1 || split_final_raw) { + VirtualFree(region->base, region->reserve_size, MEM_RELEASE|MEM_COALESCE_PLACEHOLDERS); + } + VirtualFree(region->base, 0, MEM_RELEASE); +} + +internal void +lnk_cobj_discard_unpublished_region(LNK_CompressedObj *region) +{ + if (!region) { return; } + region->active = 0; + lnk_cobj_unmap_portable_raw_runs(region); + lnk_cobj_release_placeholder_reservation(region); + if (region->portable_mapping) { CloseHandle(region->portable_mapping); } + HeapFree(GetProcessHeap(), 0, region); +} + +//////////////////////////////// +// Container validation and lifetime + +internal B32 +lnk_cobj_open_portable(LNK_Input *input, String8 mapped_file) +{ + if (mapped_file.size < sizeof(LNK_CObjHeader)) { return 0; } + LNK_CObjHeader *header = (LNK_CObjHeader *)mapped_file.str; + if (header->magic != LNK_COBJ_MAGIC) { return 0; } + B32 has_invalid_flag_dependency = + ((header->flags & LNK_COBJ_FLAG_UDT_HASH_INDEX) && + !(header->flags & LNK_COBJ_FLAG_TYPE_INDEX)) || + ((header->flags & LNK_COBJ_FLAG_DEBUG_S_SUMMARY) && + !(header->flags & LNK_COBJ_FLAG_DEBUG_S_INDEX)) || + ((header->flags & LNK_COBJ_FLAG_PACKED_TYPE_OFFSETS_V2) && + !(header->flags & LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR)) || + ((header->flags & LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR) && + !(header->flags & LNK_COBJ_FLAG_PACKED_TYPE_OFFSETS_V2)) || + ((header->flags & LNK_COBJ_TYPE_INDEX_COUNT_MASK) && + !(header->flags & LNK_COBJ_FLAG_TYPE_INDEX)); + if (header->version != LNK_COBJ_VERSION || header->header_size != sizeof(*header) || + header->raw_size == 0 || + header->segment_size < LNK_COBJ_FILE_MAPPING_GRANULARITY || + (header->segment_size & (header->segment_size - 1)) != 0 || + header->raw_size > max_U64 - (header->segment_size - 1) || + header->segment_count != CeilIntegerDiv(header->raw_size, header->segment_size) || + header->directory_offset < header->header_size || header->directory_offset > mapped_file.size || + (header->flags & ~LNK_COBJ_KNOWN_HEADER_FLAGS) != 0 || has_invalid_flag_dependency || + !(header->flags & LNK_COBJ_FLAG_PORTABLE_RAW_MAP) || + (U64)header->segment_count * sizeof(LNK_CObjSegment) > mapped_file.size - header->directory_offset) { + lnk_error(LNK_Error_IllData, "invalid compressed object container: %S", input->path); + return 0; + } + LNK_CObjSegment *directory = (LNK_CObjSegment *)(mapped_file.str + header->directory_offset); + B32 has_portable_raw_segments = 0; + for (U32 i = 0; i < header->segment_count; ++i) { + LNK_CObjSegment *entry = &directory[i]; + U64 segment_raw_offset = (U64)i * header->segment_size; + U32 expected_raw_size = (U32)Min((U64)header->segment_size, + header->raw_size - segment_raw_offset); + if (entry->raw_size != expected_raw_size || entry->stored_size == 0 || + (entry->flags & ~LNK_COBJ_SEGMENT_RAW) != 0 || + ((entry->flags & LNK_COBJ_SEGMENT_RAW) && entry->stored_size != entry->raw_size) || + entry->file_offset > mapped_file.size || entry->stored_size > mapped_file.size - entry->file_offset) { + lnk_error(LNK_Error_IllData, "invalid compressed object segment: %S", input->path); + return 0; + } + has_portable_raw_segments |= !!(entry->flags & LNK_COBJ_SEGMENT_RAW); + } + U32 type_index_count = 0; + LNK_CObjTypeIndex *type_indices = 0; + LNK_CObjUdtHashIndex *udt_hash_indices = 0; + LNK_CObjBaseRelocIndex *base_reloc_index = 0; + LNK_CObjDebugSIndex *debug_s_index = 0; + if (header->flags & LNK_COBJ_FLAG_TYPE_INDEX) { + type_index_count = (header->flags & LNK_COBJ_TYPE_INDEX_COUNT_MASK) >> LNK_COBJ_TYPE_INDEX_COUNT_SHIFT; + U64 dir_bytes = (U64)type_index_count * sizeof(LNK_CObjTypeIndex); + if (type_index_count == 0 || header->reserved > mapped_file.size || dir_bytes > mapped_file.size - header->reserved) { + lnk_error(LNK_Error_IllData, "invalid compressed object type index: %S", input->path); + return 0; + } + type_indices = (LNK_CObjTypeIndex *)(mapped_file.str + header->reserved); + for (U32 i = 0; i < type_index_count; ++i) { + LNK_CObjTypeIndex *idx = &type_indices[i]; + U64 offset_count = idx->leaf_count; + U64 offsets_bytes = offset_count * sizeof(U32); + U64 sizes_bytes = offset_count * sizeof(U16); + U64 kinds_bytes = offset_count * sizeof(U16); + if (header->flags & LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR) { + sizes_bytes = 256 * sizeof(U16); + kinds_bytes = offset_count; + U64 group_size = (U64)1 << LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT; + U64 group_count = (offset_count + group_size - 1) / group_size; + U64 group_bytes = group_count * 2 * sizeof(U32); + U64 payload_rel = (group_bytes + 7) & ~(U64)7; + if (idx->offsets_file_offset > mapped_file.size || group_bytes > mapped_file.size - idx->offsets_file_offset || + idx->sizes_file_offset < idx->offsets_file_offset || + idx->sizes_file_offset - idx->offsets_file_offset < payload_rel) { + lnk_error(LNK_Error_IllData, "invalid packed v2 type offset directory: %S", input->path); + return 0; + } + U64 payload_bytes = idx->sizes_file_offset - idx->offsets_file_offset - payload_rel; + U32 *groups = (U32 *)(mapped_file.str + idx->offsets_file_offset); + for (U64 group_idx = 0; group_idx < group_count; ++group_idx) { + U64 first = group_idx * group_size; + U64 count = Min(group_size, offset_count - first); + U32 descriptor = groups[group_idx*2 + 1]; + U64 rel = descriptor & ~(U32)1; + U64 width = (descriptor & 1) ? 3 : 2; + if (rel > payload_bytes || count * width > payload_bytes - rel) { + lnk_error(LNK_Error_IllData, "invalid packed v2 type offset payload: %S", input->path); + return 0; + } + } + offsets_bytes = idx->sizes_file_offset - idx->offsets_file_offset; + } + if (idx->raw_section_offset > header->raw_size || idx->raw_section_size > header->raw_size - idx->raw_section_offset || + idx->offsets_file_offset > mapped_file.size || offsets_bytes > mapped_file.size - idx->offsets_file_offset || + idx->sizes_file_offset > mapped_file.size || sizes_bytes > mapped_file.size - idx->sizes_file_offset || + idx->kinds_file_offset > mapped_file.size || kinds_bytes > mapped_file.size - idx->kinds_file_offset) { + lnk_error(LNK_Error_IllData, "invalid compressed object type index arrays: %S", input->path); + return 0; + } + } + if (header->flags & LNK_COBJ_FLAG_UDT_HASH_INDEX) { + U64 udt_dir_off = header->reserved + dir_bytes; + U64 udt_dir_bytes = (U64)type_index_count * sizeof(LNK_CObjUdtHashIndex); + if (udt_dir_off > mapped_file.size || udt_dir_bytes > mapped_file.size - udt_dir_off) return 0; + udt_hash_indices = (LNK_CObjUdtHashIndex *)(mapped_file.str + udt_dir_off); + for (U32 i = 0; i < type_index_count; ++i) { + LNK_CObjUdtHashIndex *idx = &udt_hash_indices[i]; + U64 bytes = (U64)idx->hash_count * sizeof(U64); + if (idx->hashes_file_offset > mapped_file.size || bytes > mapped_file.size - idx->hashes_file_offset) return 0; + } + } + } + + if (header->flags & LNK_COBJ_FLAG_BASE_RELOC_INDEX) { + U64 dir_off = header->reserved; + if (header->flags & LNK_COBJ_FLAG_TYPE_INDEX) { + dir_off += (U64)type_index_count * sizeof(LNK_CObjTypeIndex); + if (header->flags & LNK_COBJ_FLAG_UDT_HASH_INDEX) { + dir_off += (U64)type_index_count * sizeof(LNK_CObjUdtHashIndex); + } + } + if (dir_off > mapped_file.size || sizeof(LNK_CObjBaseRelocIndex) > mapped_file.size - dir_off) { + lnk_error(LNK_Error_IllData, "invalid compressed object base relocation index: %S", input->path); + return 0; + } + base_reloc_index = (LNK_CObjBaseRelocIndex *)(mapped_file.str + dir_off); + U64 entry_bytes = (U64)base_reloc_index->entry_count * sizeof(LNK_CObjBaseRelocEntry); + if (base_reloc_index->entries_file_offset > mapped_file.size || + entry_bytes > mapped_file.size - base_reloc_index->entries_file_offset) { + lnk_error(LNK_Error_IllData, "invalid compressed object base relocation entries: %S", input->path); + return 0; + } + } + if (header->flags & LNK_COBJ_FLAG_DEBUG_S_INDEX) { + U64 dir_off = header->reserved; + if (header->flags & LNK_COBJ_FLAG_TYPE_INDEX) { + dir_off += (U64)type_index_count * sizeof(LNK_CObjTypeIndex); + if (header->flags & LNK_COBJ_FLAG_UDT_HASH_INDEX) { + dir_off += (U64)type_index_count * sizeof(LNK_CObjUdtHashIndex); + } + } + if (header->flags & LNK_COBJ_FLAG_BASE_RELOC_INDEX) { dir_off += sizeof(LNK_CObjBaseRelocIndex); } + if (dir_off > mapped_file.size || sizeof(LNK_CObjDebugSIndex) > mapped_file.size - dir_off) { + lnk_error(LNK_Error_IllData, "invalid compressed object .debug$S index: %S", input->path); + return 0; + } + debug_s_index = (LNK_CObjDebugSIndex *)(mapped_file.str + dir_off); + U64 entry_bytes = (U64)debug_s_index->entry_count * sizeof(LNK_CObjDebugSEntry); + if (debug_s_index->entries_file_offset > mapped_file.size || + entry_bytes > mapped_file.size - debug_s_index->entries_file_offset) { + lnk_error(LNK_Error_IllData, "invalid compressed object .debug$S entries: %S", input->path); + return 0; + } + LNK_CObjDebugSEntry *entries = (LNK_CObjDebugSEntry *)(mapped_file.str + debug_s_index->entries_file_offset); + for (U32 i = 0; i < debug_s_index->entry_count; ++i) { + if ((U64)entries[i].raw_payload_offset + entries[i].raw_payload_size > header->raw_size || + (i && entries[i].raw_section_offset < entries[i-1].raw_section_offset)) { + lnk_error(LNK_Error_IllData, "invalid compressed object .debug$S entry: %S", input->path); + return 0; + } + } + if (header->flags & LNK_COBJ_FLAG_DEBUG_S_SUMMARY) { + U64 summary_off = AlignPow2(debug_s_index->entries_file_offset + entry_bytes, 8); + U64 summary_bytes = (U64)debug_s_index->entry_count * sizeof(LNK_CObjDebugSSummary); + if (summary_off > mapped_file.size || summary_bytes > mapped_file.size - summary_off) { + lnk_error(LNK_Error_IllData, "invalid compressed object .debug$S summaries: %S", input->path); + return 0; + } + } + } + if (!lnk_cobj_cache_init(header->segment_size)) { + lnk_error(LNK_Error_IllData, "compressed object cache initialization failed"); + return 0; + } + + HANDLE portable_mapping = 0; + if (has_portable_raw_segments) { + Temp scratch = scratch_begin(0, 0); + String16 path16 = str16_from_8(scratch.arena, input->path); + HANDLE file = CreateFileW(path16.str, GENERIC_READ, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + if (file != INVALID_HANDLE_VALUE) { + // Match ordinary OBJ mapping semantics: expose raw runs through a read-only view of a + // WRITECOPY section. Untouched pages stay file-backed without up-front commit, while the + // generic COW VEH can privately promote the rare page patched in place by the linker. + portable_mapping = CreateFileMappingW(file, 0, PAGE_WRITECOPY, 0, 0, 0); + CloseHandle(file); + } + scratch_end(scratch); + } + // A fully compressed container is self-contained in mapped_file and deliberately needs no + // second handle to its path. Only raw-stored segments require a WRITECOPY section that can be + // mapped at their logical OBJ offsets. + if (has_portable_raw_segments && !portable_mapping) { + lnk_error(LNK_Error_IllData, "unable to open portable compressed object mapping: %S", input->path); + return 0; + } + + U64 reserve_size = (header->raw_size + (U64)header->segment_size - 1) & ~((U64)header->segment_size - 1); + U8 *base = g_lnk_cobj_cache.virtual_alloc_2(GetCurrentProcess(), 0, reserve_size, + MEM_RESERVE|MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, 0, 0); + if (!base) { + if (portable_mapping) { CloseHandle(portable_mapping); } + lnk_error(LNK_Error_IllData, "unable to reserve compressed object view: %S", input->path); + return 0; + } + + // These descriptors need stable addresses, not independent VADs. One heap block per OBJ avoids + // two VirtualAlloc VADs and also halves allocator calls versus separate descriptor/table blocks. + U64 segments_off = AlignPow2(sizeof(LNK_CompressedObj), 64); + U64 metadata_size = segments_off + sizeof(LNK_CObjRuntimeSegment) * header->segment_count; + LNK_CompressedObj *region = HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, metadata_size); + LNK_CObjRuntimeSegment *segments = region ? (LNK_CObjRuntimeSegment *)((U8 *)region + segments_off) : 0; + if (!region) { + VirtualFree(base, 0, MEM_RELEASE); + if (portable_mapping) { CloseHandle(portable_mapping); } + lnk_error(LNK_Error_IllData, "unable to allocate compressed object metadata: %S", input->path); + return 0; + } + for (U32 i = 0; i < header->segment_count; ++i) { + InitializeSRWLock(&segments[i].lock); + segments[i].slot_idx = max_U32; + segments[i].address = base + (U64)i * header->segment_size; + } + region->base = base; + region->reserve_size = reserve_size; + region->raw_size = header->raw_size; + region->segment_size = header->segment_size; + region->segment_count = header->segment_count; + region->mapped_file = mapped_file; + region->header = header; + region->directory = directory; + region->type_indices = type_indices; + region->udt_hash_indices = udt_hash_indices; + region->segments = segments; + region->portable_mapping = portable_mapping; + if (base_reloc_index) { + region->base_reloc_entries = (LNK_CObjBaseRelocEntry *)(mapped_file.str + base_reloc_index->entries_file_offset); + region->base_reloc_entry_count = base_reloc_index->entry_count; + region->has_base_reloc_index = 1; + } + if (debug_s_index) { + region->debug_s_entries = (LNK_CObjDebugSEntry *)(mapped_file.str + debug_s_index->entries_file_offset); + region->debug_s_entry_count = debug_s_index->entry_count; + if (header->flags & LNK_COBJ_FLAG_DEBUG_S_SUMMARY) { + U64 bytes = (U64)debug_s_index->entry_count * sizeof(LNK_CObjDebugSEntry); + U64 off = AlignPow2(debug_s_index->entries_file_offset + bytes, 8); + region->debug_s_summaries = (LNK_CObjDebugSSummary *)(mapped_file.str + off); + } + } + region->type_index_count = type_index_count; + region->active = 1; + + B32 placeholders_ok = lnk_cobj_split_portable_raw_boundaries(region) && + lnk_cobj_map_portable_raw_runs(region); + if (!placeholders_ok) { + lnk_cobj_discard_unpublished_region(region); + lnk_error(LNK_Error_IllData, "unable to map portable compressed object view: %S", input->path); + return 0; + } + + B32 registered = 0; + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + if (g_lnk_cobj_cache.region_count < g_lnk_cobj_cache.region_cap) { + g_lnk_cobj_cache.regions[g_lnk_cobj_cache.region_count++] = region; + registered = 1; + } + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + if (!registered) { + lnk_cobj_discard_unpublished_region(region); + lnk_error(LNK_Error_IllData, "too many compressed object inputs"); + return 0; + } + + input->compressed_obj = region; + input->compressed_data = mapped_file; + input->data = str8(base, header->raw_size); + return 1; +} + +internal B32 +lnk_compressed_obj_open(LNK_Input *input, String8 mapped_file) +{ + return lnk_cobj_open_portable(input, mapped_file); +} + +internal void +lnk_compressed_obj_finalize_open(void) +{ + // Parallel input mapping appends regions in completion order. The caller invokes this joined + // boundary before starting any COFF parser task, so no logical compressed range can fault until + // the address table has been sorted for lock-free VEH lookup. + if (!g_lnk_cobj_cache.initialized || g_lnk_cobj_cache.region_count < 2) { return; } + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + qsort(g_lnk_cobj_cache.regions, g_lnk_cobj_cache.region_count, + sizeof(*g_lnk_cobj_cache.regions), lnk_cobj_region_ptr_compare); + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); +} + +internal void +lnk_compressed_obj_close(LNK_Input *input) +{ + LNK_CompressedObj *region = input->compressed_obj; + if (!region) { return; } + if (g_lnk_cobj_cache.skip_cleanup) { + // Benchmark/diagnostic mode: the process is about to exit and Windows tears the address + // space down more efficiently than issuing one UnmapViewOfFile2 call per resident segment. + // Useful for separating link work from explicit VAD teardown; not the bounded-cache path. + input->owns_file_map = 0; + return; + } + InterlockedExchange(®ion->active, 0); + lnk_cobj_unmap_portable_raw_runs(region); + for (U32 i = 0; i < region->segment_count; ++i) { + LNK_CObjRuntimeSegment *segment = ®ion->segments[i]; + if (segment->state == LNK_CObjSegState_Ready || segment->state == LNK_CObjSegState_Pinned) { + g_lnk_cobj_cache.unmap_view_of_file_2(GetCurrentProcess(), region->base + (U64)i * region->segment_size, + MEM_PRESERVE_PLACEHOLDER); + if (segment->slot_idx != max_U32 && segment->slot_idx < g_lnk_cobj_cache.slot_count) { + AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + if (g_lnk_cobj_cache.slot_segments[segment->slot_idx] == segment) { + g_lnk_cobj_cache.slot_segments[segment->slot_idx] = 0; + // Destructive shrinking can leave pinned slots above the active prefix. They remain + // mapped for pointer stability but are intentionally excluded from resident_count. + if (segment->slot_idx < g_lnk_cobj_cache.active_slot_count) { + Assert(g_lnk_cobj_cache.resident_count > 0); + g_lnk_cobj_cache.resident_count -= 1; + } + } + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + } + } + } + lnk_cobj_release_placeholder_reservation(region); + if (region->portable_mapping) { CloseHandle(region->portable_mapping); } + file_map_view_close((FileMap){0}, input->compressed_data.str, r1u64(0, input->compressed_data.size)); + // The sorted address lookup retains region pointers for the life of the process. Keep this + // small inactive descriptor allocated so an unrelated later access violation cannot make the + // vectored exception handler dereference freed metadata. + region->mapped_file = str8_zero(); + region->header = 0; + region->directory = 0; + region->type_indices = 0; + region->udt_hash_indices = 0; + region->base_reloc_entries = 0; + region->debug_s_entries = 0; + region->debug_s_summaries = 0; + region->portable_mapping = 0; + input->data = str8_zero(); + input->compressed_data = str8_zero(); + input->compressed_obj = 0; + input->owns_file_map = 0; +} + +//////////////////////////////// +// Statistics + +internal void +lnk_compressed_obj_log_stats(void) +{ + if (!g_lnk_cobj_cache.initialized || g_lnk_cobj_cache.region_count == 0) { + if (g_lnk_cobj_window_decodes) { lnk_log(LNK_Log_Timers, "[cobj window] decodes=%llu stored=%llu MiB", g_lnk_cobj_window_decodes, g_lnk_cobj_window_bytes / MB(1)); } + return; + } + U64 unique_decodes = g_lnk_cobj_cache.decodes - g_lnk_cobj_cache.redecodes; + U64 amp_x100 = unique_decodes ? (g_lnk_cobj_cache.decodes * 100) / unique_decodes : 0; + lnk_log(LNK_Log_Timers, + "[cobj] regions=%llu segment=%llu KiB cache=%llu MiB resident=%llu faults=%llu decodes=%llu unique=%llu redecodes=%llu amp=%llu.%02llux evictions=%llu decoded=%llu MiB stored=%llu MiB", + g_lnk_cobj_cache.region_count, + g_lnk_cobj_cache.segment_size / KB(1), + (g_lnk_cobj_cache.slot_count * g_lnk_cobj_cache.segment_size) / MB(1), + g_lnk_cobj_cache.occupancy_hwm, g_lnk_cobj_cache.faults, g_lnk_cobj_cache.decodes, + unique_decodes, g_lnk_cobj_cache.redecodes, amp_x100 / 100, amp_x100 % 100, + g_lnk_cobj_cache.evictions, + g_lnk_cobj_cache.decoded_bytes / MB(1), g_lnk_cobj_cache.stored_bytes_read / MB(1)); + if (g_lnk_cobj_cache.frozen_segments) { + lnk_log(LNK_Log_Timers, "[cobj frozen] segments=%llu", g_lnk_cobj_cache.frozen_segments); + } + if (g_lnk_cobj_cache.redecodes) { + lnk_log(LNK_Log_Timers, "[cobj redecodes] debug_s=%llu types=%llu segment0=%llu", + g_lnk_cobj_redecode_debug_s, g_lnk_cobj_redecode_types, + g_lnk_cobj_redecode_segment_zero); + } + if (g_lnk_cobj_window_decodes) { + lnk_log(LNK_Log_Timers, "[cobj window] decodes=%llu stored=%llu MiB", + g_lnk_cobj_window_decodes, g_lnk_cobj_window_bytes / MB(1)); + } +} + +#else +internal void lnk_compressed_obj_trim_working_set(void) {} +internal String8 lnk_compressed_obj_direct_range(LNK_CompressedObj *obj, Rng1U64 range) { return str8_zero(); } +internal B32 lnk_compressed_obj_type_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjTypeIndexView *out) { MemoryZeroStruct(out); return 0; } +internal B32 lnk_compressed_obj_base_reloc_index(LNK_CompressedObj *obj, LNK_CObjBaseRelocView *out) { MemoryZeroStruct(out); return 0; } +internal B32 lnk_compressed_obj_debug_s_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjDebugSView *out) { MemoryZeroStruct(out); return 0; } +internal B32 lnk_compressed_obj_copy_range(LNK_CompressedObj *obj, Rng1U64 range, void *dst, LNK_CObjDecodeWindow *window) { return 0; } +internal B32 lnk_compressed_obj_copy_string(LNK_CompressedObj *obj, String8 src, void *dst, LNK_CObjDecodeWindow *window) { return 0; } +internal void lnk_compressed_obj_release_window(LNK_CObjDecodeWindow *window) {} +internal U32 lnk_compressed_obj_segment_count(LNK_CompressedObj *obj) { return 0; } +internal U32 lnk_compressed_obj_segment_size(LNK_CompressedObj *obj) { return 0; } +internal Rng1U64 lnk_compressed_obj_stored_segment_range(LNK_CompressedObj *obj, U32 segment_idx) { Rng1U64 r = {0}; return r; } +internal B32 +lnk_compressed_obj_open(struct LNK_Input *input, String8 mapped_file) +{ + if (mapped_file.size >= sizeof(LNK_CObjHeader) && + ((LNK_CObjHeader *)mapped_file.str)->magic == LNK_COBJ_MAGIC) { + lnk_error(LNK_Error_IllData, + "compressed object input requires an Oodle-enabled RAD Link build: %S", input->path); + } + return 0; +} +internal void lnk_compressed_obj_finalize_open(void) {} +internal void lnk_compressed_obj_close(struct LNK_Input *input) {} +internal void lnk_compressed_obj_log_stats(void) {} +internal void lnk_compressed_obj_log_phase_stats(char *tag) {} +#endif diff --git a/src/linker/lnk_compressed_obj.h b/src/linker/lnk_compressed_obj.h new file mode 100644 index 000000000..e19aa80da --- /dev/null +++ b/src/linker/lnk_compressed_obj.h @@ -0,0 +1,67 @@ +#pragma once + +// The portable container layout is codec-independent and is also needed by the no-Oodle stubs. +// Oodle headers and calls remain private to the enabled branch in lnk_compressed_obj.c. +#include "lnk_compressed_obj_format.h" + +typedef struct LNK_CompressedObj LNK_CompressedObj; +typedef struct LNK_CObjBaseRelocEntry LNK_CObjBaseRelocEntry; +typedef struct LNK_CObjDebugSEntry LNK_CObjDebugSEntry; +typedef struct LNK_CObjDebugSSummary LNK_CObjDebugSSummary; + +typedef struct LNK_CObjTypeIndexView +{ + U64 count; + U32 *offsets; + U16 *sizes; + U16 *kinds; + U32 *packed_v2_offset_groups; + U8 *packed_v2_offset_payload; + U16 *packed_kind_dictionary; + U8 *packed_kind_codes; + U64 *complete_udt_hashes; + U64 complete_udt_hash_count; + U8 offset_checkpoint_shift; + B8 packed_sidecar; +} LNK_CObjTypeIndexView; + +typedef struct LNK_CObjBaseRelocView +{ + U32 count; + LNK_CObjBaseRelocEntry *v; +} LNK_CObjBaseRelocView; + +typedef struct LNK_CObjDebugSView +{ + U32 count; + LNK_CObjDebugSEntry *v; + LNK_CObjDebugSSummary *summaries; +} LNK_CObjDebugSView; + +typedef struct LNK_CObjDecodeWindow LNK_CObjDecodeWindow; +struct LNK_CObjDecodeWindow +{ + LNK_CompressedObj *obj; + U8 *buffer; + U64 buffer_cap; + U32 seg_idx; + U32 valid_size; +}; + + +internal void lnk_compressed_obj_trim_working_set(void); +internal String8 lnk_compressed_obj_direct_range(LNK_CompressedObj *obj, Rng1U64 range); +internal B32 lnk_compressed_obj_type_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjTypeIndexView *out); +internal B32 lnk_compressed_obj_base_reloc_index(LNK_CompressedObj *obj, LNK_CObjBaseRelocView *out); +internal B32 lnk_compressed_obj_debug_s_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjDebugSView *out); +internal B32 lnk_compressed_obj_copy_range(LNK_CompressedObj *obj, Rng1U64 range, void *dst, LNK_CObjDecodeWindow *window); +internal B32 lnk_compressed_obj_copy_string(LNK_CompressedObj *obj, String8 src, void *dst, LNK_CObjDecodeWindow *window); +internal void lnk_compressed_obj_release_window(LNK_CObjDecodeWindow *window); +internal U32 lnk_compressed_obj_segment_count(LNK_CompressedObj *obj); +internal U32 lnk_compressed_obj_segment_size(LNK_CompressedObj *obj); +internal Rng1U64 lnk_compressed_obj_stored_segment_range(LNK_CompressedObj *obj, U32 segment_idx); +internal B32 lnk_compressed_obj_open(struct LNK_Input *input, String8 mapped_file); +internal void lnk_compressed_obj_finalize_open(void); +internal void lnk_compressed_obj_close(struct LNK_Input *input); +internal void lnk_compressed_obj_log_stats(void); +internal void lnk_compressed_obj_log_phase_stats(char *tag); diff --git a/src/linker/lnk_compressed_obj_format.h b/src/linker/lnk_compressed_obj_format.h new file mode 100644 index 000000000..b76b2c3fc --- /dev/null +++ b/src/linker/lnk_compressed_obj_format.h @@ -0,0 +1,124 @@ +// Portable segmented-object format shared by the writer and RAD Link reader. +// Files are self-contained, all integers are little endian, and each compressed +// segment is independently decodable. + +#pragma once + +#define LNK_COBJ_MAGIC 0x3130304A424F4C52ull /* "RLOBJ001" */ +#define LNK_COBJ_VERSION 1u +#define LNK_COBJ_SEGMENT_RAW 0x00000001u +#define LNK_COBJ_FLAG_TYPE_INDEX 0x00000001u +#define LNK_COBJ_FLAG_RETIRED_SPARSE_RAW_VIEW 0x00000002u // Reserved; never accepted by the portable reader. +#define LNK_COBJ_FLAG_UDT_HASH_INDEX 0x00000004u +#define LNK_COBJ_FLAG_BASE_RELOC_INDEX 0x00000008u +#define LNK_COBJ_FLAG_PORTABLE_RAW_MAP 0x00000010u +#define LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR 0x00000040u +#define LNK_COBJ_FLAG_PACKED_TYPE_OFFSETS_V2 0x00000080u +#define LNK_COBJ_FLAG_DEBUG_S_INDEX 0x00002000u +#define LNK_COBJ_FLAG_DEBUG_S_SUMMARY 0x00004000u +#define LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT 9u +#define LNK_COBJ_TYPE_INDEX_COUNT_SHIFT 16u +#define LNK_COBJ_TYPE_INDEX_COUNT_MASK 0xffff0000u + +typedef struct LNK_CObjHeader +{ + U64 magic; + U32 version; + U32 header_size; + U64 raw_size; + U32 segment_size; + U32 segment_count; + U64 directory_offset; + U64 data_offset; + U32 compressor; + U32 flags; + U64 reserved; +} LNK_CObjHeader; + +typedef struct LNK_CObjSegment +{ + U64 file_offset; + U32 stored_size; + U32 raw_size; + U32 flags; + U32 reserved; +} LNK_CObjSegment; + +// Optional leaf index for .debug$T/.debug$P sections. The three arrays are +// deliberately stored separately: type dedup scans only kinds, while offsets +// and sizes are touched only for the winning leaves. All offsets are file +// offsets in the compressed container; leaf offsets are relative to the first +// byte after the four-byte CodeView signature. +typedef struct LNK_CObjTypeIndex +{ + U64 raw_section_offset; + U32 raw_section_size; + U32 leaf_count; + // Normal layout: U32 offsets, U16 sizes, U16 kinds. + // PACKED_TYPE_SIDECAR: + // U32 group[ceil(leaf_count/512)][2] = {absolute_base, payload_offset|is_u24}; + // followed by U16 deltas for normal groups and U24 deltas for rare wide groups; + // sizes_file_offset -> U16 kind_dictionary[256] (sizes derive from adjacent offsets); + // kinds_file_offset -> U8 kind_code[leaf_count]. + U64 offsets_file_offset; + U64 sizes_file_offset; + U64 kinds_file_offset; +} LNK_CObjTypeIndex; + +typedef struct LNK_CObjUdtHashIndex +{ + U64 hashes_file_offset; + U32 hash_count; + U32 reserved; +} LNK_CObjUdtHashIndex; + +// Compact candidates for PE base relocation generation. The compressor filters the full COFF +// relocation streams down to address relocations; RAD Link still applies section-liveness and +// symbol-interpretation checks because those depend on the final link. +typedef struct LNK_CObjBaseRelocIndex +{ + U64 entries_file_offset; + U32 entry_count; + U32 reserved; +} LNK_CObjBaseRelocIndex; + +typedef struct LNK_CObjBaseRelocEntry +{ + U32 sect_idx; + U32 apply_off; + U32 isymbol; + U8 addr_size; + U8 reserved[3]; +} LNK_CObjBaseRelocEntry; + +// Compact directory for C13 subsection payloads. Parsing .debug$S normally walks every +// subsection header in the logical COFF view, which needlessly decodes all compressed symbol +// segments just to construct String8 slices. Entries are grouped by raw_section_offset and let +// the linker construct those slices without touching payload pages. COFF section file offsets +// are U32 by definition, so the complete entry stays at 16 bytes. +typedef struct LNK_CObjDebugSIndex +{ + U64 entries_file_offset; + U32 entry_count; + U32 reserved; +} LNK_CObjDebugSIndex; + +typedef struct LNK_CObjDebugSEntry +{ + U32 raw_section_offset; + U32 raw_payload_offset; + U32 raw_payload_size; + U32 kind; +} LNK_CObjDebugSEntry; + +// Size-only module/GSI prepass results for the corresponding DEBUG_S_INDEX entry. These values +// depend only on symbol size/kind fields, which relocation and type-index fixups never alter. +typedef struct LNK_CObjDebugSSummary +{ + U32 module_symbol_size; + U32 gsi_candidate_count; + U32 proc_ref_count; + U32 flags; +} LNK_CObjDebugSSummary; + +#define LNK_COBJ_DEBUG_S_SUMMARY_HAS_LOCALS 0x1u From 1216958e5b7d8de91f1b57631b9b8f0b04696c3f Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Sun, 16 Aug 2026 09:55:59 -0700 Subject: [PATCH 109/121] radlink: consume indexed debug data from compressed OBJs --- src/linker/codeview_ext/codeview.c | 66 ++- src/linker/codeview_ext/codeview.h | 21 + src/linker/lnk.c | 241 ++++++++-- src/linker/lnk.h | 3 + src/linker/lnk_debug_info.c | 740 +++++++++++++++++++++++++---- src/linker/lnk_debug_info.h | 19 +- src/linker/lnk_io.c | 49 +- src/linker/lnk_obj.c | 122 ++++- src/linker/lnk_obj.h | 2 + src/linker/lnk_symbol_table.c | 3 +- src/linker/pdb_ext/pdb_builder.c | 2 + 11 files changed, 1090 insertions(+), 178 deletions(-) diff --git a/src/linker/codeview_ext/codeview.c b/src/linker/codeview_ext/codeview.c index afcba22b5..c981210d6 100644 --- a/src/linker/codeview_ext/codeview.c +++ b/src/linker/codeview_ext/codeview.c @@ -574,6 +574,10 @@ cv_debug_s_prov_list_push(Arena *arena, CV_DebugSProvList *list, U64 off, U64 si node->size = size; node->sect_idx = sect_idx; node->is_synthetic = is_synthetic; + node->module_symbol_size = 0; + node->gsi_candidate_count = 0; + node->proc_ref_count = 0; + node->symbol_summary_valid = 0; cv_debug_s_prov_list_push_node(list, node); } @@ -1284,13 +1288,32 @@ cv_debug_t_from_data(Arena *arena, String8 data, U64 align) return debug_t; } +internal U64 +cv_debug_t_get_leaf_offset(CV_DebugT *debug_t, U64 leaf_idx) +{ + Assert(leaf_idx < debug_t->count); + U64 actual_idx = leaf_idx + debug_t->sidecar_leaf_bias; + if (debug_t->sidecar_packed) { + U64 group_idx = actual_idx >> debug_t->sidecar_offset_checkpoint_shift; + U64 within = actual_idx & (((U64)1 << debug_t->sidecar_offset_checkpoint_shift) - 1); + U32 base = debug_t->sidecar_packed_v2_offset_groups[group_idx*2 + 0]; + U32 descriptor = debug_t->sidecar_packed_v2_offset_groups[group_idx*2 + 1]; + U32 width = (descriptor & 1) ? 3 : 2; + U8 *p = debug_t->sidecar_packed_v2_offset_payload + (descriptor & ~(U32)1) + within * width; + U32 delta = (U32)p[0] | ((U32)p[1] << 8); + if (width == 3) { delta |= (U32)p[2] << 16; } + return (U64)base + delta; + } + return debug_t->offsets[actual_idx]; +} + internal CV_Leaf cv_debug_t_get_leaf(CV_DebugT *debug_t, U64 leaf_idx) { CV_Leaf leaf = {0}; if (debug_t->count > 0) { Assert(leaf_idx < debug_t->count); - cv_read_leaf(debug_t->data, debug_t->offsets[leaf_idx], 1, &leaf); + cv_read_leaf(debug_t->data, cv_debug_t_get_leaf_offset(debug_t, leaf_idx), 1, &leaf); Assert(cv_header_struct_size_from_leaf_kind(leaf.kind) <= leaf.data.size); } return leaf; @@ -1339,9 +1362,9 @@ internal String8 cv_debug_t_get_raw_leaf(CV_DebugT *debug_t, U64 leaf_idx) { Assert(leaf_idx < debug_t->count); - U8 *leaf_ptr = debug_t->data.str + debug_t->offsets[leaf_idx]; - CV_LeafSize leaf_size = memory_read16(leaf_ptr); - return str8(leaf_ptr, leaf_size + sizeof(leaf_size)); + U8 *leaf_ptr = debug_t->data.str + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + U64 leaf_size = cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx); + return str8(leaf_ptr, leaf_size); } internal CV_LeafHeader * @@ -1349,11 +1372,40 @@ cv_debug_t_get_leaf_header(CV_DebugT *debug_t, U64 leaf_idx) { CV_LeafHeader *header = 0; if (leaf_idx < debug_t->count) { - header = (CV_LeafHeader *)(debug_t->data.str + debug_t->offsets[leaf_idx]); + header = (CV_LeafHeader *)(debug_t->data.str + cv_debug_t_get_leaf_offset(debug_t, leaf_idx)); } return header; } +internal CV_LeafKind +cv_debug_t_get_leaf_kind(CV_DebugT *debug_t, U64 leaf_idx) +{ + Assert(leaf_idx < debug_t->count); + U64 actual_idx = leaf_idx + debug_t->sidecar_leaf_bias; + if (debug_t->sidecar_packed_kind_codes) { + return debug_t->sidecar_packed_kind_dictionary[debug_t->sidecar_packed_kind_codes[actual_idx]]; + } + if (debug_t->sidecar_kinds) { return debug_t->sidecar_kinds[actual_idx]; } + return cv_debug_t_get_leaf_header(debug_t, leaf_idx)->kind; +} + +internal U64 +cv_debug_t_get_raw_leaf_size(CV_DebugT *debug_t, U64 leaf_idx) +{ + Assert(leaf_idx < debug_t->count); + U64 actual_idx = leaf_idx + debug_t->sidecar_leaf_bias; + if (debug_t->sidecar_packed) { + U64 offset = cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + U64 total_count = debug_t->count + debug_t->sidecar_leaf_bias; + U64 next_offset = actual_idx + 1 < total_count ? cv_debug_t_get_leaf_offset(debug_t, leaf_idx + 1) : debug_t->data.size; + Assert(next_offset >= offset); + return next_offset - offset; + } + if (debug_t->sidecar_sizes) { return (U64)debug_t->sidecar_sizes[actual_idx] + sizeof(CV_LeafSize); } + U8 *leaf_ptr = debug_t->data.str + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + return (U64)memory_read16(leaf_ptr) + sizeof(CV_LeafSize); +} + internal CV_TypeIndex cv_debug_t_get_type_index(CV_DebugT *debug_t, CV_TypeIndexSource ti_source, U64 leaf_idx) { @@ -1373,13 +1425,13 @@ cv_debug_t_get_leaf_index(CV_DebugT *debug_t, CV_TypeIndexSource ti_source, CV_T internal B32 cv_debug_t_is_pch(CV_DebugT *debug_t) { - return cv_is_leaf_pch(cv_debug_t_get_leaf(debug_t, 0).kind); + return debug_t->count && cv_is_leaf_pch(cv_debug_t_get_leaf_kind(debug_t, 0)); } internal B32 cv_debug_t_is_type_server_ref(CV_DebugT *debug_t) { - return cv_is_leaf_type_server(cv_debug_t_get_leaf(debug_t, 0).kind); + return debug_t->count && cv_is_leaf_type_server(cv_debug_t_get_leaf_kind(debug_t, 0)); } // $$Symbols diff --git a/src/linker/codeview_ext/codeview.h b/src/linker/codeview_ext/codeview.h index 75991b58f..22a42c830 100644 --- a/src/linker/codeview_ext/codeview.h +++ b/src/linker/codeview_ext/codeview.h @@ -236,6 +236,10 @@ typedef struct CV_DebugSProvNode U64 size; U32 sect_idx; B32 is_synthetic; + U32 module_symbol_size; + U32 gsi_candidate_count; + U32 proc_ref_count; + B32 symbol_summary_valid; } CV_DebugSProvNode; typedef struct CV_DebugSProvList @@ -262,6 +266,20 @@ typedef struct CV_DebugT String8 data; U64 count; U32 *offsets; + // Optional container sidecar. Keeping these as mapped arrays avoids both + // scanning compressed leaf bodies and privately allocating the index. + U16 *sidecar_sizes; + U16 *sidecar_kinds; + U32 *sidecar_packed_v2_offset_groups; + U8 *sidecar_packed_v2_offset_payload; + U16 *sidecar_packed_kind_dictionary; + U8 *sidecar_packed_kind_codes; + B8 sidecar_packed; + U8 sidecar_offset_checkpoint_shift; + U32 sidecar_leaf_bias; + U64 sidecar_raw_base; + U64 *sidecar_complete_udt_hashes; + U64 sidecar_complete_udt_hash_count; // type server U64 source_counts [CV_TypeIndexSource_COUNT]; @@ -435,9 +453,12 @@ internal CV_DebugT cv_debug_t_from_data (Arena *arena, String8 dat internal U64 cv_leaf_idx_from_ti (CV_DebugT *debug_t, CV_TypeIndexSource source, CV_TypeIndex ti); internal CV_TypeIndex cv_ti_from_leaf_idx (CV_DebugT *debug_t, CV_TypeIndexSource source, U64 leaf_idx); internal CV_Leaf cv_debug_t_get_leaf (CV_DebugT *debug_t, U64 leaf_idx); +internal U64 cv_debug_t_get_leaf_offset (CV_DebugT *debug_t, U64 leaf_idx); internal CV_Leaf cv_debug_t_get_leaf_from_ti (CV_DebugT *debug_t, CV_TypeIndexSource source, CV_TypeIndex ti); internal String8 cv_debug_t_get_raw_leaf (CV_DebugT *debug_t, U64 leaf_idx); internal CV_LeafHeader * cv_debug_t_get_leaf_header (CV_DebugT *debug_t, U64 leaf_idx); +internal CV_LeafKind cv_debug_t_get_leaf_kind (CV_DebugT *debug_t, U64 leaf_idx); +internal U64 cv_debug_t_get_raw_leaf_size (CV_DebugT *debug_t, U64 leaf_idx); internal CV_TypeIndex cv_debug_t_get_type_index (CV_DebugT *debug_t, CV_TypeIndexSource ti_source, U64 leaf_idx); internal U64 cv_debug_t_get_leaf_index (CV_DebugT *debug_t, CV_TypeIndexSource ti_source, CV_TypeIndex ti); internal B32 cv_debug_t_is_pch (CV_DebugT *debug_t); diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 6fcd75606..df6ca60f7 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -120,12 +120,14 @@ #include "lnk_lib.h" #include "codeview_ext/ifc.h" #include "lnk_debug_info.h" +#include "lnk_compressed_obj.h" #include "lnk.h" #include "lnk_log.c" #include "lnk_timer.c" #include "lnk_hasher.c" #include "lnk_io.c" +#include "lnk_compressed_obj.c" #include "lnk_cmd_line.c" #include "lnk_config.c" #include "lnk_symbol_table.c" @@ -990,6 +992,49 @@ lnk_inputer_has_items(LNK_Inputer *inputer) return 0; } +typedef struct LNK_InputOpenTask +{ + LNK_Input **inputs; + String8 *datas; +} LNK_InputOpenTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_input_open_task) +{ + LNK_InputOpenTask *task = raw_task; + LNK_Input *input = task->inputs[task_id]; + String8 data = task->datas[task_id]; + if (!input->has_disk_read_failed) { + lnk_compressed_obj_open(input, data); + } +} + +typedef struct LNK_InputClassifyTask +{ + String8 *datas; + U8 *tags; +} LNK_InputClassifyTask; + +enum +{ + LNK_InputClassify_PortableCompressed = (1 << 0), +}; + +internal +THREAD_POOL_TASK_FUNC(lnk_input_classify_task) +{ + LNK_InputClassifyTask *task = raw_task; + String8 data = task->datas[task_id]; + + U8 tag = 0; + if (data.size >= sizeof(LNK_CObjHeader) && ((LNK_CObjHeader *)data.str)->magic == LNK_COBJ_MAGIC) { + if (((LNK_CObjHeader *)data.str)->flags & LNK_COBJ_FLAG_PORTABLE_RAW_MAP) { + tag |= LNK_InputClassify_PortableCompressed; + } + } + task->tags[task_id] = tag; +} + internal LNK_InputPtrArray lnk_inputer_flush(Arena *arena, TP_Context *tp, LNK_Inputer *inputer, LNK_IO_Flags io_flags, LNK_InputList *all_inputs, LNK_InputList *new_inputs) { @@ -1020,7 +1065,6 @@ lnk_inputer_flush(Arena *arena, TP_Context *tp, LNK_Inputer *inputer, LNK_IO_Fla ProfEnd(); ProfBegin("Load Inputs From Disk"); - B8 *thin_input_was_read = push_array(scratch.arena, B8, thin_input_paths.count); String8Array thin_input_datas = lnk_read_data_from_file_path_parallel(tp, inputer->arena, io_flags, thin_input_paths, thin_input_was_read); B32 is_mapped = !!(io_flags & (LNK_IO_Flags_MemoryMapFilesReadWrite|LNK_IO_Flags_MemoryMapFilesReadOnly)); @@ -1031,6 +1075,29 @@ lnk_inputer_flush(Arena *arena, TP_Context *tp, LNK_Inputer *inputer, LNK_IO_Fla thin_inputs[thin_input_idx]->data = thin_input_datas.v[thin_input_idx]; } + // Keep the ordinary-object path as direct mmap followed by cheap first-page classification. + // Portable compressed objects have independent metadata and can be opened in parallel. + B32 has_portable_cobj = 0; + U8 *classify_tags = push_array_no_zero(scratch.arena, U8, thin_inputs_count); + LNK_InputClassifyTask classify_task = {thin_input_datas.v, classify_tags}; + if (thin_inputs_count >= 64) { + lnk_tp_for_parallel_capped(tp, 0, 32, thin_inputs_count, lnk_input_classify_task, &classify_task); + } else { + for EachIndex(i, thin_inputs_count) { lnk_input_classify_task(0, 0, i, &classify_task, tp); } + } + for EachIndex(thin_input_idx, thin_inputs_count) { + has_portable_cobj |= !!(classify_tags[thin_input_idx] & LNK_InputClassify_PortableCompressed); + } + if (has_portable_cobj) { + LNK_InputOpenTask open_task = {thin_inputs, thin_input_datas.v}; + // Reserving/splitting thousands of logical OBJ views contends on the Windows process VAD + // lock. Four lanes measured substantially faster than 8-64 on the UEFN corpus. + lnk_tp_for_parallel_capped(tp, 0, 4, thin_inputs_count, lnk_input_open_task, &open_task); + } + // Compressed inputs register their reserved address regions independently while the open tasks + // run. Sort once after all tasks have joined; sorting after every insertion serialized the + // parallel open and performed thousands of increasingly large qsorts. + lnk_compressed_obj_finalize_open(); ProfEnd(); ProfBegin("Disk Read Check"); @@ -1055,9 +1122,13 @@ THREAD_POOL_TASK_FUNC(lnk_release_file_map_task) { LNK_Input **mapped_inputs = raw_task; LNK_Input *input = mapped_inputs[task_id]; - file_map_view_close((FileMap){0}, input->data.str, r1u64(0, input->data.size)); - input->data = str8_zero(); - input->owns_file_map = 0; + if (input->compressed_obj) { + lnk_compressed_obj_close(input); + } else { + file_map_view_close((FileMap){0}, input->data.str, r1u64(0, input->data.size)); + input->data = str8_zero(); + input->owns_file_map = 0; + } } internal void @@ -4082,6 +4153,16 @@ lnk_icf_obj_file_chksms_scan(LNK_Obj *obj) if (~flags & LNK_SECTION_FLAG_DEBUG) { continue; } if ( flags & COFF_SectionFlag_LnkCOMDAT) { continue; } if (!str8_match(lnk_obj_section_name_from_section_number(obj, it.v.section_number), str8_lit(".debug$S"), 0)) { continue; } + LNK_CObjDebugSView indexed = {0}; + if (lnk_compressed_obj_debug_s_index(obj->compressed_obj, it.v.frange, &indexed)) { + for EachIndex(i, indexed.count) { + LNK_CObjDebugSEntry *entry = &indexed.v[i]; + if (entry->kind == CV_C13SubSectionKind_FileChksms) { + return str8(obj->coff.data.str + entry->raw_payload_offset, entry->raw_payload_size); + } + } + continue; + } String8 raw = lnk_obj_section_data_from_number(obj, it.v.section_number); if (raw.size < sizeof(CV_Signature) || cv_signature_from_debug_s(raw) != CV_Signature_C13) { continue; } for (U64 cursor = sizeof(CV_Signature); cursor + sizeof(CV_C13SubSectionHeader) <= raw.size; ) { @@ -4129,12 +4210,25 @@ lnk_icf_src_key_from_fn(Arena *scratch, LNK_Obj *obj, U32 fn_sn, String8 chksms) LNK_ICFSrcKey key = {0}; U32 child_sn = lnk_icf_debug_s_child_from_section(obj, fn_sn); if (child_sn == 0 || chksms.size == 0) { return key; } - String8 raw = lnk_obj_section_data_from_number(obj, child_sn); - CV_DebugS ds = cv_debug_s_from_data(scratch, raw); - cv_debug_s_tag_prov_sect(&ds, child_sn-1); - String8List lines = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Lines); - if (lines.node_count == 0) { return key; } - String8 frag = lines.first->string; + LNK_ObjSection sect = lnk_obj_section_from_section_number(obj, child_sn); + String8 frag = {0}; + LNK_CObjDebugSView indexed = {0}; + if (lnk_compressed_obj_debug_s_index(obj->compressed_obj, sect.frange, &indexed)) { + for EachIndex(i, indexed.count) { + LNK_CObjDebugSEntry *entry = &indexed.v[i]; + if (entry->kind == CV_C13SubSectionKind_Lines) { + frag = str8(obj->coff.data.str + entry->raw_payload_offset, entry->raw_payload_size); + break; + } + } + } else { + String8 raw = lnk_obj_section_data_from_number(obj, child_sn); + CV_DebugS ds = cv_debug_s_from_data(scratch, raw); + cv_debug_s_tag_prov_sect(&ds, child_sn-1); + String8List lines = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Lines); + if (lines.node_count) { frag = lines.first->string; } + } + if (frag.size == 0) { return key; } if (frag.size < sizeof(CV_C13SubSecLinesHeader) + sizeof(CV_C13File) + sizeof(CV_C13Line)) { return key; } CV_C13File *file = (CV_C13File *)(frag.str + sizeof(CV_C13SubSecLinesHeader)); CV_C13Line *l0 = (CV_C13Line *)((U8 *)file + sizeof(CV_C13File)); @@ -4152,10 +4246,29 @@ lnk_icf_src_key_from_fn(Arena *scratch, LNK_Obj *obj, U32 fn_sn, String8 chksms) internal B32 lnk_icf_debug_s_has_locals(Arena *scratch, LNK_Obj *obj, U32 child_sn) { - String8 raw = lnk_obj_section_data_from_number(obj, child_sn); - CV_DebugS ds = cv_debug_s_from_data(scratch, raw); - cv_debug_s_tag_prov_sect(&ds, child_sn-1); - String8List syms = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Symbols); + LNK_ObjSection sect = lnk_obj_section_from_section_number(obj, child_sn); + LNK_CObjDebugSView indexed = {0}; + String8List syms = {0}; + if (lnk_compressed_obj_debug_s_index(obj->compressed_obj, sect.frange, &indexed)) { + if (indexed.summaries) { + for EachIndex(i, indexed.count) { + if (indexed.v[i].kind == CV_C13SubSectionKind_Symbols && + (indexed.summaries[i].flags & LNK_COBJ_DEBUG_S_SUMMARY_HAS_LOCALS)) { return 1; } + } + return 0; + } + for EachIndex(i, indexed.count) { + if (indexed.v[i].kind == CV_C13SubSectionKind_Symbols) { + str8_list_push(scratch, &syms, str8(obj->coff.data.str + indexed.v[i].raw_payload_offset, + indexed.v[i].raw_payload_size)); + } + } + } else { + String8 raw = lnk_obj_section_data_from_number(obj, child_sn); + CV_DebugS ds = cv_debug_s_from_data(scratch, raw); + cv_debug_s_tag_prov_sect(&ds, child_sn-1); + syms = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Symbols); + } for EachNode(n, String8Node, syms.first) { String8 s = n->string; for (U64 o = 0; o + 4 <= s.size; ) { @@ -5536,6 +5649,42 @@ THREAD_POOL_TASK_FUNC(lnk_patch_section_symbols_task) ProfEnd(); } +internal +void +lnk_gather_base_reloc_candidate(Arena *arena, LNK_BaseRelocsTask *task, LNK_Obj *obj, + HashTable *page_ht, LNK_BaseRelocPageList *pages, + U32 sect_idx, U32 apply_off, U32 isymbol, U64 is_addr) +{ + COFF_ParsedSymbol symbol = lnk_parsed_symbol_from_coff_symbol_idx_no_name(obj, isymbol); + COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol); + if (symbol_interp == COFF_SymbolValueInterp_Abs) { return; } + + U64 reloc_voff = obj->coff.sections.headers[sect_idx + 1].voff + apply_off; + U64 page_voff = AlignDownPow2(reloc_voff, task->page_size); + LNK_BaseRelocPageNode *page = hash_table_search_u64_raw(page_ht, page_voff); + if (page == 0) { + page = push_array(arena, LNK_BaseRelocPageNode, 1); + page->v.voff = page_voff; + page->v.entries_addr32 = push_array(arena, U64List, 1); + page->v.entries_addr64 = push_array(arena, U64List, 1); + SLLQueuePush(pages->first, pages->last, page); + pages->count += 1; + hash_table_push_u64_raw(arena, page_ht, page_voff, page); + } + + switch (is_addr) { + case 4: { + if (task->is_large_addr_aware) { + lnk_error_obj(LNK_Error_LargeAddrAwareRequired, obj, "found out of range ADDR32 relocation for '%S', link with /LARGEADDRESSAWARE:NO", lnk_symbol_name_from_coff_symbol_idx(obj, isymbol)); + } else { + u64_list_push(arena, page->v.entries_addr32, reloc_voff); + } + } break; + case 8: { u64_list_push(arena, page->v.entries_addr64, reloc_voff); } break; + default: { InvalidPath; } break; + } +} + internal THREAD_POOL_TASK_FUNC(lnk_gather_base_reloc_pages_task) { @@ -5545,7 +5694,25 @@ THREAD_POOL_TASK_FUNC(lnk_gather_base_reloc_pages_task) LNK_Obj *obj = task->gather.objs[task_id]; ProfBeginV("%S", obj->path); + LNK_CObjBaseRelocView compressed_index = {0}; + if (lnk_compressed_obj_base_reloc_index(obj->compressed_obj, &compressed_index)) { + for EachIndex(i, compressed_index.count) { + LNK_CObjBaseRelocEntry *entry = &compressed_index.v[i]; + if (entry->sect_idx >= obj->coff.sections.count_no_null || entry->isymbol >= obj->coff.header.symbol_count || + (entry->addr_size != 4 && entry->addr_size != 8)) { + lnk_error_obj(LNK_Error_IllData, obj, "invalid compressed base relocation sidecar entry"); + continue; + } + if (obj->coff.sections.headers[entry->sect_idx + 1].flags & COFF_SectionFlag_LnkRemove) { continue; } + lnk_gather_base_reloc_candidate(arena, task, obj, page_ht, pages, entry->sect_idx, + entry->apply_off, entry->isymbol, entry->addr_size); + } + ProfEnd(); + return; + } + for LNK_EachCoffSection(it, obj) { + U32 sect_idx = safe_cast_u32(it.v.section_number - 1); COFF_SectionHeader *sect_header = it.v.header; if (*it.v.flags & COFF_SectionFlag_LnkRemove) { continue; } @@ -5553,44 +5720,10 @@ THREAD_POOL_TASK_FUNC(lnk_gather_base_reloc_pages_task) for EachIndex(reloc_idx, relocs.count) { COFF_Reloc *r = &relocs.v[reloc_idx]; - COFF_ParsedSymbol symbol = lnk_parsed_symbol_from_coff_symbol_idx_no_name(obj, r->isymbol); - COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol); - if (symbol_interp == COFF_SymbolValueInterp_Abs) { continue; } - U64 is_addr = coff_is_addr_reloc(obj->coff.header.machine, r->type); if (is_addr == 0) { continue; } - - U64 reloc_voff = sect_header->voff + r->apply_off; - U64 page_voff = AlignDownPow2(reloc_voff, task->page_size); - LNK_BaseRelocPageNode *page = hash_table_search_u64_raw(page_ht, page_voff); - if (page == 0) { - // fill out page - page = push_array(arena, LNK_BaseRelocPageNode, 1); - page->v.voff = page_voff; - page->v.entries_addr32 = push_array(arena, U64List, 1); - page->v.entries_addr64 = push_array(arena, U64List, 1); - - // push page - SLLQueuePush(pages->first, pages->last, page); - pages->count += 1; - - // register page voff - hash_table_push_u64_raw(arena, page_ht, page_voff, page); - } - - switch (is_addr) { - case 4: { - if (task->is_large_addr_aware) { - lnk_error_obj(LNK_Error_LargeAddrAwareRequired, obj, "found out of range ADDR32 relocation for '%S', link with /LARGEADDRESSAWARE:NO", lnk_symbol_name_from_coff_symbol_idx(obj, r->isymbol)); - } else { - u64_list_push(arena, page->v.entries_addr32, reloc_voff); - } - } break; - case 8: { - u64_list_push(arena, page->v.entries_addr64, reloc_voff); - } break; - default: { InvalidPath; } break; - } + lnk_gather_base_reloc_candidate(arena, task, obj, page_ht, pages, (U32)sect_idx, + r->apply_off, r->isymbol, is_addr); } } @@ -6095,7 +6228,6 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT tp_barrier_end(tp); tp_temp_end(temp); } - // ensure determinism by sorting section contribs in chunks by input index ProfScope("Sort Section Contribs") { @@ -7497,6 +7629,11 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) ProfEnd(); } + // Type merging is the last bulk consumer of type payload pages. Apply the configured cache + // generation transition before PDB construction so type residency does not stack with the + // later GSI and module-stream allocations. + lnk_compressed_obj_trim_working_set(); + // // Debug Info // @@ -7683,6 +7820,8 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // Timers // { + lnk_compressed_obj_log_stats(); + lnk_obj_log_compressed_census(); char *phase_log_env = getenv("RADLINK_PHASE_LOG"); if (lnk_get_log_status(LNK_Log_Timers) || (phase_log_env != 0 && phase_log_env[0] != 0)) { lnk_log_timers(); diff --git a/src/linker/lnk.h b/src/linker/lnk.h index 64b1f30a3..b35a701c9 100644 --- a/src/linker/lnk.h +++ b/src/linker/lnk.h @@ -32,6 +32,9 @@ typedef struct LNK_Input { String8 path; String8 data; + String8 compressed_data; + LNK_CompressedObj *compressed_obj; + U8 *compressed_eager_base; B32 disallow; B32 is_thin; B32 owns_file_map; diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index b466e3322..26601ddec 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -260,6 +260,38 @@ lnk_discard_cv_debug_info(LNK_CodeViewInput *input, U64 obj_idx) MemoryZeroStruct(cv_sub_section_prov_ptr_from_debug_s(&input->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines)); } +// Build the same lightweight C13 lists/provenance as cv_debug_s_from_data without walking the +// compressed logical section. Payload pointers are intentionally not dereferenced here; later +// consumers retain the exact original virtual addresses and fault/decode only if they truly need +// record bytes. +internal B32 +lnk_debug_s_from_compressed_index(Arena *arena, LNK_Obj *obj, U32 sect_idx, CV_DebugS *out) +{ + if (!obj->compressed_obj) { return 0; } + LNK_ObjSection section = lnk_obj_section_from_section_number(obj, sect_idx + 1); + LNK_CObjDebugSView view = {0}; + if (!lnk_compressed_obj_debug_s_index(obj->compressed_obj, section.frange, &view)) { return 0; } + for EachIndex(i, view.count) { + LNK_CObjDebugSEntry *entry = &view.v[i]; + U64 payload_min = entry->raw_payload_offset; + U64 payload_max = payload_min + entry->raw_payload_size; + if (payload_min < section.frange.min || payload_max > section.frange.max) { return 0; } + U64 idx = cv_c13_sub_section_idx_from_kind(entry->kind); + str8_list_push(arena, &out->data_list[idx], str8(obj->coff.data.str + payload_min, entry->raw_payload_size)); + cv_debug_s_prov_list_push(arena, &out->prov_list[idx], payload_min - section.frange.min, + entry->raw_payload_size, sect_idx, 0); + if (view.summaries && entry->kind == CV_C13SubSectionKind_Symbols) { + LNK_CObjDebugSSummary *summary = &view.summaries[i]; + CV_DebugSProvNode *prov = out->prov_list[idx].last; + prov->module_symbol_size = summary->module_symbol_size; + prov->gsi_candidate_count = summary->gsi_candidate_count; + prov->proc_ref_count = summary->proc_ref_count; + prov->symbol_summary_valid = 1; + } + } + return 1; +} + internal THREAD_POOL_TASK_FUNC(lnk_parse_debug_s_task) { @@ -275,8 +307,12 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_s_task) U64 input_ordinal = 0; for (String8Node *n = sect_list.first; n != 0; n = n->next, input_ordinal += 1) { // parse & merge sub sections - CV_DebugS ds = cv_debug_s_from_data(arena, n->string); - cv_debug_s_tag_prov_sect(&ds, sect_indices.v[input_ordinal]); + U32 sect_idx = sect_indices.v[input_ordinal]; + CV_DebugS ds = {0}; + if (!lnk_debug_s_from_compressed_index(arena, task->obj_arr[obj_idx], sect_idx, &ds)) { + ds = cv_debug_s_from_data(arena, n->string); + cv_debug_s_tag_prov_sect(&ds, sect_idx); + } cv_debug_s_concat_in_place(debug_s, &ds); // make sure there is one string table @@ -306,9 +342,12 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_s_task) if (obj->icf_lines_only != 0) { for (U32 section_number = 1; section_number <= obj->coff.sections.count_no_null; section_number += 1) { if (!obj->icf_lines_only[section_number]) { continue; } - String8 raw_data = lnk_obj_section_data_from_number(obj, section_number); - CV_DebugS ds = cv_debug_s_from_data(arena, raw_data); - cv_debug_s_tag_prov_sect(&ds, section_number - 1); // merged from a different child section + CV_DebugS ds = {0}; + if (!lnk_debug_s_from_compressed_index(arena, obj, section_number - 1, &ds)) { + String8 raw_data = lnk_obj_section_data_from_number(obj, section_number); + ds = cv_debug_s_from_data(arena, raw_data); + cv_debug_s_tag_prov_sect(&ds, section_number - 1); // merged from a different child section + } if (obj->icf_lines_only[section_number] == 2) { cv_debug_s_concat_in_place(debug_s, &ds); } else { @@ -425,8 +464,8 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_t_task) { ProfBeginFunction(); LNK_ParseCvTypes *task = raw_task; - if (task->out_types[task_id].offsets != 0) { - // giant obj, already parsed by lnk_parse_giant_debug_t + if (task->out_types[task_id].offsets != 0 || task->out_types[task_id].sidecar_packed) { + // Container sidecar or giant obj already supplied the index. } else if (task->raw_types[task_id].count > 0) { task->out_types[task_id] = cv_debug_t_from_data(arena, task->raw_types[task_id].v[0], CV_LeafAlign); } else { @@ -435,6 +474,115 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_t_task) ProfEnd(); } +typedef struct LNK_BackgroundPrefetch +{ + U64 range_count; + Rng1U64 *ranges; +} LNK_BackgroundPrefetch; + +internal void +lnk_background_prefetch_thread(void *raw_task) +{ + LNK_BackgroundPrefetch *task = raw_task; + lnk_prefetch_ranges(0, 0, task->range_count, task->ranges); +} + +#define LNK_WINNER_PREFETCH_WORKER_MAX 32 +typedef struct LNK_WinnerPrefetchFlight +{ + Thread threads[LNK_WINNER_PREFETCH_WORKER_MAX]; + LNK_BackgroundPrefetch tasks[LNK_WINNER_PREFETCH_WORKER_MAX]; + U32 worker_count; + U64 range_count; + U64 stored_bytes; + U64 begin_us; +} LNK_WinnerPrefetchFlight; + +internal int +lnk_prefetch_range_compare(const void *a, const void *b) +{ + Rng1U64 *ra = (Rng1U64 *)a; + Rng1U64 *rb = (Rng1U64 *)b; + return ra->min < rb->min ? -1 : ra->min > rb->min; +} + +internal void +lnk_winner_prefetch_start(LNK_WinnerPrefetchFlight *flight, U32 worker_count, + U64 range_count, Rng1U64 *ranges, U64 stored_bytes) +{ + if (range_count == 0) { return; } + qsort(ranges, range_count, sizeof(*ranges), lnk_prefetch_range_compare); + flight->worker_count = Min(worker_count, range_count); + flight->range_count = range_count; + flight->stored_bytes = stored_bytes; + flight->begin_us = now_time_us(); + for EachIndex(worker_idx, flight->worker_count) { + U64 min = range_count * worker_idx / flight->worker_count; + U64 max = range_count * (worker_idx + 1) / flight->worker_count; + flight->tasks[worker_idx] = (LNK_BackgroundPrefetch){max - min, ranges + min}; + flight->threads[worker_idx] = thread_launch(lnk_background_prefetch_thread, + &flight->tasks[worker_idx]); + } +} + +internal void +lnk_winner_prefetch_join(LNK_WinnerPrefetchFlight *flight) +{ + for EachIndex(worker_idx, flight->worker_count) { + if (flight->threads[worker_idx].u64[0]) { + thread_join(flight->threads[worker_idx], max_U64); + } + } + if (flight->worker_count) { + lnk_log(LNK_Log_Timers, "[cobj prefetch] workers=%u ranges=%llu stored=%llu MiB elapsed=%.3fs", + flight->worker_count, flight->range_count, flight->stored_bytes / MB(1), + (F64)(now_time_us() - flight->begin_us) / 1e6); + } +} + +// Install the mapped leaf index carried by a compressed-object container. +// This replaces the full leaf-body chain walk with a compact kind scan; the +// body bytes remain behind the bounded decoder until a real consumer asks for +// a leaf. +internal +THREAD_POOL_TASK_FUNC(lnk_parse_debug_t_sidecar_task) +{ + LNK_ParseCvTypes *task = raw_task; + U64 obj_idx = task_id; + if (task->raw_types[obj_idx].count == 0) { return; } + LNK_Obj *obj = task->input->obj_arr[obj_idx]; + U32 section_number = task->is_debug_p ? obj->coff.debug_p_section_number : obj->coff.debug_t_section_number; + if (!obj->compressed_obj || section_number == 0 || section_number > obj->coff.sections.count_no_null) { return; } + LNK_ObjSection sect = lnk_obj_section_from_section_number(obj, section_number); + Rng1U64 leaf_range = rng_1u64(sect.frange.min + sizeof(CV_Signature), sect.frange.max); + LNK_CObjTypeIndexView view = {0}; + if (!lnk_compressed_obj_type_index(obj->compressed_obj, leaf_range, &view)) { return; } + + CV_DebugT *out = &task->out_types[obj_idx]; + out->data = task->raw_types[obj_idx].v[0]; + out->count = view.count; + out->offsets = view.offsets; + out->sidecar_sizes = view.sizes; + out->sidecar_kinds = view.kinds; + out->sidecar_packed_v2_offset_groups = view.packed_v2_offset_groups; + out->sidecar_packed_v2_offset_payload = view.packed_v2_offset_payload; + out->sidecar_packed_kind_dictionary = view.packed_kind_dictionary; + out->sidecar_packed_kind_codes = view.packed_kind_codes; + out->sidecar_packed = view.packed_sidecar; + out->sidecar_offset_checkpoint_shift = view.offset_checkpoint_shift; + out->sidecar_raw_base = leaf_range.min; + out->sidecar_complete_udt_hashes = view.complete_udt_hashes; + out->sidecar_complete_udt_hash_count = view.complete_udt_hash_count; + for EachIndex(i, view.count) { + out->source_counts[cv_type_index_source_from_leaf_kind(cv_debug_t_get_leaf_kind(out, i))] += 1; + } + for EachElement(i, out->ti_ranges) { out->ti_ranges[i] = r1u64(CV_MinComplexTypeIndex, CV_MinComplexTypeIndex + out->count); } + if (out->count && cv_debug_t_get_leaf_kind(out, 0) == CV_LeafKind_PRECOMP) { + CV_PrecompInfo precomp_info = cv_precomp_info_from_leaf(cv_debug_t_get_leaf(out, 0)); + for EachElement(i, out->ti_ranges) { out->ti_ranges[i].max += precomp_info.leaf_count; } + } +} + // Giant .debug$T parse: a single SharedPCH-scale .debug$T is a multi-second // SERIAL pointer-chase (each leaf's offset depends on the previous leaf's // size), and a handful of such objs bound the whole capped parse stage @@ -547,7 +695,8 @@ lnk_parse_giant_debug_t(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 giant_count = 0; LNK_GiantDebugT *giants = push_array(scratch.arena, LNK_GiantDebugT, obj_count); for EachIndex(obj_idx, obj_count) { - if (parse->raw_types[obj_idx].count > 0 && parse->raw_types[obj_idx].v[0].size >= LNK_GIANT_DEBUG_T_SIZE) { + if (parse->out_types[obj_idx].offsets == 0 && !parse->out_types[obj_idx].sidecar_packed && + parse->raw_types[obj_idx].count > 0 && parse->raw_types[obj_idx].v[0].size >= LNK_GIANT_DEBUG_T_SIZE) { giants[giant_count].obj_idx = obj_idx; giants[giant_count].data = parse->raw_types[obj_idx].v[0]; giant_count += 1; @@ -1269,6 +1418,11 @@ THREAD_POOL_TASK_FUNC(lnk_ifc_complete_scan_task) LNK_IfcCompleteScanTask *task = raw_task; U64 obj_idx = task_id; CV_DebugT *dt = &task->input->debug_t_arr[obj_idx]; + if (dt->sidecar_complete_udt_hashes) { + task->out_hashes[obj_idx] = dt->sidecar_complete_udt_hashes; + task->out_counts[obj_idx] = dt->sidecar_complete_udt_hash_count; + return; + } U64 *hashes = push_array_no_zero(arena, U64, dt->count ? dt->count : 1); U64 n = 0; for EachIndex(leaf_idx, dt->count) { @@ -1347,8 +1501,7 @@ THREAD_POOL_TASK_FUNC(lnk_ifc_scan_task) // count 0x1522 leaves first to size the per-obj record array U64 ifc_leaf_count = 0; for EachIndex(leaf_idx, debug_t->count) { - CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); - if (hdr->kind == 0x1522) { ifc_leaf_count += 1; } + if (cv_debug_t_get_leaf_kind(debug_t, leaf_idx) == 0x1522) { ifc_leaf_count += 1; } } if (ifc_leaf_count == 0) { task->out_recs[obj_idx] = 0; task->out_counts[obj_idx] = 0; return; } @@ -1356,8 +1509,7 @@ THREAD_POOL_TASK_FUNC(lnk_ifc_scan_task) U64 n = 0; for EachIndex(leaf_idx, debug_t->count) { - CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); - if (hdr->kind != 0x1522) { continue; } + if (cv_debug_t_get_leaf_kind(debug_t, leaf_idx) != 0x1522) { continue; } CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); LNK_IfcRecord rec = lnk_parse_ifc_record(leaf.data); @@ -1776,6 +1928,14 @@ lnk_apply_ifc_debug_records(TP_Context *tp, TP_Arena *tp_arena, LNK_CodeViewInpu scan.out_hashes = push_array(scratch.arena, U64 *, nonblob_count); scan.out_counts = push_array(scratch.arena, U64, nonblob_count); tp_for_parallel(tp, tp_arena, nonblob_count, lnk_ifc_complete_scan_task, &scan); + U64 scan_hash_count = 0; + U64 scan_sidecar_count = 0; + for EachIndex(obj_idx, nonblob_count) { + scan_hash_count += scan.out_counts[obj_idx]; + scan_sidecar_count += (input->debug_t_arr[obj_idx].sidecar_complete_udt_hashes != 0); + } + lnk_log(LNK_Log_Timers, "[IFC] complete UDT scan: hashes=%llu sidecars=%llu/%llu", + scan_hash_count, scan_sidecar_count, nonblob_count); // parallel atomic-CAS merge (replaces the serial lnk_u64set_add loop): output-identical // because set membership is order-independent + idempotent (see lnk_u64set_add_atomic). LNK_IfcSetMergeTask merge = {0}; @@ -2056,7 +2216,25 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, { // parse .debug$S input.debug_s_arr = push_array(tp_arena->v[0], CV_DebugS, input.obj_count); + lnk_compressed_obj_log_phase_stats("before parse $S"); lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, obj_count, lnk_parse_debug_s_task, &input, "Parse .debug$S"); + lnk_compressed_obj_log_phase_stats("after parse $S"); + { + U64 bytes[CV_C13SubSectionIdxKind_COUNT] = {0}; + U64 nodes[CV_C13SubSectionIdxKind_COUNT] = {0}; + for EachIndex(obj_idx, obj_count) { + for EachElement(k, input.debug_s_arr[obj_idx].data_list) { + bytes[k] += input.debug_s_arr[obj_idx].data_list[k].total_size; + nodes[k] += input.debug_s_arr[obj_idx].data_list[k].node_count; + } + } + for EachIndex(k, CV_C13SubSectionIdxKind_COUNT) { + if (bytes[k] || nodes[k]) { + lnk_log(LNK_Log_Timers, "[debugS parsed] kind=%x bytes=%llu nodes=%llu", + cv_c13_sub_section_kind_from_idx(k), bytes[k], nodes[k]); + } + } + } // collect .debug$P and .debug$T String8Array *raw_debug_p_arr = push_array(scratch.arena, String8Array, obj_count); @@ -2083,14 +2261,18 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, debug_p_arr = push_array(tp_arena->v[0], CV_DebugT, obj_count); parse_types.raw_types = raw_debug_p_arr; parse_types.out_types = debug_p_arr; + parse_types.is_debug_p = 1; lnk_tp_for_parallel_capped_prof(tp, 0, config->debug_worker_cap, obj_count, lnk_strip_debug_t_sig_task, &parse_types, "Strip .debug$P"); + lnk_tp_for_parallel_capped_prof(tp, 0, config->debug_worker_cap, obj_count, lnk_parse_debug_t_sidecar_task, &parse_types, "Index .debug$P sidecars"); lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, obj_count, lnk_parse_debug_t_task, &parse_types, "Parse .debug$P"); // parse .debug$T input.debug_t_arr = push_array(tp_arena->v[0], CV_DebugT, obj_count); parse_types.raw_types = raw_debug_t_arr; parse_types.out_types = input.debug_t_arr; + parse_types.is_debug_p = 0; lnk_tp_for_parallel_capped_prof(tp, 0, config->debug_worker_cap, obj_count, lnk_strip_debug_t_sig_task, &parse_types, "Strip .debug$T"); + lnk_tp_for_parallel_capped_prof(tp, 0, config->debug_worker_cap, obj_count, lnk_parse_debug_t_sidecar_task, &parse_types, "Index .debug$T sidecars"); lnk_parse_giant_debug_t(tp, tp_arena, config, obj_count, &parse_types); lnk_tp_for_parallel_capped_prof(tp, tp_arena, config->debug_worker_cap, obj_count, lnk_parse_debug_t_task, &parse_types, "Parse .debug$T"); @@ -2101,6 +2283,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, } } ProfEnd(); + lnk_compressed_obj_log_phase_stats("after parse CodeView"); // sort objs based on type: PCH, /Zi (external), /Z7 (internal) input.debug_p_indices.v = push_array(tp_arena->v[0], U32, obj_count); @@ -2377,7 +2560,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, // remove CV_LeafKind_PRECOMP debug_t->count -= 1; - debug_t->offsets += 1; + debug_t->sidecar_leaf_bias += 1; } // remove LF_ENDPRECOMP from .debug$P -- P4: journal the NOTYPE rewrite instead of dirtying @@ -2388,8 +2571,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, CV_DebugT *debug_p = &input.debug_t_arr[debug_p_idx]; for EachIndex(i, debug_p->count) { U64 lf_idx = debug_p->count - (i + 1); - CV_LeafHeader *lf = cv_debug_t_get_leaf_header(debug_p, lf_idx); - if (lf->kind == CV_LeafKind_ENDPRECOMP) { + if (cv_debug_t_get_leaf_kind(debug_p, lf_idx) == CV_LeafKind_ENDPRECOMP) { lnk_notype_journal_push(tp_arena->v[0], &input.notype_journal[debug_p_idx], (U32)lf_idx, 0, debug_p->count); break; } @@ -2489,6 +2671,7 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, } } ProfEnd(); + lnk_compressed_obj_log_phase_stats("after make symbol inputs"); scratch_end(scratch); ProfEnd(); @@ -2582,10 +2765,26 @@ lnk_notype_journal_find(LNK_NotypeJournal *journal, U32 leaf_idx, B32 *kind_only // journaled view of the leaf to the hashers (identical to what the old in-place writes produced: // full rewrite => { kind=LF_NOTYPE, empty payload }; KIND_ONLY => kind=LF_NOTYPE, payload kept). internal CV_Leaf -lnk_cv_leaf_from_leaf_ref(LNK_CodeViewInput *input, U32 obj_idx, U32 leaf_idx) +lnk_cv_leaf_from_leaf_ref(Arena *arena, LNK_CObjDecodeWindow *decode_window, + LNK_CodeViewInput *input, U32 obj_idx, U32 leaf_idx) { CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + CV_Leaf leaf = {0}; + LNK_Obj *obj = obj_idx < input->obj_count ? input->obj_arr[obj_idx] : 0; + if (obj && obj->compressed_obj && (debug_t->sidecar_sizes || debug_t->sidecar_packed) && decode_window) { + U64 raw_size = cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx); + U8 *raw_leaf = push_array_no_zero(arena, U8, raw_size); + U64 leaf_off = cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + Rng1U64 raw_range = rng_1u64(debug_t->sidecar_raw_base + leaf_off, + debug_t->sidecar_raw_base + leaf_off + raw_size); + if (lnk_compressed_obj_copy_range(obj->compressed_obj, raw_range, raw_leaf, decode_window)) { + cv_read_leaf(str8(raw_leaf, raw_size), 0, 1, &leaf); + } else { + leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + } + } else { + leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + } // hot path: one bitmap test; journals are empty for ~all objs if (obj_idx < input->obj_count && lnk_notype_journal_test(&input->notype_journal[obj_idx], leaf_idx)) { B32 kind_only = 0; @@ -2608,7 +2807,7 @@ lnk_leaf_ref_materialize_meta(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; - U64 raw_size = cv_debug_t_get_raw_leaf(debug_t, leaf_idx).size; + U64 raw_size = cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx); U64 rewrite = 0; if (obj_idx < input->obj_count && lnk_notype_journal_test(&input->notype_journal[obj_idx], leaf_idx)) { B32 kind_only = 0; @@ -2782,7 +2981,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, Arena *journal_arena, LNK_LeafRef lea if (obj_idx < input->obj_count) { lnk_notype_journal_push(journal_arena, &input->notype_journal[obj_idx], leaf_idx, 0, debug_t->count); } else { - U8 *leaf_header = debug_t->data.str + debug_t->offsets[leaf_idx]; + U8 *leaf_header = debug_t->data.str + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); memory_write16(leaf_header + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); memory_write16(leaf_header + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); } @@ -2809,7 +3008,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, Arena *journal_arena, LNK_LeafRef lea if (obj_idx < input->obj_count) { lnk_notype_journal_push(journal_arena, &input->notype_journal[obj_idx], leaf_idx, 0, debug_t->count); } else { - U8 *leaf_header = debug_t->data.str + debug_t->offsets[leaf_idx]; + U8 *leaf_header = debug_t->data.str + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); memory_write16(leaf_header + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); memory_write16(leaf_header + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); } @@ -2847,6 +3046,13 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, Arena *journal_arena, LNK_LeafRef lea lnk_notype_journal_test(&input->notype_journal[obj_idx], leaf_idx)) { CV_LeafHeader leaf_header = { .size = (CV_LeafSize)(leaf.data.size + sizeof(CV_LeafKind)), .kind = leaf.kind }; lnk_hasher_update_struct(&hasher, &leaf_header); + } else if (debug_t->sidecar_sizes || debug_t->sidecar_packed) { + // The sparse raw view deliberately leaves .debug$T bodies (including the + // four-byte leaf header) as holes. The body above came from the bounded + // decode window; reconstruct the byte-identical header from its sidecar + // instead of mixing four zero bytes from the hole into the type hash. + CV_LeafHeader leaf_header = { .size = (CV_LeafSize)(cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx) - sizeof(CV_LeafSize)), .kind = leaf.kind }; + lnk_hasher_update_struct(&hasher, &leaf_header); } else { lnk_hasher_update_struct(&hasher, cv_debug_t_get_leaf_header(debug_t, leaf_idx)); } @@ -2864,6 +3070,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, Arena *journal_arena, LNK_LeafRef lea internal void lnk_hash_cv_leaf_deep(Arena *arena, + LNK_CObjDecodeWindow *decode_window, LNK_CodeViewInput *input, LNK_LeafRef root_leaf_ref, CV_TiOffsets root_ti_offs) @@ -2889,7 +3096,7 @@ lnk_hash_cv_leaf_deep(Arena *arena, root_frame->ti_offs = root_ti_offs; root_frame->ti_next = 0; root_frame->ti_count = cv_ti_offsets_count(&root_ti_offs); - root_frame->leaf = lnk_cv_leaf_from_leaf_ref(input, lnk_leaf_ref_obj_idx(root_leaf_ref), lnk_leaf_ref_leaf_idx(root_leaf_ref)); + root_frame->leaf = lnk_cv_leaf_from_leaf_ref(temp.arena, decode_window, input, lnk_leaf_ref_obj_idx(root_leaf_ref), lnk_leaf_ref_leaf_idx(root_leaf_ref)); root_frame->ti_source = cv_type_index_source_from_leaf_kind(root_frame->leaf.kind); root_frame->ti = cv_ti_from_leaf_idx(root_debug_t, root_frame->ti_source, lnk_leaf_ref_leaf_idx(root_leaf_ref)); @@ -2918,7 +3125,7 @@ lnk_hash_cv_leaf_deep(Arena *arena, // recurse down to sub types HashStack *frame = push_array(temp.arena, HashStack, 1); frame->leaf_ref = leaf_ref; - frame->leaf = lnk_cv_leaf_from_leaf_ref(input, obj_idx, leaf_idx); + frame->leaf = lnk_cv_leaf_from_leaf_ref(temp.arena, decode_window, input, obj_idx, leaf_idx); frame->ti_offs = cv_leaf_ti_offsets(temp.arena, frame->leaf.kind, frame->leaf.data); frame->ti_next = 0; frame->ti_count = cv_ti_offsets_count(&frame->ti_offs); @@ -2991,7 +3198,7 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_task) CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; for EachIndex(leaf_idx, debug_t->count) { Temp temp = temp_begin(task->fixed_arenas[worker_id]); - CV_Leaf leaf = lnk_cv_leaf_from_leaf_ref(task->input, obj_idx, leaf_idx); + CV_Leaf leaf = lnk_cv_leaf_from_leaf_ref(temp.arena, &task->decode_windows[worker_id], task->input, obj_idx, leaf_idx); CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); lnk_hash_cv_leaf(task->input, arena, lnk_leaf_ref_make(obj_idx, leaf_idx), leaf, ti_offs, 1); temp_end(temp); @@ -3009,11 +3216,11 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_deep_task) B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); for EachIndex(leaf_idx, debug_t->count) { if (task->input->debug_h_arr[obj_idx].v[leaf_idx] != 0) { continue; } - if (is_ifc_blob && cv_debug_t_get_leaf_header(debug_t, leaf_idx)->kind == CV_LeafKind_NOTYPE) { continue; } // blob $T is arena-backed + mutated in place -- raw read is post-rewrite + if (is_ifc_blob && cv_debug_t_get_leaf_kind(debug_t, leaf_idx) == CV_LeafKind_NOTYPE) { continue; } // blob $T is arena-backed + mutated in place -- raw read is post-rewrite Temp temp = temp_begin(task->fixed_arenas[worker_id]); - CV_Leaf leaf = lnk_cv_leaf_from_leaf_ref(task->input, obj_idx, leaf_idx); + CV_Leaf leaf = lnk_cv_leaf_from_leaf_ref(temp.arena, &task->decode_windows[worker_id], task->input, obj_idx, leaf_idx); CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); - lnk_hash_cv_leaf_deep(temp.arena, task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_offs); + lnk_hash_cv_leaf_deep(temp.arena, &task->decode_windows[worker_id], task->input, lnk_leaf_ref_make(obj_idx, leaf_idx), ti_offs); temp_end(temp); } ProfEnd(); @@ -3058,8 +3265,7 @@ THREAD_POOL_TASK_FUNC(lnk_estimate_unique_leaves_task) notype_cap = task->input->notype_journal[obj_idx].bit_cap; } for (U64 leaf_idx = 0; leaf_idx < debug_t->count; leaf_idx += LNK_ESTIMATE_SAMPLE_STRIDE) { - CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); - CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); + CV_LeafKind kind = cv_debug_t_get_leaf_kind(debug_t, leaf_idx); if (notype_bm && leaf_idx < notype_cap && ((notype_bm[leaf_idx >> 6] >> (leaf_idx & 63)) & 1)) { kind = CV_LeafKind_NOTYPE; } if (is_ifc_blob && kind == CV_LeafKind_NOTYPE) { continue; } CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); @@ -3092,8 +3298,7 @@ THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) // pop obj is a type-server pseudo obj: arena-backed, mutated in place, never journaled -- // the raw header read is the post-rewrite kind (original code path) - CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); // leaf index -> leaf header - CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); // leaf header -> leaf kind + CV_LeafKind kind = cv_debug_t_get_leaf_kind(debug_t, leaf_idx); CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); // leaf kind -> type stream LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[leaf_source]; // type stream -> hash table U64 hash = debug_h->v[leaf_idx]; // leaf ref -> hash @@ -3150,12 +3355,11 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) // another worker overflowed an estimate-sized table -- the whole dedup result is discarded // and retried with the total-based caps, so bail out early if (ins_atomic_u32_eval(&task->leaf_ht_overflow) != 0) { break; } - if (is_ifc_blob && cv_debug_t_get_leaf_header(debug_t, leaf_idx)->kind == CV_LeafKind_NOTYPE) { continue; } B32 is_inserted_or_updated = 1; LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); - CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); // leaf index -> leaf header - CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); // leaf header -> leaf kind + CV_LeafKind kind = cv_debug_t_get_leaf_kind(debug_t, leaf_idx); + if (is_ifc_blob && kind == CV_LeafKind_NOTYPE) { continue; } if (notype_bm && leaf_idx < notype_cap && ((notype_bm[leaf_idx >> 6] >> (leaf_idx & 63)) & 1)) { kind = CV_LeafKind_NOTYPE; } CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); // leaf kind -> type stream LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[leaf_source]; // type stream -> hash table @@ -3235,7 +3439,35 @@ THREAD_POOL_TASK_FUNC(lnk_get_present_buckets_task) for EachInRange(bucket_idx, task->ranges[task_id]) { if (ht->bucket_arr[bucket_idx] != LNK_LEAF_REF_NULL) { - unique_leaf_refs.v[cursor++] = lnk_leaf_bucket_untag(ht->bucket_arr[bucket_idx]); + LNK_LeafRef ref = lnk_leaf_bucket_untag(ht->bucket_arr[bucket_idx]); + unique_leaf_refs.v[cursor++] = ref; + + // TPI/IPI materialization is the next compressed-payload consumer. Claim each segment + // once while the winner is already hot in this extraction pass. A leaf can straddle a + // segment boundary, so claim the full raw range rather than only its first byte. + if (task->winner_segment_bitmap && task->ti_source != CV_TypeIndexSource_NULL && + lnk_leaf_ref_obj_idx(ref) < task->input->obj_count) { + U32 obj_idx = lnk_leaf_ref_obj_idx(ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(ref); + LNK_Obj *obj = task->input->obj_arr[obj_idx]; + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + U32 segment_size = lnk_compressed_obj_segment_size(obj->compressed_obj); + if (segment_size && (debug_t->sidecar_sizes || debug_t->sidecar_packed)) { + U64 raw_min = debug_t->sidecar_raw_base + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + U64 raw_max = raw_min + cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx); + U32 first = (U32)(raw_min / segment_size); + U32 opl = (U32)CeilIntegerDiv(raw_max, segment_size); + U32 count = lnk_compressed_obj_segment_count(obj->compressed_obj); + opl = Min(opl, count); + for (U32 segment_idx = first; segment_idx < opl; ++segment_idx) { + U64 global_idx = task->winner_segment_offsets[obj_idx] + segment_idx; + U64 *word = &task->winner_segment_worker_bitmaps[task_id * task->winner_segment_word_count + + (global_idx >> 6)]; + U64 bit = (U64)1 << (global_idx & 63); + *word |= bit; + } + } + } } } @@ -3398,6 +3630,22 @@ THREAD_POOL_TASK_FUNC(lnk_journal_symbol_fixups_task) for EachInRange(i, range) { LNK_SymbolInput symbols = task->input->symbol_inputs[i]; + // The journal builder makes two immediate linear passes over each Symbols subsection. + // Keeping those reads behind the shared faulting view lets concurrently scheduled objs + // evict one another between passes. A subsection-local copy streams every compressed + // segment once through the worker's decode window, then both parser passes hit ordinary + // committed memory. The fixed arena temp is rewound for every input, so peak storage is + // bounded by one Symbols subsection per active worker rather than the corpus size. + Temp symbols_temp = temp_begin(task->fixed_arenas[worker_id]); + LNK_Obj *obj = task->input->obj_arr[symbols.obj_idx]; + if (obj->compressed_obj && symbols.raw_symbols.size) { + U8 *copy = push_array_no_zero(symbols_temp.arena, U8, symbols.raw_symbols.size); + if (lnk_compressed_obj_copy_string(obj->compressed_obj, symbols.raw_symbols, copy, + &task->decode_windows[worker_id])) { + symbols.raw_symbols = str8(copy, symbols.raw_symbols.size); + } + } + // upper-bound pass: TI entries (exact) + kind rewrite / itype slots for *_ID records U64 cap = 0; for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { @@ -3495,6 +3743,7 @@ THREAD_POOL_TASK_FUNC(lnk_journal_symbol_fixups_task) } } Assert(journal->count <= cap); + temp_end(symbols_temp); } ProfEnd(); } @@ -3655,6 +3904,7 @@ lnk_obj_window_debug_s(Arena *arena, LNK_CodeViewInput *cv, U64 obj_idx, U64 ima LNK_Obj *obj = cv->obj_arr[obj_idx]; CV_DebugS *src = &cv->debug_s_arr[obj_idx]; CV_DebugS out = {0}; + LNK_CObjDecodeWindow decode_window = {0}; U64 idx_symbols = cv_c13_sub_section_idx_from_kind(CV_C13SubSectionKind_Symbols); U64 idx_inlinees = cv_c13_sub_section_idx_from_kind(CV_C13SubSectionKind_InlineeLines); @@ -3689,13 +3939,25 @@ lnk_obj_window_debug_s(Arena *arena, LNK_CodeViewInput *cv, U64 obj_idx, U64 ima if (sect_state[sect_idx] == 0) { continue; } U64 section_number = sect_idx + 1; LNK_ObjSection section = lnk_obj_section_from_section_number(obj, section_number); - COFF_SectionHeader *raw_hdr = &raw_section_table[sect_idx]; - String8 raw = str8_substr(obj->coff.data, r1u64s(raw_hdr->foff, raw_hdr->fsize)); // always start from the immutable raw view + String8 raw = lnk_compressed_obj_direct_range(obj->compressed_obj, section.frange); + B32 explicitly_decoded = 0; + if (raw.size == 0 && obj->compressed_obj != 0) { + U8 *copy = push_array_no_zero(arena, U8, dim_1u64(section.frange)); + if (lnk_compressed_obj_copy_range(obj->compressed_obj, section.frange, copy, &decode_window)) { + raw = str8(copy, dim_1u64(section.frange)); + explicitly_decoded = 1; + } + } + if (raw.size == 0) { raw = str8_substr(obj->coff.data, section.frange); } if (sect_state[sect_idx] == 1) { sect_base[sect_idx] = raw.str; + if (explicitly_decoded) { window_size += raw.size; } } else { - U8 *copy = push_array_no_zero(arena, U8, raw.size); - MemoryCopy(copy, raw.str, raw.size); + U8 *copy = raw.str; + if (!explicitly_decoded) { + copy = push_array_no_zero(arena, U8, raw.size); + MemoryCopy(copy, raw.str, raw.size); + } lnk_obj_apply_relocs_to_buffer(obj, section_number, section.header, str8(copy, raw.size), image_base, image_section_table); sect_base[sect_idx] = copy; window_size += raw.size; @@ -3768,6 +4030,7 @@ lnk_obj_window_debug_s(Arena *arena, LNK_CodeViewInput *cv, U64 obj_idx, U64 ima } } + lnk_compressed_obj_release_window(&decode_window); return out; } @@ -3795,21 +4058,30 @@ THREAD_POOL_TASK_FUNC(lnk_count_unique_leaf_sizes_task) // pointed result.v into the input). result.v now points into the copy: identical bytes, identical // order, clean input pages. internal -THREAD_POOL_TASK_FUNC(lnk_materialize_unique_leaves_task) +THREAD_POOL_TASK_FUNC(lnk_assign_unique_leaf_destinations_task) { - ProfBeginFunction(); LNK_MergeTypes *task = raw_task; Rng1U64 range = task->ranges[task_id]; - Arena *fixed_arena = task->fixed_arenas[task_id]; U8 *cursor = task->leaf_buffer + task->leaf_buffer_offsets[task_id]; for EachInRange(i, range) { - LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; + U64 meta = (U64)task->result.v[task->ti_source][i]; + U64 raw_size = meta >> 2; + U64 rewrite = meta & 3; + Assert(((U64)cursor & 3) == 0); + task->result.v[task->ti_source][i] = (U8 *)((U64)cursor | rewrite); + cursor += raw_size; + } +} + +internal void +lnk_materialize_unique_leaf_to(LNK_MergeTypes *task, CV_TypeIndexSource source, U64 i, + U8 *cursor, U64 raw_size, U64 rewrite, + Arena *fixed_arena, LNK_CObjDecodeWindow *decode_window) +{ + LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[source].v[i]; U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; - U64 meta = (U64)task->result.v[task->ti_source][i]; - U64 raw_size = meta >> 2; - U64 rewrite = meta & 3; // copy raw leaf into the private buffer. Hot path = the ORIGINAL read+memcpy (one bitmap // test on top); journaled leaves replay the NOTYPE rewrite into the COPY (full rewrite => @@ -3819,24 +4091,93 @@ THREAD_POOL_TASK_FUNC(lnk_materialize_unique_leaves_task) memory_write16(cursor + OffsetOf(CV_LeafHeader, size), sizeof(CV_LeafKind)); memory_write16(cursor + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); } else { - U8 *raw_leaf = debug_t->data.str + debug_t->offsets[leaf_idx]; - MemoryCopy(cursor, raw_leaf, raw_size); + LNK_Obj *obj = obj_idx < task->input->obj_count ? task->input->obj_arr[obj_idx] : 0; + B32 copied = 0; + if (obj && obj->compressed_obj && (debug_t->sidecar_sizes || debug_t->sidecar_packed)) { + U64 leaf_off = cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + Rng1U64 raw_range = rng_1u64(debug_t->sidecar_raw_base + leaf_off, + debug_t->sidecar_raw_base + leaf_off + raw_size); + copied = lnk_compressed_obj_copy_range(obj->compressed_obj, raw_range, cursor, decode_window); + } + if (!copied) { + U8 *raw_leaf = debug_t->data.str + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + MemoryCopy(cursor, raw_leaf, raw_size); + } if (rewrite == LNK_LEAF_MATERIALIZE_KIND_NOTYPE) { memory_write16(cursor + OffsetOf(CV_LeafHeader, kind), CV_LeafKind_NOTYPE); } } - task->result.v[task->ti_source][i] = cursor; - cursor += raw_size; + task->result.v[source][i] = cursor; // fixup type indices on the copy (same math the in-place leaf patcher applied) Temp temp = temp_begin(fixed_arena); CV_Leaf leaf = { - .kind = memory_read16(task->result.v[task->ti_source][i] + OffsetOf(CV_LeafHeader, kind)), - .data = str8(task->result.v[task->ti_source][i] + sizeof(CV_LeafHeader), raw_size - sizeof(CV_LeafHeader)), + .kind = memory_read16(task->result.v[source][i] + OffsetOf(CV_LeafHeader, kind)), + .data = str8(task->result.v[source][i] + sizeof(CV_LeafHeader), raw_size - sizeof(CV_LeafHeader)), }; CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); lnk_fixup_cv_type_indices(task, obj_idx, leaf.data, ti_offs); temp_end(temp); +} + +internal void +lnk_materialize_tagged_unique_leaf(LNK_MergeTypes *task, CV_TypeIndexSource source, U64 i, + Arena *fixed_arena, LNK_CObjDecodeWindow *decode_window) +{ + LNK_LeafRef leaf_ref = task->unique_leaf_refs_arr[source].v[i]; + U32 obj_idx = lnk_leaf_ref_obj_idx(leaf_ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(leaf_ref); + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + U64 tagged = (U64)task->result.v[source][i]; + U64 rewrite = tagged & 3; + U8 *cursor = (U8 *)(tagged & ~(U64)3); + U64 raw_size = rewrite == LNK_LEAF_MATERIALIZE_FULL_NOTYPE ? sizeof(CV_LeafHeader) : + cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx); + lnk_materialize_unique_leaf_to(task, source, i, cursor, raw_size, rewrite, fixed_arena, decode_window); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_materialize_unique_leaves_task) +{ + ProfBeginFunction(); + LNK_MergeTypes *task = raw_task; + Rng1U64 range = task->ranges[task_id]; + U8 *cursor = task->leaf_buffer + task->leaf_buffer_offsets[task_id]; + for EachInRange(i, range) { + U64 meta = (U64)task->result.v[task->ti_source][i]; + U64 raw_size = meta >> 2; + U64 rewrite = meta & 3; + lnk_materialize_unique_leaf_to(task, task->ti_source, i, cursor, raw_size, rewrite, + task->fixed_arenas[worker_id], &task->decode_windows[worker_id]); + cursor += raw_size; + } + ProfEnd(); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_materialize_fused_tpi_ipi_task) +{ + ProfBeginFunction(); + LNK_MergeTypes *task = raw_task; + U64 tpi_i = task->materialize_obj_offsets[CV_TypeIndexSource_TPI][task_id]; + U64 tpi_opl = task->materialize_obj_offsets[CV_TypeIndexSource_TPI][task_id + 1]; + U64 ipi_i = task->materialize_obj_offsets[CV_TypeIndexSource_IPI][task_id]; + U64 ipi_opl = task->materialize_obj_offsets[CV_TypeIndexSource_IPI][task_id + 1]; + while (tpi_i < tpi_opl || ipi_i < ipi_opl) { + CV_TypeIndexSource source; + U64 i; + if (ipi_i >= ipi_opl || + (tpi_i < tpi_opl && + lnk_leaf_ref_leaf_idx(task->unique_leaf_refs_arr[CV_TypeIndexSource_TPI].v[tpi_i]) < + lnk_leaf_ref_leaf_idx(task->unique_leaf_refs_arr[CV_TypeIndexSource_IPI].v[ipi_i]))) { + source = CV_TypeIndexSource_TPI; + i = tpi_i++; + } else { + source = CV_TypeIndexSource_IPI; + i = ipi_i++; + } + lnk_materialize_tagged_unique_leaf(task, source, i, task->fixed_arenas[worker_id], + &task->decode_windows[worker_id]); } ProfEnd(); } @@ -3854,6 +4195,56 @@ THREAD_POOL_TASK_FUNC(lnk_unbucket_hashes_task) } } +internal void +lnk_profile_cobj_winner_segments(LNK_MergeTypes *task) +{ + char *env = getenv("RAD_COBJ_PROFILE_WINNERS"); + if (env == 0 || env[0] == '0') { return; } + enum { BucketCount = 16, SizeCount = 4 }; + U64 segment_sizes[SizeCount] = { KB(128), KB(256), KB(512), MB(1) }; + U64 touched[CV_TypeIndexSource_COUNT][BucketCount][SizeCount] = {0}; + U64 leaf_counts[CV_TypeIndexSource_COUNT][BucketCount] = {0}; + U64 leaf_bytes[CV_TypeIndexSource_COUNT][BucketCount] = {0}; + + for (U32 source = CV_TypeIndexSource_TPI; source <= CV_TypeIndexSource_IPI; ++source) { + U64 last_key[SizeCount] = { max_U64, max_U64, max_U64, max_U64 }; + LNK_LeafRefArray refs = task->unique_leaf_refs_arr[source]; + for EachIndex(i, refs.count) { + LNK_LeafRef ref = refs.v[i]; + U32 obj_idx = lnk_leaf_ref_obj_idx(ref); + U32 leaf_idx = lnk_leaf_ref_leaf_idx(ref); + if (obj_idx >= task->input->obj_count) { continue; } + LNK_Obj *obj = task->input->obj_arr[obj_idx]; + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + if (!obj->compressed_obj || (!debug_t->sidecar_sizes && !debug_t->sidecar_packed)) { continue; } + U64 bucket = (U64)obj_idx * BucketCount / task->input->obj_count; + U64 raw_off = debug_t->sidecar_raw_base + cv_debug_t_get_leaf_offset(debug_t, leaf_idx); + leaf_counts[source][bucket] += 1; + leaf_bytes[source][bucket] += cv_debug_t_get_raw_leaf_size(debug_t, leaf_idx); + for (U32 s = 0; s < SizeCount; ++s) { + U64 key = ((U64)obj_idx << 32) | (raw_off / segment_sizes[s]); + if (key != last_key[s]) { + touched[source][bucket][s] += 1; + last_key[s] = key; + } + } + } + } + + for (U32 source = CV_TypeIndexSource_TPI; source <= CV_TypeIndexSource_IPI; ++source) { + for (U32 bucket = 0; bucket < BucketCount; ++bucket) { + lnk_log(LNK_Log_Timers, + "[cobj winners] src=%u bucket=%u objs=%u-%u leaves=%llu leafMiB=%llu seg128=%llu seg256=%llu seg512=%llu seg1024=%llu", + source, bucket, + (task->input->obj_count * bucket) / BucketCount, + (task->input->obj_count * (bucket + 1)) / BucketCount - 1, + leaf_counts[source][bucket], leaf_bytes[source][bucket] / MB(1), + touched[source][bucket][0], touched[source][bucket][1], + touched[source][bucket][2], touched[source][bucket][3]); + } + } +} + internal THREAD_POOL_TASK_FUNC(lnk_build_obj_ti_map) { @@ -3869,8 +4260,7 @@ THREAD_POOL_TASK_FUNC(lnk_build_obj_ti_map) U64 notype_cap = input->notype_journal[obj_idx].bit_cap; for EachIndex(leaf_idx, debug_t->count) { - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); - CV_LeafKind kind = leaf.kind; + CV_LeafKind kind = cv_debug_t_get_leaf_kind(debug_t, leaf_idx); if (notype_bm && leaf_idx < notype_cap && ((notype_bm[leaf_idx >> 6] >> (leaf_idx & 63)) & 1)) { kind = CV_LeafKind_NOTYPE; } CV_TypeIndexSource source = cv_type_index_source_from_leaf_kind(kind); LNK_LeafRef leaf_ref = lnk_leaf_ref_make(obj_idx, leaf_idx); @@ -3893,6 +4283,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK // deep-hash stack frames; 2x the legacy per-node list bound keeps comfortable headroom U64 max_ti_list_size = 2 * sizeof(CV_TypeIndexInfo) * (max_U16 / sizeof(CV_TypeIndex)); task.fixed_arenas = alloc_fixed_size_arena_array(scratch.arena, tp->worker_count, max_ti_list_size, max_ti_list_size); + task.decode_windows = push_array(scratch.arena, LNK_CObjDecodeWindow, tp->worker_count); ProfBegin("Produce Hashes"); { @@ -4153,6 +4544,35 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK ProfBegin("Extract present buckets from the leaf hash tables"); + B32 winner_prefetch_enabled = 0; + { + char *env = getenv("RAD_COBJ_PREFETCH_WINNERS"); + winner_prefetch_enabled = (env == 0 || env[0] != '0'); + if (winner_prefetch_enabled) { + task.winner_segment_offsets = push_array(scratch.arena, U64, input->obj_count + 1); + for EachIndex(obj_idx, input->obj_count) { + task.winner_segment_offsets[obj_idx + 1] = task.winner_segment_offsets[obj_idx] + + lnk_compressed_obj_segment_count(input->obj_arr[obj_idx]->compressed_obj); + } + U64 total_segments = task.winner_segment_offsets[input->obj_count]; + if (total_segments) { + task.winner_segment_word_count = CeilIntegerDiv(total_segments, 64); + task.winner_segment_bitmap = push_array(scratch.arena, U64, task.winner_segment_word_count); + task.winner_segment_worker_bitmaps = push_array(scratch.arena, U64, + task.winner_segment_word_count * tp->worker_count); + task.winner_segment_keys = push_array_no_zero(scratch.arena, U64, total_segments); + } + } + } + + U32 winner_prefetch_worker_count = 8; + { + char *env = getenv("RAD_COBJ_PREFETCH_WORKERS"); + if (env) { winner_prefetch_worker_count = Clamp(1, atoi(env), LNK_WINNER_PREFETCH_WORKER_MAX); } + } + LNK_WinnerPrefetchFlight winner_prefetch_flights[CV_TypeIndexSource_COUNT] = {0}; + U64 winner_prefetch_key_cursor = 0; + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { task.ti_source = ti_source; task.counts[ti_source] = push_array(scratch.arena, U64, tp->worker_count); @@ -4168,6 +4588,49 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK ti_source, task.unique_leaf_refs_arr[ti_source].count, task.leaf_ht_arr[ti_source].cap, task.leaf_ht_arr[ti_source].cap ? (F64)task.unique_leaf_refs_arr[ti_source].count / (F64)task.leaf_ht_arr[ti_source].cap : 0.0); + // Each type source gets an independent flight. TPI remains in flight while IPI is extracted + // and sorted, and IPI may join it; both are joined only immediately before materialization. + // Reduce lane-private bitmaps after extraction. The bitmap is only ~180k bits for UEFN; + // scanning it is negligible and avoids a contended atomic operation for every winning leaf. + if (task.winner_segment_bitmap && ti_source != CV_TypeIndexSource_NULL) { + U64 obj_idx = 0; + for EachIndex(word_idx, task.winner_segment_word_count) { + U64 merged = 0; + for EachIndex(lane, tp->worker_count) { + merged |= task.winner_segment_worker_bitmaps[lane * task.winner_segment_word_count + word_idx]; + } + U64 new_bits = merged & ~task.winner_segment_bitmap[word_idx]; + task.winner_segment_bitmap[word_idx] |= merged; + while (new_bits) { + U64 bit_idx = ctz64(new_bits); + U64 global_idx = word_idx * 64 + bit_idx; + while (obj_idx + 1 < input->obj_count && + global_idx >= task.winner_segment_offsets[obj_idx + 1]) { + obj_idx += 1; + } + U64 segment_idx = global_idx - task.winner_segment_offsets[obj_idx]; + task.winner_segment_keys[task.winner_segment_key_count++] = (obj_idx << 32) | segment_idx; + new_bits &= new_bits - 1; + } + } + } + U64 key_opl = task.winner_segment_key_count; + if (key_opl > winner_prefetch_key_cursor) { + U64 count = key_opl - winner_prefetch_key_cursor; + Rng1U64 *ranges = push_array_no_zero(scratch.arena, Rng1U64, count); + U64 stored_bytes = 0; + for EachIndex(i, count) { + U64 key = task.winner_segment_keys[winner_prefetch_key_cursor + i]; + U32 obj_idx = (U32)(key >> 32); + U32 segment_idx = (U32)key; + ranges[i] = lnk_compressed_obj_stored_segment_range(input->obj_arr[obj_idx]->compressed_obj, segment_idx); + stored_bytes += dim_1u64(ranges[i]); + } + winner_prefetch_key_cursor = key_opl; + lnk_winner_prefetch_start(&winner_prefetch_flights[ti_source], winner_prefetch_worker_count, + count, ranges, stored_bytes); + } + // sort output leaves based on { location index, leaf index } to guarantee determinism ProfScope("Radix Sort") { u64_array_sort_radix_parallel(tp, task.unique_leaf_refs_arr[ti_source].count, task.unique_leaf_refs_arr[ti_source].v); @@ -4180,6 +4643,12 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } } + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + lnk_winner_prefetch_join(&winner_prefetch_flights[ti_source]); + } + + lnk_profile_cobj_winner_segments(&task); + // bucket_arr is fully consumed (copied into unique_leaf_refs / sorted) -- release the probe tables // now so this multi-GB working set is gone before the merge-types/PDB-build peak. Handed to a // background reaper thread: a serial VirtualFree(MEM_RELEASE) of these multi-GB committed blocks @@ -4239,6 +4708,15 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK tp_for_parallel_prof(tp, 0, input->obj_count, lnk_build_obj_ti_map, &task, "Build TI Map"); } + lnk_compressed_obj_log_phase_stats("before type materialize"); + U64 materialize_begin_us = now_time_us(); + B32 has_compressed_inputs = 0; + for EachIndex(obj_idx, input->obj_count) { + if (input->obj_arr[obj_idx]->compressed_obj) { has_compressed_inputs = 1; break; } + } + char *fuse_materialize_env = getenv("RAD_COBJ_FUSE_TYPE_MATERIALIZE"); + B32 fuse_materialize = has_compressed_inputs && + (fuse_materialize_env == 0 || fuse_materialize_env[0] != '0'); for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { LNK_LeafRefArray unique_leaf_refs = task.unique_leaf_refs_arr[ti_source]; @@ -4272,9 +4750,45 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.leaf_buffer = push_array_no_zero(tp_temp->v[0], U8, 1); } } - tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_materialize_unique_leaves_task, &task, "Materialize + Fixup Leaves"); + if (fuse_materialize && ti_source != CV_TypeIndexSource_NULL) { + tp_for_parallel(tp, 0, tp->worker_count, lnk_assign_unique_leaf_destinations_task, &task); + } else { + // Exact original raw-OBJ path: count/meta and destination copy remain a + // single lane-balanced traversal with no tagged-pointer setup pass. + tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_materialize_unique_leaves_task, &task, "Materialize NULL Leaves"); + } + } + + // TPI and IPI leaf kinds are interleaved in each .debug$T stream. Build an + // object range index for each sorted winner array, then merge the two ranges + // by leaf index inside one task. A compressed object's decode window now sees + // each physical segment once instead of once in the TPI pass and again in the + // IPI pass. Destination pointers were assigned above, so output order and + // bytes remain exactly the same as the separate passes. + if (fuse_materialize) { + for (U32 source = CV_TypeIndexSource_TPI; source <= CV_TypeIndexSource_IPI; ++source) { + LNK_LeafRefArray refs = task.unique_leaf_refs_arr[source]; + U64 *offsets = task.materialize_obj_offsets[source] = push_array_no_zero(scratch.arena, U64, input->count + 1); + U64 cursor = 0; + for EachIndex(obj_idx, input->count) { + offsets[obj_idx] = cursor; + while (cursor < refs.count && lnk_leaf_ref_obj_idx(refs.v[cursor]) == obj_idx) { cursor += 1; } + } + offsets[input->count] = cursor; + Assert(cursor == refs.count); + } + + tp_for_parallel_prof(tp, 0, input->count, lnk_materialize_fused_tpi_ipi_task, &task, "Materialize + Fixup TPI/IPI"); + } + lnk_log(LNK_Log_Timers, "[cobj materialize] fused=%u time=%.3fs", fuse_materialize, + (F64)(now_time_us() - materialize_begin_us) / 1e6); + lnk_compressed_obj_log_phase_stats("after type materialize"); - if (merge_flags & LNK_MergeTypeFlag_ExportHashes) { + if (merge_flags & LNK_MergeTypeFlag_ExportHashes) { + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + LNK_LeafRefArray unique_leaf_refs = task.unique_leaf_refs_arr[ti_source]; + task.ti_source = ti_source; + task.ranges = tp_divide_work(scratch.arena, unique_leaf_refs.count, tp->worker_count); task.result.hashes[ti_source] = push_array_no_zero(tp_temp->v[0], U64, unique_leaf_refs.count); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_unbucket_hashes_task, &task, "Export Hashes"); } @@ -4327,6 +4841,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK tp_for_parallel_prof(tp, 0, input->symbol_patch_task_count, lnk_journal_symbol_fixups_task, &task, "Journal Symbol Fixups"); tp_for_parallel_prof(tp, 0, input->count, lnk_journal_inline_fixups_task, &task, "Journal Inline Fixups"); input->has_debug_s_fixup_journal = 1; + lnk_compressed_obj_log_phase_stats("after journal $S fixups"); if (lnk_get_log_status(LNK_Log_Debug)) { U64 entry_count = 0, entry_bytes = 0, wide_runs = 0; @@ -4347,6 +4862,8 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK ProfEnd(); } + for EachIndex(i, tp->worker_count) { lnk_compressed_obj_release_window(&task.decode_windows[i]); } + MemoryCopyTyped(task.result.min_type_indices, input->min_type_indices, CV_TypeIndexSource_COUNT); temp_end(scratch); @@ -4565,6 +5082,8 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) LNK_BuildPdb *task = raw_task; PDB_GsiContext *gsi = task->pdb->gsi; PDB_PsiContext *psi = task->pdb->psi; + Assert(gsi->bucket_count == PDB_GSI_V70_BUCKET_COUNT); + Assert(psi->gsi->bucket_count == PDB_GSI_V70_BUCKET_COUNT); ProfBegin("Global Symbols"); { @@ -4740,7 +5259,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) for EachIndex(i, chunk->count) { LNK_Symbol *symbol = chunk->v[i].symbol; LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(symbol_ref.obj, symbol_ref.symbol_idx); if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); @@ -4783,7 +5302,7 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) for EachIndex(i, chunk->count) { LNK_Symbol *symbol = chunk->v[i].symbol; LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(symbol_ref.obj, symbol_ref.symbol_idx); // discard removed and non-section symbols if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } @@ -4924,46 +5443,63 @@ lnk_write_debug_s_to_pdb_module(PDB_DbiModule *mod, CV_DebugS debug_s, String8No mod->sym_data_size += sig_size; mod_cursor += sig_size; - // write symbols - U64 scope_depth = 0; - for EachNode(n, String8Node, symbols.first) { - U64 cand_depth = 0; - B32 cand_active = 1; - for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); - - if (pre != 0) { - if (cand_active && (cv_is_global_symbol(symbol.kind) || (cand_depth == 0 && cv_is_typedef(symbol.kind)))) { - pre->cand_count += 1; - } - if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32 || - symbol.kind == CV_SymKind_GPROC32_ID || symbol.kind == CV_SymKind_LPROC32_ID) { - pre->procref_count += 1; - } - if (cand_active) { - if (cv_is_scope_symbol(symbol.kind)) { - cand_depth += 1; - } else if (cv_is_end_symbol(symbol.kind)) { - if (cand_depth == 0) { Assert(0 && "malformed symbol stream"); cand_active = 0; } - else { cand_depth -= 1; } + U64 symbols_idx = cv_c13_sub_section_idx_from_kind(CV_C13SubSectionKind_Symbols); + CV_DebugSProvList *symbol_prov = &debug_s.prov_list[symbols_idx]; + B32 use_summary = (buf == 0 && pre != 0 && symbol_prov->count == symbols.node_count); + if (use_summary) { + for (CV_DebugSProvNode *prov = symbol_prov->first; prov != 0; prov = prov->next) { + if (!prov->symbol_summary_valid) { use_summary = 0; break; } + } + } + if (use_summary) { + for (CV_DebugSProvNode *prov = symbol_prov->first; prov != 0; prov = prov->next) { + mod_cursor += prov->module_symbol_size; + mod->sym_data_size += prov->module_symbol_size; + pre->cand_count += prov->gsi_candidate_count; + pre->procref_count += prov->proc_ref_count; + } + } else { + // write symbols + U64 scope_depth = 0; + for EachNode(n, String8Node, symbols.first) { + U64 cand_depth = 0; + B32 cand_active = 1; + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= n->string.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); + + if (pre != 0) { + if (cand_active && (cv_is_global_symbol(symbol.kind) || (cand_depth == 0 && cv_is_typedef(symbol.kind)))) { + pre->cand_count += 1; + } + if (symbol.kind == CV_SymKind_GPROC32 || symbol.kind == CV_SymKind_LPROC32 || + symbol.kind == CV_SymKind_GPROC32_ID || symbol.kind == CV_SymKind_LPROC32_ID) { + pre->procref_count += 1; + } + if (cand_active) { + if (cv_is_scope_symbol(symbol.kind)) { + cand_depth += 1; + } else if (cv_is_end_symbol(symbol.kind)) { + if (cand_depth == 0) { Assert(0 && "malformed symbol stream"); cand_active = 0; } + else { cand_depth -= 1; } + } } } - } - if (symbol.kind == CV_SymKind_SKIP) { continue; } - else if (cv_is_global_symbol(symbol.kind)) { continue; } - else if (cv_is_typedef(symbol.kind) && scope_depth == 0) { continue; } - else if (symbol.kind == 0x1176) { continue; } + if (symbol.kind == CV_SymKind_SKIP) { continue; } + else if (cv_is_global_symbol(symbol.kind)) { continue; } + else if (cv_is_typedef(symbol.kind) && scope_depth == 0) { continue; } + else if (symbol.kind == 0x1176) { continue; } - if (cv_is_scope_symbol(symbol.kind)) { scope_depth += 1; } - else if (cv_is_end_symbol(symbol.kind)) { scope_depth -= 1; } + if (cv_is_scope_symbol(symbol.kind)) { scope_depth += 1; } + else if (cv_is_end_symbol(symbol.kind)) { scope_depth -= 1; } - String8 raw_symbol = cv_raw_from_symbol(cv_ptr_from_symbol(symbol)); - U64 symbol_size = (raw_symbol.size & (PDB_SYMBOL_ALIGN - 1)) == 0 ? str8_buffer_write(buf, buf_pos, raw_symbol) - : cv_write_symbol_buf(buf, buf_pos, &symbol, PDB_SYMBOL_ALIGN); - mod_cursor += symbol_size; - mod->sym_data_size += symbol_size; + String8 raw_symbol = cv_raw_from_symbol(cv_ptr_from_symbol(symbol)); + U64 symbol_size = (raw_symbol.size & (PDB_SYMBOL_ALIGN - 1)) == 0 ? str8_buffer_write(buf, buf_pos, raw_symbol) + : cv_write_symbol_buf(buf, buf_pos, &symbol, PDB_SYMBOL_ALIGN); + mod_cursor += symbol_size; + mod->sym_data_size += symbol_size; + } } } } @@ -5217,19 +5753,15 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) String8List mod_data = msf_data_from_sn(temp.arena, task->pdb->msf, mod->sn); if (mod_data.node_count) { -#if BUILD_DEBUG // raw-header sizing vs post-transform write must agree byte-for-byte U64 size_check[4] = { mod->sym_data_size, mod->c11_data_size, mod->c13_data_size, mod->globrefs_size }; -#endif String8Node buf = *mod_data.first; U64 pos = 0; lnk_write_debug_s_to_pdb_module(mod, debug_s, &buf, &pos, 0); -#if BUILD_DEBUG - Assert(mod->sym_data_size == size_check[0]); - Assert(mod->c11_data_size == size_check[1]); - Assert(mod->c13_data_size == size_check[2]); - Assert(mod->globrefs_size == size_check[3]); -#endif + AssertAlways(mod->sym_data_size == size_check[0]); + AssertAlways(mod->c11_data_size == size_check[1]); + AssertAlways(mod->c13_data_size == size_check[2]); + AssertAlways(mod->globrefs_size == size_check[3]); // sub range symbol data pages and patch symbol tree offsets if (mod->sym_data_size) { @@ -5356,6 +5888,7 @@ THREAD_POOL_TASK_FUNC(lnk_write_pdb_modules) if (task->output != 0) { lnk_pdb_output_enqueue_stream(task->output, task->pdb->msf, mod->sn); } + } barrier_wait(tp->barrier); } @@ -6139,6 +6672,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config pdb_strtab_add_cv_string_hash_table(&task.pdb->info->strtab, task.string_ht); ProfEnd(); pdb_build_types(tp, task.pdb, &build_hooks); + lnk_compressed_obj_log_phase_stats("before write modules"); lnk_pdb_output_log_mark(output_ptr, "after pdb_build_types (TPI/IPI sealed)"); // merged leaf bytes are now in MSF pages (and on their way to disk); no consumer of @@ -6194,6 +6728,7 @@ ProfScope("Write Modules") #endif tp_for_parallel_reserve(tp, 0, C, lnk_write_pdb_modules, &task); // BARRIER pass (path B): barrier_wait/tp_broadcast tp_barrier_end(tp); + lnk_compressed_obj_log_phase_stats("after write modules"); lnk_log(LNK_Log_Timers, "[pdb] write modules in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); lnk_pdb_output_log_mark(output_ptr, "after write modules"); if (g_debug_s_window && lnk_get_log_status(LNK_Log_Debug)) { @@ -6312,8 +6847,11 @@ ProfScope("Move Global Symbols") !(config->io_flags & LNK_IO_Flags_MemoryMapFilesReadWrite)) { ProfBegin("Release Input Views Early"); U64 unmap_begin_us = now_time_us(); + // All thread-pool consumers above have joined. Keep compressed-object teardown at this + // explicit quiescence point; per-object or background close can race lazy materialization. lnk_inputer_release_file_maps(tp, config->debug_worker_cap, inputer); - lnk_log(LNK_Log_Timers, "[pdb] early input-view release in %.2f ms", (F64)(now_time_us() - unmap_begin_us) / 1000.0); + lnk_log(LNK_Log_Timers, "[pdb] early input-view release in %.2f ms", + (F64)(now_time_us() - unmap_begin_us) / 1000.0); ProfEnd(); } diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 8c3e89393..c220d2f3e 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -3,6 +3,8 @@ #pragma once +typedef struct LNK_CObjDecodeWindow LNK_CObjDecodeWindow; + //////////////////////////////// // RRT @@ -204,6 +206,7 @@ typedef struct LNK_CodeViewInput *input; String8Array *raw_types; // [obj_count] CV_DebugT *out_types; // [obj_count] + B32 is_debug_p; } LNK_ParseCvTypes; //////////////////////////////// @@ -267,6 +270,7 @@ typedef struct LNK_LeafHashTable leaf_ht_arr[CV_TypeIndexSource_COUNT]; LNK_AssignedTiHash assigned_ti_arr[CV_TypeIndexSource_COUNT]; Arena **fixed_arenas; + struct LNK_CObjDecodeWindow *decode_windows; CV_TypeIndexSource ti_source; U32Array indices; Rng1U64 *ranges; @@ -304,6 +308,17 @@ typedef struct // fixup never dirties the copy-on-write input mapping) U64 *leaf_buffer_offsets; // [worker_count+1] per-lane byte offsets into leaf_buffer U8 *leaf_buffer; + U64 *materialize_obj_offsets[CV_TypeIndexSource_COUNT]; // [input->count+1], sorted-ref ranges + + // Compressed winner payload prefetch. segment_offsets maps an OBJ-local segment index to a + // compact global bit/key index. Copy-present atomically claims segments as it discovers the + // winning leaves, so no later scan of the multi-million-entry winner arrays is needed. + U64 *winner_segment_offsets; // [input->obj_count+1] + U64 *winner_segment_bitmap; + U64 *winner_segment_worker_bitmaps; // [worker_count * winner_segment_word_count] + U64 winner_segment_word_count; + U64 *winner_segment_keys; // obj_idx:32 | segment_idx:32 + U64 winner_segment_key_count; // $S fixup journal build: per-worker arenas the LNK_DebugSPatch entry arrays land on // (must outlive the merge -- replay happens at module write) @@ -427,10 +442,10 @@ internal B32 lnk_match_leaf_ref (LNK_CodeViewInput internal void lnk_notype_journal_push (Arena *arena, LNK_NotypeJournal *journal, U32 leaf_idx, B32 kind_only, U64 bit_cap); internal B32 lnk_notype_journal_test (LNK_NotypeJournal *journal, U64 leaf_idx); internal B32 lnk_notype_journal_find (LNK_NotypeJournal *journal, U32 leaf_idx, B32 *kind_only_out); -internal CV_Leaf lnk_cv_leaf_from_leaf_ref (LNK_CodeViewInput *input, U32 obj_idx, U32 leaf_idx); +internal CV_Leaf lnk_cv_leaf_from_leaf_ref (Arena *arena, LNK_CObjDecodeWindow *decode_window, LNK_CodeViewInput *input, U32 obj_idx, U32 leaf_idx); internal U64 lnk_leaf_ref_materialize_meta (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, Arena *journal_arena, LNK_LeafRef leaf_ref, CV_Leaf leaf, CV_TiOffsets ti_offs, B32 discard_cycles); -internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs); +internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CObjDecodeWindow *decode_window, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs); internal CV_TypeIndex lnk_assigned_ti_hash_search (LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); internal LNK_MergedTypes lnk_merge_types (TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK_MergeTypeFlags merge_flags); internal void lnk_apply_debug_s_fixups_for_obj (LNK_CodeViewInput *cv, U64 obj_idx); diff --git a/src/linker/lnk_io.c b/src/linker/lnk_io.c index 3b0049a55..2cea45899 100644 --- a/src/linker/lnk_io.c +++ b/src/linker/lnk_io.c @@ -340,6 +340,27 @@ THREAD_POOL_TASK_FUNC(lnk_memory_map_file_task) scratch_end(scratch); } +// Experimental lane cap for opening thousands of independent object mappings. Each lane pulls +// file indices from a shared cursor, so this preserves the ordinary per-file mapping operation and +// output ordering while letting us measure the Windows object-manager concurrency knee. +typedef struct +{ + LNK_DiskReader *reader; + U64 item_count; + U64 cursor; +} LNK_MemoryMapCappedTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_memory_map_file_capped_task) +{ + LNK_MemoryMapCappedTask *task = raw_task; + for (;;) { + U64 item_idx = ins_atomic_u64_inc_eval(&task->cursor) - 1; + if (item_idx >= task->item_count) { break; } + lnk_memory_map_file_task(arena, worker_id, item_idx, task->reader, tp); + } +} + internal String8Array lnk_read_data_from_file_path_parallel(TP_Context *tp, Arena *arena, LNK_IO_Flags io_flags, String8Array path_arr, B8 *was_read) { @@ -352,7 +373,33 @@ lnk_read_data_from_file_path_parallel(TP_Context *tp, Arena *arena, LNK_IO_Flags reader.data_arr.count = path_arr.count; reader.data_arr.v = push_array(arena, String8, path_arr.count); reader.was_read = was_read; - tp_for_parallel(tp, 0, path_arr.count, lnk_memory_map_file_task, &reader); + char *map_worker_env = getenv("RAD_COBJ_MAP_WORKERS"); + U64 map_worker_cap = map_worker_env ? strtoull(map_worker_env, 0, 10) : 0; +#if OS_WINDOWS + if (!map_worker_env && path_arr.count >= 64) { + // Standalone compressed objects expose a sparse raw COFF view. A small sample detects the + // homogeneous compressed corpus without opening or touching ordinary raw object contents. + // Raw links retain the original unrestricted tp_for_parallel call below. + Temp detect_scratch = scratch_begin(&arena, 1); + U64 sample_count = Min(path_arr.count, 32); + for EachIndex(i, sample_count) { + String16 path16 = str16_from_8(detect_scratch.arena, path_arr.v[i]); + DWORD attrs = GetFileAttributesW(path16.str); + if (attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_SPARSE_FILE)) { + map_worker_cap = 8; + break; + } + } + scratch_end(detect_scratch); + } +#endif + if (map_worker_cap > 0 && map_worker_cap < path_arr.count) { + LNK_MemoryMapCappedTask map_task = { .reader = &reader, .item_count = path_arr.count }; + tp_for_parallel(tp, 0, map_worker_cap, lnk_memory_map_file_capped_task, &map_task); + } else { + // Default/raw behavior remains the original unrestricted parallel-for. + tp_for_parallel(tp, 0, path_arr.count, lnk_memory_map_file_task, &reader); + } } else { Temp scratch = scratch_begin(&arena,1); diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index 9a30e8045..220e325fe 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -1,6 +1,25 @@ // Copyright (c) Epic Games Tools // Licensed under the MIT license (https://opensource.org/license/mit/) +typedef struct LNK_CompressedObjCensus +{ + U64 obj_count; + U64 raw_bytes; + U64 section_bytes; + U64 debug_s_bytes; + U64 debug_t_bytes; + U64 debug_p_bytes; + U64 debug_h_bytes; + U64 other_section_bytes; + U64 symbol_bytes; + U64 string_bytes; + U64 section_table_bytes; + U64 reloc_bytes; +} LNK_CompressedObjCensus; + +global LNK_CompressedObjCensus g_lnk_compressed_obj_census; +global B32 g_lnk_compressed_obj_census_enabled; + internal String8 lnk_loc_from_obj(Arena *arena, LNK_Obj *obj) { @@ -107,6 +126,12 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) U32 debug_p_section_number = 0; U32 debug_h_section_number = 0; U32 llvm_addrsig_section_number = 0; + U64 census_section_bytes = 0; + U64 census_debug_s_bytes = 0; + U64 census_debug_t_bytes = 0; + U64 census_debug_p_bytes = 0; + U64 census_debug_h_bytes = 0; + U64 census_reloc_bytes = 0; for EachIndex(sect_idx, header.section_count_no_null) { U64 section_number = sect_idx + 1; COFF_SectionHeader *coff_sect_header = &coff_section_table[sect_idx]; @@ -114,6 +139,15 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) String8 sect_name = coff_name_from_section_header(raw_coff_string_table, coff_sect_header); *section_flags = coff_sect_header->flags & ~3; // linker reserves low 2 bits for internal flags + if (g_lnk_compressed_obj_census_enabled) { + census_section_bytes += coff_sect_header->fsize; + census_reloc_bytes += (U64)coff_sect_header->reloc_count * sizeof(COFF_Reloc); + if (str8_match(sect_name, str8_lit(".debug$S"), 0)) { census_debug_s_bytes += coff_sect_header->fsize; } + if (str8_match(sect_name, str8_lit(".debug$T"), 0)) { census_debug_t_bytes += coff_sect_header->fsize; } + if (str8_match(sect_name, str8_lit(".debug$P"), 0)) { census_debug_p_bytes += coff_sect_header->fsize; } + if (str8_match(sect_name, str8_lit(".debug$H"), 0)) { census_debug_h_bytes += coff_sect_header->fsize; } + } + if (str8_starts_with(sect_name, str8_lit(".debug$"))) { *section_flags |= LNK_SECTION_FLAG_DEBUG; } @@ -362,21 +396,44 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) String8 name = str8_cstring_capped(sect_header->name, sect_header->name+sizeof(sect_header->name)); if (str8_match(name, str8_lit(".debug$S"), 0)) { Temp temp = temp_begin(scratch.arena); - String8 debug_s_data = str8_substr(input->data, rng_1u64(sect_header->foff, sect_header->foff+sect_header->fsize)); - CV_DebugS debug_s = cv_debug_s_from_data(temp.arena, debug_s_data); - cv_debug_s_tag_prov_sect(&debug_s, (U32)sect_idx); - String8List symbols_list = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); - for EachNode(symbols_n, String8Node, symbols_list.first) { - for (U64 cursor = 0, count = 0; cursor < symbols_n->string.size && count < 2; count += 1) { - CV_SymbolHeader symbol_header; - TryReadBreak(str8_deserial_read_struct(symbols_n->string, cursor, &symbol_header), cursor); - if (symbol_header.kind == CV_SymKind_COMPILE3) { - String8 raw_symbol = str8_substr(symbols_n->string, r1u64(cursor, cursor + symbol_header.size + sizeof(CV_SymSize))); - comp_symbol = cv_symbol_from_ptr(raw_symbol.str); - goto found_comp_symbol; + Rng1U64 debug_s_range = rng_1u64(sect_header->foff, sect_header->foff+sect_header->fsize); + LNK_CObjDebugSView indexed = {0}; + if (lnk_compressed_obj_debug_s_index(input->compressed_obj, debug_s_range, &indexed)) { + // COMPILE3 is expected in the first two records of a Symbols payload. The sidecar + // lets this early object-feature probe skip the otherwise full C13 header walk. + for EachIndex(entry_idx, indexed.count) { + LNK_CObjDebugSEntry *entry = &indexed.v[entry_idx]; + if (entry->kind != CV_C13SubSectionKind_Symbols) { continue; } + String8 symbols = str8(input->data.str + entry->raw_payload_offset, entry->raw_payload_size); + for (U64 cursor = 0, count = 0; cursor < symbols.size && count < 2; count += 1) { + CV_SymbolHeader symbol_header; + TryReadBreak(str8_deserial_read_struct(symbols, cursor, &symbol_header), cursor); + if (symbol_header.kind == CV_SymKind_COMPILE3) { + String8 raw_symbol = str8_substr(symbols, r1u64(cursor, cursor + symbol_header.size + sizeof(CV_SymSize))); + comp_symbol = cv_symbol_from_ptr(raw_symbol.str); + goto found_comp_symbol; + } + cursor += symbol_header.size + sizeof(CV_SymSize); + cursor = AlignPow2(cursor, CV_SymbolAlign); + } + } + } else { + String8 debug_s_data = str8_substr(input->data, debug_s_range); + CV_DebugS debug_s = cv_debug_s_from_data(temp.arena, debug_s_data); + cv_debug_s_tag_prov_sect(&debug_s, (U32)sect_idx); + String8List symbols_list = cv_sub_section_from_debug_s(debug_s, CV_C13SubSectionKind_Symbols); + for EachNode(symbols_n, String8Node, symbols_list.first) { + for (U64 cursor = 0, count = 0; cursor < symbols_n->string.size && count < 2; count += 1) { + CV_SymbolHeader symbol_header; + TryReadBreak(str8_deserial_read_struct(symbols_n->string, cursor, &symbol_header), cursor); + if (symbol_header.kind == CV_SymKind_COMPILE3) { + String8 raw_symbol = str8_substr(symbols_n->string, r1u64(cursor, cursor + symbol_header.size + sizeof(CV_SymSize))); + comp_symbol = cv_symbol_from_ptr(raw_symbol.str); + goto found_comp_symbol; + } + cursor += symbol_header.size + sizeof(CV_SymSize); + cursor = AlignPow2(cursor, CV_SymbolAlign); } - cursor += symbol_header.size + sizeof(CV_SymSize); - cursor = AlignPow2(cursor, CV_SymbolAlign); } } temp_end(temp); @@ -411,10 +468,41 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) .llvm_addrsig_section_number = llvm_addrsig_section_number, .hotpatch = hotpatch, }; + obj->compressed_obj = input->compressed_obj; obj->path = push_str8_copy(arena, input->path); obj->exclude_from_debug_info = input->exclude_from_debug_info; obj->self = &task->objs[task_id]; obj->link_member = input->link_member; + + if (input->compressed_obj != 0 && g_lnk_compressed_obj_census_enabled) { + U64 known_debug = census_debug_s_bytes + census_debug_t_bytes + census_debug_p_bytes + census_debug_h_bytes; + InterlockedIncrement64((volatile LONG64 *)&g_lnk_compressed_obj_census.obj_count); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.raw_bytes, input->data.size); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.section_bytes, census_section_bytes); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.debug_s_bytes, census_debug_s_bytes); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.debug_t_bytes, census_debug_t_bytes); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.debug_p_bytes, census_debug_p_bytes); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.debug_h_bytes, census_debug_h_bytes); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.other_section_bytes, census_section_bytes - known_debug); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.symbol_bytes, raw_coff_symbol_table.size); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.string_bytes, raw_coff_string_table.size); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.section_table_bytes, raw_coff_section_table.size); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.reloc_bytes, census_reloc_bytes); + } +} + +internal void +lnk_obj_log_compressed_census(void) +{ + if (!g_lnk_compressed_obj_census_enabled || g_lnk_compressed_obj_census.obj_count == 0) { return; } + LNK_CompressedObjCensus *c = &g_lnk_compressed_obj_census; + lnk_log(LNK_Log_Timers, + "[cobj census] objs=%llu raw=%.2f GiB sections=%.2f GiB debugS=%.2f GiB debugT=%.2f GiB debugP=%.2f GiB debugH=%.2f GiB other=%.2f GiB symbols=%.2f GiB strings=%.2f GiB sect_headers=%.2f GiB relocs=%.2f GiB", + c->obj_count, (F64)c->raw_bytes/GB(1), (F64)c->section_bytes/GB(1), + (F64)c->debug_s_bytes/GB(1), (F64)c->debug_t_bytes/GB(1), (F64)c->debug_p_bytes/GB(1), + (F64)c->debug_h_bytes/GB(1), (F64)c->other_section_bytes/GB(1), + (F64)c->symbol_bytes/GB(1), (F64)c->string_bytes/GB(1), + (F64)c->section_table_bytes/GB(1), (F64)c->reloc_bytes/GB(1)); } internal LNK_ObjNode * @@ -422,6 +510,8 @@ lnk_obj_from_input_many(TP_Context *tp, TP_Arena *arena, LNK_Config *config, U64 { LNK_ObjNode *objs = 0; if (inputs_count) { + char *census_env = getenv("RAD_COBJ_CENSUS"); + g_lnk_compressed_obj_census_enabled = census_env != 0 && census_env[0] != 0 && census_env[0] != '0'; objs = push_array(arena->v[0], LNK_ObjNode, inputs_count); LNK_ObjIniter task = { .inputs = inputs, @@ -701,7 +791,9 @@ lnk_obj_section_data_from_number(LNK_Obj *obj, U64 section_number) } COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(obj->coff.data, obj->coff.header.section_table_range).str; COFF_SectionHeader *section = §ion_table[section_number-1]; - return str8_substr(obj->coff.data, r1u64s(section->foff, section->fsize)); + Rng1U64 range = r1u64s(section->foff, section->fsize); + String8 direct = lnk_compressed_obj_direct_range(obj->compressed_obj, range); + return direct.size ? direct : str8_substr(obj->coff.data, range); } internal U64 diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index c35c254f1..bca540d25 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -58,6 +58,7 @@ typedef struct LNK_Obj String8 path; LNK_ObjCoff coff; + struct LNK_CompressedObj *compressed_obj; // flags B8 exclude_from_debug_info; @@ -313,6 +314,7 @@ internal force_inline B32 lnk_obj_symbol_iter_next(LNK_Obj *obj, LNK_ObjSymbolIt internal String8List * lnk_collect_obj_sections(TP_Context *tp, TP_Arena *arena, U64 objs_count, LNK_Obj **objs, String8 name, B32 collect_discarded, U32Array **sect_indices_out); internal B32 lnk_obj_is_before(void *raw_a, void *raw_b); +internal void lnk_obj_log_compressed_census(void); // --- Directive Parser -------------------------------------------------------- diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index a5adab703..be7ae49d2 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -659,7 +659,8 @@ lnk_symbol_table_searchf(LNK_SymbolTable *symtab, char *fmt, ...) internal ISectOff lnk_sc_from_symbol(LNK_Symbol *symbol) { - COFF_ParsedSymbol parsed_symbol = lnk_parsed_from_symbol(symbol); + LNK_ObjSymbolRef ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol parsed_symbol = lnk_parsed_symbol_from_coff_symbol_idx_no_name(ref.obj, ref.symbol_idx); ISectOff sc = { .isect = parsed_symbol.section_number, .off = parsed_symbol.value }; return sc; } diff --git a/src/linker/pdb_ext/pdb_builder.c b/src/linker/pdb_ext/pdb_builder.c index f519d8944..f5f183140 100644 --- a/src/linker/pdb_ext/pdb_builder.c +++ b/src/linker/pdb_ext/pdb_builder.c @@ -2388,6 +2388,7 @@ gsi_reserve(PDB_GsiContext *gsi, U64 bucket_idx, U64 additional) internal CV_Symbol * gsi_push_(PDB_GsiContext *gsi, U32 hash, CV_Symbol *symbol) { + Assert(gsi->bucket_count == PDB_GSI_V70_BUCKET_COUNT); U64 bucket_idx = hash & (PDB_GSI_V70_BUCKET_COUNT - 1); PDB_GsiSymbolBucket *bucket = &gsi->bucket_arr[bucket_idx]; if (bucket->count == bucket->cap) { @@ -2463,6 +2464,7 @@ gsi_push_many_list(PDB_GsiContext *gsi, U64 count, U32 *hash_arr, CV_SymbolList internal CV_Symbol * gsi_search(PDB_GsiContext *gsi, CV_Symbol *symbol) { + Assert(gsi->bucket_count == PDB_GSI_V70_BUCKET_COUNT); String8 name = cv_name_from_symbol(symbol->kind, symbol->data); U32 hash = gsi_hash(gsi, name); U64 ibucket = hash & (PDB_GSI_V70_BUCKET_COUNT - 1); From 1703d0598f71ff78332ea57492d29c454dc8e40b Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Sun, 16 Aug 2026 09:56:09 -0700 Subject: [PATCH 110/121] tools: add portable compressed OBJ writer --- build.bat | 20 +- src/linker/rad_obj_compress.c | 744 ++++++++++++++++++++++++++++++++++ 2 files changed, 763 insertions(+), 1 deletion(-) create mode 100644 src/linker/rad_obj_compress.c diff --git a/build.bat b/build.bat index 660248983..0e0cf8c95 100644 --- a/build.bat +++ b/build.bat @@ -27,6 +27,7 @@ cd /D "%~dp0" :: - `ubsan`: enable undefined-behavior sanitizer :: - `telemetry`: enable RAD telemetry profiling support :: - `spall`: enable spall profiling support +:: - `oodle`: enable Oodle-compressed OBJ support in radlink (requires OODLE_SDK_DIR) :: --- Unpack Arguments ------------------------------------------------------- for %%a in (%*) do set "%%~a=1" @@ -161,7 +162,24 @@ pushd build if "%raddbg%"=="1" set didbuild=1 && %compile% ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg.exe || exit /b 1 if "%raddbg_non_graphical%"=="1" set didbuild=1 && %compile% -DWM_STUB=1 -DR_BACKEND=R_BACKEND_STUB ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg_non_graphical.exe || exit /b 1 if "%com_shim%"=="1" set didbuild=1 && %compile% ..\src\com_shim\com_shim_main.c %compile_link% %out%com_shim.exe || exit /b 1 -if "%radlink%"=="1" set didbuild=1 && %compile% ..\src\linker\lnk.c %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 +set radlink_oodle_flags= +set radlink_oodle_lib= +set rad_obj_compress_oodle_flags= +set rad_obj_compress_oodle_lib= +if "%oodle%"=="1" if "%OODLE_SDK_DIR%"=="" ( + echo OODLE_SDK_DIR must name an Oodle SDK when building radlink with oodle + exit /b 1 +) +if "%oodle%"=="1" set radlink_oodle_flags=/I"%OODLE_SDK_DIR%\include" -DLNK_OODLE=1 +if "%oodle%"=="1" set radlink_oodle_lib="%OODLE_SDK_DIR%\lib\Win64\oo2core_win64.lib" +if "%rad_obj_compress%"=="1" if "%OODLE_SDK_DIR%"=="" ( + echo OODLE_SDK_DIR must name an Oodle SDK when building rad_obj_compress + exit /b 1 +) +if "%rad_obj_compress%"=="1" set rad_obj_compress_oodle_flags=/I"%OODLE_SDK_DIR%\include" +if "%rad_obj_compress%"=="1" set rad_obj_compress_oodle_lib="%OODLE_SDK_DIR%\lib\Win64\oo2core_win64.lib" +if "%radlink%"=="1" set didbuild=1 && %compile% %radlink_oodle_flags% ..\src\linker\lnk.c %radlink_oodle_lib% %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 +if "%rad_obj_compress%"=="1" set didbuild=1 && %compile% %rad_obj_compress_oodle_flags% ..\src\linker\rad_obj_compress.c %rad_obj_compress_oodle_lib% %compile_link% %out%rad_obj_compress.exe || exit /b 1 if "%radbin%"=="1" set didbuild=1 && %compile% ..\src\radbin\radbin_main.c %compile_link% %out%radbin.exe || exit /b 1 if "%raddump%"=="1" set didbuild=1 && %compile% ..\src\raddump\raddump_main.c %compile_link% %out%raddump.exe || exit /b 1 if "%ryan_scratch%"=="1" set didbuild=1 && %compile% ..\src\scratch\ryan_scratch.c %compile_link% %out%ryan_scratch.exe || exit /b 1 diff --git a/src/linker/rad_obj_compress.c b/src/linker/rad_obj_compress.c new file mode 100644 index 000000000..626e21578 --- /dev/null +++ b/src/linker/rad_obj_compress.c @@ -0,0 +1,744 @@ +#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include +#include +#include +#include +#include + +typedef uint8_t U8; +typedef uint16_t U16; +typedef uint32_t U32; +typedef uint64_t U64; +#include "lnk_compressed_obj_format.h" + +// This standalone reference writer emits one self-contained portable OBJ. It independently +// compresses every 512 KiB segment by default, preserves incompressible segments as direct-mapped +// raw runs, builds the linker sidecars, verifies every compressed round trip, and publishes via +// an atomic rename. A compiler-integrated writer can emit the same format without the raw input. + +//////////////////////////////// +// Minimal COFF declarations + +#pragma pack(push, 1) +typedef struct ObjFileHeader { + U16 machine, section_count; U32 time_stamp, symbol_table_foff, symbol_count; + U16 optional_header_size, flags; +} ObjFileHeader; +typedef struct ObjBigHeader { + U16 sig1, sig2, version, machine; U32 time_stamp; U8 magic[16], unused[16]; + U32 section_count, symbol_table_foff, symbol_count; +} ObjBigHeader; +typedef struct ObjSectionHeader { + U8 name[8]; U32 vsize, voff, fsize, foff, relocs_foff, lines_foff; + U16 reloc_count, line_count; U32 flags; +} ObjSectionHeader; +typedef struct ObjReloc { + U32 apply_off, isymbol; U16 type; +} ObjReloc; +#pragma pack(pop) + +typedef struct TypeBuild { + LNK_CObjTypeIndex disk; + LNK_CObjUdtHashIndex udt_disk; + U32 *offsets; + U16 *sizes; + U16 *kinds; + U8 *kind_codes; + U16 kind_dictionary[256]; + U16 kind_count; + int packed_kinds_ok; + U64 *udt_hashes; + U32 udt_cap; + U32 cap; +} TypeBuild; + +typedef struct BaseRelocBuild { + LNK_CObjBaseRelocIndex disk; + LNK_CObjBaseRelocEntry *entries; + U32 cap; + int supported; +} BaseRelocBuild; + +typedef struct DebugSBuild { + LNK_CObjDebugSIndex disk; + LNK_CObjDebugSEntry *entries; + LNK_CObjDebugSSummary *summaries; + U32 cap; + int supported; +} DebugSBuild; + +//////////////////////////////// +// File output + +static int +write_at(FILE *file, U64 offset, const void *data, U64 size) +{ + if (_fseeki64(file, (long long)offset, SEEK_SET) != 0) { fprintf(stderr, "seek failed at %llu\n", offset); return 0; } + size_t written = fwrite(data, 1, (size_t)size, file); + if (written != size) { fprintf(stderr, "write failed at %llu: %zu/%llu\n", offset, written, size); return 0; } + return 1; +} + +//////////////////////////////// +// Type sidecar construction + +static U64 +numeric_size(const U8 *p, const U8 *opl) +{ + if (p + 2 > opl) return 0; + U16 k; memcpy(&k, p, 2); + if (k < 0x8000) return 2; + U32 n = 0; + switch (k) { + case 0x8000: n=1; break; + case 0x8001: case 0x8002: case 0x801c: n=2; break; + case 0x8003: case 0x8004: case 0x8005: n=4; break; + case 0x8006: case 0x8009: case 0x800a: case 0x800c: n=8; break; + case 0x8007: n=10; break; + case 0x8008: n=16; break; + case 0x800b: n=6; break; + case 0x800d: n=16; break; + case 0x800e: n=20; break; + case 0x800f: n=32; break; + case 0x8017: case 0x8018: n=16; break; + default: return 0; + } + return p + 2 + n <= opl ? 2 + n : 0; +} + +static U64 +complete_udt_hash(U16 kind, const U8 *p, U64 size) +{ + const U8 *opl = p + size; + U32 props = 0; U64 cursor = 0; + if (kind == 0x1504 || kind == 0x1505 || kind == 0x1519) { + if (size < 16) return 0; memcpy(&props, p + 2, 2); cursor = 16; + } else if (kind == 0x1506) { + if (size < 8) return 0; memcpy(&props, p + 2, 2); cursor = 8; + } else if (kind == 0x1507) { + if (size < 12) return 0; memcpy(&props, p + 2, 2); cursor = 12; + } else if (kind == 0x1608 || kind == 0x1609) { + // CV_LeafStruct2 ends in a U16 but has U32 alignment, so its serialized + // fixed header includes two bytes of tail padding (sizeof == 20). Using + // the sum of the fields (18) makes the numeric parser start in padding and + // silently drops every CLASS2/STRUCT2 complete-definition hash. + if (size < 20) return 0; memcpy(&props, p, 4); cursor = 20; + } else return 0; + if (!(props & 0x200) || (props & 0x80)) return 0; + if (kind != 0x1507) { U64 n = numeric_size(p + cursor, opl); if (!n) return 0; cursor += n; } + const U8 *name_end = cursor < size ? memchr(p + cursor, 0, (size_t)(size - cursor)) : 0; + if (!name_end) return 0; + const U8 *unique = name_end + 1; + const U8 *unique_end = unique < opl ? memchr(unique, 0, (size_t)(opl - unique)) : 0; + if (!unique_end || unique_end == unique) return 0; + U64 h = 5381; + for (const U8 *c = unique; c < unique_end; ++c) h = ((h << 5) + h) ^ *c; + return h | 1; +} + +static int +is_type_section(const U8 name[8]) +{ + static const U8 t[8] = {'.','d','e','b','u','g','$','T'}; + static const U8 p[8] = {'.','d','e','b','u','g','$','P'}; + return memcmp(name, t, 8) == 0 || memcmp(name, p, 8) == 0; +} + +static int +build_type_indices(const U8 *file, U64 file_size, TypeBuild **builds_out, U32 *count_out) +{ + *builds_out = 0; *count_out = 0; + if (file_size < sizeof(ObjFileHeader)) return 1; + U64 section_off; + U32 section_count; + const ObjBigHeader *big = (const ObjBigHeader *)file; + if (file_size >= sizeof(*big) && big->sig1 == 0 && big->sig2 == 0xffff && big->version >= 2) { + section_off = sizeof(*big); section_count = big->section_count; + } else { + const ObjFileHeader *file_header = (const ObjFileHeader *)file; + section_off = sizeof(*file_header); section_count = file_header->section_count; + } + if (section_off > file_size || (U64)section_count * sizeof(ObjSectionHeader) > file_size - section_off) return 0; + TypeBuild *builds = (TypeBuild *)calloc(section_count ? section_count : 1, sizeof(*builds)); + if (!builds) return 0; + const ObjSectionHeader *sections = (const ObjSectionHeader *)(file + section_off); + U16 *kind_to_code = (U16 *)malloc(65536 * sizeof(U16)); + if (!kind_to_code) { free(builds); return 0; } + U32 build_count = 0; + for (U32 s = 0; s < section_count; ++s) { + const ObjSectionHeader *sec = §ions[s]; + if (!is_type_section(sec->name) || sec->fsize < 4) continue; + if (sec->foff > file_size || sec->fsize > file_size - sec->foff) { free(builds); return 0; } + TypeBuild *b = &builds[build_count++]; + memset(kind_to_code, 0xff, 65536 * sizeof(U16)); + b->packed_kinds_ok = 1; + b->disk.raw_section_offset = (U64)sec->foff + 4; + b->disk.raw_section_size = sec->fsize - 4; + const U8 *data = file + sec->foff + 4; + U64 data_size = sec->fsize - 4; + for (U64 cursor = 0; cursor + 4 <= data_size; ) { + U16 size, kind; + memcpy(&size, data + cursor, 2); memcpy(&kind, data + cursor + 2, 2); + U64 stride = (U64)size + 2; + if (size < 2 || stride > data_size - cursor || cursor > UINT32_MAX) break; + if (b->disk.leaf_count == b->cap) { + U32 new_cap = b->cap ? b->cap * 2 : 4096; + U32 *new_offsets = (U32 *)realloc(b->offsets, (size_t)new_cap * sizeof(U32)); + if (!new_offsets) return 0; + b->offsets = new_offsets; + U16 *new_sizes = (U16 *)realloc(b->sizes, (size_t)new_cap * sizeof(U16)); + if (!new_sizes) return 0; + b->sizes = new_sizes; + U16 *new_kinds = (U16 *)realloc(b->kinds, (size_t)new_cap * sizeof(U16)); + if (!new_kinds) return 0; + b->kinds = new_kinds; + U8 *new_kind_codes = (U8 *)realloc(b->kind_codes, (size_t)new_cap); + if (!new_kind_codes) return 0; + b->kind_codes = new_kind_codes; + b->cap = new_cap; + } + U32 i = b->disk.leaf_count++; + b->offsets[i] = (U32)cursor; b->sizes[i] = size; b->kinds[i] = kind; + U16 code = kind_to_code[kind]; + if (code == 0xffff) { + if (b->kind_count == 256) { + b->packed_kinds_ok = 0; + code = 0; + } else { + code = b->kind_count++; + kind_to_code[kind] = code; + b->kind_dictionary[code] = kind; + } + } + b->kind_codes[i] = (U8)code; + U64 udt_hash = complete_udt_hash(kind, data + cursor + 4, (U64)size - 2); + if (udt_hash) { + if (b->udt_disk.hash_count == b->udt_cap) { + U32 cap = b->udt_cap ? b->udt_cap * 2 : 1024; + U64 *hashes = realloc(b->udt_hashes, (size_t)cap * sizeof(U64)); + if (!hashes) return 0; + b->udt_hashes = hashes; b->udt_cap = cap; + } + b->udt_hashes[b->udt_disk.hash_count++] = udt_hash; + } + cursor += stride; + } + } + free(kind_to_code); + *builds_out = builds; *count_out = build_count; + return 1; +} + +//////////////////////////////// +// .debug$S sidecar construction + +static int +debug_s_is_global_symbol(U16 kind) +{ + return kind == 0x1107 || kind == 0x0102 || kind == 0x0202 || kind == 0x1008 || + kind == 0x110d || kind == 0x020e || kind == 0x100f || kind == 0x1113; +} + +static int +debug_s_is_typedef(U16 kind) +{ + return kind == 0x0004 || kind == 0x1003 || kind == 0x1108; +} + +static int +debug_s_is_scope(U16 kind) +{ + return kind == 0x1110 || kind == 0x110f || kind == 0x1103 || kind == 0x1102 || + kind == 0x114d || kind == 0x115d || kind == 0x1104 || kind == 0x1132 || + kind == 0x1147 || kind == 0x1146; +} + +static int +debug_s_is_end(U16 kind) +{ + return kind == 0x0006 || kind == 0x114f || kind == 0x114e; +} + +static void +debug_s_summarize_symbols(const U8 *data, U64 size, U64 *module_scope_depth, LNK_CObjDebugSSummary *out) +{ + U64 cand_depth = 0; + int cand_active = 1; + for (U64 cursor = 0; cursor + 4 <= size; ) { + U16 record_size, kind; + memcpy(&record_size, data + cursor, 2); + memcpy(&kind, data + cursor + 2, 2); + U64 raw_size = (U64)record_size + 2; + U64 pdb_size = (raw_size + 3) & ~(U64)3; + // OBJ .debug$S symbol records are byte-packed (CV_SymbolAlign == 1). PDB module + // streams rewrite them at PDB_SYMBOL_ALIGN == 4, so input traversal and output sizing + // deliberately use different strides. + if (record_size < 2 || raw_size > size - cursor) break; + + if (cand_active && (debug_s_is_global_symbol(kind) || (cand_depth == 0 && debug_s_is_typedef(kind)))) { + out->gsi_candidate_count += 1; + } + if (kind == 0x1110 || kind == 0x110f || kind == 0x1147 || kind == 0x1146) { + out->proc_ref_count += 1; + } + if (kind == 0x113e || kind == 0x1111 || kind == 0x110c || kind == 0x110d || + kind == 0x1112 || kind == 0x1113 || kind == 0x1153 || kind == 0x1107) { + out->flags |= LNK_COBJ_DEBUG_S_SUMMARY_HAS_LOCALS; + } + if (cand_active) { + if (debug_s_is_scope(kind)) cand_depth += 1; + else if (debug_s_is_end(kind)) { + if (cand_depth == 0) cand_active = 0; + else cand_depth -= 1; + } + } + + int is_module = kind != 0x0007 && !debug_s_is_global_symbol(kind) && + !(debug_s_is_typedef(kind) && *module_scope_depth == 0) && kind != 0x1176; + if (is_module) { + if (debug_s_is_scope(kind)) *module_scope_depth += 1; + else if (debug_s_is_end(kind) && *module_scope_depth) *module_scope_depth -= 1; + out->module_symbol_size += (U32)pdb_size; + } + cursor += raw_size; + } +} + +static int +build_debug_s_index(const U8 *file, U64 file_size, DebugSBuild *build) +{ + memset(build, 0, sizeof(*build)); + if (file_size < sizeof(ObjFileHeader)) return 1; + U64 section_off; + U32 section_count; + const ObjBigHeader *big = (const ObjBigHeader *)file; + if (file_size >= sizeof(*big) && big->sig1 == 0 && big->sig2 == 0xffff && big->version >= 2) { + section_off = sizeof(*big); section_count = big->section_count; + } else { + const ObjFileHeader *h = (const ObjFileHeader *)file; + section_off = sizeof(*h) + h->optional_header_size; section_count = h->section_count; + } + if (section_off > file_size || (U64)section_count * sizeof(ObjSectionHeader) > file_size - section_off) return 0; + const ObjSectionHeader *sections = (const ObjSectionHeader *)(file + section_off); + static const U8 debug_s_name[8] = {'.','d','e','b','u','g','$','S'}; + U64 module_scope_depth = 0; + for (U32 sect_idx = 0; sect_idx < section_count; ++sect_idx) { + const ObjSectionHeader *sec = §ions[sect_idx]; + if (memcmp(sec->name, debug_s_name, 8) != 0) continue; + if (sec->foff > file_size || sec->fsize > file_size - sec->foff) return 0; + if (sec->fsize < sizeof(U32)) continue; + U64 c13_base = (U64)sec->foff + sizeof(U32); // skip CV_Signature + U64 c13_size = sec->fsize - sizeof(U32); + U64 cursor = 0; + while (cursor + 2 * sizeof(U32) <= c13_size) { + U32 kind, payload_size; + memcpy(&kind, file + c13_base + cursor, sizeof(kind)); + memcpy(&payload_size, file + c13_base + cursor + sizeof(kind), sizeof(payload_size)); + U64 payload_rel = cursor + 2 * sizeof(U32); + U64 clamped_size = payload_size < c13_size - payload_rel ? payload_size : c13_size - payload_rel; + U64 payload_off = c13_base + payload_rel; + // Match cv_debug_s_from_data: ignored records do not create nodes, all unknown kinds are + // retained in the NULL bucket, and an overrun is represented by a clamped String8. + if (!(kind & 0x80000000u)) { + if (build->disk.entry_count == build->cap) { + U32 new_cap = build->cap ? build->cap * 2 : 256; + LNK_CObjDebugSEntry *entries = realloc(build->entries, (size_t)new_cap * sizeof(*entries)); + if (!entries) return 0; + build->entries = entries; + LNK_CObjDebugSSummary *summaries = realloc(build->summaries, (size_t)new_cap * sizeof(*summaries)); + if (!summaries) return 0; + build->summaries = summaries; + build->cap = new_cap; + } + if (payload_off > UINT32_MAX || clamped_size > UINT32_MAX) return 0; + U32 entry_idx = build->disk.entry_count++; + LNK_CObjDebugSEntry *dst = &build->entries[entry_idx]; + dst->raw_section_offset = sec->foff; + dst->raw_payload_offset = (U32)payload_off; + dst->raw_payload_size = (U32)clamped_size; + dst->kind = kind; + LNK_CObjDebugSSummary *summary = &build->summaries[entry_idx]; + memset(summary, 0, sizeof(*summary)); + if (kind == 0xF1) { + debug_s_summarize_symbols(file + payload_off, clamped_size, &module_scope_depth, summary); + } + } + if ((U64)payload_size > UINT64_MAX - payload_rel) break; + U64 next = payload_rel + payload_size; + if (next > UINT64_MAX - 3) break; + cursor = (next + 3) & ~(U64)3; + if (cursor <= payload_rel) break; + } + } + build->supported = 1; + return 1; +} + +//////////////////////////////// +// Base-relocation sidecar construction + +static int +build_base_reloc_index(const U8 *file, U64 file_size, BaseRelocBuild *build) +{ + memset(build, 0, sizeof(*build)); + if (file_size < sizeof(ObjFileHeader)) return 1; + U64 section_off; + U32 section_count; + U16 machine; + const ObjBigHeader *big = (const ObjBigHeader *)file; + if (file_size >= sizeof(*big) && big->sig1 == 0 && big->sig2 == 0xffff && big->version >= 2) { + section_off = sizeof(*big); section_count = big->section_count; machine = big->machine; + } else { + const ObjFileHeader *h = (const ObjFileHeader *)file; + section_off = sizeof(*h); section_count = h->section_count; machine = h->machine; + } + // RAD Link's base-reloc builder currently supports x64 only as well. + if (machine != 0x8664) return 1; + build->supported = 1; + if (section_off > file_size || (U64)section_count * sizeof(ObjSectionHeader) > file_size - section_off) return 0; + const ObjSectionHeader *sections = (const ObjSectionHeader *)(file + section_off); + for (U32 sect_idx = 0; sect_idx < section_count; ++sect_idx) { + const ObjSectionHeader *sec = §ions[sect_idx]; + U64 reloc_off = sec->relocs_foff; + U64 reloc_count = sec->reloc_count; + if ((sec->flags & 0x01000000u) && sec->reloc_count == 0xffff) { + if (reloc_off > file_size || sizeof(ObjReloc) > file_size - reloc_off) return 0; + const ObjReloc *counter = (const ObjReloc *)(file + reloc_off); + if (counter->apply_off == 0) return 0; + reloc_count = counter->apply_off - 1; + reloc_off += sizeof(ObjReloc); + } + if (reloc_off > file_size || reloc_count > (file_size - reloc_off) / sizeof(ObjReloc)) return 0; + const ObjReloc *relocs = (const ObjReloc *)(file + reloc_off); + for (U64 i = 0; i < reloc_count; ++i) { + U8 addr_size = relocs[i].type == 1 ? 8 : relocs[i].type == 2 ? 4 : 0; + if (!addr_size) continue; + if (build->disk.entry_count == build->cap) { + U32 new_cap = build->cap ? build->cap * 2 : 1024; + LNK_CObjBaseRelocEntry *entries = realloc(build->entries, (size_t)new_cap * sizeof(*entries)); + if (!entries) return 0; + build->entries = entries; + build->cap = new_cap; + } + LNK_CObjBaseRelocEntry *dst = &build->entries[build->disk.entry_count++]; + memset(dst, 0, sizeof(*dst)); + dst->sect_idx = sect_idx; + dst->apply_off = relocs[i].apply_off; + dst->isymbol = relocs[i].isymbol; + dst->addr_size = addr_size; + } + } + return 1; +} + +//////////////////////////////// +// Packed type-offset encoding + +static U64 +packed_v2_offset_storage_size(TypeBuild *b) +{ + U64 group_size = (U64)1 << LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT; + U64 group_count = ((U64)b->disk.leaf_count + group_size - 1) / group_size; + U64 directory_bytes = group_count * 2 * sizeof(U32); + U64 payload_bytes = 0; + for (U64 group_idx = 0; group_idx < group_count; ++group_idx) { + U64 first = group_idx * group_size; + U64 remain = (U64)b->disk.leaf_count - first; + U64 count = group_size < remain ? group_size : remain; + U32 base = b->offsets[first]; + U32 last_delta = b->offsets[first + count - 1] - base; + payload_bytes += count * (last_delta <= 0xffffu ? 2 : 3); + } + return ((directory_bytes + 7) & ~(U64)7) + payload_bytes; +} + +//////////////////////////////// +// Entry point + +int +main(int argc, char **argv) +{ + if (argc < 3 || argc > 7) { + fprintf(stderr, "usage: rad_obj_compress [segment-kib=512] [selkie|mermaid|kraken] [space-speed=256] [superfast|veryfast|fast|normal|optimal1..5]\n"); + return 2; + } + U32 segment_size = argc >= 4 ? (U32)strtoul(argv[3], 0, 10) * 1024u : 512u * 1024u; + OodleLZ_Compressor compressor = OodleLZ_Compressor_Kraken; + if (argc >= 5) { + if (_stricmp(argv[4], "mermaid") == 0) compressor = OodleLZ_Compressor_Mermaid; + else if (_stricmp(argv[4], "selkie") == 0) compressor = OodleLZ_Compressor_Selkie; + else if (_stricmp(argv[4], "kraken") == 0) compressor = OodleLZ_Compressor_Kraken; + else { fprintf(stderr, "invalid compressor: %s\n", argv[4]); return 2; } + } + OodleLZ_CompressionLevel compression_level = OodleLZ_CompressionLevel_Normal; + if (argc >= 7) { + if (_stricmp(argv[6], "superfast") == 0) compression_level = OodleLZ_CompressionLevel_SuperFast; + else if (_stricmp(argv[6], "veryfast") == 0) compression_level = OodleLZ_CompressionLevel_VeryFast; + else if (_stricmp(argv[6], "fast") == 0) compression_level = OodleLZ_CompressionLevel_Fast; + else if (_stricmp(argv[6], "normal") == 0) compression_level = OodleLZ_CompressionLevel_Normal; + else if (_stricmp(argv[6], "optimal1") == 0) compression_level = OodleLZ_CompressionLevel_Optimal1; + else if (_stricmp(argv[6], "optimal2") == 0) compression_level = OodleLZ_CompressionLevel_Optimal2; + else if (_stricmp(argv[6], "optimal3") == 0) compression_level = OodleLZ_CompressionLevel_Optimal3; + else if (_stricmp(argv[6], "optimal4") == 0) compression_level = OodleLZ_CompressionLevel_Optimal4; + else if (_stricmp(argv[6], "optimal5") == 0) compression_level = OodleLZ_CompressionLevel_Optimal5; + else { fprintf(stderr, "invalid compression level: %s\n", argv[6]); return 2; } + } + OodleLZ_CompressOptions options = *OodleLZ_CompressOptions_GetDefault(compressor, compression_level); + options.spaceSpeedTradeoffBytes = argc >= 6 ? (OO_S32)strtol(argv[5], 0, 10) : 256; + int packed_sidecar = 1; + OodleLZ_CompressOptions_Validate(&options); + if (segment_size < 64 * 1024 || (segment_size & (segment_size - 1)) != 0) { + fprintf(stderr, "segment size must be a power of two and at least 64 KiB\n"); + return 2; + } + + HANDLE input_file = CreateFileA(argv[1], GENERIC_READ, FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + LARGE_INTEGER input_size = {0}; + if (input_file == INVALID_HANDLE_VALUE || !GetFileSizeEx(input_file, &input_size) || input_size.QuadPart <= 0) { + fprintf(stderr, "unable to open input: %s\n", argv[1]); return 1; + } + U64 raw_size = (U64)input_size.QuadPart; + HANDLE input_mapping = CreateFileMappingW(input_file, 0, PAGE_READONLY, 0, 0, 0); + U8 *mapped_input = input_mapping ? (U8 *)MapViewOfFile(input_mapping, FILE_MAP_READ, 0, 0, 0) : 0; + U8 *input_data = mapped_input; + if (!input_data) { fprintf(stderr, "unable to map input: %s\n", argv[1]); return 1; } + if (raw_size >= sizeof(LNK_CObjHeader) && ((LNK_CObjHeader *)input_data)->magic == LNK_COBJ_MAGIC) { + fprintf(stderr, "input is already a compressed object: %s\n", argv[1]); + return 1; + } + TypeBuild *type_builds = 0; + U32 type_count = 0; + if (!build_type_indices(input_data, raw_size, &type_builds, &type_count) || type_count > 0xffff) { + fprintf(stderr, "unable to build type sidecar: %s\n", argv[1]); return 1; + } + if (packed_sidecar) { + for (U32 i = 0; i < type_count; ++i) { + if (!type_builds[i].packed_kinds_ok) { packed_sidecar = 0; break; } + } + } + BaseRelocBuild base_relocs = {0}; + if (!build_base_reloc_index(input_data, raw_size, &base_relocs)) { + fprintf(stderr, "unable to build base relocation sidecar: %s\n", argv[1]); return 1; + } + DebugSBuild debug_s_index = {0}; + if (!build_debug_s_index(input_data, raw_size, &debug_s_index)) { + fprintf(stderr, "unable to build .debug$S sidecar: %s\n", argv[1]); return 1; + } + U64 segment_count_u64 = (raw_size + segment_size - 1) / segment_size; + if (segment_count_u64 > UINT32_MAX) { fprintf(stderr, "input has too many segments\n"); return 1; } + U32 segment_count = (U32)segment_count_u64; + LNK_CObjSegment *directory = (LNK_CObjSegment *)calloc(segment_count, sizeof(*directory)); + U8 *verify = (U8 *)malloc(segment_size); + OO_SINTa comp_cap = OodleLZ_GetCompressedBufferSizeNeeded(compressor, segment_size); + U8 *comp = (U8 *)malloc((size_t)comp_cap); + size_t temp_name_size = strlen(argv[2]) + 5; + char *temp_name = (char *)malloc(temp_name_size); + if (!temp_name) { fprintf(stderr, "unable to allocate output path\n"); return 1; } + snprintf(temp_name, temp_name_size, "%s.tmp", argv[2]); + FILE *out = fopen(temp_name, "wb+"); + if (!directory || !verify || !comp || !out) { fprintf(stderr, "allocation/output open failed\n"); return 1; } + + LNK_CObjHeader header = {0}; + header.magic = LNK_COBJ_MAGIC; + header.version = LNK_COBJ_VERSION; + header.header_size = sizeof(header); + header.raw_size = raw_size; + header.segment_size = segment_size; + header.segment_count = segment_count; + U64 layout_cursor = (sizeof(header) + 7) & ~(U64)7; + U64 directory_bytes = (U64)segment_count * sizeof(*directory); + header.directory_offset = layout_cursor; + layout_cursor = (layout_cursor + directory_bytes + 7) & ~(U64)7; + header.flags = LNK_COBJ_FLAG_PORTABLE_RAW_MAP; + if (packed_sidecar) { + header.flags |= LNK_COBJ_FLAG_PACKED_TYPE_SIDECAR | LNK_COBJ_FLAG_PACKED_TYPE_OFFSETS_V2; + } + if (type_count || base_relocs.supported || debug_s_index.supported) { + U64 dirs_bytes = 0; + header.flags |= LNK_COBJ_FLAG_TYPE_INDEX | LNK_COBJ_FLAG_UDT_HASH_INDEX | + (type_count << LNK_COBJ_TYPE_INDEX_COUNT_SHIFT); + if (type_count == 0) { + header.flags &= ~(LNK_COBJ_FLAG_TYPE_INDEX | LNK_COBJ_FLAG_UDT_HASH_INDEX); + } + dirs_bytes += (U64)type_count * (sizeof(LNK_CObjTypeIndex) + sizeof(LNK_CObjUdtHashIndex)); + if (base_relocs.supported) { + header.flags |= LNK_COBJ_FLAG_BASE_RELOC_INDEX; + dirs_bytes += sizeof(LNK_CObjBaseRelocIndex); + } + if (debug_s_index.supported) { + header.flags |= LNK_COBJ_FLAG_DEBUG_S_INDEX | LNK_COBJ_FLAG_DEBUG_S_SUMMARY; + dirs_bytes += sizeof(LNK_CObjDebugSIndex); + } + header.reserved = layout_cursor; + layout_cursor = (layout_cursor + dirs_bytes + 7) & ~(U64)7; + for (U32 i = 0; i < type_count; ++i) { + TypeBuild *b = &type_builds[i]; + U64 offset_count = b->disk.leaf_count; + U64 offsets_bytes = offset_count * sizeof(U32); + U64 sizes_bytes = (U64)b->disk.leaf_count * sizeof(U16); + U64 kinds_bytes = (U64)b->disk.leaf_count * sizeof(U16); + if (packed_sidecar) { + offsets_bytes = packed_v2_offset_storage_size(b); + sizes_bytes = 256 * sizeof(U16); + kinds_bytes = offset_count; + } + U64 hashes_bytes = (U64)b->udt_disk.hash_count * sizeof(U64); + b->disk.offsets_file_offset = layout_cursor; + layout_cursor = (layout_cursor + offsets_bytes + 7) & ~(U64)7; + b->disk.sizes_file_offset = layout_cursor; + layout_cursor = (layout_cursor + sizes_bytes + 7) & ~(U64)7; + b->disk.kinds_file_offset = layout_cursor; + layout_cursor = (layout_cursor + kinds_bytes + 7) & ~(U64)7; + b->udt_disk.hashes_file_offset = hashes_bytes ? layout_cursor : 0; + layout_cursor = (layout_cursor + hashes_bytes + 7) & ~(U64)7; + } + if (base_relocs.supported) { + U64 entry_bytes = (U64)base_relocs.disk.entry_count * sizeof(LNK_CObjBaseRelocEntry); + base_relocs.disk.entries_file_offset = entry_bytes ? layout_cursor : 0; + layout_cursor = (layout_cursor + entry_bytes + 7) & ~(U64)7; + } + if (debug_s_index.supported) { + U64 entry_bytes = (U64)debug_s_index.disk.entry_count * sizeof(LNK_CObjDebugSEntry); + debug_s_index.disk.entries_file_offset = entry_bytes ? layout_cursor : 0; + layout_cursor = (layout_cursor + entry_bytes + 7) & ~(U64)7; + U64 summary_bytes = (U64)debug_s_index.disk.entry_count * sizeof(LNK_CObjDebugSSummary); + layout_cursor = (layout_cursor + summary_bytes + 7) & ~(U64)7; + } + } + header.compressor = (U32)compressor; + header.data_offset = layout_cursor; + if (!write_at(out, 0, &header, sizeof(header))) return 1; + U64 sidecar_cursor = header.reserved; + for (U32 i = 0; i < type_count; ++i) { + if (!write_at(out, sidecar_cursor, &type_builds[i].disk, sizeof(LNK_CObjTypeIndex))) return 1; + sidecar_cursor += sizeof(LNK_CObjTypeIndex); + } + for (U32 i = 0; i < type_count; ++i) { + if (!write_at(out, sidecar_cursor, &type_builds[i].udt_disk, sizeof(LNK_CObjUdtHashIndex))) return 1; + sidecar_cursor += sizeof(LNK_CObjUdtHashIndex); + } + if (base_relocs.supported) { + if (!write_at(out, sidecar_cursor, &base_relocs.disk, sizeof(base_relocs.disk))) return 1; + sidecar_cursor += sizeof(base_relocs.disk); + U64 entry_bytes = (U64)base_relocs.disk.entry_count * sizeof(LNK_CObjBaseRelocEntry); + if (entry_bytes && !write_at(out, base_relocs.disk.entries_file_offset, base_relocs.entries, entry_bytes)) return 1; + } + if (debug_s_index.supported) { + if (!write_at(out, sidecar_cursor, &debug_s_index.disk, sizeof(debug_s_index.disk))) return 1; + sidecar_cursor += sizeof(debug_s_index.disk); + U64 entry_bytes = (U64)debug_s_index.disk.entry_count * sizeof(LNK_CObjDebugSEntry); + if (entry_bytes && !write_at(out, debug_s_index.disk.entries_file_offset, debug_s_index.entries, entry_bytes)) return 1; + U64 summary_offset = (debug_s_index.disk.entries_file_offset + entry_bytes + 7) & ~(U64)7; + U64 summary_bytes = (U64)debug_s_index.disk.entry_count * sizeof(LNK_CObjDebugSSummary); + if (summary_bytes && !write_at(out, summary_offset, debug_s_index.summaries, summary_bytes)) return 1; + } + for (U32 i = 0; i < type_count; ++i) { + TypeBuild *b = &type_builds[i]; + if (packed_sidecar) { + U64 group_size = (U64)1 << LNK_COBJ_PACKED_TYPE_OFFSET_V2_SHIFT; + U64 group_count = ((U64)b->disk.leaf_count + group_size - 1) / group_size; + U64 group_bytes = group_count * 2 * sizeof(U32); + U64 payload_off = b->disk.offsets_file_offset + ((group_bytes + 7) & ~(U64)7); + U64 storage_bytes = packed_v2_offset_storage_size(b); + U64 payload_cap = storage_bytes - (payload_off - b->disk.offsets_file_offset); + U32 *groups = calloc(group_count ? (size_t)group_count * 2 : 1, sizeof(U32)); + U8 *deltas = malloc((size_t)(payload_cap ? payload_cap : 1)); + if (!groups || !deltas) return 1; + U64 payload_cursor = 0; + for (U64 group_idx = 0; group_idx < group_count; ++group_idx) { + U64 first = group_idx * group_size; + U64 remain = (U64)b->disk.leaf_count - first; + U64 count = group_size < remain ? group_size : remain; + U32 base = b->offsets[first]; + U32 last_delta = b->offsets[first + count - 1] - base; + U32 width = last_delta <= 0xffffu ? 2 : 3; + groups[group_idx*2 + 0] = base; + if (payload_cursor > UINT32_MAX) { fprintf(stderr, "packed v2 payload offset overflow\n"); return 1; } + groups[group_idx*2 + 1] = (U32)payload_cursor | (width == 3); + for (U64 i = 0; i < count; ++i) { + U32 delta = b->offsets[first + i] - base; + if ((width == 2 && delta > 0xffffu) || delta > 0xffffffu) { + fprintf(stderr, "packed v2 type offset overflow\n"); return 1; + } + deltas[payload_cursor++] = (U8)(delta >> 0); + deltas[payload_cursor++] = (U8)(delta >> 8); + if (width == 3) { deltas[payload_cursor++] = (U8)(delta >> 16); } + } + } + if (!write_at(out, b->disk.offsets_file_offset, groups, group_bytes) || + !write_at(out, payload_off, deltas, payload_cursor) || + !write_at(out, b->disk.sizes_file_offset, b->kind_dictionary, sizeof(b->kind_dictionary)) || + !write_at(out, b->disk.kinds_file_offset, b->kind_codes, b->disk.leaf_count)) return 1; + free(groups); + free(deltas); + continue; + } + if (!write_at(out, b->disk.offsets_file_offset, b->offsets, + (U64)b->disk.leaf_count * sizeof(U32)) || + !write_at(out, b->disk.sizes_file_offset, b->sizes, (U64)b->disk.leaf_count * sizeof(U16)) || + !write_at(out, b->disk.kinds_file_offset, b->kinds, (U64)b->disk.leaf_count * sizeof(U16))) return 1; + if (b->udt_disk.hash_count && + !write_at(out, b->udt_disk.hashes_file_offset, b->udt_hashes, + (U64)b->udt_disk.hash_count * sizeof(U64))) return 1; + } + + U64 payload_cursor = header.data_offset; + int in_raw_run = 0; + for (U32 seg_idx = 0; seg_idx < segment_count; ++seg_idx) { + U32 raw_len = (U32)((raw_size - (U64)seg_idx * segment_size) < segment_size ? + (raw_size - (U64)seg_idx * segment_size) : segment_size); + U64 seg_min = (U64)seg_idx * segment_size; + U8 *raw = input_data + seg_min; + LNK_CObjSegment *entry = &directory[seg_idx]; + entry->raw_size = raw_len; + OO_SINTa comp_len = OodleLZ_Compress(compressor, raw, raw_len, comp, + compression_level, &options, 0, 0, 0, 0); + if (comp_len <= 0 || comp_len >= raw_len) { + entry->flags |= LNK_COBJ_SEGMENT_RAW; + entry->stored_size = raw_len; + if (!in_raw_run) { payload_cursor = (payload_cursor + 65535) & ~(U64)65535; } + entry->file_offset = payload_cursor; + if (!write_at(out, entry->file_offset, raw, raw_len)) return 1; + U64 mapped_len = (seg_idx + 1 == segment_count) ? ((raw_len + 65535) & ~(U64)65535) : segment_size; + payload_cursor += mapped_len; + in_raw_run = 1; + } else { + in_raw_run = 0; + entry->stored_size = (U32)comp_len; + OO_SINTa verify_len = OodleLZ_Decompress(comp, comp_len, verify, raw_len, + OodleLZ_FuzzSafe_Yes, OodleLZ_CheckCRC_No, OodleLZ_Verbosity_None, + 0, 0, 0, 0, 0, 0, OodleLZ_Decode_Unthreaded); + if (verify_len != raw_len || memcmp(raw, verify, raw_len) != 0) { + fprintf(stderr, "round-trip verification failed in segment %u\n", seg_idx); + return 1; + } + payload_cursor = (payload_cursor + 7) & ~(U64)7; + entry->file_offset = payload_cursor; + if (!write_at(out, entry->file_offset, comp, (U64)comp_len)) return 1; + payload_cursor += (U64)comp_len; + } + } + if (!write_at(out, header.directory_offset, directory, (U64)segment_count * sizeof(*directory))) return 1; + if (_chsize_s(_fileno(out), payload_cursor) != 0) { + fprintf(stderr, "unable to finalize portable container size\n"); return 1; + } + fclose(out); + UnmapViewOfFile(mapped_input); + CloseHandle(input_mapping); + CloseHandle(input_file); + if (!MoveFileExA(temp_name, argv[2], MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { + fprintf(stderr, "unable to publish output: %s\n", argv[2]); + DeleteFileA(temp_name); + return 1; + } + fprintf(stdout, "%s: %.2f MiB -> %.2f MiB (%.1f%%), %u segments\n", argv[1], + (double)raw_size / (1024.0 * 1024.0), (double)payload_cursor / (1024.0 * 1024.0), + raw_size ? 100.0 * (double)payload_cursor / (double)raw_size : 0.0, segment_count); + for (U32 i = 0; i < type_count; ++i) { free(type_builds[i].offsets); free(type_builds[i].sizes); free(type_builds[i].kinds); free(type_builds[i].kind_codes); free(type_builds[i].udt_hashes); } + free(debug_s_index.summaries); free(debug_s_index.entries); free(base_relocs.entries); + free(type_builds); free(temp_name); free(comp); free(verify); free(directory); + return 0; +} From 1889f8aa972caeea75a19c8ea2d2f519f6d7d838 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Sun, 16 Aug 2026 09:56:45 -0700 Subject: [PATCH 111/121] tests: add compressed OBJ conversion and benchmark tools --- scripts/bench_cobj.ps1 | 78 ++++++++++++++++++++++ scripts/build_cobj_test.bat | 18 +++++ scripts/cobj_compress_rsp.ps1 | 107 ++++++++++++++++++++++++++++++ scripts/cobj_smoke.c | 5 ++ scripts/retarget_rsp.ps1 | 45 +++++++++++++ scripts/run_cobj_smoke.bat | 15 +++++ scripts/stage_raw_rsp.ps1 | 46 +++++++++++++ scripts/test_link_determinism.ps1 | 71 ++++++++++++++++++++ scripts/warm_rsp.ps1 | 31 +++++++++ 9 files changed, 416 insertions(+) create mode 100644 scripts/bench_cobj.ps1 create mode 100644 scripts/build_cobj_test.bat create mode 100644 scripts/cobj_compress_rsp.ps1 create mode 100644 scripts/cobj_smoke.c create mode 100644 scripts/retarget_rsp.ps1 create mode 100644 scripts/run_cobj_smoke.bat create mode 100644 scripts/stage_raw_rsp.ps1 create mode 100644 scripts/test_link_determinism.ps1 create mode 100644 scripts/warm_rsp.ps1 diff --git a/scripts/bench_cobj.ps1 b/scripts/bench_cobj.ps1 new file mode 100644 index 000000000..3a0c4b9b1 --- /dev/null +++ b/scripts/bench_cobj.ps1 @@ -0,0 +1,78 @@ +param( + [Parameter(Mandatory=$true)][string]$Rsp, + [Parameter(Mandatory=$true)][string]$Output, + [string]$Radlink = '', + [string]$WorkingDirectory = '', + [string]$LogDirectory = '', + [int]$CacheGiB = 17, + [int]$CacheShrinkGiB = 6, + [switch]$CacheFreeze = $true, + [ValidateSet('','types','except_debug_s','except_debug_s_runs','all')][string]$CacheFreezeTrim = '', + [int]$SkipCleanup = 1, + [string]$Tag = 'v2', + [string]$ExtraArgs = '', + [switch]$TrimWorkingSet = $true, + [switch]$TrimProcessWorkingSet, + [switch]$TraceMemory +) +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..')) +if (!$Radlink) { $Radlink = Join-Path $root 'out_cobj\radlink.exe' } +if (!$WorkingDirectory) { $WorkingDirectory = (Get-Location).Path } +if (!$LogDirectory) { $LogDirectory = Join-Path $root 'cobj_bench_output' } +$Radlink = [IO.Path]::GetFullPath($Radlink) +$WorkingDirectory = [IO.Path]::GetFullPath($WorkingDirectory) +$LogDirectory = [IO.Path]::GetFullPath($LogDirectory) +[IO.Directory]::CreateDirectory($LogDirectory) | Out-Null +$log = Join-Path $LogDirectory "cobj_$Tag.log" +$phase = Join-Path $LogDirectory "cobj_$Tag.phase.log" +Remove-Item -LiteralPath $log,$phase -Force -ErrorAction SilentlyContinue +$psi = [Diagnostics.ProcessStartInfo]::new() +$psi.FileName = $Radlink +$psi.WorkingDirectory = $WorkingDirectory +$psi.Arguments = '@"' + [IO.Path]::GetFullPath($Rsp) + '" /RAD_LOG:timers ' + $ExtraArgs +$psi.UseShellExecute = $false +$psi.CreateNoWindow = $true +$psi.RedirectStandardOutput = $true +$psi.RedirectStandardError = $true +$psi.Environment['RAD_COBJ_CACHE_GIB'] = [string]$CacheGiB +$psi.Environment['RAD_COBJ_SKIP_CLEANUP'] = [string]$SkipCleanup +if ($CacheShrinkGiB -gt 0) { $psi.Environment['RAD_COBJ_CACHE_SHRINK_GIB'] = [string]$CacheShrinkGiB } +else { [void]$psi.Environment.Remove('RAD_COBJ_CACHE_SHRINK_GIB') } +if ($CacheFreeze) { $psi.Environment['RAD_COBJ_CACHE_FREEZE'] = '1' } +else { [void]$psi.Environment.Remove('RAD_COBJ_CACHE_FREEZE') } +if ($CacheFreezeTrim) { + $psi.Environment['RAD_COBJ_CACHE_FREEZE_TRIM'] = if ($CacheFreezeTrim -eq 'all') { '1' } else { $CacheFreezeTrim } +} else { [void]$psi.Environment.Remove('RAD_COBJ_CACHE_FREEZE_TRIM') } +if ($TrimWorkingSet) { $psi.Environment['RAD_COBJ_TRIM_WS'] = '1' } +if ($TrimProcessWorkingSet) { $psi.Environment['RAD_COBJ_TRIM_WS'] = '2' } +$psi.Environment['RADLINK_PHASE_LOG'] = $phase +$p = [Diagnostics.Process]::new(); $p.StartInfo = $psi +$sw = [Diagnostics.Stopwatch]::StartNew(); [void]$p.Start() +$stdoutTask = $p.StandardOutput.ReadToEndAsync() +$stderrTask = $p.StandardError.ReadToEndAsync() +$peakPrivate = 0L; $peakCommit = 0L; $peakWs = 0L +$memoryTrace = [Collections.Generic.List[string]]::new() +if ($TraceMemory) { $memoryTrace.Add('ElapsedSeconds,PrivateGiB,WorkingSetGiB') } +while (!$p.HasExited) { + try { + $p.Refresh() + $private = $p.PrivateMemorySize64 + $commit = $p.PeakPagedMemorySize64 + $ws = $p.WorkingSet64 + $peakPrivate=[Math]::Max($peakPrivate,$private) + $peakCommit=[Math]::Max($peakCommit,$commit) + $peakWs=[Math]::Max($peakWs,$ws) + if ($TraceMemory) { + $memoryTrace.Add(('{0:F3},{1:F6},{2:F6}' -f $sw.Elapsed.TotalSeconds,($private/1GB),($ws/1GB))) + } + } catch {} + Start-Sleep -Milliseconds 50 +} +$stdout=$stdoutTask.GetAwaiter().GetResult(); $stderr=$stderrTask.GetAwaiter().GetResult(); $sw.Stop() +[IO.File]::WriteAllText($log, $stdout + $stderr) +if ($TraceMemory) { [IO.File]::WriteAllLines((Join-Path $LogDirectory "cobj_$Tag.memory.csv"), $memoryTrace) } +$p.Refresh(); $peakPrivate=[Math]::Max($peakPrivate,$p.PrivateMemorySize64); $peakCommit=[Math]::Max($peakCommit,$p.PeakPagedMemorySize64); $peakWs=[Math]::Max($peakWs,$p.WorkingSet64) +$hash = if ($p.ExitCode -eq 0 -and (Test-Path -LiteralPath $Output)) { (Get-FileHash -LiteralPath $Output -Algorithm SHA256).Hash } else { '' } +[pscustomobject]@{Tag=$Tag;ExitCode=$p.ExitCode;WallSeconds=$sw.Elapsed.TotalSeconds;PeakCommitGiB=$peakCommit/1GB;PeakPrivateGiB=$peakPrivate/1GB;PeakWorkingSetGiB=$peakWs/1GB;UserSeconds=$p.UserProcessorTime.TotalSeconds;KernelSeconds=$p.PrivilegedProcessorTime.TotalSeconds;Sha256=$hash;Log=$log;PhaseLog=$phase} | ConvertTo-Json -Compress +Get-Content -LiteralPath $log -Tail 30 diff --git a/scripts/build_cobj_test.bat b/scripts/build_cobj_test.bat new file mode 100644 index 000000000..5d2da90eb --- /dev/null +++ b/scripts/build_cobj_test.bat @@ -0,0 +1,18 @@ +@echo off +setlocal +if "%OODLE_SDK_DIR%"=="" ( + echo OODLE_SDK_DIR must name an Oodle SDK + exit /b 1 +) +if not exist "%OODLE_SDK_DIR%\include\oodle2.h" ( + echo Oodle SDK not found at "%OODLE_SDK_DIR%" + exit /b 1 +) +where cl >nul 2>nul || ( + echo Run this script from an x64 Visual Studio developer command prompt + exit /b 1 +) +call "%~dp0..\build.bat" radlink rad_obj_compress release oodle || exit /b 1 +if not exist "%~dp0..\out_cobj" mkdir "%~dp0..\out_cobj" +copy /y "%~dp0..\build\radlink.exe" "%~dp0..\out_cobj\radlink.exe" >nul || exit /b 1 +copy /y "%~dp0..\build\rad_obj_compress.exe" "%~dp0..\out_cobj\rad_obj_compress.exe" >nul || exit /b 1 diff --git a/scripts/cobj_compress_rsp.ps1 b/scripts/cobj_compress_rsp.ps1 new file mode 100644 index 000000000..17efb37e4 --- /dev/null +++ b/scripts/cobj_compress_rsp.ps1 @@ -0,0 +1,107 @@ +param( + [Parameter(Mandatory=$true)][string]$ResponseFile, + [Parameter(Mandatory=$true)][string]$OutputDirectory, + [Parameter(Mandatory=$true)][string]$Compressor, + [int]$Workers = 16, + [int]$SegmentKiB = 512, + [ValidateSet('selkie','mermaid','kraken')][string]$Codec = 'kraken', + [int]$SpaceSpeedTradeoff = 256, + [ValidateSet('superfast','veryfast','fast','normal','optimal1','optimal2','optimal3','optimal4','optimal5')][string]$CompressionLevel = 'normal' +) + +$ErrorActionPreference = 'Stop' +Add-Type -TypeDefinition @' +using System; +using System.Runtime.InteropServices; +public static class CObjFileSize { + [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] + public static extern uint GetCompressedFileSizeW(string path, out uint high); +} +'@ +$ResponseFile = [IO.Path]::GetFullPath($ResponseFile) +$OutputDirectory = [IO.Path]::GetFullPath($OutputDirectory) +$Compressor = [IO.Path]::GetFullPath($Compressor) +[IO.Directory]::CreateDirectory($OutputDirectory) | Out-Null + +$lines = [IO.File]::ReadAllLines($ResponseFile) +$work = New-Object System.Collections.Generic.List[object] +$rewritten = New-Object string[] $lines.Length +$objIndex = 0 +for ($lineIndex = 0; $lineIndex -lt $lines.Length; ++$lineIndex) { + $line = $lines[$lineIndex] + $path = $line.Trim().Trim('"') + if ($path -match '(?i)\.obj$' -and [IO.File]::Exists($path)) { + $leaf = [IO.Path]::GetFileName($path) + $dest = Join-Path $OutputDirectory ('{0:D5}_{1}' -f $objIndex, $leaf) + $work.Add([pscustomobject]@{ Source=$path; Destination=$dest; Index=$objIndex }) + $rewritten[$lineIndex] = '"' + ($dest -replace '\\','/') + '"' + ++$objIndex + } else { + $rewritten[$lineIndex] = $line + } +} +[IO.File]::WriteAllLines((Join-Path $OutputDirectory 'compressed.rsp'), $rewritten) + +$running = New-Object System.Collections.Generic.List[object] +$completed = 0 +$failed = 0 +$timer = [Diagnostics.Stopwatch]::StartNew() + +function Reap-Compressors([bool]$waitForOne) { + do { + $reaped = $false + for ($i = $running.Count - 1; $i -ge 0; --$i) { + $job = $running[$i] + if ($job.Process.HasExited) { + $stdout = $job.Process.StandardOutput.ReadToEnd().Trim() + $stderr = $job.Process.StandardError.ReadToEnd().Trim() + if ($job.Process.ExitCode -ne 0) { + ++$script:failed + Write-Error "compression failed ($($job.Process.ExitCode)): $($job.Source)`n$stderr" + } + $job.Process.Dispose() + $running.RemoveAt($i) + ++$script:completed + $reaped = $true + if (($script:completed % 100) -eq 0 -or $script:completed -eq $work.Count) { + Write-Host ("compressed {0}/{1} objects in {2:n1}s" -f $script:completed,$work.Count,$timer.Elapsed.TotalSeconds) + } + } + } + if ($waitForOne -and !$reaped) { Start-Sleep -Milliseconds 25 } + } while ($waitForOne -and !$reaped) +} + +foreach ($item in $work) { + if ([IO.File]::Exists($item.Destination)) { + ++$completed + continue + } + while ($running.Count -ge $Workers) { Reap-Compressors $true } + $psi = New-Object Diagnostics.ProcessStartInfo + $psi.FileName = $Compressor + $psi.Arguments = ('"{0}" "{1}" {2} {3} {4} {5}' -f $item.Source,$item.Destination,$SegmentKiB,$Codec,$SpaceSpeedTradeoff,$CompressionLevel) + $psi.UseShellExecute = $false + $psi.CreateNoWindow = $true + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $process = [Diagnostics.Process]::Start($psi) + $running.Add([pscustomobject]@{ Process=$process; Source=$item.Source }) +} +while ($running.Count) { Reap-Compressors $true } + +if ($failed) { throw "$failed compression jobs failed" } +$rawBytes = ($work | ForEach-Object { [IO.FileInfo]$_.Source } | Measure-Object Length -Sum).Sum +$logicalBytes = ($work | ForEach-Object { [IO.FileInfo]$_.Destination } | Measure-Object Length -Sum).Sum +$allocatedBytes = 0L +foreach ($item in $work) { + [uint32]$high = 0 + [uint32]$low = [CObjFileSize]::GetCompressedFileSizeW($item.Destination, [ref]$high) + if ($low -eq [uint32]::MaxValue -and [Runtime.InteropServices.Marshal]::GetLastWin32Error() -ne 0) { + throw "unable to query allocated size: $($item.Destination)" + } + $allocatedBytes += ([int64]$high -shl 32) -bor $low +} +Write-Host ("done: {0} independent objects, raw {1:n2} GiB, container logical {2:n2} GiB, allocated {3:n2} GiB ({4:n1}% of raw), {5:n1}s" -f + $work.Count,($rawBytes/1GB),($logicalBytes/1GB),($allocatedBytes/1GB), + (100.0*$allocatedBytes/$rawBytes),$timer.Elapsed.TotalSeconds) diff --git a/scripts/cobj_smoke.c b/scripts/cobj_smoke.c new file mode 100644 index 000000000..77bc677e8 --- /dev/null +++ b/scripts/cobj_smoke.c @@ -0,0 +1,5 @@ +int +main(void) +{ + return 0; +} diff --git a/scripts/retarget_rsp.ps1 b/scripts/retarget_rsp.ps1 new file mode 100644 index 000000000..890b44cfd --- /dev/null +++ b/scripts/retarget_rsp.ps1 @@ -0,0 +1,45 @@ +param( + [Parameter(Mandatory=$true)][string]$ResponseFile, + [Parameter(Mandatory=$true)][string]$Destination, + [Parameter(Mandatory=$true)][string]$Output, + [Parameter(Mandatory=$true)][string]$Pdb +) + +$ErrorActionPreference = 'Stop' +$ResponseFile = [IO.Path]::GetFullPath($ResponseFile) +$Destination = [IO.Path]::GetFullPath($Destination) +$Output = [IO.Path]::GetFullPath($Output) +$Pdb = [IO.Path]::GetFullPath($Pdb) + +foreach ($directory in @([IO.Path]::GetDirectoryName($Destination), + [IO.Path]::GetDirectoryName($Output), + [IO.Path]::GetDirectoryName($Pdb))) { + [IO.Directory]::CreateDirectory($directory) | Out-Null +} + +$outputArg = '/OUT:"' + ($Output -replace '\\','/') + '"' +$pdbArg = '/PDB:"' + ($Pdb -replace '\\','/') + '"' +$foundOutput = $false +$foundPdb = $false +$rewritten = foreach ($line in [IO.File]::ReadAllLines($ResponseFile)) { + if ($line -match '(?i)^\s*/OUT:') { + $foundOutput = $true + $outputArg + } elseif ($line -match '(?i)^\s*/PDB:') { + $foundPdb = $true + $pdbArg + } else { + $line + } +} + +if (!$foundOutput) { throw "Response file has no /OUT argument: $ResponseFile" } +if (!$foundPdb) { throw "Response file has no /PDB argument: $ResponseFile" } +[IO.File]::WriteAllLines($Destination, $rewritten) + +[pscustomobject]@{ + Source = $ResponseFile + Destination = $Destination + Output = $Output + Pdb = $Pdb +} | ConvertTo-Json -Compress diff --git a/scripts/run_cobj_smoke.bat b/scripts/run_cobj_smoke.bat new file mode 100644 index 000000000..a7830e304 --- /dev/null +++ b/scripts/run_cobj_smoke.bat @@ -0,0 +1,15 @@ +@echo off +setlocal +where cl >nul 2>nul || ( + echo Run this script from an x64 Visual Studio developer command prompt + exit /b 1 +) +pushd "%~dp0..\build" +cl /nologo /c /Z7 /Od "%~dp0cobj_smoke.c" /Focobj_smoke_raw.obj || exit /b 1 +rad_obj_compress.exe cobj_smoke_raw.obj cobj_smoke_compressed.obj 512 selkie || exit /b 1 +radlink.exe /nologo /debug:ghash /pdb:cobj_smoke.pdb /entry:main /subsystem:console /out:cobj_smoke_raw.exe cobj_smoke_raw.obj || exit /b 1 +set RAD_COBJ_CACHE_MIB=16 +set RAD_COBJ_SKIP_CLEANUP=1 +radlink.exe /nologo /debug:ghash /pdb:cobj_smoke.pdb /entry:main /subsystem:console /out:cobj_smoke_compressed.exe cobj_smoke_compressed.obj || exit /b 1 +fc /b cobj_smoke_raw.exe cobj_smoke_compressed.exe || exit /b 1 +popd diff --git a/scripts/stage_raw_rsp.ps1 b/scripts/stage_raw_rsp.ps1 new file mode 100644 index 000000000..f25672650 --- /dev/null +++ b/scripts/stage_raw_rsp.ps1 @@ -0,0 +1,46 @@ +param( + [Parameter(Mandatory=$true)][string]$ResponseFile, + [Parameter(Mandatory=$true)][string]$OutputDirectory, + [int]$Workers = 16 +) + +$ErrorActionPreference = 'Stop' +$ResponseFile = [IO.Path]::GetFullPath($ResponseFile) +$OutputDirectory = [IO.Path]::GetFullPath($OutputDirectory) +[IO.Directory]::CreateDirectory($OutputDirectory) | Out-Null + +$lines = [IO.File]::ReadAllLines($ResponseFile) +$rewritten = New-Object string[] $lines.Length +$sources = [Collections.Generic.List[string]]::new() +$destinations = [Collections.Generic.List[string]]::new() +for ($lineIndex = 0; $lineIndex -lt $lines.Length; ++$lineIndex) { + $path = $lines[$lineIndex].Trim().Trim('"') + if ($path -match '(?i)\.obj$' -and [IO.File]::Exists($path)) { + $dest = Join-Path $OutputDirectory ('{0:D5}_{1}' -f $sources.Count,[IO.Path]::GetFileName($path)) + $sources.Add($path) + $destinations.Add($dest) + $rewritten[$lineIndex] = '"' + ($dest -replace '\\','/') + '"' + } else { + $rewritten[$lineIndex] = $lines[$lineIndex] + } +} + +Add-Type -TypeDefinition @' +using System; +using System.IO; +using System.Threading.Tasks; +public static class RawObjStager { + public static void Copy(string[] sources, string[] destinations, int workers) { + Parallel.For(0, sources.Length, new ParallelOptions { MaxDegreeOfParallelism = workers }, i => { + File.Copy(sources[i], destinations[i], true); + }); + } +} +'@ + +$timer = [Diagnostics.Stopwatch]::StartNew() +[RawObjStager]::Copy($sources.ToArray(), $destinations.ToArray(), $Workers) +[IO.File]::WriteAllLines((Join-Path $OutputDirectory 'raw.rsp'), $rewritten) +$bytes = ($destinations | ForEach-Object { [IO.FileInfo]$_ } | Measure-Object Length -Sum).Sum +Write-Host ("staged {0} independent raw objects, {1:n2} GiB in {2:n1}s" -f + $sources.Count,($bytes/1GB),$timer.Elapsed.TotalSeconds) diff --git a/scripts/test_link_determinism.ps1 b/scripts/test_link_determinism.ps1 new file mode 100644 index 000000000..a501ca71d --- /dev/null +++ b/scripts/test_link_determinism.ps1 @@ -0,0 +1,71 @@ +param( + [Parameter(Mandatory=$true)][string]$Rsp, + [Parameter(Mandatory=$true)][string]$Output, + [Parameter(Mandatory=$true)][string]$Pdb, + [int]$Iterations = 5, + [string]$ExtraArgs = '', + [string]$WorkingDirectory = '', + [string]$Radlink = '' +) + +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..')) +if (!$Radlink) { $Radlink = Join-Path $root 'out_cobj\radlink.exe' } +if (!$WorkingDirectory) { $WorkingDirectory = (Get-Location).Path } +$Radlink = [IO.Path]::GetFullPath($Radlink) +$Rsp = [IO.Path]::GetFullPath($Rsp) +$Output = [IO.Path]::GetFullPath($Output) +$Pdb = [IO.Path]::GetFullPath($Pdb) + +$expectedOutputHash = $null +$expectedPdbHash = $null +$results = [Collections.Generic.List[object]]::new() + +for ($iteration = 1; $iteration -le $Iterations; ++$iteration) { + # Never let a stale artifact make a successful-but-nonproducing invocation look valid. + [IO.File]::Delete($Output) + [IO.File]::Delete($Pdb) + + $psi = [Diagnostics.ProcessStartInfo]::new() + $psi.FileName = $Radlink + $psi.WorkingDirectory = $WorkingDirectory + $psi.Arguments = '@"' + $Rsp + '" ' + $ExtraArgs + $psi.UseShellExecute = $false + $psi.CreateNoWindow = $true + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + + $process = [Diagnostics.Process]::new() + $process.StartInfo = $psi + $timer = [Diagnostics.Stopwatch]::StartNew() + [void]$process.Start() + $stdout = $process.StandardOutput.ReadToEndAsync() + $stderr = $process.StandardError.ReadToEndAsync() + $process.WaitForExit() + $timer.Stop() + $combined = $stdout.GetAwaiter().GetResult() + $stderr.GetAwaiter().GetResult() + if ($process.ExitCode -ne 0) { + throw "link iteration $iteration failed with exit code $($process.ExitCode):`n$combined" + } + if (!(Test-Path -LiteralPath $Output) -or !(Test-Path -LiteralPath $Pdb)) { + throw "link iteration $iteration did not produce both requested outputs" + } + + $outputHash = (Get-FileHash -LiteralPath $Output -Algorithm SHA256).Hash + $pdbHash = (Get-FileHash -LiteralPath $Pdb -Algorithm SHA256).Hash + if ($iteration -eq 1) { + $expectedOutputHash = $outputHash + $expectedPdbHash = $pdbHash + } elseif ($outputHash -ne $expectedOutputHash -or $pdbHash -ne $expectedPdbHash) { + throw "non-deterministic output at iteration $iteration (image $outputHash, PDB $pdbHash)" + } + + $results.Add([pscustomobject]@{ + Iteration = $iteration + WallSeconds = $timer.Elapsed.TotalSeconds + ImageSha256 = $outputHash + PdbSha256 = $pdbHash + }) +} + +$results | ConvertTo-Json diff --git a/scripts/warm_rsp.ps1 b/scripts/warm_rsp.ps1 new file mode 100644 index 000000000..7fed9df49 --- /dev/null +++ b/scripts/warm_rsp.ps1 @@ -0,0 +1,31 @@ +param([Parameter(Mandatory=$true)][string]$Rsp) + +$ErrorActionPreference = 'Stop' +$Rsp = [IO.Path]::GetFullPath($Rsp) +$Buffer = New-Object byte[] (16MB) +$Files = [Collections.Generic.List[string]]::new() + +foreach ($line in [IO.File]::ReadLines($Rsp)) { + $path = $line.Trim().Trim('"') + if ($path -notmatch '(?i)\.obj$' -or -not [IO.File]::Exists($path)) { continue } + $Files.Add($path) +} + +$total = 0L +$timer = [Diagnostics.Stopwatch]::StartNew() +foreach ($path in $Files) { + $fs = [IO.FileStream]::new($path, [IO.FileMode]::Open, [IO.FileAccess]::Read, + [IO.FileShare]::ReadWrite -bor [IO.FileShare]::Delete, + 16MB, [IO.FileOptions]::SequentialScan) + try { + while (($got = $fs.Read($Buffer, 0, $Buffer.Length)) -gt 0) { $total += $got } + } finally { $fs.Dispose() } +} +$timer.Stop() +[pscustomobject]@{ + Rsp = $Rsp + Files = $Files.Count + LogicalGiBRead = $total / 1GB + Seconds = $timer.Elapsed.TotalSeconds + GiBPerSecond = if ($timer.Elapsed.TotalSeconds) { ($total / 1GB) / $timer.Elapsed.TotalSeconds } else { 0 } +} | ConvertTo-Json -Compress From 19448c208550b0721cad08e612d98e39033bb640 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 17 Aug 2026 12:58:40 -0700 Subject: [PATCH 112/121] radlink: productionize compressed OBJ runtime policy Select bounded cache capacities adaptively or from explicit command-line options, preserve the tuned post-type-merge cache behavior, and report diagnostics only when timer logging is enabled. Reuse per-thread Oodle decoder memory to remove allocator contention from segmented decode paths. --- src/linker/lnk.c | 2 + src/linker/lnk_compressed_obj.c | 394 ++++++++++++++++++++++++++++---- src/linker/lnk_compressed_obj.h | 3 +- src/linker/lnk_config.c | 31 +++ src/linker/lnk_config.h | 10 + 5 files changed, 389 insertions(+), 51 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index df6ca60f7..e9fbece82 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -1089,6 +1089,7 @@ lnk_inputer_flush(Arena *arena, TP_Context *tp, LNK_Inputer *inputer, LNK_IO_Fla has_portable_cobj |= !!(classify_tags[thin_input_idx] & LNK_InputClassify_PortableCompressed); } if (has_portable_cobj) { + lnk_compressed_obj_prepare_cache(thin_input_datas.v, thin_input_datas.count); LNK_InputOpenTask open_task = {thin_inputs, thin_input_datas.v}; // Reserving/splitting thousands of logical OBJ views contends on the Windows process VAD // lock. Four lanes measured substantially faster than 8-64 on the UEFN corpus. @@ -8094,6 +8095,7 @@ entry_point(CmdLine *cmdline) // init config from the command line LNK_Config *config = lnk_config_init(cmdline->argc, cmdline->argv); + lnk_compressed_obj_configure(config); // Snapshot summary identity immediately after command-line parsing, before // pool initialization and later scratch allocations. diff --git a/src/linker/lnk_compressed_obj.c b/src/linker/lnk_compressed_obj.c index b141a9f73..1a997c745 100644 --- a/src/linker/lnk_compressed_obj.c +++ b/src/linker/lnk_compressed_obj.c @@ -6,7 +6,10 @@ //////////////////////////////// // Runtime state -#define LNK_COBJ_DEFAULT_CACHE_GIB 8u +#define LNK_COBJ_MAX_INITIAL_CACHE_GIB 17u +#define LNK_COBJ_MAX_POST_CACHE_GIB 6u +#define LNK_COBJ_MAX_TOTAL_CACHE_GIB (LNK_COBJ_MAX_INITIAL_CACHE_GIB + LNK_COBJ_MAX_POST_CACHE_GIB) +#define LNK_COBJ_MIN_TOTAL_CACHE_GIB 4u #define LNK_COBJ_REGION_CAP (1u << 20) #define LNK_COBJ_DEFAULT_WRITE_GROUP_SLOTS 64u #define LNK_COBJ_FILE_MAPPING_GRANULARITY KB(64) @@ -109,15 +112,67 @@ typedef struct LNK_CObjCache U64 stored_bytes_read; U64 occupancy_hwm; U64 frozen_segments; + U64 raw_mapped_segments; + U64 input_compressed_bytes; + U64 input_raw_bytes; + U64 initial_cache_bytes; + U64 post_boundary_cache_bytes; + U64 decode_us; + U64 fault_us; + U64 eviction_us; + U64 freeze_us; + U64 trim_us; + U64 cleanup_us; B32 skip_cleanup; } LNK_CObjCache; +typedef enum LNK_CObjPolicySource +{ + LNK_CObjPolicySource_Adaptive, + LNK_CObjPolicySource_Environment, + LNK_CObjPolicySource_CommandLine, +} LNK_CObjPolicySource; + +typedef struct LNK_CObjPolicy +{ + U64 initial_cache_bytes; + U64 post_boundary_cache_bytes; + U32 trim_mode; + B32 freeze_generation; + B32 one_shot; + B32 configured; + LNK_CObjPolicySource initial_source; + LNK_CObjPolicySource post_source; + LNK_CObjPolicySource trim_source; + LNK_CObjPolicySource freeze_source; + LNK_CObjPolicySource one_shot_source; +} LNK_CObjPolicy; + +typedef struct LNK_CObjDecoderScratch +{ + void *memory; + OO_SINTa size; + OodleLZ_Compressor compressor; + B32 configured; +} LNK_CObjDecoderScratch; + global LNK_CObjCache g_lnk_cobj_cache; +global LNK_CObjPolicy g_lnk_cobj_policy; +global U64 g_lnk_cobj_input_capacity_hint; global volatile LONG g_lnk_cobj_cache_init_state; -global volatile LONG g_lnk_cobj_skip_cleanup_mode = -1; +// Oodle otherwise creates and destroys decoder state through the process heap for every segment. +// Links decode tens of thousands of independent segments concurrently, so those tiny allocations +// serialize on the heap lock. One block per calling thread is sufficient because every decode is +// unthreaded and synchronous. The container records its codec, so each block is sized for the +// actual Kraken, Mermaid, or Selkie decoder instead of pessimistically reserving Oodle's maximum. +global thread_static LNK_CObjDecoderScratch g_lnk_cobj_decoder_scratch; +global U64 g_lnk_cobj_decoder_scratch_count; +global U64 g_lnk_cobj_decoder_scratch_bytes; + global U64 g_lnk_cobj_window_decodes; global U64 g_lnk_cobj_window_bytes; +global U64 g_lnk_cobj_window_decoded_bytes; global B32 g_lnk_cobj_cache_shrunk; global U64 g_lnk_cobj_redecode_debug_s; global U64 g_lnk_cobj_redecode_types; @@ -126,6 +181,196 @@ global U64 g_lnk_cobj_redecode_segment_zero; //////////////////////////////// // Configuration and diagnostics +internal LNK_CObjDecoderScratch +lnk_cobj_decoder_scratch(U32 compressor_value) +{ + OodleLZ_Compressor compressor = (OodleLZ_Compressor)compressor_value; + if (g_lnk_cobj_decoder_scratch.memory != 0 && + g_lnk_cobj_decoder_scratch.configured && + g_lnk_cobj_decoder_scratch.compressor == compressor) { + return g_lnk_cobj_decoder_scratch; + } + + OO_S32 needed = OodleLZDecoder_MemorySizeNeeded(compressor, -1); + if (needed <= 0) { + compressor = OodleLZ_Compressor_Invalid; + needed = OodleLZDecoder_MemorySizeNeeded(compressor, -1); + } + if (needed > 0 && needed > g_lnk_cobj_decoder_scratch.size) { + void *memory = VirtualAlloc(0, (SIZE_T)needed, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); + if (memory != 0) { + void *old_memory = g_lnk_cobj_decoder_scratch.memory; + OO_SINTa old_size = g_lnk_cobj_decoder_scratch.size; + g_lnk_cobj_decoder_scratch.memory = memory; + g_lnk_cobj_decoder_scratch.size = needed; + if (old_memory == 0) { + InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_decoder_scratch_count); + } else { + VirtualFree(old_memory, 0, MEM_RELEASE); + } + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_decoder_scratch_bytes, + needed - old_size); + } + } + + if (needed > 0 && + g_lnk_cobj_decoder_scratch.memory != 0 && + g_lnk_cobj_decoder_scratch.size >= needed) { + g_lnk_cobj_decoder_scratch.compressor = compressor; + g_lnk_cobj_decoder_scratch.configured = 1; + return g_lnk_cobj_decoder_scratch; + } + + // Preserve Oodle's existing internal-allocation fallback if caller memory could not be made. + LNK_CObjDecoderScratch fallback = {0}; + return fallback; +} + +internal OO_SINTa +lnk_cobj_oodle_decompress(U32 compressor, const void *stored, OO_SINTa stored_size, + void *raw, OO_SINTa raw_size) +{ + LNK_CObjDecoderScratch scratch = lnk_cobj_decoder_scratch(compressor); + return OodleLZ_Decompress(stored, stored_size, raw, raw_size, + OodleLZ_FuzzSafe_Yes, OodleLZ_CheckCRC_No, OodleLZ_Verbosity_None, + 0, 0, 0, 0, scratch.memory, scratch.size, + OodleLZ_Decode_Unthreaded); +} + +internal char * +lnk_cobj_policy_source_string(LNK_CObjPolicySource source) +{ + switch (source) { + case LNK_CObjPolicySource_CommandLine: return "command-line"; + case LNK_CObjPolicySource_Environment: return "environment"; + default: return "adaptive"; + } +} + +internal B32 +lnk_cobj_env_u64(char *name, U64 *value_out) +{ + char value[64]; + DWORD len = GetEnvironmentVariableA(name, value, sizeof(value)); + if (len == 0 || len >= sizeof(value)) { return 0; } + char *end = 0; + U64 value_u64 = strtoull(value, &end, 10); + if (end == value || *end != 0) { return 0; } + *value_out = value_u64; + return 1; +} + +internal B32 +lnk_cobj_env_switch(char *name, B32 *enabled_out) +{ + char value[16]; + DWORD len = GetEnvironmentVariableA(name, value, sizeof(value)); + if (len == 0 || len >= sizeof(value)) { return 0; } + *enabled_out = value[0] != '0' && _stricmp(value, "no") != 0 && _stricmp(value, "false") != 0; + return 1; +} + +internal void +lnk_cobj_adaptive_capacities(U64 *initial_bytes_out, U64 *post_bytes_out) +{ + U64 total_gib = 8; + MEMORYSTATUSEX memory = { sizeof(memory) }; + if (GlobalMemoryStatusEx(&memory)) { + U64 physical_gib = memory.ullTotalPhys / GB(1); + U64 commit_available_gib = memory.ullAvailPageFile / GB(1); + U64 physical_budget_gib = (physical_gib * 3) / 8; + U64 commit_budget_gib = commit_available_gib / 2; + total_gib = Clamp(LNK_COBJ_MIN_TOTAL_CACHE_GIB, + Min(physical_budget_gib, commit_budget_gib), + LNK_COBJ_MAX_TOTAL_CACHE_GIB); + } + U64 post_gib = Max(1, (total_gib * LNK_COBJ_MAX_POST_CACHE_GIB) / LNK_COBJ_MAX_TOTAL_CACHE_GIB); + *initial_bytes_out = (total_gib - post_gib) * GB(1); + *post_bytes_out = post_gib * GB(1); +} + +internal void +lnk_compressed_obj_configure(LNK_Config *config) +{ + LNK_CObjPolicy policy = {0}; + lnk_cobj_adaptive_capacities(&policy.initial_cache_bytes, &policy.post_boundary_cache_bytes); + policy.freeze_generation = 1; + policy.trim_mode = 1; + + U64 env_u64 = 0; + if (lnk_cobj_env_u64("RAD_COBJ_CACHE_GIB", &env_u64) && env_u64 > 0) { + policy.initial_cache_bytes = env_u64 * GB(1); + policy.initial_source = LNK_CObjPolicySource_Environment; + } + if (lnk_cobj_env_u64("RAD_COBJ_CACHE_MIB", &env_u64) && env_u64 > 0) { + policy.initial_cache_bytes = env_u64 * MB(1); + policy.initial_source = LNK_CObjPolicySource_Environment; + } + if (lnk_cobj_env_u64("RAD_COBJ_CACHE_SHRINK_GIB", &env_u64) && env_u64 > 0) { + policy.post_boundary_cache_bytes = env_u64 * GB(1); + policy.post_source = LNK_CObjPolicySource_Environment; + } + + B32 env_switch = 0; + if (lnk_cobj_env_switch("RAD_COBJ_CACHE_FREEZE", &env_switch)) { + policy.freeze_generation = env_switch; + policy.freeze_source = LNK_CObjPolicySource_Environment; + } + if (lnk_cobj_env_u64("RAD_COBJ_TRIM_WS", &env_u64)) { + policy.trim_mode = (U32)Clamp(0, env_u64, 2); + policy.trim_source = LNK_CObjPolicySource_Environment; + } + if (lnk_cobj_env_switch("RAD_COBJ_SKIP_CLEANUP", &env_switch)) { + policy.one_shot = env_switch; + policy.one_shot_source = LNK_CObjPolicySource_Environment; + } + + if (config != 0) { + if (config->cobj_cache_gib > 0) { + policy.initial_cache_bytes = config->cobj_cache_gib * GB(1); + policy.initial_source = LNK_CObjPolicySource_CommandLine; + } + if (config->cobj_cache_shrink_gib > 0) { + policy.post_boundary_cache_bytes = config->cobj_cache_shrink_gib * GB(1); + policy.post_source = LNK_CObjPolicySource_CommandLine; + } + if (config->cobj_cache_freeze != LNK_SwitchState_Null) { + policy.freeze_generation = config->cobj_cache_freeze == LNK_SwitchState_Yes; + policy.freeze_source = LNK_CObjPolicySource_CommandLine; + } + if (config->cobj_trim_ws != LNK_SwitchState_Null) { + policy.trim_mode = config->cobj_trim_ws == LNK_SwitchState_Yes; + policy.trim_source = LNK_CObjPolicySource_CommandLine; + } + if (config->cobj_one_shot != LNK_SwitchState_Null) { + policy.one_shot = config->cobj_one_shot == LNK_SwitchState_Yes; + policy.one_shot_source = LNK_CObjPolicySource_CommandLine; + } + } + + policy.initial_cache_bytes = Max(MB(1), policy.initial_cache_bytes); + policy.post_boundary_cache_bytes = Max(MB(1), policy.post_boundary_cache_bytes); + policy.configured = 1; + g_lnk_cobj_policy = policy; +} + +internal void +lnk_compressed_obj_prepare_cache(String8 *mapped_files, U64 count) +{ + if (g_lnk_cobj_cache_init_state != 0) { return; } + U64 raw_bytes = 0; + for EachIndex(i, count) { + String8 mapped_file = mapped_files[i]; + if (mapped_file.size < sizeof(LNK_CObjHeader)) { continue; } + LNK_CObjHeader *header = (LNK_CObjHeader *)mapped_file.str; + if (header->magic != LNK_COBJ_MAGIC || header->version != LNK_COBJ_VERSION) { continue; } + raw_bytes = header->raw_size > max_U64 - raw_bytes ? max_U64 : raw_bytes + header->raw_size; + } + if (raw_bytes > g_lnk_cobj_input_capacity_hint) { + g_lnk_cobj_input_capacity_hint = raw_bytes; + } +} + internal LNK_CompressedObj *lnk_cobj_region_from_address(U8 *address); internal int @@ -189,13 +434,7 @@ lnk_compressed_obj_log_phase_stats(char *tag) internal B32 lnk_cobj_skip_cleanup_enabled(void) { - if (g_lnk_cobj_skip_cleanup_mode < 0) { - char value[8]; - DWORD len = GetEnvironmentVariableA("RAD_COBJ_SKIP_CLEANUP", value, sizeof(value)); - LONG enabled = len > 0 && len < sizeof(value) && value[0] != '0'; - InterlockedCompareExchange(&g_lnk_cobj_skip_cleanup_mode, enabled, -1); - } - return g_lnk_cobj_skip_cleanup_mode != 0; + return g_lnk_cobj_policy.one_shot; } //////////////////////////////// @@ -322,14 +561,15 @@ lnk_compressed_obj_copy_range(LNK_CompressedObj *obj, Rng1U64 range, void *raw_d if (entry->flags & LNK_COBJ_SEGMENT_RAW) { MemoryCopy(decode_dst, stored, entry->raw_size); } else { - OO_SINTa got = OodleLZ_Decompress(stored, entry->stored_size, decode_dst, entry->raw_size, - OodleLZ_FuzzSafe_Yes, OodleLZ_CheckCRC_No, OodleLZ_Verbosity_None, - 0, 0, 0, 0, 0, 0, OodleLZ_Decode_Unthreaded); + OO_SINTa got = lnk_cobj_oodle_decompress(obj->header->compressor, + stored, entry->stored_size, + decode_dst, entry->raw_size); ok = got == entry->raw_size; } if (!ok) { return 0; } InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_window_decodes); InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_window_bytes, entry->stored_size); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_window_decoded_bytes, entry->raw_size); if (decode_direct) { // Large $S and symbol-fixup walks consume complete interior segments once. Decode those // bytes into their final arena instead of filling the window and copying them again. @@ -458,12 +698,12 @@ lnk_cobj_write_group_complete(U32 group_idx) internal void lnk_compressed_obj_trim_working_set(void) { - char *trim_env = getenv("RAD_COBJ_TRIM_WS"); - if (trim_env == 0 || trim_env[0] == 0 || trim_env[0] == '0') { return; } + if (g_lnk_cobj_policy.trim_mode == 0) { return; } U64 begin_us = now_time_us(); - if (trim_env[0] == '2') { + if (g_lnk_cobj_policy.trim_mode == 2) { SetProcessWorkingSetSize(GetCurrentProcess(), (SIZE_T)-1, (SIZE_T)-1); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.trim_us, now_time_us() - begin_us); lnk_log(LNK_Log_Timers, "[cobj trim] whole-process elapsed=%.2f ms", (F64)(now_time_us() - begin_us) / 1000.0); return; @@ -478,18 +718,13 @@ lnk_compressed_obj_trim_working_set(void) U64 cache_view_count = 0; U64 cache_view_bytes = 0; U64 purged_count = 0; - U64 shrink_gib = 0; - { - char shrink_env[64]; - DWORD shrink_env_len = GetEnvironmentVariableA("RAD_COBJ_CACHE_SHRINK_GIB", shrink_env, sizeof(shrink_env)); - if (shrink_env_len > 0 && shrink_env_len < sizeof(shrink_env)) { shrink_gib = strtoull(shrink_env, 0, 10); } - } - if (shrink_gib > 0 && !g_lnk_cobj_cache_shrunk && g_lnk_cobj_cache.segment_size) { - U64 wanted_slots = Clamp(1, shrink_gib * GB(1) / g_lnk_cobj_cache.segment_size, + U64 post_boundary_cache_bytes = g_lnk_cobj_cache.post_boundary_cache_bytes; + if (post_boundary_cache_bytes > 0 && !g_lnk_cobj_cache_shrunk && g_lnk_cobj_cache.segment_size) { + U64 wanted_slots = Clamp(1, post_boundary_cache_bytes / g_lnk_cobj_cache.segment_size, g_lnk_cobj_cache.slot_count); - char *freeze_env = getenv("RAD_COBJ_CACHE_FREEZE"); - B32 freeze_generation = freeze_env != 0 && freeze_env[0] != 0 && freeze_env[0] != '0'; + B32 freeze_generation = g_lnk_cobj_policy.freeze_generation; if (freeze_generation) { + U64 freeze_begin_us = now_time_us(); // Preserve the complete first-generation mapping but never recycle one of its slots again. // Existing logical OBJ pointers therefore remain valid. PDB misses use a fresh, independently // evicting cache generation; this costs shrink_gib additional system commit but avoids the @@ -548,8 +783,10 @@ lnk_compressed_obj_trim_working_set(void) CloseHandle(old_mapping); U64 trimmed_frozen = 0; U64 trim_calls = 0; + U64 frozen_trim_us = 0; char *freeze_trim_env = getenv("RAD_COBJ_CACHE_FREEZE_TRIM"); if (freeze_trim_env != 0 && freeze_trim_env[0] != 0 && freeze_trim_env[0] != '0') { + U64 frozen_trim_begin_us = now_time_us(); // Keep every view valid but offer its resident pages back to the OS. Later consumers // incur only soft page faults into the frozen pagefile section, never Oodle re-decodes. // This is deliberately optional: it trades some kernel fault work for a lower PDB-phase @@ -598,11 +835,15 @@ lnk_compressed_obj_trim_working_set(void) } VirtualFree(trim_addresses, 0, MEM_RELEASE); } + frozen_trim_us = now_time_us() - frozen_trim_begin_us; + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.trim_us, frozen_trim_us); } VirtualFree(old_slots, 0, MEM_RELEASE); + U64 freeze_us = now_time_us() - freeze_begin_us; + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.freeze_us, freeze_us); lnk_log(LNK_Log_Timers, "[cobj freeze] retained=%llu trimmed=%llu trim-calls=%llu new-cache=%llu MiB elapsed=%.2f ms", g_lnk_cobj_cache.frozen_segments, trimmed_frozen, trim_calls, new_mapping_size / MB(1), - (F64)(now_time_us() - begin_us) / 1000.0); + (F64)freeze_us / 1000.0); return; } ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); @@ -652,7 +893,7 @@ lnk_compressed_obj_trim_working_set(void) g_lnk_cobj_cache_shrunk = 1; ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); } - if (shrink_gib == 0) { + if (post_boundary_cache_bytes == 0) { for EachIndex(region_idx, g_lnk_cobj_cache.region_count) { LNK_CompressedObj *region = g_lnk_cobj_cache.regions[region_idx]; if (!region || !region->active) { continue; } @@ -667,6 +908,7 @@ lnk_compressed_obj_trim_working_set(void) } } } + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.trim_us, now_time_us() - begin_us); lnk_log(LNK_Log_Timers, "[cobj trim] purged=%llu active=%llu MiB cache_views=%llu cache=%llu GiB elapsed=%.2f ms", purged_count, (g_lnk_cobj_cache.active_slot_count * g_lnk_cobj_cache.segment_size) / MB(1), cache_view_count, cache_view_bytes / GB(1), @@ -728,7 +970,13 @@ lnk_cobj_evict_one_locked(U32 *slot_idx_out) } U8 *victim_address = victim->address; - if (victim_address == 0 || !g_lnk_cobj_cache.unmap_view_of_file_2(GetCurrentProcess(), victim_address, MEM_PRESERVE_PLACEHOLDER)) { + U64 eviction_begin_us = now_time_us(); + B32 unmapped = victim_address != 0 && + g_lnk_cobj_cache.unmap_view_of_file_2(GetCurrentProcess(), victim_address, + MEM_PRESERVE_PLACEHOLDER); + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.eviction_us, + now_time_us() - eviction_begin_us); + if (!unmapped) { InterlockedExchange(&victim->state, LNK_CObjSegState_Ready); return 0; } @@ -782,15 +1030,16 @@ lnk_cobj_materialize(LNK_CompressedObj *region, U32 seg_idx) (DWORD)(slot_off >> 32), (DWORD)slot_off, region->segment_size); } B32 ok = write_view != 0; + U64 decode_begin_us = now_time_us(); if (ok) { U8 *stored = region->mapped_file.str + entry->file_offset; if (entry->flags & LNK_COBJ_SEGMENT_RAW) { MemoryCopy(write_view, stored, entry->raw_size); InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_cache.raw_segments); } else { - OO_SINTa decoded = OodleLZ_Decompress(stored, entry->stored_size, write_view, entry->raw_size, - OodleLZ_FuzzSafe_Yes, OodleLZ_CheckCRC_No, OodleLZ_Verbosity_None, - 0, 0, 0, 0, 0, 0, OodleLZ_Decode_Unthreaded); + OO_SINTa decoded = lnk_cobj_oodle_decompress(region->header->compressor, + stored, entry->stored_size, + write_view, entry->raw_size); ok = decoded == entry->raw_size; } if (ok && entry->raw_size < region->segment_size) { @@ -798,6 +1047,8 @@ lnk_cobj_materialize(LNK_CompressedObj *region, U32 seg_idx) } if (temporary_write_view) { UnmapViewOfFile(write_view); } } + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.decode_us, + now_time_us() - decode_begin_us); U8 *target = region->base + (U64)seg_idx * region->segment_size; if (ok) { ok = lnk_cobj_isolate_portable_segment(region, seg_idx); } if (ok) { @@ -871,11 +1122,14 @@ lnk_cobj_veh(EXCEPTION_POINTERS *info) U8 *address = (U8 *)er->ExceptionInformation[1]; LNK_CompressedObj *region = lnk_cobj_region_from_address(address); if (region != 0) { + U64 fault_begin_us = now_time_us(); InterlockedIncrement64((volatile LONG64 *)&g_lnk_cobj_cache.faults); U32 seg_idx = (U32)((address - region->base) / region->segment_size); U64 access_kind = er->ExceptionInformation[0]; if (access_kind == 0 && seg_idx < region->segment_count && lnk_cobj_materialize(region, seg_idx)) { + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.fault_us, + now_time_us() - fault_begin_us); return EXCEPTION_CONTINUE_EXECUTION; } if (access_kind == 1 && seg_idx < region->segment_count && @@ -887,10 +1141,14 @@ lnk_cobj_veh(EXCEPTION_POINTERS *info) void *page = (void *)((UINT_PTR)address & ~(UINT_PTR)(KB(4) - 1)); DWORD old_protect = 0; if (VirtualProtect(page, KB(4), PAGE_WRITECOPY, &old_protect)) { + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.fault_us, + now_time_us() - fault_begin_us); return EXCEPTION_CONTINUE_EXECUTION; } } } + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.fault_us, + now_time_us() - fault_begin_us); } } return EXCEPTION_CONTINUE_SEARCH; @@ -917,21 +1175,21 @@ lnk_cobj_cache_init(U32 segment_size) return 0; } - U64 cache_gib = LNK_COBJ_DEFAULT_CACHE_GIB; - char cache_env[64]; - DWORD cache_env_len = GetEnvironmentVariableA("RAD_COBJ_CACHE_GIB", cache_env, sizeof(cache_env)); - if (cache_env_len > 0 && cache_env_len < sizeof(cache_env)) { - U64 parsed = strtoull(cache_env, 0, 10); - if (parsed > 0) { cache_gib = parsed; } - } - U64 cache_bytes = cache_gib * GB(1); - char cache_mib_env[64]; - DWORD cache_mib_env_len = GetEnvironmentVariableA("RAD_COBJ_CACHE_MIB", cache_mib_env, sizeof(cache_mib_env)); - if (cache_mib_env_len > 0 && cache_mib_env_len < sizeof(cache_mib_env)) { - U64 parsed = strtoull(cache_mib_env, 0, 10); - if (parsed > 0) { cache_bytes = parsed * MB(1); } + if (!g_lnk_cobj_policy.configured) { lnk_compressed_obj_configure(0); } + U64 cache_bytes = g_lnk_cobj_policy.initial_cache_bytes; + U64 post_boundary_cache_bytes = g_lnk_cobj_policy.post_boundary_cache_bytes; + U64 segment_size_u64 = segment_size; + if (g_lnk_cobj_input_capacity_hint > 0) { + if (g_lnk_cobj_input_capacity_hint < cache_bytes) { + cache_bytes = AlignPow2(g_lnk_cobj_input_capacity_hint, segment_size_u64); + } + if (g_lnk_cobj_input_capacity_hint < post_boundary_cache_bytes) { + post_boundary_cache_bytes = AlignPow2(g_lnk_cobj_input_capacity_hint, segment_size_u64); + } } g_lnk_cobj_cache.skip_cleanup = lnk_cobj_skip_cleanup_enabled(); + g_lnk_cobj_cache.initial_cache_bytes = cache_bytes; + g_lnk_cobj_cache.post_boundary_cache_bytes = post_boundary_cache_bytes; g_lnk_cobj_cache.slot_count = Max(1, cache_bytes / segment_size); g_lnk_cobj_cache.active_slot_count = g_lnk_cobj_cache.slot_count; g_lnk_cobj_cache.segment_size = segment_size; @@ -1143,6 +1401,7 @@ lnk_cobj_open_portable(LNK_Input *input, String8 mapped_file) } LNK_CObjSegment *directory = (LNK_CObjSegment *)(mapped_file.str + header->directory_offset); B32 has_portable_raw_segments = 0; + U64 raw_mapped_segment_count = 0; for (U32 i = 0; i < header->segment_count; ++i) { LNK_CObjSegment *entry = &directory[i]; U64 segment_raw_offset = (U64)i * header->segment_size; @@ -1156,6 +1415,7 @@ lnk_cobj_open_portable(LNK_Input *input, String8 mapped_file) return 0; } has_portable_raw_segments |= !!(entry->flags & LNK_COBJ_SEGMENT_RAW); + raw_mapped_segment_count += !!(entry->flags & LNK_COBJ_SEGMENT_RAW); } U32 type_index_count = 0; LNK_CObjTypeIndex *type_indices = 0; @@ -1378,6 +1638,9 @@ lnk_cobj_open_portable(LNK_Input *input, String8 mapped_file) AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); if (g_lnk_cobj_cache.region_count < g_lnk_cobj_cache.region_cap) { g_lnk_cobj_cache.regions[g_lnk_cobj_cache.region_count++] = region; + g_lnk_cobj_cache.raw_mapped_segments += raw_mapped_segment_count; + g_lnk_cobj_cache.input_compressed_bytes += mapped_file.size; + g_lnk_cobj_cache.input_raw_bytes += header->raw_size; registered = 1; } ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); @@ -1417,11 +1680,14 @@ lnk_compressed_obj_close(LNK_Input *input) { LNK_CompressedObj *region = input->compressed_obj; if (!region) { return; } + U64 cleanup_begin_us = now_time_us(); if (g_lnk_cobj_cache.skip_cleanup) { // Benchmark/diagnostic mode: the process is about to exit and Windows tears the address // space down more efficiently than issuing one UnmapViewOfFile2 call per resident segment. // Useful for separating link work from explicit VAD teardown; not the bounded-cache path. input->owns_file_map = 0; + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.cleanup_us, + now_time_us() - cleanup_begin_us); return; } InterlockedExchange(®ion->active, 0); @@ -1465,6 +1731,8 @@ lnk_compressed_obj_close(LNK_Input *input) input->compressed_data = str8_zero(); input->compressed_obj = 0; input->owns_file_map = 0; + InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.cleanup_us, + now_time_us() - cleanup_begin_us); } //////////////////////////////// @@ -1478,16 +1746,40 @@ lnk_compressed_obj_log_stats(void) return; } U64 unique_decodes = g_lnk_cobj_cache.decodes - g_lnk_cobj_cache.redecodes; - U64 amp_x100 = unique_decodes ? (g_lnk_cobj_cache.decodes * 100) / unique_decodes : 0; + U64 total_decodes = g_lnk_cobj_cache.decodes + g_lnk_cobj_window_decodes; + U64 total_decoded_bytes = g_lnk_cobj_cache.decoded_bytes + g_lnk_cobj_window_decoded_bytes; + U64 total_stored_bytes = g_lnk_cobj_cache.stored_bytes_read + g_lnk_cobj_window_bytes; lnk_log(LNK_Log_Timers, - "[cobj] regions=%llu segment=%llu KiB cache=%llu MiB resident=%llu faults=%llu decodes=%llu unique=%llu redecodes=%llu amp=%llu.%02llux evictions=%llu decoded=%llu MiB stored=%llu MiB", + "[cobj] files=%llu input=%llu/%llu MiB segment=%llu KiB cache=%llu MiB(%s)->%llu MiB(%s) freeze=%u(%s) trim=%u(%s) one-shot=%u(%s) resident-hwm=%llu faults=%llu unique=%llu decodes=%llu decoded=%llu MiB read=%llu MiB evictions=%llu redecodes=%llu frozen=%llu/%llu MiB raw-mapped=%llu decoder-scratch=%llu/%llu KiB time-ms{decode=%.2f fault=%.2f evict=%.2f freeze=%.2f trim=%.2f cleanup=%.2f}\n", g_lnk_cobj_cache.region_count, + g_lnk_cobj_cache.input_compressed_bytes / MB(1), + g_lnk_cobj_cache.input_raw_bytes / MB(1), g_lnk_cobj_cache.segment_size / KB(1), - (g_lnk_cobj_cache.slot_count * g_lnk_cobj_cache.segment_size) / MB(1), - g_lnk_cobj_cache.occupancy_hwm, g_lnk_cobj_cache.faults, g_lnk_cobj_cache.decodes, - unique_decodes, g_lnk_cobj_cache.redecodes, amp_x100 / 100, amp_x100 % 100, + g_lnk_cobj_cache.initial_cache_bytes / MB(1), + lnk_cobj_policy_source_string(g_lnk_cobj_policy.initial_source), + g_lnk_cobj_cache.post_boundary_cache_bytes / MB(1), + lnk_cobj_policy_source_string(g_lnk_cobj_policy.post_source), + g_lnk_cobj_policy.freeze_generation, + lnk_cobj_policy_source_string(g_lnk_cobj_policy.freeze_source), + g_lnk_cobj_policy.trim_mode, + lnk_cobj_policy_source_string(g_lnk_cobj_policy.trim_source), + g_lnk_cobj_policy.one_shot, + lnk_cobj_policy_source_string(g_lnk_cobj_policy.one_shot_source), + g_lnk_cobj_cache.occupancy_hwm, g_lnk_cobj_cache.faults, unique_decodes, total_decodes, + total_decoded_bytes / MB(1), total_stored_bytes / MB(1), g_lnk_cobj_cache.evictions, - g_lnk_cobj_cache.decoded_bytes / MB(1), g_lnk_cobj_cache.stored_bytes_read / MB(1)); + g_lnk_cobj_cache.redecodes, + g_lnk_cobj_cache.frozen_segments, + (g_lnk_cobj_cache.frozen_segments * g_lnk_cobj_cache.segment_size) / MB(1), + g_lnk_cobj_cache.raw_mapped_segments, + g_lnk_cobj_decoder_scratch_count, + g_lnk_cobj_decoder_scratch_bytes / KB(1), + (F64)g_lnk_cobj_cache.decode_us / 1000.0, + (F64)g_lnk_cobj_cache.fault_us / 1000.0, + (F64)g_lnk_cobj_cache.eviction_us / 1000.0, + (F64)g_lnk_cobj_cache.freeze_us / 1000.0, + (F64)g_lnk_cobj_cache.trim_us / 1000.0, + (F64)g_lnk_cobj_cache.cleanup_us / 1000.0); if (g_lnk_cobj_cache.frozen_segments) { lnk_log(LNK_Log_Timers, "[cobj frozen] segments=%llu", g_lnk_cobj_cache.frozen_segments); } @@ -1503,6 +1795,8 @@ lnk_compressed_obj_log_stats(void) } #else +internal void lnk_compressed_obj_configure(struct LNK_Config *config) { (void)config; } +internal void lnk_compressed_obj_prepare_cache(String8 *mapped_files, U64 count) { (void)mapped_files; (void)count; } internal void lnk_compressed_obj_trim_working_set(void) {} internal String8 lnk_compressed_obj_direct_range(LNK_CompressedObj *obj, Rng1U64 range) { return str8_zero(); } internal B32 lnk_compressed_obj_type_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjTypeIndexView *out) { MemoryZeroStruct(out); return 0; } diff --git a/src/linker/lnk_compressed_obj.h b/src/linker/lnk_compressed_obj.h index e19aa80da..0631de85c 100644 --- a/src/linker/lnk_compressed_obj.h +++ b/src/linker/lnk_compressed_obj.h @@ -48,7 +48,8 @@ struct LNK_CObjDecodeWindow U32 valid_size; }; - +internal void lnk_compressed_obj_configure(struct LNK_Config *config); +internal void lnk_compressed_obj_prepare_cache(String8 *mapped_files, U64 count); internal void lnk_compressed_obj_trim_working_set(void); internal String8 lnk_compressed_obj_direct_range(LNK_CompressedObj *obj, Rng1U64 range); internal B32 lnk_compressed_obj_type_index(LNK_CompressedObj *obj, Rng1U64 section_range, LNK_CObjTypeIndexView *out); diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index 0480309cd..1b441e784 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -68,6 +68,11 @@ global read_only LNK_CmdSwitch g_cmd_switch_map[] = { LNK_CmdSwitch_WholeArchive, 0, LNK_CmdValueKind_Scalar, "WHOLEARCHIVE", "[:LIBNAME]", "Force linker to pull in all objs from the specified lib." }, { LNK_CmdSwitch_Rad_Age, 0, LNK_CmdValueKind_Scalar, "RAD_AGE", ":#", "Age embeded in EXE and PDB, used to validate incremental build. Default is 1." }, + { LNK_CmdSwitch_Rad_CObjCacheGiB, 0, LNK_CmdValueKind_Scalar, "RAD_COBJ_CACHE_GIB", ":#", "Initial decoded compressed-OBJ cache capacity in GiB." }, + { LNK_CmdSwitch_Rad_CObjCacheShrinkGiB, 0, LNK_CmdValueKind_Scalar, "RAD_COBJ_CACHE_SHRINK_GIB", ":#", "Post-type-merge decoded compressed-OBJ cache capacity in GiB." }, + { LNK_CmdSwitch_Rad_CObjCacheFreeze, 0, LNK_CmdValueKind_Scalar, "RAD_COBJ_CACHE_FREEZE", "[:NO]", "Retain the initial decoded generation when starting the post-boundary cache." }, + { LNK_CmdSwitch_Rad_CObjTrimWs, 0, LNK_CmdValueKind_Scalar, "RAD_COBJ_TRIM_WS", "[:NO]", "Trim decoded cache working-set pages at the type-merge boundary." }, + { LNK_CmdSwitch_Rad_CObjOneShot, 0, LNK_CmdValueKind_Scalar, "RAD_COBJ_ONE_SHOT", "[:NO]", "Use process-teardown cleanup for this one-shot linker invocation." }, //{ LNK_CmdSwitch_Rad_BuildExp, 0, LNK_CmdValueKind_Scalar, "RAD_BUILD_EXP", "[:NO]", "Build export data." }, { LNK_CmdSwitch_Rad_BuildInfo, 0, LNK_CmdValueKind_Null, "RAD_BUILD_INFO", "", "Print build info and exit." }, { LNK_CmdSwitch_Rad_BuildImpLib, 0, LNK_CmdValueKind_Scalar, "RAD_BUILD_IMPLIB", "[:NO]", "Build import library." }, @@ -1973,6 +1978,32 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8 val lnk_cmd_switch_parse_u32(obj, cmd_switch, value, &config->age, 0); } break; + case LNK_CmdSwitch_Rad_CObjCacheGiB: { + if (lnk_cmd_switch_parse_u64(obj, cmd_switch, value, &config->cobj_cache_gib, 0) && + config->cobj_cache_gib == 0) { + lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "cache capacity must be greater than zero"); + } + } break; + + case LNK_CmdSwitch_Rad_CObjCacheShrinkGiB: { + if (lnk_cmd_switch_parse_u64(obj, cmd_switch, value, &config->cobj_cache_shrink_gib, 0) && + config->cobj_cache_shrink_gib == 0) { + lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "cache capacity must be greater than zero"); + } + } break; + + case LNK_CmdSwitch_Rad_CObjCacheFreeze: { + lnk_cmd_switch_parse_flag(obj, cmd_switch, value, &config->cobj_cache_freeze); + } break; + + case LNK_CmdSwitch_Rad_CObjTrimWs: { + lnk_cmd_switch_parse_flag(obj, cmd_switch, value, &config->cobj_trim_ws); + } break; + + case LNK_CmdSwitch_Rad_CObjOneShot: { + lnk_cmd_switch_parse_flag(obj, cmd_switch, value, &config->cobj_one_shot); + } break; + //case LNK_CmdSwitch_Rad_BuildExp: { // LNK_SwitchState state; // if (lnk_cmd_switch_parse_flag(obj, cmd_switch, value, &state)) { diff --git a/src/linker/lnk_config.h b/src/linker/lnk_config.h index 6e606a3ae..8ffdd5cde 100644 --- a/src/linker/lnk_config.h +++ b/src/linker/lnk_config.h @@ -102,6 +102,11 @@ typedef enum LNK_CmdSwitch_Rad_Age, LNK_CmdSwitch_Rad_BootMode, + LNK_CmdSwitch_Rad_CObjCacheGiB, + LNK_CmdSwitch_Rad_CObjCacheShrinkGiB, + LNK_CmdSwitch_Rad_CObjCacheFreeze, + LNK_CmdSwitch_Rad_CObjTrimWs, + LNK_CmdSwitch_Rad_CObjOneShot, LNK_CmdSwitch_Rad_BuildExp, LNK_CmdSwitch_Rad_BuildImpLib, LNK_CmdSwitch_Rad_BuildInfo, @@ -350,6 +355,11 @@ typedef struct LNK_Config U64 max_worker_count; U64 debug_worker_cap; String8 shared_thread_pool_name; + U64 cobj_cache_gib; + U64 cobj_cache_shrink_gib; + LNK_SwitchState cobj_cache_freeze; + LNK_SwitchState cobj_trim_ws; + LNK_SwitchState cobj_one_shot; LNK_SwitchState do_function_pad_min; B32 infer_function_pad_min; U64 function_pad_min; From 31a7063a3a694f93791f054e8123f3304ae7d2bd Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 17 Aug 2026 12:58:51 -0700 Subject: [PATCH 113/121] docs: specify the portable compressed OBJ format Document the version-1 container, segmented Oodle payload contract, optional linker sidecars, validation rules, and writer checklist used by the reference converter. --- docs/compressed_obj_format.md | 695 ++++++++++++++++++++++++++++++++ docs/compressed_obj_workflow.md | 200 +++++++++ 2 files changed, 895 insertions(+) create mode 100644 docs/compressed_obj_format.md create mode 100644 docs/compressed_obj_workflow.md diff --git a/docs/compressed_obj_format.md b/docs/compressed_obj_format.md new file mode 100644 index 000000000..b4ba6ada5 --- /dev/null +++ b/docs/compressed_obj_format.md @@ -0,0 +1,695 @@ +# RAD Link portable compressed OBJ format + +This document is the implementation contract for producing portable compressed OBJ files that the +current RAD Link compressed-OBJ reader accepts. It describes format version 1, identified by the +eight-byte ASCII magic `RLOBJ001`. + +The intended reader and reference writer are: + +- `src/linker/lnk_compressed_obj_format.h` +- `src/linker/lnk_compressed_obj.c` +- `src/linker/rad_obj_compress.c` + +If this document and the reader ever disagree, the reader is authoritative. A format change must +use a new version or magic; silently changing version 1 would invalidate cached objects. + +## 1. Implementation summary + +For the tested production profile, the writer should: + +1. Finish producing one normal COFF or BigObj byte stream in memory. +2. Split that byte stream into independent 512 KiB logical segments. +3. Compress each segment independently with Oodle Kraken, level Normal, and + `spaceSpeedTradeoffBytes = 256`. +4. Store a segment raw when compression fails or does not make it smaller. +5. Emit the 64-byte `RLOBJ001` header, the 24-byte-per-segment directory, the performance + sidecars described below, and then the segment payloads. +6. Verify every compressed segment by decoding it and comparing it with the original bytes. +7. Publish through a temporary file plus an atomic rename. + +The following settings produced the measured 22.97 GiB FortniteClient and 24.39 GiB UEFN corpora: + +| Setting | Required/recommended value | +|---|---| +| Container | `RLOBJ001`, version 1 | +| Byte order | Little-endian | +| Segment size | 512 KiB (`524288`) | +| Oodle compressor | Kraken (`OodleLZ_Compressor_Kraken`, value 8 in Oodle 2.9.16) | +| Oodle level | Normal | +| `spaceSpeedTradeoffBytes` | 256 | +| Segment dependencies | None; every segment is a separate Oodle stream | +| Incompressible data | Store raw | +| Filesystem features | None required; ordinary portable file | + +All compressed OBJ inputs in a single RAD Link invocation **must use exactly the same segment +size**. The cache is initialized from the first compressed OBJ and rejects a later compressed OBJ +with a different segment size. Use 512 KiB consistently for the entire corpus. + +This is **not** the UBA 12-byte-header format with an eight-byte size prefix between segments. UBA +payloads can use the same Oodle codec, but they must be repackaged into the `RLOBJ001` header and +directory layout described here before this RAD Link reader can consume them. + +## 2. Compatibility levels + +There are two useful conformance levels. + +### 2.1 Core container: correct but not performance-equivalent + +A minimal compatible file needs only: + +- the fixed header; +- the segment directory; +- independently compressed or raw-stored segment payloads; and +- the `PORTABLE_RAW_MAP` header flag. + +With no sidecar flags, set `header.reserved` to zero. RAD Link can reconstruct the original OBJ +view and link it correctly, but it must fault or decode more debug information through the generic +cache. This level is useful for bringing up and validating a new writer, but it is not expected to +match the benchmarked link time or memory use. + +### 2.2 Optimized container: the measured production profile + +To reproduce the measured behavior, also emit: + +- the packed `.debug$T`/`.debug$P` leaf index; +- the complete-UDT hash index; +- the AMD64 base-relocation candidate index; +- the `.debug$S` subsection index; and +- the `.debug$S` summaries. + +These sidecars are optimization hints derived entirely from the original OBJ. They do not replace +the original bytes. Incorrect sidecar data can change linker behavior, so omit an optional sidecar +until its implementation is known to be correct rather than emitting an approximation. + +## 3. General encoding rules + +- All integers are unsigned and little-endian. +- File offsets are absolute offsets from byte zero of the compressed file. +- Logical or raw offsets refer to offsets in the original uncompressed OBJ byte stream. +- `align_up(x, n)` means `(x + n - 1) & ~(n - 1)`, with `n` a power of two. +- All fields named `reserved` must be written as zero unless this document assigns them a meaning. +- Do not serialize the records with `#pragma pack(1)`. Use the byte offsets below, or use the C + declarations with their normal layout and assert every size. +- The ordinary output file length must cover all metadata, stored payloads, and required raw-run + padding. The format does not depend on sparse files, filesystem compression, bundles, or another + OBJ. +- Direct compressed inputs must currently use the `.obj` or `.o` extension. Use `.obj` for the + Windows toolchain. RAD Link's command-line parser classifies the path by extension before reading + the magic, so a path ending in `.radobj` is rejected as an unknown file format even when its + contents are a valid `RLOBJ001` container. The stored bytes are a RAD Link container rather than + a COFF file, and other linkers will not understand it. + +Recommended compile-time checks for a C or C++ implementation are: + +```cpp +static_assert(sizeof(LNK_CObjHeader) == 64); +static_assert(sizeof(LNK_CObjSegment) == 24); +static_assert(sizeof(LNK_CObjTypeIndex) == 40); +static_assert(sizeof(LNK_CObjUdtHashIndex) == 16); +static_assert(sizeof(LNK_CObjBaseRelocIndex) == 16); +static_assert(sizeof(LNK_CObjBaseRelocEntry) == 16); +static_assert(sizeof(LNK_CObjDebugSIndex) == 16); +static_assert(sizeof(LNK_CObjDebugSEntry) == 16); +static_assert(sizeof(LNK_CObjDebugSSummary) == 16); +``` + +## 4. Top-level file layout + +The reference writer uses this order: + +```text +0 ++-------------------------------+ +| 64-byte header | ++-------------------------------+ 8-byte aligned +| segment directory | segment_count * 24 bytes ++-------------------------------+ 8-byte aligned +| optional sidecar directories | ++-------------------------------+ 8-byte aligned +| optional sidecar payloads | ++-------------------------------+ 8-byte aligned +| segment payloads and padding | ++-------------------------------+ file length +``` + +Sidecar and segment offsets make the format relocatable inside the file. A production writer +should nevertheless use the canonical order above, keep ranges non-overlapping, and monotonically +advance a checked 64-bit layout cursor. + +## 5. Fixed header + +The header is exactly 64 bytes. + +| Offset | Type | Field | Value and meaning | +|---:|---:|---|---| +| 0 | `U64` | `magic` | `0x3130304A424F4C52`; bytes spell `RLOBJ001` | +| 8 | `U32` | `version` | `1` | +| 12 | `U32` | `header_size` | `64` | +| 16 | `U64` | `raw_size` | Exact byte length of the original OBJ; must be nonzero | +| 24 | `U32` | `segment_size` | Power of two, at least 64 KiB; use 512 KiB | +| 28 | `U32` | `segment_count` | `ceil(raw_size / segment_size)` | +| 32 | `U64` | `directory_offset` | Absolute offset of the segment directory; canonical value is 64 | +| 40 | `U64` | `data_offset` | Canonical first segment-payload position before payload alignment | +| 48 | `U32` | `compressor` | Oodle compressor enum; write 8 for Kraken | +| 52 | `U32` | `flags` | Container flags plus type-index count in bits 16-31 | +| 56 | `U64` | `reserved` | When sidecars exist, absolute offset of the sidecar directory block; otherwise zero | + +`data_offset` and `compressor` are descriptive in the version-1 reader; individual directory +entries locate payloads and Oodle identifies its stream internally. Writers must still populate +both fields correctly so future readers, diagnostics, and validators can rely on them. + +### 5.1 Header flags + +| Value | Name | Meaning | +|---:|---|---| +| `0x00000001` | `TYPE_INDEX` | One or more type-index directory records are present | +| `0x00000002` | retired | Never set; the portable reader rejects it | +| `0x00000004` | `UDT_HASH_INDEX` | One UDT-hash directory follows each type-index directory | +| `0x00000008` | `BASE_RELOC_INDEX` | A base-relocation directory is present | +| `0x00000010` | `PORTABLE_RAW_MAP` | Required for every version-1 portable file | +| `0x00000040` | `PACKED_TYPE_SIDECAR` | Type sidecars use the packed representation | +| `0x00000080` | `PACKED_TYPE_OFFSETS_V2` | Packed offset representation is version 2 | +| `0x00002000` | `DEBUG_S_INDEX` | A `.debug$S` directory is present | +| `0x00004000` | `DEBUG_S_SUMMARY` | One summary follows each `.debug$S` entry | +| `0xffff0000` | type count | Number of type-index records, shifted left by 16 | + +All unlisted bits must be zero. The reader rejects unknown bits. + +Flag dependencies are: + +- `UDT_HASH_INDEX` requires `TYPE_INDEX`. +- `DEBUG_S_SUMMARY` requires `DEBUG_S_INDEX`. +- `PACKED_TYPE_SIDECAR` and `PACKED_TYPE_OFFSETS_V2` must either both be set or both be clear. +- A nonzero type count requires `TYPE_INDEX`. +- `TYPE_INDEX` requires a nonzero type count, with a maximum of 65535. + +The packed flags have no purpose without a type index; a new writer should leave them clear when +there are no type records. + +## 6. Segment directory + +The segment directory contains `segment_count` consecutive 24-byte entries. Entry `i` describes +logical bytes beginning at `i * segment_size` in the original OBJ. + +| Entry offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U64` | `file_offset` | Absolute offset of the stored payload | +| 8 | `U32` | `stored_size` | Number of meaningful bytes stored at `file_offset` | +| 12 | `U32` | `raw_size` | Number of original bytes represented by this segment | +| 16 | `U32` | `flags` | Zero for Oodle data; `0x1` for raw data | +| 20 | `U32` | `reserved` | Zero | + +The only segment flag is: + +```text +LNK_COBJ_SEGMENT_RAW = 0x00000001 +``` + +For every entry: + +```text +raw_offset = i * segment_size +raw_size = min(segment_size, header.raw_size - raw_offset) +``` + +`stored_size` must be nonzero, and `[file_offset, file_offset + stored_size)` must be inside the +file. If `RAW` is set, `stored_size` must equal `raw_size`. If `RAW` is clear, the stored bytes must +be one complete Oodle stream that decodes to exactly `raw_size` bytes. + +The writer should choose raw storage when `OodleLZ_Compress` returns a nonpositive length or a +length greater than or equal to `raw_size`. This prevents compression from growing the file. + +### 6.1 Oodle call contract + +The tested writer initializes options from: + +```cpp +OodleLZ_Compressor compressor = OodleLZ_Compressor_Kraken; +OodleLZ_CompressionLevel level = OodleLZ_CompressionLevel_Normal; +OodleLZ_CompressOptions options = + *OodleLZ_CompressOptions_GetDefault(compressor, level); +options.spaceSpeedTradeoffBytes = 256; +OodleLZ_CompressOptions_Validate(&options); +``` + +It calls `OodleLZ_Compress` once per segment, with no dictionary base, long-range matcher, or shared +state. Do not continue an Oodle stream from one segment into the next. RAD Link may request any +segment first and may decode several segments concurrently. + +RAD Link decodes with fuzz safety enabled, quantum CRC checking disabled, no caller scratch, and +`OodleLZ_Decode_Unthreaded`. The writer currently does not request Oodle quantum CRCs. Integrity of +the complete portable object should be provided by the cache system's normal content hash. + +Do not mix Oodle codecs within one object. Version 1 has only one header-level compressor field, +even though the current Oodle decode entry point can identify a stream without consulting it. + +### 6.2 Raw-segment placement + +Raw segments are mapped directly from the portable file into RAD Link's logical OBJ address range. +Their layout therefore has additional requirements beyond the basic directory validation. + +For each maximal run of consecutive raw segments: + +1. Align the first raw segment's `file_offset` to 64 KiB. +2. Store full raw segments contiguously at `segment_size` intervals. +3. Because `segment_size` is a power of two and at least 64 KiB, every segment in the run remains + 64 KiB aligned. +4. If the final OBJ segment is raw and shorter than `segment_size`, extend the physical file through + `align_up(raw_size, 64 KiB)` bytes for that final segment. Padding bytes may be zero and are not + part of `stored_size`. + +Canonical payload-cursor logic is: + +```text +in_raw_run = false +cursor = header.data_offset + +for each segment: + if storing raw: + if !in_raw_run: + cursor = align_up(cursor, 64 KiB) + entry.file_offset = cursor + write raw_size bytes + cursor += is_final_segment + ? align_up(raw_size, 64 KiB) + : segment_size + in_raw_run = true + else: + in_raw_run = false + cursor = align_up(cursor, 8) + entry.file_offset = cursor + write stored_size compressed bytes + cursor += stored_size +``` + +The final file length is the final cursor, including raw-run padding. Do not rely on a sparse range +to provide the padding. + +## 7. Optional sidecar directory block + +When any sidecar is present, `header.reserved` points to one tightly packed directory block. The +block has no separate header. Its record order is determined entirely by `header.flags`: + +```text +LNK_CObjTypeIndex[type_count] if TYPE_INDEX +LNK_CObjUdtHashIndex[type_count] if UDT_HASH_INDEX +LNK_CObjBaseRelocIndex if BASE_RELOC_INDEX +LNK_CObjDebugSIndex if DEBUG_S_INDEX +``` + +There is no padding between these directory record arrays. Align the end of the complete block to +8 bytes before writing sidecar payload arrays. Sidecar payload offsets are absolute and each +payload should begin at an 8-byte-aligned file offset. + +## 8. Type leaf sidecar + +Emit one `LNK_CObjTypeIndex` for each `.debug$T` or `.debug$P` COFF section that has at least the +four-byte CodeView signature. Preserve COFF section order. + +The record is 40 bytes: + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U64` | `raw_section_offset` | Original OBJ offset of the first byte after the 4-byte CodeView signature | +| 8 | `U32` | `raw_section_size` | Section raw size minus 4 | +| 12 | `U32` | `leaf_count` | Number of valid leaf records indexed | +| 16 | `U64` | `offsets_file_offset` | Offset array or packed-offset block | +| 24 | `U64` | `sizes_file_offset` | Size array, or packed kind dictionary | +| 32 | `U64` | `kinds_file_offset` | Kind array, or packed kind-code array | + +For a section whose `PointerToRawData` is `foff` and `SizeOfRawData` is `fsize`: + +```text +raw_section_offset = foff + 4 +raw_section_size = fsize - 4 +``` + +Starting at `raw_section_offset`, parse leaves as follows: + +```text +cursor = 0 +while cursor + 4 <= raw_section_size: + size = read_u16(cursor + 0) + kind = read_u16(cursor + 2) + stride = size + 2 + if size < 2 or stride > raw_section_size - cursor: + stop + append offset=cursor, size=size, kind=kind + cursor += stride +``` + +Leaf offsets are relative to the first byte after the CodeView signature, not relative to the OBJ +or compressed container. The stored 16-bit `size` is the CodeView record size field: it excludes +the size field itself and includes the kind and payload. + +### 8.1 Normal type representation + +When the packed flags are clear, write three separate arrays: + +```text +offsets_file_offset -> U32 offsets[leaf_count] +sizes_file_offset -> U16 sizes[leaf_count] +kinds_file_offset -> U16 kinds[leaf_count] +``` + +### 8.2 Packed type representation, version 2 + +The tested profile uses the packed representation for every type section if every section has at +most 256 distinct leaf kinds. If any type section exceeds that limit, clear both packed flags and +use the normal representation for all type sections in that object. + +Packed offsets use groups of 512 leaves: + +```text +group_size = 512 +group_count = ceil(leaf_count / 512) +``` + +At `offsets_file_offset`, write `group_count` pairs of little-endian `U32` values: + +```text +struct Group { + U32 absolute_base; + U32 payload_offset_and_width; +}; +``` + +The group directory is followed by padding to an eight-byte boundary relative to +`offsets_file_offset`, then the delta payload: + +```text +group_bytes = group_count * 8 +payload_start = offsets_file_offset + align_up(group_bytes, 8) +``` + +For each group: + +- `absolute_base` is the first leaf offset in that group. +- Select a two-byte delta if the last offset minus the base is at most `0xffff`; otherwise select a + three-byte delta. Offsets are monotonically increasing, so checking the last delta is sufficient. +- `payload_offset_and_width & ~1` is the byte offset from `payload_start` to this group's deltas. +- Bit zero is 0 for two-byte deltas and 1 for three-byte deltas. +- Write one little-endian two- or three-byte delta per leaf, including a zero delta for the first + leaf. +- A three-byte delta must not exceed `0xffffff`. + +The second and third packed arrays repurpose the legacy field names: + +```text +sizes_file_offset -> U16 kind_dictionary[256] // exactly 512 bytes +kinds_file_offset -> U8 kind_code[leaf_count] +``` + +Assign dictionary codes in first-seen leaf order. Zero-fill unused dictionary entries. A leaf's +kind is `kind_dictionary[kind_code[i]]`. Leaf sizes are derived from adjacent offsets; the final +leaf ends at `raw_section_size`. + +## 9. Complete-UDT hash sidecar + +When `UDT_HASH_INDEX` is set, write exactly `type_count` consecutive 16-byte directory records +immediately after all `LNK_CObjTypeIndex` records: + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U64` | `hashes_file_offset` | Absolute offset of `U64` hashes, or zero when count is zero | +| 8 | `U32` | `hash_count` | Number of hashes | +| 12 | `U32` | `reserved` | Zero | + +Each record corresponds by index to the type-index record at the same index. Hashes are appended in +leaf order, but only for complete UDT definitions with nonempty unique names. + +The reference algorithm operates on the leaf payload after the two-byte kind field: + +| Leaf kind | Minimum payload bytes | Properties field | Cursor after fixed header | +|---:|---:|---|---:| +| `0x1504`, `0x1505`, `0x1519` | 16 | `U16` at `payload + 2` | 16 | +| `0x1506` | 8 | `U16` at `payload + 2` | 8 | +| `0x1507` | 12 | `U16` at `payload + 2` | 12 | +| `0x1608`, `0x1609` | 20 | `U32` at `payload + 0` | 20 | + +Ignore other kinds. Require property bit `0x0200` and reject property bit `0x0080`. For every kind +except `0x1507`, skip one CodeView numeric value at the cursor. Then skip the first NUL-terminated +name and hash the following nonempty NUL-terminated unique name. + +CodeView numeric length is: + +- two bytes when the first `U16` is below `0x8000`; +- otherwise two bytes for the numeric tag plus payload sizes: `0x8000:1`, + `0x8001/0x8002/0x801c:2`, `0x8003/0x8004/0x8005:4`, + `0x8006/0x8009/0x800a/0x800c:8`, `0x8007:10`, `0x8008:16`, `0x800b:6`, + `0x800d:16`, `0x800e:20`, `0x800f:32`, and `0x8017/0x8018:16`. + +Hash the unique-name bytes with wrapping 64-bit arithmetic: + +```text +h = 5381 +for byte in unique_name: + h = ((h << 5) + h) XOR byte +h = h OR 1 +``` + +The `OR 1` is mandatory because zero is the linker's empty-set sentinel. + +## 10. AMD64 base-relocation sidecar + +This optional sidecar is supported for AMD64 COFF machine `0x8664`. Omit the flag on unsupported +machines. + +The 16-byte directory is: + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U64` | `entries_file_offset` | Absolute entry-array offset, or zero when count is zero | +| 8 | `U32` | `entry_count` | Number of candidate relocations | +| 12 | `U32` | `reserved` | Zero | + +Each 16-byte entry is: + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U32` | `sect_idx` | Zero-based COFF section index | +| 4 | `U32` | `apply_off` | Relocation's section-relative application offset | +| 8 | `U32` | `isymbol` | COFF symbol-table index | +| 12 | `U8` | `addr_size` | 8 for `IMAGE_REL_AMD64_ADDR64`, 4 for `IMAGE_REL_AMD64_ADDR32` | +| 13 | `U8[3]` | `reserved` | Zero | + +Scan sections in COFF order and relocations in table order. Retain only AMD64 relocation type 1 +(`ADDR64`) and type 2 (`ADDR32`). RAD Link still applies symbol interpretation and section +liveness checks after reading this candidate list. + +Support `IMAGE_SCN_LNK_NRELOC_OVFL` (`0x01000000`) in the usual COFF way. When the section header's +relocation count is `0xffff`, the first relocation is the overflow counter; require its +`VirtualAddress`/`apply_off` to be nonzero, use `apply_off - 1` real entries, and skip the counter. + +## 11. `.debug$S` sidecar + +The 16-byte directory is: + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U64` | `entries_file_offset` | Absolute entry-array offset, or zero when count is zero | +| 8 | `U32` | `entry_count` | Number of retained C13 subsections | +| 12 | `U32` | `reserved` | Zero | + +Each 16-byte entry is: + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U32` | `raw_section_offset` | Original OBJ offset of the `.debug$S` section, including its signature | +| 4 | `U32` | `raw_payload_offset` | Original OBJ offset of the subsection payload | +| 8 | `U32` | `raw_payload_size` | Payload length, clamped to bytes remaining in the section | +| 12 | `U32` | `kind` | C13 subsection kind | + +Scan `.debug$S` sections in COFF section order. Skip the four-byte CodeView signature and parse +subsections as: + +```text +cursor = 0 +while cursor + 8 <= bytes_after_signature: + kind = read_u32(cursor + 0) + payload_size = read_u32(cursor + 4) + payload_rel = cursor + 8 + clamped_size = min(payload_size, bytes_after_signature - payload_rel) + + if (kind & 0x80000000) == 0: + append entry for this payload + + cursor = align_up(payload_rel + payload_size, 4) + stop on arithmetic overflow or failure to advance +``` + +Unknown kinds with bit 31 clear are retained. Kinds with bit 31 set are ignored. Entries must be +grouped by and nondecreasing in `raw_section_offset`, which naturally follows from scanning COFF +sections in order. + +### 11.1 `.debug$S` summary array + +When `DEBUG_S_SUMMARY` is set, one 16-byte summary exists for every debug-S entry. It begins at: + +```text +summary_offset = align_up(entries_file_offset + entry_count * 16, 8) +``` + +There is no summary offset in the directory; the reader derives it using this formula. + +| Offset | Type | Field | Meaning | +|---:|---:|---|---| +| 0 | `U32` | `module_symbol_size` | Four-byte-aligned size of symbols retained in the PDB module stream | +| 4 | `U32` | `gsi_candidate_count` | Number of candidate GSI records | +| 8 | `U32` | `proc_ref_count` | Number of procedure-reference candidates | +| 12 | `U32` | `flags` | Bit 0 means the subsection contains local data | + +Write an all-zero summary for subsection kinds other than `0xF1` (symbols). For an `0xF1` payload, +walk byte-packed symbol records. Each record starts with `U16 record_size, U16 kind`; its input +stride is `record_size + 2`, while its contribution to `module_symbol_size` is that stride aligned +up to four bytes. Stop at a record smaller than four bytes or one that overruns the payload. + +Use these kind sets: + +```text +global = {1107,0102,0202,1008,110d,020e,100f,1113} +typedef = {0004,1003,1108} +scope = {1110,110f,1103,1102,114d,115d,1104,1132,1147,1146} +end = {0006,114f,114e} +procedure-reference = {1110,110f,1147,1146} +has-locals = {113e,1111,110c,110d,1112,1113,1153,1107} +``` + +For `gsi_candidate_count`, begin each symbol subsection with candidate scope depth zero and active +state true. Count a global kind, or a typedef at candidate depth zero, while active. Scope kinds +increment candidate depth. An end kind decrements a positive depth; an end at depth zero makes the +candidate state inactive for the rest of that subsection. + +Maintain a separate module scope depth across all symbol subsections in the OBJ. A symbol belongs +to the module stream unless it is kind `0x0007`, a global kind, a typedef at module depth zero, or +kind `0x1176`. For a retained symbol, update module depth for scope/end kinds and add +`align_up(record_size + 2, 4)` to `module_symbol_size`. + +Set summary flag bit `0x1` if any symbol kind is in the `has-locals` set. + +## 12. Canonical layout algorithm + +The following order matches the reference writer: + +```text +cursor = align_up(64, 8) + +header.directory_offset = cursor +cursor = align_up(cursor + segment_count * 24, 8) + +if any sidecar directory is present: + header.reserved = cursor + cursor += type_count * 40 if TYPE_INDEX + cursor += type_count * 16 if UDT_HASH_INDEX + cursor += 16 if BASE_RELOC_INDEX + cursor += 16 if DEBUG_S_INDEX + cursor = align_up(cursor, 8) +else: + header.reserved = 0 + +for each type index: + assign offsets block; align cursor to 8 + assign sizes/dictionary block; align cursor to 8 + assign kinds/codes block; align cursor to 8 + assign UDT hashes block if nonempty; align cursor to 8 + +assign base-relocation entries if nonempty; align cursor to 8 +assign debug-S entries if nonempty; align cursor to 8 +assign debug-S summaries; align cursor to 8 + +header.data_offset = cursor +write segment payloads using the rules in section 6 +rewrite the completed segment directory +truncate the file to the final payload cursor +``` + +Use checked 64-bit addition and multiplication at every step. Reject any value that cannot be +represented by its destination field. + +## 13. Reader acceptance rules + +Before publishing a container, validate at least the same conditions RAD Link applies: + +### Header and directory + +- File size is at least 64 bytes. +- Magic, version, and header size are exact. +- `raw_size` is nonzero. +- `segment_size` is a power of two and at least 64 KiB. +- `segment_count == ceil(raw_size / segment_size)` without arithmetic overflow. +- The directory range is in the file. +- `PORTABLE_RAW_MAP` is set, no unknown flags are set, and all flag dependencies hold. +- Every segment has the expected raw size, a nonzero stored size, known flags, and an in-file + payload range. +- Every raw segment has `stored_size == raw_size`. +- Every raw run satisfies the 64 KiB mapping rules in section 6.2. + +### Type sidecars + +- The sidecar directory and every payload array are in the file. +- Every indexed raw section range is inside the original OBJ. +- Packed group directories and payload ranges are in bounds. +- Every packed descriptor selects width 2 or 3 through bit zero, points inside the delta payload, + and has enough bytes for its group. +- Offsets are monotonic, inside the indexed raw section, and reproduce the actual leaf boundaries. +- Packed kind codes resolve to the intended dictionary entry. + +### Other sidecars + +- Every UDT hash, base-relocation entry, debug-S entry, and summary array is in the file. +- Debug-S payload ranges are inside the original OBJ. +- Debug-S entries are nondecreasing by `raw_section_offset`. +- Sidecar contents are regenerated from the exact raw OBJ bytes and tested against a link without + sidecars. + +Finally, decode every compressed segment with the same Oodle decode options as RAD Link and compare +it byte-for-byte with the corresponding original segment. + +## 14. Publication and cache integrity + +Write to a temporary path in the destination directory. Flush and close it, complete all validation, +then atomically replace the destination. A failed conversion must never leave a file beginning with +`RLOBJ001` at the final path. + +Version 1 has no container checksum and does not enable Oodle quantum CRCs. The artifact cache must +continue to verify its normal whole-file content hash on upload and download. That hash covers the +header, sidecars, compressed payloads, raw payloads, and alignment padding. + +The artifact key must distinguish at least: + +- container magic and version; +- segment size; +- codec and compression level; +- compression options, including space/speed tradeoff; and +- sidecar schema/implementation version. + +Even when two settings decode to the same original OBJ, they are different stored artifacts. + +## 15. Bring-up and correctness test plan + +Use this order for a new writer: + +1. Emit a segment-only container with all segments compressed, avoiding raw mapping initially. +2. Decode every segment in a standalone validator and reconstruct the original OBJ byte-for-byte. +3. Link a small target from raw OBJs and compressed OBJs; compare image and PDB hashes. +4. Enable raw fallback and test files whose first, middle, and final segments are raw, including a + short final raw segment. +5. Link a response file mixing raw OBJs, compressed direct OBJ inputs, and ordinary `.lib` files. +6. Add sidecars one at a time, comparing image and PDB hashes after each addition. +7. Convert a complete large corpus with one uniform 512 KiB segment size and run warm and cold-cache + performance tests. +8. Corrupt each metadata field in validator tests and confirm rejection rather than an out-of-bounds + read or a partially accepted container. + +Compressed direct OBJ inputs can coexist with raw direct OBJ inputs and ordinary `.lib` files. +Compressed members inside a `.lib` archive are not supported by the current reader. + +The repository's reference smoke path is: + +```bat +scripts\build_cobj_test.bat +scripts\run_cobj_smoke.bat +``` + +For corpus conversion and deterministic image/PDB comparisons, see +`docs/compressed_obj_workflow.md` and the scripts under `scripts/`. diff --git a/docs/compressed_obj_workflow.md b/docs/compressed_obj_workflow.md new file mode 100644 index 000000000..6afccb046 --- /dev/null +++ b/docs/compressed_obj_workflow.md @@ -0,0 +1,200 @@ +# Portable compressed OBJ files + +## Overview + +RAD Link can optionally consume independently compressed, self-contained OBJ files. The format is +intended for targets whose raw OBJ corpus is large enough that disk traffic and file-backed pages +dominate link time and peak working set. + +Each source OBJ remains one ordinary portable file. The format does not use sparse allocation, +filesystem compression, bundles, shared manifests, or dependencies between OBJ files. Raw OBJ +files, compressed OBJ files, and ordinary `.lib` inputs can coexist in one link. + +Oodle support is opt-in. This repository does not contain or distribute Oodle headers, libraries, +or binaries. An Oodle-enabled build consumes a separately supplied SDK through `OODLE_SDK_DIR`. +The ordinary RAD Link release build has no Oodle compile-time or link-time dependency. + +## Measured results + +These are process-start-to-process-exit measurements from the final reviewed reader. File sizes are +ordinary file lengths on disk; neither corpus uses filesystem compression or sparse allocation. +Peak commit and peak working set are process peaks sampled at 50 ms intervals. + +| Target and input | Warm wall | Peak commit | Peak working set | OBJ corpus | +|---|---:|---:|---:|---:| +| FortniteClient, raw mmap | 15.36 s | 18.12 GiB | 95.50 GiB | 81.01 GiB | +| FortniteClient, compressed | 13.27 s | 15.06 GiB | 35.90 GiB | 22.97 GiB | +| UEFN DLL, raw mmap | 23.55 s | 28.45 GiB | 111.58 GiB | 89.22 GiB | +| UEFN DLL, compressed | 21.92 s | 22.27 GiB | 47.86 GiB | 24.39 GiB | + +For the cold-cache comparison, the OS file cache was purged separately before each link: + +| Target | Raw mmap | Compressed | +|---|---:|---:| +| FortniteClient | 28.51 s | 14.87 s | +| UEFN DLL | 30.56 s | 21.26 s | + +The compressed format reduced the tested corpora to roughly 28% of their original size and cut +peak working set by 57-62%. Warm results vary by roughly 0.5-1.0 seconds because PDB output is +asynchronous; a fully cached raw mmap link can occasionally be faster. The cold-cache and memory +improvements were consistent. + +Correctness was checked against raw-OBJ control links using the same output paths: + +- FortniteClient EXE SHA-256 (raw and compressed): + `1E126B7C8FAA2B730C380D9CF61131CE2E9175FAEC37A833D474952C5168D8F3` +- FortniteClient PDB SHA-256 (raw and compressed): + `39E040E711381CE48CDDE5238E7909F2083DE5DF785BF9FAAA819061AE6F56D7` +- UEFN DLL SHA-256 (raw and compressed): + `71EFFECB950BBF6E6DC6CBFFD20BA434E69388F024CAC21E083F0EA48DC7D1B5` + +## Build configurations + +Build the ordinary release without Oodle: + +```bat +build.bat radlink release +``` + +This succeeds without an Oodle SDK even if `OODLE_SDK_DIR` is unset. If a compressed OBJ is passed +to this binary it exits with a clear unsupported-input error. + +To opt into compressed OBJ support, point at an external SDK and pass the explicit build option: + +```bat +set OODLE_SDK_DIR=C:\path\to\OodleDataCompression\Sdk +build.bat radlink release oodle +``` + +The standalone reference writer also requires the external SDK: + +```bat +build.bat rad_obj_compress release +``` + +From an x64 Visual Studio developer command prompt, `scripts\build_cobj_test.bat` builds both +Oodle-enabled tools and copies them to `out_cobj`. `scripts\run_cobj_smoke.bat` compresses a small +OBJ, links both versions, and byte-compares the resulting executables. + +## Selected format + +The measured writer settings are: + +| Setting | Value | +|---|---:| +| Codec | Kraken | +| Compression level | Normal | +| Independent segment size | 512 KiB | +| Space/speed tradeoff | 256 | +| Incompressible segments | Stored raw | + +Each file contains: + +1. a fixed header describing the original COFF size and segment geometry; +2. a segment directory with stored offsets, stored sizes, raw sizes, and raw/compressed flags; +3. compact indexes for type leaves, complete UDT hashes, base relocations, and `.debug$S` data; +4. independently decodable segment payloads. + +The declarations are in `src\linker\lnk_compressed_obj_format.h`. The complete writer-facing byte +layout and sidecar contract are documented in `docs\compressed_obj_format.md`. All integers are +little-endian, and all file offsets and sidecar ranges are validated before the logical OBJ view is +published. + +The 512 KiB segment size was the measured knee for both targets. Smaller segments reduced sparse +decode amplification but added too many decoder calls; larger segments saved little disk space and +decoded too many unused bytes. + +## Reader design + +The reader preserves RAD Link's pointer-based COFF parsers while bounding private commit: + +1. Input classification checks one magic value. Ordinary OBJs stay on the existing mmap path. +2. `VirtualAlloc2` reserves a logical range matching the original OBJ using placeholders. +3. Raw-stored segments map directly from the portable file at their logical offsets. +4. A read fault on a compressed segment decodes into a bounded pagefile-backed slot and maps that + slot into the placeholder. A pinned write-copy path preserves rare in-place COFF patches. +5. Type and `.debug$S` sidecars let dominant debug passes stream compressed ranges directly into + their final destinations instead of repeatedly faulting the generic cache. +6. At the PDB boundary, the first cache generation is frozen and later misses use a smaller second + generation. This keeps published pointers stable and eliminates measured generic redecodes. + +The best measured UEFN configuration used a 17 GiB first generation and a 6 GiB second generation. +The equivalent explicit RAD Link options are: + +```bat +/RAD_COBJ_CACHE_GIB:17 +/RAD_COBJ_CACHE_SHRINK_GIB:6 +/RAD_COBJ_CACHE_FREEZE +/RAD_COBJ_TRIM_WS +/RAD_COBJ_ONE_SHOT +``` + +FortniteClient needed 27,727 first-generation segments (13.54 GiB) and ran with a 14+1 GiB +configuration. Cache capacity is not part of the file format. When these options are absent, RAD +Link derives a bounded capacity from physical memory and available commit, then caps it to the +logical size of the compressed inputs. Environment variables remain available as development +overrides; precedence is command line, environment, then the adaptive policy. + +`/RAD_COBJ_ONE_SHOT` is only appropriate for `radlink.exe` because the process is about to exit. +Leave it unset in embedded or long-lived hosts. `/RAD_LOG:TIMERS` prints the final cache, decode, +eviction, and timing statistics when diagnostics are needed; ordinary links remain silent. + +## Create and link a corpus + +Convert the direct OBJ entries in a response file while leaving switches and `.lib` paths intact: + +```powershell +& .\scripts\cobj_compress_rsp.ps1 ` + -ResponseFile C:\path\target.rsp ` + -OutputDirectory D:\scratch\compressed-objs ` + -Compressor .\out_cobj\rad_obj_compress.exe ` + -Workers 16 ` + -SegmentKiB 512 ` + -Codec kraken ` + -SpaceSpeedTradeoff 256 ` + -CompressionLevel normal +``` + +The output directory receives one independently portable file per OBJ and a rewritten +`compressed.rsp`. Use a fresh directory when changing the format or writer settings. + +Measure a link with process wall time and peak memory: + +```powershell +& .\scripts\bench_cobj.ps1 ` + -Rsp D:\scratch\compressed-objs\compressed.rsp ` + -Output D:\scratch\link\target.exe ` + -WorkingDirectory C:\path\to\link\working-directory ` + -CacheGiB 17 ` + -CacheShrinkGiB 6 ` + -Tag compressed_warm +``` + +`scripts\stage_raw_rsp.ps1`, `scripts\warm_rsp.ps1`, and `scripts\retarget_rsp.ps1` support fair +raw/compressed comparisons. `scripts\test_link_determinism.ps1` checks both image and PDB hashes +across repeated links. + +## Writer and integrity invariants + +A production writer must preserve these rules: + +1. Each output is self-contained and has ordinary non-sparse file length. +2. Segment size is a power of two and every segment is independently decodable. +3. Directory and sidecar offsets are aligned, ordered where required, and bounds-checked. +4. Type offsets are relative to the first byte after the CodeView signature. +5. The `.debug$S` index follows the reader's accepted-subsection rules exactly. +6. Incompressible segments may be stored raw without changing logical segment geometry. +7. Write a temporary file, validate every compressed round trip, and atomically rename it. + +The reference writer decodes and byte-compares every compressed segment before publishing. The +container does not currently enable Oodle quantum CRCs, and raw-stored segments bypass the decoder. +Production caches should retain their normal whole-file content hash verification when uploading +and downloading these objects; that covers metadata, compressed payloads, and raw payloads without +adding work to the linker's hot path. + +## Current limitations + +- The lazy placeholder reader is Windows-only. +- Oodle must be supplied and licensed separately; it is not part of this repository. +- Direct compressed OBJ inputs are supported. Compressed members inside `.lib` archives are not. +- The cache is process-global and tuned for the one-link command-line executable. From c8c334c083f1cf4388465236b0e24cc0189beeb9 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 17 Aug 2026 13:03:35 -0700 Subject: [PATCH 114/121] radlink: use portable atomics for cobj census --- src/linker/lnk_obj.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index 220e325fe..d55e003f1 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -476,18 +476,18 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) if (input->compressed_obj != 0 && g_lnk_compressed_obj_census_enabled) { U64 known_debug = census_debug_s_bytes + census_debug_t_bytes + census_debug_p_bytes + census_debug_h_bytes; - InterlockedIncrement64((volatile LONG64 *)&g_lnk_compressed_obj_census.obj_count); - InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.raw_bytes, input->data.size); - InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.section_bytes, census_section_bytes); - InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.debug_s_bytes, census_debug_s_bytes); - InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.debug_t_bytes, census_debug_t_bytes); - InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.debug_p_bytes, census_debug_p_bytes); - InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.debug_h_bytes, census_debug_h_bytes); - InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.other_section_bytes, census_section_bytes - known_debug); - InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.symbol_bytes, raw_coff_symbol_table.size); - InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.string_bytes, raw_coff_string_table.size); - InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.section_table_bytes, raw_coff_section_table.size); - InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_compressed_obj_census.reloc_bytes, census_reloc_bytes); + ins_atomic_u64_inc_eval(&g_lnk_compressed_obj_census.obj_count); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.raw_bytes, input->data.size); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.section_bytes, census_section_bytes); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.debug_s_bytes, census_debug_s_bytes); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.debug_t_bytes, census_debug_t_bytes); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.debug_p_bytes, census_debug_p_bytes); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.debug_h_bytes, census_debug_h_bytes); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.other_section_bytes, census_section_bytes - known_debug); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.symbol_bytes, raw_coff_symbol_table.size); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.string_bytes, raw_coff_string_table.size); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.section_table_bytes, raw_coff_section_table.size); + ins_atomic_u64_add_eval(&g_lnk_compressed_obj_census.reloc_bytes, census_reloc_bytes); } } From 4f006441cd057e43a808834863790f8c68e8ac57 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Mon, 17 Aug 2026 15:55:57 -0700 Subject: [PATCH 115/121] build: package matching radlink PDB --- scripts/build_cobj_test.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_cobj_test.bat b/scripts/build_cobj_test.bat index 5d2da90eb..33275495b 100644 --- a/scripts/build_cobj_test.bat +++ b/scripts/build_cobj_test.bat @@ -15,4 +15,5 @@ where cl >nul 2>nul || ( call "%~dp0..\build.bat" radlink rad_obj_compress release oodle || exit /b 1 if not exist "%~dp0..\out_cobj" mkdir "%~dp0..\out_cobj" copy /y "%~dp0..\build\radlink.exe" "%~dp0..\out_cobj\radlink.exe" >nul || exit /b 1 +copy /y "%~dp0..\build\radlink.pdb" "%~dp0..\out_cobj\radlink.pdb" >nul || exit /b 1 copy /y "%~dp0..\build\rad_obj_compress.exe" "%~dp0..\out_cobj\rad_obj_compress.exe" >nul || exit /b 1 From 34ef6652b4241b50f12cdf949aea4092342e5a11 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 19 Aug 2026 09:43:38 -0700 Subject: [PATCH 116/121] build: make the Oodle path compatible with Clang The Oodle build path always emitted MSVC /I syntax, so Clang treated the include option as an input filename. Select /I for MSVC and -I for Clang.\n\nAlso declare the portable-segment isolation helper before its first use. MSVC accepted the implicit declaration, while C99 Clang correctly rejected it. Together these fixes enable the documented Oodle-enabled PGO build without changing either compiler's default configuration. --- build.bat | 6 ++++-- src/linker/lnk_compressed_obj.c | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/build.bat b/build.bat index 0e0cf8c95..2aa2245dd 100644 --- a/build.bat +++ b/build.bat @@ -170,13 +170,15 @@ if "%oodle%"=="1" if "%OODLE_SDK_DIR%"=="" ( echo OODLE_SDK_DIR must name an Oodle SDK when building radlink with oodle exit /b 1 ) -if "%oodle%"=="1" set radlink_oodle_flags=/I"%OODLE_SDK_DIR%\include" -DLNK_OODLE=1 +if "%oodle%"=="1" if "%msvc%"=="1" set radlink_oodle_flags=/I"%OODLE_SDK_DIR%\include" -DLNK_OODLE=1 +if "%oodle%"=="1" if "%clang%"=="1" set radlink_oodle_flags=-I"%OODLE_SDK_DIR%\include" -DLNK_OODLE=1 if "%oodle%"=="1" set radlink_oodle_lib="%OODLE_SDK_DIR%\lib\Win64\oo2core_win64.lib" if "%rad_obj_compress%"=="1" if "%OODLE_SDK_DIR%"=="" ( echo OODLE_SDK_DIR must name an Oodle SDK when building rad_obj_compress exit /b 1 ) -if "%rad_obj_compress%"=="1" set rad_obj_compress_oodle_flags=/I"%OODLE_SDK_DIR%\include" +if "%rad_obj_compress%"=="1" if "%msvc%"=="1" set rad_obj_compress_oodle_flags=/I"%OODLE_SDK_DIR%\include" +if "%rad_obj_compress%"=="1" if "%clang%"=="1" set rad_obj_compress_oodle_flags=-I"%OODLE_SDK_DIR%\include" if "%rad_obj_compress%"=="1" set rad_obj_compress_oodle_lib="%OODLE_SDK_DIR%\lib\Win64\oo2core_win64.lib" if "%radlink%"=="1" set didbuild=1 && %compile% %radlink_oodle_flags% ..\src\linker\lnk.c %radlink_oodle_lib% %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 if "%rad_obj_compress%"=="1" set didbuild=1 && %compile% %rad_obj_compress_oodle_flags% ..\src\linker\rad_obj_compress.c %rad_obj_compress_oodle_lib% %compile_link% %out%rad_obj_compress.exe || exit /b 1 diff --git a/src/linker/lnk_compressed_obj.c b/src/linker/lnk_compressed_obj.c index 1a997c745..b451b0b3d 100644 --- a/src/linker/lnk_compressed_obj.c +++ b/src/linker/lnk_compressed_obj.c @@ -991,6 +991,8 @@ lnk_cobj_evict_one_locked(U32 *slot_idx_out) return 1; } +internal B32 lnk_cobj_isolate_portable_segment(LNK_CompressedObj *region, U32 seg_idx); + internal B32 lnk_cobj_materialize(LNK_CompressedObj *region, U32 seg_idx) { From 2907df1dd1453696b6be7bc96bc71b8a0d80a232 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 20 Aug 2026 12:13:55 -0700 Subject: [PATCH 117/121] radlink: prevent compressed cache write-group starvation Count slots completed through the per-slot fallback when a grouped write mapping cannot be created. Otherwise the group can remain permanently unsealed and make its resident cache slots non-evictable. Add anomaly-only cache stall diagnostics after five seconds and terminate after thirty seconds rather than spinning forever in the exception handler. The report bypasses the linker logger to remain safe when materialization interrupted code holding the log mutex. --- src/linker/lnk_compressed_obj.c | 128 ++++++++++++++++++++++++++++---- 1 file changed, 112 insertions(+), 16 deletions(-) diff --git a/src/linker/lnk_compressed_obj.c b/src/linker/lnk_compressed_obj.c index b451b0b3d..8a4bc3f34 100644 --- a/src/linker/lnk_compressed_obj.c +++ b/src/linker/lnk_compressed_obj.c @@ -13,6 +13,8 @@ #define LNK_COBJ_REGION_CAP (1u << 20) #define LNK_COBJ_DEFAULT_WRITE_GROUP_SLOTS 64u #define LNK_COBJ_FILE_MAPPING_GRANULARITY KB(64) +#define LNK_COBJ_STALL_REPORT_US 5000000ull +#define LNK_COBJ_STALL_ABORT_US 30000000ull #define LNK_COBJ_KNOWN_HEADER_FLAGS (LNK_COBJ_FLAG_TYPE_INDEX | \ LNK_COBJ_FLAG_UDT_HASH_INDEX | \ LNK_COBJ_FLAG_BASE_RELOC_INDEX | \ @@ -76,7 +78,7 @@ typedef struct LNK_CObjWriteGroup { SRWLOCK lock; U8 *view; - U32 completed_count; + volatile LONG completed_count; U32 slot_count; volatile LONG sealed; } LNK_CObjWriteGroup; @@ -107,6 +109,8 @@ typedef struct LNK_CObjCache U64 decodes; U64 redecodes; U64 evictions; + U64 eviction_failures; + U32 last_eviction_error; U64 raw_segments; U64 decoded_bytes; U64 stored_bytes_read; @@ -126,6 +130,19 @@ typedef struct LNK_CObjCache B32 skip_cleanup; } LNK_CObjCache; +typedef struct LNK_CObjStallStats +{ + U64 free_slots; + U64 empty_state_slots; + U64 loading_slots; + U64 ready_slots; + U64 pinned_slots; + U64 evicting_slots; + U64 unsealed_groups; + U64 incomplete_group_slots; + U64 open_group_views; +} LNK_CObjStallStats; + typedef enum LNK_CObjPolicySource { LNK_CObjPolicySource_Adaptive, @@ -664,18 +681,23 @@ lnk_cobj_write_group_acquire(U32 slot_idx, U32 *group_idx_out) U32 group_idx = slot_idx / g_lnk_cobj_cache.write_group_slot_count; LNK_CObjWriteGroup *group = &g_lnk_cobj_cache.write_groups[group_idx]; AcquireSRWLockExclusive(&group->lock); - if (!group->sealed && !group->view) { - U64 group_off = (U64)group_idx * g_lnk_cobj_cache.write_group_slot_count * - g_lnk_cobj_cache.segment_size; - U64 group_size = (U64)group->slot_count * g_lnk_cobj_cache.segment_size; - group->view = MapViewOfFile(g_lnk_cobj_cache.slot_mapping, FILE_MAP_WRITE, - (DWORD)(group_off >> 32), (DWORD)group_off, group_size); - } U8 *result = 0; - if (!group->sealed && group->view) { - U32 in_group_idx = slot_idx % g_lnk_cobj_cache.write_group_slot_count; - result = group->view + (U64)in_group_idx * g_lnk_cobj_cache.segment_size; + if (!group->sealed) { + // Return the group index even when the large write view cannot be mapped. The caller can + // populate this slot through a temporary per-slot view, but must still count it complete; + // otherwise the group never seals and none of its slots can ever be recycled. *group_idx_out = group_idx; + if (!group->view) { + U64 group_off = (U64)group_idx * g_lnk_cobj_cache.write_group_slot_count * + g_lnk_cobj_cache.segment_size; + U64 group_size = (U64)group->slot_count * g_lnk_cobj_cache.segment_size; + group->view = MapViewOfFile(g_lnk_cobj_cache.slot_mapping, FILE_MAP_WRITE, + (DWORD)(group_off >> 32), (DWORD)group_off, group_size); + } + if (group->view) { + U32 in_group_idx = slot_idx % g_lnk_cobj_cache.write_group_slot_count; + result = group->view + (U64)in_group_idx * g_lnk_cobj_cache.segment_size; + } } ReleaseSRWLockExclusive(&group->lock); return result; @@ -686,9 +708,9 @@ lnk_cobj_write_group_complete(U32 group_idx) { LNK_CObjWriteGroup *group = &g_lnk_cobj_cache.write_groups[group_idx]; AcquireSRWLockExclusive(&group->lock); - group->completed_count += 1; - if (group->completed_count == group->slot_count) { - UnmapViewOfFile(group->view); + LONG completed_count = InterlockedIncrement(&group->completed_count); + if (completed_count == group->slot_count) { + if (group->view) { UnmapViewOfFile(group->view); } group->view = 0; InterlockedExchange(&group->sealed, 1); } @@ -941,6 +963,60 @@ lnk_cobj_region_from_address(U8 *address) return 0; } +// This runs only after an impossible multi-second cache-allocation stall. Keep it independent of +// the linker logger: materialization can run inside the access-violation handler, where taking the +// logger mutex could deadlock if the interrupted code already owns it. +internal void +lnk_cobj_report_stall_locked(LNK_CompressedObj *region, U32 seg_idx, U64 elapsed_us, B32 aborting) +{ + LNK_CObjStallStats stats = {0}; + for EachIndex(slot_idx, g_lnk_cobj_cache.active_slot_count) { + LNK_CObjRuntimeSegment *segment = g_lnk_cobj_cache.slot_segments[slot_idx]; + if (!segment) { + stats.free_slots += 1; + continue; + } + switch (segment->state) { + case LNK_CObjSegState_Empty: stats.empty_state_slots += 1; break; + case LNK_CObjSegState_Loading: stats.loading_slots += 1; break; + case LNK_CObjSegState_Ready: stats.ready_slots += 1; break; + case LNK_CObjSegState_Pinned: stats.pinned_slots += 1; break; + case LNK_CObjSegState_Evicting: stats.evicting_slots += 1; break; + } + } + for EachIndex(group_idx, g_lnk_cobj_cache.write_group_count) { + LNK_CObjWriteGroup *group = &g_lnk_cobj_cache.write_groups[group_idx]; + LONG sealed = InterlockedCompareExchange(&group->sealed, 0, 0); + if (!sealed) { + U32 completed_count = (U32)InterlockedCompareExchange(&group->completed_count, 0, 0); + stats.unsealed_groups += 1; + stats.incomplete_group_slots += group->slot_count - Min(completed_count, group->slot_count); + stats.open_group_views += InterlockedCompareExchangePointer((void *volatile *)&group->view, 0, 0) != 0; + } + } + + char message[1024]; + U64 message_size = raddbg_snprintf( + message, sizeof(message), + "[radlink cobj stall] %s elapsed=%.1fs target=%p/%u cache=%llu/%llu " + "slots{free=%llu empty-state=%llu loading=%llu ready=%llu pinned=%llu evicting=%llu} " + "groups{unsealed=%llu incomplete=%llu open=%llu} " + "unmap-failures=%llu last-error=%u\n", + aborting ? "aborting" : "waiting", + (F64)elapsed_us / 1000000.0, + region->base, seg_idx, + g_lnk_cobj_cache.resident_count, g_lnk_cobj_cache.active_slot_count, + stats.free_slots, stats.empty_state_slots, stats.loading_slots, stats.ready_slots, + stats.pinned_slots, stats.evicting_slots, + stats.unsealed_groups, stats.incomplete_group_slots, stats.open_group_views, + g_lnk_cobj_cache.eviction_failures, g_lnk_cobj_cache.last_eviction_error); + HANDLE stderr_handle = GetStdHandle(STD_ERROR_HANDLE); + if (stderr_handle && stderr_handle != INVALID_HANDLE_VALUE) { + DWORD written = 0; + WriteFile(stderr_handle, message, (DWORD)Min(message_size, sizeof(message) - 1), &written, 0); + } +} + internal B32 lnk_cobj_evict_one_locked(U32 *slot_idx_out) { @@ -977,6 +1053,8 @@ lnk_cobj_evict_one_locked(U32 *slot_idx_out) InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.eviction_us, now_time_us() - eviction_begin_us); if (!unmapped) { + g_lnk_cobj_cache.eviction_failures += 1; + g_lnk_cobj_cache.last_eviction_error = GetLastError(); InterlockedExchange(&victim->state, LNK_CObjSegState_Ready); return 0; } @@ -1006,11 +1084,26 @@ lnk_cobj_materialize(LNK_CompressedObj *region, U32 seg_idx) InterlockedExchange(&runtime->state, LNK_CObjSegState_Loading); U32 slot_idx = max_U32; + U64 stall_begin_us = now_time_us(); + B32 stall_reported = 0; AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); while (!lnk_cobj_evict_one_locked(&slot_idx)) { ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); SwitchToThread(); AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); + U64 elapsed_us = now_time_us() - stall_begin_us; + if (!stall_reported && elapsed_us >= LNK_COBJ_STALL_REPORT_US) { + lnk_cobj_report_stall_locked(region, seg_idx, elapsed_us, 0); + stall_reported = 1; + } + if (elapsed_us >= LNK_COBJ_STALL_ABORT_US) { + lnk_cobj_report_stall_locked(region, seg_idx, elapsed_us, 1); + ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); + InterlockedExchange(&runtime->state, LNK_CObjSegState_Empty); + InterlockedExchange(&g_lnk_cobj_cache.failed, 1); + ReleaseSRWLockExclusive(&runtime->lock); + return 0; + } } g_lnk_cobj_cache.slot_segments[slot_idx] = runtime; runtime->slot_idx = slot_idx; @@ -1025,13 +1118,13 @@ lnk_cobj_materialize(LNK_CompressedObj *region, U32 seg_idx) U64 slot_off = (U64)slot_idx * region->segment_size; U32 write_group_idx = max_U32; U8 *write_view = lnk_cobj_write_group_acquire(slot_idx, &write_group_idx); - B32 grouped_write_view = write_view != 0 && write_group_idx != max_U32; B32 temporary_write_view = write_view == 0; if (temporary_write_view) { write_view = (U8 *)MapViewOfFile(g_lnk_cobj_cache.slot_mapping, FILE_MAP_WRITE, (DWORD)(slot_off >> 32), (DWORD)slot_off, region->segment_size); } B32 ok = write_view != 0; + B32 slot_written = 0; U64 decode_begin_us = now_time_us(); if (ok) { U8 *stored = region->mapped_file.str + entry->file_offset; @@ -1047,6 +1140,7 @@ lnk_cobj_materialize(LNK_CompressedObj *region, U32 seg_idx) if (ok && entry->raw_size < region->segment_size) { MemoryZero(write_view + entry->raw_size, region->segment_size - entry->raw_size); } + slot_written = ok; if (temporary_write_view) { UnmapViewOfFile(write_view); } } InterlockedExchangeAdd64((volatile LONG64 *)&g_lnk_cobj_cache.decode_us, @@ -1059,7 +1153,9 @@ lnk_cobj_materialize(LNK_CompressedObj *region, U32 seg_idx) PAGE_READONLY, 0, 0); ok = mapped == target; } - if (grouped_write_view) { lnk_cobj_write_group_complete(write_group_idx); } + if (slot_written && write_group_idx != max_U32) { + lnk_cobj_write_group_complete(write_group_idx); + } if (!ok) { AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); if (g_lnk_cobj_cache.slot_segments[slot_idx] == runtime) { From 337a8d0a5bb48812cd836851031d160effbb127e Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 20 Aug 2026 14:34:10 -0700 Subject: [PATCH 118/121] radlink: size small compressed caches by segment capacity Each compressed OBJ consumes a whole cache slot. Summing raw byte counts before aligning severely undersized targets made from many small OBJs; size from segment_count times segment_size instead and retain a 64-slot adaptive concurrency floor. Remove synchronous stall reporting from the VEH allocation loop. The reporter wrote to a UBA-detoured stderr pipe while holding the cache lock, so a blocked WriteFile converted temporary cache pressure into a permanent all-thread hang. --- src/linker/lnk_compressed_obj.c | 105 +++++++------------------------- 1 file changed, 22 insertions(+), 83 deletions(-) diff --git a/src/linker/lnk_compressed_obj.c b/src/linker/lnk_compressed_obj.c index 8a4bc3f34..56802a8dd 100644 --- a/src/linker/lnk_compressed_obj.c +++ b/src/linker/lnk_compressed_obj.c @@ -13,7 +13,6 @@ #define LNK_COBJ_REGION_CAP (1u << 20) #define LNK_COBJ_DEFAULT_WRITE_GROUP_SLOTS 64u #define LNK_COBJ_FILE_MAPPING_GRANULARITY KB(64) -#define LNK_COBJ_STALL_REPORT_US 5000000ull #define LNK_COBJ_STALL_ABORT_US 30000000ull #define LNK_COBJ_KNOWN_HEADER_FLAGS (LNK_COBJ_FLAG_TYPE_INDEX | \ LNK_COBJ_FLAG_UDT_HASH_INDEX | \ @@ -130,19 +129,6 @@ typedef struct LNK_CObjCache B32 skip_cleanup; } LNK_CObjCache; -typedef struct LNK_CObjStallStats -{ - U64 free_slots; - U64 empty_state_slots; - U64 loading_slots; - U64 ready_slots; - U64 pinned_slots; - U64 evicting_slots; - U64 unsealed_groups; - U64 incomplete_group_slots; - U64 open_group_views; -} LNK_CObjStallStats; - typedef enum LNK_CObjPolicySource { LNK_CObjPolicySource_Adaptive, @@ -375,16 +361,21 @@ internal void lnk_compressed_obj_prepare_cache(String8 *mapped_files, U64 count) { if (g_lnk_cobj_cache_init_state != 0) { return; } - U64 raw_bytes = 0; + U64 segment_capacity = 0; for EachIndex(i, count) { String8 mapped_file = mapped_files[i]; if (mapped_file.size < sizeof(LNK_CObjHeader)) { continue; } LNK_CObjHeader *header = (LNK_CObjHeader *)mapped_file.str; if (header->magic != LNK_COBJ_MAGIC || header->version != LNK_COBJ_VERSION) { continue; } - raw_bytes = header->raw_size > max_U64 - raw_bytes ? max_U64 : raw_bytes + header->raw_size; + // Every OBJ consumes an integral number of cache slots. Summing raw byte counts before + // aligning underestimates capacity badly for targets containing many small OBJs: sixty-four + // 20 KiB OBJs need sixty-four slots, not three slots for their ~1.25 MiB byte sum. + U64 obj_capacity = (U64)header->segment_count * header->segment_size; + segment_capacity = obj_capacity > max_U64 - segment_capacity ? max_U64 : + segment_capacity + obj_capacity; } - if (raw_bytes > g_lnk_cobj_input_capacity_hint) { - g_lnk_cobj_input_capacity_hint = raw_bytes; + if (segment_capacity > g_lnk_cobj_input_capacity_hint) { + g_lnk_cobj_input_capacity_hint = segment_capacity; } } @@ -963,60 +954,6 @@ lnk_cobj_region_from_address(U8 *address) return 0; } -// This runs only after an impossible multi-second cache-allocation stall. Keep it independent of -// the linker logger: materialization can run inside the access-violation handler, where taking the -// logger mutex could deadlock if the interrupted code already owns it. -internal void -lnk_cobj_report_stall_locked(LNK_CompressedObj *region, U32 seg_idx, U64 elapsed_us, B32 aborting) -{ - LNK_CObjStallStats stats = {0}; - for EachIndex(slot_idx, g_lnk_cobj_cache.active_slot_count) { - LNK_CObjRuntimeSegment *segment = g_lnk_cobj_cache.slot_segments[slot_idx]; - if (!segment) { - stats.free_slots += 1; - continue; - } - switch (segment->state) { - case LNK_CObjSegState_Empty: stats.empty_state_slots += 1; break; - case LNK_CObjSegState_Loading: stats.loading_slots += 1; break; - case LNK_CObjSegState_Ready: stats.ready_slots += 1; break; - case LNK_CObjSegState_Pinned: stats.pinned_slots += 1; break; - case LNK_CObjSegState_Evicting: stats.evicting_slots += 1; break; - } - } - for EachIndex(group_idx, g_lnk_cobj_cache.write_group_count) { - LNK_CObjWriteGroup *group = &g_lnk_cobj_cache.write_groups[group_idx]; - LONG sealed = InterlockedCompareExchange(&group->sealed, 0, 0); - if (!sealed) { - U32 completed_count = (U32)InterlockedCompareExchange(&group->completed_count, 0, 0); - stats.unsealed_groups += 1; - stats.incomplete_group_slots += group->slot_count - Min(completed_count, group->slot_count); - stats.open_group_views += InterlockedCompareExchangePointer((void *volatile *)&group->view, 0, 0) != 0; - } - } - - char message[1024]; - U64 message_size = raddbg_snprintf( - message, sizeof(message), - "[radlink cobj stall] %s elapsed=%.1fs target=%p/%u cache=%llu/%llu " - "slots{free=%llu empty-state=%llu loading=%llu ready=%llu pinned=%llu evicting=%llu} " - "groups{unsealed=%llu incomplete=%llu open=%llu} " - "unmap-failures=%llu last-error=%u\n", - aborting ? "aborting" : "waiting", - (F64)elapsed_us / 1000000.0, - region->base, seg_idx, - g_lnk_cobj_cache.resident_count, g_lnk_cobj_cache.active_slot_count, - stats.free_slots, stats.empty_state_slots, stats.loading_slots, stats.ready_slots, - stats.pinned_slots, stats.evicting_slots, - stats.unsealed_groups, stats.incomplete_group_slots, stats.open_group_views, - g_lnk_cobj_cache.eviction_failures, g_lnk_cobj_cache.last_eviction_error); - HANDLE stderr_handle = GetStdHandle(STD_ERROR_HANDLE); - if (stderr_handle && stderr_handle != INVALID_HANDLE_VALUE) { - DWORD written = 0; - WriteFile(stderr_handle, message, (DWORD)Min(message_size, sizeof(message) - 1), &written, 0); - } -} - internal B32 lnk_cobj_evict_one_locked(U32 *slot_idx_out) { @@ -1085,19 +1022,13 @@ lnk_cobj_materialize(LNK_CompressedObj *region, U32 seg_idx) U32 slot_idx = max_U32; U64 stall_begin_us = now_time_us(); - B32 stall_reported = 0; AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); while (!lnk_cobj_evict_one_locked(&slot_idx)) { ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); SwitchToThread(); AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); U64 elapsed_us = now_time_us() - stall_begin_us; - if (!stall_reported && elapsed_us >= LNK_COBJ_STALL_REPORT_US) { - lnk_cobj_report_stall_locked(region, seg_idx, elapsed_us, 0); - stall_reported = 1; - } if (elapsed_us >= LNK_COBJ_STALL_ABORT_US) { - lnk_cobj_report_stall_locked(region, seg_idx, elapsed_us, 1); ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); InterlockedExchange(&runtime->state, LNK_CObjSegState_Empty); InterlockedExchange(&g_lnk_cobj_cache.failed, 1); @@ -1277,12 +1208,20 @@ lnk_cobj_cache_init(U32 segment_size) U64 cache_bytes = g_lnk_cobj_policy.initial_cache_bytes; U64 post_boundary_cache_bytes = g_lnk_cobj_policy.post_boundary_cache_bytes; U64 segment_size_u64 = segment_size; - if (g_lnk_cobj_input_capacity_hint > 0) { - if (g_lnk_cobj_input_capacity_hint < cache_bytes) { - cache_bytes = AlignPow2(g_lnk_cobj_input_capacity_hint, segment_size_u64); + U64 input_capacity_hint = g_lnk_cobj_input_capacity_hint; + if (g_lnk_cobj_policy.initial_source == LNK_CObjPolicySource_Adaptive) { + // Keep at least one first-generation slot per write-group lane. This is only 32 MiB for + // 512 KiB segments, avoids serializing a wave of parallel first faults on tiny targets, + // and does not override explicit cache-size experiments. + input_capacity_hint = Max(input_capacity_hint, + segment_size_u64 * LNK_COBJ_DEFAULT_WRITE_GROUP_SLOTS); + } + if (input_capacity_hint > 0) { + if (input_capacity_hint < cache_bytes) { + cache_bytes = AlignPow2(input_capacity_hint, segment_size_u64); } - if (g_lnk_cobj_input_capacity_hint < post_boundary_cache_bytes) { - post_boundary_cache_bytes = AlignPow2(g_lnk_cobj_input_capacity_hint, segment_size_u64); + if (input_capacity_hint < post_boundary_cache_bytes) { + post_boundary_cache_bytes = AlignPow2(input_capacity_hint, segment_size_u64); } } g_lnk_cobj_cache.skip_cleanup = lnk_cobj_skip_cleanup_enabled(); From 6650522ba6accd67526aaf4d502e8598df1e42d1 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 2 Sep 2026 15:10:46 -0700 Subject: [PATCH 119/121] lnk: avoid re-sorting unchanged compressed object regions --- src/linker/lnk_compressed_obj.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk_compressed_obj.c b/src/linker/lnk_compressed_obj.c index 56802a8dd..7178aa741 100644 --- a/src/linker/lnk_compressed_obj.c +++ b/src/linker/lnk_compressed_obj.c @@ -97,6 +97,7 @@ typedef struct LNK_CObjCache LNK_CObjRuntimeSegment **slot_segments; LNK_CompressedObj **regions; U64 region_count; + U64 sorted_region_count; U64 region_cap; PVOID (WINAPI *map_view_of_file_3)(HANDLE,HANDLE,PVOID,ULONG64,SIZE_T,ULONG,ULONG,MEM_EXTENDED_PARAMETER*,ULONG); BOOL (WINAPI *unmap_view_of_file_2)(HANDLE,PVOID,ULONG); @@ -1705,10 +1706,14 @@ lnk_compressed_obj_finalize_open(void) // Parallel input mapping appends regions in completion order. The caller invokes this joined // boundary before starting any COFF parser task, so no logical compressed range can fault until // the address table has been sorted for lock-free VEH lookup. - if (!g_lnk_cobj_cache.initialized || g_lnk_cobj_cache.region_count < 2) { return; } + // Archive, generated-object and empty input batches usually add no compressed + // regions. The table is append-only; do not sort the same addresses again. + if (!g_lnk_cobj_cache.initialized || g_lnk_cobj_cache.region_count < 2 || + g_lnk_cobj_cache.region_count == g_lnk_cobj_cache.sorted_region_count) { return; } AcquireSRWLockExclusive(&g_lnk_cobj_cache.lock); qsort(g_lnk_cobj_cache.regions, g_lnk_cobj_cache.region_count, sizeof(*g_lnk_cobj_cache.regions), lnk_cobj_region_ptr_compare); + g_lnk_cobj_cache.sorted_region_count = g_lnk_cobj_cache.region_count; ReleaseSRWLockExclusive(&g_lnk_cobj_cache.lock); } From 3084b79c8dc2934c2f16cc107937375138a1c5bc Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 2 Sep 2026 23:08:48 -0700 Subject: [PATCH 120/121] tests: verify compressed debug relocation and raw artifact parity --- src/torture/tests/torture_linker_tests.c | 90 ++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/src/torture/tests/torture_linker_tests.c b/src/torture/tests/torture_linker_tests.c index 4fd16c769..0ebe0f079 100644 --- a/src/torture/tests/torture_linker_tests.c +++ b/src/torture/tests/torture_linker_tests.c @@ -356,6 +356,96 @@ TEST(link_large_import_object) } } +TEST(compressed_debug_reloc_parity) +{ +#if OS_WINDOWS + // The optional Oodle compressor is built with `rad_obj_compress`; ordinary + // torture builds do not require its SDK. + String8 compressor = str8f(arena, "%S/build/rad_obj_compress.exe", t_cwd_path()); + if (!file_path_exists(compressor)) { TestSkip(); } + + // Two relocations per record: overflow COFF's 16-bit relocation count, + // cross compressed-segment boundaries, and finish with a partial segment. + U32 record_count = 32769; + U32 *record_offsets = push_array(arena, U32, record_count); + CV_DebugS debug_s = {0}; + String8List *symbols = cv_sub_section_ptr_from_debug_s(&debug_s, CV_C13SubSectionKind_Symbols); + for EachIndex(i, record_count) { + record_offsets[i] = sizeof(CV_Signature) + sizeof(CV_C13SubSectionHeader) + safe_cast_u32(symbols->total_size); + String8 raw = cv_make_symbol(arena, CV_SymKind_GDATA32, + cv_make_data32(arena, (CV_SymData32){.itype = 0x74, .off = (U32)i*4}, str8f(arena, "relocated_%05u", (U32)i))); + CV_Symbol symbol = cv_symbol_from_ptr(raw.str); + str8_list_push(arena, symbols, cv_data_from_symbol(arena, &symbol, CV_SymbolAlign)); + } + String8List debug_data = cv_data_from_debug_s_c13(arena, &debug_s, 1); + COFF_ObjWriter *writer = coff_obj_writer_alloc(0, COFF_MachineType_X64); + COFF_ObjSection *data = coff_obj_writer_push_section(writer, str8_lit(".data"), + COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemWrite, + str8(push_array(arena, U8, record_count*4 + 16), record_count*4 + 16)); + COFF_ObjSymbol *target = coff_obj_writer_push_symbol_static(writer, str8_lit("target"), 16, data); + coff_obj_writer_push_section(writer, str8_lit(".debug$S"), + COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemDiscardable, + str8_list_join(arena, &debug_data, 0)); + // Many small relocation tables share compressed segments. Exercise their + // decoding interleaved with section-data reads. + for EachIndex(i, 512) { + CV_DebugS small_debug_s = {0}; + String8 raw = cv_make_symbol(arena, CV_SymKind_GDATA32, + cv_make_data32(arena, (CV_SymData32){.itype = 0x74, .off = (U32)i*4}, str8f(arena, "small_%04u", (U32)i))); + CV_Symbol symbol = cv_symbol_from_ptr(raw.str); + str8_list_push(arena, cv_sub_section_ptr_from_debug_s(&small_debug_s, CV_C13SubSectionKind_Symbols), + cv_data_from_symbol(arena, &symbol, CV_SymbolAlign)); + String8List small_data = cv_data_from_debug_s_c13(arena, &small_debug_s, 1); + COFF_ObjSection *small_section = coff_obj_writer_push_section(writer, str8_lit(".debug$S"), + COFF_SectionFlag_CntInitializedData|COFF_SectionFlag_MemRead|COFF_SectionFlag_MemDiscardable, + str8_list_join(arena, &small_data, 0)); + U32 off = sizeof(CV_Signature) + sizeof(CV_C13SubSectionHeader) + sizeof(CV_SymbolHeader); + coff_obj_writer_section_push_reloc(writer, small_section, off + OffsetOf(CV_SymData32, sec), target, COFF_Reloc_X64_Section); + coff_obj_writer_section_push_reloc(writer, small_section, off + OffsetOf(CV_SymData32, off), target, COFF_Reloc_X64_SecRel); + } + String8 obj = coff_obj_writer_serialize(arena, writer); + // The fixture writer only emits 16-bit relocation counts. Append an explicit + // overflow table after its ordinary object data; symbol indices are now final. + U32 reloc_count = record_count*2; + COFF_Reloc *relocs = push_array(arena, COFF_Reloc, reloc_count + 1); + relocs[0].apply_off = reloc_count + 1; + U32 reloc_idx = 1; + // Deliberately reversed, so the copy must preserve records and their sort keys. + for (U32 i = record_count; i-- > 0;) { + U32 off = record_offsets[i] + sizeof(CV_SymbolHeader); + relocs[reloc_idx++] = (COFF_Reloc){off + OffsetOf(CV_SymData32, sec), target->idx, COFF_Reloc_X64_Section}; + relocs[reloc_idx++] = (COFF_Reloc){off + OffsetOf(CV_SymData32, off), target->idx, COFF_Reloc_X64_SecRel}; + } + coff_obj_writer_release(&writer); + COFF_FileHeaderInfo header = coff_file_header_info_from_data(obj); + COFF_SectionHeader *sections = (COFF_SectionHeader *)(obj.str + header.section_table_range.min); + sections[1].relocs_foff = safe_cast_u32(obj.size); + sections[1].reloc_count = max_U16; + sections[1].flags |= COFF_SectionFlag_LnkNRelocOvfl; + String8List obj_parts = {0}; + str8_list_push(arena, &obj_parts, obj); + str8_list_push(arena, &obj_parts, str8_array(relocs, reloc_count + 1)); + obj = str8_list_join(arena, &obj_parts, 0); + T_Ok(t_write_file(str8_lit("raw.obj"), obj)); + T_Ok(t_write_file(str8_lit("reloc_input.obj"), obj)); + T_Ok(t_write_entry_obj()); + char *args = "/subsystem:console /entry:entry /debug:full /rad_time_stamp:0 /rad_workers:1 /out:reloc.exe /pdbaltpath:reloc.pdb entry.obj reloc_input.obj"; + t_invoke_linkerf("%s", args); + T_Ok(g_last_exit_code == 0); + String8 expected_image = t_read_file(arena, str8_lit("reloc.exe")); + String8 expected_pdb = t_read_file(arena, str8_lit("reloc.pdb")); + T_Ok(t_invoke(compressor, str8_lit("raw.obj compressed.obj 64 kraken 256 fast"), TIMEOUT_SEC(30))); + T_Ok(g_last_exit_code == 0); + T_Ok(t_write_file(str8_lit("reloc_input.obj"), t_read_file(arena, str8_lit("compressed.obj")))); + t_invoke_linkerf("%s", args); + T_Ok(g_last_exit_code == 0); + T_Ok(str8_match(expected_image, t_read_file(arena, str8_lit("reloc.exe")), 0)); + T_Ok(str8_match(expected_pdb, t_read_file(arena, str8_lit("reloc.pdb")), 0)); +#else + TestSkip(); +#endif +} + TEST(pdbstripped_coff_artifact_parity) { String8 path = str8f(arena, "%S/linker/tests/pdbstripped.tst", t_src_path()); From 3b1cb0bd30791fe04d91c2a8d5f7ca9ac90a2752 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 2 Sep 2026 17:26:21 -0700 Subject: [PATCH 121/121] radlink: skip ICF source reads when summaries rule out locals --- src/linker/lnk.c | 32 ++++++++-- src/torture/tests/torture_linker_tests.c | 75 ++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 6 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index e9fbece82..ead725ee4 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -4243,6 +4243,16 @@ lnk_icf_src_key_from_fn(Arena *scratch, LNK_Obj *obj, U32 fn_sn, String8 chksms) return key; } +internal B32 +lnk_icf_debug_s_summary_has_locals(LNK_CObjDebugSView *indexed) +{ + for EachIndex(i, indexed->count) { + if (indexed->v[i].kind == CV_C13SubSectionKind_Symbols && + (indexed->summaries[i].flags & LNK_COBJ_DEBUG_S_SUMMARY_HAS_LOCALS)) { return 1; } + } + return 0; +} + // does the function's record tree have anything a watch window would show? internal B32 lnk_icf_debug_s_has_locals(Arena *scratch, LNK_Obj *obj, U32 child_sn) @@ -4252,11 +4262,7 @@ lnk_icf_debug_s_has_locals(Arena *scratch, LNK_Obj *obj, U32 child_sn) String8List syms = {0}; if (lnk_compressed_obj_debug_s_index(obj->compressed_obj, sect.frange, &indexed)) { if (indexed.summaries) { - for EachIndex(i, indexed.count) { - if (indexed.v[i].kind == CV_C13SubSectionKind_Symbols && - (indexed.summaries[i].flags & LNK_COBJ_DEBUG_S_SUMMARY_HAS_LOCALS)) { return 1; } - } - return 0; + return lnk_icf_debug_s_summary_has_locals(&indexed); } for EachIndex(i, indexed.count) { if (indexed.v[i].kind == CV_C13SubSectionKind_Symbols) { @@ -4332,7 +4338,21 @@ THREAD_POOL_TASK_FUNC(lnk_icf_mark_folded_lines_task) { Temp fold_temp = temp_begin(scratch.arena); U32 child_sn = lnk_icf_debug_s_child_from_section(obj, section_number); - if (child_sn != 0) { + B32 needs_src_key = child_sn != 0; + if (needs_src_key && obj->compressed_obj != 0) { + // Only a follower with locals can escalate to a full record tree. A + // negative summary proves that source-key reads cannot change the mark; + // avoid faulting compressed Lines/checksum payloads just to compare them. + // Without summaries, retain the old source-key-first order so raw symbol + // trees are not parsed unnecessarily for same-source folds. + LNK_ObjSection child = lnk_obj_section_from_section_number(obj, child_sn); + LNK_CObjDebugSView indexed = {0}; + if (lnk_compressed_obj_debug_s_index(obj->compressed_obj, child.frange, &indexed) && indexed.summaries && + !lnk_icf_debug_s_summary_has_locals(&indexed)) { + needs_src_key = 0; + } + } + if (needs_src_key) { String8 follower_chksms = lnk_icf_obj_file_chksms(obj); // per-obj memo -- leaders String8 leader_chksms = lnk_icf_obj_file_chksms(leader_obj); // shared across follower objs LNK_ICFSrcKey fk = lnk_icf_src_key_from_fn(fold_temp.arena, obj, section_number, follower_chksms); diff --git a/src/torture/tests/torture_linker_tests.c b/src/torture/tests/torture_linker_tests.c index 0ebe0f079..008cba9d8 100644 --- a/src/torture/tests/torture_linker_tests.c +++ b/src/torture/tests/torture_linker_tests.c @@ -446,6 +446,81 @@ TEST(compressed_debug_reloc_parity) #endif } +TEST(compressed_icf_debug_record_selection) +{ +#if OS_WINDOWS + String8 compressor = str8f(arena, "%S/build/rad_obj_compress.exe", t_cwd_path()); + if (!file_path_exists(compressor)) { TestSkip(); } + // Three folded pairs: no locals, locals at different source lines, and locals + // at the same source line. Only the second pair needs both full record trees. + String8 source = str8_lit( + "__declspec(noinline) int empty_a(void) { return 3; }\n" + "__declspec(noinline) int empty_b(void) { return 3; }\n" + "__declspec(noinline) int different_a(int x) { int a = x+1; return a; }\n" + "__declspec(noinline) int different_b(int x) { int b = x+1; return b; }\n" + "#line 200 \"shared.h\"\n" + "__declspec(noinline) int same_a(int x) { int a = x+2; return a; }\n" + "#line 200 \"shared.h\"\n" + "__declspec(noinline) int same_b(int x) { int b = x+2; return b; }\n" + "int (*volatile functions[])(int) = {different_a,different_b,same_a,same_b};\n" + "int (*volatile empty[])(void) = {empty_a,empty_b};\n" + "int entry(void) { return functions[0] != functions[1] || functions[2] != functions[3] || empty[0] != empty[1]; }\n"); + T_Ok(t_write_file(str8_lit("icf_source.c"), source)); + // Permit ICF despite the address comparisons used to verify the folds below. + T_Ok(t_invoke(t_clang_path(), str8_lit("--target=x86_64-pc-windows-msvc -c -O0 -g -gcodeview -ffunction-sections -fno-addrsig -o raw.obj icf_source.c"), TIMEOUT_SEC(30))); + T_Ok(g_last_exit_code == 0); + String8 raw_obj = t_read_file(arena, str8_lit("raw.obj")); + T_Ok(t_invoke(compressor, str8_lit("raw.obj compressed.obj 64 kraken 256 fast"), TIMEOUT_SEC(30))); + T_Ok(g_last_exit_code == 0); + String8 inputs[] = {raw_obj, t_read_file(arena, str8_lit("compressed.obj"))}; + String8 expected_image = {0}, expected_pdb = {0}; + for EachElement(input_idx, inputs) { + T_Ok(t_write_file(str8_lit("icf_input.obj"), inputs[input_idx])); + t_invoke_linkerf("/nodefaultlib /subsystem:console /entry:entry /debug:full /opt:ref,icf /rad_time_stamp:0 /rad_workers:1 /out:icf.exe /pdbaltpath:icf.pdb icf_input.obj"); + T_Ok(g_last_exit_code == 0); + T_Ok(t_invoke(t_make_file_path(arena, str8_lit("icf.exe")), str8_zero(), TIMEOUT_SEC(5))); + T_Ok(g_last_exit_code == 0); // prove all three function pairs actually folded + String8 image_data = t_read_file(arena, str8_lit("icf.exe")); + String8 pdb_data = t_read_file(arena, str8_lit("icf.pdb")); + if (input_idx == 0) { + expected_image = image_data; + expected_pdb = pdb_data; + } else { + T_Ok(str8_match(expected_image, image_data, 0)); + T_Ok(str8_match(expected_pdb, pdb_data, 0)); + } + + MSF_Parsed *msf = msf_parsed_from_data(arena, pdb_data); + T_Ok(msf != 0); + if (!msf) { continue; } + PDB_DbiParsed *dbi = pdb_dbi_from_data(arena, msf_data_from_stream(msf, PDB_FixedStream_Dbi)); + PDB_CompUnitArray *modules = pdb_comp_unit_array_from_data(arena, pdb_data_from_dbi_range(dbi, PDB_DbiRange_ModuleInfo)); + U32 empty_count = 0, different_count = 0, same_count = 0; + for EachIndex(module_idx, modules->count) { + String8 symbols = pdb_data_from_unit_range(msf, modules->units[module_idx], PDB_DbiCompUnitRange_Symbols); + for (U64 cursor = 0; cursor < symbols.size;) { + CV_Symbol symbol = {0}; U64 read_size = 0; String8 error = {0}; + B32 ok = t_codec_pdb_read_symbol(symbols, cursor, PDB_SYMBOL_ALIGN, &symbol, &read_size, &error); + T_Ok(ok); + if (!ok) { break; } + if (CV_IsProc32(symbol.kind)) { + String8 name = cv_name_from_symbol(symbol.kind, symbol.data); + if (str8_match(name, str8_lit("empty_a"), 0) || str8_match(name, str8_lit("empty_b"), 0)) { empty_count += 1; } + if (str8_match(name, str8_lit("different_a"), 0) || str8_match(name, str8_lit("different_b"), 0)) { different_count += 1; } + if (str8_match(name, str8_lit("same_a"), 0) || str8_match(name, str8_lit("same_b"), 0)) { same_count += 1; } + } + cursor += read_size; + } + } + T_Ok(empty_count == 1); + T_Ok(different_count == 2); + T_Ok(same_count == 1); + } +#else + TestSkip(); +#endif +} + TEST(pdbstripped_coff_artifact_parity) { String8 path = str8f(arena, "%S/linker/tests/pdbstripped.tst", t_src_path());