radlink: portable compressed OBJ support (restacked on #842)

.github/workflows/builds.yml

@@ -45,6 +45,7 @@ jobs:
       - name: build
         shell: bash
         run: |
+          [[ "${{ matrix.compiler }}" == "clang-22" ]] && export AR=llvm-ar-22
           CC="${{ matrix.compiler }}" ./build.sh "${{ matrix.target }}" "${{ matrix.mode }}"
 
   build-windows-2022:
@@ -103,6 +104,7 @@ jobs:
       - name: Run
         shell: bash
         run: |
+          [[ "${{ matrix.compiler }}" == "clang-22" ]] && export AR=llvm-ar-22
           CC="${{ matrix.compiler }}" ./run_tests.sh clang "${{ matrix.mode }}" -- \* $TORTURE_FLAGS 
       - name: Upload Artifacts
         if: failure()

build.bat

@@ -27,6 +27,7 @@ cd /D "%~dp0"
 :: - `ubsan`: enable undefined-behavior sanitizer
 :: - `telemetry`: enable RAD telemetry profiling support
 :: - `spall`: enable spall profiling support
+:: - `oodle`: enable Oodle-compressed OBJ support in radlink (requires OODLE_SDK_DIR)
 
 :: --- Unpack Arguments -------------------------------------------------------
 for %%a in (%*) do set "%%~a=1"
@@ -53,13 +54,15 @@ if "%pgo%"=="1" (
   where llvm-profdata /q || echo llvm-profdata is not in the PATH || exit /b 1 
   if "%clang%"=="1" (
     if "%pgo_run%" == "1" (
-      call llvm-profdata merge %LLVM_PROFILE_FILE% -output=%~dp0build\build.profdata || exit /b 1
+      call llvm-profdata merge %~dp0build\pgo_raw\*.profraw -output=%~dp0build\build.profdata || exit /b 1
       set auto_compile_flags=%auto_compile_flags% -fprofile-use=%~dp0build\build.profdata
       set pgo_run=0
     ) else (
       echo [pgo enabled]
       set auto_compile_flags=%auto_compile_flags% -fprofile-generate -mllvm -vp-counters-per-site=5
-      set LLVM_PROFILE_FILE=%~dp0build\build.profraw
+      if not exist %~dp0build\pgo_raw mkdir %~dp0build\pgo_raw
+      del /q %~dp0build\pgo_raw\*.profraw 2>nul
+      set LLVM_PROFILE_FILE=%~dp0build\pgo_raw\build.%%p.profraw
       set pgo_run=1
     )
   ) else (
@@ -98,6 +101,10 @@ if "%msvc%"=="1"    set rc=call rc
 if "%clang%"=="1"   set rc=call llvm-rc
 if "%msvc%"=="1"    set link_dll=/link /DLL
 if "%clang%"=="1"   set link_dll=-Xlinker -DLL
+if "%msvc%"=="1"    set asm=call ml64 -nologo -c -Zi -Fo
+if "%clang%"=="1"   set asm=call llvm-ml -m64 /c /Fo
+if "%msvc%"=="1"    set mklib=call lib -nologo
+if "%clang%"=="1"   set mklib=call llvm-lib
 
 :: --- Choose Compile/Link Lines ----------------------------------------------
 if "%msvc%"=="1"      set compile_debug=%cl_debug%
@@ -138,12 +145,43 @@ if "%no_meta%"=="" if exist metagen.exe (
 )
 popd
 
+:: --- Assemble BLAKE3 --------------------------------------------------------
+if not exist build\blake3.lib if "%VSCMD_ARG_TGT_ARCH%" equ "x64" (
+  echo [assembling blake3]
+  pushd build
+  %asm% blake3_sse2_x86-64_windows_msvc.obj   ..\src\third_party\blake3\blake3_sse2_x86-64_windows_msvc.asm   || exit /b 1
+  %asm% blake3_sse41_x86-64_windows_msvc.obj  ..\src\third_party\blake3\blake3_sse41_x86-64_windows_msvc.asm  || exit /b 1
+  %asm% blake3_avx2_x86-64_windows_msvc.obj   ..\src\third_party\blake3\blake3_avx2_x86-64_windows_msvc.asm   || exit /b 1
+  %asm% blake3_avx512_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_avx512_x86-64_windows_msvc.asm || exit /b 1
+  %mklib% -out:blake3.lib blake3_*_msvc.obj                                                                   || exit /b 1
+  popd
+)
+
 :: --- Build Everything (@build_targets) --------------------------------------
 pushd build
 if "%raddbg%"=="1"                     set didbuild=1 && %compile% ..\src\raddbg\raddbg_main.c                               %compile_link% %link_icon% %out%raddbg.exe || exit /b 1
 if "%raddbg_non_graphical%"=="1"       set didbuild=1 && %compile% -DWM_STUB=1 -DR_BACKEND=R_BACKEND_STUB ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg_non_graphical.exe || exit /b 1
 if "%com_shim%"=="1"                   set didbuild=1 && %compile% ..\src\com_shim\com_shim_main.c                           %compile_link% %out%com_shim.exe || exit /b 1
-if "%radlink%"=="1"                    set didbuild=1 && %compile% ..\src\linker\lnk.c                                       %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1
+set radlink_oodle_flags=
+set radlink_oodle_lib=
+set rad_obj_compress_oodle_flags=
+set rad_obj_compress_oodle_lib=
+if "%oodle%"=="1" if "%OODLE_SDK_DIR%"=="" (
+  echo OODLE_SDK_DIR must name an Oodle SDK when building radlink with oodle
+  exit /b 1
+)
+if "%oodle%"=="1" if "%msvc%"=="1"  set radlink_oodle_flags=/I"%OODLE_SDK_DIR%\include" -DLNK_OODLE=1
+if "%oodle%"=="1" if "%clang%"=="1" set radlink_oodle_flags=-I"%OODLE_SDK_DIR%\include" -DLNK_OODLE=1
+if "%oodle%"=="1" set radlink_oodle_lib="%OODLE_SDK_DIR%\lib\Win64\oo2core_win64.lib"
+if "%rad_obj_compress%"=="1" if "%OODLE_SDK_DIR%"=="" (
+  echo OODLE_SDK_DIR must name an Oodle SDK when building rad_obj_compress
+  exit /b 1
+)
+if "%rad_obj_compress%"=="1" if "%msvc%"=="1"  set rad_obj_compress_oodle_flags=/I"%OODLE_SDK_DIR%\include"
+if "%rad_obj_compress%"=="1" if "%clang%"=="1" set rad_obj_compress_oodle_flags=-I"%OODLE_SDK_DIR%\include"
+if "%rad_obj_compress%"=="1" set rad_obj_compress_oodle_lib="%OODLE_SDK_DIR%\lib\Win64\oo2core_win64.lib"
+if "%radlink%"=="1"                    set didbuild=1 && %compile% %radlink_oodle_flags% ..\src\linker\lnk.c %radlink_oodle_lib% %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1
+if "%rad_obj_compress%"=="1"           set didbuild=1 && %compile% %rad_obj_compress_oodle_flags% ..\src\linker\rad_obj_compress.c %rad_obj_compress_oodle_lib% %compile_link% %out%rad_obj_compress.exe || exit /b 1
 if "%radbin%"=="1"                     set didbuild=1 && %compile% ..\src\radbin\radbin_main.c                               %compile_link% %out%radbin.exe || exit /b 1
 if "%raddump%"=="1"                    set didbuild=1 && %compile% ..\src\raddump\raddump_main.c                             %compile_link% %out%raddump.exe || exit /b 1
 if "%ryan_scratch%"=="1"               set didbuild=1 && %compile% ..\src\scratch\ryan_scratch.c                             %compile_link% %out%ryan_scratch.exe || exit /b 1
@@ -184,6 +222,10 @@ if "%pgo_run%"=="1" (
   if "%radlink%"=="1" (
     pushd local\lyra_pgo
     call %~dp0build\radlink @lyra.rsp || exit /b 1
+    rem lyra.rsp trains with /OPT:NOREF /OPT:NOICF; run a second link with REF+ICF on
+    rem (later switches win) so the profile also covers the /OPT:REF walk and the ICF
+    rem refinement -- otherwise PGO marks them cold and the build regresses on real links
+    call %~dp0build\radlink @lyra.rsp /OPT:REF /OPT:ICF || exit /b 1
     popd
   )
   goto restart

build.sh

@@ -42,8 +42,8 @@ fi
 cc_icon="-DLNX_WM_ICON=1"
 
 # --- Choose Compile/Link Lines -----------------------------------------------
-if   [[ "${gcc:-0}"   == "1" ]]; then compiler="${CC:-gcc}   $cc_cflags_gcc";   echo "[gcc compile]";
-elif [[ "${clang:-1}" == "1" ]]; then compiler="${CC:-clang} $cc_cflags_clang"; echo "[clang compile]";
+if   [[ "${gcc:-0}"   == "1" ]]; then compiler="${CC:-gcc}   $cc_cflags_gcc";   ar="${AR:-ar}";      echo "[gcc compile]";
+elif [[ "${clang:-1}" == "1" ]]; then compiler="${CC:-clang} $cc_cflags_clang"; ar="${AR:-llvm-ar}"; echo "[clang compile]";
 fi
 if   [[ "${release:-0}" == "1" ]]; then echo "[release mode]"; compile="$compiler $cc_release";
 elif [[ "${debug:-1}"   == "1" ]]; then echo "[debug mode]";   compile="$compiler $cc_debug";
@@ -62,12 +62,23 @@ then
   cd ..
 fi
 
+# --- Assemble BLAKE3 ---------------------------------------------------------
+if [[ ! -f "build/blake3.a" ]]
+then
+  echo "[assembling blake3]"
+  $compiler -c -g -o build/blake3_sse2_x86-64_unix.o   src/third_party/blake3/blake3_sse2_x86-64_unix.S
+  $compiler -c -g -o build/blake3_sse41_x86-64_unix.o  src/third_party/blake3/blake3_sse41_x86-64_unix.S
+  $compiler -c -g -o build/blake3_avx2_x86-64_unix.o   src/third_party/blake3/blake3_avx2_x86-64_unix.S
+  $compiler -c -g -o build/blake3_avx512_x86-64_unix.o src/third_party/blake3/blake3_avx512_x86-64_unix.S
+  $ar rs build/blake3.a build/blake3_*_unix.o
+fi
+
 # --- Build Everything (@build_targets) ---------------------------------------
 cd build
 if [[ "${raddbg:-0}"               == "1" ]]; then didbuild=1 && $compile ../src/raddbg/raddbg_main.c $cc_icon $cc_link $cc_os_gfx $cc_render $cc_font_provider -o raddbg; fi
 if [[ "${raddbg_non_graphical:-0}" == "1" ]]; then didbuild=1 && $compile ../src/raddbg/raddbg_main.c -DWM_STUB=1 -DR_BACKEND=R_BACKEND_STUB $cc_link $cc_os_gfx $cc_render $cc_font_provider -o raddbg_non_graphical; fi
 if [[ "${radbin:-0}"               == "1" ]]; then didbuild=1 && $compile ../src/radbin/radbin_main.c   $cc_link -o radbin; fi
-if [[ "${radlink:-0}"              == "1" ]]; then didbuild=1 && $compile ../src/linker/lnk.c           $cc_link -o radlink; fi
+if [[ "${radlink:-0}"              == "1" ]]; then didbuild=1 && $compile ../src/linker/lnk.c           $cc_link -o radlink blake3.a; fi
 if [[ "${torture:-0}"              == "1" ]]; then didbuild=1 && $compile ../src/torture/torture_main.c $cc_link $cc_os_gfx $cc_render $cc_font_provider -o torture; fi
 cd ..