diff --git a/.github/workflows/c_bindings.yml b/.github/workflows/c_bindings.yml index 63f0092..08dbae5 100644 --- a/.github/workflows/c_bindings.yml +++ b/.github/workflows/c_bindings.yml @@ -9,12 +9,10 @@ jobs: test-c-bindings: name: Test C bindings runs-on: ubuntu-latest - strategy: - fail-fast: false steps: - name: Checkout repo - uses: actions/checkout@v5 + uses: actions/checkout@main - name: Make C test program run: make -C c_bindings BUILD_MODE=release @@ -24,27 +22,72 @@ jobs: release: name: Release ${{ matrix.crate-type }} for ${{ matrix.target }} - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - target: [x86_64-pc-windows-gnu, x86_64-pc-windows-msvc, x86_64-apple-darwin, x86_64-unknown-linux-musl, x86_64-unknown-linux-gnu] - crate-type: [staticlib] + target: + - x86_64-pc-windows-gnu + - x86_64-pc-windows-msvc + - aarch64-pc-windows-msvc + - x86_64-apple-darwin + - aarch64-apple-darwin + - x86_64-unknown-linux-musl + - x86_64-unknown-linux-gnu + - aarch64-unknown-linux-musl + - aarch64-unknown-linux-gnu + crate-type: + - staticlib include: - target: x86_64-pc-windows-gnu archive: zip + os_name: windows + runner: windows-latest + architecture: x86 - target: x86_64-pc-windows-msvc archive: zip + os_name: windows + runner: windows-latest + architecture: x86 + - target: aarch64-pc-windows-msvc + archive: zip + os_name: windows + runner: windows-latest + architecture: aarch64 - target: x86_64-apple-darwin archive: zip + os_name: macos + runner: macos-15-intel + architecture: x86 + - target: aarch64-apple-darwin + archive: zip + os_name: macos + runner: macos-latest + architecture: aarch64 - target: x86_64-unknown-linux-musl archive: tar.gz + os_name: linux + runner: ubuntu-22.04 + architecture: x86 - target: x86_64-unknown-linux-gnu archive: tar.gz + os_name: linux + runner: ubuntu-22.04 + architecture: x86 + - target: aarch64-unknown-linux-musl + archive: tar.gz + os_name: linux + architecture: aarch64 + runner: ubuntu-22.04 + - target: aarch64-unknown-linux-gnu + archive: tar.gz + os_name: linux + architecture: aarch64 + runner: ubuntu-22.04 steps: - name: Checkout repo - uses: actions/checkout@v5 + uses: actions/checkout@main - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -54,55 +97,121 @@ jobs: - name: Build lib run: cargo rustc --manifest-path lib/Cargo.toml --lib --features c_bindings --release --crate-type ${{ matrix.crate-type }} --target ${{ matrix.target }} - - name: Print built files + - name: Install tree + if: matrix.os_name == 'macos' + run: + brew install tree + + - name: Print built files (unix) + if: matrix.os_name == 'linux' || matrix.os_name == 'macos' run: | tree target/${{ matrix.target }}/release + - name: Print built files (windows) + if: matrix.os_name == 'windows' + run: | + tree /f target/${{ matrix.target }}/release + + # Move versioned files and built lib to a new folder, this will contain the final release files - name: Move files for packaging run: | - mkdir -p package/lib package/include - cp target/${{ matrix.target }}/release/libcrunch64.a package/lib/ || cp target/${{ matrix.target }}/release/crunch64.lib package/lib/ + mkdir package + mkdir package/lib + mkdir package/include cp -r c_bindings/include/* package/include/ cp LICENSE package/crunch64.LICENSE cp README.md package/crunch64.README.md + + - name: Copy package (unix) + if: matrix.os_name == 'linux' || matrix.os_name == 'macos' || matrix.target == 'x86_64-pc-windows-gnu' + run: | + cp target/${{ matrix.target }}/release/libcrunch64.a package/lib/ tree package + - name: Copy package (windows) + if: matrix.os_name == 'windows' && matrix.target != 'x86_64-pc-windows-gnu' + run: | + cp target/${{ matrix.target }}/release/crunch64.lib package/lib/ + tree /f package + + # Compress the package files + # Two steps because windows silly - name: Package .tar.gz if: matrix.archive == 'tar.gz' run: | cd package && tar -czf ../crunch64-${{ matrix.crate-type }}-${{ matrix.target }}.tar.gz * - name: Package .zip - if: matrix.archive == 'zip' + if: matrix.archive == 'zip' && matrix.os_name == 'macos' run: | cd package && zip -r ../crunch64-${{ matrix.crate-type }}-${{ matrix.target }}.zip * - - name: Upload .tar.gz archive - if: matrix.archive == 'tar.gz' - uses: actions/upload-artifact@v4 - with: - name: crunch64-${{ matrix.crate-type }}-${{ matrix.target }} - path: | - crunch64-${{ matrix.crate-type }}-${{ matrix.target }}.tar.gz - if-no-files-found: error + - name: Package .zip + if: matrix.archive == 'zip' && matrix.os_name == 'windows' + run: | + cd package; Compress-Archive -Path * -DestinationPath ../crunch64-${{ matrix.crate-type }}-${{ matrix.target }}.zip - - name: Upload .zip archive - if: matrix.archive == 'zip' - uses: actions/upload-artifact@v4 + # Upload the artifact, useful for inspecting everything is ok + - name: Upload ${{ matrix.archive }} archive + uses: actions/upload-artifact@main with: name: crunch64-${{ matrix.crate-type }}-${{ matrix.target }} path: | - crunch64-${{ matrix.crate-type }}-${{ matrix.target }}.zip + crunch64-${{ matrix.crate-type }}-${{ matrix.target }}.${{ matrix.archive }} if-no-files-found: error - - name: Publish .tar.gz release - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') && matrix.archive == 'tar.gz' - with: - files: crunch64-${{ matrix.crate-type }}-${{ matrix.target }}.tar.gz + # Build and run the archives. Multi step because each os/arch combo is weird + - name: Test built archive (x86, unix like) + if: (matrix.os_name == 'linux' && matrix.architecture == 'x86') || matrix.os_name == 'macos' + run: | + cc -I c_bindings/include/ c_bindings/tests.c -L package/lib/ -l crunch64 -o tests.elf + ./tests.elf + + # Full static binary + - name: Test built archive (x86, unix like, musl, clang) + if: matrix.target == 'x86_64-unknown-linux-musl' + run: | + clang -fuse-ld=lld -I/usr/include/x86_64-linux-musl -L/usr/lib/x86_64-linux-musl --target=x86_64-unknown-linux-musl -static -Wl,--gc-sections -Wl,--strip-all -ffunction-sections -fdata-sections -I c_bindings/include/ c_bindings/tests.c -L package/lib/ -l crunch64 -o tests.elf + file ./tests.elf + ./tests.elf + + - name: Setup qemu + uses: docker/setup-qemu-action@v4 + if: matrix.os_name == 'linux' && matrix.architecture == 'aarch64' + - name: Test built archive (aarch64, linux) + if: matrix.os_name == 'linux' && matrix.architecture == 'aarch64' + run: | + docker run --platform linux/arm64 --rm -v ${{ github.workspace }}:/work -w /work gcc:latest \ + sh -c "cc -I c_bindings/include/ c_bindings/tests.c -L package/lib/ -l crunch64 -o tests.elf && ./tests.elf" + # Full static binary + - name: Test built archive (aarch64, linux, musl, clang) + if: matrix.os_name == 'linux' && matrix.architecture == 'aarch64' && matrix.target == 'aarch64-unknown-linux-musl' + run: | + docker run --platform linux/arm64 --rm -v ${{ github.workspace }}:/work -w /work alpine:latest \ + sh -c "apk add clang lld && clang -fuse-ld=lld -I/usr/include/aarch64-linux-musl -L/usr/lib/aarch64-linux-musl --target=aarch64-unknown-linux-musl -static -Wl,--gc-sections -Wl,--strip-all -ffunction-sections -fdata-sections -I c_bindings/include/ c_bindings/tests.c -L package/lib/ -l crunch64 -o tests.elf && ./tests.elf" + + - name: Setup MSVC in PATH + uses: ilammy/msvc-dev-cmd@v1 + if: (matrix.os_name == 'windows' && matrix.architecture == 'x86') && matrix.target != 'x86_64-pc-windows-gnu' + # Windows needs extra flags + - name: Test built archive (x86 msvc) + if: (matrix.os_name == 'windows' && matrix.architecture == 'x86') && matrix.target != 'x86_64-pc-windows-gnu' + run: | + cl /I c_bindings/include/ c_bindings/tests.c package/lib/crunch64.lib ws2_32.lib ntdll.lib bcrypt.lib advapi32.lib userenv.lib /Fe:tests.exe + ./tests.exe + + # Windows needs extra flags + - name: Test built archive (x86 windows, unix like) + if: matrix.target == 'x86_64-pc-windows-gnu' + run: | + cc -I c_bindings/include/ c_bindings/tests.c -L package/lib/ -l crunch64 -lws2_32 -lntdll -lbcrypt -ladvapi32 -luserenv -o tests.exe + ./tests.exe + + # We don't have a way to build and run aarch64 windows binaries on GHA :c - - name: Publish .zip release + # Actually publish the built files to github releases + - name: Publish ${{ matrix.archive }} release uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') && matrix.archive == 'zip' + if: startsWith(github.ref, 'refs/tags/') with: - files: crunch64-${{ matrix.crate-type }}-${{ matrix.target }}.zip + files: crunch64-${{ matrix.crate-type }}-${{ matrix.target }}.${{ matrix.archive }} diff --git a/.gitignore b/.gitignore index 0b7379e..87faefd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,7 @@ target/ *.Yay0 *.Yaz0 *.elf +*.a +*.lib .vscode/settings.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 742b6e6..562b571 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.6.2] - 2026-03-28 + +### Added + +- Prebuilt C archive libraries for the following targets: + - `aarch64-pc-windows-msvc` + - `aarch64-apple-darwin` + - `aarch64-unknown-linux-gnu` + - `aarch64-unknown-linux-musl` +- Most prebuilt C libs are now tested in CI. + - The following targets are being tested in CI: + - `x86_64-pc-windows-gnu` + - `x86_64-pc-windows-msvc` + - `x86_64-apple-darwin` + - `aarch64-apple-darwin` + - `x86_64-unknown-linux-musl` + - Builds/links using GCC default and Clang with musl target. + - `x86_64-unknown-linux-gnu` + - `aarch64-unknown-linux-musl` + - `aarch64-unknown-linux-gnu` + - The following targets are NOT being tested in CI: + - `aarch64-pc-windows-msvc` + +### Changed + +- Prebuilt C libs are now built under the corresponding native OS instead of + crosscompiling from an Ubuntu runner. +- Rewrite the C bindings test file to make it compatible with MSVC. + ### Fixed - Fix CI not running Rust tests. @@ -139,7 +168,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Python bindings. - C bindings. -[unreleased]: https://github.com/decompals/crunch64/compare/0.6.1...HEAD +[unreleased]: https://github.com/decompals/crunch64/compare/0.6.2...HEAD +[0.6.2]: https://github.com/decompals/crunch64/compare/0.6.1...0.6.2 [0.6.1]: https://github.com/decompals/crunch64/compare/0.6.0...0.6.1 [0.6.0]: https://github.com/decompals/crunch64/compare/0.5.4...0.6.0 [0.5.4]: https://github.com/decompals/crunch64/compare/0.5.3...0.5.4 diff --git a/Cargo.lock b/Cargo.lock index 87a0700..249957f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,7 +130,7 @@ dependencies = [ [[package]] name = "crunch64" -version = "0.6.1" +version = "0.6.2" dependencies = [ "crc32fast", "pyo3", @@ -140,7 +140,7 @@ dependencies = [ [[package]] name = "crunch64-cli" -version = "0.6.1" +version = "0.6.2" dependencies = [ "clap", "crunch64", diff --git a/Cargo.toml b/Cargo.toml index 527dc33..f937cf7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ resolver = "2" [workspace.package] # Version should be synced with lib/pyproject.toml and lib/crunch64/__init__.py -version = "0.6.1" +version = "0.6.2" edition = "2021" repository = "https://github.com/decompals/crunch64" license = "MIT" diff --git a/c_bindings/tests.c b/c_bindings/tests.c index 2d1a267..9f03fb7 100644 --- a/c_bindings/tests.c +++ b/c_bindings/tests.c @@ -1,13 +1,85 @@ #include "crunch64.h" #include -#include #include #include #include #include #include +#ifdef _MSC_VER +#include +#else +#include +#endif + + +typedef struct CrossDirEntry { +#ifdef _MSC_VER + WIN32_FIND_DATA find_data; + HANDLE hFind; + bool first; +#else + DIR *dir; +#endif +} CrossDirEntry; + + +bool cross_open_dir(CrossDirEntry *out, const char *dir_path) { +#ifdef _MSC_VER + char buffer[0x400] = {0}; + snprintf(buffer, sizeof(buffer), "%s/*", dir_path); + out->hFind = FindFirstFile(buffer, &out->find_data); + out->first = true; + if (out->hFind == INVALID_HANDLE_VALUE) { + return false; + } +#else + out->dir = opendir(dir_path); + if (!out->dir) { + return false; + } +#endif + return true; +} + +void cross_close_dir(CrossDirEntry *out) { +#ifdef _MSC_VER + FindClose(out->hFind); +#else + closedir(out->dir); +#endif +} + +const char *cross_next_filename(CrossDirEntry *out) { +#ifdef _MSC_VER + if (!out->first) { + if (!FindNextFile(out->hFind, &out->find_data)) { + return NULL; + } + } + + do { + const char *filename = out->find_data.cFileName; + out->first = false; + + if (strcmp(filename, ".") == 0 || strcmp(filename, "..") == 0) { + continue; + } + + return filename; + + } while (FindNextFile(out->hFind, &out->find_data)); +#else + struct dirent *entry = readdir(out->dir); + if (entry) { + return entry->d_name; + } +#endif + return NULL; +} + + typedef Crunch64Error (*compress_bound_fn)(size_t *dst_size, size_t src_len, const uint8_t *const src); typedef Crunch64Error (*compress_fn)(size_t *dst_size, uint8_t *dst, size_t src_size, const uint8_t *src); @@ -21,6 +93,7 @@ const char *const crunch64_error_str[] = { [Crunch64Error_ByteConversion] = "Byte conversion", [Crunch64Error_OutOfBounds] = "Out of bounds", [Crunch64Error_NullPointer] = "Null pointer", + [Crunch64Error_InvalidCompressionLevel] = "Invalid compression level", }; const char *get_crunch64_error_str(Crunch64Error error) { @@ -182,9 +255,8 @@ int errors = 0; void run_tests(const char *name, const char *file_extension, compress_bound_fn compress_bound, compress_fn compress, compress_bound_fn decompress_bound, compress_fn decompress) { - struct dirent *entry; - DIR *dir = opendir("test_data"); - if (!dir) { + CrossDirEntry dir; + if (!cross_open_dir(&dir, "test_data")) { fprintf(stderr, "Could not open test_data directory\n"); errors++; return; @@ -194,19 +266,21 @@ void run_tests(const char *name, const char *file_extension, compress_bound_fn c fprintf(stderr, "\n"); bool found_tests = false; - while ((entry = readdir(dir)) != NULL) { - if (!has_suffix(entry->d_name, file_extension)) { + + const char *filename; + while ((filename = cross_next_filename(&dir)) != NULL) { + if (!has_suffix(filename, file_extension)) { continue; } found_tests = true; char bin_path[512]; - snprintf(bin_path, sizeof(bin_path), "test_data/%s", entry->d_name); + snprintf(bin_path, sizeof(bin_path), "test_data/%s", filename); bin_path[strlen(bin_path) - strlen(file_extension)] = '\0'; // remove file extension char compressed_path[512]; - snprintf(compressed_path, sizeof(compressed_path), "test_data/%s", entry->d_name); + snprintf(compressed_path, sizeof(compressed_path), "test_data/%s", filename); fprintf(stderr, "Reading file %s\n", bin_path); size_t bin_size = 0; @@ -233,6 +307,8 @@ void run_tests(const char *name, const char *file_extension, compress_bound_fn c free(compressed_data); } + cross_close_dir(&dir); + if (!found_tests) { fprintf(stderr, "No test files found for %s\n", name); errors++; diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 1adc0b4..bf27e5f 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -12,5 +12,5 @@ name = "crunch64" path = "src/main.rs" [dependencies] -crunch64 = { version = "0.6.1", path = "../lib" } +crunch64 = { version = "0.6.2", path = "../lib" } clap = { version = "4.4.11", features = ["derive"] } diff --git a/lib/crunch64/__init__.py b/lib/crunch64/__init__.py index 8330fc7..5625640 100644 --- a/lib/crunch64/__init__.py +++ b/lib/crunch64/__init__.py @@ -3,7 +3,7 @@ from __future__ import annotations # Version should be synced with lib/Cargo.toml and lib/pyproject.toml -__version_info__ = (0, 6, 1) +__version_info__ = (0, 6, 2) __version__ = ".".join(map(str, __version_info__)) __author__ = "decompals" diff --git a/lib/pyproject.toml b/lib/pyproject.toml index 82fa5b8..ae3018d 100644 --- a/lib/pyproject.toml +++ b/lib/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "crunch64" # Version should be synced with lib/Cargo.toml and lib/crunch64/__init__.py -version = "0.6.1" +version = "0.6.2" description = "A library for handling common compression formats for N64 games" requires-python = ">=3.7" # Required by PyO3 dependencies = [