From db2f18ebd756e27208080e5a2131c88337f4b947 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Wed, 16 Sep 2026 21:34:10 +0800 Subject: [PATCH 1/6] ci(github): add GPU unittest workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # 🔧 CI/CD ## Run unit tests on host GPU runners - Trigger GPU unit tests automatically for pull requests and pushes to main. - Use the public latest image with a fixed four-GPU allocation per runner, a shared host uv cache, and a private container workspace. - Keep dependency installation and the complete pytest run in separate workflow steps, preserving test failures and existing skips. - Always remove the container and upload pytest logs and JUnit results. Co-authored-by: Codex --- .github/workflows/unittest.yml | 99 ++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 .github/workflows/unittest.yml diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml new file mode 100644 index 00000000..ba0d1424 --- /dev/null +++ b/.github/workflows/unittest.yml @@ -0,0 +1,99 @@ +name: GPU Unit Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: gpu-unittest-${{ github.ref }} + cancel-in-progress: true + +jobs: + unittest: + name: Unit Tests (H20, 4 GPUs) + runs-on: [self-hosted, linux, x64, h20, gpu] + timeout-minutes: 30 + env: + CI_IMAGE: ghcr.io/redai-studio/relaxrl:latest + CI_CONTAINER_NAME: relax-unittest-${{ github.run_id }}-${{ github.run_attempt }} + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + persist-credentials: false + + # Set CI_GPU_DEVICES in each runner's .env before starting its service: + # runner 1: 0,1,2,3; runner 2: 4,5,6,7. Never infer free GPUs at job startup. + - name: Create test container + run: | + set -euo pipefail + : "${CI_GPU_DEVICES:?Set CI_GPU_DEVICES to the four GPUs assigned to this runner}" + IFS=',' read -r -a gpu_devices <<< "$CI_GPU_DEVICES" + if [[ ${#gpu_devices[@]} -ne 4 ]]; then + echo "CI_GPU_DEVICES must list exactly four GPU indices or UUIDs" >&2 + exit 1 + fi + uv_cache_dir="${CI_UV_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/uv}" + artifact_dir="$RUNNER_TEMP/unittest-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + mkdir -p "$uv_cache_dir" "$artifact_dir" + uv_cache_dir="$(cd -- "$uv_cache_dir" && pwd)" + + # Keep network, IPC and /tmp private for parallel Ray/NCCL tests. + # Create before starting so cancellation during a pull cannot start tests. + docker create --pull always --rm --init \ + --name "$CI_CONTAINER_NAME" \ + --gpus "\"device=$CI_GPU_DEVICES\"" \ + --shm-size=32g \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/source,readonly" \ + --mount "type=bind,source=$artifact_dir,target=/artifacts" \ + --mount "type=bind,source=$uv_cache_dir,target=/uv-cache" \ + --workdir /workspace/Relax \ + --env CUDA_VISIBLE_DEVICES=0,1,2,3 \ + --env RELAX_RUN_SMOKE=0 \ + --env UV_CACHE_DIR=/uv-cache \ + --env UV_LINK_MODE=copy \ + --env "http_proxy=${http_proxy:-${HTTP_PROXY:-}}" \ + --env "https_proxy=${https_proxy:-${HTTPS_PROXY:-}}" \ + --env "no_proxy=${no_proxy:-${NO_PROXY:-localhost,127.0.0.1}}" \ + --entrypoint /bin/bash "$CI_IMAGE" -c 'sleep infinity' + docker inspect --format 'Test image: {{.Config.Image}} ({{.Image}})' "$CI_CONTAINER_NAME" + docker start "$CI_CONTAINER_NAME" + + - name: Prepare workspace and show GPUs + run: | + docker exec "$CI_CONTAINER_NAME" bash -euc ' + cp -a --no-preserve=ownership /source/. /workspace/Relax/ + nvidia-smi + ' + + - name: Install dependencies + run: | + docker exec "$CI_CONTAINER_NAME" uv pip install \ + --system --break-system-packages -r requirements.txt \ + --extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + + - name: Run unittest + run: | + docker exec "$CI_CONTAINER_NAME" bash -euo pipefail -c ' + python -m pytest tests/ -v -ra --tb=short --junitxml=/artifacts/junit.xml \ + 2>&1 | tee /artifacts/pytest.log + ' + + - name: Remove test container + if: always() + run: docker rm --force "$CI_CONTAINER_NAME" >/dev/null 2>&1 || true + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: unittest-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unittest-${{ github.run_id }}-${{ github.run_attempt }}/ + if-no-files-found: warn + retention-days: 14 From 95fa901ea8d03c3886f72da27a34f8904a714055 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Thu, 17 Sep 2026 15:06:54 +0800 Subject: [PATCH 2/6] test(mtp): isolate CPU threads in loss tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # ✅ Tests - Run chunked MTP loss comparisons with one CPU thread using a module-scoped fixture. - Restore the previous thread count after the module finishes. - Verify all 13 tests pass in both CI images and thread counts return from 1 to 192. Co-authored-by: Codex --- tests/backends/megatron/test_chunked_mtp_loss.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/backends/megatron/test_chunked_mtp_loss.py b/tests/backends/megatron/test_chunked_mtp_loss.py index e9cc1b13..3ce8e029 100644 --- a/tests/backends/megatron/test_chunked_mtp_loss.py +++ b/tests/backends/megatron/test_chunked_mtp_loss.py @@ -17,6 +17,7 @@ from __future__ import annotations import types +from collections.abc import Iterator import pytest import torch @@ -35,6 +36,18 @@ S, B, H, V, LAYERS = 16, 2, 8, 32, 2 +@pytest.fixture(scope="module", autouse=True) +def _single_threaded_cpu() -> Iterator[None]: + """Run bitwise CPU comparisons with one thread, then restore the previous + setting.""" + previous_num_threads = torch.get_num_threads() + torch.set_num_threads(1) + try: + yield + finally: + torch.set_num_threads(previous_num_threads) + + class _FakeHead(nn.Module): """Stand-in for ColumnParallelLinear output_layer. From effca4b4dafbce67bdcfecc680564cd21e9931c2 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Thu, 17 Sep 2026 15:58:10 +0800 Subject: [PATCH 3/6] ci(github): simplify unittest proxy variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # 🔧 CI/CD - Pass the runner's http_proxy, https_proxy, and no_proxy directly to Docker. - Remove the runner GPU environment setup comments. Co-authored-by: Codex --- .github/workflows/unittest.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index ba0d1424..560144ad 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -28,8 +28,6 @@ jobs: with: persist-credentials: false - # Set CI_GPU_DEVICES in each runner's .env before starting its service: - # runner 1: 0,1,2,3; runner 2: 4,5,6,7. Never infer free GPUs at job startup. - name: Create test container run: | set -euo pipefail @@ -58,9 +56,9 @@ jobs: --env RELAX_RUN_SMOKE=0 \ --env UV_CACHE_DIR=/uv-cache \ --env UV_LINK_MODE=copy \ - --env "http_proxy=${http_proxy:-${HTTP_PROXY:-}}" \ - --env "https_proxy=${https_proxy:-${HTTPS_PROXY:-}}" \ - --env "no_proxy=${no_proxy:-${NO_PROXY:-localhost,127.0.0.1}}" \ + --env http_proxy \ + --env https_proxy \ + --env no_proxy \ --entrypoint /bin/bash "$CI_IMAGE" -c 'sleep infinity' docker inspect --format 'Test image: {{.Config.Image}} ({{.Image}})' "$CI_CONTAINER_NAME" docker start "$CI_CONTAINER_NAME" From d0018b30058d0ae467a25cd455d9952f347f421b Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Thu, 17 Sep 2026 16:12:57 +0800 Subject: [PATCH 4/6] ci(github): simplify unittest cache path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # 🔧 CI/CD - Use $HOME/.cache/uv for the shared runner cache. - Remove cache path fallbacks and path normalization. Co-authored-by: Codex --- .github/workflows/unittest.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 560144ad..e5bc62b7 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -37,10 +37,9 @@ jobs: echo "CI_GPU_DEVICES must list exactly four GPU indices or UUIDs" >&2 exit 1 fi - uv_cache_dir="${CI_UV_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/uv}" + uv_cache_dir="$HOME/.cache/uv" artifact_dir="$RUNNER_TEMP/unittest-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" mkdir -p "$uv_cache_dir" "$artifact_dir" - uv_cache_dir="$(cd -- "$uv_cache_dir" && pwd)" # Keep network, IPC and /tmp private for parallel Ray/NCCL tests. # Create before starting so cancellation during a pull cannot start tests. From eec0662404350f0d96913306bff97dcf33f9317a Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Thu, 17 Sep 2026 16:29:10 +0800 Subject: [PATCH 5/6] ci(github): simplify container configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # 🔧 CI/CD - Define container settings and artifact paths in the step environment. - Mount the shared uv cache at the same absolute path on the host and container. - Forward container environment variables by name and use concise bind mounts. Co-authored-by: Codex --- .github/workflows/unittest.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index e5bc62b7..e03fc5a4 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -29,6 +29,12 @@ jobs: persist-credentials: false - name: Create test container + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3 + RELAX_RUN_SMOKE: "0" + UV_CACHE_DIR: /home/relax-ci/.cache/uv + UV_LINK_MODE: copy + ARTIFACT_DIR: ${{ runner.temp }}/unittest-${{ github.run_id }}-${{ github.run_attempt }} run: | set -euo pipefail : "${CI_GPU_DEVICES:?Set CI_GPU_DEVICES to the four GPUs assigned to this runner}" @@ -37,9 +43,7 @@ jobs: echo "CI_GPU_DEVICES must list exactly four GPU indices or UUIDs" >&2 exit 1 fi - uv_cache_dir="$HOME/.cache/uv" - artifact_dir="$RUNNER_TEMP/unittest-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" - mkdir -p "$uv_cache_dir" "$artifact_dir" + mkdir -p "$UV_CACHE_DIR" "$ARTIFACT_DIR" # Keep network, IPC and /tmp private for parallel Ray/NCCL tests. # Create before starting so cancellation during a pull cannot start tests. @@ -47,14 +51,14 @@ jobs: --name "$CI_CONTAINER_NAME" \ --gpus "\"device=$CI_GPU_DEVICES\"" \ --shm-size=32g \ - --mount "type=bind,source=$GITHUB_WORKSPACE,target=/source,readonly" \ - --mount "type=bind,source=$artifact_dir,target=/artifacts" \ - --mount "type=bind,source=$uv_cache_dir,target=/uv-cache" \ + -v "$GITHUB_WORKSPACE:/source:ro" \ + -v "$ARTIFACT_DIR:/artifacts" \ + -v "$UV_CACHE_DIR:$UV_CACHE_DIR" \ --workdir /workspace/Relax \ - --env CUDA_VISIBLE_DEVICES=0,1,2,3 \ - --env RELAX_RUN_SMOKE=0 \ - --env UV_CACHE_DIR=/uv-cache \ - --env UV_LINK_MODE=copy \ + --env CUDA_VISIBLE_DEVICES \ + --env RELAX_RUN_SMOKE \ + --env UV_CACHE_DIR \ + --env UV_LINK_MODE \ --env http_proxy \ --env https_proxy \ --env no_proxy \ From 64f730f591d41c2b53a04490c0505ce3aa5206eb Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Thu, 17 Sep 2026 16:36:07 +0800 Subject: [PATCH 6/6] ci(github): clarify unittest cache paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # 🔧 CI/CD ## Separate host and container cache paths - Name the host uv cache separately from the container cache path. - Stop forwarding RELAX_RUN_SMOKE because the smoke test already defaults to disabled. Co-authored-by: Codex --- .github/workflows/unittest.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index e03fc5a4..c923880c 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -31,8 +31,8 @@ jobs: - name: Create test container env: CUDA_VISIBLE_DEVICES: 0,1,2,3 - RELAX_RUN_SMOKE: "0" - UV_CACHE_DIR: /home/relax-ci/.cache/uv + HOST_UV_CACHE_DIR: /home/relax-ci/.cache/uv + UV_CACHE_DIR: /uv-cache UV_LINK_MODE: copy ARTIFACT_DIR: ${{ runner.temp }}/unittest-${{ github.run_id }}-${{ github.run_attempt }} run: | @@ -43,7 +43,7 @@ jobs: echo "CI_GPU_DEVICES must list exactly four GPU indices or UUIDs" >&2 exit 1 fi - mkdir -p "$UV_CACHE_DIR" "$ARTIFACT_DIR" + mkdir -p "$HOST_UV_CACHE_DIR" "$ARTIFACT_DIR" # Keep network, IPC and /tmp private for parallel Ray/NCCL tests. # Create before starting so cancellation during a pull cannot start tests. @@ -53,10 +53,9 @@ jobs: --shm-size=32g \ -v "$GITHUB_WORKSPACE:/source:ro" \ -v "$ARTIFACT_DIR:/artifacts" \ - -v "$UV_CACHE_DIR:$UV_CACHE_DIR" \ + -v "$HOST_UV_CACHE_DIR:$UV_CACHE_DIR" \ --workdir /workspace/Relax \ --env CUDA_VISIBLE_DEVICES \ - --env RELAX_RUN_SMOKE \ --env UV_CACHE_DIR \ --env UV_LINK_MODE \ --env http_proxy \