From c81be55950da02a0eb87a711e8a4cb8b80599fe6 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 17 Apr 2026 14:22:30 +0200 Subject: [PATCH 01/13] Migrate subpackage test runners to ParallelTestRunner. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each subpackage's `Pkg.test` runner is now a minimal call to PTR's `runtests`, which spawns one worker process per test file and runs them concurrently. `setup.jl` is loaded via `init_code` so each worker picks up the shared fixtures. Side effects: - Extract cuTENSOR's inline "kernel cache" testset to its own file (lib/cutensor/test/kernel_cache.jl) since runtests.jl is no longer the place for test code. - cuSPARSE's array.jl had three show-output tests that implicitly relied on `using cuSPARSE, SparseArrays` being in Main (where CUDA.jl's top-level runner incidentally loaded them). PTR workers run tests in an isolated submodule, so pass an explicit `:module => @__MODULE__` context to `sprint(show, …)` so the type names are qualified against the worker module's bindings rather than Main's. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/cublas/test/Project.toml | 4 ++++ lib/cublas/test/runtests.jl | 18 ++++++++-------- lib/cudnn/test/Project.toml | 4 ++++ lib/cudnn/test/runtests.jl | 25 ++++++++-------------- lib/cufft/test/Project.toml | 4 ++++ lib/cufft/test/ensure_increasing.jl | 10 +++++++++ lib/cufft/test/runtests.jl | 21 ++++++++----------- lib/curand/test/Project.toml | 4 ++++ lib/curand/test/runtests.jl | 14 +++++++++---- lib/cusolver/test/Project.toml | 4 ++++ lib/cusolver/test/runtests.jl | 18 ++++++++-------- lib/cusparse/test/Project.toml | 4 ++++ lib/cusparse/test/array.jl | 11 +++++----- lib/cusparse/test/runtests.jl | 19 ++++++++--------- lib/custatevec/test/Project.toml | 4 ++++ lib/custatevec/test/runtests.jl | 18 ++++++++-------- lib/cutensor/test/Project.toml | 4 ++++ lib/cutensor/test/kernel_cache.jl | 14 +++++++++++++ lib/cutensor/test/runtests.jl | 32 ++++++++--------------------- lib/cutensornet/test/Project.toml | 4 ++++ lib/cutensornet/test/runtests.jl | 16 ++++++++------- 21 files changed, 144 insertions(+), 108 deletions(-) create mode 100644 lib/cufft/test/ensure_increasing.jl create mode 100644 lib/cutensor/test/kernel_cache.jl diff --git a/lib/cublas/test/Project.toml b/lib/cublas/test/Project.toml index 997d2d07c1..24a7abfa6f 100644 --- a/lib/cublas/test/Project.toml +++ b/lib/cublas/test/Project.toml @@ -3,7 +3,11 @@ Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" BFloat16s = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" CUDACore = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" cuBLAS = "182d3088-87b7-4494-8cad-fc6afaa545bc" + +[compat] +ParallelTestRunner = "2" diff --git a/lib/cublas/test/runtests.jl b/lib/cublas/test/runtests.jl index 58dda52d4f..b10b05e2a3 100644 --- a/lib/cublas/test/runtests.jl +++ b/lib/cublas/test/runtests.jl @@ -1,11 +1,11 @@ -include("setup.jl") +using cuBLAS +using ParallelTestRunner -@testset verbose=true "cuBLAS" begin - for (root, _, files) in walkdir(@__DIR__) - for file in sort(files) - endswith(file, ".jl") || continue - file in ("setup.jl", "runtests.jl") && continue - include(joinpath(root, file)) - end - end +const init_code = quote + include(joinpath(@__DIR__, "setup.jl")) end + +testsuite = find_tests(@__DIR__) +delete!(testsuite, "setup") + +runtests(cuBLAS, ARGS; init_code, testsuite) diff --git a/lib/cudnn/test/Project.toml b/lib/cudnn/test/Project.toml index eafa4a062f..f84b126c5e 100644 --- a/lib/cudnn/test/Project.toml +++ b/lib/cudnn/test/Project.toml @@ -2,7 +2,11 @@ CUDACore = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" + +[compat] +ParallelTestRunner = "2" diff --git a/lib/cudnn/test/runtests.jl b/lib/cudnn/test/runtests.jl index 1a4fef1963..7a52d6782e 100644 --- a/lib/cudnn/test/runtests.jl +++ b/lib/cudnn/test/runtests.jl @@ -1,20 +1,11 @@ -include("setup.jl") -@test cuDNN.functional() +using cuDNN +using ParallelTestRunner -@testset verbose=true "cuDNN" begin - -# include all tests -for entry in readdir(@__DIR__) - endswith(entry, ".jl") || continue - entry in ["runtests.jl", "setup.jl"] && continue - - # generate a testset - name = splitext(entry)[1] - @eval begin - @testset $name begin - include($entry) - end - end +const init_code = quote + include(joinpath(@__DIR__, "setup.jl")) end -end +testsuite = find_tests(@__DIR__) +delete!(testsuite, "setup") + +runtests(cuDNN, ARGS; init_code, testsuite) diff --git a/lib/cufft/test/Project.toml b/lib/cufft/test/Project.toml index eab2115dcb..2584ad66e3 100644 --- a/lib/cufft/test/Project.toml +++ b/lib/cufft/test/Project.toml @@ -4,5 +4,9 @@ Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" CUDACore = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" cuFFT = "533571aa-0936-420e-b4be-9c66f5f626ca" + +[compat] +ParallelTestRunner = "2" diff --git a/lib/cufft/test/ensure_increasing.jl b/lib/cufft/test/ensure_increasing.jl new file mode 100644 index 0000000000..eeef11e2dd --- /dev/null +++ b/lib/cufft/test/ensure_increasing.jl @@ -0,0 +1,10 @@ +using cuFFT + +@testset "ensure_increasing" begin + @test (1,2,3) == cuFFT.ensure_increasing((1,2,3)) + @test (1,2,3) == cuFFT.ensure_increasing((2,1,3)) + @test (1,2,3) == cuFFT.ensure_increasing((1,3,2)) + @test (1,2,3) == cuFFT.ensure_increasing((3,1,2)) + @test (1,2,3) == cuFFT.ensure_increasing((2,3,1)) + @test (10,20,30) == cuFFT.ensure_increasing((30,20,10)) +end diff --git a/lib/cufft/test/runtests.jl b/lib/cufft/test/runtests.jl index a3c154cb0f..cad38ee47a 100644 --- a/lib/cufft/test/runtests.jl +++ b/lib/cufft/test/runtests.jl @@ -1,16 +1,11 @@ -include("setup.jl") -@test cuFFT.version() isa VersionNumber +using cuFFT +using ParallelTestRunner -@testset "ensure_increasing" begin - @test (1,2,3) == cuFFT.ensure_increasing((1,2,3)) - @test (1,2,3) == cuFFT.ensure_increasing((2,1,3)) - @test (1,2,3) == cuFFT.ensure_increasing((1,3,2)) - @test (1,2,3) == cuFFT.ensure_increasing((3,1,2)) - @test (1,2,3) == cuFFT.ensure_increasing((2,3,1)) - @test (10,20,30) == cuFFT.ensure_increasing((30,20,10)) +const init_code = quote + include(joinpath(@__DIR__, "setup.jl")) end -include("complex.jl") -include("real.jl") -include("integer.jl") -include("issues.jl") +testsuite = find_tests(@__DIR__) +delete!(testsuite, "setup") + +runtests(cuFFT, ARGS; init_code, testsuite) diff --git a/lib/curand/test/Project.toml b/lib/curand/test/Project.toml index 03e6b0edac..aed94eda49 100644 --- a/lib/curand/test/Project.toml +++ b/lib/curand/test/Project.toml @@ -1,5 +1,9 @@ [deps] CUDACore = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" +ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" cuRAND = "20fd9a0b-12d5-4c2f-a8af-7c34e9e60431" + +[compat] +ParallelTestRunner = "2" diff --git a/lib/curand/test/runtests.jl b/lib/curand/test/runtests.jl index 175834f58a..317e68496d 100644 --- a/lib/curand/test/runtests.jl +++ b/lib/curand/test/runtests.jl @@ -1,5 +1,11 @@ -include("setup.jl") -@test cuRAND.version() isa VersionNumber +using cuRAND +using ParallelTestRunner -include("high_level.jl") -include("native.jl") +const init_code = quote + include(joinpath(@__DIR__, "setup.jl")) +end + +testsuite = find_tests(@__DIR__) +delete!(testsuite, "setup") + +runtests(cuRAND, ARGS; init_code, testsuite) diff --git a/lib/cusolver/test/Project.toml b/lib/cusolver/test/Project.toml index ffb8aebc58..81527f1de6 100644 --- a/lib/cusolver/test/Project.toml +++ b/lib/cusolver/test/Project.toml @@ -2,8 +2,12 @@ Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" CUDACore = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" cuBLAS = "182d3088-87b7-4494-8cad-fc6afaa545bc" cuSOLVER = "887afef0-6a32-4de5-add4-7827692ba8fc" cuSPARSE = "b26da814-b3bc-49ef-b0ee-c816305aa060" + +[compat] +ParallelTestRunner = "2" diff --git a/lib/cusolver/test/runtests.jl b/lib/cusolver/test/runtests.jl index 882d936339..71059a3a71 100644 --- a/lib/cusolver/test/runtests.jl +++ b/lib/cusolver/test/runtests.jl @@ -1,11 +1,11 @@ -include("setup.jl") +using cuSOLVER +using ParallelTestRunner -@testset verbose=true "cuSOLVER" begin - for (root, _, files) in walkdir(@__DIR__) - for file in sort(files) - endswith(file, ".jl") || continue - file in ("setup.jl", "runtests.jl") && continue - include(joinpath(root, file)) - end - end +const init_code = quote + include(joinpath(@__DIR__, "setup.jl")) end + +testsuite = find_tests(@__DIR__) +delete!(testsuite, "setup") + +runtests(cuSOLVER, ARGS; init_code, testsuite) diff --git a/lib/cusparse/test/Project.toml b/lib/cusparse/test/Project.toml index 2774a290bb..84c09f67f0 100644 --- a/lib/cusparse/test/Project.toml +++ b/lib/cusparse/test/Project.toml @@ -4,6 +4,7 @@ CUDACore = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" SparseMatricesCSR = "a0a7dd2c-ebf4-11e9-1f05-cf50bc540ca1" @@ -11,3 +12,6 @@ SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" cuSPARSE = "b26da814-b3bc-49ef-b0ee-c816305aa060" + +[compat] +ParallelTestRunner = "2" diff --git a/lib/cusparse/test/array.jl b/lib/cusparse/test/array.jl index 7281f565e3..8d4ce19666 100644 --- a/lib/cusparse/test/array.jl +++ b/lib/cusparse/test/array.jl @@ -8,9 +8,10 @@ @test ndims(d_x) == 1 dense_d_x = CuVector(x) dense_d_x2 = CuVector(d_x) + ctx = :module => @__MODULE__ CUDACore.@allowscalar begin - @test sprint(show, d_x) == replace(sprint(show, x), "SparseVector{Float64, Int64}"=>"cuSPARSE.CuSparseVector{Float64, Int32}", "sparsevec(["=>"sparsevec(Int32[") - @test sprint(show, MIME"text/plain"(), d_x) == replace(sprint(show, MIME"text/plain"(), x), "SparseVector{Float64, Int64}"=>"CuSparseVector{Float64, Int32}", "sparsevec(["=>"sparsevec(Int32[") + @test sprint(show, d_x; context=ctx) == replace(sprint(show, x; context=ctx), "SparseVector{Float64, Int64}"=>"cuSPARSE.CuSparseVector{Float64, Int32}", "sparsevec(["=>"sparsevec(Int32[") + @test sprint(show, MIME"text/plain"(), d_x; context=ctx) == replace(sprint(show, MIME"text/plain"(), x; context=ctx), "SparseVector{Float64, Int64}"=>"CuSparseVector{Float64, Int32}", "sparsevec(["=>"sparsevec(Int32[") @test Array(d_x[:]) == x[:] @test d_x[firstindex(d_x)] == x[firstindex(x)] @test d_x[div(end, 2)] == x[div(end, 2)] @@ -45,8 +46,8 @@ @test size(d_x,3) == 1 @test ndims(d_x) == 2 CUDACore.@allowscalar begin - @test sprint(show, d_x) == sprint(show, SparseMatrixCSC(d_x)) - @test sprint(show, MIME"text/plain"(), d_x) == replace(sprint(show, MIME"text/plain"(), x), "SparseMatrixCSC{Float64, Int64}"=>"CuSparseMatrixCSC{Float64, Int32}") + @test sprint(show, d_x; context=ctx) == sprint(show, SparseMatrixCSC(d_x); context=ctx) + @test sprint(show, MIME"text/plain"(), d_x; context=ctx) == replace(sprint(show, MIME"text/plain"(), x; context=ctx), "SparseMatrixCSC{Float64, Int64}"=>"CuSparseMatrixCSC{Float64, Int32}") @test Array(d_x[:]) == x[:] @test d_x[:, :] == x[:, :] @test d_tx[:, :] == transpose(x)[:, :] @@ -247,7 +248,7 @@ # to hit the CuSparseArrayCSR path CUDACore.unsafe_free!(repeat(d_x, 1, 1, 3)) CUDACore.@allowscalar begin - @test startswith(sprint(show, MIME"text/plain"(), repeat(d_x, 1, 1, 2)), "$m×$n×2 CuSparseArrayCSR{Float64, Int32, 3} with $(2*nnz(d_x)) stored entries:\n") + @test startswith(sprint(show, MIME"text/plain"(), repeat(d_x, 1, 1, 2); context=ctx), "$m×$n×2 CuSparseArrayCSR{Float64, Int32, 3} with $(2*nnz(d_x)) stored entries:\n") end @test length(d_x) == m*n @test_throws ArgumentError copyto!(d_y,d_x) diff --git a/lib/cusparse/test/runtests.jl b/lib/cusparse/test/runtests.jl index ada5a8c918..486131663e 100644 --- a/lib/cusparse/test/runtests.jl +++ b/lib/cusparse/test/runtests.jl @@ -1,12 +1,11 @@ -include("setup.jl") -@test cuSPARSE.version() isa VersionNumber +using cuSPARSE +using ParallelTestRunner -@testset verbose=true "cuSPARSE" begin - for (root, _, files) in walkdir(@__DIR__) - for file in sort(files) - endswith(file, ".jl") || continue - file in ("setup.jl", "runtests.jl") && continue - include(joinpath(root, file)) - end - end +const init_code = quote + include(joinpath(@__DIR__, "setup.jl")) end + +testsuite = find_tests(@__DIR__) +delete!(testsuite, "setup") + +runtests(cuSPARSE, ARGS; init_code, testsuite) diff --git a/lib/custatevec/test/Project.toml b/lib/custatevec/test/Project.toml index 90a281b7a5..7ce6ba0b38 100644 --- a/lib/custatevec/test/Project.toml +++ b/lib/custatevec/test/Project.toml @@ -2,5 +2,9 @@ CUDACore = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" cuStateVec = "92f7fd98-d22e-4c0d-85a8-6ade11b672fb" + +[compat] +ParallelTestRunner = "2" diff --git a/lib/custatevec/test/runtests.jl b/lib/custatevec/test/runtests.jl index 11a6afd55b..9695abff71 100644 --- a/lib/custatevec/test/runtests.jl +++ b/lib/custatevec/test/runtests.jl @@ -1,13 +1,11 @@ -include("setup.jl") -@test cuStateVec.functional() +using cuStateVec +using ParallelTestRunner -@testset verbose=true "cuStateVec" begin - include("errors.jl") - include("apply_matrix.jl") - include("apply_pauli.jl") - include("measure.jl") - include("collapse.jl") - include("utilities.jl") +const init_code = quote + include(joinpath(@__DIR__, "setup.jl")) end -include("multigpu.jl") +testsuite = find_tests(@__DIR__) +delete!(testsuite, "setup") + +runtests(cuStateVec, ARGS; init_code, testsuite) diff --git a/lib/cutensor/test/Project.toml b/lib/cutensor/test/Project.toml index 38bfea9489..e261e0907f 100644 --- a/lib/cutensor/test/Project.toml +++ b/lib/cutensor/test/Project.toml @@ -2,6 +2,10 @@ CUDACore = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" cuTENSOR = "011b41b2-24ef-40a8-b3eb-fa098493e9e1" + +[compat] +ParallelTestRunner = "2" diff --git a/lib/cutensor/test/kernel_cache.jl b/lib/cutensor/test/kernel_cache.jl new file mode 100644 index 0000000000..be06dd17ce --- /dev/null +++ b/lib/cutensor/test/kernel_cache.jl @@ -0,0 +1,14 @@ +using cuTENSOR +using CUDACore + +if CUDACore.runtime_version() >= v"11.8" && capability(device()) >= v"8.0" + @testset "kernel cache" begin + mktempdir() do dir + cd(dir) do + cuTENSOR.write_cache!("kernelCache.bin") + @test isfile("kernelCache.bin") + cuTENSOR.read_cache!("kernelCache.bin") + end + end + end +end diff --git a/lib/cutensor/test/runtests.jl b/lib/cutensor/test/runtests.jl index 5a61ce3251..9ad51e4acf 100644 --- a/lib/cutensor/test/runtests.jl +++ b/lib/cutensor/test/runtests.jl @@ -1,27 +1,11 @@ -include("setup.jl") -@test cuTENSOR.functional() +using cuTENSOR +using ParallelTestRunner -@testset verbose=true "cuTENSOR" begin - -include("base.jl") - -include("elementwise_binary.jl") -include("elementwise_trinary.jl") -include("permutations.jl") -include("contractions.jl") -include("reductions.jl") - -# we should have some kernels in the cache after this -if CUDACore.runtime_version() >= v"11.8" && capability(device()) >= v"8.0" -@testset "kernel cache" begin - mktempdir() do dir - cd(dir) do - cuTENSOR.write_cache!("kernelCache.bin") - @test isfile("kernelCache.bin") - cuTENSOR.read_cache!("kernelCache.bin") - end - end -end +const init_code = quote + include(joinpath(@__DIR__, "setup.jl")) end -end +testsuite = find_tests(@__DIR__) +delete!(testsuite, "setup") + +runtests(cuTENSOR, ARGS; init_code, testsuite) diff --git a/lib/cutensornet/test/Project.toml b/lib/cutensornet/test/Project.toml index bc6ee7c689..02b63522e1 100644 --- a/lib/cutensornet/test/Project.toml +++ b/lib/cutensornet/test/Project.toml @@ -2,6 +2,10 @@ CUDACore = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" cuTENSOR = "011b41b2-24ef-40a8-b3eb-fa098493e9e1" cuTensorNet = "448d79b3-4b49-4e06-a5ea-00c62c0dc3db" + +[compat] +ParallelTestRunner = "2" diff --git a/lib/cutensornet/test/runtests.jl b/lib/cutensornet/test/runtests.jl index 9dca7d7fa1..f422497950 100644 --- a/lib/cutensornet/test/runtests.jl +++ b/lib/cutensornet/test/runtests.jl @@ -1,9 +1,11 @@ -include("setup.jl") -@test cuTENSOR.functional() -@test cuTensorNet.functional() +using cuTensorNet +using ParallelTestRunner -@testset verbose=true "cuTensorNet" begin - include("helpers.jl") - include("errors.jl") - include("contractions.jl") +const init_code = quote + include(joinpath(@__DIR__, "setup.jl")) end + +testsuite = find_tests(@__DIR__) +delete!(testsuite, "setup") + +runtests(cuTensorNet, ARGS; init_code, testsuite) From 39756267762dbbc0bf9af67328c403275d86e697 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 17 Apr 2026 16:10:28 +0200 Subject: [PATCH 02/13] Migrate top-level test runner to ParallelTestRunner. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrites `test/runtests.jl` on top of `ParallelTestRunner.runtests`, unifying CUDA.jl with the subpackages migrated in 94dedf5b5. The homegrown runner (~580 lines in runtests.jl + ~210 in setup.jl) becomes a thin wrapper plus a CUDA-specific `AbstractTestRecord`: - `CUDATestRecord` carries the standard fields plus `gpu_bytes`, `gpu_time`, and `gpu_rss`; an `execute(::Type{CUDATestRecord}, ...)` method uses `CUDA.@timed` to capture GPU alloc stats and queries NVML for per-process GPU RSS. `print_test_finished`/`print_test_failed` overrides add `GPU Alloc (MB)` and `GPU RSS (MB)` columns. - Worker count is capped by free GPU memory (~2 GiB/worker) in addition to PTR's CPU/RAM default. - `--sanitize[=tool]` wraps every worker by passing a compute-sanitizer `Cmd` as `runtests`'s `exename` kwarg (new in PTR 2.6). - `--all` (or an explicit `libraries/*` positional) includes subpackage tests under `lib/*/test/`, using `Base.set_active_project` to activate the subpackage's Project.toml. - Context-destroying tests (`core/initialization`, `core/cudadrv`) are isolated on a fresh worker via the `test_worker` hook and use plain Julia timing (since CUDA events invalidate with the context). Per-worker setup (`CUDATestRecord`, NVML helpers, GPUArrays TestSuite include, `CUDA.precompile_runtime`) lives in `test/setup.jl` and runs via `init_worker_code`. Per-test helpers (`testf`, `sink`, `@grab_output`, `@on_device`, `julia_exec`) are in a new `test/helpers.jl` included via `init_code`, so subpackage setup.jl's `testf` redefinitions don't clash with an imported binding. Drops: `--gpu=…` multi-device selection, exclusive-mode downgrade, interactive `?` key. GPU selection now goes through `CUDA_VISIBLE_DEVICES`. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/Project.toml | 1 + test/helpers.jl | 73 ++++++ test/runtests.jl | 639 +++++++++------------------------------------- test/setup.jl | 323 ++++++++++++----------- 4 files changed, 360 insertions(+), 676 deletions(-) create mode 100644 test/helpers.jl diff --git a/test/Project.toml b/test/Project.toml index 8bb025e557..da6f5274d8 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -20,6 +20,7 @@ KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" NVML = "611af6d1-644e-4c5d-bd58-854d7d1254b9" NVTX = "5da4648a-3479-48b8-97b9-01cb529c0a1f" +ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" diff --git a/test/helpers.jl b/test/helpers.jl new file mode 100644 index 0000000000..e01c729051 --- /dev/null +++ b/test/helpers.jl @@ -0,0 +1,73 @@ +# Per-test helpers, `include`d into each test's sandbox module. Kept separate +# from `setup.jl` (which runs once per worker and defines `CUDATestRecord`) +# so subpackage tests' own setup.jl can redefine `testf` without fighting a +# `using`-imported binding. + +using CUDA, CUDACore, GPUArrays +using CUDA: i32 +using Adapt +using ..Main: TestSuite, can_use_cupti, sanitize + +testf(f, xs...; kwargs...) = TestSuite.compare(f, CuArray, xs...; kwargs...) + +# NOTE: based on test/pkg.jl::capture_stdout, but doesn't discard exceptions +macro grab_output(ex) + quote + mktemp() do fname, fout + ret = nothing + open(fname, "w") do fout + redirect_stdout(fout) do + ret = $(esc(ex)) + + # NOTE: CUDA requires a 'proper' sync to flush its printf buffer + synchronize(context()) + end + end + ret, read(fname, String) + end + end +end + +# Run some code on-device +macro on_device(ex...) + code = ex[end] + kwargs = ex[1:end-1] + + @gensym kernel + esc(quote + let + function $kernel() + $code + return + end + + CUDA.@sync @cuda $(kwargs...) $kernel() + end + end) +end + +# helper function for sinking a value to prevent the callee from getting optimized away +@inline sink(i::Int32) = + Base.llvmcall("""%slot = alloca i32 + store volatile i32 %0, i32* %slot + %value = load volatile i32, i32* %slot + ret i32 %value""", Int32, Tuple{Int32}, i) +@inline sink(i::Int64) = + Base.llvmcall("""%slot = alloca i64 + store volatile i64 %0, i64* %slot + %value = load volatile i64, i64* %slot + ret i64 %value""", Int64, Tuple{Int64}, i) + +function julia_exec(args::Cmd, env...) + # FIXME: this doesn't work when the compute mode is set to exclusive + cmd = Base.julia_cmd() + cmd = `$cmd --project=$(Base.active_project()) --color=no $args` + + out = Pipe() + err = Pipe() + proc = run(pipeline(addenv(cmd, env...), stdout=out, stderr=err), wait=false) + close(out.in) + close(err.in) + wait(proc) + proc, read(out, String), read(err, String) +end diff --git a/test/runtests.jl b/test/runtests.jl index cc2e1d2119..ad6596494d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,580 +1,175 @@ -using Distributed -using InteractiveUtils -using Dates -import REPL -using Printf: @sprintf +using CUDA +using CUDACore +using cuBLAS, cuSPARSE, cuSOLVER, cuFFT, cuRAND +using cuDNN, cuTENSOR, cuTensorNet, cuStateVec +using GPUArrays +using ParallelTestRunner +using Pkg +using InteractiveUtils: versioninfo using Base.Filesystem: path_separator -# parse some command-line arguments -function extract_flag!(args, flag, default=nothing; typ=typeof(default)) - for f in args - if startswith(f, flag) - # Check if it's just `--flag` or if it's `--flag=foo` - if f != flag - val = split(f, '=')[2] - if !(typ === Nothing || typ <: AbstractString) - val = parse(typ, val) - end - else - val = default - end - - # Drop this value from our args - filter!(x -> x != f, args) - return (true, val) - end - end - return (false, default) -end -do_help, _ = extract_flag!(ARGS, "--help") -if do_help - println(""" - Usage: runtests.jl [--help] [--list] [--jobs=N] [--all] [TESTS...] +# load helpers, CUDATestRecord, and ParallelTestRunner overrides on the main +# process. The same file is re-`include`d on every worker via `init_worker_code` +# below; the record type must exist on both sides of the Malt boundary. +include(joinpath(@__DIR__, "setup.jl")) - --help Show this text. - --list List all available tests. - --all Also include subpackage tests (libraries/*). - --verbose Print more information during testing. - --quickfail Fail the entire run as soon as a single test errored. - --jobs=N Launch `N` processes to perform tests (default: Sys.CPU_THREADS). - --gpu=0,1,... Comma-separated list of GPUs to use (default: 0) - --sanitize[=tool] Run the tests under `compute-sanitizer`. +# parse command-line arguments (--sanitize[=tool] and --all are CUDA-specific) +args = parse_args(ARGS; custom = ["sanitize", "all"]) - Remaining arguments filter the tests that will be executed.""") - exit(0) -end -set_jobs, jobs = extract_flag!(ARGS, "--jobs"; typ=Int) -do_sanitize, sanitize_tool = extract_flag!(ARGS, "--sanitize", "memcheck") -do_verbose, _ = extract_flag!(ARGS, "--verbose") -do_quickfail, _ = extract_flag!(ARGS, "--quickfail") -do_all, _ = extract_flag!(ARGS, "--all") -do_gpu_list, gpu_list = extract_flag!(ARGS, "--gpu") -do_list, _ = extract_flag!(ARGS, "--list") -## no options should remain -optlike_args = filter(startswith("-"), ARGS) -if !isempty(optlike_args) - error("Unknown test options `$(join(optlike_args, " "))` (try `--help` for usage instructions)") +# check that CI is using the requested toolkit +toolkit_release = Base.thisminor(CUDA.runtime_version()) +label_match = match(r"^CUDA ([\d.]+)$", get(ENV, "BUILDKITE_LABEL", "")) +if label_match !== nothing + @test toolkit_release == VersionNumber(label_match.captures[1]) end -include("setup.jl") # make sure everything is precompiled +# forcibly precompile the current environment in parallel (Pkg somehow ignores +# the dependencies that are pointed through via [sources]) +Pkg.precompile() -# choose tests -const tests = [] -const test_runners = Dict() -## files in the test folder -for (rootpath, dirs, files) in walkdir(@__DIR__) - # find Julia files - filter!(files) do file - endswith(file, ".jl") && file !== "setup.jl" && file !== "runtests.jl" - end - isempty(files) && continue +@info "Julia information:\n" * sprint(io -> versioninfo(io)) +@info "CUDA information:\n" * sprint(io -> CUDA.versioninfo(io)) - # strip extension - files = map(files) do file - file[1:end-3] - end - # prepend subdir - subdir = relpath(rootpath, @__DIR__) - if subdir != "." - files = map(files) do file - joinpath(subdir, file) - end - end +## test discovery - # unify path separators - files = map(files) do file - replace(file, path_separator => '/') - end +# core tests from the top-level test directory, plus GPUArrays' testsuite. +testsuite = find_tests(@__DIR__) +delete!(testsuite, "setup") - append!(tests, files) - for file in files - test_runners[file] = ()->include("$(@__DIR__)/$file.jl") - end +# scalar-indexing tests only make sense when device memory supports it +for name in keys(TestSuite.tests) + testsuite["gpuarrays/$name"] = :(TestSuite.tests[$name](CuArray)) +end +if CUDACore.default_memory != CUDA.DeviceMemory + delete!(testsuite, "gpuarrays/indexing scalar") end -sort!(tests; by=(file)->stat("$(@__DIR__)/$file.jl").size, rev=true) -## subpackage tests (enabled with --all, or by selecting libraries/*) + +# subpackage tests under lib/*/test/ — include when requested via `--all` or +# by explicitly selecting a `libraries/*` positional. const subpackages = ["cublas", "cusparse", "cusolver", "cufft", "curand", "cudnn", "cutensor", "cutensornet", "custatevec"] -for pkg in subpackages - testdir = normpath(@__DIR__, "..", "lib", pkg, "test") - isdir(testdir) || continue - for (rootpath, dirs, files) in walkdir(testdir) - filter!(files) do file - endswith(file, ".jl") && file ∉ ("setup.jl", "runtests.jl") - end - isempty(files) && continue - for file in files - name = file[1:end-3] - subdir = relpath(rootpath, testdir) - if subdir != "." - name = joinpath(subdir, name) - end - name = replace(name, path_separator => '/') - testname = "libraries/$pkg/$name" - filepath = joinpath(rootpath, file) - setuppath = joinpath(testdir, "setup.jl") - projectpath = joinpath(testdir, "Project.toml") - push!(tests, testname) - test_runners[testname] = () -> begin +function include_subpackages!(testsuite) + for pkg in subpackages + testdir = normpath(@__DIR__, "..", "lib", pkg, "test") + isdir(testdir) || continue + sub_tests = find_tests(testdir) + delete!(sub_tests, "setup") + delete!(sub_tests, "runtests") + setuppath = joinpath(testdir, "setup.jl") + projectpath = joinpath(testdir, "Project.toml") + for (name, include_expr) in sub_tests + # include_expr is of the form `:(include($path))` + path = include_expr.args[end] + testsuite["libraries/$pkg/$name"] = quote old_project = Base.active_project() try - if isfile(projectpath) - Base.set_active_project(projectpath) + if isfile($projectpath) + Base.set_active_project($projectpath) end - include(setuppath) - include(filepath) + include($setuppath) + include($path) finally Base.set_active_project(old_project) end end end end + return testsuite end -## GPUArrays testsuite -for name in keys(TestSuite.tests) - pushfirst!(tests, "gpuarrays/$name") - test_runners["gpuarrays/$name"] = ()->TestSuite.tests[name](CuArray) + +want_all = args.custom["all"] !== nothing +want_libraries = any(startswith("libraries/"), args.positionals) +if want_all || want_libraries + include_subpackages!(testsuite) end -## finalize -pushfirst!(tests, "core/initialization") -unique!(tests) -# list tests, if requested -if do_list - println("Available tests:") - for test in sort(tests) - println(" - $test") +# default filter (no positionals given): package extensions require extra +# deps not in test/Project.toml, and subpackages are opt-in via `--all`. +if isempty(args.positionals) + filter!(testsuite) do (name, _) + if startswith(name, "extensions") + return false + end + if startswith(name, "libraries/") && !want_all + return false + end + return true end - exit(0) end -# filter tests -if isempty(ARGS) - # default to running all tests, except: - filter!(tests) do test - # package extensions often require additional dependencies, - # which we don't want to put in our test env by default. - startswith(test, "extensions") && return false - - # subpackage tests are only included with --all - if !do_all && startswith(test, "libraries/") - return false - end - if CUDACore.default_memory != CUDA.DeviceMemory && test == "gpuarrays/indexing scalar" - # GPUArrays' scalar indexing tests assume that indexing is not supported - return false - end +## GPU-memory-based parallelism - return true - end -else - # let the user filter - filter!(tests) do test - any(arg->startswith(test, arg), ARGS) - end +# pick the first visible device and use its free memory to cap worker count. +# (Set `CUDA_VISIBLE_DEVICES` to choose which device is used.) +first_gpu = first(devices()) +gpu_free = device!(first_gpu) do + mem = CUDA.free_memory() + device_reset!() + mem end +gpu_jobs = max(1, Int(gpu_free) ÷ (2 * 2^30)) -# check that CI is using the requested toolkit -toolkit_release = Base.thisminor(CUDA.runtime_version()) -label_match = match(r"^CUDA ([\d.]+)$", get(ENV, "BUILDKITE_LABEL", "")) -if label_match !== nothing - @test toolkit_release == VersionNumber(label_match.captures[1]) +if args.jobs === nothing + default_jobs = min(ParallelTestRunner.default_njobs(), gpu_jobs) + args = ParallelTestRunner.ParsedArgs( + Some(default_jobs), args.verbose, args.quickfail, args.list, + args.custom, args.positionals, + ) end -# forcibly precompile the current environment in parallel (Pkg somehow ignores -# the dependencies that are pointed through via [sources]) -using Pkg -Pkg.precompile() - -@info "Julia information:\n" * sprint(io->versioninfo(io)) - -using cuBLAS, cuSPARSE, cuSOLVER, cuFFT, cuRAND -using cuDNN, cuTENSOR, cuTensorNet, cuStateVec -@info "CUDA information:\n" * sprint(io->CUDA.versioninfo(io)) - -# select devices -function gpu_entry(dev) - id = deviceid(dev) - name = CUDA.name(dev) - uuid = CUDA.uuid(dev) - cap = capability(dev) - mig = uuid != parent_uuid(dev) - compute_mode = attribute(dev, CUDA.DEVICE_ATTRIBUTE_COMPUTE_MODE) - free_memory = device!(dev) do - mem = CUDA.free_memory() - device_reset!() - mem - end - (; id, name, cap, uuid="$(mig ? "MIG" : "GPU")-$uuid", compute_mode, free_memory) -end -gpus = if do_gpu_list - # parse the list of GPUs - map(split(gpu_list, ',')) do str - id = parse(Int, str) - gpu_entry(CuDevice(id)) - end -else - # pick the first non-exclusive GPU - entries = map(gpu_entry, devices()) - filter!(entry->entry.compute_mode != CUDA.COMPUTEMODE_PROHIBITED, entries) - first(entries, 1) -end -@info("Testing using device " * join(map(gpu->"$(gpu.id) ($(gpu.name))", gpus), ", ", " and ") * - ". To change this, specify the `--gpu` argument to the tests, or set the `CUDA_VISIBLE_DEVICES` environment variable.") -ENV["CUDA_VISIBLE_DEVICES"] = join(map(gpu->gpu.uuid, gpus), ",") -# determine parallelism -if !set_jobs - cpu_jobs = Sys.CPU_THREADS - cpu_memory_jobs = Int(Sys.free_memory()) ÷ (2 * 2^30) - gpu_memory_jobs = first(gpus).free_memory ÷ (2 * 2^30) - jobs = max(1, min(cpu_jobs, cpu_memory_jobs, gpu_memory_jobs)) -end -@info "Running $jobs tests in parallel. If this is too many, specify the `--jobs` argument to the tests, or set the `JULIA_CPU_THREADS` environment variable." -if first(gpus).compute_mode == CUDACore.COMPUTEMODE_EXCLUSIVE_PROCESS - @warn "Running tests on a GPU in exclusive mode; reducing parallelism to 1." - jobs = 1 -end +## compute-sanitizer wrapping (optional) -# install compute sanitizer -if do_sanitize +exename = nothing +if args.custom["sanitize"] !== nothing + tool_val = args.custom["sanitize"].value + tool = tool_val === nothing ? "memcheck" : tool_val # install CUDA_SDK_jll in a temporary environment project = Base.active_project() - Pkg.activate(; temp=true) + Pkg.activate(; temp = true) Pkg.add("CUDA_SDK_jll") - using CUDA_SDK_jll + @eval using CUDA_SDK_jll Pkg.activate(project) - compute_sanitizer = joinpath(CUDA_SDK_jll.artifact_dir, "cuda/compute-sanitizer/compute-sanitizer") + compute_sanitizer = joinpath(CUDA_SDK_jll.artifact_dir, + "cuda", "compute-sanitizer", "compute-sanitizer") if Sys.iswindows() compute_sanitizer *= ".exe" end @info "Running under " * read(`$compute_sanitizer --version`, String) + exename = `$compute_sanitizer --tool=$tool --launch-timeout=0 --target-processes=all --report-api-errors=no $(Base.julia_cmd()[1])` end -# add workers -const test_exeflags = Base.julia_cmd() -filter!(test_exeflags.exec) do c - return !(startswith(c, "--depwarn") || startswith(c, "--check-bounds")) -end -push!(test_exeflags.exec, "--check-bounds=yes") -push!(test_exeflags.exec, "--startup-file=no") -push!(test_exeflags.exec, "--depwarn=yes") -push!(test_exeflags.exec, "--project=$(Base.active_project())") -const test_exename = popfirst!(test_exeflags.exec) -function addworker(X; kwargs...) - exename = if do_sanitize - ```$compute_sanitizer --tool $sanitize_tool - --launch-timeout=0 - --target-processes=all - --report-api-errors=no - $test_exename``` - else - test_exename - end - withenv("JULIA_NUM_THREADS" => 1, "OPENBLAS_NUM_THREADS" => 1) do - procs = addprocs(X; exename=exename, exeflags=test_exeflags, kwargs...) - @everywhere procs include($(joinpath(@__DIR__, "setup.jl"))) - procs - end -end -addworker(min(jobs, length(tests))) +## worker setup -# pretty print information about gc and mem usage -testgroupheader = "Test" -workerheader = "(Worker)" -name_align = maximum([textwidth(testgroupheader) + textwidth(" ") + - textwidth(workerheader); map(x -> textwidth(x) + - 3 + ndigits(nworkers()), tests)]) -elapsed_align = textwidth("Time (s)") -gc_align = textwidth("GC (s)") -percent_align = textwidth("GC %") -alloc_align = textwidth("Alloc (MB)") -rss_align = textwidth("RSS (MB)") -printstyled(" "^(name_align + textwidth(testgroupheader) - 3), " | ") -printstyled(" | ---------------- GPU ---------------- | ---------------- CPU ---------------- |\n", color=:white) -printstyled(testgroupheader, color=:white) -printstyled(lpad(workerheader, name_align - textwidth(testgroupheader) + 1), " | ", color=:white) -printstyled("Time (s) | GC (s) | GC % | Alloc (MB) | RSS (MB) | GC (s) | GC % | Alloc (MB) | RSS (MB) |\n", color=:white) -print_lock = stdout isa Base.LibuvStream ? stdout.lock : ReentrantLock() -if stderr isa Base.LibuvStream - stderr.lock = print_lock +const init_worker_code = quote + include($(joinpath(@__DIR__, "setup.jl"))) end -function print_testworker_stats(test, wrkr, resp) - @nospecialize resp - lock(print_lock) - try - printstyled(test, color=:white) - printstyled(lpad("($wrkr)", name_align - textwidth(test) + 1, " "), " | ", color=:white) - time_str = @sprintf("%7.2f",resp[2]) - printstyled(lpad(time_str, elapsed_align, " "), " | ", color=:white) - - gpu_gc_str = @sprintf("%5.2f", resp[7]) - printstyled(lpad(gpu_gc_str, gc_align, " "), " | ", color=:white) - # since there may be quite a few digits in the percentage, - # the left-padding here is less to make sure everything fits - gpu_percent_str = @sprintf("%4.1f", 100 * resp[7] / resp[2]) - printstyled(lpad(gpu_percent_str, percent_align, " "), " | ", color=:white) - gpu_alloc_str = @sprintf("%5.2f", resp[6] / 2^20) - printstyled(lpad(gpu_alloc_str, alloc_align, " "), " | ", color=:white) - - gpu_rss_str = ismissing(resp[10]) ? "N/A" : @sprintf("%5.2f", resp[10] / 2^20) - printstyled(lpad(gpu_rss_str, rss_align, " "), " | ", color=:white) - - cpu_gc_str = @sprintf("%5.2f", resp[4]) - printstyled(lpad(cpu_gc_str, gc_align, " "), " | ", color=:white) - cpu_percent_str = @sprintf("%4.1f", 100 * resp[4] / resp[2]) - printstyled(lpad(cpu_percent_str, percent_align, " "), " | ", color=:white) - cpu_alloc_str = @sprintf("%5.2f", resp[3] / 2^20) - printstyled(lpad(cpu_alloc_str, alloc_align, " "), " | ", color=:white) - cpu_rss_str = @sprintf("%5.2f", resp[9] / 2^20) - printstyled(lpad(cpu_rss_str, rss_align, " "), " |\n", color=:white) - finally - unlock(print_lock) - end -end -global print_testworker_started = (name, wrkr)->begin - if do_sanitize || do_verbose - lock(print_lock) - try - printstyled(name, color=:white) - printstyled(lpad("($wrkr)", name_align - textwidth(name) + 1, " "), " |", - " "^elapsed_align, "started at $(now())\n", color=:white) - finally - unlock(print_lock) - end - end -end -function print_testworker_errored(name, wrkr) - lock(print_lock) - try - printstyled(name, color=:red) - printstyled(lpad("($wrkr)", name_align - textwidth(name) + 1, " "), " |", - " "^elapsed_align, " failed at $(now())\n", color=:red) - finally - unlock(print_lock) - end +const init_code = quote + include($(joinpath(@__DIR__, "helpers.jl"))) end -# run tasks -t0 = now() -results = [] -all_tasks = Task[] -try - # Monitor stdin and kill this task on ^C - # but don't do this on Windows, because it may deadlock in the kernel - t = current_task() - running_tests = Dict{String, DateTime}() - if !Sys.iswindows() && isa(stdin, Base.TTY) - stdin_monitor = @async begin - term = REPL.Terminals.TTYTerminal("xterm", stdin, stdout, stderr) - try - REPL.Terminals.raw!(term, true) - while true - c = read(term, Char) - if c == '\x3' - Base.throwto(t, InterruptException()) - break - elseif c == '?' - println("Currently running: ") - tests = sort(collect(running_tests), by=x->x[2]) - foreach(tests) do (test, date) - println(test, " (running for ", round(now()-date, Minute), ")") - end - end - end - catch e - isa(e, InterruptException) || rethrow() - finally - REPL.Terminals.raw!(term, false) - end - end +# `core/initialization` and `core/cudadrv` destroy the CUDA context mid-test. +# CUDA events become invalid once the context is gone, so these tests use plain +# Julia timing. We also isolate them on a fresh worker so subsequent tests +# aren't affected by the torn-down context. +const context_destroying_tests = ("core/initialization", "core/cudadrv") +function test_worker(name, init_worker_code) + if name in context_destroying_tests + return addworker(; exename, init_worker_code) end - @sync begin - function recycle_worker(p) - if isdefined(CUDA, :to) - to = remotecall_fetch(p) do - CUDA.to - end - push!(timings, to) - end - - rmprocs(p, waitfor=30) - - return nothing - end - - for p in workers() - @async begin - push!(all_tasks, current_task()) - while length(tests) > 0 - test = popfirst!(tests) - - # sometimes a worker failed, and we need to spawn a new one - if p === nothing - p = addworker(1)[1] - end - wrkr = p - - local resp - - # tests that muck with the context should not be timed with CUDA events, - # since they won't be valid at the end of the test anymore. - time_source = in(test, ["core/initialization", - "core/cudadrv"]) ? :julia : :cuda - - # run the test - running_tests[test] = now() - try - resp = remotecall_fetch(runtests, wrkr, test_runners[test], test, time_source) - catch e - isa(e, InterruptException) && return - resp = Any[e] - end - delete!(running_tests, test) - push!(results, (test, resp)) - - # act on the results - if resp[1] isa Exception - print_testworker_errored(test, wrkr) - do_quickfail && Base.throwto(t, InterruptException()) - - # the worker encountered some failure, recycle it - # so future tests get a fresh environment - p = recycle_worker(p) - else - print_testworker_stats(test, wrkr, resp) - end + return nothing +end - # resetting the context breaks certain CUDA libraries, - # so spawn a new worker when the test did so - if test in ["core/initialization", "core/cudadrv"] - p = recycle_worker(p) - end - end - if p !== nothing - recycle_worker(p) - end - end - end - end -catch e - isa(e, InterruptException) || rethrow() - # If the test suite was merely interrupted, still print the - # summary, which can be useful to diagnose what's going on - foreach(task -> begin - istaskstarted(task) || return - istaskdone(task) && return - try - schedule(task, InterruptException(); error=true) - catch ex - @error "InterruptException" exception=ex,catch_backtrace() - end - end, all_tasks) - for t in all_tasks - # NOTE: we can't just wait, but need to discard the exception, - # because the throwto for --quickfail also kills the worker. - try - wait(t) - catch e - showerror(stderr, e) - end - end -finally - if @isdefined stdin_monitor - schedule(stdin_monitor, InterruptException(); error=true) - end -end -t1 = now() -elapsed = canonicalize(Dates.CompoundPeriod(t1-t0)) -println("Testing finished in $elapsed") +## run -# construct a testset to render the test results -o_ts = Test.DefaultTestSet("Overall") -function with_testset(f, testset) - @static if VERSION >= v"1.13.0-DEV.1044" - Test.@with_testset testset f() - else - Test.push_testset(testset) - try - f() - finally - Test.pop_testset() - end - end -end -with_testset(o_ts) do - completed_tests = Set{String}() - for (testname, (resp,)) in results - push!(completed_tests, testname) - if isa(resp, Test.DefaultTestSet) - with_testset(resp) do - Test.record(o_ts, resp) - end - elseif isa(resp, Tuple{Int,Int}) - fake = Test.DefaultTestSet(testname) - for i in 1:resp[1] - Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing)) - end - for i in 1:resp[2] - Test.record(fake, Test.Broken(:test, nothing)) - end - with_testset(fake) do - Test.record(o_ts, fake) - end - elseif isa(resp, RemoteException) && isa(resp.captured.ex, Test.TestSetException) - println("Worker $(resp.pid) failed running test $(testname):") - Base.showerror(stdout, resp.captured) - println() - fake = Test.DefaultTestSet(testname) - for i in 1:resp.captured.ex.pass - Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing)) - end - for i in 1:resp.captured.ex.broken - Test.record(fake, Test.Broken(:test, nothing)) - end - for t in resp.captured.ex.errors_and_fails - Test.record(fake, t) - end - with_testset(fake) do - Test.record(o_ts, fake) - end - else - if !isa(resp, Exception) - resp = ErrorException(string("Unknown result type : ", typeof(resp))) - end - # If this test raised an exception that is not a remote testset exception, - # i.e. not a RemoteException capturing a TestSetException that means - # the test runner itself had some problem, so we may have hit a segfault, - # deserialization errors or something similar. Record this testset as Errored. - fake = Test.DefaultTestSet(testname) - Test.record(fake, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack([(exception=resp,backtrace=[])]), LineNumberNode(1))) - with_testset(fake) do - Test.record(o_ts, fake) - end - end - end - for test in tests - (test in completed_tests) && continue - fake = Test.DefaultTestSet(test) - Test.record(fake, Test.Error(:test_interrupted, test, nothing, Base.ExceptionStack([(exception="skipped",backtrace=[])]), LineNumberNode(1))) - with_testset(fake) do - Test.record(o_ts, fake) - end - end -end -println() -Test.print_test_results(o_ts, 1) -if (VERSION >= v"1.13.0-DEV.1037" && !Test.anynonpass(o_ts)) || - (VERSION < v"1.13.0-DEV.1037" && !o_ts.anynonpass) - println(" \033[32;1mSUCCESS\033[0m") -else - println(" \033[31;1mFAILURE\033[0m\n") - Test.print_test_errors(o_ts) - throw(Test.FallbackTestSetException("Test run finished with errors")) -end +runtests(CUDA, args; + testsuite, init_code, init_worker_code, test_worker, + RecordType = CUDATestRecord, + custom_args = (; julia_timed_tests = context_destroying_tests), + exename) diff --git a/test/setup.jl b/test/setup.jl index e72f229384..ef48797f24 100644 --- a/test/setup.jl +++ b/test/setup.jl @@ -1,23 +1,26 @@ -using Distributed, Test, CUDA -using CUDA: i32 +using Test +using CUDA +using CUDACore +using GPUArrays +using NVML: has_nvml, NVML +using ParallelTestRunner +using ParallelTestRunner: AbstractTestRecord, WorkerTestSet +using Test: DefaultTestSet +using Printf: @sprintf +using Random # ensure CUDA.jl is functional @assert CUDA.functional(true) -# GPUArrays has a testsuite that isn't part of the main package. -# Include it directly. -import GPUArrays -gpuarrays = pathof(GPUArrays) -gpuarrays_root = dirname(dirname(gpuarrays)) -include(joinpath(gpuarrays_root, "test", "testsuite.jl")) -testf(f, xs...; kwargs...) = TestSuite.compare(f, CuArray, xs...; kwargs...) - -using Random - -using Adapt +# GPUArrays has a testsuite that isn't part of the main package; include it +# directly. After this runs, module `TestSuite` is available to tests. +let gpuarrays = pathof(GPUArrays), + gpuarrays_root = dirname(dirname(gpuarrays)) + include(joinpath(gpuarrays_root, "test", "testsuite.jl")) +end if VERSION >= v"1.13.0-DEV.1044" -using Base.ScopedValues + using Base.ScopedValues end # detect compute-sanitizer, to disable incompatible tests (e.g. using CUPTI) @@ -32,7 +35,7 @@ function can_use_cupti() return false end - # Tegra requires running as root and and modifying the device tree + # Tegra requires running as root and modifying the device tree if CUDA.is_tegra() return false end @@ -44,164 +47,176 @@ end CUDA.precompile_runtime() -## entry point - -function runtests(f, name, time_source=:cuda) - function inner() - # generate a temporary module to execute the tests in - mod_name = Symbol("Test", rand(1:100), "Main_", replace(name, '/' => '_')) - mod = @eval(Main, module $mod_name end) - @eval(mod, using Test, Random, CUDA) +## custom test record capturing CUDA-specific statistics + +struct CUDATestRecord <: AbstractTestRecord + value::DefaultTestSet + time::Float64 + bytes::UInt64 + gctime::Float64 + compile_time::Float64 + rss::UInt64 + total_time::Float64 + # CUDA-specific: + gpu_bytes::UInt64 + gpu_time::Float64 + gpu_rss::Union{UInt64, Missing} +end - let id = myid() - wait(@spawnat 1 print_testworker_started(name, id)) +ParallelTestRunner.memory_usage(rec::CUDATestRecord) = rec.rss + +# GPU per-process memory via NVML. Returns `missing` in containers or on +# devices without NVML support. +function gpu_rss_nvml() + has_nvml() || return missing + cuda_dev = device() + mig = uuid(cuda_dev) != parent_uuid(cuda_dev) + nvml_dev = NVML.Device(uuid(cuda_dev); mig) + try + gpu_processes = NVML.compute_processes(nvml_dev) + if haskey(gpu_processes, getpid()) + return gpu_processes[getpid()].used_gpu_memory + else + return missing end + catch err + (isa(err, NVML.NVMLError) && err.code == NVML.ERROR_NOT_SUPPORTED) || rethrow() + return missing + end +end - ex = quote - GC.gc(true) - Random.seed!(1) - - if $(QuoteNode(time_source)) == :cuda - CUDA.@timed @testset $name begin - $f() - end - elseif $(QuoteNode(time_source)) == :julia - res = @timed @testset $name begin - $f() +function ParallelTestRunner.execute(::Type{CUDATestRecord}, mod::Module, f, name, + start_time, custom_args) + use_cuda_timing = !(name in custom_args.julia_timed_tests) + data = @eval mod begin + GC.gc(true) + Random.seed!(1) + if $use_cuda_timing + stats = CUDA.@timed @testset WorkerTestSet "placeholder" begin + @testset DefaultTestSet $name begin + $f end - res..., 0, 0, 0 - else - error("Unknown time source: " * $(string(time_source))) end - end - data = Core.eval(mod, ex) - #data[1] is the testset - - # process results - cpu_rss = Sys.maxrss() - cuda_dev = device() - gpu_rss = if has_nvml() - mig = uuid(cuda_dev) != parent_uuid(cuda_dev) - nvml_dev = NVML.Device(uuid(cuda_dev); mig) - try - gpu_processes = NVML.compute_processes(nvml_dev) - if haskey(gpu_processes, getpid()) - gpu_processes[getpid()].used_gpu_memory - else - # happens when we didn't do compute, or when using containers: - # https://github.com/NVIDIA/gpu-monitoring-tools/issues/63 - missing + (; testset = stats.value, stats.time, + bytes = UInt64(stats.cpu_bytes), gctime = Float64(stats.cpu_gctime), + compile_time = 0.0, + gpu_bytes = UInt64(stats.gpu_bytes), + gpu_time = Float64(stats.gpu_memtime)) + else + stats = @timed @testset WorkerTestSet "placeholder" begin + @testset DefaultTestSet $name begin + $f end - catch err - (isa(err, NVML.NVMLError) && err.code == NVML.ERROR_NOT_SUPPORTED) || rethrow() - missing end - else - missing - end - if VERSION >= v"1.11.0-DEV.1529" - tc = Test.get_test_counts(data[1]) - passes,fails,error,broken,c_passes,c_fails,c_errors,c_broken = - tc.passes, tc.fails, tc.errors, tc.broken, tc.cumulative_passes, - tc.cumulative_fails, tc.cumulative_errors, tc.cumulative_broken - else - passes,fails,errors,broken,c_passes,c_fails,c_errors,c_broken = - Test.get_test_counts(data[1]) + compile_time = @static VERSION >= v"1.11" ? stats.compile_time : 0.0 + (; testset = stats.value, stats.time, stats.bytes, stats.gctime, + compile_time, gpu_bytes = UInt64(0), gpu_time = 0.0) end - if data[1].anynonpass == false - data = ((passes+c_passes,broken+c_broken), - data[2], - data[3], - data[4], - data[5], - data[6], - data[7], - data[8]) - end - res = vcat(collect(data), cpu_rss, gpu_rss) - - GC.gc(true) - res end - @static if VERSION >= v"1.13.0-DEV.1044" - @with Test.TESTSET_PRINT_ENABLE=>false begin - inner() - end - else - old_print_setting = Test.TESTSET_PRINT_ENABLE[] - Test.TESTSET_PRINT_ENABLE[] = false - try - inner() - finally - Test.TESTSET_PRINT_ENABLE[] = old_print_setting - end - end + gpu_rss = use_cuda_timing ? gpu_rss_nvml() : missing + rss = Sys.maxrss() + record = CUDATestRecord( + data.testset, data.time, data.bytes, data.gctime, data.compile_time, + rss, time() - start_time, + data.gpu_bytes, data.gpu_time, gpu_rss, + ) + GC.gc(true) + return record end -## auxiliary stuff - -# NOTE: based on test/pkg.jl::capture_stdout, but doesn't discard exceptions -macro grab_output(ex) - quote - mktemp() do fname, fout - ret = nothing - open(fname, "w") do fout - redirect_stdout(fout) do - ret = $(esc(ex)) +## print overrides: extend the default layout with GPU columns + +const GPU_TIME_ALIGN = textwidth("GC (s)") +const GPU_ALLOC_ALIGN = textwidth("Alloc (MB)") +const GPU_RSS_ALIGN = textwidth("RSS (MB)") + +function ParallelTestRunner.print_header(::Type{CUDATestRecord}, ctx::ParallelTestRunner.TestIOContext, + testgroupheader, workerheader) + lock(ctx.lock) + try + # upper band + printstyled(ctx.stdout, " "^(ctx.name_align + textwidth(testgroupheader) - 3), " │ ", color = :white) + printstyled(ctx.stdout, " Test │", color = :white) + ctx.verbose && printstyled(ctx.stdout, " Init │", color = :white) + VERSION >= v"1.11" && ctx.verbose && printstyled(ctx.stdout, " Compile │", color = :white) + printstyled(ctx.stdout, " ─────────── GPU ─────────── │", color = :white) + printstyled(ctx.stdout, " ──────────────── CPU ──────────────── │\n", color = :white) + + # lower band + printstyled(ctx.stdout, testgroupheader, color = :white) + printstyled(ctx.stdout, lpad(workerheader, ctx.name_align - textwidth(testgroupheader) + 1), " │ ", color = :white) + printstyled(ctx.stdout, "time (s) │", color = :white) + ctx.verbose && printstyled(ctx.stdout, " time (s) │", color = :white) + VERSION >= v"1.11" && ctx.verbose && printstyled(ctx.stdout, " (%) │", color = :white) + printstyled(ctx.stdout, " GC (s) │ Alloc (MB) │ RSS (MB) │", color = :white) + printstyled(ctx.stdout, " GC (s) │ GC % │ Alloc (MB) │ RSS (MB) │\n", color = :white) + flush(ctx.stdout) + finally + unlock(ctx.lock) + end +end - # NOTE: CUDA requires a 'proper' sync to flush its printf buffer - synchronize(context()) - end - end - ret, read(fname, String) +function print_cuda_row(io::IO, record::CUDATestRecord, wrkr, test, ctx::ParallelTestRunner.TestIOContext; + color::Symbol = :white) + printstyled(io, test, color = color) + printstyled(io, lpad("($wrkr)", ctx.name_align - textwidth(test) + 1, " "), " │ ", color = color) + + time_str = @sprintf("%7.2f", record.time) + printstyled(io, lpad(time_str, ctx.elapsed_align, " "), " │ ", color = color) + + if ctx.verbose + init_time_str = @sprintf("%7.2f", record.total_time - record.time) + printstyled(io, lpad(init_time_str, ctx.elapsed_align, " "), " │ ", color = color) + if VERSION >= v"1.11" + ct = record.time > 0 ? 100 * record.compile_time / record.time : 0.0 + ct_str = @sprintf("%7.2f", Float64(ct)) + printstyled(io, lpad(ct_str, ctx.compile_align, " "), " │ ", color = color) end end -end -# Run some code on-device -macro on_device(ex...) - code = ex[end] - kwargs = ex[1:end-1] - - @gensym kernel - esc(quote - let - function $kernel() - $code - return - end + # GPU columns + gpu_time_str = @sprintf("%5.2f", record.gpu_time) + printstyled(io, lpad(gpu_time_str, GPU_TIME_ALIGN, " "), " │ ", color = color) + gpu_alloc_str = @sprintf("%5.2f", record.gpu_bytes / 2^20) + printstyled(io, lpad(gpu_alloc_str, GPU_ALLOC_ALIGN, " "), " │ ", color = color) + gpu_rss_str = ismissing(record.gpu_rss) ? "N/A" : @sprintf("%5.2f", record.gpu_rss / 2^20) + printstyled(io, lpad(gpu_rss_str, GPU_RSS_ALIGN, " "), " │ ", color = color) + + # CPU columns + gc_str = @sprintf("%5.2f", record.gctime) + printstyled(io, lpad(gc_str, ctx.gc_align, " "), " │ ", color = color) + pct = record.time > 0 ? 100 * record.gctime / record.time : 0.0 + pct_str = @sprintf("%4.1f", pct) + printstyled(io, lpad(pct_str, ctx.percent_align, " "), " │ ", color = color) + alloc_str = @sprintf("%5.2f", record.bytes / 2^20) + printstyled(io, lpad(alloc_str, ctx.alloc_align, " "), " │ ", color = color) + rss_str = @sprintf("%5.2f", record.rss / 2^20) + printstyled(io, lpad(rss_str, ctx.rss_align, " "), " │\n", color = color) +end - CUDA.@sync @cuda $(kwargs...) $kernel() - end - end) +function ParallelTestRunner.print_test_finished(record::CUDATestRecord, wrkr, test, + ctx::ParallelTestRunner.TestIOContext) + lock(ctx.lock) + try + print_cuda_row(ctx.stdout, record, wrkr, test, ctx; color = :white) + flush(ctx.stdout) + finally + unlock(ctx.lock) + end end -# helper function for sinking a value to prevent the callee from getting optimized away -@inline sink(i::Int32) = - Base.llvmcall("""%slot = alloca i32 - store volatile i32 %0, i32* %slot - %value = load volatile i32, i32* %slot - ret i32 %value""", Int32, Tuple{Int32}, i) -@inline sink(i::Int64) = - Base.llvmcall("""%slot = alloca i64 - store volatile i64 %0, i64* %slot - %value = load volatile i64, i64* %slot - ret i64 %value""", Int64, Tuple{Int64}, i) - -function julia_exec(args::Cmd, env...) - # FIXME: this doesn't work when the compute mode is set to exclusive - cmd = Base.julia_cmd() - cmd = `$cmd --project=$(Base.active_project()) --color=no $args` - - out = Pipe() - err = Pipe() - proc = run(pipeline(addenv(cmd, env...), stdout=out, stderr=err), wait=false) - close(out.in) - close(err.in) - wait(proc) - proc, read(out, String), read(err, String) +function ParallelTestRunner.print_test_failed(record::CUDATestRecord, wrkr, test, + ctx::ParallelTestRunner.TestIOContext) + lock(ctx.lock) + try + print_cuda_row(ctx.stderr, record, wrkr, test, ctx; color = :red) + flush(ctx.stderr) + finally + unlock(ctx.lock) + end end -nothing # File is loaded via a remotecall to "include". Ensure it returns "nothing". + +nothing # File is loaded via include; ensure it returns "nothing". From 719422090b60b6cc749c4cd754ae6e7df95349a2 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 17 Apr 2026 16:30:19 +0200 Subject: [PATCH 03/13] Route compute-sanitizer output via per-process log files. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A compute-sanitizer-wrapped worker starts by printing its banner ('========= COMPUTE-SANITIZER') to stdout, which collides with Malt's port handshake (the first stdout line must be parseable as a UInt16 port number). Passing `--log-file=/%p.log` redirects sanitizer text to a per-process file, leaving the worker's stdout clean for Malt. After `runtests` returns (or throws), scan the directory and surface any logs missing the "ERROR SUMMARY: 0 errors" line; emit a colored one-liner summary otherwise. This preserves the signal while keeping clean runs quiet. Also silence the `Pkg.activate`/`Pkg.add` chatter during CUDA_SDK_jll install (`io = devnull`) — the only output we want is the sanitizer version banner we explicitly print. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/runtests.jl | 59 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index ad6596494d..7905c5c006 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -123,15 +123,17 @@ end ## compute-sanitizer wrapping (optional) exename = nothing +sanitizer_log_dir = nothing if args.custom["sanitize"] !== nothing tool_val = args.custom["sanitize"].value tool = tool_val === nothing ? "memcheck" : tool_val - # install CUDA_SDK_jll in a temporary environment + # install CUDA_SDK_jll in a temporary environment so we only grab the + # tool for this run. project = Base.active_project() - Pkg.activate(; temp = true) - Pkg.add("CUDA_SDK_jll") + Pkg.activate(; temp = true, io = devnull) + Pkg.add("CUDA_SDK_jll"; io = devnull) @eval using CUDA_SDK_jll - Pkg.activate(project) + Pkg.activate(project, io = devnull) compute_sanitizer = joinpath(CUDA_SDK_jll.artifact_dir, "cuda", "compute-sanitizer", "compute-sanitizer") @@ -139,7 +141,13 @@ if args.custom["sanitize"] !== nothing compute_sanitizer *= ".exe" end @info "Running under " * read(`$compute_sanitizer --version`, String) - exename = `$compute_sanitizer --tool=$tool --launch-timeout=0 --target-processes=all --report-api-errors=no $(Base.julia_cmd()[1])` + # Route sanitizer output to per-process log files: its banner/reports on + # stdout would otherwise collide with Malt's port handshake on the worker. + # `%p` expands to the sanitizer process's PID. We surface relevant logs + # after the run finishes. + sanitizer_log_dir = mktempdir(prefix = "cuda-sanitizer-") + log_pattern = joinpath(sanitizer_log_dir, "%p.log") + exename = `$compute_sanitizer --tool=$tool --launch-timeout=0 --target-processes=all --report-api-errors=no --log-file=$log_pattern $(Base.julia_cmd()[1])` end @@ -168,8 +176,39 @@ end ## run -runtests(CUDA, args; - testsuite, init_code, init_worker_code, test_worker, - RecordType = CUDATestRecord, - custom_args = (; julia_timed_tests = context_destroying_tests), - exename) +function report_sanitizer_logs(log_dir) + log_dir === nothing && return + files = filter(endswith(".log"), readdir(log_dir; join = true)) + isempty(files) && return + total_workers = length(files) + flagged = String[] + for file in files + content = read(file, String) + if !occursin("ERROR SUMMARY: 0 errors", content) + push!(flagged, file) + end + end + println() + if isempty(flagged) + printstyled("compute-sanitizer: $total_workers workers checked, no errors\n"; + color = :green, bold = true) + else + printstyled("compute-sanitizer: $(length(flagged))/$total_workers workers reported issues:\n"; + color = :red, bold = true) + for file in flagged + println("\n--- ", basename(file), " ---") + print(read(file, String)) + end + end + println() +end + +try + runtests(CUDA, args; + testsuite, init_code, init_worker_code, test_worker, + RecordType = CUDATestRecord, + custom_args = (; julia_timed_tests = context_destroying_tests), + exename) +finally + report_sanitizer_logs(sanitizer_log_dir) +end From 300a0e3342b9d66808a6136af46e06c698bc883e Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 18 Apr 2026 09:39:17 +0200 Subject: [PATCH 04/13] Fix module look-up. --- test/core/device/intrinsics/wmma.jl | 10 +++++----- test/core/execution.jl | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/core/device/intrinsics/wmma.jl b/test/core/device/intrinsics/wmma.jl index 3295a62ded..f4ef5fdf4c 100644 --- a/test/core/device/intrinsics/wmma.jl +++ b/test/core/device/intrinsics/wmma.jl @@ -199,15 +199,15 @@ end shape = CUDA.WMMA.get_hl_shape(mnk[1], mnk[2], mnk[3]) # Get the function names - lda_func = getfield(Main, Symbol("llvm_wmma_load_a_$(a_layout)_$(shape)_global_stride_$(ab_elem_type)")) - ldb_func = getfield(Main, Symbol("llvm_wmma_load_b_$(b_layout)_$(shape)_global_stride_$(ab_elem_type)")) - ldc_func = getfield(Main, Symbol("llvm_wmma_load_c_col_$(shape)_global_stride_$(c_elem_type)")) + lda_func = getfield(@__MODULE__, Symbol("llvm_wmma_load_a_$(a_layout)_$(shape)_global_stride_$(ab_elem_type)")) + ldb_func = getfield(@__MODULE__, Symbol("llvm_wmma_load_b_$(b_layout)_$(shape)_global_stride_$(ab_elem_type)")) + ldc_func = getfield(@__MODULE__, Symbol("llvm_wmma_load_c_col_$(shape)_global_stride_$(c_elem_type)")) # Account for half and int/subint/bf16 mma different naming conventions # Int/subint and bf16 mma functions are distinguished by the a/b element type mma_sym = (d_ty == Int32 || ab_elem_type == "bf16") ? Symbol("llvm_wmma_mma_$(a_layout)_$(b_layout)_$(shape)_$(ab_elem_type)") : Symbol("llvm_wmma_mma_$(a_layout)_$(b_layout)_$(shape)_$(d_elem_type)_$(c_elem_type)") - mma_func = getfield(Main, mma_sym) - std_func = getfield(Main, Symbol("llvm_wmma_store_d_col_$(shape)_global_stride_$(d_elem_type)")) + mma_func = getfield(@__MODULE__, mma_sym) + std_func = getfield(@__MODULE__, Symbol("llvm_wmma_store_d_col_$(shape)_global_stride_$(d_elem_type)")) a_shape = get_array_shape("a", mnk, a_layout) a = rand(ab_ty, a_shape) diff --git a/test/core/execution.jl b/test/core/execution.jl index a37220cbf4..551cb9eef9 100644 --- a/test/core/execution.jl +++ b/test/core/execution.jl @@ -98,10 +98,10 @@ end export external_dummy external_dummy() = return end - import ...KernelModule + import .KernelModule @cuda KernelModule.external_dummy() @eval begin - using ...KernelModule + using .KernelModule @cuda external_dummy() end From c741a15e46a074b7f1323308cd203fd670383075 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 18 Apr 2026 09:56:20 +0200 Subject: [PATCH 05/13] Avoid binding warning. --- test/tools/reflection.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tools/reflection.jl b/test/tools/reflection.jl index e8a935272e..5362d32ef6 100644 --- a/test/tools/reflection.jl +++ b/test/tools/reflection.jl @@ -1,6 +1,6 @@ -dummy() = return - @testset "reflection" begin + dummy() = return + CUDA.code_lowered(dummy, Tuple{}) CUDA.code_typed(dummy, Tuple{}) CUDA.code_warntype(devnull, dummy, Tuple{}) From 3798fa8170bd646668cf85f65a997e0576cdf91a Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 18 Apr 2026 10:00:40 +0200 Subject: [PATCH 06/13] Reclaim pool-cached GPU memory between tests. Workers run many tests back-to-back, and pool-cached buffers stay resident because the release threshold is unbounded and the idle pool-cleanup task only runs when `isinteractive()`. Calling `CUDA.reclaim()` after the post-test GC trims the pool and empties library handle caches, reducing GPU RSS accumulation without invalidating compiled kernels. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/setup.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/setup.jl b/test/setup.jl index ef48797f24..eaba7852e8 100644 --- a/test/setup.jl +++ b/test/setup.jl @@ -122,6 +122,7 @@ function ParallelTestRunner.execute(::Type{CUDATestRecord}, mod::Module, f, name data.gpu_bytes, data.gpu_time, gpu_rss, ) GC.gc(true) + CUDA.reclaim() return record end From 5dd7b6e29e0abeb6c6658cb4817eab509bdfa4b5 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 18 Apr 2026 10:16:26 +0200 Subject: [PATCH 07/13] Wrap TestRecord in CUDATestRecord. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match ParallelTestRunner's new composition pattern for `AbstractTestRecord`: carry a `base::TestRecord` field and delegate Julia-timed execution to `ParallelTestRunner.execute(TestRecord, …)` instead of redeclaring every baseline field and re-implementing the non-CUDA timing path inline. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/setup.jl | 74 +++++++++++++++++++++------------------------------ 1 file changed, 30 insertions(+), 44 deletions(-) diff --git a/test/setup.jl b/test/setup.jl index eaba7852e8..a3be95a757 100644 --- a/test/setup.jl +++ b/test/setup.jl @@ -4,7 +4,7 @@ using CUDACore using GPUArrays using NVML: has_nvml, NVML using ParallelTestRunner -using ParallelTestRunner: AbstractTestRecord, WorkerTestSet +using ParallelTestRunner: AbstractTestRecord, TestRecord, WorkerTestSet using Test: DefaultTestSet using Printf: @sprintf using Random @@ -50,21 +50,12 @@ CUDA.precompile_runtime() ## custom test record capturing CUDA-specific statistics struct CUDATestRecord <: AbstractTestRecord - value::DefaultTestSet - time::Float64 - bytes::UInt64 - gctime::Float64 - compile_time::Float64 - rss::UInt64 - total_time::Float64 - # CUDA-specific: + base::TestRecord gpu_bytes::UInt64 gpu_time::Float64 gpu_rss::Union{UInt64, Missing} end -ParallelTestRunner.memory_usage(rec::CUDATestRecord) = rec.rss - # GPU per-process memory via NVML. Returns `missing` in containers or on # devices without NVML support. function gpu_rss_nvml() @@ -87,40 +78,34 @@ end function ParallelTestRunner.execute(::Type{CUDATestRecord}, mod::Module, f, name, start_time, custom_args) - use_cuda_timing = !(name in custom_args.julia_timed_tests) + # Context-destroying tests use plain Julia timing: CUDA events can't survive + # the context teardown. Delegate to PTR's default `execute` for those and + # wrap with empty GPU stats. + if name in custom_args.julia_timed_tests + base = ParallelTestRunner.execute(TestRecord, mod, f, name, start_time, custom_args) + return CUDATestRecord(base, UInt64(0), 0.0, missing) + end + data = @eval mod begin GC.gc(true) Random.seed!(1) - if $use_cuda_timing - stats = CUDA.@timed @testset WorkerTestSet "placeholder" begin - @testset DefaultTestSet $name begin - $f - end - end - (; testset = stats.value, stats.time, - bytes = UInt64(stats.cpu_bytes), gctime = Float64(stats.cpu_gctime), - compile_time = 0.0, - gpu_bytes = UInt64(stats.gpu_bytes), - gpu_time = Float64(stats.gpu_memtime)) - else - stats = @timed @testset WorkerTestSet "placeholder" begin - @testset DefaultTestSet $name begin - $f - end + stats = CUDA.@timed @testset WorkerTestSet "placeholder" begin + @testset DefaultTestSet $name begin + $f end - compile_time = @static VERSION >= v"1.11" ? stats.compile_time : 0.0 - (; testset = stats.value, stats.time, stats.bytes, stats.gctime, - compile_time, gpu_bytes = UInt64(0), gpu_time = 0.0) end + (; testset = stats.value, + stats.time, + cpu_bytes = UInt64(stats.cpu_bytes), + cpu_gctime = Float64(stats.cpu_gctime), + gpu_bytes = UInt64(stats.gpu_bytes), + gpu_time = Float64(stats.gpu_memtime)) end - gpu_rss = use_cuda_timing ? gpu_rss_nvml() : missing rss = Sys.maxrss() - record = CUDATestRecord( - data.testset, data.time, data.bytes, data.gctime, data.compile_time, - rss, time() - start_time, - data.gpu_bytes, data.gpu_time, gpu_rss, - ) + base = TestRecord(data.testset, data.time, data.cpu_bytes, data.cpu_gctime, + 0.0, rss, time() - start_time) + record = CUDATestRecord(base, data.gpu_bytes, data.gpu_time, gpu_rss_nvml()) GC.gc(true) CUDA.reclaim() return record @@ -161,17 +146,18 @@ end function print_cuda_row(io::IO, record::CUDATestRecord, wrkr, test, ctx::ParallelTestRunner.TestIOContext; color::Symbol = :white) + base = record.base printstyled(io, test, color = color) printstyled(io, lpad("($wrkr)", ctx.name_align - textwidth(test) + 1, " "), " │ ", color = color) - time_str = @sprintf("%7.2f", record.time) + time_str = @sprintf("%7.2f", base.time) printstyled(io, lpad(time_str, ctx.elapsed_align, " "), " │ ", color = color) if ctx.verbose - init_time_str = @sprintf("%7.2f", record.total_time - record.time) + init_time_str = @sprintf("%7.2f", base.total_time - base.time) printstyled(io, lpad(init_time_str, ctx.elapsed_align, " "), " │ ", color = color) if VERSION >= v"1.11" - ct = record.time > 0 ? 100 * record.compile_time / record.time : 0.0 + ct = base.time > 0 ? 100 * base.compile_time / base.time : 0.0 ct_str = @sprintf("%7.2f", Float64(ct)) printstyled(io, lpad(ct_str, ctx.compile_align, " "), " │ ", color = color) end @@ -186,14 +172,14 @@ function print_cuda_row(io::IO, record::CUDATestRecord, wrkr, test, ctx::Paralle printstyled(io, lpad(gpu_rss_str, GPU_RSS_ALIGN, " "), " │ ", color = color) # CPU columns - gc_str = @sprintf("%5.2f", record.gctime) + gc_str = @sprintf("%5.2f", base.gctime) printstyled(io, lpad(gc_str, ctx.gc_align, " "), " │ ", color = color) - pct = record.time > 0 ? 100 * record.gctime / record.time : 0.0 + pct = base.time > 0 ? 100 * base.gctime / base.time : 0.0 pct_str = @sprintf("%4.1f", pct) printstyled(io, lpad(pct_str, ctx.percent_align, " "), " │ ", color = color) - alloc_str = @sprintf("%5.2f", record.bytes / 2^20) + alloc_str = @sprintf("%5.2f", base.bytes / 2^20) printstyled(io, lpad(alloc_str, ctx.alloc_align, " "), " │ ", color = color) - rss_str = @sprintf("%5.2f", record.rss / 2^20) + rss_str = @sprintf("%5.2f", base.rss / 2^20) printstyled(io, lpad(rss_str, ctx.rss_align, " "), " │\n", color = color) end From d23acf453db890522015c800031800e62f058fb6 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 18 Apr 2026 10:57:05 +0200 Subject: [PATCH 08/13] Shrink high-RSS tests to fit the per-worker budget. Per-test GPU RSS data (collected after adding post-test CUDA.reclaim()) showed a handful of tests blowing well past the 4 GiB per-worker budget: - test/core/array.jl: the 512^3 sum!() case allocated ~1 GiB Float64 to exercise the big-mapreduce path; (85, 1320, 100) already exercises the same serial kernel path. Drop the 512^3 case. - test/core/sorting.jl: the "large sizes" quicksort input at 2^25 Float32 was 128 MiB; 2^22 still exercises the multi-block quicksort path. - examples/peakflops.jl: default n=5000 built four 5000x5000 Float32 matrices (~400 MiB); n=1024 is enough to demonstrate the example. - lib/cutensornet/test/contractions.jl: max_ws_size=2^32 (4 GiB workspace hint) was inflating cuTensorNet to ~1.5 GiB; 2^28 covers the same tuning paths. Library tests (cusolver/cusparse/cudnn/cutensor/etc.) still sit at 1-2 GiB due to persistent library workspace that's not pool-allocated and therefore not released by CUDA.reclaim() between tests. Co-Authored-By: Claude Opus 4.7 (1M context) --- examples/peakflops.jl | 2 +- lib/cutensornet/test/contractions.jl | 2 +- test/core/array.jl | 1 - test/core/sorting.jl | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/peakflops.jl b/examples/peakflops.jl index f5d742322b..7e8538f207 100644 --- a/examples/peakflops.jl +++ b/examples/peakflops.jl @@ -23,7 +23,7 @@ function kernel_100fma(a, b, c, out) return end -function peakflops(n::Integer=5000, dev::CuDevice=CuDevice(0)) +function peakflops(n::Integer=1024, dev::CuDevice=CuDevice(0)) device!(dev) do dims = (n, n) a = round.(rand(Float32, dims) * 100) diff --git a/lib/cutensornet/test/contractions.jl b/lib/cutensornet/test/contractions.jl index f4300ad7be..6449a02239 100644 --- a/lib/cutensornet/test/contractions.jl +++ b/lib/cutensornet/test/contractions.jl @@ -27,7 +27,7 @@ m = 16 aligns_in = UInt32.([256, 256, 256]) aligns_out = UInt32(256) ctn = CuTensorNetwork(elty, modes_in, extents_in, [C_NULL, C_NULL, C_NULL], Int32[0, 0, 0], Int32.(modesD), extentsD, C_NULL) - @testset for max_ws_size in [2^28, 2^32] + @testset for max_ws_size in [2^28] @testset for tuning in [NoAutoTune(), AutoTune()] ctn.input_arrs = raw_data_in info = rehearse_contraction(ctn, max_ws_size) diff --git a/test/core/array.jl b/test/core/array.jl index 73cc87947d..ae70eb227b 100644 --- a/test/core/array.jl +++ b/test/core/array.jl @@ -782,7 +782,6 @@ end @test test_cuda_sum!(32, 32, 32) @test test_cuda_sum!(256, 256, 256) - @test test_cuda_sum!(512, 512, 512) @test test_cuda_sum!(85, 1320, 100) end end diff --git a/test/core/sorting.jl b/test/core/sorting.jl index 91223e4fe2..012365fcd2 100644 --- a/test/core/sorting.jl +++ b/test/core/sorting.jl @@ -315,7 +315,7 @@ end @test check_sort!(Int32, (2, 2, 50000); dims=3, rev=true) # large sizes - @test check_sort!(Float32, 2^25; alg=CUDA.QuickSort) + @test check_sort!(Float32, 2^22; alg=CUDA.QuickSort) # various sync depths for depth in 0:4 From a214af735268728e0e1502a1b3800c913c76e2b0 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 18 Apr 2026 14:39:29 +0200 Subject: [PATCH 09/13] Move fresh-context init checks to a subprocess. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `@test_throws UndefRefError current_context()` / `current_device()` assertions at the top of initialization.jl require that CUDA hasn't been touched yet in the current Julia process. With PTR, every worker runs `setup.jl` as `init_worker_code`, and that already does `CUDA.functional(true)` / `precompile_runtime` / pool config — so the worker is never in a fresh state by the time the test runs, and these assertions fail ("Expected: UndefRefError, No exception thrown"). Run those four assertions (and the paired "now cause initialization" check) in a subprocess instead, the same way the issue-1331 test at the bottom of the file already does. The rest of initialization.jl doesn't depend on fresh state and runs fine on a normal worker. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/core/initialization.jl | 39 +++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/test/core/initialization.jl b/test/core/initialization.jl index 6ea9be306c..a2b4b597da 100644 --- a/test/core/initialization.jl +++ b/test/core/initialization.jl @@ -1,21 +1,36 @@ @test has_cuda(true) @test has_cuda_gpu(true) -# the API shouldn't have been initialized -@test_throws UndefRefError current_context() -@test_throws UndefRefError current_device() +# The "CUDA hasn't been initialized yet" assertions only hold in a Julia +# process that truly has a fresh runtime state, which isn't the case for us: +# ParallelTestRunner's `init_worker_code` runs `setup.jl`, which already +# touches `CUDA.functional`, `precompile_runtime`, the memory pool, etc. +# Run the fresh-state checks in a subprocess. +@testset "initialization semantics (subprocess)" begin + script = """ + using CUDA, Test + # the API shouldn't have been initialized + @test_throws UndefRefError current_context() + @test_throws UndefRefError current_device() -ctx = context() -dev = device() + ctx = context() + dev = device() + + # querying Julia's side of things shouldn't cause initialization + @test_throws UndefRefError current_context() + @test_throws UndefRefError current_device() -# querying Julia's side of things shouldn't cause initialization -@test_throws UndefRefError current_context() -@test_throws UndefRefError current_device() + # now cause initialization + a = CuArray([42]) + @test current_context() == ctx + @test current_device() == dev + """ + proc, _, _ = julia_exec(`-e $script`) + @test success(proc) +end -# now cause initialization -a = CuArray([42]) -@test current_context() == ctx -@test current_device() == dev +ctx = context() +dev = device() # ... on a different task task = @async begin From b65a01473004b258ff15a79c675692646fd4c269 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 18 Apr 2026 14:57:21 +0200 Subject: [PATCH 10/13] Fix GPU header width in CUDATestRecord print_header. The top-band GPU label spanned 30 columns but the bottom-band GPU cells (GC/Alloc/RSS) sum to 33, shifting every pipe after the GPU section three columns left. Widen the dashes (12 + 13) to match. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/setup.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/setup.jl b/test/setup.jl index a3be95a757..841e99a371 100644 --- a/test/setup.jl +++ b/test/setup.jl @@ -127,7 +127,7 @@ function ParallelTestRunner.print_header(::Type{CUDATestRecord}, ctx::ParallelTe printstyled(ctx.stdout, " Test │", color = :white) ctx.verbose && printstyled(ctx.stdout, " Init │", color = :white) VERSION >= v"1.11" && ctx.verbose && printstyled(ctx.stdout, " Compile │", color = :white) - printstyled(ctx.stdout, " ─────────── GPU ─────────── │", color = :white) + printstyled(ctx.stdout, " ──────────── GPU ───────────── │", color = :white) printstyled(ctx.stdout, " ──────────────── CPU ──────────────── │\n", color = :white) # lower band From 63a68c459e309ddb08d950dabbafb13e594bdd2d Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 18 Apr 2026 15:52:54 +0200 Subject: [PATCH 11/13] Bump ParallelTestRunner. --- test/Project.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/Project.toml b/test/Project.toml index da6f5274d8..b37fcf9ed4 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -55,3 +55,6 @@ cuSPARSE = {path = "../lib/cusparse"} cuStateVec = {path = "../lib/custatevec"} cuTENSOR = {path = "../lib/cutensor"} cuTensorNet = {path = "../lib/cutensornet"} + +[compat] +ParallelTestRunner = "2.6.0" From 489ca2204f09a3fc8d20a6bb2ebb0ac5eaf86829 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 20 Apr 2026 17:23:52 +0200 Subject: [PATCH 12/13] Remove pre-commit hook. --- .pre-commit-config.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 3e2823c366..0000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -repos: - - repo: https://github.com/fredrikekre/runic-pre-commit - rev: v2.0.1 - hooks: - - id: runic From 2aa23aa0afe7a7781c368f267fa359c77919ab4c Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 20 Apr 2026 17:24:00 +0200 Subject: [PATCH 13/13] Fix multigpu job. [only special] --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2aa73bacfd..530a884ab7 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -348,7 +348,7 @@ steps: - JuliaCI/julia#v1: version: "1.12" - JuliaCI/julia-test#v1: - test_args: "--gpu=0,1 --quickfail core libraries/cublas/xt" + test_args: "--quickfail core libraries/cublas/xt" - JuliaCI/julia-coverage#v1: dirs: - src