From 25f81d379a96c3cb75262f6df67ce1a42759e920 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Mon, 6 Apr 2026 09:13:00 +1200 Subject: [PATCH 1/5] add overloading of describe with tests and docstring --- Project.toml | 6 ++- src/MLJBase.jl | 7 ++- src/resampling/describe.jl | 91 ++++++++++++++++++++++++++++++++++++++ test/resampling.jl | 27 +++++++++-- 4 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 src/resampling/describe.jl diff --git a/Project.toml b/Project.toml index def08b91..3491b2df 100644 --- a/Project.toml +++ b/Project.toml @@ -1,12 +1,13 @@ name = "MLJBase" uuid = "a7f614a8-145f-11e9-1d2a-a57a1082229d" -authors = ["Anthony D. Blaom "] version = "1.12.1" +authors = ["Anthony D. Blaom "] [deps] CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" CategoricalDistributions = "af321ab8-2d2e-40a6-b165-3d674595d28e" ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3" +DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" @@ -17,6 +18,7 @@ InvertedIndices = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" LearnAPI = "92ad9a40-7767-427a-9ee6-6e577f1266cb" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" +Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" Missings = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a" @@ -43,12 +45,14 @@ DefaultMeasuresExt = "StatisticalMeasures" CategoricalArrays = "1" CategoricalDistributions = "0.2" ComputationalResources = "0.3" +DataAPI = "1.16.0" DelimitedFiles = "1" Distributions = "0.25.3" FillArrays = "1.14.0" InvertedIndices = "1" LearnAPI = "2" MLJModelInterface = "1.11" +Measurements = "2.14" Missings = "0.4, 1" OrderedCollections = "1.1" Parameters = "0.12" diff --git a/src/MLJBase.jl b/src/MLJBase.jl index c118507f..22bb354f 100644 --- a/src/MLJBase.jl +++ b/src/MLJBase.jl @@ -20,6 +20,7 @@ end import LearnAPI import StatisticalTraits.snakecase import StatisticalTraits.info +import Measurements # Interface @@ -66,7 +67,8 @@ import PrettyTables using DelimitedFiles using OrderedCollections using CategoricalArrays -import CategoricalArrays.DataAPI.unwrap +import DataAPI +import DataAPI: unwrap, describe import InvertedIndices: Not import Dates @@ -175,6 +177,7 @@ include("resampling/evaluation_results.jl") include("resampling/logging.jl") include("resampling/evaluate.jl") include("resampling/resampler.jl") +include("resampling/describe.jl") include("hyperparam/one_dimensional_ranges.jl") include("hyperparam/one_dimensional_range_methods.jl") @@ -303,7 +306,7 @@ export TransformedTargetModel # resampling.jl: export ResamplingStrategy, InSample, Holdout, CV, StratifiedCV, TimeSeriesCV, - evaluate!, Resampler, PerformanceEvaluation, CompactPerformanceEvaluation + evaluate!, Resampler, PerformanceEvaluation, CompactPerformanceEvaluation, describe # `MLJType` and the abstract `Model` subtypes are exported from within # src/composition/abstract_types.jl diff --git a/src/resampling/describe.jl b/src/resampling/describe.jl new file mode 100644 index 00000000..9639053a --- /dev/null +++ b/src/resampling/describe.jl @@ -0,0 +1,91 @@ +""" + individualize(iterator, delim="_") + +Given an `iterator` of abstract strings, return a vector of the same length, with numeric +suffixes that make the new elements unique, as in the following example: + +```julia-repl +julia> individualize(["cat", "dog", "cat", "mouse", "cat", "mouse"]) +6-element Vector{String}: + "cat_1" + "dog" + "cat_2" + "mouse_1" + "cat_3" + "mouse_2" +``` + +""" +function individualize(iterator, delim="_") + occurences_given_string = StatsBase.countmap(iterator) + d = deepcopy(occurences_given_string) + map(iterator) do s + occurences_given_string[s] == 1 && return s + digit = occurences_given_string[s] - d[s] + 1 + d[s] = d[s] - 1 + return "$s$delim$digit" + end +end + +""" + describe(evaluation::MLJBase.AbstractPerformanceEvaluation) + +Return a named tuple summarizing an MLJ performance evaluation, as returned by the methods +[`evaluate`](@ref) and [`evaluate!`](@ref). The summary includes all aggregated +measurements and their 95% radii of uncertainty. See also [`PerformanceEvaluation`](@ref). + +This is particularly useful for tabulating multiple evaluations, as shown in the following +example, which assumes you have MLJ, NearestNeighborModels, and DecisionTree in your +package environment. + +```julia-repl +using MLJ +X, y = @load_iris # a vector and a table + +# instantiate two models: +knn = (@load KNNClassifier pkg=NearestNeighborModels)() +tree = (@load DecisionTreeClassifier pkg=DecisionTree)() + +named_models = [ + "Dummy" => ConstantClassifier(), # a built-in model + "K-nearest neighbors" => knn, + "Decision Tree" => tree, +] +performance_evaluations = evaluate(named_models, X, y; measures=[brier_score, accuracy]) +julia> describe(performance_evaluations[1]) +(tag = "Dummy", BrierScore = -0.573 ± 0.1, Accuracy = 0.33 ± 0.23) + +table = describe.(performance_evaluations) +julia> pretty(table) +┌─────────────────────┬──────────────────────┬──────────────────────┐ +│ tag │ BrierScore │ Accuracy │ +│ String │ Measurement{Float64} │ Measurement{Float64} │ +│ Textual │ Continuous │ Continuous │ +├─────────────────────┼──────────────────────┼──────────────────────┤ +│ Dummy │ -0.573±0.1 │ 0.33±0.23 │ +│ K-nearest neighbors │ -0.21±0.21 │ 0.92±0.18 │ +│ Decision Tree │ -0.00118977±0.0 │ 1.0±0.0 │ +└─────────────────────┴──────────────────────┴──────────────────────┘ +``` + + +""" +function DataAPI.describe(e::AbstractPerformanceEvaluation) + key_value_pairs = Any[:tag=>e.tag] + measure_names = individualize( + map(e.measure) do measure + split(_repr_(measure), "(") |> first + end, + "", + ) + for (i, name) in enumerate(measure_names) + value = e.measurement[i] + δ = e.uncertainty_radius_95[i] + if !isnothing(δ) && δ isa Real && !isinf(δ) + # decorate with uncertainty radius: + value = Measurements.measurement(value, δ) + end + push!(key_value_pairs, Symbol(name) => value) + end + NamedTuple(key_value_pairs) +end diff --git a/test/resampling.jl b/test/resampling.jl index 48cb73cb..3f7ddbb2 100644 --- a/test/resampling.jl +++ b/test/resampling.jl @@ -8,6 +8,7 @@ using StatisticalMeasures import LearnAPI import CategoricalDistributions import MLJModelInterface +import Measurements @everywhere begin using .Models @@ -20,9 +21,7 @@ using Test using MLJBase import Distributions import StatsBase -@static if VERSION >= v"1.3.0-DEV.573" - using .Threads -end +using .Threads struct DummyInterval <: Interval end dummy_interval=DummyInterval() @@ -1262,4 +1261,26 @@ MLJModelInterface.target_scitype(::Type{<:UnivariateFiniteFitter}) = @test e.measurement[1] ≈ by_hand end +@test MLJBase.individualize(["cat", "dog", "cat", "mouse", "cat", "mouse"]) == + ["cat_1", "dog", "cat_2", "mouse_1", "cat_3", "mouse_2"] +@test MLJBase.individualize(["cat", "dog", "cat", "mouse", "cat", "mouse"], "") == + ["cat1", "dog", "cat2", "mouse1", "cat3", "mouse2"] + +@testset "describe" begin + X, y = make_moons(12) + y = coerce(y, OrderedFactor) + model = ConstantClassifier() + performance_evaluation = evaluate( + "const"=>model, X, y; + measures=[FScore(0.6), FScore(0.7), brier_loss, confmat], + ) + summary = describe(performance_evaluation) + @test keys(summary) == (:tag, :FScore1, :FScore2, :BrierLoss, :ConfusionMatrix) + @test summary.tag == "const" + bl = summary.BrierLoss + @test Measurements.value(bl) == performance_evaluation.measurement[3] + @test Measurements.uncertainty(bl) == performance_evaluation.uncertainty_radius_95[3] + @test summary.ConfusionMatrix == performance_evaluation.measurement[4] +end + true From 24d7d65c0b5fa8bc68c362dade8cc1b1025c1901 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Mon, 6 Apr 2026 11:37:55 +1200 Subject: [PATCH 2/5] fix docstring --- src/resampling/describe.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/resampling/describe.jl b/src/resampling/describe.jl index 9639053a..e235af91 100644 --- a/src/resampling/describe.jl +++ b/src/resampling/describe.jl @@ -43,8 +43,10 @@ using MLJ X, y = @load_iris # a vector and a table # instantiate two models: -knn = (@load KNNClassifier pkg=NearestNeighborModels)() -tree = (@load DecisionTreeClassifier pkg=DecisionTree)() +KNNClassifier = @load KNNClassifier pkg=NearestNeighborModels +DecisionTreeClassifier = @load DecisionTreeClassifier pkg=DecisionTree +knn = KNNClassifier() +tree = DecisionTreeClassifier() named_models = [ "Dummy" => ConstantClassifier(), # a built-in model From 24fa8bc426a0f7f21f2136a6738da1806d44520b Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Tue, 7 Apr 2026 14:29:52 +1200 Subject: [PATCH 3/5] refactor individuate to replace indivualize --- src/composition/models/pipelines.jl | 21 +------------ src/resampling/describe.jl | 33 ++----------------- src/utilities.jl | 47 ++++++++++++++++++++++++++++ test/composition/models/pipelines.jl | 5 --- test/resampling.jl | 7 +---- test/utilities.jl | 14 +++++++++ 6 files changed, 65 insertions(+), 62 deletions(-) diff --git a/src/composition/models/pipelines.jl b/src/composition/models/pipelines.jl index 4ba1e65a..31c124bb 100644 --- a/src/composition/models/pipelines.jl +++ b/src/composition/models/pipelines.jl @@ -12,25 +12,6 @@ # # HELPERS -# modify collection of symbols to guarantee uniqueness. For example, -# individuate([:x, :y, :x, :x]) = [:x, :y, :x2, :x3]) -function individuate(v) - isempty(v) && return v - ret = Symbol[first(v),] - for s in v[2:end] - s in ret || (push!(ret, s); continue) - n = 2 - candidate = s - while true - candidate = string(s, n) |> Symbol - candidate in ret || break - n += 1 - end - push!(ret, candidate) - end - return ret -end - function as_type(prediction_type::Symbol) if prediction_type == :deterministic return Deterministic @@ -151,7 +132,7 @@ function pipe_named_tuple(names, components) isempty(names) && throw(ERR_EMPTY_PIPELINE) # make keys unique: - names = names |> individuate |> Tuple + names = names |> MLJBase.individuate |> Tuple # check sequence: supervised_components = filter(components) do c diff --git a/src/resampling/describe.jl b/src/resampling/describe.jl index e235af91..b5ec4082 100644 --- a/src/resampling/describe.jl +++ b/src/resampling/describe.jl @@ -1,32 +1,3 @@ -""" - individualize(iterator, delim="_") - -Given an `iterator` of abstract strings, return a vector of the same length, with numeric -suffixes that make the new elements unique, as in the following example: - -```julia-repl -julia> individualize(["cat", "dog", "cat", "mouse", "cat", "mouse"]) -6-element Vector{String}: - "cat_1" - "dog" - "cat_2" - "mouse_1" - "cat_3" - "mouse_2" -``` - -""" -function individualize(iterator, delim="_") - occurences_given_string = StatsBase.countmap(iterator) - d = deepcopy(occurences_given_string) - map(iterator) do s - occurences_given_string[s] == 1 && return s - digit = occurences_given_string[s] - d[s] + 1 - d[s] = d[s] - 1 - return "$s$delim$digit" - end -end - """ describe(evaluation::MLJBase.AbstractPerformanceEvaluation) @@ -74,11 +45,11 @@ julia> pretty(table) """ function DataAPI.describe(e::AbstractPerformanceEvaluation) key_value_pairs = Any[:tag=>e.tag] - measure_names = individualize( + measure_names = MLJBase.individuate( map(e.measure) do measure split(_repr_(measure), "(") |> first end, - "", + delim="", ) for (i, name) in enumerate(measure_names) value = e.measurement[i] diff --git a/src/utilities.jl b/src/utilities.jl index 44a1856a..9e2e02d6 100644 --- a/src/utilities.jl +++ b/src/utilities.jl @@ -555,3 +555,50 @@ converted to a vector. """ guess_model_target_observation_scitype(model) = observation(target_scitype(model)) + + +# ## MAKING A COLLECTION OF STRINGS OR SYMBOLS UNIQUE + +append_digit(s::AbstractString, delim, n) = string(s, delim, n) +append_digit(s::Symbol, args...) = Symbol(append_digit(string(s), args...)) + +""" + MLJBase.individuate(collection, delim="", use_one=false) + +Given a `collection` of strings or symbols, add numeric suffixes to some elements, to +distinguish repeated elements, and return these unique elements as a vector of the same +length as `collection`: + +```julia-repl +julia> collection = ["cat", "dog", "cat", "mouse", "cat", "mouse"] +julia> MLJBase.individuate(collection) + "cat" + "dog" + "cat2" + "mouse" + "cat3" + "mouse2" + +julia> MLJBase.individuate(collection, delim="_", use_one=true) +6-element Vector{String}: + "cat_1" + "dog" + "cat_2" + "mouse_1" + "cat_3" + "mouse_2" + +``` + +""" +function individuate(iterator; delim="", use_one=false) + occurences_given_string = StatsBase.countmap(iterator) + d = deepcopy(occurences_given_string) + map(iterator) do s + occurences_given_string[s] == 1 && return s + digit = occurences_given_string[s] - d[s] + 1 + d[s] = d[s] - 1 + digit == 1 && !use_one && return s + return append_digit(s, delim, digit) + end +end diff --git a/test/composition/models/pipelines.jl b/test/composition/models/pipelines.jl index c90143a7..cc2fddbe 100644 --- a/test/composition/models/pipelines.jl +++ b/test/composition/models/pipelines.jl @@ -10,11 +10,6 @@ import MLJBase: Pred, Trans rng = StableRNG(698790187) -@testset "helpers" begin - @test MLJBase.individuate([:x, :y, :x, :z, :y, :x]) == - [:x, :y, :x2, :z, :y2, :x3] -end - # # DUMMY MODELS diff --git a/test/resampling.jl b/test/resampling.jl index 3f7ddbb2..13e365d8 100644 --- a/test/resampling.jl +++ b/test/resampling.jl @@ -1261,11 +1261,6 @@ MLJModelInterface.target_scitype(::Type{<:UnivariateFiniteFitter}) = @test e.measurement[1] ≈ by_hand end -@test MLJBase.individualize(["cat", "dog", "cat", "mouse", "cat", "mouse"]) == - ["cat_1", "dog", "cat_2", "mouse_1", "cat_3", "mouse_2"] -@test MLJBase.individualize(["cat", "dog", "cat", "mouse", "cat", "mouse"], "") == - ["cat1", "dog", "cat2", "mouse1", "cat3", "mouse2"] - @testset "describe" begin X, y = make_moons(12) y = coerce(y, OrderedFactor) @@ -1275,7 +1270,7 @@ end measures=[FScore(0.6), FScore(0.7), brier_loss, confmat], ) summary = describe(performance_evaluation) - @test keys(summary) == (:tag, :FScore1, :FScore2, :BrierLoss, :ConfusionMatrix) + @test keys(summary) == (:tag, :FScore, :FScore2, :BrierLoss, :ConfusionMatrix) @test summary.tag == "const" bl = summary.BrierLoss @test Measurements.value(bl) == performance_evaluation.measurement[3] diff --git a/test/utilities.jl b/test/utilities.jl index 1548e7b1..fec3bc53 100644 --- a/test/utilities.jl +++ b/test/utilities.jl @@ -221,5 +221,19 @@ MLJBase.target_scitype(::Type{<:DRegressor2}) = @test !contains(str, "Int64") end +@testset "individuate" begin + @test MLJBase.individuate([:x, :y, :x, :z, :y, :x]) == + [:x, :y, :x2, :z, :y2, :x3] + @test MLJBase.individuate([:x, :y, :x, :z, :y, :x], delim="_") == + [:x, :y, :x_2, :z, :y_2, :x_3] + @test MLJBase.individuate( + ["cat", "dog", "cat", "mouse", "cat", "mouse"], + use_one=true, + delim="_", + ) == ["cat_1", "dog", "cat_2", "mouse_1", "cat_3", "mouse_2"] + @test MLJBase.individuate(["cat", "dog", "cat", "mouse", "cat", "mouse"]) == + ["cat", "dog", "cat2", "mouse", "cat3", "mouse2"] +end + end # module true From 57e2787b6f854750905d39abd6ac0b4c0224502d Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Mon, 13 Apr 2026 08:51:05 +1200 Subject: [PATCH 4/5] update resampling docstring --- src/resampling/evaluate.jl | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/resampling/evaluate.jl b/src/resampling/evaluate.jl index 608086a9..467bbc1f 100644 --- a/src/resampling/evaluate.jl +++ b/src/resampling/evaluate.jl @@ -265,11 +265,12 @@ log_evaluation(logger, performance_evaluation) = nothing Estimate the performance of a machine `mach` wrapping a supervised model in data, using the specified `resampling` strategy (defaulting to 6-fold cross-validation) and `measure`, -which can be a single measure or vector. Returns a [`PerformanceEvaluation`](@ref) -object. +which can be a single measure or vector. Returns a [`PerformanceEvaluation`](@ref) or +[`CompactPerformanceEvaluation`](@ref) object. To obtain a brief named tuple summary of +this object, suitable for tabulation, apply [`describe`](@ref). In place of `mach`, one can use `tag_string => mach`, or a vector of either of these forms, -to return a vector of performance evaluation objects. +to return a vector of performance evaluation objects. Available resampling strategies are $RESAMPLING_STRATEGIES_LIST. If `resampling` is not an instance of one of these, then a vector of tuples of the form `(train_rows, test_rows)` @@ -374,18 +375,32 @@ show(e) Evaluate multiple machines: -```julia +```julia-repl @load KNNClassifier pkg=NearestNeighborModels mach1 = machine(ConstantClassifier(), X, y) mach2 = machine(KNNClassifier(), X , y) -evaluate!(["const" => mach1, "knn" => mach2]) -# 2-element Vector{...} -# PerformanceEvaluation("const", 0.698 ± 0.0062) -# PerformanceEvaluation("knn", 2.22e-16 ± 0.0) +julia> performance_evaluations = evaluate!(["const" => mach1, "knn" => mach2]) +2-element Vector{...} + PerformanceEvaluation("const", 0.698 ± 0.0062) + PerformanceEvaluation("knn", 2.22e-16 ± 0.0) +``` + +Tabulate the results: + +```julia-repl +describe.(performance_evaluations) |> pretty # or `|> DataFrames.DataFrame` +┌─────────┬──────────────────────┐ +│ tag │ LogLoss │ +│ String │ Measurement{Float64} │ +│ Textual │ Continuous │ +├─────────┼──────────────────────┤ +│ const │ 0.6977±0.0062 │ +│ knn │ 2.22045e-16±0.0 │ +└─────────┴──────────────────────┘ ``` See also [`evaluate`](@ref), [`PerformanceEvaluation`](@ref), -[`CompactPerformanceEvaluation`](@ref). +[`CompactPerformanceEvaluation`](@ref), [`describe`](@ref). """ function evaluate!( From 1d84f4e40f1fdcbc6306ef35e0c5c22933d9540a Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Mon, 13 Apr 2026 08:52:12 +1200 Subject: [PATCH 5/5] bump 1.13.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 3491b2df..d581eca0 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "MLJBase" uuid = "a7f614a8-145f-11e9-1d2a-a57a1082229d" -version = "1.12.1" +version = "1.13.0" authors = ["Anthony D. Blaom "] [deps]