From e56c7543a8f0e09ef7f21d64d77df6a11df5c2a0 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sat, 18 Jul 2026 14:51:07 +0200 Subject: [PATCH 1/3] Fix as_forecast_quantile() from samples for asymmetric probs (#1196) as_forecast_quantile.forecast_sample() built the quantile_level column from a symmetrised vector unique(round(c(probs, 1 - probs), 10)) while computing the quantiles at the raw probs, a refactoring regression from 4d9ecf75 (#876). For asymmetric probs this silently paired quantile values with the wrong levels (via length-1 recycling) or errored; for duplicated probs it triggered a duplicate-forecast assertion. Quantiles are now computed at exactly the requested (deduplicated) probs, matching the documentation. Symmetric prob sets, including the default, produce byte-identical output to before. probs is now also bounded to [0, 1] for a clearer early error. Co-Authored-By: Claude Fable 5 --- NEWS.md | 1 + R/class-forecast-sample.R | 8 +-- tests/testthat/test-class-forecast-quantile.R | 72 +++++++++++++++++++ 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index e1ee239eb..a2d078c19 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ # scoringutils (development version) +- Fixed `as_forecast_quantile()` for sample-based forecasts producing silently wrong quantiles or erroring when `probs` was not symmetric around 0.5 (e.g. `probs = 0.4` or `probs = c(0.1, 0.2)`). Quantiles are now computed at exactly the requested `probs` (deduplicated), and out-of-range `probs` produce a clear assertion error (#1196). - Added `filter_scores()` and `impute_missing_scores()` for handling missing forecasts before summarisation. `filter_scores()` removes target combinations with insufficient model coverage, while `impute_missing_scores()` fills in missing scores using configurable strategies (worst, mean, NA, or reference model). Both use a strategy function pattern for extensibility. See `vignette("handling-missing-forecasts")` for details (#1122). - Added `plot_discrimination()` to visualise the discrimination ability of binary forecasts by plotting the distribution of predicted probabilities, stratified by the observed outcome. The function requires a `forecast_binary` object (created with `as_forecast_binary()`) (#942). - Fixed `summarise_scores()` producing a data.table with duplicate column names when the input `scores` object had no score columns (e.g. because every metric in `score()` warned and returned nothing). `summarise_scores()` now matches metric columns by exact name rather than regex partial match, and errors with a clear message when there is nothing to summarise (#1179). diff --git a/R/class-forecast-sample.R b/R/class-forecast-sample.R index 27c1f653f..c0fde70cd 100644 --- a/R/class-forecast-sample.R +++ b/R/class-forecast-sample.R @@ -103,17 +103,15 @@ as_forecast_quantile.forecast_sample <- function( ... ) { forecast <- as.data.table(data) - assert_numeric(probs, min.len = 1) + assert_numeric(probs, min.len = 1, lower = 0, upper = 1) reserved_columns <- c("predicted", "sample_id") by <- setdiff(colnames(forecast), reserved_columns) - quantile_level <- unique( - round(c(probs, 1 - probs), digits = 10) - ) + quantile_level <- unique(round(probs, digits = 10)) forecast <- forecast[, .(quantile_level = quantile_level, - predicted = quantile(x = predicted, probs = ..probs, + predicted = quantile(x = predicted, probs = ..quantile_level, type = ..type, na.rm = TRUE)), by = by] diff --git a/tests/testthat/test-class-forecast-quantile.R b/tests/testthat/test-class-forecast-quantile.R index 9a372bec6..1f5bfd5b0 100644 --- a/tests/testthat/test-class-forecast-quantile.R +++ b/tests/testthat/test-class-forecast-quantile.R @@ -148,6 +148,78 @@ test_that("as_forecast_quantiles works", { ) }) +test_that("as_forecast_quantiles handles a single asymmetric prob", { + samples <- data.table( + model = "model1", + observed = 50, + predicted = 1:100, + sample_id = 1:100 + ) |> + as_forecast_sample() + + quantile <- as_forecast_quantile(samples, probs = 0.4) + + expect_identical(nrow(quantile), 1L) + expect_identical(quantile$quantile_level, 0.4) + expect_identical( + quantile$predicted, + unname(quantile(1:100, probs = 0.4, type = 7)) + ) +}) + +test_that("as_forecast_quantiles handles multiple asymmetric probs", { + samples <- data.table( + model = "model1", + observed = 50, + predicted = 1:100, + sample_id = 1:100 + ) |> + as_forecast_sample() + + expect_no_condition( + quantile <- as_forecast_quantile(samples, probs = c(0.1, 0.2)) + ) + expect_identical(quantile$quantile_level, c(0.1, 0.2)) + expect_identical( + quantile$predicted, + unname(quantile(1:100, probs = c(0.1, 0.2), type = 7)) + ) +}) + +test_that("as_forecast_quantiles deduplicates repeated probs", { + samples <- data.table( + model = "model1", + observed = 50, + predicted = 1:100, + sample_id = 1:100 + ) |> + as_forecast_sample() + + quantile <- as_forecast_quantile(samples, probs = c(0.5, 0.5)) + + expect_identical(nrow(quantile), 1L) + expect_identical(quantile$quantile_level, 0.5) + expect_identical( + quantile$predicted, + unname(quantile(1:100, probs = 0.5, type = 7)) + ) +}) + +test_that("as_forecast_quantiles errors on out-of-range probs", { + samples <- data.table( + model = "model1", + observed = 50, + predicted = 1:100, + sample_id = 1:100 + ) |> + as_forecast_sample() + + expect_error( + as_forecast_quantile(samples, probs = c(0.5, 1.5)), + "Assertion on 'probs' failed" + ) +}) + test_that("as_forecast_quantiles issue 557 fix", { out <- example_sample_discrete |> na.omit() |> From d49191520b66b71ac01313cd2e79e7e6b41edd44 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sat, 18 Jul 2026 15:26:37 +0200 Subject: [PATCH 2/3] Add ..quantile_level to globalVariables to fix lint Co-Authored-By: Claude Fable 5 --- R/z-globalVariables.R | 1 + 1 file changed, 1 insertion(+) diff --git a/R/z-globalVariables.R b/R/z-globalVariables.R index 99f3a7b51..d84d33024 100644 --- a/R/z-globalVariables.R +++ b/R/z-globalVariables.R @@ -6,6 +6,7 @@ globalVariables(c( "..forecast_unit", "..index", "..probs", + "..quantile_level", "..samplecols", "..type", ".BY", From 5cacd5833d4cc58bf2112c8781cb5f036a9a3c5b Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sun, 19 Jul 2026 09:31:57 +0200 Subject: [PATCH 3/3] Reject missing values in probs in as_forecast_quantile() Follow-up to the review of the asymmetric-probs fix (#1196): probs containing NA (e.g. c(0.5, NA)) previously passed the assert_numeric() bounds check (checkmate bounds ignore NAs) and silently produced rows with quantile_level = NA / predicted = NA. Add any.missing = FALSE so this now fails with a clear assertion error, plus a regression test. Co-Authored-By: Claude Fable 5 --- R/class-forecast-sample.R | 2 +- tests/testthat/test-class-forecast-quantile.R | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/R/class-forecast-sample.R b/R/class-forecast-sample.R index c0fde70cd..6c9be0b25 100644 --- a/R/class-forecast-sample.R +++ b/R/class-forecast-sample.R @@ -103,7 +103,7 @@ as_forecast_quantile.forecast_sample <- function( ... ) { forecast <- as.data.table(data) - assert_numeric(probs, min.len = 1, lower = 0, upper = 1) + assert_numeric(probs, min.len = 1, lower = 0, upper = 1, any.missing = FALSE) reserved_columns <- c("predicted", "sample_id") by <- setdiff(colnames(forecast), reserved_columns) diff --git a/tests/testthat/test-class-forecast-quantile.R b/tests/testthat/test-class-forecast-quantile.R index 1f5bfd5b0..8db02a1b7 100644 --- a/tests/testthat/test-class-forecast-quantile.R +++ b/tests/testthat/test-class-forecast-quantile.R @@ -220,6 +220,21 @@ test_that("as_forecast_quantiles errors on out-of-range probs", { ) }) +test_that("as_forecast_quantiles errors on missing values in probs", { + samples <- data.table( + model = "model1", + observed = 50, + predicted = 1:100, + sample_id = 1:100 + ) |> + as_forecast_sample() + + expect_error( + as_forecast_quantile(samples, probs = c(0.5, NA)), + "Assertion on 'probs' failed" + ) +}) + test_that("as_forecast_quantiles issue 557 fix", { out <- example_sample_discrete |> na.omit() |>