From 48d9321d16706db8fa953226747b5a417b8740e3 Mon Sep 17 00:00:00 2001 From: Hugo Gruson Date: Fri, 15 May 2026 16:43:31 +0200 Subject: [PATCH 1/8] S3 reading support --- R/read.R | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/R/read.R b/R/read.R index d59a521..158e2d7 100644 --- a/R/read.R +++ b/R/read.R @@ -51,8 +51,8 @@ NULL # https://ngff.openmicroscopy.org/specifications/0.5/index.html#images # The name of the array is arbitrary with the ordering defined by # by the "multiscales" metadata, but is often a sequence starting at 0. - ds <- .validate_multiscales_paths(x, datasets(mdattr)) - ds <- file.path(x, as.character(ds)) + # ds <- .validate_multiscales_paths(x, datasets(mdattr)) + ds <- paste0(x, ds) as <- lapply(ds, ZarrArray) list(array=as, mdattr=mdattr) } @@ -77,7 +77,7 @@ readLabel <- function(x, ...) { #' @importFrom dplyr sql #' @export readPoint <- function(x, ...) { - pq <- list.files(x, "\\.parquet$", full.names=TRUE) + pq <- paste0(x, file.path("points.parquet", "part.0.parquet")) md <- read_zarr_attributes(x) ax <- unlist(md$axes) df <- ddbs_open_dataset(pq, conn=.conn()) |> @@ -94,7 +94,8 @@ readPoint <- function(x, ...) { #' @export readShape <- function(x, ...) { md <- read_zarr_attributes(x) - pq <- list.files(x, "\\.parquet$", full.names=TRUE) + # "shapes.parquet" currently hardcoded in SpatialData.io + pq <- paste0(x, "shapes.parquet") df <- ddbs_open_dataset(pq, conn=.conn(), crs=NA_character_) attr(df, "source_path") <- pq SpatialDataShape(data=df, meta=SpatialDataAttrs(md)) @@ -134,11 +135,22 @@ readSpatialData <- function(x, args <- as.list(environment())[.LAYERS] skip <- vapply(args, isFALSE, logical(1)) + x <- Rarr:::.normalize_array_path(x) + store_meta <- Rarr:::.read_consolidated_metadata(x)$metadata + # is.null(.$data_type) is a hack that works for both v2 and v3 Zarr stores, to keep only + # groups, but not arrays + # In v3, we could just do .$node_type == "group", but in v2, there is no node_type. + store_groups <- names(store_meta[vapply(store_meta, \(.) is.null(.$data_type), logical(1))]) + # helper for layer reading .readLayer <- \(l) { - # 'j' are the paths on disk, 'nms' are their basenames - j <- list.dirs(file.path(x, l), recursive=FALSE, full.names=TRUE) - nms <- names(j) <- basename(j) + message(" reading ", l, "...") + j <- store_groups[startsWith(store_groups, paste0(l, "/"))] + j <- setNames( + paste0(x, j, "/", recycle0 = TRUE), + basename(j) + ) + opt <- args[[l]] if (!isTRUE(opt)) { # validate each requested element From a1da313f2bf4269ce98ef49aa7929fafba79f001 Mon Sep 17 00:00:00 2001 From: Hugo Gruson Date: Fri, 15 May 2026 16:58:50 +0200 Subject: [PATCH 2/8] Restore validation on local paths to avoid regression --- R/read.R | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/R/read.R b/R/read.R index 158e2d7..b071fa6 100644 --- a/R/read.R +++ b/R/read.R @@ -51,7 +51,11 @@ NULL # https://ngff.openmicroscopy.org/specifications/0.5/index.html#images # The name of the array is arbitrary with the ordering defined by # by the "multiscales" metadata, but is often a sequence starting at 0. - # ds <- .validate_multiscales_paths(x, datasets(mdattr)) + if (!any(startsWith(x, c("http://", "https://", "s3://")))) { + # Until we have a complete store interface (https://github.com/Huber-group-EMBL/Rarr/pull/176), + # only local objects can be fully validated. + ds <- .validate_multiscales_paths(x, datasets(mdattr)) + } ds <- paste0(x, ds) as <- lapply(ds, ZarrArray) list(array=as, mdattr=mdattr) From cfec3962923794471b4ffaabe2398fa98f9ddab5 Mon Sep 17 00:00:00 2001 From: Hugo Gruson Date: Fri, 15 May 2026 17:13:52 +0200 Subject: [PATCH 3/8] Set minimum Rarr version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index c444ccc..060be80 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -37,7 +37,7 @@ Imports: graph, Matrix, methods, - Rarr, + Rarr (>= 2.1.9), RBGL, rlang, sf, From 53da9b754b67860301d2c4f667d4e05689683d2f Mon Sep 17 00:00:00 2001 From: Hugo Gruson Date: Fri, 15 May 2026 22:13:05 +0200 Subject: [PATCH 4/8] Ensure ds is defined for remote stores --- R/read.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/R/read.R b/R/read.R index b071fa6..fc9ed13 100644 --- a/R/read.R +++ b/R/read.R @@ -55,6 +55,9 @@ NULL # Until we have a complete store interface (https://github.com/Huber-group-EMBL/Rarr/pull/176), # only local objects can be fully validated. ds <- .validate_multiscales_paths(x, datasets(mdattr)) + } else { + # For remote objects, we skip validation and assume that the datasets are in the expected location. + ds <- datasets(mdattr) } ds <- paste0(x, ds) as <- lapply(ds, ZarrArray) From 6f58e377312a7111c42103f0db8b5ef13ee5b471 Mon Sep 17 00:00:00 2001 From: Hugo Gruson Date: Thu, 21 May 2026 16:55:51 +0200 Subject: [PATCH 5/8] Improve v2 handling --- R/read.R | 12 +++++++----- inst/extdata/blobs.zarr/{zmetadata => .zmetadata} | 0 2 files changed, 7 insertions(+), 5 deletions(-) rename inst/extdata/blobs.zarr/{zmetadata => .zmetadata} (100%) diff --git a/R/read.R b/R/read.R index fc9ed13..94bb4e0 100644 --- a/R/read.R +++ b/R/read.R @@ -142,12 +142,14 @@ readSpatialData <- function(x, args <- as.list(environment())[.LAYERS] skip <- vapply(args, isFALSE, logical(1)) - x <- Rarr:::.normalize_array_path(x) + x <- Rarr:::.normalize_array_path(x) store_meta <- Rarr:::.read_consolidated_metadata(x)$metadata - # is.null(.$data_type) is a hack that works for both v2 and v3 Zarr stores, to keep only - # groups, but not arrays - # In v3, we could just do .$node_type == "group", but in v2, there is no node_type. - store_groups <- names(store_meta[vapply(store_meta, \(.) is.null(.$data_type), logical(1))]) + + # We have to treat v2 and v3 separately in the next 3 lines but we unify them again as `store_groups`. + store_groups_v3 <- store_meta[vapply(store_meta, \(.) !is.null(.$node_type) && .$node_type == "group", logical(1))] + store_groups_v2 <- store_meta[endsWith(names(store_meta), ".zgroup")] + names(store_groups_v2) <- dirname(names(store_groups_v2)) + store_groups <- names(c(store_groups_v3, store_groups_v2)) # helper for layer reading .readLayer <- \(l) { diff --git a/inst/extdata/blobs.zarr/zmetadata b/inst/extdata/blobs.zarr/.zmetadata similarity index 100% rename from inst/extdata/blobs.zarr/zmetadata rename to inst/extdata/blobs.zarr/.zmetadata From c7a43e85d8d2d5a3bfb37e0bf5c85260e1b56439 Mon Sep 17 00:00:00 2001 From: Hugo Gruson Date: Thu, 21 May 2026 17:03:15 +0200 Subject: [PATCH 6/8] Remove extra messages --- R/read.R | 1 - 1 file changed, 1 deletion(-) diff --git a/R/read.R b/R/read.R index 94bb4e0..b7b1723 100644 --- a/R/read.R +++ b/R/read.R @@ -153,7 +153,6 @@ readSpatialData <- function(x, # helper for layer reading .readLayer <- \(l) { - message(" reading ", l, "...") j <- store_groups[startsWith(store_groups, paste0(l, "/"))] j <- setNames( paste0(x, j, "/", recycle0 = TRUE), From 1c6d852aafe9c3e858822f53c34a840f2b59342b Mon Sep 17 00:00:00 2001 From: Hugo Gruson Date: Thu, 21 May 2026 17:03:28 +0200 Subject: [PATCH 7/8] Make sure trailing slash is present in paths --- tests/testthat/test-read.R | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/testthat/test-read.R b/tests/testthat/test-read.R index 433342d..367a3a6 100644 --- a/tests/testthat/test-read.R +++ b/tests/testthat/test-read.R @@ -11,6 +11,7 @@ test_that("readElement()", { for (l in names(typ)) { f <- paste0(toupper(substr(l, 1, 1)), substr(l, 2, nchar(l)-1)) y <- list.files(file.path(x, l), full.names=TRUE)[1] + y <- paste0(y, "/", recycle0 = TRUE) expect_is(get(paste0("read", f))(y), typ[l]) } }) From aebc8c164fa9f1ae87abf5299520ad8b3d9916b9 Mon Sep 17 00:00:00 2001 From: Hugo Gruson Date: Tue, 26 May 2026 08:38:25 +0200 Subject: [PATCH 8/8] Create store_meta if missing --- R/read.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/R/read.R b/R/read.R index b7b1723..f56e005 100644 --- a/R/read.R +++ b/R/read.R @@ -144,6 +144,9 @@ readSpatialData <- function(x, x <- Rarr:::.normalize_array_path(x) store_meta <- Rarr:::.read_consolidated_metadata(x)$metadata + if (is.null(store_meta)) { + store_meta <- Rarr::consolidate_metadata(x, action = "return") + } # We have to treat v2 and v3 separately in the next 3 lines but we unify them again as `store_groups`. store_groups_v3 <- store_meta[vapply(store_meta, \(.) !is.null(.$node_type) && .$node_type == "group", logical(1))]