Skip to content

Write and use consolidated metadata #492

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Bisaloo wants to merge 14 commits into
base: devel
Choose a base branch
from
+85 −13
Open

Write and use consolidated metadata #492

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
b9c1468
Add consolidated metadata when writing zarr anndata
Bisaloo Jul 28, 2026
5627868
Leverage consolidated metadata to get encoding info in a single file …
Bisaloo Jul 28, 2026
0584102
Use hashed env for consolidated metadata
Bisaloo Jul 29, 2026
f7c8d94
Define helper function to get attrs from consolidated
Bisaloo Jul 29, 2026
325cfad
Add missing trailing slash to root path
Bisaloo Jul 29, 2026
0c6712f
Support both v2 and v3 when creating consolidated on the fly
Bisaloo Jul 29, 2026
a46c404
Use new read_consolidate_metadata(consolidate = ) arg
Bisaloo Jul 30, 2026
8d8dc28
Simplify .get_attrs_from_consolidated()
Bisaloo Jul 30, 2026
4ef8184
Require more recent Rarr version
Bisaloo Jul 30, 2026
a5a7051
Use new read_zarr_consolidated_metadata() name
Bisaloo Jul 30, 2026
c36a712
Add NEWS bullet
Bisaloo Jul 31, 2026
f84626c
Run air
Bisaloo Aug 3, 2026
c32da76
Bump version
Bisaloo Aug 3, 2026
c8cdc88
Merge branch 'devel' into consolidated-metadata
rcannood Aug 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions DESCRIPTION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Package: anndataR
Title: AnnData interoperability in R
Version: 1.3.1
Version: 1.3.2

@lazappi lazappi Aug 9, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if we want to bump this yet (I've lost track of how we were managing versions, @rcannood might have a plan)

Authors@R: c(
person("Robrecht", "Cannoodt", , "rcannood@gmail.com", role = c("aut", "cre"),
comment = c(ORCID = "0000-0003-3641-729X", github = "rcannood")),
Expand Down Expand Up @@ -56,7 +56,7 @@ Suggests:
knitr,
processx,
rhdf5 (>= 2.52.1),
Rarr (>= 2.1.13),
Rarr (>= 2.1.31),
rmarkdown,
S4Vectors,
Seurat,
Expand Down
1 change: 1 addition & 0 deletions NEWS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
- Add support for `nullable-string-array` elements in H5AD and Zarr (PR #480)
- Check `RELEASE_*` branches against Bioconductor release and `devel` against Bioconductor devel, instead of checking `devel` against both (PR #502).
- Run BiocCheck on `RELEASE_*` branches as well as on `devel` (PR #504).
- Write consolidated metadata on non-readonly anndata stores. Consolidated metadata is used to read encoding type and version of elements one level deep into the store (PR #492).

# anndataR 1.3.1

Expand Down
93 changes: 82 additions & 11 deletions R/ZarrAnnData.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ ZarrAnnData <- R6::R6Class(
private = list(
.zarrobj = NULL,
.zarrformat = NULL,
.consolidated_metadata = NULL,
.compression = NULL,
.readonly = NULL,

Expand All @@ -39,6 +40,10 @@ ZarrAnnData <- R6::R6Class(
call = rlang::caller_env()
)
}
},

.get_attrs_from_consolidated = function(slot, attr) {
private$.consolidated_metadata[[slot]]$attributes[[attr]]
}
),
active = list(
Expand All @@ -48,7 +53,12 @@ ZarrAnnData <- R6::R6Class(

if (missing(value)) {
# trackstatus: class=ZarrAnnData, feature=get_X, status=done
read_zarr_element(private$.zarrobj, "X") |>
read_zarr_element(
private$.zarrobj,
"X",
private$.get_attrs_from_consolidated("X", "encoding-type"),
private$.get_attrs_from_consolidated("X", "encoding-version")
) |>
private$.add_matrix_dimnames("X")
} else {
private$.check_writeable()
Expand All @@ -74,7 +84,12 @@ ZarrAnnData <- R6::R6Class(

if (missing(value)) {
# trackstatus: class=ZarrAnnData, feature=get_layers, status=done
read_zarr_element(private$.zarrobj, "layers") |>
read_zarr_element(
private$.zarrobj,
"layers",
private$.get_attrs_from_consolidated("layers", "encoding-type"),
private$.get_attrs_from_consolidated("layers", "encoding-version")
) |>
private$.add_mapping_dimnames("layers")
} else {
private$.check_writeable()
Expand All @@ -100,7 +115,12 @@ ZarrAnnData <- R6::R6Class(

if (missing(value)) {
# trackstatus: class=ZarrAnnData, feature=get_obsm, status=done
read_zarr_element(private$.zarrobj, "obsm") |>
read_zarr_element(
private$.zarrobj,
"obsm",
private$.get_attrs_from_consolidated("obsm", "encoding-type"),
private$.get_attrs_from_consolidated("obsm", "encoding-version")
) |>
private$.add_mapping_dimnames("obsm")
} else {
private$.check_writeable()
Expand Down Expand Up @@ -128,7 +148,12 @@ ZarrAnnData <- R6::R6Class(

if (missing(value)) {
# trackstatus: class=ZarrAnnData, feature=get_varm, status=done
read_zarr_element(private$.zarrobj, "varm") |>
read_zarr_element(
private$.zarrobj,
"varm",
private$.get_attrs_from_consolidated("varm", "encoding-type"),
private$.get_attrs_from_consolidated("varm", "encoding-version")
) |>
private$.add_mapping_dimnames("varm")
} else {
private$.check_writeable()
Expand Down Expand Up @@ -156,7 +181,12 @@ ZarrAnnData <- R6::R6Class(

if (missing(value)) {
# trackstatus: class=ZarrAnnData, feature=get_obsp, status=done
read_zarr_element(private$.zarrobj, "obsp") |>
read_zarr_element(
private$.zarrobj,
"obsp",
private$.get_attrs_from_consolidated("obsp", "encoding-type"),
private$.get_attrs_from_consolidated("obsp", "encoding-version")
) |>
private$.add_mapping_dimnames("obsp")
} else {
private$.check_writeable()
Expand All @@ -182,7 +212,12 @@ ZarrAnnData <- R6::R6Class(

if (missing(value)) {
# trackstatus: class=ZarrAnnData, feature=get_varp, status=done
read_zarr_element(private$.zarrobj, "varp") |>
read_zarr_element(
private$.zarrobj,
"varp",
private$.get_attrs_from_consolidated("varp", "encoding-type"),
private$.get_attrs_from_consolidated("varp", "encoding-version")
) |>
private$.add_mapping_dimnames("varp")
} else {
private$.check_writeable()
Expand All @@ -208,7 +243,12 @@ ZarrAnnData <- R6::R6Class(

if (missing(value)) {
# trackstatus: class=ZarrAnnData, feature=get_obs, status=done
read_zarr_element(private$.zarrobj, "obs")
read_zarr_element(
private$.zarrobj,
"obs",
private$.get_attrs_from_consolidated("obs", "encoding-type"),
private$.get_attrs_from_consolidated("obs", "encoding-version")
)
} else {
private$.check_writeable()
# trackstatus: class=ZarrAnnData, feature=set_obs, status=done
Expand All @@ -227,7 +267,12 @@ ZarrAnnData <- R6::R6Class(

if (missing(value)) {
# trackstatus: class=ZarrAnnData, feature=get_var, status=done
read_zarr_element(private$.zarrobj, "var")
read_zarr_element(
private$.zarrobj,
"var",
private$.get_attrs_from_consolidated("var", "encoding-type"),
private$.get_attrs_from_consolidated("var", "encoding-version")
)
} else {
private$.check_writeable()
# trackstatus: class=ZarrAnnData, feature=set_var, status=done
Expand All @@ -246,7 +291,13 @@ ZarrAnnData <- R6::R6Class(

if (missing(value)) {
# trackstatus: class=ZarrAnnData, feature=get_obs_names, status=done
read_zarr_element_keys(private$.zarrobj, "obs", dim = "rows")
read_zarr_element_keys(
private$.zarrobj,
"obs",
private$.get_attrs_from_consolidated("obs", "encoding-type"),
private$.get_attrs_from_consolidated("obs", "encoding-version"),
dim = "rows"
)
} else {
private$.check_writeable()
# trackstatus: class=ZarrAnnData, feature=set_obs_names, status=done
Expand All @@ -259,7 +310,13 @@ ZarrAnnData <- R6::R6Class(

if (missing(value)) {
# trackstatus: class=ZarrAnnData, feature=get_var_names, status=done
read_zarr_element_keys(private$.zarrobj, "var", dim = "rows")
read_zarr_element_keys(
private$.zarrobj,
"var",
private$.get_attrs_from_consolidated("var", "encoding-type"),
private$.get_attrs_from_consolidated("var", "encoding-version"),
dim = "rows"
)
} else {
private$.check_writeable()
# trackstatus: class=ZarrAnnData, feature=set_var_names, status=done
Expand All @@ -272,7 +329,12 @@ ZarrAnnData <- R6::R6Class(

if (missing(value)) {
# trackstatus: class=ZarrAnnData, feature=get_uns, status=done
read_zarr_element(private$.zarrobj, "uns")
read_zarr_element(
private$.zarrobj,
"uns",
private$.get_attrs_from_consolidated("uns", "encoding-type"),
private$.get_attrs_from_consolidated("uns", "encoding-version")
)
} else {
private$.check_writeable()
# trackstatus: class=ZarrAnnData, feature=set_uns, status=done
Expand Down Expand Up @@ -493,8 +555,17 @@ ZarrAnnData <- R6::R6Class(
self[[slot]] <- value
}
}
Rarr::zarr_consolidate_metadata(file)
}

# Create consolidated metadata on the fly, in memory,
# if it doesn't exist on disk (only happens for readonly stores).
private$.consolidated_metadata <- Rarr::read_zarr_consolidated_metadata(
file,
consolidate = "missing"
) |>
list2env(hash = TRUE, parent = emptyenv())

self
},

Expand Down