Implement ZarrADMatrix objects

DESCRIPTION

@@ -42,4 +42,6 @@ Collate:
 	writeZarrArray.R
 	ZarrSparseMatrixSeed-class.R
 	ZarrSparseMatrix-class.R
+	ZarrADMatrixSeed-class.R
+	ZarrADMatrix-class.R
 	zzz.R

NAMESPACE

@@ -21,7 +21,10 @@ exportClasses(
     ZarrArray, ZarrMatrix,
     ZarrRealizationSink,
     ZarrSparseMatrixSeed, CSC_ZarrSparseMatrixSeed, CSR_ZarrSparseMatrixSeed,
-    ZarrSparseMatrix
+    ZarrSparseMatrix,
+    ZarrADMatrixSeed,
+    Dense_ZarrADMatrixSeed, CSC_ZarrADMatrixSeed, CSR_ZarrADMatrixSeed,
+    ZarrADMatrix
 )
 
 
@@ -31,14 +34,18 @@ exportClasses(
 
 S3method(t, CSC_ZarrSparseMatrixSeed)
 S3method(t, CSR_ZarrSparseMatrixSeed)
+S3method(t, CSC_ZarrADMatrixSeed)
+S3method(t, CSR_ZarrADMatrixSeed)
 
 ### We also export them thru the export() directive so that (a) they can be
 ### called directly, (b) tab-completion on the name of the generic shows them,
 ### and (c) methods() doesn't asterisk them.
 
 export(
     t.CSC_ZarrSparseMatrixSeed,
-    t.CSR_ZarrSparseMatrixSeed
+    t.CSR_ZarrSparseMatrixSeed,
+    t.CSC_ZarrADMatrixSeed,
+    t.CSR_ZarrADMatrixSeed
 )
 
 
@@ -93,6 +100,11 @@ export(
     ZarrSparseMatrixSeed,
 
     ## ZarrSparseMatrix-class.R:
-    ZarrSparseMatrix
-)
-
+    ZarrSparseMatrix,
+
+    ## ZarrADMatrixSeed-class.R
+    ZarrADMatrixSeed,
+
+    ## ZarrADMatrix-class.R
+    ZarrADMatrix
+)

R/ZarrADMatrix-class.R

@@ -0,0 +1,33 @@
+### =========================================================================
+### ZarrADMatrix objects
+### -------------------------------------------------------------------------
+###
+
+setClass("ZarrADMatrix",
+         contains="DelayedMatrix",
+         representation(seed="ZarrADMatrixSeed")
+)
+
+
+### - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+### Constructor
+###
+
+setMethod("DelayedArray", "ZarrADMatrixSeed",
+          function(seed) new_DelayedArray(seed, Class="ZarrADMatrix")
+)
+
+### Works directly on an ZarrADMatrixSeed derivative, in which case it must
+### be called with a single argument.
+ZarrADMatrix <- function(zarr_store, layer=NULL)
+{
+  if (is(zarr_store, "ZarrADMatrixSeed")) {
+    if (!is.null(layer))
+      stop(wmsg("ZarrADMatrix() must be called with a single argument ",
+                "when passed an ZarrADMatrixSeed derivative"))
+    seed <- zarr_store
+  } else {
+    seed <- ZarrADMatrixSeed(zarr_store, layer=layer)
+  }
+  DelayedArray(seed)
+}

R/ZarrADMatrixSeed-class.R

@@ -0,0 +1,125 @@
+### =========================================================================
+### ZarrADMatrixSeed objects
+### -------------------------------------------------------------------------
+
+
+setClass("ZarrADMatrixSeed",
+         contains=c("Array", "OutOfMemoryObject"),
+         representation("VIRTUAL")
+)
+
+setClass("Dense_ZarrADMatrixSeed",
+         contains=c("ZarrADMatrixSeed", "ZarrArraySeed"),
+         representation(dimnames="list"),
+         prototype(dimnames=list(NULL, NULL))
+)
+setClass("CSC_ZarrADMatrixSeed",
+         contains=c("ZarrADMatrixSeed", "CSC_ZarrSparseMatrixSeed")
+)
+setClass("CSR_ZarrADMatrixSeed",
+         contains=c("ZarrADMatrixSeed", "CSR_ZarrSparseMatrixSeed")
+)
+
+
+### - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+### dimnames() method for Dense_ZarrADMatrixSeed objects
+###
+
+### We overwrite the method for ZarrArraySeed objects with a method that
+### accesses the slot, not the store
+setMethod("dimnames", "Dense_ZarrADMatrixSeed",
+          function(x) S4Arrays:::simplify_NULL_dimnames(x@dimnames)
+)
+
+
+### - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+### Transposition
+###
+
+### S3/S4 combo for t.CSC_ZarrADMatrixSeed
+t.CSC_ZarrADMatrixSeed <- function(x)
+{
+  new2("CSR_ZarrADMatrixSeed", callNextMethod())
+}
+setMethod("t", "CSC_ZarrADMatrixSeed", t.CSC_ZarrADMatrixSeed)
+
+### S3/S4 combo for t.CSR_ZarrADMatrixSeed
+t.CSR_ZarrADMatrixSeed <- function(x)
+{
+  new2("CSC_ZarrADMatrixSeed", callNextMethod())
+}
+setMethod("t", "CSR_ZarrADMatrixSeed", t.CSR_ZarrADMatrixSeed)
+
+
+### - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+### Constructor
+###
+
+.load_zarr_ad_rownames <- function(zarr_store, name="var")
+{
+  ok <- try(zarr_node_is_group(zarr_store, name), silent=TRUE)
+  if (!isTRUE(ok))
+    return(NULL)
+  ROWNAMES_DATASET <- file.path(name, "_index")
+  ok <- try(zarr_node_is_dataset(zarr_store, ROWNAMES_DATASET), silent=TRUE)
+  if (!isTRUE(ok))
+    return(NULL)
+  read_zarr_array(file.path(zarr_store, ROWNAMES_DATASET))
+}
+
+### Must return a list of length 2.
+.load_zarr_ad_dimnames <- function(zarr_store)
+{
+  ans_rownames <- .load_zarr_ad_rownames(zarr_store)
+  ans_colnames <- .load_zarr_ad_rownames(zarr_store, name="obs")
+  if (is.null(ans_rownames) && is.null(ans_colnames))
+    warning(wmsg("could not find dimnames in this anndata-zarr store"))
+  list(ans_rownames, ans_colnames)
+}
+
+### Returns an ZarrADMatrixSeed derivative (can be either a Dense_ZarrADMatrixSeed,
+### or a CSC_ZarrSparseMatrixSeed, or a CSR_ZarrSparseMatrixSeed object).
+ZarrADMatrixSeed <- function(zarr_store, layer=NULL)
+{
+  if (!isSingleString(zarr_store))
+    stop(wmsg("'zarr_store' must be a single string specifying the ",
+              "path to the anndata-zarr store"))
+  zarr_store <- file_path_as_absolute(zarr_store)
+  if (is.null(layer)) {
+    name <- "X"
+  } else {
+    if (!isSingleString(layer) || !nzchar(layer))
+      stop(wmsg("'layer' must be NULL or a single non-empty string"))
+    name <- file.path("layers", layer)
+  }
+  if (!zarr_exists(zarr_store, name)) {
+    msg <- c("Zarr object \"", name, "\" does not exist ",
+             "in this Zarr store")
+    if (is.null(layer))
+      msg <- c(msg, " Is this a valid anndata-zarr store?")
+    stop(wmsg(msg))
+  }
+  dimnames <- .load_zarr_ad_dimnames(zarr_store)
+
+  if (zarr_node_is_dataset(zarr_store, name)) {
+    ans0 <- ZarrArraySeed(zarr_store, name)
+    if (length(dim(ans0)) != 2L)
+      stop(wmsg("Zarr dataset \"", name, "\" in store \"", zarr_store, "\" ",
+                "does not have exactly 2 dimensions. Please consider ",
+                "using the ZarrArray() constructor to access this ",
+                "dataset."))
+    ans <- new2("Dense_ZarrADMatrixSeed", ans0, dimnames=dimnames)
+  } else if (zarr_node_is_group(zarr_store, name)) {
+    ans0 <- ZarrSparseMatrixSeed(zarr_store, name)
+    if (is(ans0, "CSC_ZarrSparseMatrixSeed"))
+      ans_class <- "CSC_ZarrADMatrixSeed"
+    else
+      ans_class <- "CSR_ZarrADMatrixSeed"
+    ans <- new2(ans_class, ans0, dimnames=dimnames)
+  } else {
+    stop(wmsg("Zarr object \"", name, "\" in store \"", zarr_store, "\" ",
+              "is neither a dataset or a group. Is this a valid ",
+              "anndata-zarr store?"))
+  }
+  ans
+}

R/ZarrSparseMatrixSeed-class.R

@@ -324,7 +324,7 @@ ZarrSparseMatrixSeed <- function(zarr_store, group, subdata=NULL,
     dim <- .get_sparse_matrix_dim(zarr_store, group, dim=dim)
 
     ## Get sparse layout to use ("CSC" or "CSR").
-    ## For consistency with H5SparseMatrixSeed, we flip the notions of rows
+    ## For consistency with ZarrSparseMatrixSeed, we flip the notions of rows
     ## and columns w.r.t. to the AnnData convention. So:
     ## - "compressed sparse row" in the AnnData-style Zarr store
     ##   becomes "compressed sparse column" at the R level,

TODO

@@ -1,6 +1,3 @@
-o Implement ZarrADMatrixSeed and ZarrADMatrix objects (analog to
-  H5ADMatrixSeed and H5ADMatrix objects from the HDF5Array package).
-
 o Does Rarr provide an equivalent of h5ls() for quick inspection of
   a Zarr store? This would be very useful in the context of using
   things like ZarrSparseMatrixSeed() or ZarrSparseMatrix() where the

man/ZarrADMatrix-class.Rd

@@ -0,0 +1,118 @@
+\name{ZarrADMatrix-class}
+\docType{class}
+
+\alias{class:ZarrADMatrix}
+\alias{ZarrADMatrix-class}
+\alias{ZarrADMatrix}
+
+\alias{DelayedArray,ZarrADMatrixSeed-method}
+
+\alias{nzcount,ZarrADMatrix-method}
+\alias{extractNonzeroDataByCol,ZarrADMatrix-method}
+\alias{extractNonzeroDataByRow,ZarrADMatrix-method}
+
+\title{anndata-zarr central matrices (or matrices in the /layers group)
+       as DelayedMatrix objects}
+
+\description{
+  \code{anndata-zarr} stores are Zarr stores used for on-disk representation
+  of AnnData Python objects. At the very minimum, they contain a central
+  data matrix, named \code{X}, of shape #observations x #variables, and
+  possibly additional data matrices (stored in the Zarr group \code{/layers})
+  that share the shape and dimnames of \code{X}.
+  See \url{https://anndata.readthedocs.io/} for more information.
+
+  The ZarrADMatrix class is a \link[DelayedArray]{DelayedMatrix} subclass
+  for representing and operating on the central matrix of an \code{anndata-zarr}
+  store, or any matrix in its \code{/layers} group.
+
+  All the operations available for \link[DelayedArray]{DelayedMatrix}
+  objects work on ZarrADMatrix objects.
+}
+
+\usage{
+## Constructor function:
+ZarrADMatrix(zarr_store, layer=NULL)
+}
+
+\arguments{
+  \item{zarr_store}{
+    The path (as a single string) to the \code{anndata-zarr} store.
+  }
+  \item{layer}{
+    \code{NULL} (the default) or the name of a matrix in the \code{/layers}
+    group. By default (i.e. when \code{layer} is not specified)
+    \code{ZarrADMatrix()} returns the central matrix (\code{X}).
+  }
+}
+
+\value{
+  \code{ZarrADMatrix()} returns an ZarrADMatrix object of shape #variables x
+  #observations. Note that in Python and Zarr the shape of this matrix is
+  considered to be #observations x #variables, but in R it is transposed.
+  This follows the widely adopted convention of transposing Zarr matrices
+  when they get loaded into R.
+}
+
+\references{
+  \url{https://anndata.readthedocs.io/} for AnnData Python objects
+  and the \code{anndata-zarr} format.
+}
+
+\seealso{
+  \itemize{
+    \item \link{ZarrArray} objects for representing conventional (a.k.a.
+          dense) Zarr datasets as \link[DelayedArray]{DelayedArray} objects.
+
+    \item \link{ZarrSparseMatrix} objects for representing Zarr sparse matrices
+          as \link[DelayedArray]{DelayedMatrix} objects.
+
+    \item \link[DelayedArray]{DelayedMatrix} objects in the \pkg{DelayedArray}
+          package.
+
+    \item The \link{ZarrADMatrixSeed} helper class.
+
+    \item \code{\link[anndataR]{read_zarr}} and
+          \code{\link[anndataR]{write_zarr}} in
+          the \pkg{anndataR} package for
+          importing/exporting an \code{anndata-zarr} store as/from a
+          \link[SingleCellExperiment]{SingleCellExperiment} object.
+
+    \item \link[SparseArray]{SparseArray} objects in the \pkg{SparseArray}
+          package.
+  }
+}
+
+\examples{
+library(anndataR)
+zarr_zip <- system.file("extdata", 
+                        paste0("example_v2.zarr.zip"), 
+                        package = "anndataR")
+td <- tempdir(check = TRUE)
+unzip(zarr_zip, exdir = td)
+zarr_store <- file.path(td, paste0("example_v2.zarr"))
+
+X <- ZarrADMatrix(zarr_store)
+X
+
+class(X)  # ZarrADMatrix
+is(X, "DelayedMatrix")  # TRUE
+
+class(seed(X))  # Dense_ZarrADMatrixSeed
+
+dim(X)
+path(X)
+is_sparse(X)  # FALSE
+
+## Use coercion to load the full dataset into memory:
+as.matrix(X)          # as ordinary array (usually not recommended)
+
+\dontrun{
+  ## Works only if ZarrADMatrix object is sparse!
+  as(X, "dgCMatrix")    # as dgCMatrix
+  as(X, "SparseArray")  # as SparseArray object (most efficient)
+  SparseArray(X)        # equivalent to 'as(X, "SparseArray")'
+}
+}
+\keyword{classes}
+\keyword{methods}