Fixes

src/datasets/loaders/bruker_cosmx/script.py

@@ -199,6 +199,59 @@ def extract_zip(input_zip: Path, output_dir: Path, strip_root: bool = False):
 else:
     raise AssertionError(f"No CellLabels folder and no per-FOV CellLabels tifs found in {DATA_DIR}")
 
+#############################################
+# Memory optimization: lean transcript read #
+#############################################
+# sopa's CosMx reader loads the ENTIRE `*_tx_file.csv` into a pandas DataFrame in
+# one `pd.read_csv` (sopa/io/reader/cosmx.py::_CosMXReader.read_transcripts). For
+# the mouse-brain half-hemisphere that's hundreds of millions of rows, and the
+# string columns ('target', 'CellComp', 'cell', ...) come in as object dtype
+# (~60 B/row) — this is the peak that blows past even the 480 GB veryhighmem cap.
+#
+# We wrap the `read_csv` used inside sopa's cosmx module so that, *only for the
+# transcripts file*, we (a) read the 'target' feature column as categorical and
+# (b) keep just the columns sopa's stitched-FOV path (read_transcripts +
+# _get_global_cell_id) and the downstream transcript schema actually use. Every
+# other sopa read (fov positions, metadata, counts, polygons) is left untouched,
+# and all of sopa's coordinate/stitching math runs unchanged on the leaner frame.
+from sopa.io.reader import cosmx as _cosmx_mod
+
+_real_pd = _cosmx_mod.pd
+_real_read_csv = _real_pd.read_csv
+
+# Columns used by read_transcripts (fov=None) + _get_global_cell_id, plus 'z'
+# which the downstream transcript schema allows. Everything else in the tx file
+# (x_local_px, y_local_px, cell, CellComp, ...) is unused downstream.
+_TX_KEEP = ["fov", "cell_ID", "target", "x_global_px", "y_global_px", "z"]
+_TX_REQUIRED = {"target", "x_global_px", "y_global_px", "fov", "cell_ID"}
+
+def _lean_read_csv(filepath_or_buffer, *args, **kwargs):
+    name = str(filepath_or_buffer)
+    if "_tx_file.csv" in name and "usecols" not in kwargs:
+        header = _real_read_csv(
+            filepath_or_buffer, nrows=0,
+            compression=kwargs.get("compression", "infer"),
+        )
+        cols = set(header.columns)
+        if _TX_REQUIRED.issubset(cols):
+            kwargs["usecols"] = [c for c in _TX_KEEP if c in cols]
+            dtype = dict(kwargs.get("dtype") or {})
+            dtype.setdefault("target", "category")
+            kwargs["dtype"] = dtype
+            log(f"Lean transcript read of {Path(name).name}: usecols={kwargs['usecols']}, target=category")
+    return _real_read_csv(filepath_or_buffer, *args, **kwargs)
+
+class _PandasReadCsvProxy:
+    """Forwards every attribute to pandas, but leans the transcripts read_csv."""
+    def __init__(self, real):
+        self.__dict__["_real"] = real
+    def read_csv(self, *args, **kwargs):
+        return _lean_read_csv(*args, **kwargs)
+    def __getattr__(self, item):
+        return getattr(self._real, item)
+
+_cosmx_mod.pd = _PandasReadCsvProxy(_real_pd)
+
 #########################################
 # Convert raw files to spatialdata zarr #
 #########################################

src/methods_cell_type_annotation/rctd/script.R

@@ -18,7 +18,7 @@ par <- list(
 )
 
 meta <- list(
-  'cpus': 4,
+  cpus = 4
 )
 
 ## VIASH END

src/methods_transcript_assignment/comseg/script.py

@@ -37,22 +37,37 @@
 
 # Read input files
 print('Reading input files', flush=True)
-sdata = sd.read_zarr(par['input_ist'])
+sdata_src = sd.read_zarr(par['input_ist'])
 sdata_segm = sd.read_zarr(par['input_segmentation'])
 
 # Multi-partition parquet restarts its index at 0 per partition, producing duplicate
 # index labels that break sopa's cell_id assignment and the final write with
 # "cannot reindex on an axis with duplicate labels". Rebuild the transcripts as a
 # single-partition frame with a clean RangeIndex before any sopa op touches them.
-_tx = sdata[par['transcripts_key']]
+_tx = sdata_src[par['transcripts_key']]
 _tx_reset = dd.from_pandas(_tx.compute().reset_index(drop=True), npartitions=1)
 _tx_reset.attrs.update(_tx.attrs)
-del sdata[par['transcripts_key']]
-sdata[par['transcripts_key']] = _tx_reset
 
 # Convert the prior segmentation to polygons
-sdata["segmentation_boundaries"] = sd.to_polygons(sdata_segm["segmentation"])
-del sdata["segmentation_boundaries"]["label"] # make_transcript_patches will create a new label column and fails if one exists.
+segmentation_boundaries = sd.to_polygons(sdata_segm["segmentation"])
+del segmentation_boundaries["label"]  # make_transcript_patches will create a new label column and fails if one exists.
+
+# Run sopa/ComSeg on a FRESH, in-memory SpatialData. sd.read_zarr() leaves `sdata_src`
+# backed by the shared source zarr (sdata_src.path -> input_ist), and sopa's
+# make_image_patches / make_transcript_patches / segmentation.comseg all write their new
+# elements (image_patches, transcripts_patches, comseg_boundaries) and a .sopa_cache back
+# through that path -- mutating the supposed-to-be-immutable source dataset in place. If a
+# concurrent run or an OOM kill interrupts one of those writes, it leaves a half-written
+# element (e.g. shapes/image_patches with no shapes.parquet) that then breaks EVERY
+# downstream reader of the dataset. baysor and proseg avoid this by running sopa on a fresh
+# in-memory object; do the same here (carrying the image for make_image_patches and the
+# prior boundaries for the transcript patches) so all sopa writes land in the ephemeral
+# work dir, never the source zarr.
+sdata = sd.SpatialData(
+    images={"image": sdata_src["image"]},
+    points={par['transcripts_key']: _tx_reset},
+    shapes={"segmentation_boundaries": segmentation_boundaries},
+)
 
 # Make patches
 sopa.make_image_patches(sdata, image_key="image", patch_width=par["patch_width"], patch_overlap=par["patch_overlap"])

src/methods_transcript_assignment/segger/config.vsh.yaml

@@ -116,14 +116,25 @@ engines:
       #    supported numba/numpy range is a latent risk for segger's phenograph step.
       #    Validated set: numba 0.60.0 / numpy 2.0.2 / llvmlite 0.43.0 (cudf .values,
       #    cuml kNN, cugraph louvain, torch, spatialdata/anndata all import + run).
+      #  - PIN polars>=1.24,<1.26 (== what cudf-polars 25.4 requires). segger's github
+      #    install leaves polars unpinned; this keeps it at the RAPIDS-compatible 1.24.
+      #    See the writer.py from_torch shim below for why we must stay <1.26.
       #  - swap opencv-python -> opencv-python-headless: segger pulls opencv-python,
       #    whose cv2 needs libGL.so.1 (a system lib we cannot apt-install as non-root);
       #    the headless build provides the same cv2 without it. See NOTES.md.
       - type: docker
         run:
-          - pip install --no-cache-dir "pandas>=2.0,<2.2.4" "anndata>=0.12,<0.13" "scanpy>=1.11,<1.12" "spatialdata>=0.7,<0.8" "dask==2025.2.0" "distributed==2025.2.0" "numba>=0.59.1,<0.61" "numpy>=2.0,<2.1"
+          - pip install --no-cache-dir "pandas>=2.0,<2.2.4" "anndata>=0.12,<0.13" "scanpy>=1.11,<1.12" "spatialdata>=0.7,<0.8" "dask==2025.2.0" "distributed==2025.2.0" "numba>=0.59.1,<0.61" "numpy>=2.0,<2.1" "polars>=1.24,<1.26"
           - pip uninstall -y opencv-python || true
           - pip install --no-cache-dir opencv-python-headless
+          # segger's data/writer.py calls pl.from_torch to turn the prediction tensors into a
+          # polars frame, but polars only added from_torch AFTER 1.27 (PR pola-rs/polars#22177)
+          # -- and cudf-polars 25.4 caps polars at <1.26, so we cannot get the real one without
+          # breaking the RAPIDS stack. Inject the missing function (torch tensor -> from_numpy,
+          # honoring segger's list/dict schema) right after writer.py's `import torch`. getattr
+          # preserves a real from_torch if a future polars provides one. Path is fixed by the
+          # py3.12 base image; a build failure here (missing file) is a loud signal it moved.
+          - "sed -i '/^import torch$/a pl.from_torch = getattr(pl, \"from_torch\", lambda data, schema=None, **k: pl.from_numpy(data.detach().cpu().numpy(), schema=schema))' /opt/conda/lib/python3.12/site-packages/segger/data/writer.py"
   - type: native
 
 runners: