Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions benchmarks/to_xarray.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
"""Benchmarks for to_xarray() methods,
like UxDataset.to_xarray() and UxDataArray.to_xarray().
"""

import numpy as np
import uxarray as ux


def _assign_coords(obj, n_coords):
n_face = obj.sizes["n_face"]
return obj.assign_coords(
{f"c{i}": ("n_face", np.arange(n_face, dtype=float)) for i in range(n_coords)}
)


class UxDatasetToXarray:
"""Benchmark ``UxDataset.to_xarray()``, which is dominated by per-variable overhead."""

params = [[(100, 10), (1000, 100)]]
param_names = ["n_vars_and_n_coords"]

def setup(self, n_vars_and_n_coords):
n_vars, n_coords = n_vars_and_n_coords
uxds = ux.tutorial.open_dataset("outCSne30-vortex")
uxds = uxds.assign({f"v{i}": uxds["psi"]+i for i in range(n_vars-1)})
self.uxds = _assign_coords(uxds, n_coords)

def time_to_xarray(self, n_vars_and_n_coords):
self.uxds.to_xarray()
Comment thread
Sevans711 marked this conversation as resolved.
Resolved

def time_to_xarray_healpix(self, n_vars_and_n_coords):
self.uxds.to_xarray("HEALPix")


class UxDataArrayToXarray:
"""Benchmark ``UxDataArray.to_xarray()`` with many (n_face-sized) coords."""

params = [[10]]
param_names = ["n_coords"]

def setup(self, n_coords):
uxds = ux.tutorial.open_dataset("outCSne30-vortex")
self.uxda = _assign_coords(uxds["psi"], n_coords)

def time_to_xarray(self, n_coords):
self.uxda.to_xarray()
151 changes: 151 additions & 0 deletions test/core/test_dataarray.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import warnings
import dask
import numpy as np
import uxarray as ux
from uxarray.errors import DataCenteringError, DimensionError, GridInvalidError
from uxarray.grid import neighbors
from uxarray.grid.geometry import _build_polygon_shells, _build_corrected_polygon_shells
from uxarray.core.dataset import UxDataset, UxDataArray
import xarray as xr
import pytest


Expand Down Expand Up @@ -599,3 +601,152 @@ def test_uxdataarray_astype_returns_uxdataarray():
result = obj.astype('float32')
assert obj.dtype == np.float32 # the original dtype was float32
assert result.identical(obj) # so astype() should be a no-op.


class TestUxDataArrayToXarray:
"""``UxDataArray.to_xarray()`` must behave exactly like old "naive" implementation:
result = xr.DataArray(uxda)
"""

def get_uxds(self):
"""quad-hexagon plus a time-dependent var, extra coords, attrs and encodings."""
uxds = ux.tutorial.open_dataset("quad-hexagon")
uxds = uxds.assign(
{
"var_with_time": uxds["t2m"]
* xr.DataArray([10, 20, 30], dims="time", coords={"time": np.arange(3)})
}
)
uxds = uxds.assign_coords(
face_id=("n_face", np.arange(uxds.sizes["n_face"]), {"role": "id"}),
level=850.0,
)
uxds.variables["var_with_time"].attrs["units"] = "K"
uxds.variables["face_id"].encoding = {"dtype": "int32"}
uxds.variables["time"].encoding = {"dtype": "int64"}
return uxds

def get_uxda(self):
"""A named UxDataArray with an indexed coord, non-index coords and encodings."""
uxda = self.get_uxds()["var_with_time"]
# Set on the extracted array so the main-variable encoding is guaranteed to be present
uxda.variable.encoding = {"dtype": "float32", "_FillValue": -999.0}
return uxda

def old(self, uxda):
"""The previous, "naive" implementation."""
return xr.DataArray(uxda)

def assert_same_encodings(self, new, old):
"""Main-variable encoding, coord types/encodings and index types match."""
assert new.variable.encoding == old.variable.encoding
assert set(new.coords) == set(old.coords)
for k in old.coords:
assert type(new.coords[k].variable) is type(old.coords[k].variable), k
np.testing.assert_equal(new.coords[k].values, old.coords[k].values, err_msg=k)
assert set(new.xindexes) == set(old.xindexes)
for k in old.xindexes:
assert type(new.xindexes[k]) is type(old.xindexes[k]), k

def test_identical_to_old(self):
"""Contents, name, dims, encodings, and ``_close`` match the old result."""
uxda = self.get_uxda()
assert uxda.variable.encoding # sanity: there is encoding that could leak through
new, old = uxda.to_xarray(), self.old(uxda)
xr.testing.assert_identical(new, old)
assert new.name == old.name == "var_with_time"
assert new.dims == old.dims
assert new._close is old._close is None
self.assert_same_encodings(new, old)

def test_data_sharing_matches_old(self):
"""Whether the data and each coord's data is shared with the original matches the old path."""
uxda = self.get_uxda()
new, old = uxda.to_xarray(), self.old(uxda)
shares_new = np.shares_memory(new.values, uxda.values)
shares_old = np.shares_memory(old.values, uxda.values)
assert shares_new == shares_old
for k in uxda.coords:
shares_new = np.shares_memory(new.coords[k].values, uxda.coords[k].values)
shares_old = np.shares_memory(old.coords[k].values, uxda.coords[k].values)
assert shares_new == shares_old, k

def test_modifying_result_leaves_original_unchanged(self):
"""Mutating or restructuring the result never changes the original, same as the old path."""

def snapshot(da):
return (
da.name,
dict(da.attrs),
dict(da.variable.encoding),
{k: (dict(c.attrs), dict(c.encoding)) for k, c in da.coords.items()},
)

for convert in (self.old, lambda u: u.to_xarray()):
uxda = self.get_uxda()
before = snapshot(uxda)
result = convert(uxda)

result.attrs["added"] = 1
result.variable.encoding["added"] = 1
for k in ("face_id", "time"):
result.coords[k].attrs["added"] = 1
result.coords[k].encoding["added"] = 1
result.variable.attrs = {"replaced": 1}
result.variable.encoding = {"replaced": 1}
result.coords["extra"] = ("n_face", np.zeros(uxda.sizes["n_face"]))
del result.coords["level"]
result.name = "renamed"

assert snapshot(uxda) == before

def test_nested_mutables_match_old(self):
"""Whether nested attrs/encoding values leak into the original matches the old path."""
leaked = []
for convert in (self.old, lambda u: u.to_xarray()):
uxda = self.get_uxda()
uxda.attrs["nested"] = [0]
uxda.coords["face_id"].attrs["nested"] = [0]
uxda.coords["face_id"].encoding["nested"] = [0]
result = convert(uxda)
result.attrs["nested"].append(1)
result.coords["face_id"].attrs["nested"].append(1)
result.coords["face_id"].encoding["nested"].append(1)
leaked_here = (
uxda.attrs["nested"],
uxda.coords["face_id"].attrs["nested"],
uxda.coords["face_id"].encoding["nested"],
)
leaked.append(leaked_here)
assert leaked[0] == leaked[1]

def test_lazy(self, tmp_path):
Comment thread
Sevans711 marked this conversation as resolved.
Resolved
"""Just like the old path: dask-backed and file-backed data stay lazy,
and closing the result doesn't close the original.
"""

def fail(*args, **kwargs):
raise RuntimeError("compute was called")

# dask-backed data and coordinates
uxds = self.get_uxds()
chunked = uxds["var_with_time"].chunk({"n_face": 2})
with dask.config.set(scheduler=fail):
new = chunked.to_xarray()
old = self.old(chunked)
assert new.chunks == old.chunks == chunked.chunks
assert type(new.variable._data) is type(old.variable._data)
for k in old.coords:
assert type(new.coords[k].variable._data) is type(old.coords[k].variable._data), k
xr.testing.assert_identical(new, old)

# lazily loaded file-backed data
path = tmp_path / "data.nc"
uxds.to_netcdf(path)
opened = ux.UxDataset(xr.open_dataset(path), uxgrid=uxds.uxgrid)
uxda = opened["var_with_time"]
new, old = uxda.to_xarray(), self.old(uxda)
assert new.variable._in_memory == old.variable._in_memory is False
self.assert_same_encodings(new, old)
xr.testing.assert_identical(new.load(), old.load())
opened.close()
157 changes: 155 additions & 2 deletions test/core/test_dataset.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import dask
import numpy as np
import numpy.testing as nt
import xarray as xr
import uxarray as ux
import uxarray.errors
from uxarray import UxDataset
import pytest

import numpy as np


@pytest.fixture()
def healpix_sample_ds():
Expand Down Expand Up @@ -374,6 +374,159 @@ def test_uxdataset_astype_returns_uxdataset():
assert result.identical(obj) # so astype() should be a no-op.


class TestUxDatasetToXarray:
"""``UxDataset.to_xarray()`` must behave exactly like old "naive" implementation:
result = xr.Dataset(uxds.data_vars, coords=uxds.coords, attrs=uxds.attrs)
"""

def get_uxds(self):
"""quad-hexagon plus a time-dependent var, extra coords, attrs and encodings."""
uxds = ux.tutorial.open_dataset("quad-hexagon")
uxds = uxds.assign(
{
"var_with_time": uxds["t2m"]
* xr.DataArray([10, 20, 30], dims="time", coords={"time": np.arange(3)})
}
)
uxds = uxds.assign_coords(
face_id=("n_face", np.arange(uxds.sizes["n_face"]), {"role": "id"}),
level=850.0,
)
uxds.attrs["title"] = "test"
uxds.variables["t2m"].attrs["units"] = "K"
uxds.variables["t2m"].encoding = {"dtype": "float32", "_FillValue": -999.0}
uxds.variables["face_id"].encoding = {"dtype": "int32"}
uxds.variables["time"].encoding = {"dtype": "int64"}
uxds.encoding = {"unlimited_dims": {"time"}}
return uxds

def old(self, uxds):
"""The previous, "naive" implementation."""
return xr.Dataset(uxds.data_vars, coords=uxds.coords, attrs=uxds.attrs)

def assert_same_variables(self, new, old):
"""Variable types, attrs, encodings and index types match."""
assert set(new.variables) == set(old.variables)
for k in old.variables:
assert type(new.variables[k]) is type(old.variables[k]), k
assert new.variables[k].attrs == old.variables[k].attrs, k
assert new.variables[k].encoding == old.variables[k].encoding, k
assert set(new.xindexes) == set(old.xindexes)
for k in old.xindexes:
assert type(new.xindexes[k]) is type(old.xindexes[k]), k

def test_identical_to_old(self):
"""Contents, variable encodings, dataset encoding and ``_close`` match the old result."""
uxds = self.get_uxds()
new, old = uxds.to_xarray(), self.old(uxds)
xr.testing.assert_identical(new, old)
self.assert_same_variables(new, old)
assert new.encoding == old.encoding == {}
assert new._close is old._close is None

def test_data_sharing_matches_old(self):
"""Whether each variable's data is shared with the original matches the old path."""
uxds = self.get_uxds()
new, old = uxds.to_xarray(), self.old(uxds)
for k in uxds.variables:
shares_new = np.shares_memory(new[k].values, uxds[k].values)
shares_old = np.shares_memory(old[k].values, uxds[k].values)
assert shares_new == shares_old, k

def test_modifying_result_leaves_original_unchanged(self):
"""Mutating or restructuring the result never changes the original, same as the old path."""

def snapshot(ds):
return (
dict(ds.attrs),
{k: (dict(v.attrs), dict(v.encoding)) for k, v in ds.variables.items()},
)

for convert in (self.old, lambda u: u.to_xarray()):
uxds = self.get_uxds()
before = snapshot(uxds)
result = convert(uxds)

result.attrs["added"] = 1
for k in ("t2m", "face_id", "time"):
result.variables[k].attrs["added"] = 1
result.variables[k].encoding["added"] = 1
result.variables["var_with_time"].attrs = {"replaced": 1}
result.variables["var_with_time"].encoding = {"replaced": 1}
result["extra"] = ("n_face", np.zeros(uxds.sizes["n_face"]))
del result["var_with_time"]

assert snapshot(uxds) == before

def test_nested_mutables_match_old(self):
"""Whether nested attrs/encoding values leak into the original matches the old path."""
leaked = []
for convert in (self.old, lambda u: u.to_xarray()):
uxds = self.get_uxds()
uxds.attrs["nested"] = [0]
uxds.variables["t2m"].attrs["nested"] = [0]
uxds.variables["face_id"].attrs["nested"] = [0]
uxds.variables["face_id"].encoding["nested"] = [0]
result = convert(uxds)
result.attrs["nested"].append(1)
result.variables["t2m"].attrs["nested"].append(1)
result.variables["face_id"].attrs["nested"].append(1)
result.variables["face_id"].encoding["nested"].append(1)
leaked_here = (
uxds.attrs["nested"],
uxds.variables["t2m"].attrs["nested"],
uxds.variables["face_id"].attrs["nested"],
uxds.variables["face_id"].encoding["nested"],
)
leaked.append(leaked_here)
assert leaked[0] == leaked[1]

@pytest.mark.parametrize("grid_format", ["UGRID", "HEALPix"])
def test_close_not_shared(self, grid_format):
"""Just like the old path, closing the result never closes the original."""
for i, convert in enumerate((self.old, lambda u: u.to_xarray(grid_format))):
uxds = self.get_uxds()
closed = []
uxds.set_close(lambda: closed.append("uhoh, uxds.close was called!"))
result = convert(uxds)
result.close()
assert len(closed) == 0, f"failed at i={i}"
assert result._close is None, f"failed at i={i}"

def test_lazy(self, tmp_path):
"""Just like the old path: dask-backed and file-backed data stay lazy,
and closing the result doesn't close the original.
"""

def fail(*args, **kwargs):
raise RuntimeError("compute was called")

# dask-backed variables and coordinates
uxds = self.get_uxds()
chunked = uxds.chunk({"n_face": 2})
with dask.config.set(scheduler=fail):
new = chunked.to_xarray()
old = self.old(chunked)
assert new.chunks == old.chunks == chunked.chunks
for k in old.variables:
assert type(new[k].variable._data) is type(old[k].variable._data), k
xr.testing.assert_identical(new, old)

# lazily loaded file-backed variables
path = tmp_path / "data.nc"
uxds.to_netcdf(path)
opened = ux.UxDataset(xr.open_dataset(path), uxgrid=uxds.uxgrid)
new, old = opened.to_xarray(), self.old(opened)
for k in old.data_vars:
assert new[k].variable._in_memory == old[k].variable._in_memory is False
self.assert_same_variables(new, old)

new.close()
opened.load() # original's file handles are still open and readable
xr.testing.assert_identical(new.load(), old.load())
opened.close()


class TestUxDatasetMimicsUxDataArrayMethods:
"""Testing behavior of UxDataset methods which simply apply UxDataArray methods iteratively,
such as UxDataset.zonal_mean().
Expand Down
Loading
Loading