diff --git a/benchmarks/to_xarray.py b/benchmarks/to_xarray.py new file mode 100644 index 000000000..a419e9aee --- /dev/null +++ b/benchmarks/to_xarray.py @@ -0,0 +1,46 @@ +"""Benchmarks for to_xarray() methods, +like UxDataset.to_xarray() and UxDataArray.to_xarray(). +""" + +import numpy as np +import uxarray as ux + + +def _assign_coords(obj, n_coords): + n_face = obj.sizes["n_face"] + return obj.assign_coords( + {f"c{i}": ("n_face", np.arange(n_face, dtype=float)) for i in range(n_coords)} + ) + + +class UxDatasetToXarray: + """Benchmark ``UxDataset.to_xarray()``, which is dominated by per-variable overhead.""" + + params = [[(100, 10), (1000, 100)]] + param_names = ["n_vars_and_n_coords"] + + def setup(self, n_vars_and_n_coords): + n_vars, n_coords = n_vars_and_n_coords + uxds = ux.tutorial.open_dataset("outCSne30-vortex") + uxds = uxds.assign({f"v{i}": uxds["psi"]+i for i in range(n_vars-1)}) + self.uxds = _assign_coords(uxds, n_coords) + + def time_to_xarray(self, n_vars_and_n_coords): + self.uxds.to_xarray() + + def time_to_xarray_healpix(self, n_vars_and_n_coords): + self.uxds.to_xarray("HEALPix") + + +class UxDataArrayToXarray: + """Benchmark ``UxDataArray.to_xarray()`` with many (n_face-sized) coords.""" + + params = [[10]] + param_names = ["n_coords"] + + def setup(self, n_coords): + uxds = ux.tutorial.open_dataset("outCSne30-vortex") + self.uxda = _assign_coords(uxds["psi"], n_coords) + + def time_to_xarray(self, n_coords): + self.uxda.to_xarray() diff --git a/test/core/test_dataarray.py b/test/core/test_dataarray.py index 970389221..dcd42309f 100644 --- a/test/core/test_dataarray.py +++ b/test/core/test_dataarray.py @@ -1,10 +1,12 @@ import warnings +import dask import numpy as np import uxarray as ux from uxarray.errors import DataCenteringError, DimensionError, GridInvalidError from uxarray.grid import neighbors from uxarray.grid.geometry import _build_polygon_shells, _build_corrected_polygon_shells from uxarray.core.dataset import UxDataset, UxDataArray +import xarray as xr import pytest @@ -599,3 +601,152 @@ def test_uxdataarray_astype_returns_uxdataarray(): result = obj.astype('float32') assert obj.dtype == np.float32 # the original dtype was float32 assert result.identical(obj) # so astype() should be a no-op. + + +class TestUxDataArrayToXarray: + """``UxDataArray.to_xarray()`` must behave exactly like old "naive" implementation: + result = xr.DataArray(uxda) + """ + + def get_uxds(self): + """quad-hexagon plus a time-dependent var, extra coords, attrs and encodings.""" + uxds = ux.tutorial.open_dataset("quad-hexagon") + uxds = uxds.assign( + { + "var_with_time": uxds["t2m"] + * xr.DataArray([10, 20, 30], dims="time", coords={"time": np.arange(3)}) + } + ) + uxds = uxds.assign_coords( + face_id=("n_face", np.arange(uxds.sizes["n_face"]), {"role": "id"}), + level=850.0, + ) + uxds.variables["var_with_time"].attrs["units"] = "K" + uxds.variables["face_id"].encoding = {"dtype": "int32"} + uxds.variables["time"].encoding = {"dtype": "int64"} + return uxds + + def get_uxda(self): + """A named UxDataArray with an indexed coord, non-index coords and encodings.""" + uxda = self.get_uxds()["var_with_time"] + # Set on the extracted array so the main-variable encoding is guaranteed to be present + uxda.variable.encoding = {"dtype": "float32", "_FillValue": -999.0} + return uxda + + def old(self, uxda): + """The previous, "naive" implementation.""" + return xr.DataArray(uxda) + + def assert_same_encodings(self, new, old): + """Main-variable encoding, coord types/encodings and index types match.""" + assert new.variable.encoding == old.variable.encoding + assert set(new.coords) == set(old.coords) + for k in old.coords: + assert type(new.coords[k].variable) is type(old.coords[k].variable), k + np.testing.assert_equal(new.coords[k].values, old.coords[k].values, err_msg=k) + assert set(new.xindexes) == set(old.xindexes) + for k in old.xindexes: + assert type(new.xindexes[k]) is type(old.xindexes[k]), k + + def test_identical_to_old(self): + """Contents, name, dims, encodings, and ``_close`` match the old result.""" + uxda = self.get_uxda() + assert uxda.variable.encoding # sanity: there is encoding that could leak through + new, old = uxda.to_xarray(), self.old(uxda) + xr.testing.assert_identical(new, old) + assert new.name == old.name == "var_with_time" + assert new.dims == old.dims + assert new._close is old._close is None + self.assert_same_encodings(new, old) + + def test_data_sharing_matches_old(self): + """Whether the data and each coord's data is shared with the original matches the old path.""" + uxda = self.get_uxda() + new, old = uxda.to_xarray(), self.old(uxda) + shares_new = np.shares_memory(new.values, uxda.values) + shares_old = np.shares_memory(old.values, uxda.values) + assert shares_new == shares_old + for k in uxda.coords: + shares_new = np.shares_memory(new.coords[k].values, uxda.coords[k].values) + shares_old = np.shares_memory(old.coords[k].values, uxda.coords[k].values) + assert shares_new == shares_old, k + + def test_modifying_result_leaves_original_unchanged(self): + """Mutating or restructuring the result never changes the original, same as the old path.""" + + def snapshot(da): + return ( + da.name, + dict(da.attrs), + dict(da.variable.encoding), + {k: (dict(c.attrs), dict(c.encoding)) for k, c in da.coords.items()}, + ) + + for convert in (self.old, lambda u: u.to_xarray()): + uxda = self.get_uxda() + before = snapshot(uxda) + result = convert(uxda) + + result.attrs["added"] = 1 + result.variable.encoding["added"] = 1 + for k in ("face_id", "time"): + result.coords[k].attrs["added"] = 1 + result.coords[k].encoding["added"] = 1 + result.variable.attrs = {"replaced": 1} + result.variable.encoding = {"replaced": 1} + result.coords["extra"] = ("n_face", np.zeros(uxda.sizes["n_face"])) + del result.coords["level"] + result.name = "renamed" + + assert snapshot(uxda) == before + + def test_nested_mutables_match_old(self): + """Whether nested attrs/encoding values leak into the original matches the old path.""" + leaked = [] + for convert in (self.old, lambda u: u.to_xarray()): + uxda = self.get_uxda() + uxda.attrs["nested"] = [0] + uxda.coords["face_id"].attrs["nested"] = [0] + uxda.coords["face_id"].encoding["nested"] = [0] + result = convert(uxda) + result.attrs["nested"].append(1) + result.coords["face_id"].attrs["nested"].append(1) + result.coords["face_id"].encoding["nested"].append(1) + leaked_here = ( + uxda.attrs["nested"], + uxda.coords["face_id"].attrs["nested"], + uxda.coords["face_id"].encoding["nested"], + ) + leaked.append(leaked_here) + assert leaked[0] == leaked[1] + + def test_lazy(self, tmp_path): + """Just like the old path: dask-backed and file-backed data stay lazy, + and closing the result doesn't close the original. + """ + + def fail(*args, **kwargs): + raise RuntimeError("compute was called") + + # dask-backed data and coordinates + uxds = self.get_uxds() + chunked = uxds["var_with_time"].chunk({"n_face": 2}) + with dask.config.set(scheduler=fail): + new = chunked.to_xarray() + old = self.old(chunked) + assert new.chunks == old.chunks == chunked.chunks + assert type(new.variable._data) is type(old.variable._data) + for k in old.coords: + assert type(new.coords[k].variable._data) is type(old.coords[k].variable._data), k + xr.testing.assert_identical(new, old) + + # lazily loaded file-backed data + path = tmp_path / "data.nc" + uxds.to_netcdf(path) + opened = ux.UxDataset(xr.open_dataset(path), uxgrid=uxds.uxgrid) + uxda = opened["var_with_time"] + new, old = uxda.to_xarray(), self.old(uxda) + assert new.variable._in_memory == old.variable._in_memory is False + self.assert_same_encodings(new, old) + xr.testing.assert_identical(new.load(), old.load()) + opened.close() diff --git a/test/core/test_dataset.py b/test/core/test_dataset.py index 20f061dcb..9fd2c502a 100644 --- a/test/core/test_dataset.py +++ b/test/core/test_dataset.py @@ -1,3 +1,5 @@ +import dask +import numpy as np import numpy.testing as nt import xarray as xr import uxarray as ux @@ -5,8 +7,6 @@ from uxarray import UxDataset import pytest -import numpy as np - @pytest.fixture() def healpix_sample_ds(): @@ -374,6 +374,159 @@ def test_uxdataset_astype_returns_uxdataset(): assert result.identical(obj) # so astype() should be a no-op. +class TestUxDatasetToXarray: + """``UxDataset.to_xarray()`` must behave exactly like old "naive" implementation: + result = xr.Dataset(uxds.data_vars, coords=uxds.coords, attrs=uxds.attrs) + """ + + def get_uxds(self): + """quad-hexagon plus a time-dependent var, extra coords, attrs and encodings.""" + uxds = ux.tutorial.open_dataset("quad-hexagon") + uxds = uxds.assign( + { + "var_with_time": uxds["t2m"] + * xr.DataArray([10, 20, 30], dims="time", coords={"time": np.arange(3)}) + } + ) + uxds = uxds.assign_coords( + face_id=("n_face", np.arange(uxds.sizes["n_face"]), {"role": "id"}), + level=850.0, + ) + uxds.attrs["title"] = "test" + uxds.variables["t2m"].attrs["units"] = "K" + uxds.variables["t2m"].encoding = {"dtype": "float32", "_FillValue": -999.0} + uxds.variables["face_id"].encoding = {"dtype": "int32"} + uxds.variables["time"].encoding = {"dtype": "int64"} + uxds.encoding = {"unlimited_dims": {"time"}} + return uxds + + def old(self, uxds): + """The previous, "naive" implementation.""" + return xr.Dataset(uxds.data_vars, coords=uxds.coords, attrs=uxds.attrs) + + def assert_same_variables(self, new, old): + """Variable types, attrs, encodings and index types match.""" + assert set(new.variables) == set(old.variables) + for k in old.variables: + assert type(new.variables[k]) is type(old.variables[k]), k + assert new.variables[k].attrs == old.variables[k].attrs, k + assert new.variables[k].encoding == old.variables[k].encoding, k + assert set(new.xindexes) == set(old.xindexes) + for k in old.xindexes: + assert type(new.xindexes[k]) is type(old.xindexes[k]), k + + def test_identical_to_old(self): + """Contents, variable encodings, dataset encoding and ``_close`` match the old result.""" + uxds = self.get_uxds() + new, old = uxds.to_xarray(), self.old(uxds) + xr.testing.assert_identical(new, old) + self.assert_same_variables(new, old) + assert new.encoding == old.encoding == {} + assert new._close is old._close is None + + def test_data_sharing_matches_old(self): + """Whether each variable's data is shared with the original matches the old path.""" + uxds = self.get_uxds() + new, old = uxds.to_xarray(), self.old(uxds) + for k in uxds.variables: + shares_new = np.shares_memory(new[k].values, uxds[k].values) + shares_old = np.shares_memory(old[k].values, uxds[k].values) + assert shares_new == shares_old, k + + def test_modifying_result_leaves_original_unchanged(self): + """Mutating or restructuring the result never changes the original, same as the old path.""" + + def snapshot(ds): + return ( + dict(ds.attrs), + {k: (dict(v.attrs), dict(v.encoding)) for k, v in ds.variables.items()}, + ) + + for convert in (self.old, lambda u: u.to_xarray()): + uxds = self.get_uxds() + before = snapshot(uxds) + result = convert(uxds) + + result.attrs["added"] = 1 + for k in ("t2m", "face_id", "time"): + result.variables[k].attrs["added"] = 1 + result.variables[k].encoding["added"] = 1 + result.variables["var_with_time"].attrs = {"replaced": 1} + result.variables["var_with_time"].encoding = {"replaced": 1} + result["extra"] = ("n_face", np.zeros(uxds.sizes["n_face"])) + del result["var_with_time"] + + assert snapshot(uxds) == before + + def test_nested_mutables_match_old(self): + """Whether nested attrs/encoding values leak into the original matches the old path.""" + leaked = [] + for convert in (self.old, lambda u: u.to_xarray()): + uxds = self.get_uxds() + uxds.attrs["nested"] = [0] + uxds.variables["t2m"].attrs["nested"] = [0] + uxds.variables["face_id"].attrs["nested"] = [0] + uxds.variables["face_id"].encoding["nested"] = [0] + result = convert(uxds) + result.attrs["nested"].append(1) + result.variables["t2m"].attrs["nested"].append(1) + result.variables["face_id"].attrs["nested"].append(1) + result.variables["face_id"].encoding["nested"].append(1) + leaked_here = ( + uxds.attrs["nested"], + uxds.variables["t2m"].attrs["nested"], + uxds.variables["face_id"].attrs["nested"], + uxds.variables["face_id"].encoding["nested"], + ) + leaked.append(leaked_here) + assert leaked[0] == leaked[1] + + @pytest.mark.parametrize("grid_format", ["UGRID", "HEALPix"]) + def test_close_not_shared(self, grid_format): + """Just like the old path, closing the result never closes the original.""" + for i, convert in enumerate((self.old, lambda u: u.to_xarray(grid_format))): + uxds = self.get_uxds() + closed = [] + uxds.set_close(lambda: closed.append("uhoh, uxds.close was called!")) + result = convert(uxds) + result.close() + assert len(closed) == 0, f"failed at i={i}" + assert result._close is None, f"failed at i={i}" + + def test_lazy(self, tmp_path): + """Just like the old path: dask-backed and file-backed data stay lazy, + and closing the result doesn't close the original. + """ + + def fail(*args, **kwargs): + raise RuntimeError("compute was called") + + # dask-backed variables and coordinates + uxds = self.get_uxds() + chunked = uxds.chunk({"n_face": 2}) + with dask.config.set(scheduler=fail): + new = chunked.to_xarray() + old = self.old(chunked) + assert new.chunks == old.chunks == chunked.chunks + for k in old.variables: + assert type(new[k].variable._data) is type(old[k].variable._data), k + xr.testing.assert_identical(new, old) + + # lazily loaded file-backed variables + path = tmp_path / "data.nc" + uxds.to_netcdf(path) + opened = ux.UxDataset(xr.open_dataset(path), uxgrid=uxds.uxgrid) + new, old = opened.to_xarray(), self.old(opened) + for k in old.data_vars: + assert new[k].variable._in_memory == old[k].variable._in_memory is False + self.assert_same_variables(new, old) + + new.close() + opened.load() # original's file handles are still open and readable + xr.testing.assert_identical(new.load(), old.load()) + opened.close() + + class TestUxDatasetMimicsUxDataArrayMethods: """Testing behavior of UxDataset methods which simply apply UxDataArray methods iteratively, such as UxDataset.zonal_mean(). diff --git a/uxarray/core/dataarray.py b/uxarray/core/dataarray.py index 111a77c1d..b1b840f22 100644 --- a/uxarray/core/dataarray.py +++ b/uxarray/core/dataarray.py @@ -626,7 +626,20 @@ def to_dataset( return uxds def to_xarray(self) -> xr.DataArray: - return xr.DataArray(self) + """returns a shallow copy of this ``UxDataArray`` as a plain ``xr.DataArray``.""" + # Equivalent to the "naive" xr.DataArray(self), but internally here + # uses _construct_direct() for efficiency, to avoid validation checks, + # since a valid UxDataArray is already guaranteed to be a valid xr.DataArray. + variable = self._variable.copy(deep=False) + variable.encoding = {} # matches xr.DataArray(self) behavior + coords = {k: v.copy(deep=True) for k, v in self._coords.items()} + # ^ also matches xr.DataArray(self) behavior, which copies coords deeply. + return xr.DataArray._construct_direct( + variable, + coords, + self._name, + dict(self._indexes), + ) def astype(self, dtype, **kw_super): """Copy of this uxarray object, with data cast to a specified type. diff --git a/uxarray/core/dataset.py b/uxarray/core/dataset.py index 69c0b95c9..ee3e6b195 100644 --- a/uxarray/core/dataset.py +++ b/uxarray/core/dataset.py @@ -938,25 +938,37 @@ def neighborhood(self, r: float = 1.0) -> DatasetNeighborhood: return DatasetNeighborhood(self, r=r) def to_xarray(self, grid_format: str = "UGRID") -> xr.Dataset: - """ - Converts a ``ux.UXDataset`` to a ``xr.Dataset``. + """returns a shallow copy of this ``UxDataset`` as a plain ``xr.Dataset``. Parameters ---------- grid_format : str, default="UGRID" - The format in which to convert the grid. Supported values are "UGRID" and "HEALPix". The dimensions will - match the selected grid format. + The format in which to convert the grid. Supported values are "UGRID" and "HEALPix". + If "HEALPix", renames "n_face" dimension to "cell" in the result. + Otherwise, dimensions are kept unchanged. Returns ------- xr.Dataset - The ``ux.UXDataset`` represented as a ``xr.Dataset`` + The ``ux.UxDataset`` represented as a ``xr.Dataset`` """ + # Equivalent to the "naive" xr.Dataset(self.data_vars, coords=self.coords, attrs=self.attrs) + # but internally here uses _construct_direct() for efficiency, to avoid validation checks, + # since a valid UxDataset is already guaranteed to be a valid xr.Dataset. + variables = {k: v.copy(deep=False) for k, v in self._variables.items()} + for k, v in variables.items(): + if k not in self._coord_names: + v.encoding = {} # matches the "naive" implementation's behavior + result = xr.Dataset._construct_direct( + variables=variables, + coord_names=set(self._coord_names), + dims=dict(self._dims), + attrs=dict(self.attrs), + indexes=dict(self._indexes), + ) if grid_format == "HEALPix": - ds = self.rename_dims({"n_face": "cell"}) - return xr.Dataset(ds.data_vars, coords=ds.coords, attrs=ds.attrs) - - return xr.Dataset(self.data_vars, coords=self.coords, attrs=self.attrs) + result = result.rename_dims({"n_face": "cell"}) + return result def astype(self, dtype, **kw_super): """Copy of this uxarray object, with data cast to a specified type.