AssetCollection and generic storage client

CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.58.0] - 2026-07-31
+
+### Added
+
+- `tilebox-datasets`: Added `AssetCollection.from_datapoint()` for resolving STAC assets and their generated metadata,
+  storage, and authentication messages from queried datapoints.
+- `tilebox-datasets`: Added `Asset`, `Band`, and `AssetLocation` authoring types together with
+  `AssetCollection.from_assets()` and `AssetCollection.to_fields()` for compiling optimized STAC asset metadata for
+  ingestion, plus common string-valued `MediaType` constants.
+- `tilebox-datasets`: Added public schema field types for creating datasets containing supported STAC Assets, Storage,
+  Authentication, Links, Provider, and ProcessingSoftware messages, with aliases for Geometry and UUID fields.
+- `tilebox-storage`: Added async asset resolution, reading, downloading, and GeoTIFF access backed by object stores.
+
 ## [0.57.0] - 2026-07-27
 
 ### Added
@@ -429,7 +442,8 @@ the first client that does not cache data (since it's already on the local file
 - Released under the [MIT](https://opensource.org/license/mit) license.
 - Released packages: `tilebox-datasets`, `tilebox-workflows`, `tilebox-storage`, `tilebox-grpc`
 
-[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.57.0...HEAD
+[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.58.0...HEAD
+[0.58.0]: https://github.com/tilebox/tilebox-python/compare/v0.57.0...v0.58.0 
 [0.57.0]: https://github.com/tilebox/tilebox-python/compare/v0.56.0...v0.57.0
 [0.56.0]: https://github.com/tilebox/tilebox-python/compare/v0.55.1...v0.56.0
 [0.55.1]: https://github.com/tilebox/tilebox-python/compare/v0.55.0...v0.55.1

buf.gen.datasets-bufpy.yaml

@@ -0,0 +1,13 @@
+# yaml-language-server: $schema=https://www.schemastore.org/buf.gen.json
+version: v2
+plugins:
+  - remote: buf.build/bufbuild/py:v0.2.0
+    out: tilebox-datasets/tilebox/datasets
+    opt: init_files=false
+inputs:
+  - module: buf.build/tilebox/api
+    paths:
+      - "datasets/stac/v1"
+    # Keep this experimental package validation-blind, matching the existing generated clients.
+    exclude_types:
+      - "buf.validate.**"

prek.toml

@@ -19,7 +19,7 @@ hooks = [
 
 [[repos]]
 repo = "https://github.com/charliermarsh/ruff-pre-commit"
-rev = "v0.16.0"
+rev = "v0.16.1"
 hooks = [
   {
     id = "ruff-check",

pyproject.toml

@@ -119,4 +119,5 @@ exclude = [
     "**/*_pb2.py",
     "**/*_pb2.pyi",
     "**/*pb2_grpc.py",
+    "**/*pb.py",
 ]

tilebox-datasets/pyproject.toml

@@ -32,6 +32,8 @@ dependencies = [
     "pandas>=2.1",
     "shapely>=2",
     "promise>=2.3",
+    "typing-extensions>=4.5",
+    "protobuf-py>=0.1.1",
 ]
 
 [dependency-groups]

tilebox-datasets/tests/data/test_datasets.py

@@ -2,6 +2,7 @@
 from uuid import uuid4
 
 import pytest
+from google.protobuf.descriptor_pb2 import FieldDescriptorProto
 from hypothesis import given
 
 from tests.data.datasets import (
@@ -26,6 +27,7 @@
     FieldRole,
     ListDatasetsResponse,
 )
+from tilebox.datasets.schema import Assets, Authentication, Links, ProcessingSoftware, Provider, Storage
 from tilebox.datasets.service import TileboxDatasetService
 
 
@@ -61,6 +63,28 @@ def test_field_from_dict(field_dict: FieldDict) -> None:
     ]
 
 
+@pytest.mark.parametrize(
+    ("message_type", "message_name"),
+    [
+        (Assets, "datasets.stac.v1.Assets"),
+        (Authentication, "datasets.stac.v1.Authentication"),
+        (Links, "datasets.stac.v1.Links"),
+        (Provider, "datasets.stac.v1.Provider"),
+        (ProcessingSoftware, "datasets.stac.v1.ProcessingSoftware"),
+        (Storage, "datasets.stac.v1.Storage"),
+    ],
+)
+def test_message_types_can_define_dataset_fields(message_type: type, message_name: str) -> None:
+    scalar = Field.from_dict({"name": "metadata", "type": message_type})
+    repeated = Field.from_dict({"name": "metadata", "type": list[message_type]})  # type: ignore[typeddict-item,valid-type]
+
+    assert scalar.descriptor.type == FieldDescriptorProto.TYPE_MESSAGE
+    assert scalar.descriptor.type_name == f".{message_name}"
+    assert scalar.descriptor.label == FieldDescriptorProto.LABEL_OPTIONAL
+    assert repeated.descriptor.type_name == f".{message_name}"
+    assert repeated.descriptor.label == FieldDescriptorProto.LABEL_REPEATED
+
+
 @given(fields())
 def test_fields_to_message_and_back(field: Field) -> None:
     assert Field.from_message(field.to_message()) == field

tilebox-datasets/tests/protobuf_conversion/test_protobuf_xarray.py

@@ -14,8 +14,22 @@
 
 from tests.data.datapoint import example_datapoints
 from tests.example_dataset.example_dataset_pb2 import ExampleDatapoint
-from tilebox.datasets.datasets.stac.v1.asset_pb2 import Assets
+from tilebox.datasets.datasets.stac.v1.asset_pb import Assets
+from tilebox.datasets.datasets.stac.v1.asset_pb2 import Asset as AssetPB2
+from tilebox.datasets.datasets.stac.v1.asset_pb2 import Assets as AssetsPB2
+from tilebox.datasets.datasets.stac.v1.authentication_pb import Authentication
+from tilebox.datasets.datasets.stac.v1.authentication_pb2 import Authentication as AuthenticationPB2
+from tilebox.datasets.datasets.stac.v1.core_pb import Links, Provider
+from tilebox.datasets.datasets.stac.v1.core_pb2 import Link as LinkPB2
+from tilebox.datasets.datasets.stac.v1.core_pb2 import Links as LinksPB2
+from tilebox.datasets.datasets.stac.v1.core_pb2 import Provider as ProviderPB2
+from tilebox.datasets.datasets.stac.v1.processing_pb import ProcessingSoftware
+from tilebox.datasets.datasets.stac.v1.processing_pb2 import ProcessingSoftware as ProcessingSoftwarePB2
+from tilebox.datasets.datasets.stac.v1.storage_pb import Storage
+from tilebox.datasets.datasets.stac.v1.storage_pb2 import Storage as StoragePB2
+from tilebox.datasets.protobuf_conversion.field_types import _AssetsDisplay
 from tilebox.datasets.protobuf_conversion.protobuf_xarray import MessageToXarrayConverter
+from tilebox.datasets.protobuf_conversion.to_protobuf import to_messages
 from tilebox.datasets.query.time_interval import timestamp_to_datetime, us_to_datetime
 
 
@@ -103,43 +117,100 @@ def test_convert_datapoint(datapoint: ExampleDatapoint) -> None:  # noqa: PLR091
         assert isinstance(dataset.some_repeated_geometry[i].item(), Polygon | MultiPolygon)
 
 
-def test_convert_unknown_message_fields_as_objects() -> None:
+def test_convert_stac_messages_to_protobuf_py() -> None:
+    message_types = {
+        "assets": ("datasets.stac.v1.Assets", AssetsPB2(assets=[AssetPB2(key="preview")]), _AssetsDisplay),
+        "authentication": ("datasets.stac.v1.Authentication", AuthenticationPB2(), Authentication),
+        "links": ("datasets.stac.v1.Links", LinksPB2(links=[LinkPB2(href="https://example.com")]), Links),
+        "provider": ("datasets.stac.v1.Provider", ProviderPB2(name="Tilebox"), Provider),
+        "processing_software": (
+            "datasets.stac.v1.ProcessingSoftware",
+            ProcessingSoftwarePB2(versions={"processor": "1.0"}),
+            ProcessingSoftware,
+        ),
+        "storage": ("datasets.stac.v1.Storage", StoragePB2(), Storage),
+    }
     file_descriptor = descriptor_pb2.FileDescriptorProto(
         name="tests/protobuf_conversion/stac_datapoint.proto",
         package="tests.protobuf_conversion",
-        dependency=["datasets/stac/v1/asset.proto"],
+        dependency=[
+            "datasets/stac/v1/asset.proto",
+            "datasets/stac/v1/authentication.proto",
+            "datasets/stac/v1/core.proto",
+            "datasets/stac/v1/processing.proto",
+            "datasets/stac/v1/storage.proto",
+        ],
     )
     message_descriptor = file_descriptor.message_type.add(name="StacDatapoint")
-    message_descriptor.field.add(
-        name="assets",
-        number=1,
-        label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL,
-        type=descriptor_pb2.FieldDescriptorProto.TYPE_MESSAGE,
-        type_name=".datasets.stac.v1.Assets",
-    )
-    message_descriptor.field.add(
-        name="related_assets",
-        number=2,
-        label=descriptor_pb2.FieldDescriptorProto.LABEL_REPEATED,
-        type=descriptor_pb2.FieldDescriptorProto.TYPE_MESSAGE,
-        type_name=".datasets.stac.v1.Assets",
-    )
+    for index, (field_name, (message_name, _, _)) in enumerate(message_types.items()):
+        message_descriptor.field.add(
+            name=field_name,
+            number=index * 2 + 1,
+            label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL,
+            type=descriptor_pb2.FieldDescriptorProto.TYPE_MESSAGE,
+            type_name=f".{message_name}",
+        )
+        message_descriptor.field.add(
+            name=f"related_{field_name}",
+            number=index * 2 + 2,
+            label=descriptor_pb2.FieldDescriptorProto.LABEL_REPEATED,
+            type=descriptor_pb2.FieldDescriptorProto.TYPE_MESSAGE,
+            type_name=f".{message_name}",
+        )
     descriptor = Default().AddSerializedFile(file_descriptor.SerializeToString())
     message_type = GetMessageClass(descriptor.message_types_by_name["StacDatapoint"])
 
-    assets = Assets()
-    related_assets = Assets()
-    messages = [message_type(), message_type(assets=assets, related_assets=[related_assets])]
+    values = {}
+    for field_name, (_, source_value, _) in message_types.items():
+        values[field_name] = source_value
+        values[f"related_{field_name}"] = [source_value]
+    messages = [message_type(), message_type(**values)]
 
     converter = MessageToXarrayConverter()
     converter.convert_all(messages)
     dataset = converter.finalize("time")
 
-    assert dataset.assets.dtype == object
-    assert dataset.assets[0].item() is None
-    assert dataset.assets[1].item() == assets
-    assert dataset.related_assets.dtype == object
-    assert dataset.related_assets[1, 0].item() == related_assets
+    for field_name, (_, source_value, target_type) in message_types.items():
+        assert dataset[field_name].dtype == object
+        assert dataset[field_name][0].item() is None
+        converted_value = dataset[field_name][1].item()
+        assert isinstance(converted_value, target_type)
+        assert converted_value.to_binary() == source_value.SerializeToString()
+        if field_name == "assets":
+            assert isinstance(converted_value, Assets)
+            assert str(converted_value) == "1 assets"
+            assert repr(converted_value) == "[preview]"
+
+        repeated_field_name = f"related_{field_name}"
+        assert dataset[repeated_field_name].dtype == object
+        converted_repeated_value = dataset[repeated_field_name][1, 0].item()
+        assert isinstance(converted_repeated_value, target_type)
+        assert converted_repeated_value.to_binary() == source_value.SerializeToString()
+
+    converted_assets = dataset["assets"][1].item()
+    assert isinstance(converted_assets, Assets)
+    assert repr(converted_assets) == "[preview]"
+    assert str(converted_assets) == "1 assets"
+
+    roundtripped = to_messages(dataset, message_type)
+    assert roundtripped == messages
+
+    for repeated_container in (list, tuple):
+        input_data = {}
+        record = {}
+        for field_name, (_, source_value, target_type) in message_types.items():
+            value_type = Assets if field_name == "assets" else target_type
+            target_value = value_type.from_binary(source_value.SerializeToString())
+            input_data[field_name] = [target_value]
+            input_data[f"related_{field_name}"] = [repeated_container([target_value])]
+            record[field_name] = target_value
+            record[f"related_{field_name}"] = repeated_container([target_value])
+        assert to_messages(input_data, message_type) == [message_type(**values)]
+        assert to_messages([record], message_type) == [message_type(**values)]
+
+    html = dataset._repr_html_()
+    assert "preview" in html
+    assert "access_profiles" not in html
 
 
 @given(lists(example_datapoints(generated_fields=True, missing_fields=True), min_size=5, max_size=30))

tilebox-datasets/tests/protobuf_conversion/test_to_protobuf.py

@@ -1,4 +1,8 @@
+from datetime import datetime, timezone
+
+import numpy as np
 import pandas as pd
+import pytest
 import xarray as xr
 from hypothesis import given
 from hypothesis.strategies import lists
@@ -39,3 +43,75 @@ def test_pandas_to_protobuf_messages(datapoints: list[pd.DataFrame]) -> None:
     dataframe = pd.concat(datapoints)
     converted = to_messages(dataframe, ExampleDatapoint)
     assert len(converted) == len(datapoints)
+
+
+def test_record_oriented_data_preserves_absent_fields_and_filters_missing_values() -> None:
+    time = datetime(2026, 7, 31, tzinfo=timezone.utc)
+    converted = to_messages(
+        [
+            {
+                "time": time,
+                "some_bool": True,
+                "some_int": 4,
+                "some_repeated_int": [1, None, np.nan, 2],
+            },
+            {"time": time, "some_bool": np.nan},
+        ],
+        ExampleDatapoint,
+        required_fields=["time"],
+    )
+
+    assert converted[0].some_bool is True
+    assert converted[0].some_int == 4
+    assert converted[0].some_repeated_int == [1, 2]
+    assert converted[1].some_bool is False
+    assert converted[1].some_int == 0
+
+
+def test_record_oriented_data_requires_every_record_to_have_required_fields() -> None:
+    with pytest.raises(ValueError, match=r"Record 1: Missing required field.*time"):
+        to_messages(
+            [{"time": datetime(2026, 7, 31, tzinfo=timezone.utc)}, {"some_int": 1}],
+            ExampleDatapoint,
+            required_fields=["time"],
+        )
+
+
+def test_record_oriented_data_rejects_required_values_that_convert_to_unset() -> None:
+    with pytest.raises(ValueError, match="Record 0: Field 'some_identifier': Invalid value for required field"):
+        to_messages(
+            [{"some_identifier": ""}],
+            ExampleDatapoint,
+            required_fields=["some_identifier"],
+        )
+
+
+def test_dataframe_missing_values_leave_optional_fields_unset() -> None:
+    time = datetime(2026, 7, 31, tzinfo=timezone.utc)
+    dataframe = pd.DataFrame([{"time": time, "some_bool": True}, {"time": time}])
+
+    converted = to_messages(dataframe, ExampleDatapoint, required_fields=["time"])
+
+    assert converted[0].some_bool is True
+    assert converted[1].some_bool is False
+
+
+def test_iterable_of_column_tuples_is_rejected_as_invalid_records() -> None:
+    with pytest.raises(TypeError, match="record 0 is tuple"):
+        to_messages([("time", [datetime(2026, 7, 31, tzinfo=timezone.utc)])], ExampleDatapoint)  # type: ignore[arg-type]
+
+
+def test_ignored_columns_do_not_participate_in_shape_validation() -> None:
+    converted = to_messages(
+        {"time": [datetime(2026, 7, 31, tzinfo=timezone.utc)], "id": []},
+        ExampleDatapoint,
+        required_fields=["time"],
+        ignore_fields=["id"],
+    )
+
+    assert len(converted) == 1
+
+
+def test_conversion_errors_include_record_and_field_context() -> None:
+    with pytest.raises(TypeError, match="Record 0: Field 'some_repeated_int': Expected an iterable"):
+        to_messages([{"some_repeated_int": 1}], ExampleDatapoint)