Merge branch 'master' into retire/strict-provenance

Author: dimitri-yatsenko

.github/workflows/test.yaml

@@ -27,6 +27,14 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # Exercise both ends of the supported range (requires-python
+        # >=3.10,<3.15). The version-pinned pixi environments are defined in
+        # pyproject.toml under [tool.pixi.environments].
+        environment: [test-py310, test-py314]
+    name: test (${{ matrix.environment }})
     steps:
       - uses: actions/checkout@v4
 
@@ -35,9 +43,10 @@ jobs:
         with:
           cache: true
           locked: false
+          environments: ${{ matrix.environment }}
 
       - name: Run tests
-        run: pixi run -e test test-cov
+        run: pixi run -e ${{ matrix.environment }} test-cov
 
   # Unit tests run without containers (faster feedback)
   unit-tests:

pyproject.toml

@@ -19,7 +19,7 @@ dependencies = [
   "packaging",
 ]
 
-requires-python = ">=3.10,<3.14"
+requires-python = ">=3.10,<3.15"
 authors = [
   {name = "Dimitri Yatsenko", email = "dimitri@datajoint.com"},
   {name = "Thinh Nguyen", email = "thinh@datajoint.com"},
@@ -57,6 +57,11 @@ keywords = [
 # https://pypi.org/classifiers/
 classifiers = [
   "Programming Language :: Python",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
   "Development Status :: 5 - Production/Stable",
   "Intended Audience :: Science/Research",
   "Intended Audience :: Healthcare Industry",
@@ -250,10 +255,22 @@ datajoint = { path = ".", editable = true, extras = ["test"] }
 [tool.pixi.feature.dev.pypi-dependencies]
 datajoint = { path = ".", editable = true, extras = ["dev", "test"] }
 
+# Pin the interpreter for the min/max supported versions so CI can exercise
+# both ends of the requires-python range (see .github/workflows/test.yaml).
+[tool.pixi.feature.py310.dependencies]
+python = "3.10.*"
+
+[tool.pixi.feature.py314.dependencies]
+python = "3.14.*"
+
 [tool.pixi.environments]
 default = { solve-group = "default" }
 dev = { features = ["dev"], solve-group = "default" }
 test = { features = ["test"], solve-group = "default" }
+# Version-pinned test environments (each solves independently — a solve-group
+# cannot span different Python versions).
+test-py310 = { features = ["test", "py310"] }
+test-py314 = { features = ["test", "py314"] }
 
 [tool.pixi.tasks]
 # Tests use testcontainers - no manual setup required
@@ -265,7 +282,7 @@ services-down = "docker compose down"
 test-external = { cmd = "DJ_USE_EXTERNAL_CONTAINERS=1 pytest tests/", depends-on = ["services-up"] }
 
 [tool.pixi.dependencies]
-python = ">=3.10,<3.14"
+python = ">=3.10,<3.15"
 graphviz = ">=13.1.2,<14"
 
 [tool.pixi.activation]

src/datajoint/__init__.py

@@ -284,7 +284,7 @@ def FreeTable(conn_or_name, full_table_name: str | None = None) -> _FreeTable:
     "diagram": (".diagram", None),  # Return the module itself
     # cli imports click
     "cli": (".cli", "cli"),
-    # gc — exposed lazily so `dj.gc.scan(...)` works as documented in gc.py
+    # gc — exposed lazily so `dj.gc.GarbageCollector(...)` works as documented in gc.py
     # and in the user docs (how-to/garbage-collection.md).
     "gc": (".gc", None),  # Return the module itself
 }

src/datajoint/autopopulate.py

@@ -22,6 +22,14 @@
 
 logger = logging.getLogger(__name__.split(".")[0])
 
+# Parallel populate hands the live table and its open connection to workers by
+# process inheritance, so it requires the `fork` start method. Python 3.14
+# changed the default from `fork` to `forkserver`, which pickles the payload
+# instead — that neither carries a live DB connection nor the table's internal
+# state, and it deadlocks. Pin to `fork` where available (all POSIX platforms);
+# fall back to the platform default elsewhere.
+_MP_START_METHOD = "fork" if "fork" in mp.get_all_start_methods() else None
+
 
 # --- helper functions for multiprocessing --
 
@@ -30,7 +38,8 @@ def _initialize_populate(table: Table, jobs: Job | None, populate_kwargs: dict[s
     """
     Initialize a worker process for multiprocessing.
 
-    Saves the unpickled table to the current process and reconnects to database.
+    Stores the inherited table on the worker process and reconnects to database
+    (the parent closes its connection before forking; each worker reopens one).
 
     Parameters
     ----------
@@ -477,7 +486,9 @@ def _populate_direct(
                 if hasattr(self.connection._conn, "ctx"):
                     del self.connection._conn.ctx
                 with (
-                    mp.Pool(processes, _initialize_populate, (self, None, populate_kwargs)) as pool,
+                    mp.get_context(_MP_START_METHOD).Pool(
+                        processes, _initialize_populate, (self, None, populate_kwargs)
+                    ) as pool,
                     tqdm(desc="Processes: ", total=nkeys) if display_progress else contextlib.nullcontext() as progress_bar,
                 ):
                     for status in pool.imap(_call_populate1, keys, chunksize=1):
@@ -573,7 +584,9 @@ def handler(signum, frame):
                     if hasattr(self.connection._conn, "ctx"):
                         del self.connection._conn.ctx  # SSLContext is not pickleable
                     with (
-                        mp.Pool(processes, _initialize_populate, (self, self.jobs, populate_kwargs)) as pool,
+                        mp.get_context(_MP_START_METHOD).Pool(
+                            processes, _initialize_populate, (self, self.jobs, populate_kwargs)
+                        ) as pool,
                         tqdm(desc="Processes: ", total=nkeys)
                         if display_progress
                         else contextlib.nullcontext() as progress_bar,
@@ -834,8 +847,6 @@ def _update_job_metadata(self, key, start_time, duration, version):
 
         pk_condition = make_condition(self, key, set())
         self.connection.query(
-            f"UPDATE {self.full_table_name} SET "
-            "_job_start_time=%s, _job_duration=%s, _job_version=%s "
-            f"WHERE {pk_condition}",
+            f"UPDATE {self.full_table_name} SET _job_start_time=%s, _job_duration=%s, _job_version=%s WHERE {pk_condition}",
             args=(start_time, duration, version[:64] if version else ""),
         )

src/datajoint/builtin_codecs/hash.py

@@ -21,7 +21,7 @@ class HashCodec(Codec):
     The database column stores JSON metadata: ``{hash, store, size}``.
     Duplicate content is automatically deduplicated across all tables.
 
-    Deletion: Requires garbage collection via ``dj.gc.collect()``.
+    Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
 
     External only - requires @ modifier.
 

src/datajoint/builtin_codecs/npy.py

@@ -272,7 +272,7 @@ class Recording(dj.Manual):
         - Path: ``{schema}/{table}/{pk}/{attribute}.npy``
         - Database column: JSON with ``{path, store, dtype, shape}``
 
-    Deletion: Requires garbage collection via ``dj.gc.collect()``.
+    Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
 
     See Also
     --------

src/datajoint/builtin_codecs/object.py

@@ -53,7 +53,7 @@ def make(self, key):
 
             {store_root}/{schema}/{table}/{pk}/{field}/
 
-    Deletion: Requires garbage collection via ``dj.gc.collect()``.
+    Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
 
     Comparison with hash-addressed::
 

src/datajoint/builtin_codecs/schema.py

@@ -114,7 +114,7 @@ def _build_path(
         Build schema-addressed storage path.
 
         Constructs a path that mirrors the database schema structure:
-        ``{schema}/{table}/{pk_values}/{field}{ext}``
+        ``{schema_prefix}/{schema}/{table}/{pk_values}/{field}{ext}``
 
         Supports partitioning if configured in the store.
 
@@ -150,6 +150,7 @@ def _build_path(
         spec = config.get_store_spec(store_name)
         partition_pattern = spec.get("partition_pattern")
         token_length = spec.get("token_length", 8)
+        schema_prefix = spec["schema_prefix"]  # always present: settings applies the default
 
         return build_object_path(
             schema=schema,
@@ -159,6 +160,7 @@ def _build_path(
             ext=ext,
             partition_pattern=partition_pattern,
             token_length=token_length,
+            schema_prefix=schema_prefix,
         )
 
     def _get_backend(self, store_name: str | None = None, config=None):

src/datajoint/codecs.py

@@ -38,6 +38,7 @@ class MyTable(dj.Manual):
 
 from __future__ import annotations
 
+import json
 import logging
 from abc import ABC, abstractmethod
 from typing import Any
@@ -219,6 +220,45 @@ def validate(self, value: Any) -> None:
         """
         pass
 
+    def referenced_paths(self, stored: Any) -> list[tuple[str, str | None]]:
+        """
+        Return the external store paths this stored value references.
+
+        Garbage collection and delete-time cleanup call this on the *stored*
+        (encoded) representation of a column value — the JSON/dict already in
+        the database — so discovery stays metadata-only (no download or decode).
+
+        The default recognizes DataJoint's standard external-storage metadata:
+        a dict (or JSON string) carrying a ``path`` and optional ``store``. This
+        covers hash-addressed (``<hash@>``/``<blob@>``/``<attach@>``) and
+        schema-addressed (``<object@>``/``<npy@>`` and any :class:`SchemaCodec`
+        subclass) storage with no extra work, and returns an empty list for
+        values that are not externally stored (e.g. in-table ``<blob>``).
+
+        Override only if your codec stores external references in a non-standard
+        shape. Returning the paths here is what lets garbage collection see a
+        custom codec's files as *referenced* rather than orphaned (see #1469).
+
+        Parameters
+        ----------
+        stored : any
+            The stored (encoded) value as read from the database column.
+
+        Returns
+        -------
+        list[tuple[str, str | None]]
+            ``(path, store_name)`` for each external artifact referenced.
+        """
+        value = stored
+        if isinstance(value, str):
+            try:
+                value = json.loads(value)
+            except (json.JSONDecodeError, TypeError, ValueError):
+                return []
+        if isinstance(value, dict) and "path" in value:
+            return [(value["path"], value.get("store"))]
+        return []
+
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}(name={self.name!r})>"