diff --git a/poetry.lock b/poetry.lock index e6c6898f..6f45500f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "aioftp" @@ -199,6 +199,18 @@ tzdata = {version = "*", markers = "python_version >= \"3.9\""} doc = ["doc8", "sphinx (>=7.0.0)", "sphinx-autobuild", "sphinx-autodoc-typehints", "sphinx_rtd_theme (>=1.3.0)"] test = ["dateparser (==1.*)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2025.2)", "simplejson (==3.*)"] +[[package]] +name = "astroid" +version = "4.0.4" +description = "An abstract syntax tree for Python with inference support." +optional = false +python-versions = ">=3.10.0" +groups = ["dev"] +files = [ + {file = "astroid-4.0.4-py3-none-any.whl", hash = "sha256:52f39653876c7dec3e3afd4c2696920e05c83832b9737afc21928f2d2eb7a753"}, + {file = "astroid-4.0.4.tar.gz", hash = "sha256:986fed8bcf79fb82c78b18a53352a0b287a73817d6dbcfba3162da36667c49a0"}, +] + [[package]] name = "asttokens" version = "3.0.1" @@ -1271,6 +1283,22 @@ files = [ {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, ] +[[package]] +name = "dill" +version = "0.4.1" +description = "serialize all of Python" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d"}, + {file = "dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa"}, +] + +[package.extras] +graph = ["objgraph (>=1.7.2)"] +profile = ["gprof2dot (>=2022.7.29)"] + [[package]] name = "distlib" version = "0.4.0" @@ -3247,8 +3275,8 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -3895,6 +3923,34 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pylint" +version = "4.0.8" +description = "python code static checker" +optional = false +python-versions = ">=3.10.0" +groups = ["dev"] +files = [ + {file = "pylint-4.0.8-py3-none-any.whl", hash = "sha256:3341c08c0aabaa4adc71516de0969f3ba5c692b56c75af4dcb4d242823fbe363"}, + {file = "pylint-4.0.8.tar.gz", hash = "sha256:1c1b2128bde5ff5e966801413080b6384d42a5782718d528c906dbb6beab94ed"}, +] + +[package.dependencies] +astroid = ">=4.0.2,<=4.1.dev0" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +dill = [ + {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, + {version = ">=0.3.6", markers = "python_version == \"3.11\""}, +] +isort = ">=5,<5.13 || >5.13,<10" +mccabe = ">=0.6,<0.8" +platformdirs = ">=2.2" +tomlkit = ">=0.10.1" + +[package.extras] +spelling = ["pyenchant (>=3.2,<4.0)"] +testutils = ["gitpython (>3)"] + [[package]] name = "pyparsing" version = "3.3.2" @@ -5332,6 +5388,18 @@ files = [ {file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"}, ] +[[package]] +name = "tomlkit" +version = "0.15.1" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "tomlkit-0.15.1-py3-none-any.whl", hash = "sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304"}, + {file = "tomlkit-0.15.1.tar.gz", hash = "sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97"}, +] + [[package]] name = "tornado" version = "6.5.7" @@ -5802,4 +5870,4 @@ web = ["streamlit"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.14" -content-hash = "e977e2830aeca92c8938c314aa38478261d87d1ab172aa480b87389ab20add4b" +content-hash = "c7b74c65dd4997b3f7ce0031157af5d2f604f82c7485a9e0d5fd6818d1bfa73b" diff --git a/pyproject.toml b/pyproject.toml index 72671b0a..8fdd021c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,7 @@ pytest-retry = "1.7.0" datamodel-code-generator = {extras = ["http"], version = ">=0.54,<0.65"} pytest-asyncio = "^1.3.0" pytest-cov = "^7.1.0" +pylint = "^4.0.8" [tool.poetry.group.docs.dependencies] sphinx = "^5.1.1" diff --git a/pysus/management/catalog.py b/pysus/management/catalog.py index 7b0eed3a..8fdc142e 100644 --- a/pysus/management/catalog.py +++ b/pysus/management/catalog.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any from pysus.api.errors import CatalogError +from pysus.api.metadata.models import DatasetGroup if TYPE_CHECKING: # pragma: no cover from pysus.api.ducklake.client import DuckLake @@ -154,19 +155,21 @@ def ensure_group( self, cursor, dataset_id: int, - name: str | None, - long_name: str | None = None, - description: str | None = None, + group: DatasetGroup | str | None, ) -> int | None: """Return the group id for (dataset, name), creating it if needed.""" - if not name: + if not group: return None - name = name.strip().upper() - # ``long_name`` is NOT NULL; fall back to the group code itself when - # the source carries no long name (e.g. SIM's legacy "D" group). - if not long_name: + if isinstance(group, DatasetGroup): + name = group.name.strip().upper() + long_name = group.long_name or name + description = group.description + else: + name = str(group).strip().upper() long_name = name + description = None + cursor.execute( "SELECT id, long_name, description FROM pysus.dataset_groups " "WHERE dataset_id = ? AND name = ?", diff --git a/pysus/management/sync.py b/pysus/management/sync.py index 0d73d7e9..441a7c74 100644 --- a/pysus/management/sync.py +++ b/pysus/management/sync.py @@ -29,6 +29,7 @@ import anyio import httpx +from pysus.api.metadata.models import DatasetGroup from pysus import CACHEPATH from pysus.api.ducklake.functional import upload_s3 from pysus.api.errors import AuthenticationError, ConnectionError @@ -1191,14 +1192,10 @@ def _catalog_rows( ) group = getattr(file, "group", None) - group_name = getattr(group, "name", None) if group is not None else None - group_name = str(group_name) if group_name else None group_id = writer.ensure_group( dataset_cursor, dataset_id, - group_name, - getattr(group, "long_name", None) if group else None, - getattr(group, "description", None) if group else None, + group, ) writer.upsert_file( @@ -1436,8 +1433,11 @@ async def _fix_misparsed_metadata( group_id = self.writer.ensure_group( cursor, dataset_id, - str(group) if group else None, - group_long_name or (str(group) if group else None), + DatasetGroup( + name=str(group), + long_name=group_long_name or str(group), + description="", + ) if group else None, ) cursor.execute( "UPDATE pysus.files SET path = ?, month = ?, " diff --git a/pysus/tests/management/test_catalog.py b/pysus/tests/management/test_catalog.py index fb763e8b..7850f828 100644 --- a/pysus/tests/management/test_catalog.py +++ b/pysus/tests/management/test_catalog.py @@ -7,6 +7,7 @@ import pyarrow as pa import pytest from pysus.management.catalog import CatalogWriter +from pysus.api.metadata.models import DatasetGroup _SCHEMA = """ CREATE SCHEMA pysus; @@ -492,7 +493,7 @@ def test_returns_none_for_empty_name(self, full_catalog): def test_creates_new_group(self, full_catalog): writer, cursor, _ = full_catalog gid = writer.ensure_group( - cursor, dataset_id=1, name="deng", long_name="Dengue" + cursor, dataset_id=1, group=DatasetGroup(name="deng", long_name="Dengue", description="") ) assert gid == 1 cursor.execute( @@ -505,10 +506,10 @@ def test_creates_new_group(self, full_catalog): def test_reuses_existing_group(self, full_catalog): writer, cursor, _ = full_catalog gid1 = writer.ensure_group( - cursor, dataset_id=1, name="deng", long_name="Dengue" + cursor, dataset_id=1, group=DatasetGroup(name="deng", long_name="Dengue", description="") ) gid2 = writer.ensure_group( - cursor, dataset_id=1, name="deng", long_name="Dengue v2" + cursor, dataset_id=1, group=DatasetGroup(name="deng", long_name="Dengue v2", description="") ) assert gid1 == gid2 cursor.execute( @@ -519,7 +520,7 @@ def test_reuses_existing_group(self, full_catalog): def test_group_name_uppercased(self, full_catalog): writer, cursor, _ = full_catalog - gid = writer.ensure_group(cursor, dataset_id=1, name=" sinh ") + gid = writer.ensure_group(cursor, dataset_id=1, group=" sinh ") cursor.execute( "SELECT name FROM pysus.dataset_groups WHERE id = ?", (gid,) )