diff --git a/CHANGES/1358.feature b/CHANGES/1358.feature new file mode 100644 index 00000000..5cb68bb9 --- /dev/null +++ b/CHANGES/1358.feature @@ -0,0 +1 @@ +Added a repository endpoint that lists Python packages with limit/offset pagination. diff --git a/docs/index.md b/docs/index.md index 80c26e43..ff9bb16d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,7 @@ The REST API documentation for `pulp_python` is available [here](site:pulp_pytho - [Create local mirrors of PyPI](site:pulp_python/docs/user/guides/sync/) that you have full control over - [Upload your own Python packages](site:pulp_python/docs/user/guides/upload/) +- [Browse the package catalog](site:pulp_python/docs/user/guides/catalog/) over the REST API - [Perform pip install](site:pulp_python/docs/user/guides/host/) from your Pulp Python repositories - Download packages on-demand to reduce disk usage - Every operation creates a restorable snapshot with Versioned Repositories @@ -34,5 +35,4 @@ Users may also find pulpcore’s conceptual docs useful. This documentation falls into two main categories: 1. `How-to Guides` shows the **major features** of the Python plugin, with links to reference docs. -2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each -minor feature, including all fields and options. +2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each minor feature, including all fields and options. diff --git a/docs/user/guides/_SUMMARY.md b/docs/user/guides/_SUMMARY.md index d9c01bdb..36699ede 100644 --- a/docs/user/guides/_SUMMARY.md +++ b/docs/user/guides/_SUMMARY.md @@ -1,6 +1,7 @@ * [Set up your own PyPI](pypi.md) * [Sync from Remote Repositories](sync.md) * [Upload and Manage Content](upload.md) +* [Browse the package catalog](catalog.md) * [Host Python Content](host.md) * [Vulnerability Report](vulnerability_report.md) * [Attestation Hosting](attestation.md) diff --git a/docs/user/guides/catalog.md b/docs/user/guides/catalog.md new file mode 100644 index 00000000..e7b5bcfc --- /dev/null +++ b/docs/user/guides/catalog.md @@ -0,0 +1,49 @@ +# Browse the package catalog + +The package list returns **one row per package name**. It defaults to the +repository's latest complete version. Pass `repository_version` (HREF or PRN) +to read an older snapshot. `${REPO_PK}` is the repository UUID. + +## List packages + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/?limit=10" +``` + +`count` is the number of distinct packages, not files. Each row is one package +name. `summary`, `description`, `author`, `author_email`, and `license` come +from the newest version. `versions` has one entry per stored version string, +newest first, with that version's `license_expression`. A wheel and an sdist +of the same version are one entry. A rebuild such as `1.0.0+test.1` stays its +own entry because the stored version differs. + +```json +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "name": "shelf-reader", + "name_normalized": "shelf-reader", + "summary": "A small example package", + "description": "", + "author": "", + "author_email": "", + "license": "", + "versions": [ + {"version": "0.1", "license_expression": ""} + ] + } + ] +} +``` + +Results are ordered by `name_normalized`. Within a package, versions are newest +first. Page with `limit` and `offset` to read the whole catalog. Pass +`repository_version` when the export must stay on one snapshot. The selected version +must belong to this repository and be complete. + +Filter names with `name_normalized`, `name_normalized__icontains`, or +`name_normalized__istartswith`. Use `ordering=-name_normalized` to list names +in descending order. diff --git a/pulp_python/app/catalog.py b/pulp_python/app/catalog.py new file mode 100644 index 00000000..aa242423 --- /dev/null +++ b/pulp_python/app/catalog.py @@ -0,0 +1,99 @@ +"""Helpers for the repository package list.""" + +from collections import defaultdict + +from django.db.models import Case, IntegerField, Value, When +from packaging.version import InvalidVersion, Version + +from pulp_python.app.models import PythonPackageContent + + +def _version_sort_key(version): + """PEP 440 sort key. Use with `reverse=True` for newest first. + + Invalid versions sort after all valid ones when `reverse=True`. + """ + if not version: + return (-1, "") + try: + return (0, Version(version)) + except InvalidVersion: + return (-1, str(version)) + + +def python_packages_in_version(repository_version): + """Python package content contained in `repository_version`.""" + content = PythonPackageContent.objects.all() + if repository_version is None: + return content.none() + return repository_version.get_content(content) + + +def distinct_package_names_qs(content_qs): + """One row per distinct `name_normalized`, ordered for stable pagination.""" + return ( + content_qs.values("name_normalized") + .distinct() + .order_by(*(content_qs.query.order_by or ("name_normalized",))) + ) + + +def assemble_package_index(content_qs, name_rows, repository_version): + """One catalog row per package name, with one content unit per stored version. + + Wheel and sdist of the same `(name_normalized, version)` collapse to one + row. An sdist is preferred; otherwise the newest `pulp_created` is kept. + Version strings are the values stored on the content unit, newest first. + Summary, description, author, author email, and license are taken from the + newest version because they do not vary by version. + """ + if not name_rows or repository_version is None: + return [] + + names = [row["name_normalized"] for row in name_rows] + by_name = defaultdict(list) + # DISTINCT ON keeps the first row for each name and stored version. + units = ( + content_qs.filter(name_normalized__in=names) + .defer("description", "classifiers", "requires_dist", "project_urls", "license_file") + .order_by( + "name_normalized", + "version", + Case( + When(packagetype="sdist", then=Value(0)), + default=Value(1), + output_field=IntegerField(), + ), + "-pulp_created", + "pk", + ) + .distinct("name_normalized", "version") + ) + for unit in units: + by_name[unit.name_normalized].append(unit) + + result = [] + for row in name_rows: + normalized = row["name_normalized"] + versions = sorted( + by_name.get(normalized, []), + key=lambda unit: (_version_sort_key(unit.version), unit.filename), + reverse=True, + ) + source = versions[0] if versions else None + result.append( + { + "name": source.name if source else normalized, + "name_normalized": normalized, + "summary": source.summary if source else "", + "description": source.description if source else "", + "author": source.author if source else "", + "author_email": source.author_email if source else "", + "license": source.license if source else "", + "versions": [ + {"version": unit.version, "license_expression": unit.license_expression} + for unit in versions + ], + } + ) + return result diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index 8d580a2f..ae974832 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -349,7 +349,7 @@ def list(self, request, path): if self.should_redirect(repo_version=repo_version): return redirect(urljoin(self.base_content_url, f"{path}/simple/")) names = ( - content.order_by("name_normalized") + content.order_by("name_normalized", "name") .values_list("name", flat=True) .distinct("name_normalized") .iterator() diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index c59a5398..f7fd7ee6 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -641,6 +641,57 @@ class Meta: model = python_models.PythonPackageContent +class PythonRepositoryPackageVersionSerializer(serializers.Serializer): + """One stored version of a package.""" + + version = serializers.CharField(help_text=_("The package's version number.")) + license_expression = serializers.CharField( + allow_blank=True, + help_text=_("Text string that is a valid SPDX license expression."), + ) + + +class PythonRepositoryPackageSerializer(serializers.Serializer): + """One distinct package name and its stored versions.""" + + name = serializers.CharField( + help_text=_("A project name stored on one of this package's files."), + ) + name_normalized = serializers.CharField( + help_text=_("PEP 503 normalized package name. Index rows are unique on this field."), + ) + summary = serializers.CharField( + allow_blank=True, + help_text=_("A one-line summary of what the package does."), + ) + description = serializers.CharField( + allow_blank=True, + help_text=_("A longer description of the package that can run to several paragraphs."), + ) + author = serializers.CharField( + allow_blank=True, + help_text=_( + "Text containing the author's name. Contact information can also be added," + " separated with newlines." + ), + ) + author_email = serializers.CharField( + allow_blank=True, + help_text=_("The author's e-mail address."), + ) + license = serializers.CharField( + allow_blank=True, + help_text=_("Text indicating the license covering the distribution."), + ) + versions = PythonRepositoryPackageVersionSerializer( + many=True, + help_text=_( + "One entry per stored version, newest first. A wheel and an sdist of the " + "same version are one entry. Version strings are unchanged." + ), + ) + + class PackageProvenanceSerializer(core_serializers.NoArtifactContentUploadSerializer): """ A Serializer for PackageProvenance. diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 400c9398..3cb3d93e 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -2,9 +2,12 @@ from bandersnatch.configuration import BandersnatchConfig from django.db import transaction -from django_filters import CharFilter +from django.http import Http404 +from django_filters import CharFilter, OrderingFilter +from django_filters.rest_framework import DjangoFilterBackend, FilterSet from django_filters.rest_framework import filters as drf_filters -from drf_spectacular.utils import extend_schema, extend_schema_view +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.utils import canonicalize_name from rest_framework import status @@ -20,18 +23,23 @@ from pulpcore.plugin import viewsets as core_viewsets from pulpcore.plugin.actions import ModifyRepositoryActionMixin -from pulpcore.plugin.models import RepositoryVersion +from pulpcore.plugin.models import RepositoryContent, RepositoryVersion from pulpcore.plugin.serializers import ( AsyncOperationResponseSerializer, RepositoryAddRemoveContentSerializer, RepositorySyncURLSerializer, ) from pulpcore.plugin.tasking import dispatch -from pulpcore.plugin.util import extract_pk +from pulpcore.plugin.util import extract_pk, get_domain_pk from pulp_python.app import models as python_models from pulp_python.app import serializers as python_serializers from pulp_python.app import tasks +from pulp_python.app.catalog import ( + assemble_package_index, + distinct_package_names_qs, + python_packages_in_version, +) class PythonRepositoryViewSet( @@ -248,6 +256,116 @@ def sync(self, request, pk, **kwargs): return core_viewsets.OperationPostponedResponse(result, request) +class PythonRepositoryPackageFilter(FilterSet): + """Filter catalog rows by package name and order distinct names.""" + + ordering = OrderingFilter(fields=("name_normalized",)) + + class Meta: + model = python_models.PythonPackageContent + fields = {"name_normalized": core_viewsets.NAME_FILTER_OPTIONS} + + +class PythonRepositoryPackageFilterBackend(DjangoFilterBackend): + """Apply package filters while the viewset is registered for repository membership.""" + + def get_filterset_class(self, view, queryset=None): + return view.filterset_class + + +class PythonRepositoryPackageViewSet(core_viewsets.NamedModelViewSet, ListModelMixin): + """Read-only package catalog for a Python repository.""" + + endpoint_name = "packages" + parent_viewset = PythonRepositoryViewSet + parent_lookup_kwargs = {"repository_pk": "repository__pk"} + serializer_class = python_serializers.PythonRepositoryPackageSerializer + # Catalog rows describe repository membership, not individual content resources. + queryset = RepositoryContent.objects.none() + filterset_class = PythonRepositoryPackageFilter + filter_backends = (PythonRepositoryPackageFilterBackend,) + + DEFAULT_ACCESS_POLICY = { + "statements": [ + { + "action": ["list"], + "principal": "authenticated", + "effect": "allow", + "condition": "has_repository_model_or_domain_or_obj_perms:python.view_pythonrepository", # noqa: E501 + }, + ], + } + + def get_parent_field_and_object(self): + """Require the parent repository to belong to the current domain.""" + field, repository = super().get_parent_field_and_object() + if repository.pulp_domain_id != get_domain_pk(): + raise Http404 + return field, repository + + def _requested_repository_version(self, repository): + """Resolve a complete snapshot belonging to the parent repository.""" + href = self.request.query_params.get("repository_version") + if not href: + return repository.latest_version() + try: + repo_version = self.get_resource(href, RepositoryVersion) + except ValidationError as exc: + raise ValidationError({"repository_version": exc.detail}) from exc + if repo_version.repository_id != repository.pk: + raise ValidationError({"repository_version": "Must be a version of this repository."}) + if not repo_version.complete: + raise ValidationError({"repository_version": "Must be a complete repository version."}) + return repo_version + + def get_queryset(self): + """Select package content through the requested repository snapshot.""" + if getattr(self, "swagger_fake_view", False): + return python_models.PythonPackageContent.objects.none() + repository = self.get_parent_object() + self.repository_version = self._requested_repository_version(repository) + return python_packages_in_version(self.repository_version) + + @extend_schema( + summary="List packages", + description=( + "Return one row per distinct package name in a repository version " + "(latest complete version if repository_version is omitted). " + "Each versions entry is the stored version and its license_expression " + "(wheel and sdist of the same version are a single entry). Results are " + "ordered by name_normalized. Pagination count is the number of " + "distinct packages, not files." + ), + parameters=[ + OpenApiParameter( + name="repository_version", + type=OpenApiTypes.URI, + location=OpenApiParameter.QUERY, + required=False, + description=( + "HREF or PRN of a version of this repository. " + "Defaults to the latest complete version." + ), + ), + ], + responses={200: python_serializers.PythonRepositoryPackageSerializer(many=True)}, + ) + def list(self, request, *args, **kwargs): + """List distinct packages in a repository version.""" + content_qs = self.filter_queryset(self.get_queryset()) + names_qs = distinct_package_names_qs(content_qs) + page = self.paginate_queryset(names_qs) + rows = assemble_package_index( + content_qs, + page if page is not None else list(names_qs), + self.repository_version, + ) + serializer = self.get_serializer(rows, many=True) + if page is not None: + return self.get_paginated_response(serializer.data) + return Response(serializer.data) + + class PythonBlocklistEntryViewSet( core_viewsets.NamedModelViewSet, CreateModelMixin, diff --git a/pulp_python/tests/functional/api/test_catalog.py b/pulp_python/tests/functional/api/test_catalog.py new file mode 100644 index 00000000..4c6c9f53 --- /dev/null +++ b/pulp_python/tests/functional/api/test_catalog.py @@ -0,0 +1,287 @@ +"""Catalog API tests.""" + +import io +import tarfile +import uuid +from urllib.parse import urljoin + +import pytest +import requests + +from pulp_python.tests.functional.constants import PYTHON_SM_PROJECT_SPECIFIER + + +def _api_get(bindings_cfg, path, **params): + url = urljoin(bindings_cfg.host + "/", path.lstrip("/")) + response = requests.get(url, params=params, auth=(bindings_cfg.username, bindings_cfg.password)) + assert response.status_code == 200, response.text + return response.json() + + +_PACKAGE_FIELDS = { + "name", + "name_normalized", + "summary", + "description", + "author", + "author_email", + "license", + "versions", +} + + +def _assert_package_row(pkg): + assert set(pkg) == _PACKAGE_FIELDS + assert pkg["name"] + assert pkg["name_normalized"] + versions = [item["version"] for item in pkg["versions"]] + assert versions + assert len(versions) == len(set(versions)) + for item in pkg["versions"]: + assert set(item) == {"version", "license_expression"} + assert item["version"] + + +def _write_sdist(directory, name, version): + """Write a minimal sdist whose PKG-INFO Name/Version pkginfo can read.""" + pkg_dir = f"{name}-{version}" + filename = f"{pkg_dir}.tar.gz" + path = directory / filename + pkg_info = f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n".encode() + with tarfile.open(path, "w:gz") as tar: + info = tarfile.TarInfo(name=f"{pkg_dir}/PKG-INFO") + info.size = len(pkg_info) + tar.addfile(info, io.BytesIO(pkg_info)) + return filename, str(path) + + +def _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, version): + filename, path = _write_sdist(tmp_path, name, version) + python_content_factory(relative_path=filename, file=path, repository=repo) + return python_bindings.RepositoriesPythonApi.read(repo.pulp_href) + + +@pytest.fixture +def sm_repo(python_repo_with_sync, python_remote_factory): + remote = python_remote_factory(includes=PYTHON_SM_PROJECT_SPECIFIER) + return python_repo_with_sync(remote) + + +@pytest.mark.parallel +def test_package_list_grouping_and_pagination(bindings_cfg, sm_repo): + """Package index is one row per name, and count is distinct packages not files.""" + data = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=1) + assert data["count"] == 3 + assert len(data["results"]) == 1 + _assert_package_row(data["results"][0]) + + page2 = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=1, offset=1) + assert page2["count"] == 3 + assert page2["results"][0]["name_normalized"] != data["results"][0]["name_normalized"] + + all_rows = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=100)["results"] + names = [pkg["name_normalized"] for pkg in all_rows] + assert names == sorted(names) + assert set(names) == {"aiohttp", "celery", "django"} + django = next(pkg for pkg in all_rows if pkg["name_normalized"] == "django") + assert {item["version"] for item in django["versions"]} >= { + "1.10.4", + "1.10.3", + "1.10.2", + "1.10.1", + } + assert all("+" not in item["version"] for item in django["versions"]) + + +@pytest.mark.parallel +def test_package_list_empty_repository(python_bindings, python_repo_factory): + repo = python_repo_factory() + data = python_bindings.RepositoriesPythonPackagesApi.list(repo.pulp_href) + assert data.count == 0 + assert data.results == [] + + +@pytest.mark.parallel +def test_packages_repository_version(bindings_cfg, sm_repo, python_repo_factory): + """repository_version selects a snapshot; omitted uses the latest complete version.""" + latest_href = sm_repo.latest_version_href + v0_href = f"{sm_repo.pulp_href}versions/0/" + + default_pkgs = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/") + explicit_pkgs = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", repository_version=latest_href + ) + assert default_pkgs["count"] == explicit_pkgs["count"] == 3 + + version = _api_get(bindings_cfg, latest_href) + prn_pkgs = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", repository_version=version["prn"] + ) + assert prn_pkgs["count"] == 3 + assert version["prn"].startswith("prn:core.repositoryversion:") + + v0_pkgs = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", repository_version=v0_href) + assert v0_pkgs["count"] == 0 + assert v0_pkgs["results"] == [] + + other = python_repo_factory() + url = urljoin(bindings_cfg.host + "/", f"{sm_repo.pulp_href}packages/".lstrip("/")) + response = requests.get( + url, + params={"repository_version": other.latest_version_href}, + auth=(bindings_cfg.username, bindings_cfg.password), + ) + assert response.status_code == 400, response.text + + +@pytest.mark.parallel +def test_package_list_one_entry_per_stored_version( + bindings_cfg, python_bindings, python_repo_with_sync +): + """A wheel and an sdist of the same version are one version string.""" + repo = python_repo_with_sync() + files = python_bindings.ContentPackagesApi.list(repository_version=repo.latest_version_href) + assert files.count == 2 + assert {item.packagetype for item in files.results} == {"sdist", "bdist_wheel"} + + pkgs = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert pkgs["count"] == 1 + row = pkgs["results"][0] + _assert_package_row(row) + assert row["name_normalized"] == "shelf-reader" + assert [item["version"] for item in row["versions"]] == ["0.1"] + + +@pytest.mark.parallel +def test_package_list_version_order( + bindings_cfg, python_bindings, python_content_factory, python_repo_factory, tmp_path +): + """Stored versions are newest-first by PEP 440, not lexicographically.""" + repo = python_repo_factory() + name = f"ordered-{uuid.uuid4().hex[:8]}" + for version in ("1.10", "1.9", "1.2"): + repo = _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, version) + data = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert data["count"] == 1 + pkg = data["results"][0] + _assert_package_row(pkg) + assert [item["version"] for item in pkg["versions"]] == ["1.10", "1.9", "1.2"] + + +@pytest.mark.parallel +def test_package_list_keeps_stored_versions( + bindings_cfg, python_bindings, python_content_factory, python_repo_factory, tmp_path +): + """A rebuild stays its own file, with the version string stored in Pulp.""" + repo = python_repo_factory() + suffix = uuid.uuid4().hex[:8] + name = f"pkg-{suffix}" + + repo = _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, "2.0.0") + original_version = repo.latest_version_href + repo = _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, "1.0.0+test.3") + row = _api_get(bindings_cfg, f"{repo.pulp_href}packages/")["results"][0] + _assert_package_row(row) + assert [item["version"] for item in row["versions"]] == ["2.0.0", "1.0.0+test.3"] + historical = _api_get( + bindings_cfg, f"{repo.pulp_href}packages/", repository_version=original_version + )["results"][0] + assert [item["version"] for item in historical["versions"]] == ["2.0.0"] + + +@pytest.mark.parallel +def test_package_list_keeps_rebuild_version_string( + bindings_cfg, python_bindings, python_content_factory, python_repo_factory, tmp_path +): + """A rebuild is a separate entry because its stored version string differs.""" + repo = python_repo_factory() + name = f"rebuild-{uuid.uuid4().hex[:8]}" + repo = _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, "5.3.17") + repo = _add_sdist( + python_content_factory, + python_bindings, + tmp_path, + repo, + name, + "5.3.17+test.1.n1", + ) + + pkgs = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert pkgs["count"] == 1 + pkg = pkgs["results"][0] + _assert_package_row(pkg) + assert {item["version"] for item in pkg["versions"]} == {"5.3.17", "5.3.17+test.1.n1"} + + +@pytest.mark.parallel +def test_package_list_filters_and_ordering(bindings_cfg, sm_repo): + """Name filters and ordering apply to packages within the selected snapshot.""" + path = f"{sm_repo.pulp_href}packages/" + for params in ( + {"name_normalized": "django"}, + {"name_normalized__istartswith": "DJAN"}, + {"name_normalized__icontains": "JANG"}, + {"name_normalized__in": "django,absent"}, + ): + data = _api_get(bindings_cfg, path, **params) + assert data["count"] == 1 + assert [row["name_normalized"] for row in data["results"]] == ["django"] + data = _api_get(bindings_cfg, path, ordering="-name_normalized", limit=2) + assert data["count"] == 3 + assert [row["name_normalized"] for row in data["results"]] == ["django", "celery"] + data = _api_get(bindings_cfg, path, ordering="-name_normalized", limit=2, offset=2) + assert [row["name_normalized"] for row in data["results"]] == ["aiohttp"] + + +@pytest.mark.parallel +@pytest.mark.parametrize("selector", ["invalid", "repository", "missing", "wrong_type_prn"]) +def test_package_list_invalid_version(bindings_cfg, python_repo_factory, selector): + """Malformed, missing, and non-version resources return a validation error.""" + repo = python_repo_factory() + selectors = { + "invalid": "/not-a-pulp-resource/", + "repository": repo.pulp_href, + "missing": f"{repo.pulp_href}versions/999/", + "wrong_type_prn": repo.prn, + } + response = requests.get( + urljoin(bindings_cfg.host, f"{repo.pulp_href}packages/"), + params={"repository_version": selectors[selector]}, + auth=(bindings_cfg.username, bindings_cfg.password), + ) + assert response.status_code == 400, response.text + assert "repository_version" in response.json() + + +@pytest.mark.parallel +def test_package_list_permissions(bindings_cfg, python_repo_factory, gen_user): + """Catalog access requires view permission on the parent repository.""" + repo = python_repo_factory() + other = python_repo_factory() + user = gen_user() + viewer = gen_user(object_roles=[("python.pythonrepository_viewer", repo.pulp_href)]) + model_viewer = gen_user(model_roles=["python.pythonrepository_viewer"]) + for actor, repository, status in ( + (user, repo, 403), + (viewer, repo, 200), + (viewer, other, 403), + (model_viewer, other, 200), + ): + response = requests.get( + urljoin(bindings_cfg.host, f"{repository.pulp_href}packages/"), + auth=(actor.username, actor.password), + ) + assert response.status_code == status, response.text + + response = requests.post( + urljoin(bindings_cfg.host, f"{repo.pulp_href}packages/"), + json={}, + auth=(bindings_cfg.username, bindings_cfg.password), + ) + assert response.status_code == 405, response.text + + response = requests.get( + urljoin(bindings_cfg.host, f"{repo.pulp_href}packages/"), + auth=("", ""), + ) + assert response.status_code == 401, response.text diff --git a/pulp_python/tests/functional/api/test_crud_publications.py b/pulp_python/tests/functional/api/test_crud_publications.py index e7941ac1..fb65cbd1 100644 --- a/pulp_python/tests/functional/api/test_crud_publications.py +++ b/pulp_python/tests/functional/api/test_crud_publications.py @@ -1,8 +1,10 @@ +import io import random +import tarfile +import zipfile from urllib.parse import urljoin import pytest -from pypi_simple import PyPISimple from pulp_python.tests.functional.constants import ( PYTHON_EGG_FILENAME, @@ -113,27 +115,57 @@ def test_new_content_is_published(python_publication_workflow, python_distributi assert proper is True, msgs +def _write_sdist_with_name(directory, name, version): + """Write a minimal sdist whose PKG-INFO Name differs from PEP 503 canonical form.""" + pkg_dir = f"{name}-{version}" + filename = f"{pkg_dir}.tar.gz" + path = directory / filename + pkg_info = f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n".encode() + with tarfile.open(path, "w:gz") as tar: + info = tarfile.TarInfo(name=f"{pkg_dir}/PKG-INFO") + info.size = len(pkg_info) + tar.addfile(info, io.BytesIO(pkg_info)) + return filename, str(path) + + +def _write_wheel_with_name(directory, metadata_name, version, filename): + """Write a minimal wheel whose METADATA Name can differ from the sdist Name.""" + dist_info = f"{filename.split('-')[0]}-{version}.dist-info" + metadata = f"Metadata-Version: 2.1\nName: {metadata_name}\nVersion: {version}\n" + wheel = ( + "Wheel-Version: 1.0\nGenerator: pulp-python-test\nRoot-Is-Purelib: true\n" + "Tag: py2.py3-none-any\n" + ) + path = directory / filename + with zipfile.ZipFile(path, "w") as zf: + zf.writestr(f"{dist_info}/METADATA", metadata) + zf.writestr(f"{dist_info}/WHEEL", wheel) + return filename, str(path) + + @pytest.mark.parallel def test_non_matching_canonicalized_name( - python_repo, python_content_factory, python_publication_factory, python_distribution_factory + python_repo, + python_content_factory, + python_publication_factory, + python_distribution_factory, + tmp_path, ): """Ensures a package with dists that have non-matching canonicalized names is published.""" - packages = [] - filenames = ["msg_parser-1.0.0-py2.py3-none-any.whl", "msg_parser-1.0.0.tar.gz"] - with PyPISimple() as client: - page = client.get_project_page("msg-parser") - for pkg in page.packages: - if pkg.filename in filenames: - c = python_content_factory(pkg.filename, url=pkg.url, repository=python_repo) - if c.filename.endswith(".tar.gz"): - # The metadata name in the SDist is not the same as the Wheel's name - assert c.name == "msg_parser" - else: - assert c.name == "msg-parser" - packages.append(c) + version = "1.0.0" + wheel_filename = "msg_parser-1.0.0-py2.py3-none-any.whl" + sdist_filename, sdist_path = _write_sdist_with_name(tmp_path, "msg_parser", version) + _, wheel_path = _write_wheel_with_name(tmp_path, "msg-parser", version, wheel_filename) + + wheel = python_content_factory(wheel_filename, file=wheel_path, repository=python_repo) + sdist = python_content_factory(sdist_filename, file=sdist_path, repository=python_repo) + # The metadata name in the SDist is not the same as the Wheel's name + assert wheel.name == "msg-parser" + assert sdist.name == "msg_parser" + pub = python_publication_factory(repository=python_repo) distro = python_distribution_factory(publication=pub) url = urljoin(distro.base_url, "simple/") - proper, msgs = ensure_simple(url, {"msg-parser": filenames}) + proper, msgs = ensure_simple(url, {"msg-parser": [wheel_filename, sdist_filename]}) assert proper is True, msgs diff --git a/pulp_python/tests/functional/api/test_domains.py b/pulp_python/tests/functional/api/test_domains.py index 391af15c..07069247 100644 --- a/pulp_python/tests/functional/api/test_domains.py +++ b/pulp_python/tests/functional/api/test_domains.py @@ -1,9 +1,10 @@ import json import subprocess import uuid -from urllib.parse import urlsplit +from urllib.parse import urljoin, urlsplit import pytest +import requests from pulpcore.app import settings # noqa: TID251 @@ -83,6 +84,51 @@ def test_domain_object_creation( } +def _catalog_api_get(bindings_cfg, path, **params): + url = urljoin(bindings_cfg.host + "/", path.lstrip("/")) + response = requests.get(url, params=params, auth=(bindings_cfg.username, bindings_cfg.password)) + assert response.status_code == 200, response.text + return response.json() + + +@pytest.mark.parallel +def test_domain_catalog_packages(bindings_cfg, domain_factory, python_repo_factory, gen_user): + """Domain-prefixed repository and repository_version hrefs, including PRN.""" + domain = domain_factory() + repo = python_repo_factory(pulp_domain=domain.name) + assert f"{domain.name}/api/" in repo.pulp_href + assert f"{domain.name}/api/" in repo.latest_version_href + + pkgs = _catalog_api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert pkgs["count"] == 0 + assert pkgs["results"] == [] + + by_href = _catalog_api_get( + bindings_cfg, f"{repo.pulp_href}packages/", repository_version=repo.latest_version_href + ) + assert by_href["count"] == 0 + version = _catalog_api_get(bindings_cfg, repo.latest_version_href) + by_prn = _catalog_api_get( + bindings_cfg, f"{repo.pulp_href}packages/", repository_version=version["prn"] + ) + assert by_prn["count"] == 0 + assert version["prn"].startswith("prn:core.repositoryversion:") + + default_path = f"{repo.pulp_href}packages/".replace(f"/{domain.name}/api/", "/default/api/") + response = requests.get( + urljoin(bindings_cfg.host, default_path), + auth=(bindings_cfg.username, bindings_cfg.password), + ) + assert response.status_code == 404, response.text + + viewer = gen_user(domain_roles=[("python.pythonrepository_viewer", domain.pulp_href)]) + response = requests.get( + urljoin(bindings_cfg.host, f"{repo.pulp_href}packages/"), + auth=(viewer.username, viewer.password), + ) + assert response.status_code == 200, response.text + + @pytest.mark.parallel def test_domain_content_upload( domain_factory, diff --git a/pulp_python/tests/unit/test_catalog.py b/pulp_python/tests/unit/test_catalog.py new file mode 100644 index 00000000..12a5edfd --- /dev/null +++ b/pulp_python/tests/unit/test_catalog.py @@ -0,0 +1,116 @@ +"""Unit tests for the repository package index helpers.""" + +import uuid + +import pytest +from django.db import connection +from django.test.utils import CaptureQueriesContext +from rest_framework.test import APIRequestFactory, force_authenticate + +from pulpcore.plugin.models import RepositoryVersion + +from pulp_python.app.catalog import ( + assemble_package_index, + distinct_package_names_qs, + python_packages_in_version, +) +from pulp_python.app.models import PythonPackageContent, PythonRepository +from pulp_python.app.viewsets import PythonRepositoryPackageViewSet + +_EMPTY_TEXT = ( + "author", + "author_email", + "description", + "home_page", + "keywords", + "license", + "metadata_version", + "platform", + "summary", + "download_url", + "supported_platform", + "maintainer", + "maintainer_email", + "project_url", + "requires_python", + "description_content_type", + "license_expression", + "python_version", +) + + +def _create_package(name, version): + fields = {field: "" for field in _EMPTY_TEXT} + fields.update( + name=name, + version=version, + filename=f"{name}-{version}.tar.gz", + packagetype="sdist", + sha256=uuid.uuid4().hex + uuid.uuid4().hex, + description=f"{name} {version} description", + ) + return PythonPackageContent.objects.create(**fields) + + +@pytest.mark.django_db +def test_package_list_pagination_and_deferred_metadata(django_assert_num_queries): + """Page distinct names and load large metadata only for the newest version.""" + repository = PythonRepository.objects.create(name=str(uuid.uuid4())) + created = [ + _create_package(name, f"1.0.{number}") + for name in ("alpha", "beta", "gamma", "delta") + for number in range(5) + ] + with repository.new_version() as version: + version.add_content( + PythonPackageContent.objects.filter(pk__in=[item.pk for item in created]) + ) + + repo_version = repository.latest_version() + content_qs = python_packages_in_version(repo_version) + names_qs = distinct_package_names_qs(content_qs) + + with django_assert_num_queries(1): + assert names_qs.count() == 4 + with django_assert_num_queries(1): + page = list(names_qs[:2]) + with CaptureQueriesContext(connection) as queries: + rows = assemble_package_index(content_qs, page, repo_version) + + assert [row["name_normalized"] for row in rows] == ["alpha", "beta"] + for row in rows: + assert [item["version"] for item in row["versions"]] == [ + f"1.0.{number}" for number in reversed(range(5)) + ] + assert row["description"] == f"{row['name_normalized']} 1.0.4 description" + + bulk_select = queries[0]["sql"].split(" FROM ", 1)[0] + for field in ("description", "classifiers", "requires_dist", "project_urls", "license_file"): + assert f'"{field}"' not in bulk_select + description_queries = [ + query for query in queries if '"description"' in query["sql"].split(" FROM ", 1)[0] + ] + assert len(description_queries) == len(rows) + + +@pytest.mark.django_db +@pytest.mark.parametrize("explicit", [True, False]) +def test_package_list_incomplete_version(django_user_model, explicit): + """Reject an explicit incomplete snapshot and ignore it when selecting the latest.""" + repository = PythonRepository.objects.create(name=str(uuid.uuid4())) + incomplete = RepositoryVersion.objects.create(repository=repository, number=1, complete=False) + user = django_user_model.objects.create(username=str(uuid.uuid4()), is_superuser=True) + params = ( + {"repository_version": f"prn:core.repositoryversion:{incomplete.pk}"} if explicit else {} + ) + request = APIRequestFactory().get("/", params) + force_authenticate(request, user=user) + response = PythonRepositoryPackageViewSet.as_view({"get": "list"})( + request, repository_pk=str(repository.pk) + ) + assert response.status_code == (400 if explicit else 200) + if explicit: + assert "complete" in str(response.data["repository_version"]) + else: + assert response.data["count"] == 0 + assert response.data["results"] == []