Add a paginated Python repository package catalog

CHANGES/1358.feature

@@ -0,0 +1 @@
+Added a repository endpoint that lists Python packages with limit/offset pagination.

docs/index.md

@@ -12,6 +12,7 @@ The REST API documentation for `pulp_python` is available [here](site:pulp_pytho
 
 - [Create local mirrors of PyPI](site:pulp_python/docs/user/guides/sync/) that you have full control over
 - [Upload your own Python packages](site:pulp_python/docs/user/guides/upload/)
+- [Browse the package catalog](site:pulp_python/docs/user/guides/catalog/) over the REST API
 - [Perform pip install](site:pulp_python/docs/user/guides/host/) from your Pulp Python repositories
 - Download packages on-demand to reduce disk usage
 - Every operation creates a restorable snapshot with Versioned Repositories
@@ -34,5 +35,4 @@ Users may also find pulpcore’s conceptual docs useful.
 This documentation falls into two main categories:
 
 1. `How-to Guides` shows the **major features** of the Python plugin, with links to reference docs.
-2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each 
-minor feature, including all fields and options.
+2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each minor feature, including all fields and options.

docs/user/guides/_SUMMARY.md

@@ -1,6 +1,7 @@
 * [Set up your own PyPI](pypi.md)
 * [Sync from Remote Repositories](sync.md)
 * [Upload and Manage Content](upload.md)
+* [Browse the package catalog](catalog.md)
 * [Host Python Content](host.md)
 * [Vulnerability Report](vulnerability_report.md)
 * [Attestation Hosting](attestation.md)

docs/user/guides/catalog.md

@@ -0,0 +1,49 @@
+# Browse the package catalog
+
+The package list returns **one row per package name**. It defaults to the
+repository's latest complete version. Pass `repository_version` (HREF or PRN)
+to read an older snapshot. `${REPO_PK}` is the repository UUID.
+
+## List packages
+
+```bash
+http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/?limit=10"
+```
+
+`count` is the number of distinct packages, not files. Each row is one package
+name. `summary`, `description`, `author`, `author_email`, and `license` come
+from the newest version. `versions` has one entry per stored version string,
+newest first, with that version's `license_expression`. A wheel and an sdist
+of the same version are one entry. A rebuild such as `1.0.0+test.1` stays its
+own entry because the stored version differs.
+
+```json
+{
+  "count": 1,
+  "next": null,
+  "previous": null,
+  "results": [
+    {
+      "name": "shelf-reader",
+      "name_normalized": "shelf-reader",
+      "summary": "A small example package",
+      "description": "",
+      "author": "",
+      "author_email": "",
+      "license": "",
+      "versions": [
+        {"version": "0.1", "license_expression": ""}
+      ]
+    }
+  ]
+}
+```
+
+Results are ordered by `name_normalized`. Within a package, versions are newest
+first. Page with `limit` and `offset` to read the whole catalog. Pass
+`repository_version` when the export must stay on one snapshot. The selected version
+must belong to this repository and be complete.
+
+Filter names with `name_normalized`, `name_normalized__icontains`, or
+`name_normalized__istartswith`. Use `ordering=-name_normalized` to list names
+in descending order.

pulp_python/app/catalog.py

@@ -0,0 +1,99 @@
+"""Helpers for the repository package list."""
+
+from collections import defaultdict
+
+from django.db.models import Case, IntegerField, Value, When
+from packaging.version import InvalidVersion, Version
+
+from pulp_python.app.models import PythonPackageContent
+
+
+def _version_sort_key(version):
+    """PEP 440 sort key. Use with `reverse=True` for newest first.
+
+    Invalid versions sort after all valid ones when `reverse=True`.
+    """
+    if not version:
+        return (-1, "")
+    try:
+        return (0, Version(version))
+    except InvalidVersion:
+        return (-1, str(version))
+
+
+def python_packages_in_version(repository_version):
+    """Python package content contained in `repository_version`."""
+    content = PythonPackageContent.objects.all()
+    if repository_version is None:
+        return content.none()
+    return repository_version.get_content(content)
+
+
+def distinct_package_names_qs(content_qs):
+    """One row per distinct `name_normalized`, ordered for stable pagination."""
+    return (
+        content_qs.values("name_normalized")
+        .distinct()
+        .order_by(*(content_qs.query.order_by or ("name_normalized",)))
+    )
+
+
+def assemble_package_index(content_qs, name_rows, repository_version):
+    """One catalog row per package name, with one content unit per stored version.
+
+    Wheel and sdist of the same `(name_normalized, version)` collapse to one
+    row. An sdist is preferred; otherwise the newest `pulp_created` is kept.
+    Version strings are the values stored on the content unit, newest first.
+    Summary, description, author, author email, and license are taken from the
+    newest version because they do not vary by version.
+    """
+    if not name_rows or repository_version is None:
+        return []
+
+    names = [row["name_normalized"] for row in name_rows]
+    by_name = defaultdict(list)
+    # DISTINCT ON keeps the first row for each name and stored version.
+    units = (
+        content_qs.filter(name_normalized__in=names)
+        .defer("description", "classifiers", "requires_dist", "project_urls", "license_file")
+        .order_by(
+            "name_normalized",
+            "version",
+            Case(
+                When(packagetype="sdist", then=Value(0)),
+                default=Value(1),
+                output_field=IntegerField(),
+            ),
+            "-pulp_created",
+            "pk",
+        )
+        .distinct("name_normalized", "version")
+    )
+    for unit in units:
+        by_name[unit.name_normalized].append(unit)
+
+    result = []
+    for row in name_rows:
+        normalized = row["name_normalized"]
+        versions = sorted(
+            by_name.get(normalized, []),
+            key=lambda unit: (_version_sort_key(unit.version), unit.filename),
+            reverse=True,
+        )
+        source = versions[0] if versions else None
+        result.append(
+            {
+                "name": source.name if source else normalized,
+                "name_normalized": normalized,
+                "summary": source.summary if source else "",
+                "description": source.description if source else "",
+                "author": source.author if source else "",
+                "author_email": source.author_email if source else "",
+                "license": source.license if source else "",
+                "versions": [
+                    {"version": unit.version, "license_expression": unit.license_expression}
+                    for unit in versions
+                ],
+            }
+        )
+    return result

pulp_python/app/pypi/views.py

@@ -349,7 +349,7 @@ def list(self, request, path):
         if self.should_redirect(repo_version=repo_version):
             return redirect(urljoin(self.base_content_url, f"{path}/simple/"))
         names = (
-            content.order_by("name_normalized")
+            content.order_by("name_normalized", "name")
             .values_list("name", flat=True)
             .distinct("name_normalized")
             .iterator()

pulp_python/app/serializers.py

@@ -641,6 +641,57 @@ class Meta:
         model = python_models.PythonPackageContent
 
 
+class PythonRepositoryPackageVersionSerializer(serializers.Serializer):
+    """One stored version of a package."""
+
+    version = serializers.CharField(help_text=_("The package's version number."))
+    license_expression = serializers.CharField(
+        allow_blank=True,
+        help_text=_("Text string that is a valid SPDX license expression."),
+    )
+
+
+class PythonRepositoryPackageSerializer(serializers.Serializer):
+    """One distinct package name and its stored versions."""
+
+    name = serializers.CharField(
+        help_text=_("A project name stored on one of this package's files."),
+    )
+    name_normalized = serializers.CharField(
+        help_text=_("PEP 503 normalized package name. Index rows are unique on this field."),
+    )
+    summary = serializers.CharField(
+        allow_blank=True,
+        help_text=_("A one-line summary of what the package does."),
+    )
+    description = serializers.CharField(
+        allow_blank=True,
+        help_text=_("A longer description of the package that can run to several paragraphs."),
+    )
+    author = serializers.CharField(
+        allow_blank=True,
+        help_text=_(
+            "Text containing the author's name. Contact information can also be added,"
+            " separated with newlines."
+        ),
+    )
+    author_email = serializers.CharField(
+        allow_blank=True,
+        help_text=_("The author's e-mail address."),
+    )
+    license = serializers.CharField(
+        allow_blank=True,
+        help_text=_("Text indicating the license covering the distribution."),
+    )
+    versions = PythonRepositoryPackageVersionSerializer(
+        many=True,
+        help_text=_(
+            "One entry per stored version, newest first. A wheel and an sdist of the "
+            "same version are one entry. Version strings are unchanged."
+        ),
+    )
+
+
 class PackageProvenanceSerializer(core_serializers.NoArtifactContentUploadSerializer):
     """
     A Serializer for PackageProvenance.

pulp_python/app/viewsets.py

@@ -2,9 +2,12 @@
 
 from bandersnatch.configuration import BandersnatchConfig
 from django.db import transaction
-from django_filters import CharFilter
+from django.http import Http404
+from django_filters import CharFilter, OrderingFilter
+from django_filters.rest_framework import DjangoFilterBackend, FilterSet
 from django_filters.rest_framework import filters as drf_filters
-from drf_spectacular.utils import extend_schema, extend_schema_view
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view
 from packaging.specifiers import InvalidSpecifier, SpecifierSet
 from packaging.utils import canonicalize_name
 from rest_framework import status
@@ -20,18 +23,23 @@
 
 from pulpcore.plugin import viewsets as core_viewsets
 from pulpcore.plugin.actions import ModifyRepositoryActionMixin
-from pulpcore.plugin.models import RepositoryVersion
+from pulpcore.plugin.models import RepositoryContent, RepositoryVersion
 from pulpcore.plugin.serializers import (
     AsyncOperationResponseSerializer,
     RepositoryAddRemoveContentSerializer,
     RepositorySyncURLSerializer,
 )
 from pulpcore.plugin.tasking import dispatch
-from pulpcore.plugin.util import extract_pk
+from pulpcore.plugin.util import extract_pk, get_domain_pk
 
 from pulp_python.app import models as python_models
 from pulp_python.app import serializers as python_serializers
 from pulp_python.app import tasks
+from pulp_python.app.catalog import (
+    assemble_package_index,
+    distinct_package_names_qs,
+    python_packages_in_version,
+)
 
 
 class PythonRepositoryViewSet(
@@ -248,6 +256,116 @@ def sync(self, request, pk, **kwargs):
         return core_viewsets.OperationPostponedResponse(result, request)
 
 
+class PythonRepositoryPackageFilter(FilterSet):
+    """Filter catalog rows by package name and order distinct names."""
+
+    ordering = OrderingFilter(fields=("name_normalized",))
+
+    class Meta:
+        model = python_models.PythonPackageContent
+        fields = {"name_normalized": core_viewsets.NAME_FILTER_OPTIONS}
+
+
+class PythonRepositoryPackageFilterBackend(DjangoFilterBackend):
+    """Apply package filters while the viewset is registered for repository membership."""
+
+    def get_filterset_class(self, view, queryset=None):
+        return view.filterset_class
+
+
+class PythonRepositoryPackageViewSet(core_viewsets.NamedModelViewSet, ListModelMixin):
+    """Read-only package catalog for a Python repository."""
+
+    endpoint_name = "packages"
+    parent_viewset = PythonRepositoryViewSet
+    parent_lookup_kwargs = {"repository_pk": "repository__pk"}
+    serializer_class = python_serializers.PythonRepositoryPackageSerializer
+    # Catalog rows describe repository membership, not individual content resources.
+    queryset = RepositoryContent.objects.none()
+    filterset_class = PythonRepositoryPackageFilter
+    filter_backends = (PythonRepositoryPackageFilterBackend,)
+
+    DEFAULT_ACCESS_POLICY = {
+        "statements": [
+            {
+                "action": ["list"],
+                "principal": "authenticated",
+                "effect": "allow",
+                "condition": "has_repository_model_or_domain_or_obj_perms:python.view_pythonrepository",  # noqa: E501
+            },
+        ],
+    }
+
+    def get_parent_field_and_object(self):
+        """Require the parent repository to belong to the current domain."""
+        field, repository = super().get_parent_field_and_object()
+        if repository.pulp_domain_id != get_domain_pk():
+            raise Http404
+        return field, repository
+
+    def _requested_repository_version(self, repository):
+        """Resolve a complete snapshot belonging to the parent repository."""
+        href = self.request.query_params.get("repository_version")
+        if not href:
+            return repository.latest_version()
+        try:
+            repo_version = self.get_resource(href, RepositoryVersion)
+        except ValidationError as exc:
+            raise ValidationError({"repository_version": exc.detail}) from exc
+        if repo_version.repository_id != repository.pk:
+            raise ValidationError({"repository_version": "Must be a version of this repository."})
+        if not repo_version.complete:
+            raise ValidationError({"repository_version": "Must be a complete repository version."})
+        return repo_version
+
+    def get_queryset(self):
+        """Select package content through the requested repository snapshot."""
+        if getattr(self, "swagger_fake_view", False):
+            return python_models.PythonPackageContent.objects.none()
+        repository = self.get_parent_object()
+        self.repository_version = self._requested_repository_version(repository)
+        return python_packages_in_version(self.repository_version)
+
+    @extend_schema(
+        summary="List packages",
+        description=(
+            "Return one row per distinct package name in a repository version "
+            "(latest complete version if repository_version is omitted). "
+            "Each versions entry is the stored version and its license_expression "
+            "(wheel and sdist of the same version are a single entry). Results are "
+            "ordered by name_normalized. Pagination count is the number of "
+            "distinct packages, not files."
+        ),
+        parameters=[
+            OpenApiParameter(
+                name="repository_version",
+                type=OpenApiTypes.URI,
+                location=OpenApiParameter.QUERY,
+                required=False,
+                description=(
+                    "HREF or PRN of a version of this repository. "
+                    "Defaults to the latest complete version."
+                ),
+            ),
+        ],
+        responses={200: python_serializers.PythonRepositoryPackageSerializer(many=True)},
+    )
+    def list(self, request, *args, **kwargs):
+        """List distinct packages in a repository version."""
+        content_qs = self.filter_queryset(self.get_queryset())
+        names_qs = distinct_package_names_qs(content_qs)
+        page = self.paginate_queryset(names_qs)
+        rows = assemble_package_index(
+            content_qs,
+            page if page is not None else list(names_qs),
+            self.repository_version,
+        )
+        serializer = self.get_serializer(rows, many=True)
+        if page is not None:
+            return self.get_paginated_response(serializer.data)
+        return Response(serializer.data)
+
+
 class PythonBlocklistEntryViewSet(
     core_viewsets.NamedModelViewSet,
     CreateModelMixin,