Merge pull request #1244 from jobselko/perf_change

[PULP-1776] Optimize upload_time and provenance in PyPI APIs

(cherry picked from commit 43f1bf0)

Author: jobselko

CHANGES/1242.bugfix

@@ -0,0 +1 @@
+Optimized `upload_time` lookups in Simple and JSON APIs.

CHANGES/1243.bugfix

@@ -0,0 +1 @@
+Optimized `provenance` lookups in Simple API.

pulp_python/app/pypi/views.py

@@ -7,7 +7,7 @@
 from django.contrib.sessions.models import Session
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import transaction
-from django.db.models import OuterRef, Subquery
+from django.db.models import Exists, F, FilteredRelation, OuterRef, Q
 from django.db.utils import DatabaseError
 from django.http.response import (
     Http404,
@@ -26,7 +26,6 @@
 from rest_framework.response import Response
 from rest_framework.viewsets import ViewSet
 
-from pulpcore.plugin.models import RepositoryContent
 from pulpcore.plugin.tasking import dispatch
 from pulpcore.plugin.util import get_domain, get_url
 from pulpcore.plugin.viewsets import OperationPostponedResponse
@@ -363,13 +362,16 @@ def retrieve(self, request, path, package):
             return redirect(urljoin(self.base_content_url, f"{path}/simple/{normalized}/"))
         if content:
             local_packages = content.filter(name__normalize=normalized)
-            repo_added_subquery = RepositoryContent.objects.filter(
-                content_id=OuterRef("pk"),
-                repository=repo_ver.repository,
-                version_removed=None,
-            ).values("pulp_created")[:1]
             packages = local_packages.annotate(
-                repo_added_time=Subquery(repo_added_subquery)
+                active_membership=FilteredRelation(
+                    "version_memberships",
+                    condition=Q(
+                        version_memberships__repository=repo_ver.repository,
+                        version_memberships__version_removed=None,
+                    ),
+                ),
+                repo_added_time=F("active_membership__pulp_created"),
+                has_provenance=Exists(PackageProvenance.objects.filter(package_id=OuterRef("pk"))),
             ).values(
                 "filename",
                 "sha256",
@@ -378,9 +380,7 @@ def retrieve(self, request, path, package):
                 "size",
                 "repo_added_time",
                 "version",
-            )
-            provenances = PackageProvenance.objects.filter(package__in=local_packages).values_list(
-                "package__filename", flat=True
+                "has_provenance",
             )
             local_releases = {
                 p["filename"]: {
@@ -389,7 +389,7 @@ def retrieve(self, request, path, package):
                     "upload_time": p["repo_added_time"],
                     "provenance": (
                         self.get_provenance_url(normalized, p["version"], p["filename"])
-                        if p["filename"] in provenances
+                        if p["has_provenance"]
                         else None
                     ),
                 }

pulp_python/app/utils.py

@@ -11,7 +11,7 @@
 import pkginfo
 from aiohttp.client_exceptions import ClientError
 from django.conf import settings
-from django.db.models import OuterRef, Subquery
+from django.db.models import F, FilteredRelation, Q
 from django.db.utils import IntegrityError
 from jinja2 import Template
 from packaging.requirements import Requirement
@@ -20,7 +20,7 @@
 from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage
 
 from pulpcore.plugin.exceptions import TimeoutException
-from pulpcore.plugin.models import Artifact, Remote, RepositoryContent
+from pulpcore.plugin.models import Artifact, Remote
 from pulpcore.plugin.util import get_domain
 
 log = logging.getLogger(__name__)
@@ -375,12 +375,16 @@ def python_content_to_json(
     Returns None if version is specified but not found within content_query
     """
     if repository_version:
-        repo_added_subquery = RepositoryContent.objects.filter(
-            content_id=OuterRef("pk"),
-            repository=repository_version.repository,
-            version_removed=None,
-        ).values("pulp_created")[:1]
-        content_query = content_query.annotate(repo_added_time=Subquery(repo_added_subquery))
+        content_query = content_query.annotate(
+            active_membership=FilteredRelation(
+                "version_memberships",
+                condition=Q(
+                    version_memberships__repository=repository_version.repository,
+                    version_memberships__version_removed=None,
+                ),
+            ),
+            repo_added_time=F("active_membership__pulp_created"),
+        )
     full_metadata = {"last_serial": 0}  # For now the serial field isn't supported by Pulp
     latest_content = latest_content_version(content_query, version)
     if not latest_content: