diff --git a/CHANGES/1242.bugfix b/CHANGES/1242.bugfix new file mode 100644 index 000000000..94a77391e --- /dev/null +++ b/CHANGES/1242.bugfix @@ -0,0 +1 @@ +Optimized `upload_time` lookups in Simple and JSON APIs. diff --git a/CHANGES/1243.bugfix b/CHANGES/1243.bugfix new file mode 100644 index 000000000..e5138e18c --- /dev/null +++ b/CHANGES/1243.bugfix @@ -0,0 +1 @@ +Optimized `provenance` lookups in Simple API. diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index b8585df7a..e228d88dd 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -7,7 +7,7 @@ from django.contrib.sessions.models import Session from django.core.exceptions import ObjectDoesNotExist from django.db import transaction -from django.db.models import OuterRef, Subquery +from django.db.models import Exists, F, FilteredRelation, OuterRef, Q from django.db.utils import DatabaseError from django.http.response import ( Http404, @@ -26,7 +26,6 @@ from rest_framework.response import Response from rest_framework.viewsets import ViewSet -from pulpcore.plugin.models import RepositoryContent from pulpcore.plugin.tasking import dispatch from pulpcore.plugin.util import get_domain, get_url from pulpcore.plugin.viewsets import OperationPostponedResponse @@ -368,13 +367,16 @@ def retrieve(self, request, path, package): return redirect(urljoin(self.base_content_url, f"{path}/simple/{normalized}/")) if content is not None: local_packages = content.filter(name_normalized=normalized) - repo_added_subquery = RepositoryContent.objects.filter( - content_id=OuterRef("pk"), - repository=repo_ver.repository, - version_removed=None, - ).values("pulp_created")[:1] packages = local_packages.annotate( - repo_added_time=Subquery(repo_added_subquery) + active_membership=FilteredRelation( + "version_memberships", + condition=Q( + version_memberships__repository=repo_ver.repository, + version_memberships__version_removed=None, + ), + ), + repo_added_time=F("active_membership__pulp_created"), + has_provenance=Exists(PackageProvenance.objects.filter(package_id=OuterRef("pk"))), ).values( "filename", "sha256", @@ -383,9 +385,7 @@ def retrieve(self, request, path, package): "size", "repo_added_time", "version", - ) - provenances = PackageProvenance.objects.filter(package__in=local_packages).values_list( - "package__filename", flat=True + "has_provenance", ) local_releases = { p["filename"]: { @@ -394,7 +394,7 @@ def retrieve(self, request, path, package): "upload_time": p["repo_added_time"], "provenance": ( self.get_provenance_url(normalized, p["version"], p["filename"]) - if p["filename"] in provenances + if p["has_provenance"] else None ), } diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 1b30e82a4..d5b7da8bc 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -11,7 +11,7 @@ import pkginfo from aiohttp.client_exceptions import ClientError from django.conf import settings -from django.db.models import OuterRef, Subquery +from django.db.models import F, FilteredRelation, Q from django.db.utils import IntegrityError from jinja2 import Template from packaging.requirements import Requirement @@ -20,7 +20,7 @@ from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage from pulpcore.plugin.exceptions import TimeoutException -from pulpcore.plugin.models import Artifact, Remote, RepositoryContent +from pulpcore.plugin.models import Artifact, Remote from pulpcore.plugin.util import get_domain log = logging.getLogger(__name__) @@ -375,12 +375,16 @@ def python_content_to_json( Returns None if version is specified but not found within content_query """ if repository_version: - repo_added_subquery = RepositoryContent.objects.filter( - content_id=OuterRef("pk"), - repository=repository_version.repository, - version_removed=None, - ).values("pulp_created")[:1] - content_query = content_query.annotate(repo_added_time=Subquery(repo_added_subquery)) + content_query = content_query.annotate( + active_membership=FilteredRelation( + "version_memberships", + condition=Q( + version_memberships__repository=repository_version.repository, + version_memberships__version_removed=None, + ), + ), + repo_added_time=F("active_membership__pulp_created"), + ) full_metadata = {"last_serial": 0} # For now the serial field isn't supported by Pulp latest_content = latest_content_version(content_query, version) if not latest_content: