Experiment: optimize read-only content queries

pulpcore/app/migrations/0162_contentartifact_relative_path_like_index.py

@@ -0,0 +1,21 @@
+from django.contrib.postgres.indexes import BTreeIndex, OpClass
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    atomic = False  # required for CONCURRENTLY
+
+    dependencies = [
+        ("core", "0161_upstreampulp_remote_policy"),
+    ]
+
+    operations = [
+        AddIndexConcurrently(
+            model_name="contentartifact",
+            index=BTreeIndex(
+                OpClass("relative_path", name="text_pattern_ops"),
+                name="core_conten_relativ_like_idx",
+            ),
+        ),
+    ]

pulpcore/app/models/content.py

@@ -16,7 +16,7 @@
 
 from django.conf import settings
 from django.contrib.postgres.fields import HStoreField
-from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.indexes import BTreeIndex, GinIndex, OpClass
 from django.core import validators
 from django.db import IntegrityError, models, transaction
 from django.forms.models import model_to_dict
@@ -663,7 +663,15 @@ class ContentArtifact(BaseModel, QueryMixin):
 
     class Meta:
         unique_together = ("content", "relative_path")
-        indexes = [models.Index(fields=["relative_path"])]
+        indexes = [
+            models.Index(fields=["relative_path"]),
+            # Supports LIKE 'prefix%' (relative_path__startswith) regardless of collation,
+            # e.g. for directory listings served from a repository version.
+            BTreeIndex(
+                OpClass("relative_path", name="text_pattern_ops"),
+                name="core_conten_relativ_like_idx",
+            ),
+        ]
 
     @staticmethod
     def sort_key(ca):

pulpcore/app/settings.py

@@ -288,6 +288,10 @@
 CONTENT_ORIGIN = None
 CONTENT_PATH_PREFIX = "/pulp/content/"
 
+# Maximum number of entries rendered in a content app directory listing. Listings with more
+# entries are truncated and a "…" marker is shown. Set to 0 to disable the limit.
+DIRECTORY_LISTING_LIMIT = 10000
+
 API_APP_TTL = 120  # The heartbeat is called from gunicorn notify (defaulting to 45 sec).
 CONTENT_APP_TTL = 30
 WORKER_TTL = 30

pulpcore/content/handler.py

@@ -45,6 +45,7 @@
     models,
     transaction,
 )
+from django.db.models import Exists, OuterRef  # noqa: E402
 from jinja2 import Template  # noqa: E402
 
 from pulpcore.app import mime_types  # noqa: E402
@@ -662,6 +663,8 @@ async def list_directory(self, repo_version, publication, path):
 
         Returns:
             Set of strings representing the files and directories in the directory listing.
+            If DIRECTORY_LISTING_LIMIT is set and the listing exceeds it, the listing is
+            truncated and a "…" marker entry is included.
         """
 
         def update_date(dates, name, timestamp):
@@ -711,9 +714,22 @@ def list_directory_blocking():
 
             if repo_version or publication.pass_through:
                 cas = ContentArtifact.objects.select_related("artifact").filter(
-                    content__in=content_repo_ver.content, relative_path__startswith=path
+                    Exists(
+                        content_repo_ver._content_relationships().filter(
+                            content_id=OuterRef("content_id")
+                        )
+                    ),
+                    relative_path__startswith=path,
                 )
+                listing_limit = settings.DIRECTORY_LISTING_LIMIT
+                if listing_limit:
+                    # Fetch one extra row so truncation can be detected without a count query.
+                    cas = cas[: listing_limit + 1]
                 for ca in cas:
+                    if listing_limit and len(directory_list) >= listing_limit:
+                        # Too many entries to render; mark the listing as truncated.
+                        directory_list.add("…")
+                        break
                     name = file_or_directory_name(path, ca.relative_path)
                     directory_list.add(name)
                     update_content_date(content_dates, ca.content_id, ca.pulp_created)
@@ -726,9 +742,13 @@ def list_directory_blocking():
 
             if directory_list:
                 # Find the dates the content got added to the repository
-                for rc in content_repo_ver._content_relationships():
-                    if rc.content_id in content_to_find:
-                        content_dates[rc.content_id] = rc.pulp_created
+                content_dates.update(
+                    dict(
+                        content_repo_ver._content_relationships()
+                        .filter(content_id__in=list(content_to_find))
+                        .values_list("content_id", "pulp_created")
+                    )
+                )
 
                 # Prefer repository membership dates, then aggregate the newest date per entry.
                 for content_id, names in content_to_find.items():
@@ -897,9 +917,14 @@ async def _match_and_stream(self, path, request):
                             "artifact", "artifact__pulp_domain"
                         )
                         .filter(
-                            content__in=publication.repository_version.content,
+                            Exists(
+                                publication.repository_version._content_relationships().filter(
+                                    content_id=OuterRef("content_id")
+                                )
+                            ),
+                            relative_path=original_rel_path,
                         )
-                        .aget(relative_path=original_rel_path)
+                        .aget()
                     )
 
                 except MultipleObjectsReturned:
@@ -937,9 +962,25 @@ async def _match_and_stream(self, path, request):
             # Look for index.html or list the directory
             index_path = "{}index.html".format(rel_path)
 
-            contentartifact_exists = await ContentArtifact.objects.filter(
-                content__in=repo_version.content, relative_path=index_path
-            ).aexists()
+            # File-ish paths (no trailing slash, dot in the last segment, e.g.
+            # "a-1.0.0.jar") can never have a ContentArtifact at "<path>/index.html",
+            # so skip the existence probe entirely. Directory URLs (trailing slash
+            # or dot-less last segment) still probe below. A dotted *directory*
+            # name without a slash falls through to list_directory(), which issues
+            # the trailing-slash redirect; the redirected request probes again and
+            # serves index.html as before (one extra redirect, same content).
+            last_segment = rel_path.rstrip("/").rsplit("/", 1)[-1]
+            if not ends_in_slash and "." in last_segment:
+                contentartifact_exists = False
+            else:
+                contentartifact_exists = await ContentArtifact.objects.filter(
+                    Exists(
+                        repo_version._content_relationships().filter(
+                            content_id=OuterRef("content_id")
+                        )
+                    ),
+                    relative_path=index_path,
+                ).aexists()
             if contentartifact_exists:
                 original_rel_path = index_path
                 headers = self.response_headers(original_rel_path, distro)
@@ -960,9 +1001,20 @@ async def _match_and_stream(self, path, request):
                     )
 
             try:
-                ca = await ContentArtifact.objects.select_related(
-                    "artifact", "artifact__pulp_domain"
-                ).aget(content__in=repo_version.content, relative_path=original_rel_path)
+                ca = (
+                    await ContentArtifact.objects.select_related(
+                        "artifact", "artifact__pulp_domain"
+                    )
+                    .filter(
+                        Exists(
+                            repo_version._content_relationships().filter(
+                                content_id=OuterRef("content_id")
+                            )
+                        ),
+                        relative_path=original_rel_path,
+                    )
+                    .aget()
+                )
 
             except MultipleObjectsReturned:
                 log.error(

pulpcore/tests/unit/content/test_handler.py

@@ -308,7 +308,9 @@ async def test_list_directory_metadata(monkeypatch, use_publication):
         SimpleNamespace(objects=remote_artifact_manager),
     )
 
-    repo_version = SimpleNamespace(content=[], _content_relationships=lambda: [])
+    relationships = Mock()
+    relationships.filter.return_value.values_list.return_value = []
+    repo_version = SimpleNamespace(content=[], _content_relationships=lambda: relationships)
 
     if use_publication:
         publication_manager = Mock()