Optimize paginator usage

#210

district/models.py

@@ -541,10 +541,14 @@ class DistrictArticlesPage(
 
     def get_context(self, request):
         context = super().get_context(request)
-        context["articles"] = Paginator(
-            self.append_all_shared_articles(DistrictArticlePage.objects.child_of(self)),
+        context["articles"] = self.materialize_page_object_content(
+            Paginator(
+                self.append_all_shared_articles_query(
+                    DistrictArticlePage.objects.child_of(self)
+                ),
                 self.max_items,
             ).get_page(request.GET.get("page"))
+        )
         return context
 
     @route(r"^tagy/$", name="tags")
@@ -591,26 +595,30 @@ class DistrictArticlesPage(
 
         try:
             tag = Tag.objects.filter(slug=request.GET["tag"])[0]
-            article_page_qs = self.append_all_shared_articles(
+            article_page_qs = self.append_all_shared_articles_query(
                 DistrictArticlePage.objects.filter(
                     page_ptr_id__in=page_ids, tags__slug=tag.slug
                 ),
-                filter=lambda shared: shared.filter(
+                custom_article_query=lambda shared: shared.filter(
                     page_ptr_id__in=page_ids, tags__slug=tag.slug
                 ),
             )
         except (KeyError, IndexError):
             tag = None
-            article_page_qs = self.append_all_shared_articles(
+            article_page_qs = self.append_all_shared_articles_query(
                 DistrictArticlePage.objects.filter(page_ptr_id__in=page_ids),
-                filter=lambda shared: shared.filter(page_ptr_id__in=page_ids),
+                custom_article_query=lambda shared: shared.filter(
+                    page_ptr_id__in=page_ids
+                ),
             )
 
         return {
-            "article_page_list": Paginator(
+            "article_page_list": self.materialize_page_object_content(
+                Paginator(
                     article_page_qs,
                     self.max_items,
-            ).get_page(request.GET.get("page")),
+                ).get_page(request.GET.get("page"))
+            ),
             "tag": tag,
         }
 

main/models.py

@@ -58,6 +58,7 @@ class MainHomePage(
     ExtendedMetadataHomePageMixin,
     MetadataPageMixin,
     InstagramMixin,
+    ArticlesMixin,
     Page,
 ):
     # header
@@ -153,6 +154,7 @@ class MainHomePage(
 
     content_panels = Page.content_panels + [
         FieldPanel("content"),
+        *ArticleMixin.content_panels,
         FieldPanel("footer_other_links"),
         FieldPanel("footer_person_list"),
     ]
@@ -245,15 +247,21 @@ class MainHomePage(
         return context
 
     def get_region_response(self, request):
+        context = {}
         if request.GET.get("region", None) == "VSK":
             sorted_article_qs = MainArticlePage.objects.filter(
                 region__isnull=False
             ).order_by("-date")
-        else:
-            sorted_article_qs = MainArticlePage.objects.filter(
-                region=request.GET.get("region", None)
-            ).order_by("-date")
             context = {"article_data_list": sorted_article_qs[:3]}
+        else:
+            sorted_article_qs = self.append_all_shared_articles_query(
+                MainArticlePage.objects.filter(region=request.GET.get("region", None))
+            )
+            context = {
+                "article_data_list": self.materialize_shared_articles_query(
+                    sorted_article_qs[:3]
+                )
+            }
         data = {
             "html": render(
                 request, "main/includes/small_article_preview.html", context
@@ -494,12 +502,16 @@ class MainArticlesPage(
 
     def get_articles_response(self, request):
         article_paginator = Paginator(
-            MainArticlePage.objects.filter(article_type=ARTICLE_TYPES.PRESS_RELEASE)
-            .order_by("-date", "title")
-            .live(),
+            self.append_all_shared_articles_query(
+                MainArticlePage.objects.filter(
+                    article_type=ARTICLE_TYPES.PRESS_RELEASE
+                ),
+            ).order_by("union_date", "union_title"),
             10,
         )
-        article_page = article_paginator.get_page(request.GET.get("page", 1))
+        article_page = self.materialize_page_object_content(
+            article_paginator.get_page(request.GET.get("page", 1))
+        )
         context = {"article_data_list": article_page.object_list}
         html_content = render(
             request, "main/includes/person_article_preview.html", context
@@ -512,12 +524,17 @@ class MainArticlesPage(
 
     def get_all_articles_search_response(self, request):
         article_paginator = Paginator(
-            self.append_all_shared_articles(MainArticlePage.objects).search(
-                request.GET["q"]
+            self.append_all_shared_articles_query(
+                MainArticlePage.objects.search(
+                    request.GET["q"],
+                    custom_article_query=lambda query: query.search(request.GET["q"]),
+                )
             ),
             10,
         )
-        article_page = article_paginator.get_page(request.GET.get("page", 1))
+        article_page = self.materialize_page_object_content(
+            article_paginator.get_page(request.GET.get("page", 1))
+        )
         context = {"article_data_list": article_page.object_list}
         html_content = render(
             request, "main/includes/person_article_preview.html", context

shared/models.py

@@ -3,11 +3,12 @@ from enum import Enum
 from functools import reduce
 
 from django.apps import apps
+from django.core.paginator import Page as PaginatorPage
 from django.db import models
 from django.db.models.expressions import F, Value
 from django.utils import timezone
 from modelcluster.fields import ParentalKey, ParentalManyToManyField
-from taggit.models import ItemBase, Tag, TagBase
+from taggit.models import ItemBase, TagBase
 from wagtail.admin.panels import FieldPanel, MultiFieldPanel, PublishingPanel
 from wagtail.fields import StreamField
 from wagtail.models import Page
@@ -308,7 +309,7 @@ class ArticlesMixin(models.Model):
         elif self._meta.app_label == "main":
             return SharedArticlesPageType.MAIN
 
-    def unique_page_query_materialized(self, results):
+    def evaluate_page_query(self, results):
         return list(
             reduce(
                 lambda unique, item: unique | {f"{item['union_page_ptr_id']}": item},
@@ -318,7 +319,9 @@ class ArticlesMixin(models.Model):
         )
 
     def append_all_shared_articles_query(
-        self, previous_query: models.QuerySet | None = None, filter=None
+        self,
+        previous_query: models.QuerySet | None = None,
+        custom_article_query=None,
     ):
         """
         To prevent circular deps, we get class models during runtime
@@ -385,7 +388,9 @@ class ArticlesMixin(models.Model):
         main_article_query: models.QuerySet = MainArticlePage.objects
 
         apply_additional_filter = (
-            lambda query: filter(query) if filter is not None else query
+            lambda query: custom_article_query(query)
+            if custom_article_query is not None
+            else query
         )
 
         create_query_by_slug = lambda query: apply_additional_filter(
@@ -401,16 +406,16 @@ class ArticlesMixin(models.Model):
         )
 
         district_by_slug = create_query_by_slug(district_article_query)
-        if filter is not None:
-            district_by_slug = filter(district_by_slug)
+        if custom_article_query is not None:
+            district_by_slug = custom_article_query(district_by_slug)
 
         uniweb_by_slug = create_query_by_slug(uniweb_article_query)
-        if filter is not None:
-            uniweb_by_slug = filter(uniweb_by_slug)
+        if custom_article_query is not None:
+            uniweb_by_slug = custom_article_query(uniweb_by_slug)
 
         main_by_slug = create_query_by_slug(main_article_query)
-        if filter is not None:
-            main_by_slug = filter(main_by_slug)
+        if custom_article_query is not None:
+            main_by_slug = custom_article_query(main_by_slug)
 
         shared_field = Value(
             self.page_ptr.id,
@@ -478,9 +483,7 @@ class ArticlesMixin(models.Model):
 
         return results.order_by("union_date")
 
-    def append_all_shared_articles(
-        self, previous_query: models.QuerySet | None = None, filter=None
-    ):
+    def materialize_shared_articles_query(self, results):
         """
         To prevent circular deps, we get class models during runtime
         """
@@ -494,16 +497,14 @@ class ArticlesMixin(models.Model):
         district_meta_fields = DistrictArticlePage._meta.fields
         uniweb_meta_fields = UniwebArticlePage._meta.fields
 
-        results = self.append_all_shared_articles_query(previous_query, filter)
-
-        evaluated = self.unique_page_query_materialized(
-            results
-        )  # We MUST eval here since we can't turn values() into concrete class instances in QuerySet after union
-
         assign_to_model = lambda unioned: lambda assignment, field: assignment | {
             field.column: unioned[f"union_{field.column}"]
         }
 
+        evaluated = self.evaluate_page_query(
+            results
+        )  # We MUST eval here since we can't turn values() into concrete class instances in QuerySet after union
+
         if page_type == SharedArticlesPageType.DISTRICT:
             return list(
                 map(
@@ -534,12 +535,23 @@ class ArticlesMixin(models.Model):
                 )
             )
 
+    def append_all_shared_articles(
+        self, previous_query: models.QuerySet | None = None, custom_article_query=None
+    ):
+        return self.materialize_shared_articles_query(
+            self.append_all_shared_articles_query(previous_query, custom_article_query)
+        )
+
     def get_article_page_by_slug(self, slug: str):
         articles = self.append_all_shared_articles(
-            filter=lambda query: query.filter(slug=slug)
+            custom_article_query=lambda query: query.filter(slug=slug)
         )
         return articles[0]
 
+    def materialize_page_object_content(self, page: PaginatorPage):
+        page.object_list = self.materialize_page_object_content(page.object_list)
+        return page
+
     def setup_article_page_context(self, request):
         slug = request.GET.get("sdilene", "")
         return self.get_article_page_by_slug(slug).serve(request)
@@ -547,7 +559,6 @@ class ArticlesMixin(models.Model):
     def search_tags_by_unioned_id_query(
         self,
         articles: list,
-        additional_query=None,
         tags_model_query=None,
     ):
         DistrictArticleTag = apps.get_model(app_label="district.DistrictArticleTag")
@@ -568,27 +579,19 @@ class ArticlesMixin(models.Model):
             content_object_id__in=get_ids_by_page_type(
                 SharedArticlesPageType.DISTRICT.value
             )
-        )
+        ).select_related("tag_id")
         uniweb_tags = UniwebArticleTag.objects.filter(
             content_object_id__in=get_ids_by_page_type(
                 SharedArticlesPageType.UNIWEB.value
             )
-        )
+        ).select_related("tag_id")
         main_tags = MainArticleTag.objects.filter(
             content_object_id__in=get_ids_by_page_type(
                 SharedArticlesPageType.MAIN.value
             )
-        )
-
-        if additional_query is not None:
-            district_tags = additional_query(district_tags)
-            uniweb_tags = additional_query(uniweb_tags)
-            main_tags = additional_query(main_tags)
-
-        union = district_tags.union(uniweb_tags).union(main_tags)
+        ).select_related("tag_id")
 
-        union = union.values_list("tag_id", flat=True)
-        tag_query = Tag.objects.filter(id__in=union)
+        tag_query = district_tags.union(uniweb_tags).union(main_tags)
 
         if tags_model_query is not None:
             tag_query = tags_model_query(tag_query)

uniweb/models.py

@@ -552,9 +552,9 @@ class UniwebArticlesIndexPage(
 
         own_children = UniwebArticlePage.objects.child_of(self)
 
-        articles = self.append_all_shared_articles(
+        articles = self.append_all_shared_articles_query(
             own_children,
-            filter=lambda articles: articles.filter(**tag_params)
+            custom_article_query=lambda articles: articles.filter(**tag_params)
             if tag is not None
             else articles,
         )
@@ -566,7 +566,9 @@ class UniwebArticlesIndexPage(
             else articles,
         )
 
-        context["articles"] = Paginator(articles, ARTICLES_PER_PAGE).get_page(num)
+        context["articles"] = self.materialize_page_object_content(
+            Paginator(articles, ARTICLES_PER_PAGE).get_page(num)
+        )
         context["tags"] = self.search_tags_by_unioned_id_query(articles_ids)
         context["active_tag"] = tag
         return context