district: WIP celery

.isort.cfg

@@ -3,4 +3,4 @@
 line_length = 88
 multi_line_output = 3
 include_trailing_comma = true
-known_third_party = PyPDF2,arrow,bleach,bs4,captcha,django,environ,faker,ics,markdown,modelcluster,pirates,pytest,pytz,requests,sentry_sdk,snapshottest,taggit,wagtail,wagtailmetadata,weasyprint,yaml
+known_third_party = PyPDF2,arrow,bleach,bs4,captcha,celery,django,environ,faker,ics,markdown,modelcluster,pirates,pytest,pytz,requests,sentry_sdk,snapshottest,taggit,wagtail,wagtailmetadata,weasyprint,yaml

district/forms.py

@@ -3,7 +3,7 @@ from django.contrib.messages import ERROR, WARNING
 from wagtail.admin.forms import WagtailAdminPageForm
 from wagtail.core.models.collections import Collection
 
-from .jekyll_import import JekyllArticleImporter
+from .tasks import import_jekyll_articles
 
 
 class JekyllImportForm(WagtailAdminPageForm):
@@ -63,25 +63,26 @@ class JekyllImportForm(WagtailAdminPageForm):
         if len(self.errors):
             return cleaned_data
 
-        if cleaned_data.get("dry_run"):
-            import_message_list = self.handle_import()
-            for message in import_message_list:
-                if message["level"] in (WARNING, ERROR):
-                    self.add_error("jekyll_repo_url", message["text"])
+        # if cleaned_data.get("dry_run"):
+        # import_message_list = self.handle_import()
+        # for message in import_message_list:
+        #     if message["level"] in (WARNING, ERROR):
+        #         self.add_error("jekyll_repo_url", message["text"])
 
         return cleaned_data
 
     def handle_import(self):
-        import_message_list = JekyllArticleImporter(
-            article_parent_page=self.instance,
-            collection=self.cleaned_data["collection"],
+        # import_message_list =
+        import_jekyll_articles.delay(
+            article_parent_page_id=self.instance.id,
+            collection_id=self.cleaned_data["collection"].id,
             url=self.cleaned_data["jekyll_repo_url"],
             dry_run=self.cleaned_data["dry_run"],
             use_git=self.cleaned_data["use_git"],
-        ).perform_import()
+        )
 
-        self.instance.import_message_list = import_message_list
-        return import_message_list
+        # self.instance.import_message_list = import_message_list
+        # return import_message_list
 
     def save(self, commit=True):
         if self.cleaned_data.get("do_import") and not self.cleaned_data["dry_run"]:

district/jekyll_import.py

@@ -256,18 +256,27 @@ params = {}
 
 class JekyllArticleImporter:
     def __init__(
-        self, article_parent_page, collection, url: str, dry_run: bool, use_git: bool
+        self,
+        article_parent_page_id: int,
+        collection_id: int,
+        url: str,
+        dry_run: bool,
+        use_git: bool,
     ):
+        from district.models import DistrictArticlesPage
+
         # Params
-        self.article_parent_page = article_parent_page
-        self.collection = collection
+        self.article_parent_page = DistrictArticlesPage.objects.get(
+            id=article_parent_page_id
+        )
+        self.collection = Collection.objects.get(id=collection_id)
         self.dry_run = dry_run
         self.use_git = use_git
         self.url = url
 
         # Computed props
         self.path, self.repo_name = get_path_and_repo_name(self.url, self.use_git)
-        self.site = article_parent_page.get_site()
+        self.site = self.article_parent_page.get_site()
         self.site_config = get_site_config(self.path)
 
         self.article_path = self.site_config.get("articlepath", None)

district/tasks.py

+import logging
+
+from majak.celery import app
+
+from .jekyll_import import JekyllArticleImporter
+
+logger = logging.getLogger(__name__)
+
+
+@app.task
+def import_jekyll_articles(
+    article_parent_page_id: int,
+    collection_id: int,
+    url: str,
+    dry_run: bool,
+    use_git: bool,
+):
+    return JekyllArticleImporter(
+        article_parent_page_id=article_parent_page_id,
+        collection_id=collection_id,
+        url=url,
+        dry_run=dry_run,
+        use_git=use_git,
+    ).perform_import()

majak/celery.py

+import os
+
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "majak.settings.dev")
+
+app = Celery("majak")
+
+# Using a string here means the worker don't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+#
+#
+# @app.task(bind=True)
+# def debug_task(self):
+#     print('Request: {0!r}'.format(self.request))

majak/settings/base.py

@@ -190,6 +190,14 @@ CACHES = {
 CACHES["default"]["TIMEOUT"] = 60 * 60 * 24
 CACHES["renditions"]["TIMEOUT"] = 60 * 60 * 24
 
+# CELERY
+# ------------------------------------------------------------------------------
+CELERY_BROKER_URL = "redis://localhost:6379/6"  # TODO set from env
+CELERY_RESULT_BACKEND = "redis://localhost:6379/6"
+CELERY_ACCEPT_CONTENT = ["application/json"]
+CELERY_RESULT_SERIALIZER = "json"
+CELERY_TASK_SERIALIZER = "json"
+
 # SENTRY
 # ------------------------------------------------------------------------------
 

requirements/base.in

@@ -17,6 +17,7 @@ sentry-sdk
 Markdown
 beautifulsoup4
 bleach
+celery
 ipython
 weasyprint
 pypdf2