From 1ec14937db8541d6da3f23c4f2715a186e9bd5f8 Mon Sep 17 00:00:00 2001
From: Ondrej Rehounek <ondra.rehounek@seznam.cz>
Date: Fri, 1 Apr 2022 08:35:29 +0200
Subject: [PATCH] district: WIP celery
---
.isort.cfg | 2 +-
district/forms.py | 25 +++++++++++++------------
district/jekyll_import.py | 17 +++++++++++++----
district/tasks.py | 24 ++++++++++++++++++++++++
majak/celery.py | 22 ++++++++++++++++++++++
majak/settings/base.py | 8 ++++++++
requirements/base.in | 1 +
7 files changed, 82 insertions(+), 17 deletions(-)
create mode 100644 district/tasks.py
create mode 100644 majak/celery.py
diff --git a/.isort.cfg b/.isort.cfg
index 4bd2ae95..8e0a2515 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -3,4 +3,4 @@
line_length = 88
multi_line_output = 3
include_trailing_comma = true
-known_third_party = PyPDF2,arrow,bleach,bs4,captcha,django,environ,faker,ics,markdown,modelcluster,pirates,pytest,pytz,requests,sentry_sdk,snapshottest,taggit,wagtail,wagtailmetadata,weasyprint,yaml
+known_third_party = PyPDF2,arrow,bleach,bs4,captcha,celery,django,environ,faker,ics,markdown,modelcluster,pirates,pytest,pytz,requests,sentry_sdk,snapshottest,taggit,wagtail,wagtailmetadata,weasyprint,yaml
diff --git a/district/forms.py b/district/forms.py
index 38a96195..f1ec5b1e 100644
--- a/district/forms.py
+++ b/district/forms.py
@@ -3,7 +3,7 @@ from django.contrib.messages import ERROR, WARNING
from wagtail.admin.forms import WagtailAdminPageForm
from wagtail.core.models.collections import Collection
-from .jekyll_import import JekyllArticleImporter
+from .tasks import import_jekyll_articles
class JekyllImportForm(WagtailAdminPageForm):
@@ -63,25 +63,26 @@ class JekyllImportForm(WagtailAdminPageForm):
if len(self.errors):
return cleaned_data
- if cleaned_data.get("dry_run"):
- import_message_list = self.handle_import()
- for message in import_message_list:
- if message["level"] in (WARNING, ERROR):
- self.add_error("jekyll_repo_url", message["text"])
+ # if cleaned_data.get("dry_run"):
+ # import_message_list = self.handle_import()
+ # for message in import_message_list:
+ # if message["level"] in (WARNING, ERROR):
+ # self.add_error("jekyll_repo_url", message["text"])
return cleaned_data
def handle_import(self):
- import_message_list = JekyllArticleImporter(
- article_parent_page=self.instance,
- collection=self.cleaned_data["collection"],
+ # import_message_list =
+ import_jekyll_articles.delay(
+ article_parent_page_id=self.instance.id,
+ collection_id=self.cleaned_data["collection"].id,
url=self.cleaned_data["jekyll_repo_url"],
dry_run=self.cleaned_data["dry_run"],
use_git=self.cleaned_data["use_git"],
- ).perform_import()
+ )
- self.instance.import_message_list = import_message_list
- return import_message_list
+ # self.instance.import_message_list = import_message_list
+ # return import_message_list
def save(self, commit=True):
if self.cleaned_data.get("do_import") and not self.cleaned_data["dry_run"]:
diff --git a/district/jekyll_import.py b/district/jekyll_import.py
index 697709c6..e60c3325 100644
--- a/district/jekyll_import.py
+++ b/district/jekyll_import.py
@@ -256,18 +256,27 @@ params = {}
class JekyllArticleImporter:
def __init__(
- self, article_parent_page, collection, url: str, dry_run: bool, use_git: bool
+ self,
+ article_parent_page_id: int,
+ collection_id: int,
+ url: str,
+ dry_run: bool,
+ use_git: bool,
):
+ from district.models import DistrictArticlesPage
+
# Params
- self.article_parent_page = article_parent_page
- self.collection = collection
+ self.article_parent_page = DistrictArticlesPage.objects.get(
+ id=article_parent_page_id
+ )
+ self.collection = Collection.objects.get(id=collection_id)
self.dry_run = dry_run
self.use_git = use_git
self.url = url
# Computed props
self.path, self.repo_name = get_path_and_repo_name(self.url, self.use_git)
- self.site = article_parent_page.get_site()
+ self.site = self.article_parent_page.get_site()
self.site_config = get_site_config(self.path)
self.article_path = self.site_config.get("articlepath", None)
diff --git a/district/tasks.py b/district/tasks.py
new file mode 100644
index 00000000..859d217b
--- /dev/null
+++ b/district/tasks.py
@@ -0,0 +1,24 @@
+import logging
+
+from majak.celery import app
+
+from .jekyll_import import JekyllArticleImporter
+
+logger = logging.getLogger(__name__)
+
+
+@app.task
+def import_jekyll_articles(
+ article_parent_page_id: int,
+ collection_id: int,
+ url: str,
+ dry_run: bool,
+ use_git: bool,
+):
+ return JekyllArticleImporter(
+ article_parent_page_id=article_parent_page_id,
+ collection_id=collection_id,
+ url=url,
+ dry_run=dry_run,
+ use_git=use_git,
+ ).perform_import()
diff --git a/majak/celery.py b/majak/celery.py
new file mode 100644
index 00000000..9a05f6d1
--- /dev/null
+++ b/majak/celery.py
@@ -0,0 +1,22 @@
+import os
+
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "majak.settings.dev")
+
+app = Celery("majak")
+
+# Using a string here means the worker don't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+# should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+#
+#
+# @app.task(bind=True)
+# def debug_task(self):
+# print('Request: {0!r}'.format(self.request))
diff --git a/majak/settings/base.py b/majak/settings/base.py
index 927b9215..630c9337 100644
--- a/majak/settings/base.py
+++ b/majak/settings/base.py
@@ -190,6 +190,14 @@ CACHES = {
CACHES["default"]["TIMEOUT"] = 60 * 60 * 24
CACHES["renditions"]["TIMEOUT"] = 60 * 60 * 24
+# CELERY
+# ------------------------------------------------------------------------------
+CELERY_BROKER_URL = "redis://localhost:6379/6" # TODO set from env
+CELERY_RESULT_BACKEND = "redis://localhost:6379/6"
+CELERY_ACCEPT_CONTENT = ["application/json"]
+CELERY_RESULT_SERIALIZER = "json"
+CELERY_TASK_SERIALIZER = "json"
+
# SENTRY
# ------------------------------------------------------------------------------
diff --git a/requirements/base.in b/requirements/base.in
index 32bf181d..ff32bae5 100644
--- a/requirements/base.in
+++ b/requirements/base.in
@@ -17,6 +17,7 @@ sentry-sdk
Markdown
beautifulsoup4
bleach
+celery
ipython
weasyprint
pypdf2
--
GitLab