From c2c7f86e33b2eecad9cde1c2b743815a9dcfcaa1 Mon Sep 17 00:00:00 2001
From: Marco Ciotola <848222@stud.unive.it>
Date: Mon, 4 Mar 2019 21:44:35 +0100
Subject: [PATCH] [DJ1.10] Upgrade Celery to 4.2.1, Kombu to 4.2.0
Should probably integrate with django-celery-results and/or django-celery-beat
---
Procfile | 2 +-
helios/__init__.py | 6 +++-
helios/celery_app.py | 21 ++++++++++++
helios/tasks.py | 79 ++++++++++++++++++++++++--------------------
helios/tests.py | 49 ++++++++++++---------------
requirements.txt | 5 ++-
settings.py | 18 +++-------
7 files changed, 98 insertions(+), 82 deletions(-)
create mode 100644 helios/celery_app.py
diff --git a/Procfile b/Procfile
index 63d8d0c..608417d 100644
--- a/Procfile
+++ b/Procfile
@@ -1,2 +1,2 @@
web: gunicorn wsgi:application -b 0.0.0.0:$PORT -w 8
-worker: python manage.py celeryd -E -B --beat --concurrency=1
\ No newline at end of file
+worker: celery worker --app helios --events --beat --concurrency 1 --logfile celeryw.log --pidfile celeryw.pid
\ No newline at end of file
diff --git a/helios/__init__.py b/helios/__init__.py
index 3a516d7..2f59829 100644
--- a/helios/__init__.py
+++ b/helios/__init__.py
@@ -1,5 +1,9 @@
from django.conf import settings
-from django.core.urlresolvers import reverse
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from celery_app import app as celery_app
+
+__all__ = ('celery_app', 'TEMPLATE_BASE', 'ADMIN_ONLY', 'VOTERS_UPLOAD', 'VOTERS_EMAIL',)
TEMPLATE_BASE = settings.HELIOS_TEMPLATE_BASE or "helios/templates/base.html"
diff --git a/helios/celery_app.py b/helios/celery_app.py
new file mode 100644
index 0000000..89f0ecb
--- /dev/null
+++ b/helios/celery_app.py
@@ -0,0 +1,21 @@
+import os
+
+# set the default Django settings module for the 'celery' program.
+from celery import Celery
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
+
+app = Celery()
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+# should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+
+@app.task(bind=True)
+def debug_task(self):
+ print('Request: {0!r}'.format(self.request))
diff --git a/helios/tasks.py b/helios/tasks.py
index d550afc..8610097 100644
--- a/helios/tasks.py
+++ b/helios/tasks.py
@@ -5,8 +5,7 @@ Celery queued tasks for Helios
ben@adida.net
"""
import copy
-
-from celery.task import task
+from celery import shared_task
from celery.utils.log import get_logger
import signals
@@ -14,9 +13,9 @@ from models import CastVote, Election, Voter, VoterFile
from view_utils import render_template_raw
-@task()
+@shared_task
def cast_vote_verify_and_store(cast_vote_id, status_update_message=None, **kwargs):
- cast_vote = CastVote.objects.get(id = cast_vote_id)
+ cast_vote = CastVote.objects.get(id=cast_vote_id)
result = cast_vote.verify_and_store()
voter = cast_vote.voter
@@ -26,21 +25,22 @@ def cast_vote_verify_and_store(cast_vote_id, status_update_message=None, **kwarg
if result:
# send the signal
signals.vote_cast.send(sender=election, election=election, user=user, voter=voter, cast_vote=cast_vote)
-
+
if status_update_message and user.can_update_status():
user.update_status(status_update_message)
else:
logger = get_logger(cast_vote_verify_and_store.__name__)
logger.error("Failed to verify and store %d" % cast_vote_id)
-
-@task()
+
+
+@shared_task
def voters_email(election_id, subject_template, body_template, extra_vars={},
voter_constraints_include=None, voter_constraints_exclude=None):
"""
voter_constraints_include are conditions on including voters
voter_constraints_exclude are conditions on excluding voters
"""
- election = Election.objects.get(id = election_id)
+ election = Election.objects.get(id=election_id)
# select the right list of voters
voters = election.voter_set.all()
@@ -50,61 +50,66 @@ def voters_email(election_id, subject_template, body_template, extra_vars={},
voters = voters.exclude(**voter_constraints_exclude)
for voter in voters:
- single_voter_email.delay(voter.uuid, subject_template, body_template, extra_vars)
+ single_voter_email.delay(voter.uuid, subject_template, body_template, extra_vars)
+
-@task()
+@shared_task
def voters_notify(election_id, notification_template, extra_vars={}):
- election = Election.objects.get(id = election_id)
+ election = Election.objects.get(id=election_id)
for voter in election.voter_set.all():
single_voter_notify.delay(voter.uuid, notification_template, extra_vars)
-@task()
+
+@shared_task
def single_voter_email(voter_uuid, subject_template, body_template, extra_vars={}):
- voter = Voter.objects.get(uuid = voter_uuid)
+ voter = Voter.objects.get(uuid=voter_uuid)
the_vars = copy.copy(extra_vars)
- the_vars.update({'voter' : voter})
+ the_vars.update({'voter': voter})
subject = render_template_raw(None, subject_template, the_vars)
body = render_template_raw(None, body_template, the_vars)
voter.send_message(subject, body)
-@task()
+
+@shared_task
def single_voter_notify(voter_uuid, notification_template, extra_vars={}):
- voter = Voter.objects.get(uuid = voter_uuid)
+ voter = Voter.objects.get(uuid=voter_uuid)
the_vars = copy.copy(extra_vars)
- the_vars.update({'voter' : voter})
+ the_vars.update({'voter': voter})
notification = render_template_raw(None, notification_template, the_vars)
voter.send_notification(notification)
-@task()
+
+@shared_task
def election_compute_tally(election_id):
- election = Election.objects.get(id = election_id)
+ election = Election.objects.get(id=election_id)
election.compute_tally()
- election_notify_admin.delay(election_id = election_id,
- subject = "encrypted tally computed",
- body = """
+ election_notify_admin.delay(election_id=election_id,
+ subject="encrypted tally computed",
+ body="""
The encrypted tally for election %s has been computed.
--
Helios
""" % election.name)
-
+
if election.has_helios_trustee():
- tally_helios_decrypt.delay(election_id = election.id)
+ tally_helios_decrypt.delay(election_id=election.id)
-@task()
+
+@shared_task
def tally_helios_decrypt(election_id):
- election = Election.objects.get(id = election_id)
+ election = Election.objects.get(id=election_id)
election.helios_trustee_decrypt()
- election_notify_admin.delay(election_id = election_id,
- subject = 'Helios Decrypt',
- body = """
+ election_notify_admin.delay(election_id=election_id,
+ subject='Helios Decrypt',
+ body="""
Helios has decrypted its portion of the tally
for election %s.
@@ -112,13 +117,14 @@ for election %s.
Helios
""" % election.name)
-@task()
+
+@shared_task
def voter_file_process(voter_file_id):
- voter_file = VoterFile.objects.get(id = voter_file_id)
+ voter_file = VoterFile.objects.get(id=voter_file_id)
voter_file.process()
- election_notify_admin.delay(election_id = voter_file.election.id,
- subject = 'voter file processed',
- body = """
+ election_notify_admin.delay(election_id=voter_file.election.id,
+ subject='voter file processed',
+ body="""
Your voter file upload for election %s
has been processed.
@@ -128,7 +134,8 @@ has been processed.
Helios
""" % (voter_file.election.name, voter_file.num_voters))
-@task()
+
+@shared_task
def election_notify_admin(election_id, subject, body):
- election = Election.objects.get(id = election_id)
+ election = Election.objects.get(id=election_id)
election.admin.send_message(subject, body)
diff --git a/helios/tests.py b/helios/tests.py
index c87f74a..1e97188 100644
--- a/helios/tests.py
+++ b/helios/tests.py
@@ -2,29 +2,24 @@
Unit Tests for Helios
"""
-import unittest, datetime, re, urllib
-import django_webtest
-
-import models
-import datatypes
-
-from helios_auth import models as auth_models
-from views import ELGAMAL_PARAMS
-import views
-import utils
+import datetime
+import re
+import urllib
-from django.db import IntegrityError, transaction
-from django.test.client import Client
+import django_webtest
+import uuid
+from django.conf import settings
+from django.core import mail
+from django.core.files import File
from django.test import TestCase
from django.utils.html import escape as html_escape
-from django.core import mail
-from django.core.files import File
-from django.core.urlresolvers import reverse
-from django.conf import settings
-from django.core.exceptions import PermissionDenied
+import helios.datatypes as datatypes
+import helios.models as models
+import helios.utils as utils
+import helios.views as views
+from helios_auth import models as auth_models
-import uuid
class ElectionModelTests(TestCase):
fixtures = ['users.json']
@@ -42,7 +37,7 @@ class ElectionModelTests(TestCase):
self.election.questions = QUESTIONS
def setup_trustee(self):
- self.election.generate_trustee(ELGAMAL_PARAMS)
+ self.election.generate_trustee(views.ELGAMAL_PARAMS)
def setup_openreg(self):
self.election.openreg=True
@@ -115,7 +110,7 @@ class ElectionModelTests(TestCase):
self.assertEquals(len(issues), 0)
def test_helios_trustee(self):
- self.election.generate_trustee(ELGAMAL_PARAMS)
+ self.election.generate_trustee(views.ELGAMAL_PARAMS)
self.assertTrue(self.election.has_helios_trustee())
@@ -199,15 +194,15 @@ class ElectionModelTests(TestCase):
def test_voter_registration(self):
# before adding a voter
voters = models.Voter.get_by_election(self.election)
- self.assertTrue(len(voters) == 0)
+ self.assertEquals(0, len(voters))
# make sure no voter yet
voter = models.Voter.get_by_election_and_user(self.election, self.user)
- self.assertTrue(voter == None)
+ self.assertIsNone(voter)
# make sure no voter at all across all elections
voters = models.Voter.get_by_user(self.user)
- self.assertTrue(len(voters) == 0)
+ self.assertEquals(0, len(voters))
# register the voter
voter = models.Voter.register_user_in_election(self.user, self.election)
@@ -215,13 +210,13 @@ class ElectionModelTests(TestCase):
# make sure voter is there now
voter_2 = models.Voter.get_by_election_and_user(self.election, self.user)
- self.assertFalse(voter == None)
- self.assertFalse(voter_2 == None)
+ self.assertIsNotNone(voter)
+ self.assertIsNotNone(voter_2)
self.assertEquals(voter, voter_2)
# make sure voter is there in this call too
voters = models.Voter.get_by_user(self.user)
- self.assertTrue(len(voters) == 1)
+ self.assertEquals(1, len(voters))
self.assertEquals(voter, voters[0])
voter_2 = models.Voter.get_by_election_and_uuid(self.election, voter.uuid)
@@ -275,7 +270,7 @@ class DatatypeTests(TestCase):
def setUp(self):
self.election = models.Election.objects.all()[0]
- self.election.generate_trustee(ELGAMAL_PARAMS)
+ self.election.generate_trustee(views.ELGAMAL_PARAMS)
def test_instantiate(self):
ld_obj = datatypes.LDObject.instantiate(self.election.get_helios_trustee(), '2011/01/Trustee')
diff --git a/requirements.txt b/requirements.txt
index 1211f93..99e4906 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,8 @@
Django==1.9.13
anyjson==0.3.3
-celery==3.1.25
-django-celery==3.2.2
+celery==4.2.1
django-picklefield==0.3.0
-kombu==3.0.37
+kombu==4.2.0
html5lib==0.999
psycopg2==2.7.3.2
pyparsing==1.5.7
diff --git a/settings.py b/settings.py
index 34c87d1..9b30ff5 100644
--- a/settings.py
+++ b/settings.py
@@ -157,9 +157,6 @@ INSTALLED_APPS = (
'djangosecure',
'django.contrib.sessions',
'django.contrib.sites',
- ## needed for queues
- 'djcelery',
- 'kombu.transport.django',
## HELIOS stuff
'helios_auth',
'helios',
@@ -274,17 +271,10 @@ logging.basicConfig(
)
-# set up django-celery
-# BROKER_BACKEND = "kombu.transport.DatabaseTransport"
-BROKER_URL = "django://"
-CELERY_RESULT_DBURI = DATABASES['default']
-import djcelery
-djcelery.setup_loader()
-
-
-# for testing
-TEST_RUNNER = 'djcelery.contrib.test_runner.CeleryTestSuiteRunner'
-# this effectively does CELERY_ALWAYS_EAGER = True
+# set up celery
+CELERY_BROKER_URL = 'amqp://localhost'
+CELERY_TASK_ALWAYS_EAGER = True
+#database_url = DATABASES['default']
# Rollbar Error Logging
ROLLBAR_ACCESS_TOKEN = get_from_env('ROLLBAR_ACCESS_TOKEN', None)
--
GitLab