[DJ1.10] Upgrade Celery to 4.2.1, Kombu to 4.2.0

Should probably integrate with django-celery-results and/or django-celery-beat

Procfile

 web: gunicorn wsgi:application -b 0.0.0.0:$PORT -w 8
-worker: python manage.py celeryd -E -B --beat --concurrency=1
+worker: celery worker --app helios --events --beat --concurrency 1 --logfile celeryw.log --pidfile celeryw.pid
\ No newline at end of file

helios/__init__.py

 from django.conf import settings
-from django.core.urlresolvers import reverse
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from celery_app import app as celery_app
+
+__all__ = ('celery_app', 'TEMPLATE_BASE', 'ADMIN_ONLY', 'VOTERS_UPLOAD', 'VOTERS_EMAIL',)
 
 TEMPLATE_BASE = settings.HELIOS_TEMPLATE_BASE or "helios/templates/base.html"
 

helios/celery_app.py

+import os
+
+# set the default Django settings module for the 'celery' program.
+from celery import Celery
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
+
+app = Celery()
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+
+@app.task(bind=True)
+def debug_task(self):
+    print('Request: {0!r}'.format(self.request))

helios/tasks.py

@@ -5,8 +5,7 @@ Celery queued tasks for Helios
 ben@adida.net
 """
 import copy
-
-from celery.task import task
+from celery import shared_task
 from celery.utils.log import get_logger
 
 import signals
@@ -14,7 +13,7 @@ from models import CastVote, Election, Voter, VoterFile
 from view_utils import render_template_raw
 
 
-@task()
+@shared_task
 def cast_vote_verify_and_store(cast_vote_id, status_update_message=None, **kwargs):
     cast_vote = CastVote.objects.get(id=cast_vote_id)
     result = cast_vote.verify_and_store()
@@ -33,7 +32,8 @@ def cast_vote_verify_and_store(cast_vote_id, status_update_message=None, **kwarg
         logger = get_logger(cast_vote_verify_and_store.__name__)
         logger.error("Failed to verify and store %d" % cast_vote_id)
 
-@task()
+
+@shared_task
 def voters_email(election_id, subject_template, body_template, extra_vars={},
                  voter_constraints_include=None, voter_constraints_exclude=None):
     """
@@ -52,13 +52,15 @@ def voters_email(election_id, subject_template, body_template, extra_vars={},
     for voter in voters:
         single_voter_email.delay(voter.uuid, subject_template, body_template, extra_vars)
 
-@task()
+
+@shared_task
 def voters_notify(election_id, notification_template, extra_vars={}):
     election = Election.objects.get(id=election_id)
     for voter in election.voter_set.all():
         single_voter_notify.delay(voter.uuid, notification_template, extra_vars)
 
-@task()
+
+@shared_task
 def single_voter_email(voter_uuid, subject_template, body_template, extra_vars={}):
     voter = Voter.objects.get(uuid=voter_uuid)
 
@@ -70,7 +72,8 @@ def single_voter_email(voter_uuid, subject_template, body_template, extra_vars={
 
     voter.send_message(subject, body)
 
-@task()
+
+@shared_task
 def single_voter_notify(voter_uuid, notification_template, extra_vars={}):
     voter = Voter.objects.get(uuid=voter_uuid)
 
@@ -81,7 +84,8 @@ def single_voter_notify(voter_uuid, notification_template, extra_vars={}):
 
     voter.send_notification(notification)
 
-@task()
+
+@shared_task
 def election_compute_tally(election_id):
     election = Election.objects.get(id=election_id)
     election.compute_tally()
@@ -98,7 +102,8 @@ Helios
     if election.has_helios_trustee():
         tally_helios_decrypt.delay(election_id=election.id)
 
-@task()
+
+@shared_task
 def tally_helios_decrypt(election_id):
     election = Election.objects.get(id=election_id)
     election.helios_trustee_decrypt()
@@ -112,7 +117,8 @@ for election %s.
 Helios
 """ % election.name)
 
-@task()
+
+@shared_task
 def voter_file_process(voter_file_id):
     voter_file = VoterFile.objects.get(id=voter_file_id)
     voter_file.process()
@@ -128,7 +134,8 @@ has been processed.
 Helios
 """ % (voter_file.election.name, voter_file.num_voters))
 
-@task()
+
+@shared_task
 def election_notify_admin(election_id, subject, body):
     election = Election.objects.get(id=election_id)
     election.admin.send_message(subject, body)

helios/tests.py

@@ -2,29 +2,24 @@
 Unit Tests for Helios
 """
 
-import unittest, datetime, re, urllib
-import django_webtest
-
-import models
-import datatypes
-
-from helios_auth import models as auth_models
-from views import ELGAMAL_PARAMS
-import views
-import utils
+import datetime
+import re
+import urllib
 
-from django.db import IntegrityError, transaction
-from django.test.client import Client
+import django_webtest
+import uuid
+from django.conf import settings
+from django.core import mail
+from django.core.files import File
 from django.test import TestCase
 from django.utils.html import escape as html_escape
 
-from django.core import mail
-from django.core.files import File
-from django.core.urlresolvers import reverse
-from django.conf import settings
-from django.core.exceptions import PermissionDenied
+import helios.datatypes as datatypes
+import helios.models as models
+import helios.utils as utils
+import helios.views as views
+from helios_auth import models as auth_models
 
-import uuid
 
 class ElectionModelTests(TestCase):
     fixtures = ['users.json']
@@ -42,7 +37,7 @@ class ElectionModelTests(TestCase):
         self.election.questions = QUESTIONS
 
     def setup_trustee(self):
-        self.election.generate_trustee(ELGAMAL_PARAMS)
+        self.election.generate_trustee(views.ELGAMAL_PARAMS)
 
     def setup_openreg(self):
         self.election.openreg=True
@@ -115,7 +110,7 @@ class ElectionModelTests(TestCase):
         self.assertEquals(len(issues), 0)
         
     def test_helios_trustee(self):
-        self.election.generate_trustee(ELGAMAL_PARAMS)
+        self.election.generate_trustee(views.ELGAMAL_PARAMS)
 
         self.assertTrue(self.election.has_helios_trustee())
 
@@ -199,15 +194,15 @@ class ElectionModelTests(TestCase):
     def test_voter_registration(self):
         # before adding a voter
         voters = models.Voter.get_by_election(self.election)
-        self.assertTrue(len(voters) == 0)
+        self.assertEquals(0, len(voters))
 
         # make sure no voter yet
         voter = models.Voter.get_by_election_and_user(self.election, self.user)
-        self.assertTrue(voter == None)
+        self.assertIsNone(voter)
 
         # make sure no voter at all across all elections
         voters = models.Voter.get_by_user(self.user)
-        self.assertTrue(len(voters) == 0)
+        self.assertEquals(0, len(voters))
 
         # register the voter
         voter = models.Voter.register_user_in_election(self.user, self.election)
@@ -215,13 +210,13 @@ class ElectionModelTests(TestCase):
         # make sure voter is there now
         voter_2 = models.Voter.get_by_election_and_user(self.election, self.user)
 
-        self.assertFalse(voter == None)
-        self.assertFalse(voter_2 == None)
+        self.assertIsNotNone(voter)
+        self.assertIsNotNone(voter_2)
         self.assertEquals(voter, voter_2)
 
         # make sure voter is there in this call too
         voters = models.Voter.get_by_user(self.user)
-        self.assertTrue(len(voters) == 1)
+        self.assertEquals(1, len(voters))
         self.assertEquals(voter, voters[0])
 
         voter_2 = models.Voter.get_by_election_and_uuid(self.election, voter.uuid)
@@ -275,7 +270,7 @@ class DatatypeTests(TestCase):
 
     def setUp(self):
         self.election = models.Election.objects.all()[0]
-        self.election.generate_trustee(ELGAMAL_PARAMS)
+        self.election.generate_trustee(views.ELGAMAL_PARAMS)
 
     def test_instantiate(self):
         ld_obj = datatypes.LDObject.instantiate(self.election.get_helios_trustee(), '2011/01/Trustee')

requirements.txt

 Django==1.9.13
 anyjson==0.3.3
-celery==3.1.25
-django-celery==3.2.2
+celery==4.2.1
 django-picklefield==0.3.0
-kombu==3.0.37
+kombu==4.2.0
 html5lib==0.999
 psycopg2==2.7.3.2
 pyparsing==1.5.7

settings.py

@@ -157,9 +157,6 @@ INSTALLED_APPS = (
     'djangosecure',
     'django.contrib.sessions',
     'django.contrib.sites',
-    ## needed for queues
-    'djcelery',
-    'kombu.transport.django',
     ## HELIOS stuff
     'helios_auth',
     'helios',
@@ -274,17 +271,10 @@ logging.basicConfig(
 )
 
 
-# set up django-celery
-# BROKER_BACKEND = "kombu.transport.DatabaseTransport"
-BROKER_URL = "django://"
-CELERY_RESULT_DBURI = DATABASES['default']
-import djcelery
-djcelery.setup_loader()
-
-
-# for testing
-TEST_RUNNER = 'djcelery.contrib.test_runner.CeleryTestSuiteRunner'
-# this effectively does CELERY_ALWAYS_EAGER = True
+# set up celery
+CELERY_BROKER_URL = 'amqp://localhost'
+CELERY_TASK_ALWAYS_EAGER = True
+#database_url = DATABASES['default']
 
 # Rollbar Error Logging
 ROLLBAR_ACCESS_TOKEN = get_from_env('ROLLBAR_ACCESS_TOKEN', None)