Upgrade to Celery 4

src/pretix/base/services/async.py

@@ -19,7 +19,6 @@ from pretix.celery import app
 
 
 class ProfiledTask(app.Task):
-    abstract = True
 
     def __call__(self, *args, **kwargs):
 
@@ -43,7 +42,6 @@ class TransactionAwareTask(ProfiledTask):
     Task class which is aware of django db transactions and only executes tasks
     after transaction has been committed
     """
-    abstract = True
 
     def apply_async(self, *args, **kwargs):
         """

src/pretix/base/views/async.py

@@ -17,6 +17,7 @@ class AsyncAction:
     task = None
     success_url = None
     error_url = None
+    known_errortypes = []
 
     def do(self, *args):
         if not isinstance(self.task, app.Task):
@@ -53,7 +54,7 @@ class AsyncAction:
     def _return_ajax_result(self, res, timeout=.5):
         if not res.ready():
             try:
-                res.get(timeout=timeout)
+                res.get(timeout=timeout, propagate=False)
             except celery.exceptions.TimeoutError:
                 pass
 
@@ -118,8 +119,13 @@ class AsyncAction:
         return redirect(self.get_error_url())
 
     def get_error_message(self, exception):
-        logger.error('Unexpected exception: %r' % exception)
+        if isinstance(exception, dict) and exception['exc_type'] in self.known_errortypes:
-        return _('An unexpected error has occured.')
+            return exception['exc_message']
+        elif exception.__class__.__name__ in self.known_errortypes:
+            return str(exception)
+        else:
+            logger.error('Unexpected exception: %r' % exception)
+            return _('An unexpected error has occured.')
 
     def get_success_message(self, value):
         return _('The task has been completed.')

src/pretix/celery.py

@@ -1,29 +1,11 @@
 import os
 
 from celery import Celery
-from celery.utils.mail import ErrorMail
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pretix.settings")
 
 from django.conf import settings
 
 app = Celery('pretix')
-
+app.config_from_object('django.conf:settings', namespace='CELERY')
-
-class MyErrorMail(ErrorMail):
-
-    def should_send(self, context, exc):
-        from pretix.base.services.orders import OrderError
-        from pretix.base.services.cart import CartError
-
-        blacklist = (OrderError, CartError)
-        return not isinstance(exc, blacklist)
-
-
-app.config_from_object('django.conf:settings')
-app.conf.CELERY_ANNOTATIONS = {
-    '*': {
-        'ErrorMail': MyErrorMail,
-    }
-}
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

src/pretix/presale/checkoutflow.py

@@ -11,8 +11,7 @@ from django.views.generic.base import TemplateResponseMixin
 
 from pretix.base.models import Order
 from pretix.base.models.orders import InvoiceAddress
-from pretix.base.services.mail import SendMailException
+from pretix.base.services.orders import perform_order
-from pretix.base.services.orders import OrderError, perform_order
 from pretix.base.signals import register_payment_providers
 from pretix.multidomain.urlreverse import eventreverse
 from pretix.presale.forms.checkout import ContactForm, InvoiceAddressForm
@@ -294,6 +293,7 @@ class ConfirmStep(CartMixin, AsyncAction, TemplateFlowStep):
     identifier = "confirm"
     template_name = "pretixpresale/event/checkout_confirm.html"
     task = perform_order
+    known_errortypes = ['OrderError']
 
     def is_applicable(self, request):
         return True
@@ -350,11 +350,7 @@ class ConfirmStep(CartMixin, AsyncAction, TemplateFlowStep):
         return self.get_order_url(order)
 
     def get_error_message(self, exception):
-        if isinstance(exception, dict) and exception['exc_type'] == 'OrderError':
+        if exception.__class__.__name__ == 'SendMailException':
-            return exception['exc_message']
-        elif isinstance(exception, OrderError):
-            return str(exception)
-        elif isinstance(exception, SendMailException):
             return _('There was an error sending the confirmation mail. Please try again later.')
         return super().get_error_message(exception)
 

src/pretix/presale/views/cart.py

@@ -102,6 +102,7 @@ class CartActionMixin:
 
 class CartRemove(EventViewMixin, CartActionMixin, AsyncAction, View):
     task = remove_items_from_cart
+    known_errortypes = ['CartError']
 
     def get_success_message(self, value):
         if CartPosition.objects.filter(cart_id=self.request.session.session_key).exists():
@@ -109,13 +110,6 @@ class CartRemove(EventViewMixin, CartActionMixin, AsyncAction, View):
         else:
             return _('Your cart is empty.')
 
-    def get_error_message(self, exception):
-        if isinstance(exception, dict) and exception['exc_type'] == 'CartError':
-            return exception['exc_message']
-        elif isinstance(exception, CartError):
-            return str(exception)
-        return super().get_error_message(exception)
-
     def post(self, request, *args, **kwargs):
         items = self._items_from_post_data()
         if items:
@@ -131,17 +125,11 @@ class CartRemove(EventViewMixin, CartActionMixin, AsyncAction, View):
 
 class CartAdd(EventViewMixin, CartActionMixin, AsyncAction, View):
     task = add_items_to_cart
+    known_errortypes = ['CartError']
 
     def get_success_message(self, value):
         return _('The products have been successfully added to your cart.')
 
-    def get_error_message(self, exception):
-        if isinstance(exception, dict) and exception['exc_type'] == 'CartError':
-            return exception['exc_message']
-        elif isinstance(exception, CartError):
-            return str(exception)
-        return super().get_error_message(exception)
-
     def post(self, request, *args, **kwargs):
         items = self._items_from_post_data()
         if items:

src/pretix/presale/views/order.py

@@ -5,7 +5,7 @@ from django.http import FileResponse, Http404, HttpResponse
 from django.shortcuts import redirect, render
 from django.utils.functional import cached_property
 from django.utils.timezone import now
-from django.utils.translation import gettext, ugettext_lazy as _
+from django.utils.translation import ugettext_lazy as _
 from django.views.generic import TemplateView, View
 
 from pretix.base.models import CachedTicket, Invoice, Order, OrderPosition
@@ -13,7 +13,7 @@ from pretix.base.models.orders import InvoiceAddress
 from pretix.base.services.invoices import (
     generate_cancellation, generate_invoice, invoice_pdf, invoice_qualified,
 )
-from pretix.base.services.orders import OrderError, cancel_order
+from pretix.base.services.orders import cancel_order
 from pretix.base.services.tickets import generate
 from pretix.base.signals import (
     register_payment_providers, register_ticket_outputs,
@@ -462,6 +462,7 @@ class OrderCancel(EventViewMixin, OrderDetailMixin, TemplateView):
 
 class OrderCancelDo(EventViewMixin, OrderDetailMixin, AsyncAction, View):
     task = cancel_order
+    known_errortypes = ['OrderError']
 
     def get_success_url(self, value):
         return self.get_order_url()
@@ -485,13 +486,6 @@ class OrderCancelDo(EventViewMixin, OrderDetailMixin, AsyncAction, View):
     def get_success_message(self, value):
         return _('The order has been canceled.')
 
-    def get_error_message(self, exception):
-        if isinstance(exception, dict) and exception['exc_type'] == 'OrderError':
-            return gettext(exception['exc_message'])
-        elif isinstance(exception, OrderError):
-            return str(exception)
-        return super().get_error_message(exception)
-
 
 class OrderDownload(EventViewMixin, OrderDetailMixin, View):
 

src/pretix/settings.py

@@ -142,11 +142,10 @@ if not SESSION_ENGINE:
 
 HAS_CELERY = config.has_option('celery', 'broker')
 if HAS_CELERY:
-    BROKER_URL = config.get('celery', 'broker')
+    CELERY_BROKER_URL = config.get('celery', 'broker')
     CELERY_RESULT_BACKEND = config.get('celery', 'backend')
-    CELERY_SEND_TASK_ERROR_EMAILS = bool(ADMINS)
 else:
-    CELERY_ALWAYS_EAGER = True
+    CELERY_TASK_ALWAYS_EAGER = True
 
 SESSION_COOKIE_DOMAIN = config.get('pretix', 'cookie_domain', fallback=None)
 
@@ -431,9 +430,7 @@ LOGGING = {
 }
 
 CELERY_TASK_SERIALIZER = 'json'
-# We need to use pickle for now, because kombu/celery are unable to serialize
+CELERY_RESULT_SERIALIZER = 'json'
-# exceptions (that we also use as return values) into any other format.
-CELERY_RESULT_SERIALIZER = 'pickle'
 
 BOOTSTRAP3 = {
     'success_css_class': ''

src/requirements/production.txt

@@ -13,10 +13,8 @@ libsass
 django-otp==0.3.*
 python-u2flib-server==4.*
 django-formtools==1.0
-# celery>=3.1,<3.2
+celery==4.0.2
-# until the following issue is fixed, we need our own celery version
+kombu==4.0.2
-# https://github.com/celery/celery/pull/3199
-git+https://github.com/pretix/celery.git@pretix#egg=celery
 django-statici18n==1.2.*
 inlinestyler==0.2.*
 

src/requirements/redis.txt

@@ -1,5 +1,2 @@
 django-redis>=4.1,<4.2
-redis>=2.10,<2.11
+redis==2.10.5
-# until the following issue is fixed, we need our own kombu version
-# https://github.com/celery/kombu/pull/590
-git+https://github.com/pretix/kombu.git@pretix#egg=kombu