Add profiling for celery tasks

src/pretix/base/services/async.py

@@ -7,11 +7,38 @@ Usage:
     @task(base=TransactionAwareTask)
     def task_…():
 """
-from celery import Task
+import cProfile
+import os
+import random
+import time
+
+from django.conf import settings
 from django.db import transaction
 
+from pretix.celery import app
 
-class TransactionAwareTask(Task):
+
+class ProfiledTask(app.Task):
+    abstract = True
+
+    def __call__(self, *args, **kwargs):
+
+        if settings.PROFILING_RATE > 0 and random.random() < settings.PROFILING_RATE / 100:
+            profiler = cProfile.Profile()
+            profiler.enable()
+            starttime = time.time()
+            ret = super().__call__(*args, **kwargs)
+            profiler.disable()
+            tottime = time.time() - starttime
+            profiler.dump_stats(os.path.join(settings.PROFILE_DIR, '{time:.0f}_{tottime:.3f}_celery_{t}.pstat'.format(
+                t=self.name, tottime=tottime, time=time.time()
+            )))
+            return ret
+        else:
+            return super().__call__(*args, **kwargs)
+
+
+class TransactionAwareTask(ProfiledTask):
     """
     Task class which is aware of django db transactions and only executes tasks
     after transaction has been committed

src/pretix/base/services/cart.py

@@ -10,6 +10,7 @@ from pretix.base.i18n import LazyLocaleException
 from pretix.base.models import (
     CartPosition, Event, Item, ItemVariation, Quota, Voucher,
 )
+from pretix.base.services.async import ProfiledTask
 from pretix.base.services.locking import LockTimeoutException
 from pretix.celery import app
 
@@ -208,7 +209,7 @@ def _add_items_to_cart(event: Event, items: List[dict], cart_id: str=None) -> No
                 raise CartError(err)
 
 
-@app.task(bind=True, max_retries=5, default_retry_delay=1)
+@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1)
 def add_items_to_cart(self, event: int, items: List[dict], cart_id: str=None) -> None:
     """
     Adds a list of items to a user's cart.
@@ -249,7 +250,7 @@ def _remove_items_from_cart(event: Event, items: List[dict], cart_id: str) -> No
                     cp.delete()
 
 
-@app.task(bind=True, max_retries=5, default_retry_delay=1)
+@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1)
 def remove_items_from_cart(self, event: int, items: List[dict], cart_id: str=None) -> None:
     """
     Removes a list of items from a user's cart.

src/pretix/base/services/export.py

@@ -4,11 +4,12 @@ from django.core.files.base import ContentFile
 
 from pretix.base.i18n import language
 from pretix.base.models import CachedFile, Event, cachedfile_name
+from pretix.base.services.async import ProfiledTask
 from pretix.base.signals import register_data_exporters
 from pretix.celery import app
 
 
-@app.task()
+@app.task(base=ProfiledTask)
 def export(event: str, fileid: str, provider: str, form_data: Dict[str, Any]) -> None:
     event = Event.objects.get(id=event)
     file = CachedFile.objects.get(id=fileid)

src/pretix/base/services/orders.py

@@ -22,6 +22,7 @@ from pretix.base.models import (
 )
 from pretix.base.models.orders import InvoiceAddress
 from pretix.base.payment import BasePaymentProvider
+from pretix.base.services.async import ProfiledTask
 from pretix.base.services.invoices import (
     generate_cancellation, generate_invoice, invoice_qualified,
 )
@@ -585,7 +586,7 @@ class OrderChangeManager:
             raise OrderError(error_messages['internal'])
 
 
-@app.task(bind=True, max_retries=5, default_retry_delay=1)
+@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1)
 def perform_order(self, event: str, payment_provider: str, positions: List[str],
                   email: str=None, locale: str=None, address: int=None, meta_info: dict=None):
     try:
@@ -597,7 +598,7 @@ def perform_order(self, event: str, payment_provider: str, positions: List[str],
         return OrderError(error_messages['busy'])
 
 
-@app.task(bind=True, max_retries=5, default_retry_delay=1)
+@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1)
 def cancel_order(self, order: int, user: int=None):
     try:
         try:

src/pretix/base/services/tickets.py

@@ -8,12 +8,13 @@ from pretix.base.i18n import language
 from pretix.base.models import (
     CachedFile, CachedTicket, Event, Order, OrderPosition, cachedfile_name,
 )
+from pretix.base.services.async import ProfiledTask
 from pretix.base.signals import register_ticket_outputs
 from pretix.celery import app
 from pretix.helpers.database import rolledback_transaction
 
 
-@app.task
+@app.task(base=ProfiledTask)
 def generate(order_position: str, provider: str):
     order_position = OrderPosition.objects.select_related('order', 'order__event').get(id=order_position)
     ct = CachedTicket.objects.get_or_create(order_position=order_position, provider=provider)[0]