Add metrics on celery queue length

doc/admin/maintainance.rst

@@ -95,6 +95,12 @@ pretix_model_instances
     the ``model`` name. Starting with pretix 3.11, these numbers might only be approximate for
     most tables when running on PostgreSQL to mitigate performance impact.
 
+pretix_celery_tasks_queued_count
+    The number of background tasks in the worker queue, labeled with ``queue``.
+
+pretix_celery_tasks_queued_age_seconds
+    The age of the longest-waiting in the worker queue in seconds, labeled with ``queue``.
+
 .. _metric types: https://prometheus.io/docs/concepts/metric_types/
 .. _Prometheus: https://prometheus.io/
 .. _cProfile: https://docs.python.org/3/library/profile.html

src/pretix/base/metrics.py

@@ -1,4 +1,6 @@
+import json
 import math
+import time
 from collections import defaultdict
 
 from django.apps import apps
@@ -6,6 +8,7 @@ from django.conf import settings
 from django.db import connection
 
 from pretix.base.models import Event, Invoice, Order, OrderPosition, Organizer
+from pretix.celery_app import app
 
 if settings.HAS_REDIS:
     import django_redis
@@ -248,6 +251,19 @@ def metric_values():
         else:
             metrics['pretix_model_instances']['{model="%s"}' % m._meta] = estimate_count_fast(m)
 
+    if settings.HAS_CELERY:
+        client = app.broker_connection().channel().client
+        for q in settings.CELERY_TASK_QUEUES:
+            llen = client.llen(q.name)
+            lfirst = client.lindex(q.name, -1)
+            metrics['pretix_celery_tasks_queued_count']['{queue="%s"}' % q.name] = llen
+            if lfirst:
+                ldata = json.loads(lfirst)
+                dt = time.time() - ldata.get('created', 0)
+                metrics['pretix_celery_tasks_queued_age_seconds']['{queue="%s"}' % q.name] = dt
+            else:
+                metrics['pretix_celery_tasks_queued_age_seconds']['{queue="%s"}' % q.name] = 0
+
     return metrics