Refactor quota calculation (#1668)

src/pretix/api/views/item.py

@@ -20,6 +20,7 @@ from pretix.base.models import (
     CartPosition, Item, ItemAddOn, ItemBundle, ItemCategory, ItemVariation,
     Question, QuestionOption, Quota,
 )
+from pretix.base.services.quotas import QuotaAvailability
 from pretix.helpers.dicts import merge_dicts
 
 with scopes_disabled():
@@ -533,14 +534,17 @@ class QuotaViewSet(ConditionalListView, viewsets.ModelViewSet):
     def availability(self, request, *args, **kwargs):
         quota = self.get_object()
 
-        avail = quota.availability()
+        qa = QuotaAvailability()
+        qa.queue(quota)
+        qa.compute()
+        avail = qa.results[quota]
 
         data = {
-            'paid_orders': quota.count_paid_orders(),
+            'paid_orders': qa.count_paid_orders[quota],
-            'pending_orders': quota.count_pending_orders(),
+            'pending_orders': qa.count_pending_orders[quota],
-            'blocking_vouchers': quota.count_blocking_vouchers(),
+            'blocking_vouchers': qa.count_vouchers[quota],
-            'cart_positions': quota.count_in_cart(),
+            'cart_positions': qa.count_cart[quota],
-            'waiting_list': quota.count_waiting_list_pending(),
+            'waiting_list': qa.count_pending_orders[quota],
             'available_number': avail[1],
             'available': avail[0] == Quota.AVAILABILITY_OK,
             'total_size': quota.size,

src/pretix/base/exporters/orderlist.py

@@ -14,6 +14,7 @@ from pretix.base.models import (
     GiftCard, InvoiceAddress, InvoiceLine, Order, OrderPosition, Question,
 )
 from pretix.base.models.orders import OrderFee, OrderPayment, OrderRefund
+from pretix.base.services.quotas import QuotaAvailability
 from pretix.base.settings import PERSON_NAME_SCHEMES
 
 from ..exporter import ListExporter, MultiSheetListExporter
@@ -516,16 +517,21 @@ class QuotaListExporter(ListExporter):
         ]
         yield headers
 
-        for quota in self.event.quotas.all():
+        quotas = list(self.event.quotas.all())
-            avail = quota.availability()
+        qa = QuotaAvailability(full_results=True)
+        qa.queue(*quotas)
+        qa.compute()
+
+        for quota in quotas:
+            avail = qa.results[quota]
             row = [
                 quota.name,
                 _('Infinite') if quota.size is None else quota.size,
-                quota.count_paid_orders(),
+                qa.count_paid_orders[quota],
-                quota.count_pending_orders(),
+                qa.count_pending_orders[quota],
-                quota.count_blocking_vouchers(),
+                qa.count_vouchers[quota],
-                quota.count_in_cart(),
+                qa.count_cart[quota],
-                quota.count_waiting_list_pending(),
+                qa.count_waitinglist[quota],
                 _('Infinite') if avail[1] is None else avail[1]
             ]
             yield row

src/pretix/base/models/event.py

@@ -214,8 +214,10 @@ class EventMixin:
         vars_reserved = set()
         items_gone = set()
         vars_gone = set()
+
+        r = getattr(self, '_quota_cache', {})
         for q in self.active_quotas:
-            res = q.availability(allow_cache=True)
+            res = r[q] if q in r else q.availability(allow_cache=True)
 
             if res[0] == Quota.AVAILABILITY_OK:
                 if q.active_items:

src/pretix/base/models/items.py

@@ -7,11 +7,10 @@ from typing import Tuple
 
 import dateutil.parser
 import pytz
-from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.core.validators import RegexValidator
 from django.db import models
-from django.db.models import F, Func, Q, Sum
+from django.db.models import Q
 from django.utils import formats
 from django.utils.crypto import get_random_string
 from django.utils.functional import cached_property
@@ -25,7 +24,6 @@ from pretix.base.models import fields
 from pretix.base.models.base import LoggedModel
 from pretix.base.models.fields import MultiStringField
 from pretix.base.models.tax import TaxedPrice
-from pretix.base.signals import quota_availability
 
 from .event import Event, SubEvent
 
@@ -1350,6 +1348,7 @@ class Quota(LoggedModel):
             self.event.cache.clear()
 
     def save(self, *args, **kwargs):
+        # This is *not* called when the db-level cache is upated, since we use bulk_update there
         clear_cache = kwargs.pop('clear_cache', True)
         super().save(*args, **kwargs)
         if self.event and clear_cache:
@@ -1384,6 +1383,8 @@ class Quota(LoggedModel):
         :returns: a tuple where the first entry is one of the ``Quota.AVAILABILITY_`` constants
                   and the second is the number of available tickets.
         """
+        from ..services.quotas import QuotaAvailability
+
         if allow_cache and self.cache_is_hot() and count_waitinglist:
             return self.cached_availability_state, self.cached_availability_number
 
@@ -1392,141 +1393,16 @@ class Quota(LoggedModel):
 
         if _cache is not None and self.pk in _cache:
             return _cache[self.pk]
-        now_dt = now_dt or now()
+        qa = QuotaAvailability(count_waitinglist=count_waitinglist, early_out=False)
-        res = self._availability(now_dt, count_waitinglist)
+        qa.queue(self)
-        for recv, resp in quota_availability.send(sender=self.event, quota=self, result=res,
+        qa.compute(now_dt=now_dt)
-                                                  count_waitinglist=count_waitinglist):
+        res = qa.results[self]
-            res = resp
-
-        if res[0] <= Quota.AVAILABILITY_ORDERED and self.close_when_sold_out and not self.closed:
-            self.closed = True
-            self.save(update_fields=['closed'])
-            self.log_action('pretix.event.quota.closed')
-
-        self.event.cache.delete('item_quota_cache')
-        rewrite_cache = count_waitinglist and (
-            not self.cache_is_hot(now_dt) or res[0] > self.cached_availability_state
-        )
-        if rewrite_cache:
-            self.cached_availability_state = res[0]
-            self.cached_availability_number = res[1]
-            self.cached_availability_time = now_dt
-            if self.size is None:
-                self.cached_availability_paid_orders = self.count_paid_orders()
-            self.save(
-                update_fields=[
-                    'cached_availability_state', 'cached_availability_number', 'cached_availability_time',
-                    'cached_availability_paid_orders'
-                ],
-                clear_cache=False,
-                using='default'
-            )
 
         if _cache is not None:
             _cache[self.pk] = res
             _cache['_count_waitinglist'] = count_waitinglist
         return res
 
-    def _availability(self, now_dt: datetime=None, count_waitinglist=True, ignore_closed=False):
-        now_dt = now_dt or now()
-        if self.closed and not ignore_closed:
-            return Quota.AVAILABILITY_ORDERED, 0
-
-        size_left = self.size
-        if size_left is None:
-            return Quota.AVAILABILITY_OK, None
-
-        paid_orders = self.count_paid_orders()
-        self.cached_availability_paid_orders = paid_orders
-        size_left -= paid_orders
-        if size_left <= 0:
-            return Quota.AVAILABILITY_GONE, 0
-
-        size_left -= self.count_pending_orders()
-        if size_left <= 0:
-            return Quota.AVAILABILITY_ORDERED, 0
-
-        size_left -= self.count_blocking_vouchers(now_dt)
-        if size_left <= 0:
-            return Quota.AVAILABILITY_ORDERED, 0
-
-        if count_waitinglist:
-            size_left -= self.count_waiting_list_pending()
-            if size_left <= 0:
-                return Quota.AVAILABILITY_ORDERED, 0
-
-        size_left -= self.count_in_cart(now_dt)
-        if size_left <= 0:
-            return Quota.AVAILABILITY_RESERVED, 0
-
-        return Quota.AVAILABILITY_OK, size_left
-
-    def count_blocking_vouchers(self, now_dt: datetime=None) -> int:
-        from pretix.base.models import Voucher
-
-        now_dt = now_dt or now()
-        if 'sqlite3' in settings.DATABASES['default']['ENGINE']:
-            func = 'MAX'
-        else:  # NOQA
-            func = 'GREATEST'
-
-        return Voucher.objects.filter(
-            Q(event=self.event) & Q(subevent=self.subevent) &
-            Q(block_quota=True) &
-            Q(Q(valid_until__isnull=True) | Q(valid_until__gte=now_dt)) &
-            Q(Q(self._position_lookup) | Q(quota=self))
-        ).values('id').aggregate(
-            free=Sum(Func(F('max_usages') - F('redeemed'), 0, function=func))
-        )['free'] or 0
-
-    def count_waiting_list_pending(self) -> int:
-        from pretix.base.models import WaitingListEntry
-        return WaitingListEntry.objects.filter(
-            Q(voucher__isnull=True) & Q(subevent=self.subevent) &
-            self._position_lookup
-        ).distinct().count()
-
-    def count_in_cart(self, now_dt: datetime=None) -> int:
-        from pretix.base.models import CartPosition
-
-        now_dt = now_dt or now()
-        return CartPosition.objects.filter(
-            Q(event=self.event) & Q(subevent=self.subevent) &
-            Q(expires__gte=now_dt) &
-            Q(
-                Q(voucher__isnull=True)
-                | Q(voucher__block_quota=False)
-                | Q(voucher__valid_until__lt=now_dt)
-            ) &
-            self._position_lookup
-        ).count()
-
-    def count_pending_orders(self) -> dict:
-        from pretix.base.models import Order, OrderPosition
-
-        # This query has beeen benchmarked against a Count('id', distinct=True) aggregate and won by a small margin.
-        return OrderPosition.objects.filter(
-            self._position_lookup, order__status=Order.STATUS_PENDING, order__event=self.event, subevent=self.subevent
-        ).count()
-
-    def count_paid_orders(self):
-        from pretix.base.models import Order, OrderPosition
-
-        return OrderPosition.objects.filter(
-            self._position_lookup, order__status=Order.STATUS_PAID, order__event=self.event, subevent=self.subevent
-        ).count()
-
-    @cached_property
-    def _position_lookup(self) -> Q:
-        return (
-            (  # Orders for items which do not have any variations
-               Q(variation__isnull=True) &
-               Q(item_id__in=Quota.items.through.objects.filter(quota_id=self.pk).values_list('item_id', flat=True))
-            ) | (  # Orders for items which do have any variations
-                   Q(variation__in=Quota.variations.through.objects.filter(quota_id=self.pk).values_list('itemvariation_id', flat=True))
-            )
-        )
-
     class QuotaExceededException(Exception):
         pass
 
@@ -1535,7 +1411,6 @@ class Quota(LoggedModel):
         for variation in (variations or []):
             if variation.item not in items:
                 raise ValidationError(_('All variations must belong to an item contained in the items list.'))
-                break
 
     @staticmethod
     def clean_items(event, items, variations):

src/pretix/base/services/cart.py

@@ -25,6 +25,7 @@ from pretix.base.reldate import RelativeDateWrapper
 from pretix.base.services.checkin import _save_answers
 from pretix.base.services.locking import LockTimeoutException, NoLockManager
 from pretix.base.services.pricing import get_price
+from pretix.base.services.quotas import QuotaAvailability
 from pretix.base.services.tasks import ProfiledEventTask
 from pretix.base.settings import PERSON_NAME_SCHEMES
 from pretix.base.signals import validate_cart_addons
@@ -728,10 +729,14 @@ class CartManager:
 
     def _get_quota_availability(self):
         quotas_ok = defaultdict(int)
+        qa = QuotaAvailability()
+        qa.queue(*[k for k, v in self._quota_diff.items() if v > 0])
+        qa.compute(now_dt=self.now_dt)
         for quota, count in self._quota_diff.items():
             if count <= 0:
                 quotas_ok[quota] = 0
-            avail = quota.availability(self.now_dt)
+                break
+            avail = qa.results[quota]
             if avail[1] is not None and avail[1] < count:
                 quotas_ok[quota] = min(count, avail[1])
             else:

src/pretix/base/services/orders.py

@@ -44,6 +44,7 @@ from pretix.base.services.invoices import (
 from pretix.base.services.locking import LockTimeoutException, NoLockManager
 from pretix.base.services.mail import SendMailException
 from pretix.base.services.pricing import get_price
+from pretix.base.services.quotas import QuotaAvailability
 from pretix.base.services.tasks import ProfiledEventTask, ProfiledTask
 from pretix.base.signals import (
     allow_ticket_download, order_approved, order_canceled, order_changed,
@@ -1391,10 +1392,13 @@ class OrderChangeManager:
                     raise OrderError(self.error_messages['seat_subevent_mismatch'].format(seat=v['seat'].name))
 
     def _check_quotas(self):
+        qa = QuotaAvailability()
+        qa.queue(*[k for k, v in self._quotadiff.items() if v > 0])
+        qa.compute()
         for quota, diff in self._quotadiff.items():
             if diff <= 0:
                 continue
-            avail = quota.availability()
+            avail = qa.results[quota]
             if avail[0] != Quota.AVAILABILITY_OK or (avail[1] is not None and avail[1] < diff):
                 raise OrderError(self.error_messages['quota'].format(name=quota.name))
 

src/pretix/base/services/quotas.py

@@ -1,15 +1,357 @@
+import sys
+from collections import Counter, defaultdict
 from datetime import timedelta
 
 from django.conf import settings
-from django.db.models import Max, Q
+from django.db.models import Count, F, Func, Max, Q, Sum
 from django.dispatch import receiver
 from django.utils.timezone import now
 from django_scopes import scopes_disabled
 
-from pretix.base.models import Event, LogEntry
+from pretix.base.models import (
+    CartPosition, Event, LogEntry, Order, OrderPosition, Quota, Voucher,
+    WaitingListEntry,
+)
 from pretix.celery_app import app
 
-from ..signals import periodic_task
+from ..signals import periodic_task, quota_availability
+
+
+class QuotaAvailability:
+    """
+    This special object allows so compute the availability of multiple quotas, even across events, and inspect their
+    results. The maximum number of SQL queries is constant and not dependent on the number of quotas.
+
+    Usage example::
+
+        qa = QuotaAvailability()
+        qa.queue(quota1, quota2, …)
+        qa.compute()
+        print(qa.results)
+
+    Properties you can access after computation.
+
+    * results (dict mapping quotas to availability tuples)
+    * count_paid_orders (dict mapping quotas to ints)
+    * count_paid_orders (dict mapping quotas to ints)
+    * count_pending_orders (dict mapping quotas to ints)
+    * count_vouchers (dict mapping quotas to ints)
+    * count_waitinglist (dict mapping quotas to ints)
+    * count_cart (dict mapping quotas to ints)
+    """
+
+    def __init__(self, count_waitinglist=True, ignore_closed=False, full_results=False, early_out=True):
+        """
+        Initialize a new quota availability calculator
+
+        :param count_waitinglist: If ``True`` (default), the waiting list is considered. If ``False``, it is ignored.
+
+        :param ignore_closed: Quotas have a ``closed`` state that always makes the quota return as sold out. If you set
+                              ``ignore_closed`` to ``True``, we will ignore this completely. Default is ``False``.
+
+        :param full_results: Usually, the computation is as efficient as possible, i.e. if after counting the sold
+                             orders we already see that the quota is sold out, we're not going to count the carts,
+                             since it does not matter. This also means that you will not be able to get that number from
+                             ``.count_cart``. If you want all parts to be calculated (i.e. because you want to show
+                             statistics to the user), pass ``full_results`` and we'll skip that optimization.
+                             items
+
+        :param early_out: Usually, if a quota is ``closed`` or if its ``size`` is ``None`` (i.e. unlimited), we will
+                          not need database access to determine the availability and return it right away. If you set
+                          this to ``False``, however, we will *still* count the number of orders, which is required to
+                          keep the database-level quota cache up to date so backend overviews render quickly. If you
+                          do not care about keeping the cache up to date, you can set this to ``False`` for further
+                          performance improvements.
+        """
+        self._queue = []
+        self._count_waitinglist = count_waitinglist
+        self._ignore_closed = ignore_closed
+        self._full_results = full_results
+        self._item_to_quotas = defaultdict(list)
+        self._var_to_quotas = defaultdict(list)
+        self._early_out = early_out
+        self._quota_objects = {}
+        self.results = {}
+        self.count_paid_orders = defaultdict(int)
+        self.count_pending_orders = defaultdict(int)
+        self.count_vouchers = defaultdict(int)
+        self.count_waitinglist = defaultdict(int)
+        self.count_cart = defaultdict(int)
+
+        self.sizes = {}
+
+    def queue(self, *quota):
+        self._queue += quota
+
+    def compute(self, now_dt=None):
+        now_dt = now_dt or now()
+        quotas = list(self._queue)
+        quotas_original = list(self._queue)
+        self._queue.clear()
+        if not quotas:
+            return
+
+        self._compute(quotas, now_dt)
+
+        for q in quotas_original:
+            for recv, resp in quota_availability.send(sender=q.event, quota=q, result=self.results[q],
+                                                      count_waitinglist=self.count_waitinglist):
+                self.results[q] = resp
+
+        self._close(quotas)
+        self._write_cache(quotas, now_dt)
+
+    def _write_cache(self, quotas, now_dt):
+        events = {q.event for q in quotas}
+        update = []
+        for e in events:
+            e.cache.delete('item_quota_cache')
+        for q in quotas:
+            rewrite_cache = self._count_waitinglist and (
+                not q.cache_is_hot(now_dt) or self.results[q][0] > q.cached_availability_state
+                or q.cached_availability_paid_orders is None
+            )
+            if rewrite_cache:
+                q.cached_availability_state = self.results[q][0]
+                q.cached_availability_number = self.results[q][1]
+                q.cached_availability_time = now_dt
+                if q in self.count_paid_orders:
+                    q.cached_availability_paid_orders = self.count_paid_orders[q]
+                update.append(q)
+        if update:
+            Quota.objects.using('default').bulk_update(update, [
+                'cached_availability_state', 'cached_availability_number', 'cached_availability_time',
+                'cached_availability_paid_orders'
+            ], batch_size=50)
+
+    def _close(self, quotas):
+        for q in quotas:
+            if self.results[q][0] <= Quota.AVAILABILITY_ORDERED and q.close_when_sold_out and not q.closed:
+                q.closed = True
+                q.save(update_fields=['closed'])
+                q.log_action('pretix.event.quota.closed')
+
+    def _compute(self, quotas, now_dt):
+        # Quotas we want to look at now
+        self.sizes.update({q: q.size for q in quotas})
+
+        # Some helpful caches
+        self._quota_objects.update({q.pk: q for q in quotas})
+
+        # Compute result for closed or unlimited
+        self._compute_early_outs(quotas)
+
+        if self._early_out:
+            if not self._full_results:
+                quotas = [q for q in quotas if q not in self.results]
+                if not quotas:
+                    return
+
+        size_left = Counter({q: (sys.maxsize if s is None else s) for q, s in self.sizes.items()})
+        for q in quotas:
+            self.count_paid_orders[q] = 0
+            self.count_pending_orders[q] = 0
+            self.count_cart[q] = 0
+            self.count_vouchers[q] = 0
+            self.count_waitinglist[q] = 0
+
+        # Fetch which quotas belong to which items and variations
+        q_items = Quota.items.through.objects.filter(
+            quota_id__in=[q.pk for q in quotas]
+        ).values('quota_id', 'item_id')
+        for m in q_items:
+            self._item_to_quotas[m['item_id']].append(self._quota_objects[m['quota_id']])
+
+        q_vars = Quota.variations.through.objects.filter(
+            quota_id__in=[q.pk for q in quotas]
+        ).values('quota_id', 'itemvariation_id')
+        for m in q_vars:
+            self._var_to_quotas[m['itemvariation_id']].append(self._quota_objects[m['quota_id']])
+
+        self._compute_orders(quotas, q_items, q_vars, size_left)
+
+        if not self._full_results:
+            quotas = [q for q in quotas if q not in self.results]
+            if not quotas:
+                return
+
+        self._compute_vouchers(quotas, q_items, q_vars, size_left, now_dt)
+
+        if not self._full_results:
+            quotas = [q for q in quotas if q not in self.results]
+            if not quotas:
+                return
+
+        self._compute_carts(quotas, q_items, q_vars, size_left, now_dt)
+
+        if self._count_waitinglist:
+            if not self._full_results:
+                quotas = [q for q in quotas if q not in self.results]
+                if not quotas:
+                    return
+
+            self._compute_waitinglist(quotas, q_items, q_vars, size_left)
+
+        for q in quotas:
+            if q not in self.results:
+                if size_left[q] > 0:
+                    self.results[q] = Quota.AVAILABILITY_OK, size_left[q]
+                else:
+                    raise ValueError("inconclusive quota")
+
+    def _compute_orders(self, quotas, q_items, q_vars, size_left):
+        events = {q.event_id for q in quotas}
+        subevents = {q.subevent_id for q in quotas}
+        seq = Q(subevent_id__in=subevents)
+        if None in subevents:
+            seq |= Q(subevent__isnull=True)
+        op_lookup = OrderPosition.objects.filter(
+            order__status__in=[Order.STATUS_PAID, Order.STATUS_PENDING],
+            order__event_id__in=events,
+        ).filter(seq).filter(
+            Q(
+                Q(variation_id__isnull=True) &
+                Q(item_id__in={i['item_id'] for i in q_items if self._quota_objects[i['quota_id']] in quotas})
+            ) | Q(
+                variation_id__in={i['itemvariation_id'] for i in q_vars if self._quota_objects[i['quota_id']] in quotas})
+        ).order_by().values('order__status', 'item_id', 'subevent_id', 'variation_id').annotate(c=Count('*'))
+        for line in sorted(op_lookup, key=lambda li: li['order__status'], reverse=True):  # p before n
+            if line['variation_id']:
+                qs = self._var_to_quotas[line['variation_id']]
+            else:
+                qs = self._item_to_quotas[line['item_id']]
+            for q in qs:
+                if q.subevent_id == line['subevent_id']:
+                    size_left[q] -= line['c']
+                    if line['order__status'] == Order.STATUS_PAID:
+                        self.count_paid_orders[q] += line['c']
+                        q.cached_availability_paid_orders = line['c']
+                    elif line['order__status'] == Order.STATUS_PENDING:
+                        self.count_pending_orders[q] += line['c']
+                    if size_left[q] <= 0 and q not in self.results:
+                        if line['order__status'] == Order.STATUS_PAID:
+                            self.results[q] = Quota.AVAILABILITY_GONE, 0
+                        else:
+                            self.results[q] = Quota.AVAILABILITY_ORDERED, 0
+
+    def _compute_vouchers(self, quotas, q_items, q_vars, size_left, now_dt):
+        events = {q.event_id for q in quotas}
+        if 'sqlite3' in settings.DATABASES['default']['ENGINE']:
+            func = 'MAX'
+        else:  # NOQA
+            func = 'GREATEST'
+
+        subevents = {q.subevent_id for q in quotas}
+        seq = Q(subevent_id__in=subevents)
+        if None in subevents:
+            seq |= Q(subevent__isnull=True)
+        v_lookup = Voucher.objects.filter(
+            Q(event_id__in=events) &
+            seq &
+            Q(block_quota=True) &
+            Q(Q(valid_until__isnull=True) | Q(valid_until__gte=now_dt)) &
+            Q(
+                Q(
+                    Q(variation_id__isnull=True) &
+                    Q(item_id__in={i['item_id'] for i in q_items if self._quota_objects[i['quota_id']] in quotas})
+                ) | Q(
+                    variation_id__in={i['itemvariation_id'] for i in q_vars if
+                                      self._quota_objects[i['quota_id']] in quotas}
+                ) | Q(
+                    quota_id__in=[q.pk for q in quotas]
+                )
+            )
+        ).order_by().values('subevent_id', 'item_id', 'quota_id', 'variation_id').annotate(
+            free=Sum(Func(F('max_usages') - F('redeemed'), 0, function=func))
+        )
+        for line in v_lookup:
+            if line['variation_id']:
+                qs = self._var_to_quotas[line['variation_id']]
+            elif line['item_id']:
+                qs = self._item_to_quotas[line['item_id']]
+            else:
+                qs = [self._quota_objects[line['quota_id']]]
+            for q in qs:
+                if q.subevent_id == line['subevent_id']:
+                    size_left[q] -= line['free']
+                    self.count_vouchers[q] += line['free']
+                    if q not in self.results and size_left[q] <= 0:
+                        self.results[q] = Quota.AVAILABILITY_ORDERED, 0
+
+    def _compute_carts(self, quotas, q_items, q_vars, size_left, now_dt):
+        events = {q.event_id for q in quotas}
+        subevents = {q.subevent_id for q in quotas}
+        seq = Q(subevent_id__in=subevents)
+        if None in subevents:
+            seq |= Q(subevent__isnull=True)
+        cart_lookup = CartPosition.objects.filter(
+            Q(event_id__in=events) &
+            seq &
+            Q(expires__gte=now_dt) &
+            Q(
+                Q(voucher__isnull=True)
+                | Q(voucher__block_quota=False)
+                | Q(voucher__valid_until__lt=now_dt)
+            ) &
+            Q(
+                Q(
+                    Q(variation_id__isnull=True) &
+                    Q(item_id__in={i['item_id'] for i in q_items if self._quota_objects[i['quota_id']] in quotas})
+                ) | Q(
+                    variation_id__in={i['itemvariation_id'] for i in q_vars if self._quota_objects[i['quota_id']] in quotas}
+                )
+            )
+        ).order_by().values('item_id', 'subevent_id', 'variation_id').annotate(c=Count('*'))
+        for line in cart_lookup:
+            if line['variation_id']:
+                qs = self._var_to_quotas[line['variation_id']]
+            else:
+                qs = self._item_to_quotas[line['item_id']]
+            for q in qs:
+                if q.subevent_id == line['subevent_id']:
+                    size_left[q] -= line['c']
+                    self.count_cart[q] += line['c']
+                    if q not in self.results and size_left[q] <= 0:
+                        self.results[q] = Quota.AVAILABILITY_RESERVED, 0
+
+    def _compute_waitinglist(self, quotas, q_items, q_vars, size_left):
+        events = {q.event_id for q in quotas}
+        subevents = {q.subevent_id for q in quotas}
+        seq = Q(subevent_id__in=subevents)
+        if None in subevents:
+            seq |= Q(subevent__isnull=True)
+        w_lookup = WaitingListEntry.objects.filter(
+            Q(event_id__in=events) &
+            Q(voucher__isnull=True) &
+            seq &
+            Q(
+                Q(
+                    Q(variation_id__isnull=True) &
+                    Q(item_id__in={i['item_id'] for i in q_items if self._quota_objects[i['quota_id']] in quotas})
+                ) | Q(variation_id__in={i['itemvariation_id'] for i in q_vars if
+                                        self._quota_objects[i['quota_id']] in quotas})
+            )
+        ).order_by().values('item_id', 'subevent_id', 'variation_id').annotate(c=Count('*'))
+        for line in w_lookup:
+            if line['variation_id']:
+                qs = self._var_to_quotas[line['variation_id']]
+            else:
+                qs = self._item_to_quotas[line['item_id']]
+            for q in qs:
+                if q.subevent_id == line['subevent_id']:
+                    size_left[q] -= line['c']
+                    self.count_waitinglist[q] += line['c']
+                    if q not in self.results and size_left[q] <= 0:
+                        self.results[q] = Quota.AVAILABILITY_ORDERED, 0
+
+    def _compute_early_outs(self, quotas):
+        for q in quotas:
+            if q.closed and not self._ignore_closed:
+                self.results[q] = Quota.AVAILABILITY_ORDERED, 0
+            elif q.size is None:
+                self.results[q] = Quota.AVAILABILITY_OK, None
+            elif q.size == 0:
+                self.results[q] = Quota.AVAILABILITY_GONE, 0
 
 
 @receiver(signal=periodic_task)

src/pretix/control/templates/pretixcontrol/items/quotas.html

@@ -59,7 +59,7 @@
                         </td>
                         <td>
                             <ul>
-                                {% for item in q.items.all %}
+                                {% for item in cached_items%}
                                     {% if not item.has_variations %}
                                         <li><a href="{% url "control:event.item" organizer=request.event.organizer.slug event=request.event.slug item=item.id %}">{{ item }}</a></li>
                                     {% endif %}
@@ -74,7 +74,7 @@
                             <td>{{ q.subevent.name }} – {{ q.subevent.get_date_range_display }}</td>
                         {% endif %}
                         <td>{% if q.size == None %}Unlimited{% else %}{{ q.size }}{% endif %}</td>
-                        <td>{% include "pretixcontrol/items/fragment_quota_availability.html" with availability=q.availability closed=q.closed %}</td>
+                        <td>{% include "pretixcontrol/items/fragment_quota_availability.html" with availability=q.cached_avail closed=q.closed %}</td>
                         <td class="text-right flip">
                             <a href="{% url "control:event.items.quotas.edit" organizer=request.event.organizer.slug event=request.event.slug quota=q.id %}" class="btn btn-default btn-sm"><i class="fa fa-edit"></i></a>
                             <a href="{% url "control:event.items.quotas.add" organizer=request.event.organizer.slug event=request.event.slug %}?copy_from={{ q.id }}"

src/pretix/control/views/dashboards.py

@@ -25,6 +25,7 @@ from pretix.base.models import (
     Item, ItemVariation, Order, OrderPosition, OrderRefund, RequiredAction,
     SubEvent, Voucher, WaitingListEntry,
 )
+from pretix.base.services.quotas import QuotaAvailability
 from pretix.base.timeline import timeline_for_event
 from pretix.control.forms.event import CommentForm
 from pretix.control.signals import (
@@ -199,10 +200,20 @@ def waitinglist_widgets(sender, subevent=None, lazy=False, **kwargs):
 @receiver(signal=event_dashboard_widgets)
 def quota_widgets(sender, subevent=None, lazy=False, **kwargs):
     widgets = []
+    quotas = sender.quotas.filter(subevent=subevent)
 
-    for q in sender.quotas.filter(subevent=subevent):
+    quotas_to_compute = [
+        q for q in quotas
+        if not q.cache_is_hot(now() + timedelta(seconds=5))
+    ]
+    qa = QuotaAvailability()
+    if quotas_to_compute:
+        qa.queue(*quotas_to_compute)
+        qa.compute()
+
+    for q in quotas:
         if not lazy:
-            status, left = q.availability(allow_cache=True)
+            status, left = qa.results[q] if q in qa.results else q.availability(allow_cache=True)
         widgets.append({
             'content': None if lazy else NUM_WIDGET.format(
                 num='{}/{}'.format(left, q.size) if q.size is not None else '\u221e',

src/pretix/control/views/item.py

@@ -31,6 +31,7 @@ from pretix.base.models import (
 )
 from pretix.base.models.event import SubEvent
 from pretix.base.models.items import ItemAddOn, ItemBundle, ItemMetaValue
+from pretix.base.services.quotas import QuotaAvailability
 from pretix.base.services.tickets import invalidate_cache
 from pretix.base.signals import quota_availability
 from pretix.control.forms.item import (
@@ -643,9 +644,7 @@ class QuotaList(PaginationMixin, ListView):
     template_name = 'pretixcontrol/items/quotas.html'
 
     def get_queryset(self):
-        qs = Quota.objects.filter(
+        qs = self.request.event.quotas.prefetch_related(
-            event=self.request.event
-        ).prefetch_related(
             Prefetch(
                 "items",
                 queryset=Item.objects.annotate(
@@ -654,13 +653,28 @@ class QuotaList(PaginationMixin, ListView):
                 to_attr="cached_items"
             ),
             "variations",
-            "variations__item"
+            "variations__item",
+            Prefetch(
+                "subevent",
+                queryset=self.request.event.subevents.all()
+            )
         )
         if self.request.GET.get("subevent", "") != "":
             s = self.request.GET.get("subevent", "")
             qs = qs.filter(subevent_id=s)
         return qs
 
+    def get_context_data(self, **kwargs):
+        ctx = super().get_context_data()
+
+        qa = QuotaAvailability()
+        qa.queue(*ctx['quotas'])
+        qa.compute()
+        for quota in ctx['quotas']:
+            qa.cached_avail = qa.results[quota]
+
+        return ctx
+
 
 class QuotaCreate(EventPermissionRequiredMixin, CreateView):
     model = Quota
@@ -719,28 +733,30 @@ class QuotaView(ChartContainingView, DetailView):
     def get_context_data(self, *args, **kwargs):
         ctx = super().get_context_data()
 
-        avail = self.object.availability()
+        qa = QuotaAvailability(full_results=True)
-        ctx['avail'] = avail
+        qa.queue(self.object)
+        qa.compute()
+        ctx['avail'] = qa.results[self.object]
 
         data = [
             {
                 'label': gettext('Paid orders'),
-                'value': self.object.count_paid_orders(),
+                'value': qa.count_paid_orders[self.object],
                 'sum': True,
             },
             {
                 'label': gettext('Pending orders'),
-                'value': self.object.count_pending_orders(),
+                'value': qa.count_pending_orders[self.object],
                 'sum': True,
             },
             {
                 'label': gettext('Vouchers and waiting list reservations'),
-                'value': self.object.count_blocking_vouchers(),
+                'value': qa.count_vouchers[self.object],
                 'sum': True,
             },
             {
                 'label': gettext('Current user\'s carts'),
-                'value': self.object.count_in_cart(),
+                'value': qa.count_cart[self.object],
                 'sum': True,
             },
         ]
@@ -756,14 +772,14 @@ class QuotaView(ChartContainingView, DetailView):
         })
         data.append({
             'label': gettext('Waiting list (pending)'),
-            'value': self.object.count_waiting_list_pending(),
+            'value': qa.count_waitinglist[self.object],
             'sum': False,
         })
 
         if self.object.size is not None:
             data.append({
                 'label': gettext('Currently for sale'),
-                'value': avail[1],
+                'value': ctx['avail'][1],
                 'sum': False,
                 'strong': True
             })
@@ -786,7 +802,17 @@ class QuotaView(ChartContainingView, DetailView):
         ctx['has_ignore_vouchers'] = Voucher.objects.filter(
             Q(allow_ignore_quota=True) &
             Q(Q(valid_until__isnull=True) | Q(valid_until__gte=now())) &
-            Q(Q(self.object._position_lookup) | Q(quota=self.object)) &
+            Q(
+                (
+                    (  # Orders for items which do not have any variations
+                        Q(variation__isnull=True) &
+                        Q(item_id__in=Quota.items.through.objects.filter(quota_id=self.object.pk).values_list('item_id', flat=True))
+                    ) | (  # Orders for items which do have any variations
+                        Q(variation__in=Quota.variations.through.objects.filter(quota_id=self.object.pk).values_list(
+                            'itemvariation_id', flat=True))
+                    )
+                ) | Q(quota=self.object)
+            ) &
             Q(redeemed__lt=F('max_usages'))
         ).exists()
         if self.object.closed:

src/pretix/control/views/main.py

@@ -18,6 +18,7 @@ from i18nfield.strings import LazyI18nString
 from pretix.base.forms import SafeSessionWizardView
 from pretix.base.i18n import language
 from pretix.base.models import Event, EventMetaValue, Organizer, Quota, Team
+from pretix.base.services.quotas import QuotaAvailability
 from pretix.control.forms.event import (
     EventWizardBasicsForm, EventWizardCopyForm, EventWizardFoundationForm,
 )
@@ -82,19 +83,27 @@ class EventList(PaginationMixin, ListView):
             self.filter_form[k] for k in self.filter_form.fields if k.startswith('meta_')
         ]
 
+        quotas = []
         for s in ctx['events']:
             s.first_quotas = s.first_quotas[:4]
-            for q in s.first_quotas:
+            quotas += list(s.first_quotas)
-                q.cached_avail = (
+
-                    (q.cached_availability_state, q.cached_availability_number)
+        qa = QuotaAvailability(early_out=False)
-                    if q.cached_availability_time is not None
+        for q in quotas:
-                    else q.availability(allow_cache=True)
+            if q.cached_availability_time is None or q.cached_availability_paid_orders is None:
+                qa.queue(q)
+        qa.compute()
+
+        for q in quotas:
+            q.cached_avail = (
+                qa.results[q] if q in qa.results
+                else (q.cached_availability_state, q.cached_availability_number)
+            )
+            if q.size is not None:
+                q.percent_paid = min(
+                    100,
+                    round(q.cached_availability_paid_orders / q.size * 100) if q.size > 0 else 100
                 )
-                if q.size is not None:
-                    q.percent_paid = min(
-                        100,
-                        round(q.cached_availability_paid_orders / q.size * 100) if q.size > 0 else 100
-                    )
         return ctx
 
     @cached_property

src/pretix/control/views/subevents.py

@@ -24,6 +24,7 @@ from pretix.base.models.items import (
     ItemVariation, Quota, SubEventItem, SubEventItemVariation,
 )
 from pretix.base.reldate import RelativeDate, RelativeDateWrapper
+from pretix.base.services.quotas import QuotaAvailability
 from pretix.control.forms.checkin import CheckinListForm
 from pretix.control.forms.filter import SubEventFilterForm
 from pretix.control.forms.item import QuotaForm
@@ -68,19 +69,28 @@ class SubEventList(EventPermissionRequiredMixin, PaginationMixin, ListView):
     def get_context_data(self, **kwargs):
         ctx = super().get_context_data(**kwargs)
         ctx['filter_form'] = self.filter_form
+
+        quotas = []
         for s in ctx['subevents']:
             s.first_quotas = s.first_quotas[:4]
-            for q in s.first_quotas:
+            quotas += list(s.first_quotas)
-                q.cached_avail = (
+
-                    (q.cached_availability_state, q.cached_availability_number)
+        qa = QuotaAvailability(early_out=False)
-                    if q.cached_availability_time is not None
+        for q in quotas:
-                    else q.availability(allow_cache=True)
+            if q.cached_availability_time is None or q.cached_availability_paid_orders is None:
+                qa.queue(q)
+        qa.compute()
+
+        for q in quotas:
+            q.cached_avail = (
+                qa.results[q] if q in qa.results
+                else (q.cached_availability_state, q.cached_availability_number)
+            )
+            if q.size is not None:
+                q.percent_paid = min(
+                    100,
+                    round(q.cached_availability_paid_orders / q.size * 100) if q.size > 0 else 100
                 )
-                if q.size is not None:
-                    q.percent_paid = min(
-                        100,
-                        round(q.cached_availability_paid_orders / q.size * 100) if q.size > 0 else 100
-                    )
         return ctx
 
     @cached_property

src/pretix/presale/views/event.py

@@ -22,6 +22,7 @@ from pretix.base.channels import get_all_sales_channels
 from pretix.base.models import ItemVariation, Quota, SeatCategoryMapping
 from pretix.base.models.event import SubEvent
 from pretix.base.models.items import ItemBundle
+from pretix.base.services.quotas import QuotaAvailability
 from pretix.multidomain.urlreverse import eventreverse
 from pretix.presale.ical import get_ical
 from pretix.presale.signals import item_description
@@ -116,6 +117,24 @@ def get_grouped_items(event, subevent=None, voucher=None, channel='web', require
         # If a voucher is set to a specific quota, we need to filter out on that level
         restrict_vars = set(voucher.quota.variations.all())
 
+    quotas_to_compute = []
+    for item in items:
+        if item.has_variations:
+            for v in item.available_variations:
+                for q in v._subevent_quotas:
+                    if q not in quota_cache:
+                        quotas_to_compute.append(q)
+        else:
+            for q in item._subevent_quotas:
+                if q not in quota_cache:
+                    quotas_to_compute.append(q)
+
+    if quotas_to_compute:
+        qa = QuotaAvailability()
+        qa.queue(*quotas_to_compute)
+        qa.compute()
+        quota_cache.update({q.pk: r for q, r in qa.results.items()})
+
     for item in items:
         if voucher and voucher.item_id and voucher.variation_id:
             # Restrict variations if the voucher only allows one
@@ -250,7 +269,7 @@ def get_grouped_items(event, subevent=None, voucher=None, channel='web', require
 
             item._remove = not bool(item.available_variations)
 
-    if not external_quota_cache:
+    if not external_quota_cache and not voucher:
         event.cache.set('item_quota_cache', quota_cache, 5)
     items = [item for item in items
              if (len(item.available_variations) > 0 or not item.has_variations) and not item._remove]

src/pretix/presale/views/organizer.py

@@ -18,6 +18,7 @@ from pretix.base.i18n import language
 from pretix.base.models import (
     Event, EventMetaValue, SubEvent, SubEventMetaValue,
 )
+from pretix.base.services.quotas import QuotaAvailability
 from pretix.helpers.daterange import daterange
 from pretix.multidomain.urlreverse import eventreverse
 from pretix.presale.ical import get_ical
@@ -272,7 +273,21 @@ def add_subevents_for_days(qs, before, after, ebd, timezones, event=None, cart_n
     ).order_by(
         'date_from'
     )
+    quotas_to_compute = []
     for se in qs:
+        if se.presale_is_running:
+            quotas_to_compute += [
+                q for q in se.active_quotas
+                if not q.cache_is_hot(now() + timedelta(seconds=5))
+            ]
+
+    if quotas_to_compute:
+        qa = QuotaAvailability()
+        qa.queue(*quotas_to_compute)
+        qa.compute()
+    for se in qs:
+        if quotas_to_compute:
+            se._quota_cache = qa.results
         kwargs = {'subevent': se.pk}
         if cart_namespace:
             kwargs['cart_namespace'] = cart_namespace

src/tests/base/test_models.py

@@ -26,6 +26,7 @@ from pretix.base.models.items import (
 )
 from pretix.base.reldate import RelativeDate, RelativeDateWrapper
 from pretix.base.services.orders import OrderError, cancel_order, perform_order
+from pretix.base.services.quotas import QuotaAvailability
 from pretix.testutils.scope import classscope
 
 
@@ -274,7 +275,10 @@ class QuotaTestCase(BaseQuotaTestCase):
         if 'sqlite' not in settings.DATABASES['default']['ENGINE']:
             pytest.xfail('This should raise a type error on most databases')
         Voucher.objects.create(quota=self.quota, event=self.event, block_quota=True, max_usages=2, redeemed=4)
-        self.assertEqual(self.quota.count_blocking_vouchers(), 0)
+        qa = QuotaAvailability(full_results=True)
+        qa.queue(self.quota)
+        qa.compute()
+        self.assertEqual(qa.count_vouchers[self.quota], 0)
 
     @classscope(attr='o')
     def test_voucher_quota_multiuse_multiproduct(self):
@@ -293,7 +297,10 @@ class QuotaTestCase(BaseQuotaTestCase):
                                redeemed=2)
         Voucher.objects.create(item=self.item2, variation=self.var2, event=self.event, block_quota=True, max_usages=5,
                                redeemed=2)
-        self.assertEqual(self.quota.count_blocking_vouchers(), 9)
+        qa = QuotaAvailability(full_results=True)
+        qa.queue(self.quota)
+        qa.compute()
+        self.assertEqual(qa.count_vouchers[self.quota], 9)
 
     @classscope(attr='o')
     def test_voucher_quota_expiring_soon(self):
@@ -322,8 +329,11 @@ class QuotaTestCase(BaseQuotaTestCase):
         CartPosition.objects.create(event=self.event, item=self.item1, price=2,
                                     expires=now() + timedelta(days=3), voucher=v)
         self.assertTrue(v.is_in_cart())
-        self.assertEqual(self.quota.count_blocking_vouchers(), 1)
+        qa = QuotaAvailability(full_results=True)
-        self.assertEqual(self.quota.count_in_cart(), 0)
+        qa.queue(self.quota)
+        qa.compute()
+        self.assertEqual(qa.count_vouchers[self.quota], 1)
+        self.assertEqual(qa.count_cart[self.quota], 0)
         self.assertEqual(self.item1.check_quotas(), (Quota.AVAILABILITY_OK, 1))
 
     @classscope(attr='o')
@@ -332,8 +342,11 @@ class QuotaTestCase(BaseQuotaTestCase):
         v = Voucher.objects.create(quota=self.quota, event=self.event, block_quota=True)
         CartPosition.objects.create(event=self.event, item=self.item1, price=2,
                                     expires=now() + timedelta(days=3), voucher=v)
-        self.assertEqual(self.quota.count_blocking_vouchers(), 1)
+        qa = QuotaAvailability(full_results=True)
-        self.assertEqual(self.quota.count_in_cart(), 0)
+        qa.queue(self.quota)
+        qa.compute()
+        self.assertEqual(qa.count_vouchers[self.quota], 1)
+        self.assertEqual(qa.count_cart[self.quota], 0)
         self.assertEqual(self.item1.check_quotas(), (Quota.AVAILABILITY_OK, 1))
 
     @classscope(attr='o')
@@ -343,8 +356,11 @@ class QuotaTestCase(BaseQuotaTestCase):
                                    block_quota=True)
         CartPosition.objects.create(event=self.event, item=self.item1, price=2,
                                     expires=now() + timedelta(days=3), voucher=v)
-        self.assertEqual(self.quota.count_blocking_vouchers(), 0)
+        qa = QuotaAvailability(full_results=True)
-        self.assertEqual(self.quota.count_in_cart(), 1)
+        qa.queue(self.quota)
+        qa.compute()
+        self.assertEqual(qa.count_vouchers[self.quota], 0)
+        self.assertEqual(qa.count_cart[self.quota], 1)
         self.assertEqual(self.item1.check_quotas(), (Quota.AVAILABILITY_OK, 1))
 
     @classscope(attr='o')
@@ -353,8 +369,11 @@ class QuotaTestCase(BaseQuotaTestCase):
         v = Voucher.objects.create(quota=self.quota, event=self.event)
         CartPosition.objects.create(event=self.event, item=self.item1, price=2,
                                     expires=now() + timedelta(days=3), voucher=v)
-        self.assertEqual(self.quota.count_blocking_vouchers(), 0)
+        qa = QuotaAvailability(full_results=True)
-        self.assertEqual(self.quota.count_in_cart(), 1)
+        qa.queue(self.quota)
+        qa.compute()
+        self.assertEqual(qa.count_vouchers[self.quota], 0)
+        self.assertEqual(qa.count_cart[self.quota], 1)
         self.assertEqual(self.item1.check_quotas(), (Quota.AVAILABILITY_OK, 1))
 
     @classscope(attr='o')
@@ -388,6 +407,9 @@ class QuotaTestCase(BaseQuotaTestCase):
         WaitingListEntry.objects.create(
             event=self.event, item=self.item2, variation=self.var1, email='foo@bar.com', voucher=v
         )
+        qa = QuotaAvailability()
+        qa.queue(self.quota)
+        qa.compute()
         self.assertEqual(self.var1.check_quotas(), (Quota.AVAILABILITY_OK, 1))
         self.assertEqual(self.var1.check_quotas(count_waitinglist=False), (Quota.AVAILABILITY_OK, 1))