Fix #785 -- Show availability in (sub)event list (#1112)

src/pretix/base/models/event.py

@@ -11,7 +11,7 @@ from django.core.files.storage import default_storage
 from django.core.mail import get_connection
 from django.core.validators import RegexValidator
 from django.db import models
-from django.db.models import Exists, OuterRef, Q
+from django.db.models import Exists, OuterRef, Prefetch, Q, Subquery
 from django.template.defaultfilters import date as _date
 from django.utils.crypto import get_random_string
 from django.utils.functional import cached_property
@@ -22,6 +22,7 @@ from i18nfield.fields import I18nCharField, I18nTextField
 from pretix.base.models.base import LoggedModel
 from pretix.base.reldate import RelativeDateWrapper
 from pretix.base.validators import EventSlugBlacklistValidator
+from pretix.helpers.database import GroupConcat
 from pretix.helpers.daterange import daterange
 from pretix.helpers.json import safe_string
 
@@ -159,6 +160,79 @@ class EventMixin:
 
         return safe_string(json.dumps(eventdict))
 
+    @classmethod
+    def annotated(cls, qs, channel='web'):
+        from pretix.base.models import Item, ItemVariation, Quota
+
+        sq_active_item = Item.objects.filter_available(channel=channel).filter(
+            Q(variations__isnull=True)
+            & Q(quotas__pk=OuterRef('pk'))
+        ).order_by().values_list('quotas__pk').annotate(
+            items=GroupConcat('pk', delimiter=',')
+        ).values('items')
+        sq_active_variation = ItemVariation.objects.filter(
+            Q(active=True)
+            & Q(item__active=True)
+            & Q(Q(item__available_from__isnull=True) | Q(item__available_from__lte=now()))
+            & Q(Q(item__available_until__isnull=True) | Q(item__available_until__gte=now()))
+            & Q(Q(item__category__isnull=True) | Q(item__category__is_addon=False))
+            & Q(item__sales_channels__contains=channel)
+            & Q(item__hide_without_voucher=False)  # TODO: does this make sense?
+            & Q(quotas__pk=OuterRef('pk'))
+        ).order_by().values_list('quotas__pk').annotate(
+            items=GroupConcat('pk', delimiter=',')
+        ).values('items')
+        return qs.prefetch_related(
+            Prefetch(
+                'quotas',
+                to_attr='active_quotas',
+                queryset=Quota.objects.annotate(
+                    active_items=Subquery(sq_active_item, output_field=models.TextField()),
+                    active_variations=Subquery(sq_active_variation, output_field=models.TextField()),
+                ).exclude(
+                    Q(active_items="") & Q(active_variations="")
+                )
+            )
+        )
+
+    @cached_property
+    def best_availability_state(self):
+        from .items import Quota
+
+        if not hasattr(self, 'active_quotas'):
+            raise TypeError("Call this only if you fetched the subevents via Event/SubEvent.annotated()")
+        items_available = set()
+        vars_available = set()
+        items_reserved = set()
+        vars_reserved = set()
+        items_gone = set()
+        vars_gone = set()
+        for q in self.active_quotas:
+            res = q.availability(allow_cache=True)
+
+            if res[0] == Quota.AVAILABILITY_OK:
+                if q.active_items:
+                    items_available.update(q.active_items.split(","))
+                if q.active_variations:
+                    vars_available.update(q.active_variations.split(","))
+            elif res[0] == Quota.AVAILABILITY_RESERVED:
+                if q.active_items:
+                    items_reserved.update(q.active_items.split(","))
+                if q.active_variations:
+                    vars_available.update(q.active_variations.split(","))
+            elif res[0] < Quota.AVAILABILITY_RESERVED:
+                if q.active_items:
+                    items_gone.update(q.active_items.split(","))
+                if q.active_variations:
+                    vars_gone.update(q.active_variations.split(","))
+        if not self.active_quotas:
+            return None
+        if items_available - items_reserved - items_gone or vars_available - vars_reserved - vars_gone:
+            return Quota.AVAILABILITY_OK
+        if items_reserved - items_gone or vars_reserved - vars_gone:
+            return Quota.AVAILABILITY_RESERVED
+        return Quota.AVAILABILITY_GONE
+
 
 @settings_hierarkey.add(parent_field='organizer', cache_namespace='event')
 class Event(EventMixin, LoggedModel):
@@ -572,8 +646,10 @@ class Event(EventMixin, LoggedModel):
             )
         ).order_by('date_from', 'name')
 
-    @property
-    def subevent_list_subevents(self):
+    def subevents_annotated(self, channel):
+        return SubEvent.annotated(self.subevents, channel)
+
+    def subevents_sorted(self, queryset):
         ordering = self.settings.get('frontpage_subevent_ordering', default='date_ascending', as_type=str)
         orderfields = {
             'date_ascending': ('date_from', 'name'),
@@ -581,7 +657,7 @@ class Event(EventMixin, LoggedModel):
             'name_ascending': ('name', 'date_from'),
             'name_descending': ('-name', 'date_from'),
         }[ordering]
-        subevs = self.subevents.filter(
+        subevs = queryset.filter(
             Q(active=True) & (
                 Q(Q(date_to__isnull=True) & Q(date_from__gte=now()))
                 | Q(date_to__gte=now())

src/pretix/base/models/items.py

@@ -153,6 +153,30 @@ class SubEventItemVariation(models.Model):
             self.subevent.event.cache.clear()
 
 
+class ItemQuerySet(models.QuerySet):
+    def filter_available(self, channel='web', voucher=None, allow_addons=False):
+        q = (
+            # IMPORTANT: If this is updated, also update the ItemVariation query
+            # in models/event.py: EventMixin.annotated()
+            Q(active=True)
+            & Q(Q(available_from__isnull=True) | Q(available_from__lte=now()))
+            & Q(Q(available_until__isnull=True) | Q(available_until__gte=now()))
+            & Q(sales_channels__contains=channel)
+        )
+        if not allow_addons:
+            q &= Q(Q(category__isnull=True) | Q(category__is_addon=False))
+        qs = self.filter(q)
+
+        vouchq = Q(hide_without_voucher=False)
+        if voucher:
+            if voucher.item_id:
+                vouchq |= Q(pk=voucher.item_id)
+                qs = qs.filter(pk=voucher.item_id)
+            elif voucher.quota_id:
+                qs = qs.filter(quotas__in=[voucher.quota_id])
+        return qs.filter(vouchq)
+
+
 class Item(LoggedModel):
     """
     An item is a thing which can be sold. It belongs to an event and may or may not belong to a category.
@@ -200,6 +224,8 @@ class Item(LoggedModel):
     :type sales_channels: bool
     """
 
+    objects = ItemQuerySet.as_manager()
+
     event = models.ForeignKey(
         Event,
         on_delete=models.PROTECT,
@@ -930,6 +956,16 @@ class Quota(LoggedModel):
     :type size: int
     :param items: The set of :py:class:`Item` objects this quota applies to
     :param variations: The set of :py:class:`ItemVariation` objects this quota applies to
+
+    This model keeps a cache of the quota availability that is used in places where up-to-date
+    data is not important. This cache might be out of date even though a more recent quota was
+    calculated. This is intentional to keep database writes low. Currently, the cached values
+    are written whenever the quota is being calculated throughout the system and the cache is
+    at least 120 seconds old or if the new value is qualitatively "better" than the cached one
+    (i.e. more free quota).
+
+    There's also a cronjob that refreshes the cache of every quota if there is any log entry in
+    the event that is newer than the quota's cached time.
     """
 
     AVAILABILITY_GONE = 0
@@ -1012,6 +1048,15 @@ class Quota(LoggedModel):
         This method is used to determine whether Items or ItemVariations belonging
         to this quota should currently be available for sale.
 
+        :param count_waitinglist: Whether or not take waiting list reservations into account. Defaults
+                                  to ``True``.
+        :param _cache: A dictionary mapping quota IDs to availabilities. If this quota is already
+                       contained in that dictionary, this value will be used. Otherwise, the dict
+                       will be populated accordingly.
+        :param allow_cache: Allow for values to be returned from the longer-term cache, see also
+                            the documentation of this model class. Only works if ``count_waitinglist`` is
+                            set to ``True``.
+
         :returns: a tuple where the first entry is one of the ``Quota.AVAILABILITY_`` constants
                   and the second is the number of available tickets.
         """
@@ -1027,7 +1072,10 @@ class Quota(LoggedModel):
         res = self._availability(now_dt, count_waitinglist)
 
         self.event.cache.delete('item_quota_cache')
-        if count_waitinglist and not self.cache_is_hot(now_dt):
+        rewrite_cache = count_waitinglist and (
+            not self.cache_is_hot(now_dt) or res[0] > self.cached_availability_state
+        )
+        if rewrite_cache:
             self.cached_availability_state = res[0]
             self.cached_availability_number = res[1]
             self.cached_availability_time = now_dt

src/pretix/base/services/quotas.py

@@ -1,6 +1,9 @@
+from datetime import timedelta
+
 from django.db import models
 from django.db.models import F, Max, OuterRef, Q, Subquery
 from django.dispatch import receiver
+from django.utils.timezone import now
 
 from pretix.base.models import LogEntry, Quota
 from pretix.celery_app import app
@@ -26,7 +29,8 @@ def refresh_quota_caches():
         last_activity=Subquery(last_activity, output_field=models.DateTimeField())
     ).filter(
         Q(cached_availability_time__isnull=True) |
-        Q(cached_availability_time__lt=F('last_activity'))
+        Q(cached_availability_time__lt=F('last_activity')) |
+        Q(cached_availability_time__lt=now() - timedelta(hours=2), last_activity__gt=now() - timedelta(days=7))
     )
     for q in quotas:
         q.availability()

src/pretix/base/settings.py

@@ -204,6 +204,10 @@ DEFAULTS = {
         'default': 'True',
         'type': bool
     },
+    'event_list_availability': {
+        'default': 'True',
+        'type': bool
+    },
     'event_list_type': {
         'default': 'list',
         'type': str