From 5308099d8415ae8266783c36a60f091be8dec0e5 Mon Sep 17 00:00:00 2001
From: Raphael Michel <mail@raphaelmichel.de>
Date: Sun, 13 Dec 2020 15:50:02 +0100
Subject: [PATCH 1/4] Fix 5-second quota caching

---
 src/pretix/base/services/quotas.py |  7 ++++---
 src/pretix/presale/views/event.py  | 14 ++++++++------
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/pretix/base/services/quotas.py b/src/pretix/base/services/quotas.py
index bec49ba808..d9f6fc37c8 100644
--- a/src/pretix/base/services/quotas.py
+++ b/src/pretix/base/services/quotas.py
@@ -113,10 +113,11 @@ class QuotaAvailability:
                 raise e
 
     def _write_cache(self, quotas, now_dt):
-        events = {q.event for q in quotas}
+        # We used to also delete item_quota_cache:* from the event cache here, but as the cache
+        # gets more complex, this does not seem worth it. The cache is only present for up to
+        # 5 seconds to prevent high peaks, and a 5-second delay in availability is usually
+        # tolerable
         update = []
-        for e in events:
-            e.cache.delete('item_quota_cache')
         for q in quotas:
             rewrite_cache = self._count_waitinglist and (
                 not q.cache_is_hot(now_dt) or self.results[q][0] > q.cached_availability_state
diff --git a/src/pretix/presale/views/event.py b/src/pretix/presale/views/event.py
index 9172b81156..20100aee04 100644
--- a/src/pretix/presale/views/event.py
+++ b/src/pretix/presale/views/event.py
@@ -63,6 +63,7 @@ def item_group_by_category(items):
 
 def get_grouped_items(event, subevent=None, voucher=None, channel='web', require_seat=0, base_qs=None, allow_addons=False,
                       quota_cache=None, filter_items=None, filter_categories=None):
+    base_qs_set = base_qs is not None
     base_qs = base_qs if base_qs is not None else event.items
 
     requires_seat = Exists(
@@ -139,8 +140,9 @@ def get_grouped_items(event, subevent=None, voucher=None, channel='web', require
         items = items.filter(category_id__in=[a for a in filter_categories if a.isdigit()])
 
     display_add_to_cart = False
-    external_quota_cache = quota_cache or event.cache.get('item_quota_cache')
-    quota_cache = external_quota_cache or {}
+    quota_cache_key = f'item_quota_cache:{subevent.id if subevent else 0}:{channel}:{bool(require_seat)}'
+    quota_cache = quota_cache or event.cache.get(quota_cache_key) or {}
+    quota_cache_existed = bool(quota_cache)
 
     if subevent:
         item_price_override = subevent.item_price_overrides
@@ -159,11 +161,11 @@ def get_grouped_items(event, subevent=None, voucher=None, channel='web', require
         if item.has_variations:
             for v in item.available_variations:
                 for q in v._subevent_quotas:
-                    if q not in quota_cache:
+                    if q.pk not in quota_cache:
                         quotas_to_compute.append(q)
         else:
             for q in item._subevent_quotas:
-                if q not in quota_cache:
+                if q.pk not in quota_cache:
                     quotas_to_compute.append(q)
 
     if quotas_to_compute:
@@ -306,8 +308,8 @@ def get_grouped_items(event, subevent=None, voucher=None, channel='web', require
 
             item._remove = not bool(item.available_variations)
 
-    if not external_quota_cache and not voucher and not allow_addons:
-        event.cache.set('item_quota_cache', quota_cache, 5)
+    if not quota_cache_existed and not voucher and not allow_addons and not base_qs_set and not filter_items and not filter_categories:
+        event.cache.set(quota_cache_key, quota_cache, 5)
     items = [item for item in items
              if (len(item.available_variations) > 0 or not item.has_variations) and not item._remove]
     return items, display_add_to_cart

From 571fef4ed8d5d1414299a2e74272953e69ed49e7 Mon Sep 17 00:00:00 2001
From: Raphael Michel <mail@raphaelmichel.de>
Date: Sun, 13 Dec 2020 16:29:17 +0100
Subject: [PATCH 2/4] Re-structure some querying on cart and order pages to
 reduce load

---
 src/pretix/base/models/orders.py              |  4 +-
 src/pretix/presale/checkoutflow.py            |  2 +-
 .../templates/pretixpresale/event/order.html  |  6 +--
 src/pretix/presale/views/__init__.py          | 45 ++++++++++++++++---
 src/pretix/presale/views/event.py             |  2 +-
 src/pretix/presale/views/order.py             |  2 +
 src/pretix/presale/views/questions.py         | 26 +----------
 7 files changed, 48 insertions(+), 39 deletions(-)

diff --git a/src/pretix/base/models/orders.py b/src/pretix/base/models/orders.py
index caf9d49026..eef3e8e034 100644
--- a/src/pretix/base/models/orders.py
+++ b/src/pretix/base/models/orders.py
@@ -902,7 +902,7 @@ class Order(LockModel, LoggedModel):
 
     @property
     def positions_with_tickets(self):
-        for op in self.positions.all():
+        for op in self.positions.select_related('item'):
             if not op.generate_ticket:
                 continue
             yield op
@@ -1155,7 +1155,7 @@ class AbstractPosition(models.Model):
         (2) questions: a list of Question objects, extended by an 'answer' property
         """
         self.answ = {}
-        for a in self.answers.all():
+        for a in getattr(self, 'answerlist', self.answers.all()):  # use prefetch_related cache from get_cart
             self.answ[a.question_id] = a
 
         # We need to clone our question objects, otherwise we will override the cached
diff --git a/src/pretix/presale/checkoutflow.py b/src/pretix/presale/checkoutflow.py
index 4163f0c6e3..98a5590482 100644
--- a/src/pretix/presale/checkoutflow.py
+++ b/src/pretix/presale/checkoutflow.py
@@ -673,7 +673,7 @@ class QuestionsStep(QuestionsViewMixin, CartMixin, TemplateFlowStep):
         return ctx
 
 
-class PaymentStep(QuestionsViewMixin, CartMixin, TemplateFlowStep):
+class PaymentStep(CartMixin, TemplateFlowStep):
     priority = 200
     identifier = "payment"
     template_name = "pretixpresale/event/checkout_payment.html"
diff --git a/src/pretix/presale/templates/pretixpresale/event/order.html b/src/pretix/presale/templates/pretixpresale/event/order.html
index 85a21a606e..f70746370e 100644
--- a/src/pretix/presale/templates/pretixpresale/event/order.html
+++ b/src/pretix/presale/templates/pretixpresale/event/order.html
@@ -313,7 +313,7 @@
         {% endif %}
         <div class="clearfix"></div>
     </div>
-    {% if order.user_change_allowed or order.user_cancel_allowed %}
+    {% if user_change_allowed or user_cancel_allowed %}
         <div class="panel panel-primary panel-cancellation">
             <div class="panel-heading">
                 <h3 class="panel-title">
@@ -321,7 +321,7 @@
                 </h3>
             </div>
             <ul class="list-group">
-                {% if order.user_change_allowed %}
+                {% if user_change_allowed %}
                     <li class="list-group-item">
                         <p>
                             {% blocktrans trimmed %}
@@ -335,7 +335,7 @@
                         </a>
                     </li>
                 {% endif %}
-                {% if order.user_cancel_allowed %}
+                {% if user_cancel_allowed %}
                     <li class="list-group-item">
                         {% if order.status == "p" and order.total != 0 %}
                             {% if order.user_cancel_fee >= order.total %}
diff --git a/src/pretix/presale/views/__init__.py b/src/pretix/presale/views/__init__.py
index d554be65d8..149e6a8111 100644
--- a/src/pretix/presale/views/__init__.py
+++ b/src/pretix/presale/views/__init__.py
@@ -5,14 +5,14 @@ from functools import wraps
 from itertools import groupby
 
 from django.conf import settings
-from django.db.models import Prefetch, Sum
+from django.db.models import Prefetch, Sum, Exists, OuterRef
 from django.utils.functional import cached_property
 from django.utils.timezone import now
 from django_scopes import scopes_disabled
 
 from pretix.base.i18n import language
 from pretix.base.models import (
-    CartPosition, InvoiceAddress, OrderPosition, QuestionAnswer,
+    CartPosition, InvoiceAddress, OrderPosition, QuestionAnswer, ItemAddOn, Question, QuestionOption,
 )
 from pretix.base.services.cart import get_fees
 from pretix.helpers.cookies import set_cookie_without_samesite
@@ -68,12 +68,15 @@ class CartMixin:
                 prefetch.append(Prefetch('answers', queryset=QuestionAnswer.objects.prefetch_related('options')))
 
             cartpos = queryset.order_by(
-                'item__category__position', 'item__category_id', 'item__position', 'item__name', 'variation__value'
+                'item__category__position', 'item__category_id', 'item__position', 'item__name',
+                'variation__value'
             ).select_related(
-                'item', 'variation', 'addon_to', 'subevent', 'subevent__event', 'subevent__event__organizer', 'seat'
+                'item', 'variation', 'addon_to', 'subevent', 'subevent__event',
+                'subevent__event__organizer', 'seat'
             ).prefetch_related(
                 *prefetch
             )
+
         else:
             cartpos = self.positions
 
@@ -123,7 +126,7 @@ class CartMixin:
                     or pos.pk in has_addons \
                     or pos.addon_to_id \
                     or pos.item.issue_giftcard \
-                    or (answers and (has_attendee_data or pos.item.questions.exists())):
+                    or (answers and (has_attendee_data or bool(pos.item.questions.all()))):  # do not use .exists() to re-use prefetch cache
                 return (
                     # standalone positions are grouped by main product position id, addons below them also sorted by position id
                     i, addon_penalty, pos.pk,
@@ -147,7 +150,8 @@ class CartMixin:
             group.total = group.count * group.price
             group.net_total = group.count * group.net_price
             group.has_questions = answers and k[0] != ""
-            group.tax_rule = group.item.tax_rule
+            if not hasattr(group, 'tax_rule'):
+                group.tax_rule = group.item.tax_rule
 
             group.bundle_sum = group.price + sum(a.price for a in has_addons[group.pk])
             group.bundle_sum_net = group.net_price + sum(a.net_price for a in has_addons[group.pk])
@@ -214,6 +218,8 @@ def cart_exists(request):
 
 def get_cart(request):
     from pretix.presale.views.cart import get_or_create_cart_id
+    qqs = request.event.questions.all()
+    qqs = qqs.filter(ask_during_checkin=False, hidden=False)
 
     if not hasattr(request, '_cart_cache'):
         cart_id = get_or_create_cart_id(request, create=False)
@@ -222,11 +228,36 @@ def get_cart(request):
         else:
             request._cart_cache = CartPosition.objects.filter(
                 cart_id=cart_id, event=request.event
+            ).annotate(
+                has_addon_choices=Exists(
+                    ItemAddOn.objects.filter(
+                        base_item_id=OuterRef('item_id')
+                    )
+                )
             ).order_by(
-                'item', 'variation'
+                'item__category__position', 'item__category_id', 'item__position', 'item__name', 'variation__value'
             ).select_related(
                 'item', 'variation', 'subevent', 'subevent__event', 'subevent__event__organizer',
                 'item__tax_rule', 'addon_to'
+            ).select_related(
+                'addon_to'
+            ).prefetch_related(
+                'addons', 'addons__item', 'addons__variation',
+                Prefetch('answers',
+                         QuestionAnswer.objects.prefetch_related('options'),
+                         to_attr='answerlist'),
+                Prefetch('item__questions',
+                         qqs.prefetch_related(
+                             Prefetch('options', QuestionOption.objects.prefetch_related(Prefetch(
+                                 # This prefetch statement is utter bullshit, but it actually prevents Django from doing
+                                 # a lot of queries since ModelChoiceIterator stops trying to be clever once we have
+                                 # a prefetch lookup on this query...
+                                 'question',
+                                 Question.objects.none(),
+                                 to_attr='dummy'
+                             )))
+                         ).select_related('dependency_question'),
+                         to_attr='questions_to_ask')
             )
             for cp in request._cart_cache:
                 cp.event = request.event  # Populate field with known value to save queries
diff --git a/src/pretix/presale/views/event.py b/src/pretix/presale/views/event.py
index 20100aee04..c937f3143a 100644
--- a/src/pretix/presale/views/event.py
+++ b/src/pretix/presale/views/event.py
@@ -408,7 +408,7 @@ class EventIndex(EventViewMixin, EventListMixin, CartMixin, TemplateView):
         context['ev'] = self.subevent or self.request.event
         context['subevent'] = self.subevent
         context['cart'] = self.get_cart()
-        context['has_addon_choices'] = get_cart(self.request).filter(item__addons__isnull=False).exists()
+        context['has_addon_choices'] = any(cp.has_addon_choices for cp in get_cart(self.request))\
 
         if self.subevent:
             context['frontpage_text'] = str(self.subevent.frontpage_text)
diff --git a/src/pretix/presale/views/order.py b/src/pretix/presale/views/order.py
index e3f1dd065e..f82568d75b 100644
--- a/src/pretix/presale/views/order.py
+++ b/src/pretix/presale/views/order.py
@@ -245,6 +245,8 @@ class OrderDetails(EventViewMixin, OrderDetailMixin, CartMixin, TicketPageMixin,
         ).exclude(
             provider__in=('offsetting', 'reseller', 'boxoffice', 'manual')
         )
+        ctx['user_change_allowed'] = self.order.user_change_allowed
+        ctx['user_cancel_allowed'] = self.order.user_cancel_allowed
         for r in ctx['refunds']:
             if r.provider == 'giftcard':
                 gc = GiftCard.objects.get(pk=r.info_data.get('gift_card'))
diff --git a/src/pretix/presale/views/questions.py b/src/pretix/presale/views/questions.py
index e8c9934c6c..a5ecec6a95 100644
--- a/src/pretix/presale/views/questions.py
+++ b/src/pretix/presale/views/questions.py
@@ -1,7 +1,5 @@
-from django.db.models import Prefetch
 from django.utils.functional import cached_property
 
-from pretix.base.models import Question, QuestionAnswer, QuestionOption
 from pretix.base.views.mixins import BaseQuestionsViewMixin
 from pretix.presale.forms.checkout import QuestionsForm
 from pretix.presale.views import get_cart
@@ -13,27 +11,5 @@ class QuestionsViewMixin(BaseQuestionsViewMixin):
 
     @cached_property
     def _positions_for_questions(self):
-        qqs = self.request.event.questions.all()
-        if self.only_user_visible:
-            qqs = qqs.filter(ask_during_checkin=False, hidden=False)
-        cart = get_cart(self.request).select_related(
-            'addon_to'
-        ).prefetch_related(
-            'addons', 'addons__item', 'addons__variation',
-            Prefetch('answers',
-                     QuestionAnswer.objects.prefetch_related('options'),
-                     to_attr='answerlist'),
-            Prefetch('item__questions',
-                     qqs.prefetch_related(
-                         Prefetch('options', QuestionOption.objects.prefetch_related(Prefetch(
-                             # This prefetch statement is utter bullshit, but it actually prevents Django from doing
-                             # a lot of queries since ModelChoiceIterator stops trying to be clever once we have
-                             # a prefetch lookup on this query...
-                             'question',
-                             Question.objects.none(),
-                             to_attr='dummy'
-                         )))
-                     ).select_related('dependency_question'),
-                     to_attr='questions_to_ask')
-        )
+        cart = get_cart(self.request)
         return sorted(list(cart), key=self._keyfunc)

From 62b1aec3b034fbd6bd2af4a793b64b4aefadfe9e Mon Sep 17 00:00:00 2001
From: Raphael Michel <mail@raphaelmichel.de>
Date: Sun, 13 Dec 2020 15:55:36 +0100
Subject: [PATCH 3/4] Save a pointless query on non-series events

---
 src/pretix/presale/views/event.py | 44 ++++++++++++++++++-------------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/src/pretix/presale/views/event.py b/src/pretix/presale/views/event.py
index c937f3143a..f72874d120 100644
--- a/src/pretix/presale/views/event.py
+++ b/src/pretix/presale/views/event.py
@@ -415,13 +415,33 @@ class EventIndex(EventViewMixin, EventListMixin, CartMixin, TemplateView):
         else:
             context['frontpage_text'] = str(self.request.event.settings.frontpage_text)
 
+        if self.request.event.has_subevents:
+            context.update(self._subevent_list_context())
+
+        context['show_cart'] = (
+            context['cart']['positions'] and (
+                self.request.event.has_subevents or self.request.event.presale_is_running
+            )
+        )
+        if self.request.event.settings.redirect_to_checkout_directly:
+            context['cart_redirect'] = eventreverse(self.request.event, 'presale:event.checkout.start',
+                                                    kwargs={'cart_namespace': kwargs.get('cart_namespace') or ''})
+            if context['cart_redirect'].startswith('https:'):
+                context['cart_redirect'] = '/' + context['cart_redirect'].split('/', 3)[3]
+        else:
+            context['cart_redirect'] = self.request.path
+
+        return context
+
+    def _subevent_list_context(self):
+        context = {}
         context['list_type'] = self.request.GET.get("style", self.request.event.settings.event_list_type)
         if context['list_type'] not in ("calendar", "week") and self.request.event.subevents.filter(date_from__gt=now()).count() > 50:
             if self.request.event.settings.event_list_type not in ("calendar", "week"):
                 self.request.event.settings.event_list_type = "calendar"
             context['list_type'] = "calendar"
 
-        if context['list_type'] == "calendar" and self.request.event.has_subevents:
+        if context['list_type'] == "calendar":
             self._set_month_year()
             tz = pytz.timezone(self.request.event.settings.timezone)
             _, ndays = calendar.monthrange(self.year, self.month)
@@ -436,7 +456,7 @@ class EventIndex(EventViewMixin, EventListMixin, CartMixin, TemplateView):
             add_subevents_for_days(
                 filter_qs_by_attr(self.request.event.subevents_annotated(self.request.sales_channel.identifier).using(settings.DATABASE_REPLICA), self.request),
                 before, after, ebd, set(), self.request.event,
-                kwargs.get('cart_namespace')
+                self.kwargs.get('cart_namespace')
             )
 
             context['show_names'] = ebd.get('_subevents_different_names', False) or sum(
@@ -445,7 +465,7 @@ class EventIndex(EventViewMixin, EventListMixin, CartMixin, TemplateView):
             context['weeks'] = weeks_for_template(ebd, self.year, self.month)
             context['months'] = [date(self.year, i + 1, 1) for i in range(12)]
             context['years'] = range(now().year - 2, now().year + 3)
-        elif context['list_type'] == "week" and self.request.event.has_subevents:
+        elif context['list_type'] == "week":
             self._set_week_year()
             tz = pytz.timezone(self.request.event.settings.timezone)
             week = isoweek.Week(self.year, self.week)
@@ -464,7 +484,7 @@ class EventIndex(EventViewMixin, EventListMixin, CartMixin, TemplateView):
             add_subevents_for_days(
                 filter_qs_by_attr(self.request.event.subevents_annotated(self.request.sales_channel.identifier).using(settings.DATABASE_REPLICA), self.request),
                 before, after, ebd, set(), self.request.event,
-                kwargs.get('cart_namespace')
+                self.kwargs.get('cart_namespace')
             )
 
             context['show_names'] = ebd.get('_subevents_different_names', False) or sum(
@@ -479,24 +499,10 @@ class EventIndex(EventViewMixin, EventListMixin, CartMixin, TemplateView):
             context['week_format'] = get_format('WEEK_FORMAT')
             if context['week_format'] == 'WEEK_FORMAT':
                 context['week_format'] = WEEK_FORMAT
-        elif self.request.event.has_subevents:
+        else:
             context['subevent_list'] = self.request.event.subevents_sorted(
                 filter_qs_by_attr(self.request.event.subevents_annotated(self.request.sales_channel.identifier).using(settings.DATABASE_REPLICA), self.request)
             )
-
-        context['show_cart'] = (
-            context['cart']['positions'] and (
-                self.request.event.has_subevents or self.request.event.presale_is_running
-            )
-        )
-        if self.request.event.settings.redirect_to_checkout_directly:
-            context['cart_redirect'] = eventreverse(self.request.event, 'presale:event.checkout.start',
-                                                    kwargs={'cart_namespace': kwargs.get('cart_namespace') or ''})
-            if context['cart_redirect'].startswith('https:'):
-                context['cart_redirect'] = '/' + context['cart_redirect'].split('/', 3)[3]
-        else:
-            context['cart_redirect'] = self.request.path
-
         return context
 
 

From 1f21d1420ca6f003fc32a409c924fdf7763fcfb5 Mon Sep 17 00:00:00 2001
From: Raphael Michel <mail@raphaelmichel.de>
Date: Mon, 14 Dec 2020 11:45:23 +0100
Subject: [PATCH 4/4] Fix import order

---
 src/pretix/control/forms/event.py    | 4 +---
 src/pretix/presale/views/__init__.py | 5 +++--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/pretix/control/forms/event.py b/src/pretix/control/forms/event.py
index c630cdfe7d..050c140eec 100644
--- a/src/pretix/control/forms/event.py
+++ b/src/pretix/control/forms/event.py
@@ -19,9 +19,7 @@ from pytz import common_timezones, timezone
 
 from pretix.base.channels import get_all_sales_channels
 from pretix.base.email import get_available_placeholders
-from pretix.base.forms import (
-    I18nModelForm, PlaceholderValidator, SettingsForm,
-)
+from pretix.base.forms import I18nModelForm, PlaceholderValidator, SettingsForm
 from pretix.base.models import Event, Organizer, TaxRule, Team
 from pretix.base.models.event import EventMetaValue, SubEvent
 from pretix.base.reldate import RelativeDateField, RelativeDateTimeField
diff --git a/src/pretix/presale/views/__init__.py b/src/pretix/presale/views/__init__.py
index 149e6a8111..ec0f3e3167 100644
--- a/src/pretix/presale/views/__init__.py
+++ b/src/pretix/presale/views/__init__.py
@@ -5,14 +5,15 @@ from functools import wraps
 from itertools import groupby
 
 from django.conf import settings
-from django.db.models import Prefetch, Sum, Exists, OuterRef
+from django.db.models import Exists, OuterRef, Prefetch, Sum
 from django.utils.functional import cached_property
 from django.utils.timezone import now
 from django_scopes import scopes_disabled
 
 from pretix.base.i18n import language
 from pretix.base.models import (
-    CartPosition, InvoiceAddress, OrderPosition, QuestionAnswer, ItemAddOn, Question, QuestionOption,
+    CartPosition, InvoiceAddress, ItemAddOn, OrderPosition, Question,
+    QuestionAnswer, QuestionOption,
 )
 from pretix.base.services.cart import get_fees
 from pretix.helpers.cookies import set_cookie_without_samesite