Optimize rendering of very large calendars (#2406)

src/pretix/base/services/quotas.py

@@ -113,10 +113,8 @@ class QuotaAvailability:
         be a few minutes outdated. In this case, you may not rely on the results in the ``count_*`` properties.
         """
         now_dt = now_dt or now()
-        quotas = list(set(self._queue))
+        quota_ids_set = {q.id for q in self._queue}
-        quotas_original = list(self._queue)
+        if not quota_ids_set:
-        self._queue.clear()
-        if not quotas:
             return
 
         if allow_cache:
@@ -129,7 +127,7 @@ class QuotaAvailability:
             elif settings.HAS_REDIS:
                 rc = get_redis_connection("redis")
                 quotas_by_event = defaultdict(list)
-                for q in quotas_original:
+                for q in [_q for _q in self._queue if _q.id in quota_ids_set]:
                     quotas_by_event[q.event_id].append(q)
 
                 for eventid, evquotas in quotas_by_event.items():
@@ -139,16 +137,19 @@ class QuotaAvailability:
                             data = [rv for rv in redisval.decode().split(',')]
                             # Except for some rare situations, we don't want to use cache entries older than 2 minutes
                             if time.time() - int(data[2]) < 120 or allow_cache_stale:
-                                quotas_original.remove(q)
+                                quota_ids_set.remove(q.id)
-                                quotas.remove(q)
                                 if data[1] == "None":
                                     self.results[q] = int(data[0]), None
                                 else:
                                     self.results[q] = int(data[0]), int(data[1])
 
-        if not quotas:
+        if not quota_ids_set:
             return
 
+        quotas = [_q for _q in self._queue if _q.id in quota_ids_set]
+        quotas_original = list(quotas)
+        self._queue.clear()
+
         self._compute(quotas, now_dt)
 
         for q in quotas_original:
@@ -284,15 +285,16 @@ class QuotaAvailability:
         seq = Q(subevent_id__in=subevents)
         if None in subevents:
             seq |= Q(subevent__isnull=True)
+        quota_ids = {q.pk for q in quotas}
         op_lookup = OrderPosition.objects.filter(
             order__status__in=[Order.STATUS_PAID, Order.STATUS_PENDING],
             order__event_id__in=events,
         ).filter(seq).filter(
             Q(
                 Q(variation_id__isnull=True) &
-                Q(item_id__in={i['item_id'] for i in q_items if self._quota_objects[i['quota_id']] in quotas})
+                Q(item_id__in={i['item_id'] for i in q_items if i['quota_id'] in quota_ids})
             ) | Q(
-                variation_id__in={i['itemvariation_id'] for i in q_vars if self._quota_objects[i['quota_id']] in quotas})
+                variation_id__in={i['itemvariation_id'] for i in q_vars if i['quota_id'] in quota_ids})
         ).order_by()
         if any(q.release_after_exit for q in quotas):
             op_lookup = op_lookup.annotate(
@@ -359,6 +361,7 @@ class QuotaAvailability:
             func = 'GREATEST'
 
         subevents = {q.subevent_id for q in quotas}
+        quota_ids = {q.pk for q in quotas}
         seq = Q(subevent_id__in=subevents)
         if None in subevents:
             seq |= Q(subevent__isnull=True)
@@ -370,10 +373,9 @@ class QuotaAvailability:
             Q(
                 Q(
                     Q(variation_id__isnull=True) &
-                    Q(item_id__in={i['item_id'] for i in q_items if self._quota_objects[i['quota_id']] in quotas})
+                    Q(item_id__in={i['item_id'] for i in q_items if i['quota_id'] in quota_ids})
                 ) | Q(
-                    variation_id__in={i['itemvariation_id'] for i in q_vars if
+                    variation_id__in={i['itemvariation_id'] for i in q_vars if i['quota_id'] in quota_ids}
-                                      self._quota_objects[i['quota_id']] in quotas}
                 ) | Q(
                     quota_id__in=[q.pk for q in quotas]
                 )
@@ -398,6 +400,7 @@ class QuotaAvailability:
     def _compute_carts(self, quotas, q_items, q_vars, size_left, now_dt):
         events = {q.event_id for q in quotas}
         subevents = {q.subevent_id for q in quotas}
+        quota_ids = {q.pk for q in quotas}
         seq = Q(subevent_id__in=subevents)
         if None in subevents:
             seq |= Q(subevent__isnull=True)
@@ -413,9 +416,9 @@ class QuotaAvailability:
             Q(
                 Q(
                     Q(variation_id__isnull=True) &
-                    Q(item_id__in={i['item_id'] for i in q_items if self._quota_objects[i['quota_id']] in quotas})
+                    Q(item_id__in={i['item_id'] for i in q_items if i['quota_id'] in quota_ids})
                 ) | Q(
-                    variation_id__in={i['itemvariation_id'] for i in q_vars if self._quota_objects[i['quota_id']] in quotas}
+                    variation_id__in={i['itemvariation_id'] for i in q_vars if i['quota_id'] in quota_ids}
                 )
             )
         ).order_by().values('item_id', 'subevent_id', 'variation_id').annotate(c=Count('*'))
@@ -434,6 +437,7 @@ class QuotaAvailability:
     def _compute_waitinglist(self, quotas, q_items, q_vars, size_left):
         events = {q.event_id for q in quotas}
         subevents = {q.subevent_id for q in quotas}
+        quota_ids = {q.pk for q in quotas}
         seq = Q(subevent_id__in=subevents)
         if None in subevents:
             seq |= Q(subevent__isnull=True)
@@ -444,9 +448,8 @@ class QuotaAvailability:
             Q(
                 Q(
                     Q(variation_id__isnull=True) &
-                    Q(item_id__in={i['item_id'] for i in q_items if self._quota_objects[i['quota_id']] in quotas})
+                    Q(item_id__in={i['item_id'] for i in q_items if i['quota_id'] in quota_ids})
-                ) | Q(variation_id__in={i['itemvariation_id'] for i in q_vars if
+                ) | Q(variation_id__in={i['itemvariation_id'] for i in q_vars if i['quota_id'] in quota_ids})
-                                        self._quota_objects[i['quota_id']] in quotas})
             )
         ).order_by().values('item_id', 'subevent_id', 'variation_id').annotate(c=Count('*'))
         for line in w_lookup:

src/pretix/helpers/daterange.py

@@ -32,10 +32,11 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under the License.
 
-from django.template.defaultfilters import date as _date
 from django.utils.html import format_html
 from django.utils.translation import get_language, gettext_lazy as _
 
+from pretix.helpers.templatetags.date_fast import date_fast as _date
+
 
 def daterange(df, dt, as_html=False):
     lng = get_language()

src/pretix/helpers/templatetags/date_fast.py

@@ -0,0 +1,61 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020 Raphael Michel and contributors
+# Copyright (C) 2020-2021 rami.io GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+import functools
+
+from django import template
+from django.utils import dateformat
+from django.utils.formats import get_format
+from django.utils.translation import get_language
+
+register = template.Library()
+
+
+@functools.lru_cache(maxsize=32)
+def _get_format(format_type, lang):
+    return get_format(format_type, lang)
+
+
+@register.filter(expects_localtime=True, is_safe=False)
+def date_fast(value, arg=None):
+    """
+    Slightly quicker version of |date if the filter is called a lot. The speedup is achieved through
+    LRU caching for formats.
+
+    Django's built-in |date filter has a caching mechanism if you call it with a named format,
+    i.e. ``|date_fast:"SHORT_DATE_FORMAT"`` will only be ~6% faster than ``|date:"SHORT_DATE_FORMAT"``.
+
+    However, Django's built-in caching has a flaw with unnamed formats, therefore ``|date_fast:"Y-m-d"``
+    will be ~12% faster than ``|date:"Y-m-d"``.
+    """
+    if value in (None, ''):
+        return ''
+
+    lang = get_language()
+    format = _get_format(arg, lang)
+
+    try:
+        return dateformat.format(value, format)
+    except AttributeError:
+        try:
+            return format(value, arg)
+        except AttributeError:
+            return ''

src/pretix/presale/templates/pretixpresale/fragment_calendar.html

@@ -1,16 +1,17 @@
 {% load i18n %}
+{% load date_fast %}
 <div class="table-responsive">
     <table class="table table-calendar">
         <caption class="sr-only">{% trans "Calendar" %}</caption>
         <thead>
         <tr>
-            <th><span aria-hidden="true">{{ weeks.1.0.date|date:"D" }}</span><span class="sr-only">{{ weeks.1.0.date|date:"l" }}</span></th>
+            <th><span aria-hidden="true">{{ weeks.1.0.date|date_fast:"D" }}</span><span class="sr-only">{{ weeks.1.0.date|date_fast:"l" }}</span></th>
-            <th><span aria-hidden="true">{{ weeks.1.1.date|date:"D" }}</span><span class="sr-only">{{ weeks.1.1.date|date:"l" }}</span></th>
+            <th><span aria-hidden="true">{{ weeks.1.1.date|date_fast:"D" }}</span><span class="sr-only">{{ weeks.1.1.date|date_fast:"l" }}</span></th>
-            <th><span aria-hidden="true">{{ weeks.1.2.date|date:"D" }}</span><span class="sr-only">{{ weeks.1.2.date|date:"l" }}</span></th>
+            <th><span aria-hidden="true">{{ weeks.1.2.date|date_fast:"D" }}</span><span class="sr-only">{{ weeks.1.2.date|date_fast:"l" }}</span></th>
-            <th><span aria-hidden="true">{{ weeks.1.3.date|date:"D" }}</span><span class="sr-only">{{ weeks.1.3.date|date:"l" }}</span></th>
+            <th><span aria-hidden="true">{{ weeks.1.3.date|date_fast:"D" }}</span><span class="sr-only">{{ weeks.1.3.date|date_fast:"l" }}</span></th>
-            <th><span aria-hidden="true">{{ weeks.1.4.date|date:"D" }}</span><span class="sr-only">{{ weeks.1.4.date|date:"l" }}</span></th>
+            <th><span aria-hidden="true">{{ weeks.1.4.date|date_fast:"D" }}</span><span class="sr-only">{{ weeks.1.4.date|date_fast:"l" }}</span></th>
-            <th><span aria-hidden="true">{{ weeks.1.5.date|date:"D" }}</span><span class="sr-only">{{ weeks.1.5.date|date:"l" }}</span></th>
+            <th><span aria-hidden="true">{{ weeks.1.5.date|date_fast:"D" }}</span><span class="sr-only">{{ weeks.1.5.date|date_fast:"l" }}</span></th>
-            <th><span aria-hidden="true">{{ weeks.1.6.date|date:"D" }}</span><span class="sr-only">{{ weeks.1.6.date|date:"l" }}</span></th>
+            <th><span aria-hidden="true">{{ weeks.1.6.date|date_fast:"D" }}</span><span class="sr-only">{{ weeks.1.6.date|date_fast:"l" }}</span></th>
         </tr>
         </thead>
         <tbody>
@@ -19,8 +20,8 @@
                 {% for day in week %}
                     {% if day %}
                         <td class="day {% if day.events %}has-events{% else %}no-events{% endif %}"
-                            data-date="{{ day.date|date:"SHORT_DATE_FORMAT" }}">
+                            data-date="{{ day.date|date_fast:"SHORT_DATE_FORMAT" }}">
-                            <p><time datetime="{{ day.date|date:"Y-m-d" }}">{{ day.day }}</time></p>
+                            <p><time datetime="{{ day.date|date_fast:"Y-m-d" }}">{{ day.day }}</time></p>
                             <ul class="events">
                                 {% for event in day.events %}
                                     <li><a class="event {% if event.continued %}continued{% else %} {% spaceless %}
@@ -57,9 +58,9 @@
                                                     {% if not show_names|default_if_none:True %}
                                                         <strong>
                                                     {% endif %}
-                                                    <time datetime="{{ event.time|date:"H:i" }}">{{ event.time|date:"TIME_FORMAT" }}</time>
+                                                    <time datetime="{{ event.time|date_fast:"H:i" }}">{{ event.time|date_fast:"TIME_FORMAT" }}</time>
                                                     {% if event.event.settings.show_date_to and event.time_end %}
-                                                        <span aria-hidden="true">–</span><span class="sr-only">{% trans "until" context "timerange" %}</span> <time datetime="{{ event.time_end|date:"H:i" }}">{{ event.time_end|date:"TIME_FORMAT" }}</time>
+                                                        <span aria-hidden="true">–</span><span class="sr-only">{% trans "until" context "timerange" %}</span> <time datetime="{{ event.time_end|date_fast:"H:i" }}">{{ event.time_end|date_fast:"TIME_FORMAT" }}</time>
                                                     {% endif %}
                                                     {% if not show_names|default_if_none:True %}
                                                         </strong>
@@ -90,7 +91,7 @@
                                                     <span class="fa fa-ticket" aria-hidden="true"></span> {% trans "Sale over" %}
                                                 {% elif event.event.settings.presale_start_show_date and event.event.presale_start %}
                                                     <span class="fa fa-ticket" aria-hidden="true"></span>
-                                                    {% with date_human=event.event.presale_start|date:"SHORT_DATE_FORMAT" date_iso=event.event.presale_start|date:"c"  %}
+                                                    {% with date_human=event.event.presale_start|date_fast:"SHORT_DATE_FORMAT" date_iso=event.event.presale_start|date_fast:"c"  %}
                                                     {% blocktrans with start_date="<time datetime='"|add:date_iso|add:"'>"|add:date_human|add:"</time>"|safe %}
                                                         from {{ start_date }}
                                                     {% endblocktrans %}

src/pretix/presale/templates/pretixpresale/fragment_week_calendar.html

@@ -1,10 +1,11 @@
 {% load i18n %}
+{% load date_fast %}
 <div class="week-calendar">
     {% for day in days %}
         <details class="weekday {% if day.events %}has-events{% else %}no-events{% endif %} {% if day.today %}today{% endif %}"
-                data-date="{{ day.date|date:"SHORT_DATE_FORMAT" }}" open>
+                data-date="{{ day.date|date_fast:"SHORT_DATE_FORMAT" }}" open>
             <summary>
-                <h3><span class="fa fa-fw" aria-hidden="true"></span> <time datetime="{{ day.date|date:"Y-m-d" }}">{{ day.day_formatted }}</time></h3>
+                <h3><span class="fa fa-fw" aria-hidden="true"></span> <time datetime="{{ day.date|date_fast:"Y-m-d" }}">{{ day.day_formatted }}</time></h3>
             </summary>
             <ul class="events">
                 {% for event in day.events %}
@@ -42,9 +43,9 @@
                                     {% if not show_names|default_if_none:True %}
                                         <strong>
                                     {% endif %}
-                                    <time datetime="{{ event.time|date:"H:i" }}">{{ event.time|date:"TIME_FORMAT" }}</time>
+                                    <time datetime="{{ event.time|date_fast:"H:i" }}">{{ event.time|date_fast:"TIME_FORMAT" }}</time>
                                     {% if event.time_end %}
-                                        <span aria-hidden="true">–</span><span class="sr-only">{% trans "until" context "timerange" %}</span> <time datetime="{{ event.time_end|date:"H:i" }}">{{ event.time_end|date:"TIME_FORMAT" }}</time>
+                                        <span aria-hidden="true">–</span><span class="sr-only">{% trans "until" context "timerange" %}</span> <time datetime="{{ event.time_end|date_fast:"H:i" }}">{{ event.time_end|date_fast:"TIME_FORMAT" }}</time>
                                     {% endif %}
                                     {% if not show_names|default_if_none:True %}
                                         </strong>
@@ -75,7 +76,7 @@
                                     <span class="fa fa-ticket" aria-hidden="true"></span> {% trans "Sale over" %}
                                 {% elif event.event.settings.presale_start_show_date and event.event.presale_start %}
                                     <span class="fa fa-ticket" aria-hidden="true"></span>
-                                    {% blocktrans with start_date=event.event.presale_start|date:"SHORT_DATE_FORMAT" %}
+                                    {% blocktrans with start_date=event.event.presale_start|date_fast:"SHORT_DATE_FORMAT" %}
                                         from {{ start_date }}
                                     {% endblocktrans %}
                                 {% else %}

src/pretix/presale/views/organizer.py

@@ -44,7 +44,7 @@ import isoweek
 import pytz
 from django.conf import settings
 from django.core.cache import caches
-from django.db.models import Exists, Max, Min, OuterRef, Q
+from django.db.models import Exists, Max, Min, OuterRef, Prefetch, Q
 from django.db.models.functions import Coalesce, Greatest
 from django.http import Http404, HttpResponse
 from django.shortcuts import redirect
@@ -58,7 +58,7 @@ from pytz import UTC
 
 from pretix.base.i18n import language
 from pretix.base.models import (
-    Event, EventMetaValue, Quota, SubEvent, SubEventMetaValue,
+    Event, EventMetaValue, Organizer, Quota, SubEvent, SubEventMetaValue,
 )
 from pretix.base.services.quotas import QuotaAvailability
 from pretix.helpers.compat import date_fromisocalendar
@@ -416,7 +416,11 @@ def add_events_for_days(request, baseqs, before, after, ebd, timezones):
     ).order_by(
         'date_from'
     ).prefetch_related(
-        '_settings_objects', 'organizer___settings_objects'
+        '_settings_objects',
+        Prefetch(
+            'organizer',
+            queryset=Organizer.objects.prefetch_related('_settings_objects')
+        )
     )
     if hasattr(request, 'organizer'):
         qs = filter_qs_by_attr(qs, request)
@@ -553,7 +557,7 @@ def add_subevents_for_days(qs, before, after, ebd, timezones, event=None, cart_n
 
 
 def sort_ev(e):
-    return e['time'] or time(0, 0, 0), str(e['event'])
+    return e['time'] or time(0, 0, 0), str(e['event'].name)
 
 
 def days_for_template(ebd, week):
@@ -655,7 +659,16 @@ class CalendarView(OrganizerViewMixin, EventListMixin, TemplateView):
             event__live=True,
             event__sales_channels__contains=self.request.sales_channel.identifier
         ).prefetch_related(
-            'event___settings_objects', 'event__organizer___settings_objects'
+            Prefetch(
+                'event',
+                queryset=Event.objects.prefetch_related(
+                    '_settings_objects',
+                    Prefetch(
+                        'organizer',
+                        queryset=Organizer.objects.prefetch_related('_settings_objects')
+                    )
+                )
+            )
         )), self.request).using(settings.DATABASE_REPLICA), before, after, ebd, timezones)
         self._multiple_timezones = len(timezones) > 1
         return ebd
@@ -739,7 +752,16 @@ class WeekCalendarView(OrganizerViewMixin, EventListMixin, TemplateView):
             event__live=True,
             event__sales_channels__contains=self.request.sales_channel.identifier
         ).prefetch_related(
-            'event___settings_objects', 'event__organizer___settings_objects'
+            Prefetch(
+                'event',
+                queryset=Event.objects.prefetch_related(
+                    '_settings_objects',
+                    Prefetch(
+                        'organizer',
+                        queryset=Organizer.objects.prefetch_related('_settings_objects')
+                    )
+                )
+            )
         )), self.request).using(settings.DATABASE_REPLICA), before, after, ebd, timezones)
         self._multiple_timezones = len(timezones) > 1
         return ebd
@@ -1068,7 +1090,16 @@ class DayCalendarView(OrganizerViewMixin, EventListMixin, TemplateView):
             event__live=True,
             event__sales_channels__contains=self.request.sales_channel.identifier
         ).prefetch_related(
-            'event___settings_objects', 'event__organizer___settings_objects'
+            Prefetch(
+                'event',
+                queryset=Event.objects.prefetch_related(
+                    '_settings_objects',
+                    Prefetch(
+                        'organizer',
+                        queryset=Organizer.objects.prefetch_related('_settings_objects')
+                    )
+                )
+            )
         )), self.request).using(settings.DATABASE_REPLICA), before, after, ebd, timezones)
         self._multiple_timezones = len(timezones) > 1
         return ebd
@@ -1089,7 +1120,11 @@ class OrganizerIcalDownload(OrganizerViewMixin, View):
             ).order_by(
                 'date_from'
             ).prefetch_related(
-                '_settings_objects', 'organizer___settings_objects'
+                '_settings_objects',
+                Prefetch(
+                    'organizer',
+                    queryset=Organizer.objects.prefetch_related('_settings_objects')
+                )
             )
         )
         events += list(
@@ -1104,7 +1139,16 @@ class OrganizerIcalDownload(OrganizerViewMixin, View):
                 ),
                 request
             ).prefetch_related(
-                'event___settings_objects', 'event__organizer___settings_objects'
+                Prefetch(
+                    'event',
+                    queryset=Event.objects.prefetch_related(
+                        '_settings_objects',
+                        Prefetch(
+                            'organizer',
+                            queryset=Organizer.objects.prefetch_related('_settings_objects')
+                        )
+                    )
+                )
             ).order_by(
                 'date_from'
             )