diff --git a/src/pretix/base/migrations/0180_auto_20210324_1309.py b/src/pretix/base/migrations/0180_auto_20210324_1309.py new file mode 100644 index 0000000000..49b722c22b --- /dev/null +++ b/src/pretix/base/migrations/0180_auto_20210324_1309.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.12 on 2021-03-24 13:09 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('pretixbase', '0179_auto_20210311_1653'), + ] + + operations = [ + migrations.RemoveField( + model_name='quota', + name='cached_availability_number', + ), + migrations.RemoveField( + model_name='quota', + name='cached_availability_paid_orders', + ), + migrations.RemoveField( + model_name='quota', + name='cached_availability_state', + ), + migrations.RemoveField( + model_name='quota', + name='cached_availability_time', + ), + ] diff --git a/src/pretix/base/models/event.py b/src/pretix/base/models/event.py index 35a94f2a72..e2680c5092 100644 --- a/src/pretix/base/models/event.py +++ b/src/pretix/base/models/event.py @@ -657,10 +657,6 @@ class Event(EventMixin, LoggedModel): oldid = q.pk q.pk = None q.event = self - q.cached_availability_state = None - q.cached_availability_number = None - q.cached_availability_paid_orders = None - q.cached_availability_time = None q.closed = False q.save() q.log_action('pretix.object.cloned') diff --git a/src/pretix/base/models/items.py b/src/pretix/base/models/items.py index dddc76bd68..83e81b7892 100644 --- a/src/pretix/base/models/items.py +++ b/src/pretix/base/models/items.py @@ -17,6 +17,7 @@ from django.utils.functional import cached_property from django.utils.timezone import is_naive, make_aware, now from django.utils.translation import gettext_lazy as _, pgettext_lazy from django_countries.fields import Country +from django_redis import get_redis_connection from django_scopes import ScopedManager from i18nfield.fields import I18nCharField, I18nTextField @@ -25,6 +26,7 @@ from pretix.base.models.base import LoggedModel from pretix.base.models.fields import MultiStringField from pretix.base.models.tax import TaxedPrice +from ... import settings from .event import Event, SubEvent @@ -1374,10 +1376,6 @@ class Quota(LoggedModel): blank=True, verbose_name=_("Variations") ) - cached_availability_state = models.PositiveIntegerField(null=True, blank=True) - cached_availability_number = models.PositiveIntegerField(null=True, blank=True) - cached_availability_paid_orders = models.PositiveIntegerField(null=True, blank=True) - cached_availability_time = models.DateTimeField(null=True, blank=True) close_when_sold_out = models.BooleanField( verbose_name=_('Close this quota permanently once it is sold out'), @@ -1422,14 +1420,10 @@ class Quota(LoggedModel): self.event.cache.clear() def rebuild_cache(self, now_dt=None): - self.cached_availability_time = None - self.cached_availability_number = None - self.cached_availability_state = None - self.availability(now_dt=now_dt) - - def cache_is_hot(self, now_dt=None): - now_dt = now_dt or now() - return self.cached_availability_time and (now_dt - self.cached_availability_time).total_seconds() < 120 + if settings.HAS_REDIS: + rc = get_redis_connection("redis") + rc.hdel(f'quotas:{self.event_id}:availabilitycache', str(self.pk)) + self.availability(now_dt=now_dt) def availability( self, now_dt: datetime=None, count_waitinglist=True, _cache=None, allow_cache=False @@ -1452,9 +1446,6 @@ class Quota(LoggedModel): """ from ..services.quotas import QuotaAvailability - if allow_cache and self.cache_is_hot() and count_waitinglist: - return self.cached_availability_state, self.cached_availability_number - if _cache and count_waitinglist is not _cache.get('_count_waitinglist', True): _cache.clear() @@ -1462,7 +1453,7 @@ class Quota(LoggedModel): return _cache[self.pk] qa = QuotaAvailability(count_waitinglist=count_waitinglist, early_out=False) qa.queue(self) - qa.compute(now_dt=now_dt) + qa.compute(now_dt=now_dt, allow_cache=allow_cache) res = qa.results[self] if _cache is not None: diff --git a/src/pretix/base/services/quotas.py b/src/pretix/base/services/quotas.py index d9f6fc37c8..8b5459333a 100644 --- a/src/pretix/base/services/quotas.py +++ b/src/pretix/base/services/quotas.py @@ -1,25 +1,22 @@ import sys +import time from collections import Counter, defaultdict -from datetime import timedelta from itertools import zip_longest from django.conf import settings -from django.db import OperationalError, models +from django.db import models from django.db.models import ( Case, Count, F, Func, Max, OuterRef, Q, Subquery, Sum, Value, When, ) -from django.dispatch import receiver from django.utils.timezone import now -from django_scopes import scopes_disabled +from django_redis import get_redis_connection from pretix.base.models import ( - CartPosition, Checkin, Event, LogEntry, Order, OrderPosition, Quota, - Voucher, WaitingListEntry, + CartPosition, Checkin, Order, OrderPosition, Quota, Voucher, + WaitingListEntry, ) -from pretix.celery_app import app -from ...helpers.periodic import minimum_interval -from ..signals import periodic_task, quota_availability +from ..signals import quota_availability class QuotaAvailability: @@ -89,7 +86,11 @@ class QuotaAvailability: def queue(self, *quota): self._queue += quota - def compute(self, now_dt=None): + def compute(self, now_dt=None, allow_cache=False, allow_cache_stale=False): + """ + Compute the queued quotas. If ``allow_cache`` is set, results may also be taken from a cache that might + be a few minutes outdated. In this case, you may not rely on the results in the ``count_*`` properties. + """ now_dt = now_dt or now() quotas = list(set(self._queue)) quotas_original = list(self._queue) @@ -97,6 +98,35 @@ class QuotaAvailability: if not quotas: return + if allow_cache: + if self._full_results: + raise ValueError("You cannot combine full_results and allow_cache.") + + elif not self._count_waitinglist: + raise ValueError("If you set allow_cache, you need to set count_waitinglist.") + + elif settings.HAS_REDIS: + rc = get_redis_connection("redis") + quotas_by_event = defaultdict(list) + for q in quotas_original: + quotas_by_event[q.event_id].append(q) + + for eventid, evquotas in quotas_by_event.items(): + d = rc.hmget(f'quotas:{eventid}:availabilitycache', [str(q.pk) for q in evquotas]) + for redisval, q in zip(d, evquotas): + if redisval is not None: + data = [rv for rv in redisval.decode().split(',')] + if time.time() - int(data[2]) < 120 or allow_cache_stale: + quotas_original.remove(q) + quotas.remove(q) + if data[1] == "None": + self.results[q] = int(data[0]), None, int(data[2]) + else: + self.results[q] = int(data[0]), int(data[1]), int(data[2]) + + if not quotas: + return + self._compute(quotas, now_dt) for q in quotas_original: @@ -105,36 +135,48 @@ class QuotaAvailability: self.results[q] = resp self._close(quotas) - try: - self._write_cache(quotas, now_dt) - except OperationalError as e: - # Ignore deadlocks when multiple threads try to write to the cache - if 'deadlock' not in str(e).lower(): - raise e + self._write_cache(quotas, now_dt) def _write_cache(self, quotas, now_dt): + if not settings.HAS_REDIS or not quotas: + return + + rc = get_redis_connection("redis") + # We write the computed availability to redis in a per-event hash as + # + # quota_id -> (availability_state, availability_number, timestamp). + # + # We store this in a hash instead of inidividual values to avoid making two many redis requests + # which would introduce latency. + + # The individual entries in the hash are "valid" for 120 seconds. This means in a typical peak scenario with + # high load *to a specific calendar or event*, lots of parallel web requests will receive an "expired" result + # around the same time, recompute quotas and write back to the cache. To avoid overloading redis with lots of + # simultaneous write queries for the same page, we place a very naive and simple "lock" on the write process for + # these quotas. + + lock_name = '_'.join([str(p) for p in sorted([q.pk for q in quotas])]) + if rc.exists(f'quotas:availabilitycachewrite:{lock_name}'): + return + rc.setex(f'quotas:availabilitycachewrite:{lock_name}', '1', 10) + + update = defaultdict(list) + for q in quotas: + update[q.event_id].append(q) + + for eventid, quotas in update.items(): + rc.hmset(f'quotas:{eventid}:availabilitycache', { + str(q.id): ",".join( + [str(i) for i in self.results[q]] + + [str(int(time.time()))] + ) for q in quotas + }) + rc.expire(f'quotas:{eventid}:availabilitycache', 3600 * 24 * 7) + # We used to also delete item_quota_cache:* from the event cache here, but as the cache # gets more complex, this does not seem worth it. The cache is only present for up to # 5 seconds to prevent high peaks, and a 5-second delay in availability is usually # tolerable - update = [] - for q in quotas: - rewrite_cache = self._count_waitinglist and ( - not q.cache_is_hot(now_dt) or self.results[q][0] > q.cached_availability_state - or q.cached_availability_paid_orders is None - ) - if rewrite_cache: - q.cached_availability_state = self.results[q][0] - q.cached_availability_number = self.results[q][1] - q.cached_availability_time = now_dt - if q in self.count_paid_orders: - q.cached_availability_paid_orders = self.count_paid_orders[q] - update.append(q) - if update: - Quota.objects.using('default').bulk_update(update, [ - 'cached_availability_state', 'cached_availability_number', 'cached_availability_time', - 'cached_availability_paid_orders' - ], batch_size=50) def _close(self, quotas): for q in quotas: @@ -404,44 +446,8 @@ class QuotaAvailability: self.results[q] = Quota.AVAILABILITY_GONE, 0 -@receiver(signal=periodic_task) -@minimum_interval(minutes_after_success=60) -def build_all_quota_caches(sender, **kwargs): - refresh_quota_caches.apply() - - def grouper(iterable, n, fillvalue=None): """Collect data into fixed-length chunks or blocks""" # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx args = [iter(iterable)] * n return zip_longest(fillvalue=fillvalue, *args) - - -@app.task -@scopes_disabled() -def refresh_quota_caches(): - # Active events - active = LogEntry.objects.using(settings.DATABASE_REPLICA).filter( - datetime__gt=now() - timedelta(days=7) - ).order_by().values('event').annotate( - last_activity=Max('datetime') - ) - for a in active: - try: - e = Event.objects.using(settings.DATABASE_REPLICA).get(pk=a['event']) - except Event.DoesNotExist: - continue - quotas = e.quotas.filter( - Q(cached_availability_time__isnull=True) | - Q(cached_availability_time__lt=a['last_activity']) | - Q(cached_availability_time__lt=now() - timedelta(hours=2)) - ).filter( - Q(subevent__isnull=True) | - Q(subevent__date_to__isnull=False, subevent__date_to__gte=now() - timedelta(days=14)) | - Q(subevent__date_from__gte=now() - timedelta(days=14)) - ) - - for qs in grouper(quotas, 100, None): - qa = QuotaAvailability(early_out=False) - qa.queue(*[q for q in qs if q is not None]) - qa.compute() diff --git a/src/pretix/control/forms/filter.py b/src/pretix/control/forms/filter.py index 9695f3e624..9a6de35c2e 100644 --- a/src/pretix/control/forms/filter.py +++ b/src/pretix/control/forms/filter.py @@ -995,7 +995,6 @@ class EventFilterForm(FilterForm): 'date_from': 'order_from', 'date_to': 'order_to', 'live': 'live', - 'sum_tickets_paid': 'sum_tickets_paid' } status = forms.ChoiceField( label=_('Status'), diff --git a/src/pretix/control/templates/pretixcontrol/events/index.html b/src/pretix/control/templates/pretixcontrol/events/index.html index eca4f42c88..09156466a2 100644 --- a/src/pretix/control/templates/pretixcontrol/events/index.html +++ b/src/pretix/control/templates/pretixcontrol/events/index.html @@ -81,8 +81,6 @@ {% trans "Paid tickets per quota" %} - - {% trans "Status" %} diff --git a/src/pretix/control/templates/pretixcontrol/fragment_quota_box_paid.html b/src/pretix/control/templates/pretixcontrol/fragment_quota_box_paid.html index 03809d6561..19b80e9e42 100644 --- a/src/pretix/control/templates/pretixcontrol/fragment_quota_box_paid.html +++ b/src/pretix/control/templates/pretixcontrol/fragment_quota_box_paid.html @@ -1,6 +1,7 @@ {% load i18n %} -
{% blocktrans with date=q.cached_availability_time|date:"SHORT_DATETIME_FORMAT" %}Numbers as of {{ date }}{% endblocktrans %}{% if q.cached_avail.1 is not None %}
{% blocktrans with num=q.cached_avail.1 %}Currently available: {{ num }}{% endblocktrans %}{% endif %}"> +{% blocktrans with num=q.cached_avail.1 %}Currently available: {{ num }}{% endblocktrans %}{% endif %}" + href="{% url "control:event.items.quotas.show" event=q.event.slug organizer=q.event.organizer.slug quota=q.pk %}"> {% if q.size|default_if_none:"NONE" == "NONE" %}
@@ -13,4 +14,4 @@
{{ q.cached_availability_paid_orders|default_if_none:"?" }} / {{ q.size|default_if_none:"∞" }}
-
+ diff --git a/src/pretix/control/templates/pretixcontrol/subevents/index.html b/src/pretix/control/templates/pretixcontrol/subevents/index.html index 1d3e13ed9b..fbd52bf0d0 100644 --- a/src/pretix/control/templates/pretixcontrol/subevents/index.html +++ b/src/pretix/control/templates/pretixcontrol/subevents/index.html @@ -80,8 +80,6 @@ {% trans "Paid tickets per quota" %} - - {% trans "Status" %} diff --git a/src/pretix/control/views/dashboards.py b/src/pretix/control/views/dashboards.py index 7568922cb3..34f7a39427 100644 --- a/src/pretix/control/views/dashboards.py +++ b/src/pretix/control/views/dashboards.py @@ -202,14 +202,10 @@ def quota_widgets(sender, subevent=None, lazy=False, **kwargs): widgets = [] quotas = sender.quotas.filter(subevent=subevent) - quotas_to_compute = [ - q for q in quotas - if not q.cache_is_hot(now() + timedelta(seconds=5)) - ] qa = QuotaAvailability() - if quotas_to_compute: - qa.queue(*quotas_to_compute) - qa.compute() + if quotas: + qa.queue(*quotas) + qa.compute(allow_cache=True) for q in quotas: if not lazy: diff --git a/src/pretix/control/views/main.py b/src/pretix/control/views/main.py index c8d0781b30..01a8d95250 100644 --- a/src/pretix/control/views/main.py +++ b/src/pretix/control/views/main.py @@ -1,9 +1,7 @@ from django.conf import settings from django.contrib import messages from django.db import transaction -from django.db.models import ( - F, IntegerField, Max, Min, OuterRef, Prefetch, Subquery, Sum, -) +from django.db.models import F, Max, Min, Prefetch from django.db.models.functions import Coalesce, Greatest from django.http import JsonResponse from django.shortcuts import redirect @@ -52,17 +50,7 @@ class EventList(PaginationMixin, ListView): order_to=Coalesce('max_fromto', 'max_to', 'max_from', 'date_to', 'date_from'), ) - sum_tickets_paid = Quota.objects.filter( - event=OuterRef('pk'), subevent__isnull=True - ).order_by().values('event').annotate( - s=Sum('cached_availability_paid_orders') - ).values( - 's' - ) - - qs = qs.annotate( - sum_tickets_paid=Subquery(sum_tickets_paid, output_field=IntegerField()) - ).prefetch_related( + qs = qs.prefetch_related( Prefetch('quotas', queryset=Quota.objects.filter(subevent__isnull=True).annotate(s=Coalesce(F('size'), 0)).order_by('-s'), to_attr='first_quotas') @@ -90,15 +78,12 @@ class EventList(PaginationMixin, ListView): qa = QuotaAvailability(early_out=False) for q in quotas: - if q.cached_availability_time is None or q.cached_availability_paid_orders is None: - qa.queue(q) + qa.queue(q) qa.compute() for q in quotas: - q.cached_avail = ( - qa.results[q] if q in qa.results - else (q.cached_availability_state, q.cached_availability_number) - ) + q.cached_avail = qa.results[q] + q.cached_availability_paid_orders = qa.count_paid_orders.get(qa, 0) if q.size is not None: q.percent_paid = min( 100, diff --git a/src/pretix/control/views/subevents.py b/src/pretix/control/views/subevents.py index c926af98eb..eb60ee9f51 100644 --- a/src/pretix/control/views/subevents.py +++ b/src/pretix/control/views/subevents.py @@ -6,9 +6,7 @@ from dateutil.rrule import DAILY, MONTHLY, WEEKLY, YEARLY, rrule, rruleset from django.contrib import messages from django.core.files import File from django.db import connections, transaction -from django.db.models import ( - Count, F, IntegerField, OuterRef, Prefetch, Subquery, Sum, -) +from django.db.models import Count, F, Prefetch from django.db.models.functions import Coalesce, TruncDate, TruncTime from django.forms import inlineformset_factory from django.http import Http404, HttpResponse, HttpResponseRedirect @@ -57,20 +55,11 @@ class SubEventQueryMixin: return self.request.GET def get_queryset(self, list=False): - sum_tickets_paid = Quota.objects.filter( - subevent=OuterRef('pk') - ).order_by().values('subevent').annotate( - s=Sum('cached_availability_paid_orders') - ).values( - 's' - ) qs = self.request.event.subevents if list: - qs = qs.annotate( - sum_tickets_paid=Subquery(sum_tickets_paid, output_field=IntegerField()) - ).prefetch_related( + qs = qs.prefetch_related( Prefetch('quotas', - queryset=Quota.objects.annotate(s=Coalesce(F('size'), 0)).order_by('-s'), + queryset=self.request.event.quotas.annotate(s=Coalesce(F('size'), 0)).order_by('-s'), to_attr='first_quotas') ) if self.filter_form.is_valid(): @@ -108,15 +97,12 @@ class SubEventList(EventPermissionRequiredMixin, PaginationMixin, SubEventQueryM qa = QuotaAvailability(early_out=False) for q in quotas: - if q.cached_availability_time is None or q.cached_availability_paid_orders is None: - qa.queue(q) + qa.queue(q) qa.compute() for q in quotas: - q.cached_avail = ( - qa.results[q] if q in qa.results - else (q.cached_availability_state, q.cached_availability_number) - ) + q.cached_avail = qa.results[q] + q.cached_availability_paid_orders = qa.count_paid_orders.get(qa, 0) if q.size is not None: q.percent_paid = min( 100, @@ -1220,8 +1206,7 @@ class SubEventBulkEdit(SubEventQueryMixin, EventPermissionRequiredMixin, FormVie ).values( 'item_list', 'var_list', *(f.name for f in Quota._meta.fields if f.name not in ( - 'id', 'event', 'items', 'variations', 'cached_availability_state', 'cached_availability_number', - 'cached_availability_paid_orders', 'cached_availability_time', 'closed', + 'id', 'event', 'items', 'variations', 'closed', )) ).order_by('subevent_id') diff --git a/src/pretix/presale/views/organizer.py b/src/pretix/presale/views/organizer.py index e3e416f3a0..a201d29c8c 100644 --- a/src/pretix/presale/views/organizer.py +++ b/src/pretix/presale/views/organizer.py @@ -368,33 +368,33 @@ def add_subevents_for_days(qs, before, after, ebd, timezones, event=None, cart_n quotas_to_compute = [] for se in qs: if se.presale_is_running: - quotas_to_compute += [ - q for q in se.active_quotas - if not q.cache_is_hot(now() + timedelta(seconds=5)) - ] + quotas_to_compute += se.active_quotas name = None + qcache = {} if quotas_to_compute: qa = QuotaAvailability() qa.queue(*quotas_to_compute) - qa.compute() + qa.compute(allow_cache=True) + qcache.update(qa.results) + for se in qs: - if quotas_to_compute: - se._quota_cache = qa.results + if qcache: + se._quota_cache = qcache kwargs = {'subevent': se.pk} if cart_namespace: kwargs['cart_namespace'] = cart_namespace - settings = event.settings if event else se.event.settings - timezones.add(settings.timezones) - tz = pytz.timezone(settings.timezone) + s = event.settings if event else se.event.settings + timezones.add(s.timezones) + tz = pytz.timezone(s.timezone) datetime_from = se.date_from.astimezone(tz) date_from = datetime_from.date() if name is None: name = str(se.name) elif str(se.name) != name: ebd['_subevents_different_names'] = True - if se.event.settings.show_date_to and se.date_to: + if s.show_date_to and se.date_to: datetime_to = se.date_to.astimezone(tz) date_to = se.date_to.astimezone(tz).date() d = max(date_from, before.date()) @@ -402,13 +402,13 @@ def add_subevents_for_days(qs, before, after, ebd, timezones, event=None, cart_n first = d == date_from ebd[d].append({ 'continued': not first, - 'timezone': settings.timezone, - 'time': datetime_from.time().replace(tzinfo=None) if first and settings.show_times else None, + 'timezone': s.timezone, + 'time': datetime_from.time().replace(tzinfo=None) if first and s.show_times else None, 'time_end': ( datetime_to.time().replace(tzinfo=None) if (date_to == date_from or ( date_to == date_from + timedelta(days=1) and datetime_to.time() < datetime_from.time() - )) and settings.show_times + )) and s.show_times else None ), 'event': se, @@ -420,9 +420,9 @@ def add_subevents_for_days(qs, before, after, ebd, timezones, event=None, cart_n ebd[date_from].append({ 'event': se, 'continued': False, - 'time': datetime_from.time().replace(tzinfo=None) if se.event.settings.show_times else None, + 'time': datetime_from.time().replace(tzinfo=None) if s.show_times else None, 'url': eventreverse(se.event, 'presale:event.index', kwargs=kwargs), - 'timezone': se.event.settings.timezone, + 'timezone': s.timezone, }) diff --git a/src/pretix/settings.py b/src/pretix/settings.py index 8ba5aa9e70..d0e0123141 100644 --- a/src/pretix/settings.py +++ b/src/pretix/settings.py @@ -273,6 +273,7 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'django.contrib.humanize', 'pretix.base', 'pretix.control', 'pretix.presale', diff --git a/src/tests/control/test_auth.py b/src/tests/control/test_auth.py index dd2ab71e5d..70462ff779 100644 --- a/src/tests/control/test_auth.py +++ b/src/tests/control/test_auth.py @@ -398,7 +398,6 @@ class FakeRedis(object): return self def exists(self, rkey): - print(rkey in self.storage) return rkey in self.storage def setex(self, rkey, value, expiration):