From 584ff16f58fb272638d475ba23cfd12a9918f71a Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Tue, 23 Mar 2021 19:33:40 +0100 Subject: [PATCH] Add a quota cache in redis --- src/pretix/base/services/quotas.py | 28 +++++++++++----- src/pretix/presale/views/organizer.py | 47 +++++++++++++++++++-------- 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/src/pretix/base/services/quotas.py b/src/pretix/base/services/quotas.py index d9f6fc37c8..b9ce1bdf5c 100644 --- a/src/pretix/base/services/quotas.py +++ b/src/pretix/base/services/quotas.py @@ -10,6 +10,7 @@ from django.db.models import ( ) from django.dispatch import receiver from django.utils.timezone import now +from django_redis import get_redis_connection from django_scopes import scopes_disabled from pretix.base.models import ( @@ -89,7 +90,7 @@ class QuotaAvailability: def queue(self, *quota): self._queue += quota - def compute(self, now_dt=None): + def compute(self, now_dt=None, dbcache=True): now_dt = now_dt or now() quotas = list(set(self._queue)) quotas_original = list(self._queue) @@ -106,18 +107,18 @@ class QuotaAvailability: self._close(quotas) try: - self._write_cache(quotas, now_dt) + self._write_cache(quotas, now_dt, dbcache) except OperationalError as e: # Ignore deadlocks when multiple threads try to write to the cache if 'deadlock' not in str(e).lower(): raise e - def _write_cache(self, quotas, now_dt): + def _write_cache(self, quotas, now_dt, dbcache): # We used to also delete item_quota_cache:* from the event cache here, but as the cache # gets more complex, this does not seem worth it. The cache is only present for up to # 5 seconds to prevent high peaks, and a 5-second delay in availability is usually # tolerable - update = [] + update = defaultdict(list) for q in quotas: rewrite_cache = self._count_waitinglist and ( not q.cache_is_hot(now_dt) or self.results[q][0] > q.cached_availability_state @@ -129,12 +130,21 @@ class QuotaAvailability: q.cached_availability_time = now_dt if q in self.count_paid_orders: q.cached_availability_paid_orders = self.count_paid_orders[q] - update.append(q) + update[q.event_id].append(q) + if update: - Quota.objects.using('default').bulk_update(update, [ - 'cached_availability_state', 'cached_availability_number', 'cached_availability_time', - 'cached_availability_paid_orders' - ], batch_size=50) + if dbcache: + Quota.objects.using('default').bulk_update(sum((quotas for event, quotas in update.items()), []), [ + 'cached_availability_state', 'cached_availability_number', 'cached_availability_time', + 'cached_availability_paid_orders' + ], batch_size=50) + + if settings.HAS_REDIS: + rc = get_redis_connection("redis") + for eventid, quotas in update.items(): + rc.hmset(f'quotas:{eventid}:availability', { + str(q.id): ",".join([str(i) for i in self.results[q]]) for q in quotas + }) def _close(self, quotas): for q in quotas: diff --git a/src/pretix/presale/views/organizer.py b/src/pretix/presale/views/organizer.py index e3e416f3a0..2cbb0d7c63 100644 --- a/src/pretix/presale/views/organizer.py +++ b/src/pretix/presale/views/organizer.py @@ -14,6 +14,7 @@ from django.utils.timezone import get_current_timezone, now from django.views import View from django.views.decorators.cache import cache_page from django.views.generic import ListView, TemplateView +from django_redis import get_redis_connection from pytz import UTC from pretix.base.i18n import language @@ -366,35 +367,53 @@ def add_subevents_for_days(qs, before, after, ebd, timezones, event=None, cart_n 'date_from' ) quotas_to_compute = [] + quotas_by_event = defaultdict(list) for se in qs: if se.presale_is_running: quotas_to_compute += [ q for q in se.active_quotas if not q.cache_is_hot(now() + timedelta(seconds=5)) ] + quotas_by_event[se.event_id] += [ + q for q in se.active_quotas + if not q.cache_is_hot(now() + timedelta(seconds=5)) + ] name = None + qcache = {} if quotas_to_compute: - qa = QuotaAvailability() - qa.queue(*quotas_to_compute) - qa.compute() - for se in qs: + if settings.HAS_REDIS: + rc = get_redis_connection("redis") + for ev, quotas in quotas_by_event.items(): + d = rc.hmget(f'quotas:{ev}:availability', [str(q.pk) for q in quotas]) + for redisval, q in zip(d, quotas): + if redisval is not None: + quotas_to_compute.remove(q) + qcache[q] = tuple(int(rv) for rv in redisval.decode().split(',')) + if quotas_to_compute: - se._quota_cache = qa.results + qa = QuotaAvailability() + qa.queue(*quotas_to_compute) + qa.compute(dbcache=False) + qcache.update(qa.results) + + for se in qs: + if qcache: + se._quota_cache = qcache kwargs = {'subevent': se.pk} if cart_namespace: kwargs['cart_namespace'] = cart_namespace - settings = event.settings if event else se.event.settings - timezones.add(settings.timezones) - tz = pytz.timezone(settings.timezone) + s = event.settings if event else se.event.settings + timezones.add(s.timezones) + tz = pytz.timezone(s.timezone) datetime_from = se.date_from.astimezone(tz) date_from = datetime_from.date() if name is None: name = str(se.name) elif str(se.name) != name: ebd['_subevents_different_names'] = True - if se.event.settings.show_date_to and se.date_to: + if s.show_date_to and se.date_to: datetime_to = se.date_to.astimezone(tz) date_to = se.date_to.astimezone(tz).date() d = max(date_from, before.date()) @@ -402,13 +421,13 @@ def add_subevents_for_days(qs, before, after, ebd, timezones, event=None, cart_n first = d == date_from ebd[d].append({ 'continued': not first, - 'timezone': settings.timezone, - 'time': datetime_from.time().replace(tzinfo=None) if first and settings.show_times else None, + 'timezone': s.timezone, + 'time': datetime_from.time().replace(tzinfo=None) if first and s.show_times else None, 'time_end': ( datetime_to.time().replace(tzinfo=None) if (date_to == date_from or ( date_to == date_from + timedelta(days=1) and datetime_to.time() < datetime_from.time() - )) and settings.show_times + )) and s.show_times else None ), 'event': se, @@ -420,9 +439,9 @@ def add_subevents_for_days(qs, before, after, ebd, timezones, event=None, cart_n ebd[date_from].append({ 'event': se, 'continued': False, - 'time': datetime_from.time().replace(tzinfo=None) if se.event.settings.show_times else None, + 'time': datetime_from.time().replace(tzinfo=None) if s.show_times else None, 'url': eventreverse(se.event, 'presale:event.index', kwargs=kwargs), - 'timezone': se.event.settings.timezone, + 'timezone': s.timezone, })