CGM_Public/pretix_original
2
0
Fork 1
mirror of https://github.com/pretix/pretix.git synced 2026-05-09 15:54:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move quota cache from database to redis (#2010)

Browse Source
This commit is contained in:
Raphael Michel
2021-03-29 09:42:27 +02:00
committed by GitHub
parent a927b47b8b
commit d3748a6194
14 changed files with 148 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/pretix/base/migrations/0180_auto_20210324_1309.py Normal file
View File

@@ -0,0 +1,29 @@
# Generated by Django 3.0.12 on 2021-03-24 13:09
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('pretixbase', '0179_auto_20210311_1653'),
]
operations = [
migrations.RemoveField(
model_name='quota',
name='cached_availability_number',
),
migrations.RemoveField(
model_name='quota',
name='cached_availability_paid_orders',
),
migrations.RemoveField(
model_name='quota',
name='cached_availability_state',
),
migrations.RemoveField(
model_name='quota',
name='cached_availability_time',
),
]

4
src/pretix/base/models/event.py
View File

@@ -657,10 +657,6 @@ class Event(EventMixin, LoggedModel):
oldid = q.pk
q.pk = None
q.event = self
q.cached_availability_state = None
q.cached_availability_number = None
q.cached_availability_paid_orders = None
q.cached_availability_time = None
q.closed = False
q.save()
q.log_action('pretix.object.cloned')

23
src/pretix/base/models/items.py
View File

@@ -17,6 +17,7 @@ from django.utils.functional import cached_property
from django.utils.timezone import is_naive, make_aware, now
from django.utils.translation import gettext_lazy as _, pgettext_lazy
from django_countries.fields import Country
from django_redis import get_redis_connection
from django_scopes import ScopedManager
from i18nfield.fields import I18nCharField, I18nTextField
@@ -25,6 +26,7 @@ from pretix.base.models.base import LoggedModel
from pretix.base.models.fields import MultiStringField
from pretix.base.models.tax import TaxedPrice
from ... import settings
from .event import Event, SubEvent
@@ -1374,10 +1376,6 @@ class Quota(LoggedModel):
blank=True,
verbose_name=_("Variations")
)
cached_availability_state = models.PositiveIntegerField(null=True, blank=True)
cached_availability_number = models.PositiveIntegerField(null=True, blank=True)
cached_availability_paid_orders = models.PositiveIntegerField(null=True, blank=True)
cached_availability_time = models.DateTimeField(null=True, blank=True)
close_when_sold_out = models.BooleanField(
verbose_name=_('Close this quota permanently once it is sold out'),
@@ -1422,14 +1420,10 @@ class Quota(LoggedModel):
self.event.cache.clear()
def rebuild_cache(self, now_dt=None):
self.cached_availability_time = None
self.cached_availability_number = None
self.cached_availability_state = None
self.availability(now_dt=now_dt)
def cache_is_hot(self, now_dt=None):
now_dt = now_dt or now()
return self.cached_availability_time and (now_dt - self.cached_availability_time).total_seconds() < 120
if settings.HAS_REDIS:
rc = get_redis_connection("redis")
rc.hdel(f'quotas:{self.event_id}:availabilitycache', str(self.pk))
self.availability(now_dt=now_dt)
def availability(
self, now_dt: datetime=None, count_waitinglist=True, _cache=None, allow_cache=False
@@ -1452,9 +1446,6 @@ class Quota(LoggedModel):
"""
from ..services.quotas import QuotaAvailability
if allow_cache and self.cache_is_hot() and count_waitinglist:
return self.cached_availability_state, self.cached_availability_number
if _cache and count_waitinglist is not _cache.get('_count_waitinglist', True):
_cache.clear()
@@ -1462,7 +1453,7 @@ class Quota(LoggedModel):
return _cache[self.pk]
qa = QuotaAvailability(count_waitinglist=count_waitinglist, early_out=False)
qa.queue(self)
qa.compute(now_dt=now_dt)
qa.compute(now_dt=now_dt, allow_cache=allow_cache)
res = qa.results[self]
if _cache is not None:

146
src/pretix/base/services/quotas.py
View File

@@ -1,25 +1,22 @@
import sys
import time
from collections import Counter, defaultdict
from datetime import timedelta
from itertools import zip_longest
from django.conf import settings
from django.db import OperationalError, models
from django.db import models
from django.db.models import (
Case, Count, F, Func, Max, OuterRef, Q, Subquery, Sum, Value, When,
)
from django.dispatch import receiver
from django.utils.timezone import now
from django_scopes import scopes_disabled
from django_redis import get_redis_connection
from pretix.base.models import (
CartPosition, Checkin, Event, LogEntry, Order, OrderPosition, Quota,
Voucher, WaitingListEntry,
CartPosition, Checkin, Order, OrderPosition, Quota, Voucher,
WaitingListEntry,
)
from pretix.celery_app import app
from ...helpers.periodic import minimum_interval
from ..signals import periodic_task, quota_availability
from ..signals import quota_availability
class QuotaAvailability:
@@ -89,7 +86,11 @@ class QuotaAvailability:
def queue(self, *quota):
self._queue += quota
def compute(self, now_dt=None):
def compute(self, now_dt=None, allow_cache=False, allow_cache_stale=False):
"""
Compute the queued quotas. If ``allow_cache`` is set, results may also be taken from a cache that might
be a few minutes outdated. In this case, you may not rely on the results in the ``count_*`` properties.
"""
now_dt = now_dt or now()
quotas = list(set(self._queue))
quotas_original = list(self._queue)
@@ -97,6 +98,35 @@ class QuotaAvailability:
if not quotas:
return
if allow_cache:
if self._full_results:
raise ValueError("You cannot combine full_results and allow_cache.")
elif not self._count_waitinglist:
raise ValueError("If you set allow_cache, you need to set count_waitinglist.")
elif settings.HAS_REDIS:
rc = get_redis_connection("redis")
quotas_by_event = defaultdict(list)
for q in quotas_original:
quotas_by_event[q.event_id].append(q)
for eventid, evquotas in quotas_by_event.items():
d = rc.hmget(f'quotas:{eventid}:availabilitycache', [str(q.pk) for q in evquotas])
for redisval, q in zip(d, evquotas):
if redisval is not None:
data = [rv for rv in redisval.decode().split(',')]
if time.time() - int(data[2]) < 120 or allow_cache_stale:
quotas_original.remove(q)
quotas.remove(q)
if data[1] == "None":
self.results[q] = int(data[0]), None, int(data[2])
else:
self.results[q] = int(data[0]), int(data[1]), int(data[2])
if not quotas:
return
self._compute(quotas, now_dt)
for q in quotas_original:
@@ -105,36 +135,48 @@ class QuotaAvailability:
self.results[q] = resp
self._close(quotas)
try:
self._write_cache(quotas, now_dt)
except OperationalError as e:
# Ignore deadlocks when multiple threads try to write to the cache
if 'deadlock' not in str(e).lower():
raise e
self._write_cache(quotas, now_dt)
def _write_cache(self, quotas, now_dt):
if not settings.HAS_REDIS or not quotas:
return
rc = get_redis_connection("redis")
# We write the computed availability to redis in a per-event hash as
#
# quota_id -> (availability_state, availability_number, timestamp).
#
# We store this in a hash instead of inidividual values to avoid making two many redis requests
# which would introduce latency.
# The individual entries in the hash are "valid" for 120 seconds. This means in a typical peak scenario with
# high load *to a specific calendar or event*, lots of parallel web requests will receive an "expired" result
# around the same time, recompute quotas and write back to the cache. To avoid overloading redis with lots of
# simultaneous write queries for the same page, we place a very naive and simple "lock" on the write process for
# these quotas.
lock_name = '_'.join([str(p) for p in sorted([q.pk for q in quotas])])
if rc.exists(f'quotas:availabilitycachewrite:{lock_name}'):
return
rc.setex(f'quotas:availabilitycachewrite:{lock_name}', '1', 10)
update = defaultdict(list)
for q in quotas:
update[q.event_id].append(q)
for eventid, quotas in update.items():
rc.hmset(f'quotas:{eventid}:availabilitycache', {
str(q.id): ",".join(
[str(i) for i in self.results[q]] +
[str(int(time.time()))]
) for q in quotas
})
rc.expire(f'quotas:{eventid}:availabilitycache', 3600 * 24 * 7)
# We used to also delete item_quota_cache:* from the event cache here, but as the cache
# gets more complex, this does not seem worth it. The cache is only present for up to
# 5 seconds to prevent high peaks, and a 5-second delay in availability is usually
# tolerable
update = []
for q in quotas:
rewrite_cache = self._count_waitinglist and (
not q.cache_is_hot(now_dt) or self.results[q][0] > q.cached_availability_state
or q.cached_availability_paid_orders is None
)
if rewrite_cache:
q.cached_availability_state = self.results[q][0]
q.cached_availability_number = self.results[q][1]
q.cached_availability_time = now_dt
if q in self.count_paid_orders:
q.cached_availability_paid_orders = self.count_paid_orders[q]
update.append(q)
if update:
Quota.objects.using('default').bulk_update(update, [
'cached_availability_state', 'cached_availability_number', 'cached_availability_time',
'cached_availability_paid_orders'
], batch_size=50)
def _close(self, quotas):
for q in quotas:
@@ -404,44 +446,8 @@ class QuotaAvailability:
self.results[q] = Quota.AVAILABILITY_GONE, 0
@receiver(signal=periodic_task)
@minimum_interval(minutes_after_success=60)
def build_all_quota_caches(sender, **kwargs):
refresh_quota_caches.apply()
def grouper(iterable, n, fillvalue=None):
"""Collect data into fixed-length chunks or blocks"""
# grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
args = [iter(iterable)] * n
return zip_longest(fillvalue=fillvalue, *args)
@app.task
@scopes_disabled()
def refresh_quota_caches():
# Active events
active = LogEntry.objects.using(settings.DATABASE_REPLICA).filter(
datetime__gt=now() - timedelta(days=7)
).order_by().values('event').annotate(
last_activity=Max('datetime')
)
for a in active:
try:
e = Event.objects.using(settings.DATABASE_REPLICA).get(pk=a['event'])
except Event.DoesNotExist:
continue
quotas = e.quotas.filter(
Q(cached_availability_time__isnull=True) |
Q(cached_availability_time__lt=a['last_activity']) |
Q(cached_availability_time__lt=now() - timedelta(hours=2))
).filter(
Q(subevent__isnull=True) |
Q(subevent__date_to__isnull=False, subevent__date_to__gte=now() - timedelta(days=14)) |
Q(subevent__date_from__gte=now() - timedelta(days=14))
)
for qs in grouper(quotas, 100, None):
qa = QuotaAvailability(early_out=False)
qa.queue(*[q for q in qs if q is not None])
qa.compute()

1
src/pretix/control/forms/filter.py
View File

@@ -995,7 +995,6 @@ class EventFilterForm(FilterForm):
'date_from': 'order_from',
'date_to': 'order_to',
'live': 'live',
'sum_tickets_paid': 'sum_tickets_paid'
}
status = forms.ChoiceField(
label=_('Status'),

2
src/pretix/control/templates/pretixcontrol/events/index.html
View File

@@ -81,8 +81,6 @@
</th>
<th>
{% trans "Paid tickets per quota" %}
<a href="?{% url_replace request 'ordering' '-sum_tickets_paid' %}"><i class="fa fa-caret-down"></i></a>
<a href="?{% url_replace request 'ordering' 'sum_tickets_paid' %}"><i class="fa fa-caret-up"></i></a>
</th>
<th>
{% trans "Status" %}

7
src/pretix/control/templates/pretixcontrol/fragment_quota_box_paid.html
View File

@@ -1,6 +1,7 @@
{% load i18n %}
<div class="quotabox" data-toggle="tooltip_html" data-placement="top"
title="{% trans "Quota:" %} {{ q.name }}<br>{% blocktrans with date=q.cached_availability_time|date:"SHORT_DATETIME_FORMAT" %}Numbers as of {{ date }}{% endblocktrans %}{% if q.cached_avail.1 is not None %}<br>{% blocktrans with num=q.cached_avail.1 %}Currently available: {{ num }}{% endblocktrans %}{% endif %}">
<a class="quotabox" data-toggle="tooltip_html" data-placement="top"
title="{% trans "Quota:" %} {{ q.name }}{% if q.cached_avail.1 is not None %}<br>{% blocktrans with num=q.cached_avail.1 %}Currently available: {{ num }}{% endblocktrans %}{% endif %}"
href="{% url "control:event.items.quotas.show" event=q.event.slug organizer=q.event.organizer.slug quota=q.pk %}">
{% if q.size|default_if_none:"NONE" == "NONE" %}
<div class="progress">
</div>
@@ -13,4 +14,4 @@
<div class="numbers">
{{ q.cached_availability_paid_orders|default_if_none:"?" }} / {{ q.size|default_if_none:"∞" }}
</div>
</div>
</a>

2
src/pretix/control/templates/pretixcontrol/subevents/index.html
View File

@@ -80,8 +80,6 @@
</th>
<th>
{% trans "Paid tickets per quota" %}
<a href="?{% url_replace request 'filter-ordering' '-sum_tickets_paid' %}"><i class="fa fa-caret-down"></i></a>
<a href="?{% url_replace request 'filter-ordering' 'sum_tickets_paid' %}"><i class="fa fa-caret-up"></i></a>
</th>
<th>
{% trans "Status" %}

10
src/pretix/control/views/dashboards.py
View File

@@ -202,14 +202,10 @@ def quota_widgets(sender, subevent=None, lazy=False, **kwargs):
widgets = []
quotas = sender.quotas.filter(subevent=subevent)
quotas_to_compute = [
q for q in quotas
if not q.cache_is_hot(now() + timedelta(seconds=5))
]
qa = QuotaAvailability()
if quotas_to_compute:
qa.queue(*quotas_to_compute)
qa.compute()
if quotas:
qa.queue(*quotas)
qa.compute(allow_cache=True)
for q in quotas:
if not lazy:

25
src/pretix/control/views/main.py
View File

@@ -1,9 +1,7 @@
from django.conf import settings
from django.contrib import messages
from django.db import transaction
from django.db.models import (
F, IntegerField, Max, Min, OuterRef, Prefetch, Subquery, Sum,
)
from django.db.models import F, Max, Min, Prefetch
from django.db.models.functions import Coalesce, Greatest
from django.http import JsonResponse
from django.shortcuts import redirect
@@ -52,17 +50,7 @@ class EventList(PaginationMixin, ListView):
order_to=Coalesce('max_fromto', 'max_to', 'max_from', 'date_to', 'date_from'),
)
sum_tickets_paid = Quota.objects.filter(
event=OuterRef('pk'), subevent__isnull=True
).order_by().values('event').annotate(
s=Sum('cached_availability_paid_orders')
).values(
's'
)
qs = qs.annotate(
sum_tickets_paid=Subquery(sum_tickets_paid, output_field=IntegerField())
).prefetch_related(
qs = qs.prefetch_related(
Prefetch('quotas',
queryset=Quota.objects.filter(subevent__isnull=True).annotate(s=Coalesce(F('size'), 0)).order_by('-s'),
to_attr='first_quotas')
@@ -90,15 +78,12 @@ class EventList(PaginationMixin, ListView):
qa = QuotaAvailability(early_out=False)
for q in quotas:
if q.cached_availability_time is None or q.cached_availability_paid_orders is None:
qa.queue(q)
qa.queue(q)
qa.compute()
for q in quotas:
q.cached_avail = (
qa.results[q] if q in qa.results
else (q.cached_availability_state, q.cached_availability_number)
)
q.cached_avail = qa.results[q]
q.cached_availability_paid_orders = qa.count_paid_orders.get(qa, 0)
if q.size is not None:
q.percent_paid = min(
100,

29
src/pretix/control/views/subevents.py
View File

@@ -6,9 +6,7 @@ from dateutil.rrule import DAILY, MONTHLY, WEEKLY, YEARLY, rrule, rruleset
from django.contrib import messages
from django.core.files import File
from django.db import connections, transaction
from django.db.models import (
Count, F, IntegerField, OuterRef, Prefetch, Subquery, Sum,
)
from django.db.models import Count, F, Prefetch
from django.db.models.functions import Coalesce, TruncDate, TruncTime
from django.forms import inlineformset_factory
from django.http import Http404, HttpResponse, HttpResponseRedirect
@@ -57,20 +55,11 @@ class SubEventQueryMixin:
return self.request.GET
def get_queryset(self, list=False):
sum_tickets_paid = Quota.objects.filter(
subevent=OuterRef('pk')
).order_by().values('subevent').annotate(
s=Sum('cached_availability_paid_orders')
).values(
's'
)
qs = self.request.event.subevents
if list:
qs = qs.annotate(
sum_tickets_paid=Subquery(sum_tickets_paid, output_field=IntegerField())
).prefetch_related(
qs = qs.prefetch_related(
Prefetch('quotas',
queryset=Quota.objects.annotate(s=Coalesce(F('size'), 0)).order_by('-s'),
queryset=self.request.event.quotas.annotate(s=Coalesce(F('size'), 0)).order_by('-s'),
to_attr='first_quotas')
)
if self.filter_form.is_valid():
@@ -108,15 +97,12 @@ class SubEventList(EventPermissionRequiredMixin, PaginationMixin, SubEventQueryM
qa = QuotaAvailability(early_out=False)
for q in quotas:
if q.cached_availability_time is None or q.cached_availability_paid_orders is None:
qa.queue(q)
qa.queue(q)
qa.compute()
for q in quotas:
q.cached_avail = (
qa.results[q] if q in qa.results
else (q.cached_availability_state, q.cached_availability_number)
)
q.cached_avail = qa.results[q]
q.cached_availability_paid_orders = qa.count_paid_orders.get(qa, 0)
if q.size is not None:
q.percent_paid = min(
100,
@@ -1220,8 +1206,7 @@ class SubEventBulkEdit(SubEventQueryMixin, EventPermissionRequiredMixin, FormVie
).values(
'item_list', 'var_list',
*(f.name for f in Quota._meta.fields if f.name not in (
'id', 'event', 'items', 'variations', 'cached_availability_state', 'cached_availability_number',
'cached_availability_paid_orders', 'cached_availability_time', 'closed',
'id', 'event', 'items', 'variations', 'closed',
))
).order_by('subevent_id')

32
src/pretix/presale/views/organizer.py
View File

@@ -368,33 +368,33 @@ def add_subevents_for_days(qs, before, after, ebd, timezones, event=None, cart_n
quotas_to_compute = []
for se in qs:
if se.presale_is_running:
quotas_to_compute += [
q for q in se.active_quotas
if not q.cache_is_hot(now() + timedelta(seconds=5))
]
quotas_to_compute += se.active_quotas
name = None
qcache = {}
if quotas_to_compute:
qa = QuotaAvailability()
qa.queue(*quotas_to_compute)
qa.compute()
qa.compute(allow_cache=True)
qcache.update(qa.results)
for se in qs:
if quotas_to_compute:
se._quota_cache = qa.results
if qcache:
se._quota_cache = qcache
kwargs = {'subevent': se.pk}
if cart_namespace:
kwargs['cart_namespace'] = cart_namespace
settings = event.settings if event else se.event.settings
timezones.add(settings.timezones)
tz = pytz.timezone(settings.timezone)
s = event.settings if event else se.event.settings
timezones.add(s.timezones)
tz = pytz.timezone(s.timezone)
datetime_from = se.date_from.astimezone(tz)
date_from = datetime_from.date()
if name is None:
name = str(se.name)
elif str(se.name) != name:
ebd['_subevents_different_names'] = True
if se.event.settings.show_date_to and se.date_to:
if s.show_date_to and se.date_to:
datetime_to = se.date_to.astimezone(tz)
date_to = se.date_to.astimezone(tz).date()
d = max(date_from, before.date())
@@ -402,13 +402,13 @@ def add_subevents_for_days(qs, before, after, ebd, timezones, event=None, cart_n
first = d == date_from
ebd[d].append({
'continued': not first,
'timezone': settings.timezone,
'time': datetime_from.time().replace(tzinfo=None) if first and settings.show_times else None,
'timezone': s.timezone,
'time': datetime_from.time().replace(tzinfo=None) if first and s.show_times else None,
'time_end': (
datetime_to.time().replace(tzinfo=None)
if (date_to == date_from or (
date_to == date_from + timedelta(days=1) and datetime_to.time() < datetime_from.time()
)) and settings.show_times
)) and s.show_times
else None
),
'event': se,
@@ -420,9 +420,9 @@ def add_subevents_for_days(qs, before, after, ebd, timezones, event=None, cart_n
ebd[date_from].append({
'event': se,
'continued': False,
'time': datetime_from.time().replace(tzinfo=None) if se.event.settings.show_times else None,
'time': datetime_from.time().replace(tzinfo=None) if s.show_times else None,
'url': eventreverse(se.event, 'presale:event.index', kwargs=kwargs),
'timezone': se.event.settings.timezone,
'timezone': s.timezone,
})

1
src/pretix/settings.py
View File

@@ -273,6 +273,7 @@ INSTALLED_APPS = [
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.humanize',
'pretix.base',
'pretix.control',
'pretix.presale',