# # This file is part of pretix (Community Edition). # # Copyright (C) 2014-2020 Raphael Michel and contributors # Copyright (C) 2020-today pretix GmbH and contributors # # This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General # Public License as published by the Free Software Foundation in version 3 of the License. # # ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are # applicable granting you additional permissions and placing additional restrictions on your usage of this software. # Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive # this file, see . # # This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied # warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more # details. # # You should have received a copy of the GNU Affero General Public License along with this program. If not, see # . # import sys import time from collections import Counter, defaultdict from itertools import zip_longest import django_redis from django.conf import settings from django.db import connection, models from django.db.models import ( Case, Count, F, Func, Max, OuterRef, Q, Subquery, Sum, Value, When, prefetch_related_objects, ) from django.utils.timezone import now from pretix.base.models import ( CartPosition, Checkin, Order, OrderPosition, Quota, Voucher, WaitingListEntry, ) from ..signals import quota_availability class QuotaAvailability: """ This special object allows so compute the availability of multiple quotas, even across events, and inspect their results. The maximum number of SQL queries is constant and not dependent on the number of quotas. Usage example:: qa = QuotaAvailability() qa.queue(quota1, quota2, …) qa.compute() print(qa.results) Properties you can access after computation. * results (dict mapping quotas to availability tuples) * count_paid_orders (dict mapping quotas to ints) * count_paid_orders (dict mapping quotas to ints) * count_pending_orders (dict mapping quotas to ints) * count_vouchers (dict mapping quotas to ints) * count_waitinglist (dict mapping quotas to ints) * count_cart (dict mapping quotas to ints) """ def __init__(self, count_waitinglist=True, ignore_closed=False, full_results=False, early_out=True, allow_repeatable_read=False): """ Initialize a new quota availability calculator :param count_waitinglist: If ``True`` (default), the waiting list is considered. If ``False``, it is ignored. :param ignore_closed: Quotas have a ``closed`` state that always makes the quota return as sold out. If you set ``ignore_closed`` to ``True``, we will ignore this completely. Default is ``False``. :param full_results: Usually, the computation is as efficient as possible, i.e. if after counting the sold orders we already see that the quota is sold out, we're not going to count the carts, since it does not matter. This also means that you will not be able to get that number from ``.count_cart``. If you want all parts to be calculated (i.e. because you want to show statistics to the user), pass ``full_results`` and we'll skip that optimization. items :param early_out: Usually, if a quota is ``closed`` or if its ``size`` is ``None`` (i.e. unlimited), we will not need database access to determine the availability and return it right away. If you set this to ``False``, however, we will *still* count the number of orders, which is required to keep the database-level quota cache up to date so backend overviews render quickly. If you do not care about keeping the cache up to date, you can set this to ``False`` for further performance improvements. :param allow_repeatable_read: Allow to run this even in REPEATABLE READ mode, generally not advised. """ self._queue = [] self._count_waitinglist = count_waitinglist self._ignore_closed = ignore_closed self._full_results = full_results self._item_to_quotas = defaultdict(set) self._var_to_quotas = defaultdict(set) self._early_out = early_out self._quota_objects = {} self._allow_repeatable_read = allow_repeatable_read self.results = {} self.count_paid_orders = defaultdict(int) self.count_pending_orders = defaultdict(int) self.count_exited_orders = defaultdict(int) self.count_vouchers = defaultdict(int) self.count_waitinglist = defaultdict(int) self.count_cart = defaultdict(int) self._cache_key_suffix = "" if not self._count_waitinglist: self._cache_key_suffix += ":nocw" if self._ignore_closed: self._cache_key_suffix += ":igcl" self.sizes = {} def queue(self, *quota): self._queue += quota def compute(self, now_dt=None, allow_cache=False, allow_cache_stale=False): """ Compute the queued quotas. If ``allow_cache`` is set, results may also be taken from a cache that might be a few minutes outdated. In this case, you may not rely on the results in the ``count_*`` properties. """ if not self._allow_repeatable_read and getattr(connection, "tx_in_repeatable_read", False): raise ValueError("You cannot compute quotas in REPEATABLE READ mode unless you explicitly opted in to " "do so.") now_dt = now_dt or now() quota_ids_set = {q.id for q in self._queue} if not quota_ids_set: return if allow_cache: if self._full_results: raise ValueError("You cannot combine full_results and allow_cache.") elif settings.HAS_REDIS: rc = django_redis.get_redis_connection("redis") quotas_by_event = defaultdict(list) for q in [_q for _q in self._queue if _q.id in quota_ids_set]: quotas_by_event[q.event_id].append(q) for eventid, evquotas in quotas_by_event.items(): d = rc.hmget(f'quotas:{eventid}:availabilitycache{self._cache_key_suffix}', [str(q.pk) for q in evquotas]) for redisval, q in zip(d, evquotas): if redisval is not None: data = [rv for rv in redisval.decode().split(',')] # Except for some rare situations, we don't want to use cache entries older than 2 minutes if time.time() - int(data[2]) < 120 or allow_cache_stale: quota_ids_set.remove(q.id) if data[1] == "None": self.results[q] = int(data[0]), None else: self.results[q] = int(data[0]), int(data[1]) if not quota_ids_set: return quotas = [_q for _q in self._queue if _q.id in quota_ids_set] quotas_original = list(quotas) self._queue.clear() self._compute(quotas, now_dt) for q in quotas_original: for recv, resp in quota_availability.send(sender=q.event, quota=q, result=self.results[q], count_waitinglist=self.count_waitinglist): self.results[q] = resp self._close(quotas) self._write_cache(quotas, now_dt) def _write_cache(self, quotas, now_dt): if not settings.HAS_REDIS or not quotas: return rc = django_redis.get_redis_connection("redis") # We write the computed availability to redis in a per-event hash as # # quota_id -> (availability_state, availability_number, timestamp). # # We store this in a hash instead of individual values to avoid making too many redis requests # which would introduce latency. # The individual entries in the hash are "valid" for 120 seconds. This means in a typical peak scenario with # high load *to a specific calendar or event*, lots of parallel web requests will receive an "expired" result # around the same time, recompute quotas and write back to the cache. To avoid overloading redis with lots of # simultaneous write queries for the same page, we place a very naive and simple "lock" on the write process for # these quotas. We choose 10 seconds since that should be well above the duration of a write. lock_name = '_'.join([str(p) for p in sorted([q.pk for q in quotas])]) if rc.exists(f'quotas:availabilitycachewrite:{lock_name}{self._cache_key_suffix}'): return rc.setex(f'quotas:availabilitycachewrite:{lock_name}{self._cache_key_suffix}', '1', 10) update = defaultdict(list) for q in quotas: update[q.event_id].append(q) for eventid, quotas in update.items(): rc.hset(f'quotas:{eventid}:availabilitycache{self._cache_key_suffix}', mapping={ str(q.id): ",".join( [str(i) for i in self.results[q]] + [str(int(time.time()))] ) for q in quotas }) # To make sure old events do not fill up our redis instance, we set an expiry on the cache. However, we set it # on 7 days even though we mostly ignore values older than 2 monites. The reasoning is that we have some places # where we set allow_cache_stale and use the old entries anyways to save on performance. rc.expire(f'quotas:{eventid}:availabilitycache{self._cache_key_suffix}', 3600 * 24 * 7) # We used to also delete item_quota_cache:* from the event cache here, but as the cache # gets more complex, this does not seem worth it. The cache is only present for up to # 5 seconds to prevent high peaks, and a 5-second delay in availability is usually # tolerable def _close(self, quotas): for q in quotas: if self.results[q][0] <= Quota.AVAILABILITY_ORDERED and q.close_when_sold_out and not q.closed: q.closed = True q.save(update_fields=['closed']) q.log_action('pretix.event.quota.closed') def _compute(self, quotas, now_dt): # Quotas we want to look at now self.sizes.update({q: q.size for q in quotas}) # Some helpful caches self._quota_objects.update({q.pk: q for q in quotas}) # Compute result for closed or unlimited self._compute_early_outs(quotas) if self._early_out: if not self._full_results: quotas = [q for q in quotas if q not in self.results] if not quotas: return size_left = Counter({q: (sys.maxsize if s is None else s) for q, s in self.sizes.items()}) for q in quotas: self.count_paid_orders[q] = 0 self.count_pending_orders[q] = 0 self.count_cart[q] = 0 self.count_vouchers[q] = 0 self.count_waitinglist[q] = 0 # Fetch which quotas belong to which items and variations q_items = Quota.items.through.objects.filter( quota_id__in=[q.pk for q in quotas] ).values('quota_id', 'item_id') for m in q_items: self._item_to_quotas[m['item_id']].add(self._quota_objects[m['quota_id']]) q_vars = Quota.variations.through.objects.filter( quota_id__in=[q.pk for q in quotas] ).values('quota_id', 'itemvariation_id', 'itemvariation__item_id') for m in q_vars: self._var_to_quotas[m['itemvariation_id']].add(self._quota_objects[m['quota_id']]) # We can't be 100% certain that a quota, when it is connected to a variation, is also always connected to # the parent item, so we double-check here just to be sure. self._item_to_quotas[m['itemvariation__item_id']].add(self._quota_objects[m['quota_id']]) self._compute_orders(quotas, q_items, q_vars, size_left) if not self._full_results: quotas = [q for q in quotas if q not in self.results] if not quotas: return self._compute_vouchers(quotas, q_items, q_vars, size_left, now_dt) if not self._full_results: quotas = [q for q in quotas if q not in self.results] if not quotas: return self._compute_carts(quotas, q_items, q_vars, size_left, now_dt) if self._count_waitinglist: if not self._full_results: quotas = [q for q in quotas if q not in self.results] if not quotas: return self._compute_waitinglist(quotas, q_items, q_vars, size_left) for q in quotas: if q not in self.results: if size_left[q] > 0: self.results[q] = Quota.AVAILABILITY_OK, size_left[q] else: raise ValueError("inconclusive quota") def _compute_orders(self, quotas, q_items, q_vars, size_left): events = {q.event_id for q in quotas} subevents = {q.subevent_id for q in quotas} seq = Q(subevent_id__in=subevents) if None in subevents: seq |= Q(subevent__isnull=True) quota_ids = {q.pk for q in quotas} op_lookup = OrderPosition.objects.filter( order__status__in=[Order.STATUS_PAID, Order.STATUS_PENDING], order__event_id__in=events, ).filter(seq).filter( Q( Q(variation_id__isnull=True) & Q(item_id__in={i['item_id'] for i in q_items if i['quota_id'] in quota_ids}) ) | Q( variation_id__in={i['itemvariation_id'] for i in q_vars if i['quota_id'] in quota_ids}) ).filter( ~Q(Q(ignore_from_quota_while_blocked=True) & Q(blocked__isnull=False)) ).order_by() if any(q.release_after_exit for q in quotas): op_lookup = op_lookup.annotate( last_entry=Subquery( Checkin.objects.filter( position_id=OuterRef('pk'), list__allow_entry_after_exit=False, type=Checkin.TYPE_ENTRY, ).order_by().values('position_id').annotate( m=Max('datetime') ).values('m') ), last_exit=Subquery( Checkin.objects.filter( position_id=OuterRef('pk'), list__allow_entry_after_exit=False, type=Checkin.TYPE_EXIT, ).order_by().values('position_id').annotate( m=Max('datetime') ).values('m') ), ).annotate( is_exited=Case( When( Q(last_entry__isnull=False) & Q(last_exit__isnull=False) & Q(last_exit__gt=F('last_entry')), then=Value(1, output_field=models.IntegerField()), ), default=Value(0, output_field=models.IntegerField()), output_field=models.IntegerField(), ), ) else: op_lookup = op_lookup.annotate( is_exited=Value(0, output_field=models.IntegerField()) ) op_lookup = op_lookup.values('order__status', 'item_id', 'subevent_id', 'variation_id', 'is_exited').annotate(c=Count('*')) for line in sorted(op_lookup, key=lambda li: (int(li['is_exited']), li['order__status']), reverse=True): # p before n, exited before non-exited if line['variation_id']: qs = self._var_to_quotas[line['variation_id']] else: qs = self._item_to_quotas[line['item_id']] for q in qs: if q.subevent_id == line['subevent_id']: if line['order__status'] == Order.STATUS_PAID: self.count_paid_orders[q] += line['c'] q.cached_availability_paid_orders = self.count_paid_orders[q] elif line['order__status'] == Order.STATUS_PENDING: self.count_pending_orders[q] += line['c'] if q.release_after_exit and line['is_exited']: self.count_exited_orders[q] += line['c'] else: size_left[q] -= line['c'] if size_left[q] <= 0 and q not in self.results: if line['order__status'] == Order.STATUS_PAID: self.results[q] = Quota.AVAILABILITY_GONE, 0 else: self.results[q] = Quota.AVAILABILITY_ORDERED, 0 def _compute_vouchers(self, quotas, q_items, q_vars, size_left, now_dt): events = {q.event_id for q in quotas} if 'sqlite3' in settings.DATABASES['default']['ENGINE']: func = 'MAX' else: # NOQA func = 'GREATEST' subevents = {q.subevent_id for q in quotas} quota_ids = {q.pk for q in quotas} seq = Q(subevent_id__in=subevents) if None in subevents: seq |= Q(subevent__isnull=True) v_lookup = Voucher.objects.filter( Q(event_id__in=events) & seq & Q(block_quota=True) & Q(Q(valid_until__isnull=True) | Q(valid_until__gte=now_dt)) & Q( Q( Q(variation_id__isnull=True) & Q(item_id__in=( {i['item_id'] for i in q_items if i['quota_id'] in quota_ids} | {i['itemvariation__item_id'] for i in q_vars if i['quota_id'] in quota_ids} )) ) | Q( variation_id__in={i['itemvariation_id'] for i in q_vars if i['quota_id'] in quota_ids} ) | Q( quota_id__in=[q.pk for q in quotas] ) ) ).order_by().values('subevent_id', 'item_id', 'quota_id', 'variation_id').annotate( free=Sum(Func(F('max_usages') - F('redeemed'), 0, function=func)) ) for line in v_lookup: if line['variation_id']: qs = self._var_to_quotas[line['variation_id']] elif line['item_id']: qs = self._item_to_quotas[line['item_id']] else: qs = [self._quota_objects[line['quota_id']]] for q in qs: if q.subevent_id == line['subevent_id']: size_left[q] -= line['free'] self.count_vouchers[q] += line['free'] if q not in self.results and size_left[q] <= 0: self.results[q] = Quota.AVAILABILITY_ORDERED, 0 def _compute_carts(self, quotas, q_items, q_vars, size_left, now_dt): events = {q.event_id for q in quotas} subevents = {q.subevent_id for q in quotas} quota_ids = {q.pk for q in quotas} seq = Q(subevent_id__in=subevents) if None in subevents: seq |= Q(subevent__isnull=True) cart_lookup = CartPosition.objects.filter( Q(event_id__in=events) & seq & Q(expires__gte=now_dt) & Q( Q(voucher__isnull=True) | Q(voucher__block_quota=False) | Q(voucher__valid_until__lt=now_dt) ) & Q( Q( Q(variation_id__isnull=True) & Q(item_id__in={i['item_id'] for i in q_items if i['quota_id'] in quota_ids}) ) | Q( variation_id__in={i['itemvariation_id'] for i in q_vars if i['quota_id'] in quota_ids} ) ) ).order_by().values('item_id', 'subevent_id', 'variation_id').annotate(c=Count('*')) for line in cart_lookup: if line['variation_id']: qs = self._var_to_quotas[line['variation_id']] else: qs = self._item_to_quotas[line['item_id']] for q in qs: if q.subevent_id == line['subevent_id']: size_left[q] -= line['c'] self.count_cart[q] += line['c'] if q not in self.results and size_left[q] <= 0: self.results[q] = Quota.AVAILABILITY_RESERVED, 0 def _compute_waitinglist(self, quotas, q_items, q_vars, size_left): prefetch_related_objects(quotas, "event", "event__organizer") quotas = [ q for q in quotas if not q.event.settings.waiting_list_auto_disable or q.event.settings.waiting_list_auto_disable.datetime(q.subevent or q.event) > now() ] events = {q.event_id for q in quotas} subevents = {q.subevent_id for q in quotas} quota_ids = {q.pk for q in quotas} seq = Q(subevent_id__in=subevents) if None in subevents: seq |= Q(subevent__isnull=True) w_lookup = WaitingListEntry.objects.filter( Q(event_id__in=events) & Q(voucher__isnull=True) & seq & Q( Q( Q(variation_id__isnull=True) & Q(item_id__in={i['item_id'] for i in q_items if i['quota_id'] in quota_ids}) ) | Q(variation_id__in={i['itemvariation_id'] for i in q_vars if i['quota_id'] in quota_ids}) ) ).order_by().values('item_id', 'subevent_id', 'variation_id').annotate(c=Count('*')) for line in w_lookup: if line['variation_id']: qs = self._var_to_quotas[line['variation_id']] else: qs = self._item_to_quotas[line['item_id']] for q in qs: if q.subevent_id == line['subevent_id']: size_left[q] -= line['c'] self.count_waitinglist[q] += line['c'] if q not in self.results and size_left[q] <= 0: self.results[q] = Quota.AVAILABILITY_ORDERED, 0 def _compute_early_outs(self, quotas): for q in quotas: if q.closed and not self._ignore_closed: self.results[q] = Quota.AVAILABILITY_ORDERED, 0 elif q.size is None: self.results[q] = Quota.AVAILABILITY_OK, None elif q.size == 0: self.results[q] = Quota.AVAILABILITY_GONE, 0 def grouper(iterable, n, fillvalue=None): """Collect data into fixed-length chunks or blocks""" # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx args = [iter(iterable)] * n return zip_longest(fillvalue=fillvalue, *args)