CGM_Public/pretix_cgo
1
0
Fork 0
forked from CGM_Public/pretix_original
Code Pull Requests Actions Activity

Simplified the locking procedure

Browse Source
This commit is contained in:
Raphael Michel
2015-09-16 11:59:07 +02:00
parent 0680f940a6
commit c268da02a2
8 changed files with 240 additions and 248 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/pretix/base/migrations/0014_auto_20150916_1319.py Normal file
View File

@@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('pretixbase', '0013_auto_20150916_0941'),
]
operations = [
migrations.CreateModel(
name='EventLock',
fields=[
('event', models.CharField(primary_key=True, max_length=36, serialize=False)),
('date', models.DateTimeField(auto_now=True)),
],
),
migrations.RemoveField(
model_name='quota',
name='locked',
),
]

115
src/pretix/base/models.py
View File

@@ -1,8 +1,7 @@
import copy import copy
import random import random
import time
import uuid import uuid
from datetime import datetime, timedelta from datetime import datetime
from itertools import product from itertools import product
import six import six
@@ -453,6 +452,7 @@ class Event(Versionable):
null=True, blank=True, null=True, blank=True,
verbose_name=_("Plugins"), verbose_name=_("Plugins"),
) )
locked_here = False
class Meta: class Meta:
verbose_name = _("Event") verbose_name = _("Event")
@@ -531,6 +531,13 @@ class Event(Versionable):
return False return False
return True return True
def lock(self):
"""
Returns a contextmanager that can be used to lock an event for bookings
"""
from .services import locking
return locking.LockManager(self)
class EventPermission(Versionable): class EventPermission(Versionable):
""" """
@@ -1372,10 +1379,6 @@ class Quota(Versionable):
blank=True, blank=True,
verbose_name=_("Variations") verbose_name=_("Variations")
) )
locked = models.DateTimeField(
null=True, blank=True
)
locked_here = False
class Meta: class Meta:
verbose_name = _("Quota") verbose_name = _("Quota")
@@ -1443,34 +1446,9 @@ class Quota(Versionable):
return Quota.AVAILABILITY_OK, self.size - paid_orders - pending_valid_orders - valid_cart_positions return Quota.AVAILABILITY_OK, self.size - paid_orders - pending_valid_orders - valid_cart_positions
class LockTimeoutException(Exception):
pass
class QuotaExceededException(Exception): class QuotaExceededException(Exception):
pass pass
def lock(self):
"""
Issue a lock on this quota so nobody can take tickets from this quota until
you release the lock. Will retry 5 times on failure.
:raises Quota.LockTimeoutException: if the quota is locked every time we try
to obtain the lock
"""
from .services import locking
return locking.lock_quota(self)
def release(self, force=False):
"""
Release a lock placed by :py:meth:`lock()`. If the parameter force is not set to ``True``,
the lock will only be released if it was issued in _this_ python
representation of the database object.
"""
from .services import locking
return locking.release_quota(self, force)
class Order(Versionable): class Order(Versionable):
""" """
@@ -1637,22 +1615,22 @@ class Order(Versionable):
order.save() order.save()
return order return order
def _can_be_paid(self, keep_locked=False): def _can_be_paid(self):
error_messages = { error_messages = {
'late': _("The payment is too late to be accepted."), 'late': _("The payment is too late to be accepted."),
} }
if self.event.settings.get('payment_term_last') \ if self.event.settings.get('payment_term_last') \
and now() > self.event.settings.get('payment_term_last'): and now() > self.event.settings.get('payment_term_last'):
return error_messages['late'], None return error_messages['late']
if now() < self.expires: if now() < self.expires:
return True, None return True
if not self.event.settings.get('payment_term_accept_late'): if not self.event.settings.get('payment_term_accept_late'):
return error_messages['late'], None return error_messages['late']
return self._is_still_available(keep_locked) return self._is_still_available()
def _is_still_available(self, keep_locked=False): def _is_still_available(self):
error_messages = { error_messages = {
'unavailable': _('Some of the ordered products were no longer available.'), 'unavailable': _('Some of the ordered products were no longer available.'),
'busy': _('We were not able to process the request completely as the ' 'busy': _('We were not able to process the request completely as the '
@@ -1664,43 +1642,34 @@ class Order(Versionable):
'variation__values', 'variation__values__prop', 'variation__values', 'variation__values__prop',
'item__questions', 'answers' 'item__questions', 'answers'
)) ))
quotas_locked = set() quota_cache = {}
release = True
try: try:
for i, op in enumerate(positions): with self.event.lock():
quotas = list(op.item.quotas.all()) if op.variation is None else list(op.variation.quotas.all()) for i, op in enumerate(positions):
if len(quotas) == 0: quotas = list(op.item.quotas.all()) if op.variation is None else list(op.variation.quotas.all())
raise Quota.QuotaExceededException(error_messages['unavailable']) if len(quotas) == 0:
for quota in quotas:
# Lock the quota, so no other thread is allowed to perform sales covered by this
# quota while we're doing so.
if quota.identity not in [q.identity for q in quotas_locked]:
quota.lock()
quotas_locked.add(quota)
quota.cached_availability = quota.availability()[1]
else:
# Use cached version
quota = [q for q in quotas_locked if q.identity == quota.identity][0]
quota.cached_availability -= 1
if quota.cached_availability < 0:
# This quota is sold out/currently unavailable, so do not sell this at all
raise Quota.QuotaExceededException(error_messages['unavailable']) raise Quota.QuotaExceededException(error_messages['unavailable'])
for quota in quotas:
# Lock the quota, so no other thread is allowed to perform sales covered by this
# quota while we're doing so.
if quota.identity not in quota_cache:
quota_cache[quota.identity] = quota
quota.cached_availability = quota.availability()[1]
else:
# Use cached version
quota = quota_cache[quota.identity]
quota.cached_availability -= 1
if quota.cached_availability < 0:
# This quota is sold out/currently unavailable, so do not sell this at all
raise Quota.QuotaExceededException(error_messages['unavailable'])
except Quota.QuotaExceededException as e: except Quota.QuotaExceededException as e:
return str(e), None return str(e)
except Quota.LockTimeoutException: except EventLock.LockTimeoutException:
# Is raised when there are too many threads asking for quota locks and we were # Is raised when there are too many threads asking for quota locks and we were
# unaible to get one # unaible to get one
return error_messages['busy'], None return error_messages['busy']
else: return True
release = False
finally:
# Release the locks. This is important ;)
if release or not keep_locked:
for quota in quotas_locked:
quota.release()
return True, quotas_locked
class CachedTicket(models.Model): class CachedTicket(models.Model):
@@ -1905,3 +1874,11 @@ class OrganizerSetting(Versionable):
object = VersionedForeignKey(Organizer, related_name='setting_objects') object = VersionedForeignKey(Organizer, related_name='setting_objects')
key = models.CharField(max_length=255) key = models.CharField(max_length=255)
value = models.TextField() value = models.TextField()
class EventLock(models.Model):
event = models.CharField(max_length=36, primary_key=True)
date = models.DateTimeField(auto_now=True)
class LockTimeoutException(Exception):
pass

125
src/pretix/base/services/locking.py
View File

@@ -3,110 +3,117 @@ import time
from datetime import timedelta from datetime import timedelta
from django.conf import settings from django.conf import settings
from django.db.models import Q from django.db import transaction
from django.utils.timezone import now from django.utils.timezone import now
from pretix.base.models import Quota from pretix.base.models import EventLock
logger = logging.getLogger('pretix.base.locking') logger = logging.getLogger('pretix.base.locking')
def lock_quota(quota): class LockManager:
def __init__(self, event):
self.event = event
def __enter__(self):
lock_event(self.event)
def __exit__(self, exc_type, exc_val, exc_tb):
release_event(self.event)
if exc_type is not None:
return False
def lock_event(event):
""" """
Issue a lock on this quota so nobody can take tickets from this quota until Issue a lock on this event so nobody can book tickets for this event until
you release the lock. Will retry 5 times on failure. you release the lock. Will retry 5 times on failure.
:raises Quota.LockTimeoutException: if the quota is locked every time we try :raises EventLock.LockTimeoutException: if the event is locked every time we try
to obtain the lock to obtain the lock
""" """
if event.locked_here:
return True
if settings.HAS_REDIS: if settings.HAS_REDIS:
return lock_quota_redis(quota) return lock_event_redis(event)
else: else:
return lock_quota_db(quota) return lock_event_db(event)
def lock_quota_db(quota): def release_event(event, force=False):
retries = 5
for i in range(retries):
dt = now()
updated = Quota.objects.current.filter(
Q(identity=quota.identity)
& Q(Q(locked__lt=dt - timedelta(seconds=120)) | Q(locked__isnull=True))
& Q(version_end_date__isnull=True)
).update(
locked=dt
)
if updated:
quota.locked_here = dt
quota.locked = dt
return True
time.sleep(2 ** i / 100)
raise Quota.LockTimeoutException()
def release_quota(quota, force=False):
""" """
Release a lock placed by :py:meth:`lock()`. If the parameter force is not set to ``True``, Release a lock placed by :py:meth:`lock()`. If the parameter force is not set to ``True``,
the lock will only be released if it was issued in _this_ python the lock will only be released if it was issued in _this_ python
representation of the database object. representation of the database object.
""" """
if not quota.locked_here and not force: if not event.locked_here and not force:
return False return False
if settings.HAS_REDIS: if settings.HAS_REDIS:
return release_quota_redis(quota) return release_event_redis(event)
else: else:
return release_quota_db(quota) return release_event_db(event)
def release_quota_db(quota): def lock_event_db(event):
updated = Quota.objects.current.filter( retries = 5
identity=quota.identity, for i in range(retries):
version_end_date__isnull=True with transaction.atomic():
).update( dt = now()
locked=None l, created = EventLock.objects.get_or_create(event=event.identity)
) if created:
quota.locked_here = None event.locked_here = dt
quota.locked = None return True
return updated elif l.date < now() - timedelta(seconds=120):
updated = EventLock.objects.filter(event=event.identity, date=l.date).update(date=dt)
if updated:
event.locked_here = dt
return True
time.sleep(2 ** i / 100)
raise EventLock.LockTimeoutException()
def redis_lock_from_quota(quota): def release_event_db(event):
deleted = EventLock.objects.filter(event=event.identity).delete()
event.locked_here = None
return deleted
def redis_lock_from_event(event):
from django_redis import get_redis_connection from django_redis import get_redis_connection
from redis.lock import Lock from redis.lock import Lock
if not hasattr(quota, '_redis_lock'): if not hasattr(event, '_redis_lock'):
rc = get_redis_connection("redis") rc = get_redis_connection("redis")
quota._redis_lock = Lock(redis=rc, name='pretix_quota_%s' % quota.identity, timeout=120) event._redis_lock = Lock(redis=rc, name='pretix_event_%s' % event.identity, timeout=120)
return quota._redis_lock return event._redis_lock
def lock_quota_redis(quota): def lock_event_redis(event):
from redis.exceptions import RedisError from redis.exceptions import RedisError
lock = redis_lock_from_quota(quota)
lock = redis_lock_from_event(event)
retries = 5 retries = 5
for i in range(retries): for i in range(retries):
dt = now() dt = now()
try: try:
if lock.acquire(False): if lock.acquire(False):
quota.locked_here = dt event.locked_here = dt
quota.locked = dt
return True return True
except RedisError: except RedisError:
logger.exception('Error locking a quota') logger.exception('Error locking an event')
raise Quota.LockTimeoutException() raise EventLock.LockTimeoutException()
time.sleep(2 ** i / 100) time.sleep(2 ** i / 100)
raise Quota.LockTimeoutException() raise EventLock.LockTimeoutException()
def release_quota_redis(quota): def release_event_redis(event):
from redis import RedisError from redis import RedisError
lock = redis_lock_from_quota(quota)
lock = redis_lock_from_event(event)
try: try:
lock.release() lock.release()
except RedisError: except RedisError:
logger.exception('Error releasing a quota lock') logger.exception('Error releasing an event lock')
raise Quota.LockTimeoutException() raise EventLock.LockTimeoutException()
quota.locked_here = None event.locked_here = None
quota.locked = None
return True return True

86
src/pretix/base/services/orders.py
View File

@@ -4,7 +4,7 @@ from django.db import transaction
from django.utils.timezone import now from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from pretix.base.models import Order, OrderPosition, Quota from pretix.base.models import EventLock, Order, OrderPosition, Quota
from pretix.base.services.mail import mail from pretix.base.services.mail import mail
from pretix.base.signals import order_paid, order_placed from pretix.base.signals import order_paid, order_placed
from pretix.helpers.urls import build_absolute_uri from pretix.helpers.urls import build_absolute_uri
@@ -27,22 +27,19 @@ def mark_order_paid(order, provider=None, info=None, date=None, manual=None, for
:type force: boolean :type force: boolean
:raises Quota.QuotaExceededException: if the quota is exceeded and ``force`` is ``False`` :raises Quota.QuotaExceededException: if the quota is exceeded and ``force`` is ``False``
""" """
can_be_paid, quotas_locked = order._can_be_paid(keep_locked=True) with order.event.lock():
if not force and can_be_paid is not True: can_be_paid = order._can_be_paid()
raise Quota.QuotaExceededException(can_be_paid) if not force and can_be_paid is not True:
order = order.clone() raise Quota.QuotaExceededException(can_be_paid)
order.payment_provider = provider or order.payment_provider order = order.clone()
order.payment_info = info or order.payment_info order.payment_provider = provider or order.payment_provider
order.payment_date = date or now() order.payment_info = info or order.payment_info
if manual is not None: order.payment_date = date or now()
order.payment_manual = manual if manual is not None:
order.status = Order.STATUS_PAID order.payment_manual = manual
order.save() order.status = Order.STATUS_PAID
order_paid.send(order.event, order=order) order.save()
order_paid.send(order.event, order=order)
if quotas_locked:
for quota in quotas_locked:
quota.release()
from pretix.base.services.mail import mail from pretix.base.services.mail import mail
@@ -69,7 +66,7 @@ class OrderError(Exception):
pass pass
def check_positions(event, dt, positions, quotas_locked): def check_positions(event, dt, positions):
error_messages = { error_messages = {
'unavailable': _('Some of the products you selected were no longer available. ' 'unavailable': _('Some of the products you selected were no longer available. '
'Please see below for details.'), 'Please see below for details.'),
@@ -102,11 +99,6 @@ def check_positions(event, dt, positions, quotas_locked):
continue continue
quota_ok = True quota_ok = True
for quota in quotas: for quota in quotas:
# Lock the quota, so no other thread is allowed to perform sales covered by this
# quota while we're doing so.
if quota.identity not in [q.identity for q in quotas_locked]:
quota.lock()
quotas_locked.add(quota)
avail = quota.availability() avail = quota.availability()
if avail[0] != Quota.AVAILABILITY_OK: if avail[0] != Quota.AVAILABILITY_OK:
# This quota is sold out/currently unavailable, so do not sell this at all # This quota is sold out/currently unavailable, so do not sell this at all
@@ -131,35 +123,31 @@ def perform_order(event, user, payment_provider, positions):
'server was too busy. Please try again.'), 'server was too busy. Please try again.'),
} }
dt = now() dt = now()
quotas_locked = set()
try: try:
check_positions(event, dt, positions, quotas_locked) with event.lock():
order = place_order(event, user, positions, dt, payment_provider) check_positions(event, dt, positions)
mail( order = place_order(event, user, positions, dt, payment_provider)
user, _('Your order: %(code)s') % {'code': order.code}, mail(
'pretixpresale/email/order_placed.txt', user, _('Your order: %(code)s') % {'code': order.code},
{ 'pretixpresale/email/order_placed.txt',
'user': user, 'order': order, {
'event': event, 'user': user, 'order': order,
'url': build_absolute_uri('presale:event.order', kwargs={ 'event': event,
'event': event.slug, 'url': build_absolute_uri('presale:event.order', kwargs={
'organizer': event.organizer.slug, 'event': event.slug,
'order': order.code, 'organizer': event.organizer.slug,
}), 'order': order.code,
'payment': payment_provider.order_pending_mail_render(order) }),
}, 'payment': payment_provider.order_pending_mail_render(order)
event },
) event
return order )
except Quota.LockTimeoutException: return order
# Is raised when there are too many threads asking for quota locks and we were except EventLock.LockTimeoutException:
# unaible to get one # Is raised when there are too many threads asking for event locks and we were
# unable to get one
raise OrderError(error_messages['busy']) raise OrderError(error_messages['busy'])
finally:
# Release the locks. This is important ;)
for quota in quotas_locked:
quota.release()
@transaction.atomic() @transaction.atomic()

2
src/pretix/control/views/orders.py
View File

@@ -248,7 +248,7 @@ class OrderExtend(OrderView):
if oldvalue > now(): if oldvalue > now():
self.form.save() self.form.save()
else: else:
is_available, _quotas_locked = self.order._is_still_available(keep_locked=False) is_available = self.order._is_still_available()
if is_available is True: if is_available is True:
self.form.save() self.form.save()
messages.success(self.request, _('The payment term has been changed.')) messages.success(self.request, _('The payment term has been changed.'))

129
src/pretix/presale/views/cart.py
View File

@@ -10,7 +10,9 @@ from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from django.views.generic import View from django.views.generic import View
from pretix.base.models import CartPosition, Item, ItemVariation, Quota from pretix.base.models import (
CartPosition, EventLock, Item, ItemVariation, Quota,
)
from pretix.presale.views import EventLoginRequiredMixin, EventViewMixin from pretix.presale.views import EventLoginRequiredMixin, EventViewMixin
@@ -187,78 +189,71 @@ class CartAdd(EventViewMixin, CartActionMixin, View):
identity__in=[i[1] for i in self.items if i[1] is not None] identity__in=[i[1] for i in self.items if i[1] is not None]
).select_related("item", "item__event").prefetch_related("quotas", "values", "values__prop") ).select_related("item", "item__event").prefetch_related("quotas", "values", "values__prop")
} }
try:
with self.request.event.lock():
# Process the request itself
for i in self.items:
# Check whether the specified items are part of what we just fetched from the database
# If they are not, the user supplied item IDs which either do not exist or belong to
# a different event
if i[0] not in items_cache or (i[1] is not None and i[1] not in variations_cache):
self.error_message(self.error_messages['not_for_sale'])
return redirect(self.get_failure_url())
# Process the request itself item = items_cache[i[0]]
for i in self.items: variation = variations_cache[i[1]] if i[1] is not None else None
# Check whether the specified items are part of what we just fetched from the database
# If they are not, the user supplied item IDs which either do not exist or belong to
# a different event
if i[0] not in items_cache or (i[1] is not None and i[1] not in variations_cache):
self.error_message(self.error_messages['not_for_sale'])
return redirect(self.get_failure_url())
item = items_cache[i[0]] # Execute restriction plugins to check whether they (a) change the price or
variation = variations_cache[i[1]] if i[1] is not None else None # (b) make the item/variation unavailable. If neither is the case, check_restriction
# will correctly return the default price
price = item.check_restrictions() if variation is None else variation.check_restrictions()
# Execute restriction plugins to check whether they (a) change the price or # Fetch all quotas. If there are no quotas, this item is not allowed to be sold.
# (b) make the item/variation unavailable. If neither is the case, check_restriction quotas = list(item.quotas.all()) if variation is None else list(variation.quotas.all())
# will correctly return the default price
price = item.check_restrictions() if variation is None else variation.check_restrictions()
# Fetch all quotas. If there are no quotas, this item is not allowed to be sold. if price is False or len(quotas) == 0 or not item.active:
quotas = list(item.quotas.all()) if variation is None else list(variation.quotas.all()) self.error_message(self.error_messages['unavailable'])
continue
if price is False or len(quotas) == 0 or not item.active: # Assume that all quotas allow us to buy i[2] instances of the object
self.error_message(self.error_messages['unavailable']) quota_ok = i[2]
continue for quota in quotas:
avail = quota.availability()
if avail[1] < i[2]:
# This quota is not available or less than i[2] items are left, so we have to
# reduce the number of bought items
self.error_message(
self.error_messages['unavailable']
if avail[0] != Quota.AVAILABILITY_OK
else self.error_messages['in_part']
)
quota_ok = min(quota_ok, avail[1])
# Assume that all quotas allow us to buy i[2] instances of the object # Create a CartPosition for as much items as we can
quota_ok = i[2] for k in range(quota_ok):
try: if len(i) > 3 and i[2] == 1:
for quota in quotas: # Recreating
# Lock the quota, so no other thread is allowed to perform sales covered by this cp = i[3].clone()
# quota while we're doing so. cp.expires = expiry
quota.lock() cp.price = price
avail = quota.availability() cp.save()
if avail[1] < i[2]: else:
# This quota is not available or less than i[2] items are left, so we have to CartPosition.objects.create(
# reduce the number of bought items event=self.request.event,
self.error_message( user=self.request.user,
self.error_messages['unavailable'] item=item,
if avail[0] != Quota.AVAILABILITY_OK variation=variation,
else self.error_messages['in_part'] price=price,
) expires=expiry
quota_ok = min(quota_ok, avail[1]) )
# Create a CartPosition for as much items as we can self._delete_expired()
for k in range(quota_ok):
if len(i) > 3 and i[2] == 1:
# Recreating
cp = i[3].clone()
cp.expires = expiry
cp.price = price
cp.save()
else:
CartPosition.objects.create(
event=self.request.event,
user=self.request.user,
item=item,
variation=variation,
price=price,
expires=expiry
)
except Quota.LockTimeoutException:
# Is raised when there are too many threads asking for quota locks and we were
# unaible to get one
self.error_message(self.error_messages['busy'], important=True)
finally:
# Release the locks. This is important ;)
for quota in quotas:
quota.release()
self._delete_expired() if not self.msg_some_unavailable:
messages.success(self.request, _('The products have been successfully added to your cart.'))
if not self.msg_some_unavailable: return redirect(self.get_success_url())
messages.success(self.request, _('The products have been successfully added to your cart.')) except EventLock.LockTimeoutException:
# Is raised when there are too many threads asking for quota locks and we were
return redirect(self.get_success_url()) # unaible to get one
self.error_message(self.error_messages['busy'], important=True)

2
src/pretix/presale/views/order.py
View File

@@ -94,7 +94,7 @@ class OrderDetails(EventViewMixin, EventLoginRequiredMixin, OrderDetailMixin,
ctx['can_retry'] = ( ctx['can_retry'] = (
self.payment_provider.order_can_retry(self.order) self.payment_provider.order_can_retry(self.order)
and self.payment_provider.is_enabled and self.payment_provider.is_enabled
and self.order._can_be_paid(keep_locked=False) and self.order._can_be_paid()
) )
elif self.order.status == Order.STATUS_PAID: elif self.order.status == Order.STATUS_PAID:
ctx['payment'] = self.payment_provider.order_paid_render(self.request, self.order) ctx['payment'] = self.payment_provider.order_paid_render(self.request, self.order)

4
src/requirements/production.txt
View File

@@ -4,8 +4,8 @@ python-dateutil>=2.4,<2.5
pytz pytz
django-bootstrap3>=6.1,<6.2 django-bootstrap3>=6.1,<6.2
-e git+https://github.com/pretix/django-formset-js.git@master#egg=django-formset-js -e git+https://github.com/pretix/django-formset-js.git@master#egg=django-formset-js
#cleanerversion>=1.5,<1.6 cleanerversion==1.5.3
-e git+https://github.com/pretix/cleanerversion.git@pretix#egg=CleanerVersion #-e git+https://github.com/pretix/cleanerversion.git@pretix#egg=CleanerVersion
django-compressor>=1.5,<2.0 django-compressor>=1.5,<2.0
reportlab>=3.1.44,<3.2 reportlab>=3.1.44,<3.2
-e git+https://github.com/pretix/PyPDF2.git@pretix#egg=PyPDF2 -e git+https://github.com/pretix/PyPDF2.git@pretix#egg=PyPDF2