From 69a80f2540e3537051a46ef30a1a799ce11affd6 Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Thu, 13 Jun 2019 12:18:39 +0200 Subject: [PATCH] Update tasks and cronjobs --- src/pretix/api/signals.py | 3 + src/pretix/api/webhooks.py | 90 +++++------ src/pretix/base/services/cart.py | 22 +-- src/pretix/base/services/cleanup.py | 4 + src/pretix/base/services/export.py | 7 +- src/pretix/base/services/invoices.py | 23 +-- src/pretix/base/services/mail.py | 141 +++++++++--------- src/pretix/base/services/notifications.py | 25 ++-- src/pretix/base/services/orders.py | 15 +- src/pretix/base/services/quotas.py | 2 + src/pretix/base/services/shredder.py | 12 +- src/pretix/base/services/tasks.py | 26 ++++ src/pretix/base/services/tickets.py | 17 ++- src/pretix/base/services/update_check.py | 2 + src/pretix/base/services/waitinglist.py | 9 +- src/pretix/plugins/badges/tasks.py | 5 +- src/pretix/plugins/badges/views.py | 2 +- src/pretix/plugins/banktransfer/tasks.py | 91 +++++------ src/pretix/plugins/sendmail/tasks.py | 7 +- src/pretix/plugins/stripe/tasks.py | 3 +- .../management/commands/updatestyles.py | 2 + src/pretix/presale/style.py | 50 +++---- 22 files changed, 307 insertions(+), 251 deletions(-) diff --git a/src/pretix/api/signals.py b/src/pretix/api/signals.py index 0fd2824e1..689cde048 100644 --- a/src/pretix/api/signals.py +++ b/src/pretix/api/signals.py @@ -2,6 +2,7 @@ from datetime import timedelta from django.dispatch import Signal, receiver from django.utils.timezone import now +from django_scopes import scopes_disabled from pretix.api.models import ApiCall, WebHookCall from pretix.base.signals import periodic_task @@ -17,10 +18,12 @@ instances. @receiver(periodic_task) +@scopes_disabled() def cleanup_webhook_logs(sender, **kwargs): WebHookCall.objects.filter(datetime__lte=now() - timedelta(days=30)).delete() @receiver(periodic_task) +@scopes_disabled() def cleanup_api_logs(sender, **kwargs): ApiCall.objects.filter(created__lte=now() - timedelta(hours=24)).delete() diff --git a/src/pretix/api/webhooks.py b/src/pretix/api/webhooks.py index cf5d3bb18..b9ae9ca4e 100644 --- a/src/pretix/api/webhooks.py +++ b/src/pretix/api/webhooks.py @@ -8,6 +8,7 @@ from celery.exceptions import MaxRetriesExceededError from django.db.models import Exists, OuterRef, Q from django.dispatch import receiver from django.utils.translation import ugettext_lazy as _ +from django_scopes import scope, scopes_disabled from requests import RequestException from pretix.api.models import WebHook, WebHookCall, WebHookEventListener @@ -203,51 +204,52 @@ def notify_webhooks(logentry_id: int): @app.task(base=ProfiledTask, bind=True, max_retries=9) def send_webhook(self, logentry_id: int, action_type: str, webhook_id: int): # 9 retries with 2**(2*x) timing is roughly 72 hours - logentry = LogEntry.all.get(id=logentry_id) - webhook = WebHook.objects.get(id=webhook_id) + with scopes_disabled(): + webhook = WebHook.objects.get(id=webhook_id) + with scope(organizer=webhook.organizer): + logentry = LogEntry.all.get(id=logentry_id) + types = get_all_webhook_events() + event_type = types.get(action_type) + if not event_type or not webhook.enabled: + return # Ignore, e.g. plugin not installed - types = get_all_webhook_events() - event_type = types.get(action_type) - if not event_type or not webhook.enabled: - return # Ignore, e.g. plugin not installed + payload = event_type.build_payload(logentry) + t = time.time() - payload = event_type.build_payload(logentry) - t = time.time() - - try: try: - resp = requests.post( - webhook.target_url, - json=payload, - allow_redirects=False - ) - WebHookCall.objects.create( - webhook=webhook, - action_type=logentry.action_type, - target_url=webhook.target_url, - is_retry=self.request.retries > 0, - execution_time=time.time() - t, - return_code=resp.status_code, - payload=json.dumps(payload), - response_body=resp.text[:1024 * 1024], - success=200 <= resp.status_code <= 299 - ) - if resp.status_code == 410: - webhook.enabled = False - webhook.save() - elif resp.status_code > 299: + try: + resp = requests.post( + webhook.target_url, + json=payload, + allow_redirects=False + ) + WebHookCall.objects.create( + webhook=webhook, + action_type=logentry.action_type, + target_url=webhook.target_url, + is_retry=self.request.retries > 0, + execution_time=time.time() - t, + return_code=resp.status_code, + payload=json.dumps(payload), + response_body=resp.text[:1024 * 1024], + success=200 <= resp.status_code <= 299 + ) + if resp.status_code == 410: + webhook.enabled = False + webhook.save() + elif resp.status_code > 299: + raise self.retry(countdown=2 ** (self.request.retries * 2)) + except RequestException as e: + WebHookCall.objects.create( + webhook=webhook, + action_type=logentry.action_type, + target_url=webhook.target_url, + is_retry=self.request.retries > 0, + execution_time=time.time() - t, + return_code=0, + payload=json.dumps(payload), + response_body=str(e)[:1024 * 1024] + ) raise self.retry(countdown=2 ** (self.request.retries * 2)) - except RequestException as e: - WebHookCall.objects.create( - webhook=webhook, - action_type=logentry.action_type, - target_url=webhook.target_url, - is_retry=self.request.retries > 0, - execution_time=time.time() - t, - return_code=0, - payload=json.dumps(payload), - response_body=str(e)[:1024 * 1024] - ) - raise self.retry(countdown=2 ** (self.request.retries * 2)) - except MaxRetriesExceededError: - pass + except MaxRetriesExceededError: + pass diff --git a/src/pretix/base/services/cart.py b/src/pretix/base/services/cart.py index ae003fdd9..b767b08da 100644 --- a/src/pretix/base/services/cart.py +++ b/src/pretix/base/services/cart.py @@ -23,7 +23,7 @@ from pretix.base.reldate import RelativeDateWrapper from pretix.base.services.checkin import _save_answers from pretix.base.services.locking import LockTimeoutException, NoLockManager from pretix.base.services.pricing import get_price -from pretix.base.services.tasks import ProfiledTask +from pretix.base.services.tasks import ProfiledEventTask from pretix.base.settings import PERSON_NAME_SCHEMES from pretix.base.templatetags.rich_text import rich_text from pretix.celery_app import app @@ -902,7 +902,7 @@ def get_fees(event, request, total, invoice_address, provider): return fees -@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,)) +@app.task(base=ProfiledEventTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,)) def add_items_to_cart(self, event: int, items: List[dict], cart_id: str=None, locale='en', invoice_address: int=None, widget_data=None, sales_channel='web') -> None: """ @@ -913,8 +913,6 @@ def add_items_to_cart(self, event: int, items: List[dict], cart_id: str=None, lo :raises CartError: On any error that occured """ with language(locale): - event = Event.objects.get(id=event) - ia = False if invoice_address: try: @@ -934,8 +932,8 @@ def add_items_to_cart(self, event: int, items: List[dict], cart_id: str=None, lo raise CartError(error_messages['busy']) -@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,)) -def remove_cart_position(self, event: int, position: int, cart_id: str=None, locale='en') -> None: +@app.task(base=ProfiledEventTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,)) +def remove_cart_position(self, event: Event, position: int, cart_id: str=None, locale='en') -> None: """ Removes a list of items from a user's cart. :param event: The event ID in question @@ -943,7 +941,6 @@ def remove_cart_position(self, event: int, position: int, cart_id: str=None, loc :param session: Session ID of a guest """ with language(locale): - event = Event.objects.get(id=event) try: try: cm = CartManager(event=event, cart_id=cart_id) @@ -955,15 +952,14 @@ def remove_cart_position(self, event: int, position: int, cart_id: str=None, loc raise CartError(error_messages['busy']) -@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,)) -def clear_cart(self, event: int, cart_id: str=None, locale='en') -> None: +@app.task(base=ProfiledEventTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,)) +def clear_cart(self, event: Event, cart_id: str=None, locale='en') -> None: """ Removes a list of items from a user's cart. :param event: The event ID in question :param session: Session ID of a guest """ with language(locale): - event = Event.objects.get(id=event) try: try: cm = CartManager(event=event, cart_id=cart_id) @@ -975,8 +971,8 @@ def clear_cart(self, event: int, cart_id: str=None, locale='en') -> None: raise CartError(error_messages['busy']) -@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,)) -def set_cart_addons(self, event: int, addons: List[dict], cart_id: str=None, locale='en', +@app.task(base=ProfiledEventTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,)) +def set_cart_addons(self, event: Event, addons: List[dict], cart_id: str=None, locale='en', invoice_address: int=None, sales_channel='web') -> None: """ Removes a list of items from a user's cart. @@ -985,8 +981,6 @@ def set_cart_addons(self, event: int, addons: List[dict], cart_id: str=None, loc :param session: Session ID of a guest """ with language(locale): - event = Event.objects.get(id=event) - ia = False if invoice_address: try: diff --git a/src/pretix/base/services/cleanup.py b/src/pretix/base/services/cleanup.py index 3a3ba4b0d..330689ad2 100644 --- a/src/pretix/base/services/cleanup.py +++ b/src/pretix/base/services/cleanup.py @@ -2,6 +2,7 @@ from datetime import timedelta from django.dispatch import receiver from django.utils.timezone import now +from django_scopes import scopes_disabled from pretix.base.models import CachedCombinedTicket, CachedTicket @@ -10,6 +11,7 @@ from ..signals import periodic_task @receiver(signal=periodic_task) +@scopes_disabled() def clean_cart_positions(sender, **kwargs): for cp in CartPosition.objects.filter(expires__lt=now() - timedelta(days=14), addon_to__isnull=False): cp.delete() @@ -20,12 +22,14 @@ def clean_cart_positions(sender, **kwargs): @receiver(signal=periodic_task) +@scopes_disabled() def clean_cached_files(sender, **kwargs): for cf in CachedFile.objects.filter(expires__isnull=False, expires__lt=now()): cf.delete() @receiver(signal=periodic_task) +@scopes_disabled() def clean_cached_tickets(sender, **kwargs): for cf in CachedTicket.objects.filter(created__lte=now() - timedelta(days=30)): cf.delete() diff --git a/src/pretix/base/services/export.py b/src/pretix/base/services/export.py index 60ee1015e..b547d62f3 100644 --- a/src/pretix/base/services/export.py +++ b/src/pretix/base/services/export.py @@ -6,7 +6,7 @@ from django.utils.translation import ugettext from pretix.base.i18n import LazyLocaleException, language from pretix.base.models import CachedFile, Event, cachedfile_name -from pretix.base.services.tasks import ProfiledTask +from pretix.base.services.tasks import ProfiledEventTask from pretix.base.signals import register_data_exporters from pretix.celery_app import app @@ -15,9 +15,8 @@ class ExportError(LazyLocaleException): pass -@app.task(base=ProfiledTask, throws=(ExportError,)) -def export(event: str, fileid: str, provider: str, form_data: Dict[str, Any]) -> None: - event = Event.objects.get(id=event) +@app.task(base=ProfiledEventTask, throws=(ExportError,)) +def export(event: Event, fileid: str, provider: str, form_data: Dict[str, Any]) -> None: file = CachedFile.objects.get(id=fileid) with language(event.settings.locale), override(event.settings.timezone): responses = register_data_exporters.send(event) diff --git a/src/pretix/base/services/invoices.py b/src/pretix/base/services/invoices.py index 803bba212..979148de5 100644 --- a/src/pretix/base/services/invoices.py +++ b/src/pretix/base/services/invoices.py @@ -15,6 +15,7 @@ from django.utils import timezone from django.utils.timezone import now from django.utils.translation import pgettext, ugettext as _ from django_countries.fields import Country +from django_scopes import scope, scopes_disabled from i18nfield.strings import LazyI18nString from pretix.base.i18n import language @@ -244,16 +245,18 @@ def generate_invoice(order: Order, trigger_pdf=True): @app.task(base=TransactionAwareTask) def invoice_pdf_task(invoice: int): - i = Invoice.objects.get(pk=invoice) - if i.shredded: - return None - if i.file: - i.file.delete() - with language(i.locale): - fname, ftype, fcontent = i.event.invoice_renderer.generate(i) - i.file.save(fname, ContentFile(fcontent)) - i.save() - return i.file.name + with scopes_disabled(): + i = Invoice.objects.get(pk=invoice) + with scope(organizer=i.order.event.organizer): + if i.shredded: + return None + if i.file: + i.file.delete() + with language(i.locale): + fname, ftype, fcontent = i.event.invoice_renderer.generate(i) + i.file.save(fname, ContentFile(fcontent)) + i.save() + return i.file.name def invoice_qualified(order: Order): diff --git a/src/pretix/base/services/mail.py b/src/pretix/base/services/mail.py index 7e979842f..d21758e8c 100644 --- a/src/pretix/base/services/mail.py +++ b/src/pretix/base/services/mail.py @@ -10,6 +10,7 @@ from django.conf import settings from django.core.mail import EmailMultiAlternatives, get_connection from django.template.loader import get_template from django.utils.translation import ugettext as _ +from django_scopes import scope, scopes_disabled from i18nfield.strings import LazyI18nString from pretix.base.email import ClassicMailRenderer @@ -234,83 +235,87 @@ def mail_send_task(self, *args, to: List[str], subject: str, body: str, html: st pass if event: - event = Event.objects.get(id=event) + with scopes_disabled(): + event = Event.objects.get(id=event) backend = event.get_mail_backend() + cm = lambda: scope(organizer=event.organizer) # noqa else: backend = get_connection(fail_silently=False) + cm = lambda: scopes_disabled() # noqa - if event: - if order: - try: - order = event.orders.get(pk=order) - except Order.DoesNotExist: - order = None - else: - if position: - try: - position = order.positions.get(pk=position) - except OrderPosition.DoesNotExist: - attach_tickets = False - if attach_tickets: - args = [] - attach_size = 0 - for name, ct in get_tickets_for_order(order, base_position=position): - content = ct.file.read() - args.append((name, content, ct.type)) - attach_size += len(content) + with cm(): + if event: + if order: + try: + order = event.orders.get(pk=order) + except Order.DoesNotExist: + order = None + else: + if position: + try: + position = order.positions.get(pk=position) + except OrderPosition.DoesNotExist: + attach_tickets = False + if attach_tickets: + args = [] + attach_size = 0 + for name, ct in get_tickets_for_order(order, base_position=position): + content = ct.file.read() + args.append((name, content, ct.type)) + attach_size += len(content) - if attach_size < 4 * 1024 * 1024: - # Do not attach more than 4MB, it will bounce way to often. - for a in args: - try: - email.attach(*a) - except: - pass - else: - order.log_action( - 'pretix.event.order.email.attachments.skipped', - data={ - 'subject': 'Attachments skipped', - 'message': 'Attachment have not been send because {} bytes are likely too large to arrive.'.format(attach_size), - 'recipient': '', - 'invoices': [], - } - ) + if attach_size < 4 * 1024 * 1024: + # Do not attach more than 4MB, it will bounce way to often. + for a in args: + try: + email.attach(*a) + except: + pass + else: + order.log_action( + 'pretix.event.order.email.attachments.skipped', + data={ + 'subject': 'Attachments skipped', + 'message': 'Attachment have not been send because {} bytes are likely too large to arrive.'.format(attach_size), + 'recipient': '', + 'invoices': [], + } + ) - email = email_filter.send_chained(event, 'message', message=email, order=order) + email = email_filter.send_chained(event, 'message', message=email, order=order) - try: - backend.send_messages([email]) - except smtplib.SMTPResponseException as e: - if e.smtp_code in (101, 111, 421, 422, 431, 442, 447, 452): - self.retry(max_retries=5, countdown=2 ** (self.request.retries * 2)) - logger.exception('Error sending email') + try: + backend.send_messages([email]) + except smtplib.SMTPResponseException as e: + if e.smtp_code in (101, 111, 421, 422, 431, 442, 447, 452): + self.retry(max_retries=5, countdown=2 ** (self.request.retries * 2)) + logger.exception('Error sending email') - if order: - order.log_action( - 'pretix.event.order.email.error', - data={ - 'subject': 'SMTP code {}'.format(e.smtp_code), - 'message': e.smtp_error.decode() if isinstance(e.smtp_error, bytes) else str(e.smtp_error), - 'recipient': '', - 'invoices': [], - } - ) + if order: + order.log_action( + 'pretix.event.order.email.error', + data={ + 'subject': 'SMTP code {}'.format(e.smtp_code), + 'message': e.smtp_error.decode() if isinstance(e.smtp_error, bytes) else str(e.smtp_error), + 'recipient': '', + 'invoices': [], + } + ) - raise SendMailException('Failed to send an email to {}.'.format(to)) - except Exception as e: - if order: - order.log_action( - 'pretix.event.order.email.error', - data={ - 'subject': 'Internal error', - 'message': str(e), - 'recipient': '', - 'invoices': [], - } - ) - logger.exception('Error sending email') - raise SendMailException('Failed to send an email to {}.'.format(to)) + raise SendMailException('Failed to send an email to {}.'.format(to)) + except Exception as e: + if order: + order.log_action( + 'pretix.event.order.email.error', + data={ + 'subject': 'Internal error', + 'message': str(e), + 'recipient': '', + 'invoices': [], + } + ) + logger.exception('Error sending email') + raise SendMailException('Failed to send an email to {}.'.format(to)) def mail_send(*args, **kwargs): diff --git a/src/pretix/base/services/notifications.py b/src/pretix/base/services/notifications.py index 9c5460b75..35327ead0 100644 --- a/src/pretix/base/services/notifications.py +++ b/src/pretix/base/services/notifications.py @@ -1,5 +1,6 @@ from django.conf import settings from django.template.loader import get_template +from django_scopes import scope, scopes_disabled from inlinestyler.utils import inline_css from pretix.base.i18n import language @@ -12,6 +13,7 @@ from pretix.helpers.urls import build_absolute_uri @app.task(base=TransactionAwareTask) +@scopes_disabled() def notify(logentry_id: int): logentry = LogEntry.all.get(id=logentry_id) if not logentry.event: @@ -66,17 +68,22 @@ def notify(logentry_id: int): @app.task(base=ProfiledTask) def send_notification(logentry_id: int, action_type: str, user_id: int, method: str): logentry = LogEntry.all.get(id=logentry_id) - user = User.objects.get(id=user_id) - types = get_all_notification_types(logentry.event) - notification_type = types.get(action_type) - if not notification_type: - return # Ignore, e.g. plugin not active for this event + if logentry.event: + sm = lambda: scope(organizer=logentry.event.organizer) # noqa + else: + sm = lambda: scopes_disabled() # noqa + with sm(): + user = User.objects.get(id=user_id) + types = get_all_notification_types(logentry.event) + notification_type = types.get(action_type) + if not notification_type: + return # Ignore, e.g. plugin not active for this event - with language(user.locale): - notification = notification_type.build_notification(logentry) + with language(user.locale): + notification = notification_type.build_notification(logentry) - if method == "mail": - send_notification_mail(notification, user) + if method == "mail": + send_notification_mail(notification, user) def send_notification_mail(notification: Notification, user: User): diff --git a/src/pretix/base/services/orders.py b/src/pretix/base/services/orders.py index 811d2773a..7e069def3 100644 --- a/src/pretix/base/services/orders.py +++ b/src/pretix/base/services/orders.py @@ -16,6 +16,7 @@ from django.utils.formats import date_format from django.utils.functional import cached_property from django.utils.timezone import make_aware, now from django.utils.translation import ugettext as _ +from django_scopes import scopes_disabled from pretix.api.models import OAuthApplication from pretix.base.i18n import ( @@ -42,7 +43,7 @@ from pretix.base.services.invoices import ( from pretix.base.services.locking import LockTimeoutException, NoLockManager from pretix.base.services.mail import SendMailException from pretix.base.services.pricing import get_price -from pretix.base.services.tasks import ProfiledTask +from pretix.base.services.tasks import ProfiledEventTask, ProfiledTask from pretix.base.settings import PERSON_NAME_SCHEMES from pretix.base.signals import ( allow_ticket_download, order_approved, order_canceled, order_changed, @@ -715,10 +716,8 @@ def _order_placed_email_attendee(event: Event, order: Order, position: OrderPosi logger.exception('Order received email could not be sent to attendee') -def _perform_order(event: str, payment_provider: str, position_ids: List[str], +def _perform_order(event: Event, payment_provider: str, position_ids: List[str], email: str, locale: str, address: int, meta_info: dict=None, sales_channel: str='web'): - - event = Event.objects.get(id=event) if payment_provider: pprov = event.get_payment_providers().get(payment_provider) if not pprov: @@ -804,6 +803,7 @@ def _perform_order(event: str, payment_provider: str, position_ids: List[str], @receiver(signal=periodic_task) +@scopes_disabled() def expire_orders(sender, **kwargs): eventcache = {} @@ -818,6 +818,7 @@ def expire_orders(sender, **kwargs): @receiver(signal=periodic_task) +@scopes_disabled() def send_expiry_warnings(sender, **kwargs): eventcache = {} today = now().replace(hour=0, minute=0, second=0) @@ -875,6 +876,7 @@ def send_expiry_warnings(sender, **kwargs): @receiver(signal=periodic_task) +@scopes_disabled() def send_download_reminders(sender, **kwargs): today = now().replace(hour=0, minute=0, second=0, microsecond=0) @@ -1497,8 +1499,8 @@ class OrderChangeManager: return pprov -@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1, throws=(OrderError,)) -def perform_order(self, event: str, payment_provider: str, positions: List[str], +@app.task(base=ProfiledEventTask, bind=True, max_retries=5, default_retry_delay=1, throws=(OrderError,)) +def perform_order(self, event: Event, payment_provider: str, positions: List[str], email: str=None, locale: str=None, address: int=None, meta_info: dict=None, sales_channel: str='web'): with language(locale): @@ -1513,6 +1515,7 @@ def perform_order(self, event: str, payment_provider: str, positions: List[str], @app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1, throws=(OrderError,)) +@scopes_disabled() def cancel_order(self, order: int, user: int=None, send_mail: bool=True, api_token=None, oauth_application=None, device=None, cancellation_fee=None, try_auto_refund=False): try: diff --git a/src/pretix/base/services/quotas.py b/src/pretix/base/services/quotas.py index 76dab041a..0a182f877 100644 --- a/src/pretix/base/services/quotas.py +++ b/src/pretix/base/services/quotas.py @@ -4,6 +4,7 @@ from django.conf import settings from django.db.models import Max, Q from django.dispatch import receiver from django.utils.timezone import now +from django_scopes import scopes_disabled from pretix.base.models import Event, LogEntry from pretix.celery_app import app @@ -17,6 +18,7 @@ def build_all_quota_caches(sender, **kwargs): @app.task +@scopes_disabled() def refresh_quota_caches(): # Active events active = LogEntry.objects.using(settings.DATABASE_REPLICA).filter( diff --git a/src/pretix/base/services/shredder.py b/src/pretix/base/services/shredder.py index e857d19ba..9956c5d22 100644 --- a/src/pretix/base/services/shredder.py +++ b/src/pretix/base/services/shredder.py @@ -11,14 +11,13 @@ from django.utils.timezone import now from django.utils.translation import ugettext_lazy as _ from pretix.base.models import CachedFile, Event, cachedfile_name -from pretix.base.services.tasks import ProfiledTask +from pretix.base.services.tasks import ProfiledEventTask from pretix.base.shredder import ShredError from pretix.celery_app import app -@app.task(base=ProfiledTask) -def export(event: str, shredders: List[str]) -> None: - event = Event.objects.get(id=event) +@app.task(base=ProfiledEventTask) +def export(event: Event, shredders: List[str]) -> None: known_shredders = event.get_data_shredders() with NamedTemporaryFile() as rawfile: @@ -63,9 +62,8 @@ def export(event: str, shredders: List[str]) -> None: return cf.pk -@app.task(base=ProfiledTask, throws=(ShredError,)) -def shred(event: str, fileid: str, confirm_code: str) -> None: - event = Event.objects.get(id=event) +@app.task(base=ProfiledEventTask, throws=(ShredError,)) +def shred(event: Event, fileid: str, confirm_code: str) -> None: known_shredders = event.get_data_shredders() try: cf = CachedFile.objects.get(pk=fileid) diff --git a/src/pretix/base/services/tasks.py b/src/pretix/base/services/tasks.py index 145a88ae0..520405578 100644 --- a/src/pretix/base/services/tasks.py +++ b/src/pretix/base/services/tasks.py @@ -14,10 +14,12 @@ import time from django.conf import settings from django.db import transaction +from django_scopes import scope, scopes_disabled from pretix.base.metrics import ( pretix_task_duration_seconds, pretix_task_runs_total, ) +from pretix.base.models import Event from pretix.celery_app import app @@ -61,6 +63,30 @@ class ProfiledTask(app.Task): return super().on_success(retval, task_id, args, kwargs) +class EventTask(app.Task): + def __call__(self, *args, **kwargs): + if 'event_id' in kwargs: + event_id = kwargs.get('event_id') + with scopes_disabled(): + event = Event.objects.select_related('organizer').get(pk=event_id) + del kwargs['event_id'] + kwargs['event'] = event + else: + args = list(args) + event_id = args[0] + with scopes_disabled(): + event = Event.objects.select_related('organizer').get(pk=event_id) + args[0] = event + + with scope(organizer=event.organizer): + ret = super().__call__(*args, **kwargs) + return ret + + +class ProfiledEventTask(ProfiledTask, EventTask): + pass + + class TransactionAwareTask(ProfiledTask): """ Task class which is aware of django db transactions and only executes tasks diff --git a/src/pretix/base/services/tickets.py b/src/pretix/base/services/tickets.py index 2b006e89c..bb327888d 100644 --- a/src/pretix/base/services/tickets.py +++ b/src/pretix/base/services/tickets.py @@ -4,13 +4,14 @@ import os from django.core.files.base import ContentFile from django.utils.timezone import now from django.utils.translation import ugettext as _ +from django_scopes import scopes_disabled from pretix.base.i18n import language from pretix.base.models import ( CachedCombinedTicket, CachedTicket, Event, InvoiceAddress, Order, OrderPosition, ) -from pretix.base.services.tasks import ProfiledTask +from pretix.base.services.tasks import EventTask, ProfiledTask from pretix.base.settings import PERSON_NAME_SCHEMES from pretix.base.signals import allow_ticket_download, register_ticket_outputs from pretix.celery_app import app @@ -57,10 +58,11 @@ def generate_order(order: int, provider: str): @app.task(base=ProfiledTask) def generate(model: str, pk: int, provider: str): - if model == 'order': - return generate_order(pk, provider) - elif model == 'orderposition': - return generate_orderposition(pk, provider) + with scopes_disabled(): + if model == 'order': + return generate_order(pk, provider) + elif model == 'orderposition': + return generate_orderposition(pk, provider) class DummyRollbackException(Exception): @@ -165,9 +167,8 @@ def get_tickets_for_order(order, base_position=None): return tickets -@app.task(base=ProfiledTask) -def invalidate_cache(event: int, item: int=None, provider: str=None, order: int=None, **kwargs): - event = Event.objects.get(id=event) +@app.task(base=EventTask) +def invalidate_cache(event: Event, item: int=None, provider: str=None, order: int=None, **kwargs): qs = CachedTicket.objects.filter(order_position__order__event=event) qsc = CachedCombinedTicket.objects.filter(order__event=event) diff --git a/src/pretix/base/services/update_check.py b/src/pretix/base/services/update_check.py index a29df018e..eba972634 100644 --- a/src/pretix/base/services/update_check.py +++ b/src/pretix/base/services/update_check.py @@ -6,6 +6,7 @@ import requests from django.dispatch import receiver from django.utils.timezone import now from django.utils.translation import ugettext_lazy as _, ugettext_noop +from django_scopes import scopes_disabled from i18nfield.strings import LazyI18nString from pretix import __version__ @@ -29,6 +30,7 @@ def run_update_check(sender, **kwargs): @app.task +@scopes_disabled() def update_check(): gs = GlobalSettingsObject() diff --git a/src/pretix/base/services/waitinglist.py b/src/pretix/base/services/waitinglist.py index 0241e3476..ec711e7d8 100644 --- a/src/pretix/base/services/waitinglist.py +++ b/src/pretix/base/services/waitinglist.py @@ -1,17 +1,17 @@ import sys from django.dispatch import receiver +from django_scopes import scopes_disabled from pretix.base.models import Event, User, WaitingListEntry from pretix.base.models.waitinglist import WaitingListException -from pretix.base.services.tasks import ProfiledTask +from pretix.base.services.tasks import EventTask from pretix.base.signals import periodic_task from pretix.celery_app import app -@app.task(base=ProfiledTask) -def assign_automatically(event_id: int, user_id: int=None, subevent_id: int=None): - event = Event.objects.get(id=event_id) +@app.task(base=EventTask) +def assign_automatically(event: Event, user_id: int=None, subevent_id: int=None): if user_id: user = User.objects.get(id=user_id) else: @@ -69,6 +69,7 @@ def assign_automatically(event_id: int, user_id: int=None, subevent_id: int=None @receiver(signal=periodic_task) +@scopes_disabled() def process_waitinglist(sender, **kwargs): qs = Event.objects.filter( live=True diff --git a/src/pretix/plugins/badges/tasks.py b/src/pretix/plugins/badges/tasks.py index a1cd1a017..15085ca1c 100644 --- a/src/pretix/plugins/badges/tasks.py +++ b/src/pretix/plugins/badges/tasks.py @@ -7,6 +7,7 @@ from pretix.base.models import ( CachedFile, Event, OrderPosition, cachedfile_name, ) from pretix.base.services.orders import OrderError +from pretix.base.services.tasks import EventTask from pretix.celery_app import app from .exporters import render_pdf @@ -14,8 +15,8 @@ from .exporters import render_pdf logger = logging.getLogger(__name__) -@app.task(throws=(OrderError,)) -def badges_create_pdf(fileid: int, event: int, positions: List[int]) -> int: +@app.task(base=EventTask, throws=(OrderError,)) +def badges_create_pdf(event: int, fileid: int, positions: List[int]) -> int: file = CachedFile.objects.get(id=fileid) event = Event.objects.get(id=event) diff --git a/src/pretix/plugins/badges/views.py b/src/pretix/plugins/badges/views.py index a1911a89f..bb79670fc 100644 --- a/src/pretix/plugins/badges/views.py +++ b/src/pretix/plugins/badges/views.py @@ -223,7 +223,7 @@ class OrderPrintDo(EventPermissionRequiredMixin, AsyncAction, View): else: positions = [p.pk for p in order.positions.all()] return self.do( - str(cf.id), self.request.event.pk, + str(cf.id), positions, ) diff --git a/src/pretix/plugins/banktransfer/tasks.py b/src/pretix/plugins/banktransfer/tasks.py index 8d19e6fa1..356fcbff8 100644 --- a/src/pretix/plugins/banktransfer/tasks.py +++ b/src/pretix/plugins/banktransfer/tasks.py @@ -9,6 +9,7 @@ from django.db import transaction from django.db.models import Q from django.utils.formats import date_format from django.utils.translation import ugettext, ugettext_noop +from django_scopes import scope, scopes_disabled from pretix.base.i18n import language from pretix.base.models import ( @@ -194,51 +195,53 @@ def _get_unknown_transactions(job: BankImportJob, data: list, event: Event=None, @app.task(base=TransactionAwareTask, bind=True, max_retries=5, default_retry_delay=1) def process_banktransfers(self, job: int, data: list) -> None: with language("en"): # We'll translate error messages at display time - job = BankImportJob.objects.get(pk=job) - job.state = BankImportJob.STATE_RUNNING - job.save() - prefixes = [] + with scopes_disabled(): + job = BankImportJob.objects.get(pk=job) + with scope(organizer=job.organizer or job.event.organizer): + job.state = BankImportJob.STATE_RUNNING + job.save() + prefixes = [] - try: - # Delete left-over transactions from a failed run before so they can reimported - BankTransaction.objects.filter(state=BankTransaction.STATE_UNCHECKED, **job.owner_kwargs).delete() - - transactions = _get_unknown_transactions(job, data, **job.owner_kwargs) - - code_len = settings.ENTROPY['order_code'] - if job.event: - pattern = re.compile(job.event.slug.upper() + r"[ \-_]*([A-Z0-9]{%s})" % code_len) - else: - if not prefixes: - prefixes = [e.slug.upper().replace(".", r"\.").replace("-", r"[\- ]*") - for e in job.organizer.events.all()] - pattern = re.compile("(%s)[ \\-_]*([A-Z0-9]{%s})" % ("|".join(prefixes), code_len)) - - for trans in transactions: - match = pattern.search(trans.reference.replace(" ", "").replace("\n", "").upper()) - - if match: - if job.event: - code = match.group(1) - _handle_transaction(trans, code, event=job.event) - else: - slug = match.group(1) - code = match.group(2) - _handle_transaction(trans, code, organizer=job.organizer, slug=slug) - else: - trans.state = BankTransaction.STATE_NOMATCH - trans.save() - except LockTimeoutException: try: - self.retry() - except MaxRetriesExceededError: - logger.exception('Maximum number of retries exceeded for task.') + # Delete left-over transactions from a failed run before so they can reimported + BankTransaction.objects.filter(state=BankTransaction.STATE_UNCHECKED, **job.owner_kwargs).delete() + + transactions = _get_unknown_transactions(job, data, **job.owner_kwargs) + + code_len = settings.ENTROPY['order_code'] + if job.event: + pattern = re.compile(job.event.slug.upper() + r"[ \-_]*([A-Z0-9]{%s})" % code_len) + else: + if not prefixes: + prefixes = [e.slug.upper().replace(".", r"\.").replace("-", r"[\- ]*") + for e in job.organizer.events.all()] + pattern = re.compile("(%s)[ \\-_]*([A-Z0-9]{%s})" % ("|".join(prefixes), code_len)) + + for trans in transactions: + match = pattern.search(trans.reference.replace(" ", "").replace("\n", "").upper()) + + if match: + if job.event: + code = match.group(1) + _handle_transaction(trans, code, event=job.event) + else: + slug = match.group(1) + code = match.group(2) + _handle_transaction(trans, code, organizer=job.organizer, slug=slug) + else: + trans.state = BankTransaction.STATE_NOMATCH + trans.save() + except LockTimeoutException: + try: + self.retry() + except MaxRetriesExceededError: + logger.exception('Maximum number of retries exceeded for task.') + job.state = BankImportJob.STATE_ERROR + job.save() + except Exception as e: job.state = BankImportJob.STATE_ERROR job.save() - except Exception as e: - job.state = BankImportJob.STATE_ERROR - job.save() - raise e - else: - job.state = BankImportJob.STATE_COMPLETED - job.save() + raise e + else: + job.state = BankImportJob.STATE_COMPLETED + job.save() diff --git a/src/pretix/plugins/sendmail/tasks.py b/src/pretix/plugins/sendmail/tasks.py index 245cf205d..f56b3f245 100644 --- a/src/pretix/plugins/sendmail/tasks.py +++ b/src/pretix/plugins/sendmail/tasks.py @@ -5,15 +5,14 @@ from i18nfield.strings import LazyI18nString from pretix.base.i18n import language from pretix.base.models import Event, InvoiceAddress, Order, User from pretix.base.services.mail import SendMailException, mail -from pretix.base.services.tasks import ProfiledTask +from pretix.base.services.tasks import ProfiledEventTask from pretix.celery_app import app from pretix.multidomain.urlreverse import build_absolute_uri -@app.task(base=ProfiledTask) -def send_mails(event: int, user: int, subject: dict, message: dict, orders: list, items: list, recipients: str) -> None: +@app.task(base=ProfiledEventTask) +def send_mails(event: Event, user: int, subject: dict, message: dict, orders: list, items: list, recipients: str) -> None: failures = [] - event = Event.objects.get(pk=event) user = User.objects.get(pk=user) if user else None orders = Order.objects.filter(pk__in=orders, event=event) subject = LazyI18nString(subject) diff --git a/src/pretix/plugins/stripe/tasks.py b/src/pretix/plugins/stripe/tasks.py index 44d55b02f..afd6c6bbc 100644 --- a/src/pretix/plugins/stripe/tasks.py +++ b/src/pretix/plugins/stripe/tasks.py @@ -5,6 +5,7 @@ import stripe from django.conf import settings from pretix.base.models import Event +from pretix.base.services.tasks import EventTask from pretix.celery_app import app from pretix.multidomain.urlreverse import get_domain from pretix.plugins.stripe.models import RegisteredApplePayDomain @@ -27,7 +28,7 @@ def get_stripe_account_key(prov): return prov.settings.publishable_key -@app.task(max_retries=5, default_retry_delay=1) +@app.task(base=EventTask, max_retries=5, default_retry_delay=1) def stripe_verify_domain(event_id, domain): from pretix.plugins.stripe.payment import StripeCC event = Event.objects.get(pk=event_id) diff --git a/src/pretix/presale/management/commands/updatestyles.py b/src/pretix/presale/management/commands/updatestyles.py index 3d4e708e0..993ece380 100644 --- a/src/pretix/presale/management/commands/updatestyles.py +++ b/src/pretix/presale/management/commands/updatestyles.py @@ -4,6 +4,7 @@ from django.conf import settings from django.core.files.base import ContentFile, File from django.core.files.storage import default_storage from django.core.management.base import BaseCommand +from django_scopes import scopes_disabled from pretix.base.models import Event_SettingsStore, Organizer_SettingsStore from pretix.base.settings import GlobalSettingsObject @@ -15,6 +16,7 @@ from ...style import regenerate_css, regenerate_organizer_css class Command(BaseCommand): help = "Re-generate all custom stylesheets and scripts" + @scopes_disabled() def handle(self, *args, **options): for es in Organizer_SettingsStore.objects.filter(key="presale_css_file"): regenerate_organizer_css.apply_async(args=(es.object_id,)) diff --git a/src/pretix/presale/style.py b/src/pretix/presale/style.py index a7fff3e4b..76deabefc 100644 --- a/src/pretix/presale/style.py +++ b/src/pretix/presale/style.py @@ -11,9 +11,10 @@ from django.core.files.base import ContentFile from django.core.files.storage import default_storage from django.dispatch import Signal from django.templatetags.static import static as _static +from django_scopes import scope from pretix.base.models import Event, Event_SettingsStore, Organizer -from pretix.base.services.tasks import ProfiledTask +from pretix.base.services.tasks import ProfiledEventTask, ProfiledTask from pretix.celery_app import app from pretix.multidomain.urlreverse import get_domain from pretix.presale.signals import sass_postamble, sass_preamble @@ -78,10 +79,8 @@ def compile_scss(object, file="main.scss", fonts=True): return css, checksum -@app.task(base=ProfiledTask) -def regenerate_css(event_id: int): - event = Event.objects.select_related('organizer').get(pk=event_id) - +@app.task(base=ProfiledEventTask) +def regenerate_css(event): # main.scss css, checksum = compile_scss(event) fname = 'pub/{}/{}/presale.{}.css'.format(event.organizer.slug, event.slug, checksum[:16]) @@ -105,28 +104,29 @@ def regenerate_css(event_id: int): def regenerate_organizer_css(organizer_id: int): organizer = Organizer.objects.get(pk=organizer_id) - # main.scss - css, checksum = compile_scss(organizer) - fname = 'pub/{}/presale.{}.css'.format(organizer.slug, checksum[:16]) - if organizer.settings.get('presale_css_checksum', '') != checksum: - newname = default_storage.save(fname, ContentFile(css.encode('utf-8'))) - organizer.settings.set('presale_css_file', newname) - organizer.settings.set('presale_css_checksum', checksum) + with scope(organizer=organizer): + # main.scss + css, checksum = compile_scss(organizer) + fname = 'pub/{}/presale.{}.css'.format(organizer.slug, checksum[:16]) + if organizer.settings.get('presale_css_checksum', '') != checksum: + newname = default_storage.save(fname, ContentFile(css.encode('utf-8'))) + organizer.settings.set('presale_css_file', newname) + organizer.settings.set('presale_css_checksum', checksum) - # widget.scss - css, checksum = compile_scss(organizer, file='widget.scss', fonts=False) - fname = 'pub/{}/widget.{}.css'.format(organizer.slug, checksum[:16]) - if organizer.settings.get('presale_widget_css_checksum', '') != checksum: - newname = default_storage.save(fname, ContentFile(css.encode('utf-8'))) - organizer.settings.set('presale_widget_css_file', newname) - organizer.settings.set('presale_widget_css_checksum', checksum) + # widget.scss + css, checksum = compile_scss(organizer, file='widget.scss', fonts=False) + fname = 'pub/{}/widget.{}.css'.format(organizer.slug, checksum[:16]) + if organizer.settings.get('presale_widget_css_checksum', '') != checksum: + newname = default_storage.save(fname, ContentFile(css.encode('utf-8'))) + organizer.settings.set('presale_widget_css_file', newname) + organizer.settings.set('presale_widget_css_checksum', checksum) - non_inherited_events = set(Event_SettingsStore.objects.filter( - object__organizer=organizer, key__in=affected_keys - ).values_list('object_id', flat=True)) - for event in organizer.events.all(): - if event.pk not in non_inherited_events: - regenerate_css.apply_async(args=(event.pk,)) + non_inherited_events = set(Event_SettingsStore.objects.filter( + object__organizer=organizer, key__in=affected_keys + ).values_list('object_id', flat=True)) + for event in organizer.events.all(): + if event.pk not in non_inherited_events: + regenerate_css.apply_async(args=(event.pk,)) register_fonts = Signal()