Integrate django-scopes (#1319)

* Install django-scopes

* Fix tests.api

* Update tasks and cronjobs

* Fix remaining tests

* Remove unused import

* Fix tests after rebase

* Disable scopes for get_Events_with_any_permission

* Disable scopes for a management command
This commit is contained in:
Raphael Michel
2019-06-17 10:46:55 +02:00
committed by GitHub
parent b1db5dbb3e
commit d85ddb5bda
130 changed files with 6253 additions and 4509 deletions

View File

@@ -10,6 +10,7 @@ from django.db.models import Q
from django.dispatch import receiver
from django.utils.timezone import make_aware, now
from django.utils.translation import pgettext_lazy, ugettext as _
from django_scopes import scopes_disabled
from pretix.base.i18n import language
from pretix.base.models import (
@@ -23,7 +24,7 @@ from pretix.base.reldate import RelativeDateWrapper
from pretix.base.services.checkin import _save_answers
from pretix.base.services.locking import LockTimeoutException, NoLockManager
from pretix.base.services.pricing import get_price
from pretix.base.services.tasks import ProfiledTask
from pretix.base.services.tasks import ProfiledEventTask
from pretix.base.settings import PERSON_NAME_SCHEMES
from pretix.base.templatetags.rich_text import rich_text
from pretix.celery_app import app
@@ -902,7 +903,7 @@ def get_fees(event, request, total, invoice_address, provider):
return fees
@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,))
@app.task(base=ProfiledEventTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,))
def add_items_to_cart(self, event: int, items: List[dict], cart_id: str=None, locale='en',
invoice_address: int=None, widget_data=None, sales_channel='web') -> None:
"""
@@ -913,12 +914,11 @@ def add_items_to_cart(self, event: int, items: List[dict], cart_id: str=None, lo
:raises CartError: On any error that occured
"""
with language(locale):
event = Event.objects.get(id=event)
ia = False
if invoice_address:
try:
ia = InvoiceAddress.objects.get(pk=invoice_address)
with scopes_disabled():
ia = InvoiceAddress.objects.get(pk=invoice_address)
except InvoiceAddress.DoesNotExist:
pass
@@ -934,8 +934,8 @@ def add_items_to_cart(self, event: int, items: List[dict], cart_id: str=None, lo
raise CartError(error_messages['busy'])
@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,))
def remove_cart_position(self, event: int, position: int, cart_id: str=None, locale='en') -> None:
@app.task(base=ProfiledEventTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,))
def remove_cart_position(self, event: Event, position: int, cart_id: str=None, locale='en') -> None:
"""
Removes a list of items from a user's cart.
:param event: The event ID in question
@@ -943,7 +943,6 @@ def remove_cart_position(self, event: int, position: int, cart_id: str=None, loc
:param session: Session ID of a guest
"""
with language(locale):
event = Event.objects.get(id=event)
try:
try:
cm = CartManager(event=event, cart_id=cart_id)
@@ -955,15 +954,14 @@ def remove_cart_position(self, event: int, position: int, cart_id: str=None, loc
raise CartError(error_messages['busy'])
@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,))
def clear_cart(self, event: int, cart_id: str=None, locale='en') -> None:
@app.task(base=ProfiledEventTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,))
def clear_cart(self, event: Event, cart_id: str=None, locale='en') -> None:
"""
Removes a list of items from a user's cart.
:param event: The event ID in question
:param session: Session ID of a guest
"""
with language(locale):
event = Event.objects.get(id=event)
try:
try:
cm = CartManager(event=event, cart_id=cart_id)
@@ -975,8 +973,8 @@ def clear_cart(self, event: int, cart_id: str=None, locale='en') -> None:
raise CartError(error_messages['busy'])
@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,))
def set_cart_addons(self, event: int, addons: List[dict], cart_id: str=None, locale='en',
@app.task(base=ProfiledEventTask, bind=True, max_retries=5, default_retry_delay=1, throws=(CartError,))
def set_cart_addons(self, event: Event, addons: List[dict], cart_id: str=None, locale='en',
invoice_address: int=None, sales_channel='web') -> None:
"""
Removes a list of items from a user's cart.
@@ -985,12 +983,11 @@ def set_cart_addons(self, event: int, addons: List[dict], cart_id: str=None, loc
:param session: Session ID of a guest
"""
with language(locale):
event = Event.objects.get(id=event)
ia = False
if invoice_address:
try:
ia = InvoiceAddress.objects.get(pk=invoice_address)
with scopes_disabled():
ia = InvoiceAddress.objects.get(pk=invoice_address)
except InvoiceAddress.DoesNotExist:
pass
try:

View File

@@ -2,6 +2,7 @@ from datetime import timedelta
from django.dispatch import receiver
from django.utils.timezone import now
from django_scopes import scopes_disabled
from pretix.base.models import CachedCombinedTicket, CachedTicket
@@ -10,6 +11,7 @@ from ..signals import periodic_task
@receiver(signal=periodic_task)
@scopes_disabled()
def clean_cart_positions(sender, **kwargs):
for cp in CartPosition.objects.filter(expires__lt=now() - timedelta(days=14), addon_to__isnull=False):
cp.delete()
@@ -20,12 +22,14 @@ def clean_cart_positions(sender, **kwargs):
@receiver(signal=periodic_task)
@scopes_disabled()
def clean_cached_files(sender, **kwargs):
for cf in CachedFile.objects.filter(expires__isnull=False, expires__lt=now()):
cf.delete()
@receiver(signal=periodic_task)
@scopes_disabled()
def clean_cached_tickets(sender, **kwargs):
for cf in CachedTicket.objects.filter(created__lte=now() - timedelta(days=30)):
cf.delete()

View File

@@ -6,7 +6,7 @@ from django.utils.translation import ugettext
from pretix.base.i18n import LazyLocaleException, language
from pretix.base.models import CachedFile, Event, cachedfile_name
from pretix.base.services.tasks import ProfiledTask
from pretix.base.services.tasks import ProfiledEventTask
from pretix.base.signals import register_data_exporters
from pretix.celery_app import app
@@ -15,9 +15,8 @@ class ExportError(LazyLocaleException):
pass
@app.task(base=ProfiledTask, throws=(ExportError,))
def export(event: str, fileid: str, provider: str, form_data: Dict[str, Any]) -> None:
event = Event.objects.get(id=event)
@app.task(base=ProfiledEventTask, throws=(ExportError,))
def export(event: Event, fileid: str, provider: str, form_data: Dict[str, Any]) -> None:
file = CachedFile.objects.get(id=fileid)
with language(event.settings.locale), override(event.settings.timezone):
responses = register_data_exporters.send(event)

View File

@@ -15,6 +15,7 @@ from django.utils import timezone
from django.utils.timezone import now
from django.utils.translation import pgettext, ugettext as _
from django_countries.fields import Country
from django_scopes import scope, scopes_disabled
from i18nfield.strings import LazyI18nString
from pretix.base.i18n import language
@@ -244,16 +245,18 @@ def generate_invoice(order: Order, trigger_pdf=True):
@app.task(base=TransactionAwareTask)
def invoice_pdf_task(invoice: int):
i = Invoice.objects.get(pk=invoice)
if i.shredded:
return None
if i.file:
i.file.delete()
with language(i.locale):
fname, ftype, fcontent = i.event.invoice_renderer.generate(i)
i.file.save(fname, ContentFile(fcontent))
i.save()
return i.file.name
with scopes_disabled():
i = Invoice.objects.get(pk=invoice)
with scope(organizer=i.order.event.organizer):
if i.shredded:
return None
if i.file:
i.file.delete()
with language(i.locale):
fname, ftype, fcontent = i.event.invoice_renderer.generate(i)
i.file.save(fname, ContentFile(fcontent))
i.save()
return i.file.name
def invoice_qualified(order: Order):

View File

@@ -10,6 +10,7 @@ from django.conf import settings
from django.core.mail import EmailMultiAlternatives, get_connection
from django.template.loader import get_template
from django.utils.translation import ugettext as _
from django_scopes import scope, scopes_disabled
from i18nfield.strings import LazyI18nString
from pretix.base.email import ClassicMailRenderer
@@ -234,83 +235,87 @@ def mail_send_task(self, *args, to: List[str], subject: str, body: str, html: st
pass
if event:
event = Event.objects.get(id=event)
with scopes_disabled():
event = Event.objects.get(id=event)
backend = event.get_mail_backend()
cm = lambda: scope(organizer=event.organizer) # noqa
else:
backend = get_connection(fail_silently=False)
cm = lambda: scopes_disabled() # noqa
if event:
if order:
try:
order = event.orders.get(pk=order)
except Order.DoesNotExist:
order = None
else:
if position:
try:
position = order.positions.get(pk=position)
except OrderPosition.DoesNotExist:
attach_tickets = False
if attach_tickets:
args = []
attach_size = 0
for name, ct in get_tickets_for_order(order, base_position=position):
content = ct.file.read()
args.append((name, content, ct.type))
attach_size += len(content)
with cm():
if event:
if order:
try:
order = event.orders.get(pk=order)
except Order.DoesNotExist:
order = None
else:
if position:
try:
position = order.positions.get(pk=position)
except OrderPosition.DoesNotExist:
attach_tickets = False
if attach_tickets:
args = []
attach_size = 0
for name, ct in get_tickets_for_order(order, base_position=position):
content = ct.file.read()
args.append((name, content, ct.type))
attach_size += len(content)
if attach_size < 4 * 1024 * 1024:
# Do not attach more than 4MB, it will bounce way to often.
for a in args:
try:
email.attach(*a)
except:
pass
else:
order.log_action(
'pretix.event.order.email.attachments.skipped',
data={
'subject': 'Attachments skipped',
'message': 'Attachment have not been send because {} bytes are likely too large to arrive.'.format(attach_size),
'recipient': '',
'invoices': [],
}
)
if attach_size < 4 * 1024 * 1024:
# Do not attach more than 4MB, it will bounce way to often.
for a in args:
try:
email.attach(*a)
except:
pass
else:
order.log_action(
'pretix.event.order.email.attachments.skipped',
data={
'subject': 'Attachments skipped',
'message': 'Attachment have not been send because {} bytes are likely too large to arrive.'.format(attach_size),
'recipient': '',
'invoices': [],
}
)
email = email_filter.send_chained(event, 'message', message=email, order=order)
email = email_filter.send_chained(event, 'message', message=email, order=order)
try:
backend.send_messages([email])
except smtplib.SMTPResponseException as e:
if e.smtp_code in (101, 111, 421, 422, 431, 442, 447, 452):
self.retry(max_retries=5, countdown=2 ** (self.request.retries * 2))
logger.exception('Error sending email')
try:
backend.send_messages([email])
except smtplib.SMTPResponseException as e:
if e.smtp_code in (101, 111, 421, 422, 431, 442, 447, 452):
self.retry(max_retries=5, countdown=2 ** (self.request.retries * 2))
logger.exception('Error sending email')
if order:
order.log_action(
'pretix.event.order.email.error',
data={
'subject': 'SMTP code {}'.format(e.smtp_code),
'message': e.smtp_error.decode() if isinstance(e.smtp_error, bytes) else str(e.smtp_error),
'recipient': '',
'invoices': [],
}
)
if order:
order.log_action(
'pretix.event.order.email.error',
data={
'subject': 'SMTP code {}'.format(e.smtp_code),
'message': e.smtp_error.decode() if isinstance(e.smtp_error, bytes) else str(e.smtp_error),
'recipient': '',
'invoices': [],
}
)
raise SendMailException('Failed to send an email to {}.'.format(to))
except Exception as e:
if order:
order.log_action(
'pretix.event.order.email.error',
data={
'subject': 'Internal error',
'message': str(e),
'recipient': '',
'invoices': [],
}
)
logger.exception('Error sending email')
raise SendMailException('Failed to send an email to {}.'.format(to))
raise SendMailException('Failed to send an email to {}.'.format(to))
except Exception as e:
if order:
order.log_action(
'pretix.event.order.email.error',
data={
'subject': 'Internal error',
'message': str(e),
'recipient': '',
'invoices': [],
}
)
logger.exception('Error sending email')
raise SendMailException('Failed to send an email to {}.'.format(to))
def mail_send(*args, **kwargs):

View File

@@ -1,5 +1,6 @@
from django.conf import settings
from django.template.loader import get_template
from django_scopes import scope, scopes_disabled
from inlinestyler.utils import inline_css
from pretix.base.i18n import language
@@ -12,6 +13,7 @@ from pretix.helpers.urls import build_absolute_uri
@app.task(base=TransactionAwareTask)
@scopes_disabled()
def notify(logentry_id: int):
logentry = LogEntry.all.get(id=logentry_id)
if not logentry.event:
@@ -66,17 +68,22 @@ def notify(logentry_id: int):
@app.task(base=ProfiledTask)
def send_notification(logentry_id: int, action_type: str, user_id: int, method: str):
logentry = LogEntry.all.get(id=logentry_id)
user = User.objects.get(id=user_id)
types = get_all_notification_types(logentry.event)
notification_type = types.get(action_type)
if not notification_type:
return # Ignore, e.g. plugin not active for this event
if logentry.event:
sm = lambda: scope(organizer=logentry.event.organizer) # noqa
else:
sm = lambda: scopes_disabled() # noqa
with sm():
user = User.objects.get(id=user_id)
types = get_all_notification_types(logentry.event)
notification_type = types.get(action_type)
if not notification_type:
return # Ignore, e.g. plugin not active for this event
with language(user.locale):
notification = notification_type.build_notification(logentry)
with language(user.locale):
notification = notification_type.build_notification(logentry)
if method == "mail":
send_notification_mail(notification, user)
if method == "mail":
send_notification_mail(notification, user)
def send_notification_mail(notification: Notification, user: User):

View File

@@ -16,6 +16,7 @@ from django.utils.formats import date_format
from django.utils.functional import cached_property
from django.utils.timezone import make_aware, now
from django.utils.translation import ugettext as _
from django_scopes import scopes_disabled
from pretix.api.models import OAuthApplication
from pretix.base.i18n import (
@@ -42,7 +43,7 @@ from pretix.base.services.invoices import (
from pretix.base.services.locking import LockTimeoutException, NoLockManager
from pretix.base.services.mail import SendMailException
from pretix.base.services.pricing import get_price
from pretix.base.services.tasks import ProfiledTask
from pretix.base.services.tasks import ProfiledEventTask, ProfiledTask
from pretix.base.settings import PERSON_NAME_SCHEMES
from pretix.base.signals import (
allow_ticket_download, order_approved, order_canceled, order_changed,
@@ -715,10 +716,8 @@ def _order_placed_email_attendee(event: Event, order: Order, position: OrderPosi
logger.exception('Order received email could not be sent to attendee')
def _perform_order(event: str, payment_provider: str, position_ids: List[str],
def _perform_order(event: Event, payment_provider: str, position_ids: List[str],
email: str, locale: str, address: int, meta_info: dict=None, sales_channel: str='web'):
event = Event.objects.get(id=event)
if payment_provider:
pprov = event.get_payment_providers().get(payment_provider)
if not pprov:
@@ -732,7 +731,8 @@ def _perform_order(event: str, payment_provider: str, position_ids: List[str],
addr = None
if address is not None:
try:
addr = InvoiceAddress.objects.get(pk=address)
with scopes_disabled():
addr = InvoiceAddress.objects.get(pk=address)
except InvoiceAddress.DoesNotExist:
pass
@@ -804,6 +804,7 @@ def _perform_order(event: str, payment_provider: str, position_ids: List[str],
@receiver(signal=periodic_task)
@scopes_disabled()
def expire_orders(sender, **kwargs):
eventcache = {}
@@ -818,6 +819,7 @@ def expire_orders(sender, **kwargs):
@receiver(signal=periodic_task)
@scopes_disabled()
def send_expiry_warnings(sender, **kwargs):
eventcache = {}
today = now().replace(hour=0, minute=0, second=0)
@@ -875,6 +877,7 @@ def send_expiry_warnings(sender, **kwargs):
@receiver(signal=periodic_task)
@scopes_disabled()
def send_download_reminders(sender, **kwargs):
today = now().replace(hour=0, minute=0, second=0, microsecond=0)
@@ -1497,8 +1500,8 @@ class OrderChangeManager:
return pprov
@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1, throws=(OrderError,))
def perform_order(self, event: str, payment_provider: str, positions: List[str],
@app.task(base=ProfiledEventTask, bind=True, max_retries=5, default_retry_delay=1, throws=(OrderError,))
def perform_order(self, event: Event, payment_provider: str, positions: List[str],
email: str=None, locale: str=None, address: int=None, meta_info: dict=None,
sales_channel: str='web'):
with language(locale):
@@ -1513,6 +1516,7 @@ def perform_order(self, event: str, payment_provider: str, positions: List[str],
@app.task(base=ProfiledTask, bind=True, max_retries=5, default_retry_delay=1, throws=(OrderError,))
@scopes_disabled()
def cancel_order(self, order: int, user: int=None, send_mail: bool=True, api_token=None, oauth_application=None,
device=None, cancellation_fee=None, try_auto_refund=False):
try:

View File

@@ -4,6 +4,7 @@ from django.conf import settings
from django.db.models import Max, Q
from django.dispatch import receiver
from django.utils.timezone import now
from django_scopes import scopes_disabled
from pretix.base.models import Event, LogEntry
from pretix.celery_app import app
@@ -17,6 +18,7 @@ def build_all_quota_caches(sender, **kwargs):
@app.task
@scopes_disabled()
def refresh_quota_caches():
# Active events
active = LogEntry.objects.using(settings.DATABASE_REPLICA).filter(

View File

@@ -11,14 +11,13 @@ from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _
from pretix.base.models import CachedFile, Event, cachedfile_name
from pretix.base.services.tasks import ProfiledTask
from pretix.base.services.tasks import ProfiledEventTask
from pretix.base.shredder import ShredError
from pretix.celery_app import app
@app.task(base=ProfiledTask)
def export(event: str, shredders: List[str]) -> None:
event = Event.objects.get(id=event)
@app.task(base=ProfiledEventTask)
def export(event: Event, shredders: List[str]) -> None:
known_shredders = event.get_data_shredders()
with NamedTemporaryFile() as rawfile:
@@ -63,9 +62,8 @@ def export(event: str, shredders: List[str]) -> None:
return cf.pk
@app.task(base=ProfiledTask, throws=(ShredError,))
def shred(event: str, fileid: str, confirm_code: str) -> None:
event = Event.objects.get(id=event)
@app.task(base=ProfiledEventTask, throws=(ShredError,))
def shred(event: Event, fileid: str, confirm_code: str) -> None:
known_shredders = event.get_data_shredders()
try:
cf = CachedFile.objects.get(pk=fileid)

View File

@@ -14,10 +14,12 @@ import time
from django.conf import settings
from django.db import transaction
from django_scopes import scope, scopes_disabled
from pretix.base.metrics import (
pretix_task_duration_seconds, pretix_task_runs_total,
)
from pretix.base.models import Event
from pretix.celery_app import app
@@ -61,6 +63,35 @@ class ProfiledTask(app.Task):
return super().on_success(retval, task_id, args, kwargs)
class EventTask(app.Task):
def __call__(self, *args, **kwargs):
if 'event_id' in kwargs:
event_id = kwargs.get('event_id')
with scopes_disabled():
event = Event.objects.select_related('organizer').get(pk=event_id)
del kwargs['event_id']
kwargs['event'] = event
elif 'event' in kwargs:
event_id = kwargs.get('event')
with scopes_disabled():
event = Event.objects.select_related('organizer').get(pk=event_id)
kwargs['event'] = event
else:
args = list(args)
event_id = args[0]
with scopes_disabled():
event = Event.objects.select_related('organizer').get(pk=event_id)
args[0] = event
with scope(organizer=event.organizer):
ret = super().__call__(*args, **kwargs)
return ret
class ProfiledEventTask(ProfiledTask, EventTask):
pass
class TransactionAwareTask(ProfiledTask):
"""
Task class which is aware of django db transactions and only executes tasks

View File

@@ -4,13 +4,14 @@ import os
from django.core.files.base import ContentFile
from django.utils.timezone import now
from django.utils.translation import ugettext as _
from django_scopes import scopes_disabled
from pretix.base.i18n import language
from pretix.base.models import (
CachedCombinedTicket, CachedTicket, Event, InvoiceAddress, Order,
OrderPosition,
)
from pretix.base.services.tasks import ProfiledTask
from pretix.base.services.tasks import EventTask, ProfiledTask
from pretix.base.settings import PERSON_NAME_SCHEMES
from pretix.base.signals import allow_ticket_download, register_ticket_outputs
from pretix.celery_app import app
@@ -57,10 +58,11 @@ def generate_order(order: int, provider: str):
@app.task(base=ProfiledTask)
def generate(model: str, pk: int, provider: str):
if model == 'order':
return generate_order(pk, provider)
elif model == 'orderposition':
return generate_orderposition(pk, provider)
with scopes_disabled():
if model == 'order':
return generate_order(pk, provider)
elif model == 'orderposition':
return generate_orderposition(pk, provider)
class DummyRollbackException(Exception):
@@ -165,9 +167,8 @@ def get_tickets_for_order(order, base_position=None):
return tickets
@app.task(base=ProfiledTask)
def invalidate_cache(event: int, item: int=None, provider: str=None, order: int=None, **kwargs):
event = Event.objects.get(id=event)
@app.task(base=EventTask)
def invalidate_cache(event: Event, item: int=None, provider: str=None, order: int=None, **kwargs):
qs = CachedTicket.objects.filter(order_position__order__event=event)
qsc = CachedCombinedTicket.objects.filter(order__event=event)

View File

@@ -6,6 +6,7 @@ import requests
from django.dispatch import receiver
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _, ugettext_noop
from django_scopes import scopes_disabled
from i18nfield.strings import LazyI18nString
from pretix import __version__
@@ -29,6 +30,7 @@ def run_update_check(sender, **kwargs):
@app.task
@scopes_disabled()
def update_check():
gs = GlobalSettingsObject()

View File

@@ -1,17 +1,17 @@
import sys
from django.dispatch import receiver
from django_scopes import scopes_disabled
from pretix.base.models import Event, User, WaitingListEntry
from pretix.base.models.waitinglist import WaitingListException
from pretix.base.services.tasks import ProfiledTask
from pretix.base.services.tasks import EventTask
from pretix.base.signals import periodic_task
from pretix.celery_app import app
@app.task(base=ProfiledTask)
def assign_automatically(event_id: int, user_id: int=None, subevent_id: int=None):
event = Event.objects.get(id=event_id)
@app.task(base=EventTask)
def assign_automatically(event: Event, user_id: int=None, subevent_id: int=None):
if user_id:
user = User.objects.get(id=user_id)
else:
@@ -69,6 +69,7 @@ def assign_automatically(event_id: int, user_id: int=None, subevent_id: int=None
@receiver(signal=periodic_task)
@scopes_disabled()
def process_waitinglist(sender, **kwargs):
qs = Event.objects.filter(
live=True