Merge branch 'master' into per-payment-abort_pending_payment_allowed

* Fix performance issues in download reminder

* Update src/pretix/base/services/orders.py

Co-authored-by: Richard Schreiber <schreiber@pretix.eu>

* Fixes after review

* Fix check in loop

---------

Co-authored-by: Richard Schreiber <schreiber@pretix.eu>

src/pretix/api/auth/devicesecurity.py

@@ -115,7 +115,7 @@ class PretixScanSecurityProfile(AllowListSecurityProfile):
         ('GET', 'api-v1:event.settings'),
         ('POST', 'api-v1:upload'),
         ('POST', 'api-v1:checkinrpc.redeem'),
-        ('POST', 'api-v1:checkinrpc.annull'),
+        ('POST', 'api-v1:checkinrpc.annul'),
         ('GET', 'api-v1:checkinrpc.search'),
         ('GET', 'api-v1:reusablemedium-list'),
         ('POST', 'api-v1:reusablemedium-lookup'),
@@ -154,7 +154,7 @@ class PretixScanNoSyncNoSearchSecurityProfile(AllowListSecurityProfile):
         ('GET', 'api-v1:event.settings'),
         ('POST', 'api-v1:upload'),
         ('POST', 'api-v1:checkinrpc.redeem'),
-        ('POST', 'api-v1:checkinrpc.annull'),
+        ('POST', 'api-v1:checkinrpc.annul'),
         ('GET', 'api-v1:checkinrpc.search'),
     )
 
@@ -191,7 +191,7 @@ class PretixScanNoSyncSecurityProfile(AllowListSecurityProfile):
         ('GET', 'api-v1:event.settings'),
         ('POST', 'api-v1:upload'),
         ('POST', 'api-v1:checkinrpc.redeem'),
-        ('POST', 'api-v1:checkinrpc.annull'),
+        ('POST', 'api-v1:checkinrpc.annul'),
         ('GET', 'api-v1:checkinrpc.search'),
     )
 

src/pretix/base/services/orders.py

@@ -48,12 +48,12 @@ from celery.exceptions import MaxRetriesExceededError
 from django.conf import settings
 from django.core.cache import cache
 from django.core.exceptions import ValidationError
-from django.db import transaction
+from django.db import models, transaction
 from django.db.models import (
-    Count, Exists, F, IntegerField, Max, Min, OuterRef, Q, QuerySet, Sum,
-    Value,
+    Count, Exists, F, IntegerField, Max, Min, OuterRef, Q, QuerySet, Subquery,
+    Sum, Value,
 )
-from django.db.models.functions import Coalesce, Greatest
+from django.db.models.functions import Cast, Greatest
 from django.db.transaction import get_connection
 from django.dispatch import receiver
 from django.utils.functional import cached_property
@@ -71,7 +71,7 @@ from pretix.base.models import (
     Membership, Order, OrderPayment, OrderPosition, Quota, Seat,
     SeatCategoryMapping, User, Voucher,
 )
-from pretix.base.models.event import SubEvent
+from pretix.base.models.event import Event_SettingsStore, SubEvent
 from pretix.base.models.orders import (
     BlockedTicketSecret, InvoiceAddress, OrderFee, OrderRefund,
     generate_secret,
@@ -1494,83 +1494,104 @@ def send_expiry_warnings(sender, **kwargs):
 @scopes_disabled()
 def send_download_reminders(sender, **kwargs):
     today = now().replace(hour=0, minute=0, second=0, microsecond=0)
-    qs = Order.objects.annotate(
-        first_date=Coalesce(
-            Min('all_positions__subevent__date_from'),
-            F('event__date_from')
+
+    events = Event.objects.filter(
+        Q(has_subevents=False, date_from__gte=now()) |
+        (Q(has_subevents=True) & Q(Exists(
+            SubEvent.objects.filter(event_id=OuterRef('id'), date_from__gte=now())
+        )))
+    ).annotate(
+        reminder_days=Subquery(
+            Event_SettingsStore.objects.filter(
+                object=OuterRef('id'),
+                key='mail_days_download_reminder'
+            ).exclude(
+                value="None"
+            ).annotate(
+                val=Cast(F("value"), output_field=models.IntegerField()),
+            ).values("val")
         )
     ).filter(
-        download_reminder_sent=False,
-        datetime__lte=now() - timedelta(hours=2),
-        first_date__gte=today,
-    ).only(
-        'pk', 'event_id', 'sales_channel', 'datetime',
-    ).order_by('event_id')
-    event_id = None
-    days = None
-    event = None
+        reminder_days__isnull=False,
+    ).order_by()
 
-    for o in qs:
-        if o.event_id != event_id:
-            days = o.event.settings.get('mail_days_download_reminder', as_type=int)
-            event = o.event
-            event_id = o.event_id
+    for event in events.iterator(chunk_size=10_000):
+        qs = event.orders.filter(
+            download_reminder_sent=False,
+            datetime__lte=now() - timedelta(hours=2),
+        )
 
-        if days is None:
-            continue
-
-        if o.sales_channel.identifier not in event.settings.mail_sales_channel_download_reminder:
-            continue
-
-        reminder_date = (o.first_date - timedelta(days=days)).replace(hour=0, minute=0, second=0, microsecond=0)
-        if now() < reminder_date or o.datetime > reminder_date:
-            continue
-
-        with transaction.atomic():
-            o = Order.objects.select_for_update(of=OF_SELF).get(pk=o.pk)
-            if o.download_reminder_sent:
-                # Race condition
-                continue
-            positions = list(o.positions_with_tickets)
-            if not positions:
+        if event.has_subevents:
+            qs = qs.annotate(
+                first_date=Min('all_positions__subevent__date_from')
+            ).filter(
+                Q(first_date__gte=today)
+            )
+        else:
+            event_reminder_date = (event.date_from - timedelta(days=event.reminder_days)).replace(hour=0, minute=0, second=0, microsecond=0)
+            if now() < event_reminder_date:
                 continue
 
-            if not o.ticket_download_available:
+        qs = qs.only(
+            'pk', 'event_id', 'sales_channel', 'datetime',
+        ).order_by()
+
+        for o in qs:
+            if o.sales_channel.identifier not in event.settings.mail_sales_channel_download_reminder:
                 continue
 
-            if o.status != Order.STATUS_PAID:
-                if o.status != Order.STATUS_PENDING or o.require_approval or (not o.valid_if_pending and not o.event.settings.ticket_download_pending):
+            if event.has_subevents:
+                reminder_date = ((o.first_date or event.date_from) - timedelta(days=event.reminder_days)).replace(hour=0, minute=0, second=0, microsecond=0)
+            else:
+                reminder_date = event_reminder_date
+            if now() < reminder_date or o.datetime > reminder_date:
+                continue
+
+            with transaction.atomic():
+                o = Order.objects.select_for_update(of=OF_SELF).get(pk=o.pk)
+                if o.download_reminder_sent:
+                    # Race condition
+                    continue
+                positions = list(o.positions_with_tickets)
+                if not positions:
                     continue
 
-            with language(o.locale, o.event.settings.region):
-                o.download_reminder_sent = True
-                o.save(update_fields=['download_reminder_sent'])
-                email_template = event.settings.mail_text_download_reminder
-                email_subject = event.settings.mail_subject_download_reminder
-                email_context = get_email_context(event=event, order=o)
-                o.send_mail(
-                    email_subject, email_template, email_context,
-                    'pretix.event.order.email.download_reminder_sent',
-                    attach_tickets=True
-                )
+                if not o.ticket_download_available:
+                    continue
 
-                if event.settings.mail_send_download_reminder_attendee:
-                    for p in positions:
-                        if p.subevent_id:
-                            reminder_date = (p.subevent.date_from - timedelta(days=days)).replace(
-                                hour=0, minute=0, second=0, microsecond=0
-                            )
-                            if now() < reminder_date:
-                                continue
-                        if p.addon_to_id is None and p.attendee_email and p.attendee_email != o.email:
-                            email_template = event.settings.mail_text_download_reminder_attendee
-                            email_subject = event.settings.mail_subject_download_reminder_attendee
-                            email_context = get_email_context(event=event, order=o, position=p)
-                            o.send_mail(
-                                email_subject, email_template, email_context,
-                                'pretix.event.order.email.download_reminder_sent',
-                                attach_tickets=True, position=p
-                            )
+                if o.status != Order.STATUS_PAID:
+                    if o.status != Order.STATUS_PENDING or o.require_approval or (not o.valid_if_pending and not o.event.settings.ticket_download_pending):
+                        continue
+
+                with language(o.locale, o.event.settings.region):
+                    o.download_reminder_sent = True
+                    o.save(update_fields=['download_reminder_sent'])
+                    email_template = event.settings.mail_text_download_reminder
+                    email_subject = event.settings.mail_subject_download_reminder
+                    email_context = get_email_context(event=event, order=o)
+                    o.send_mail(
+                        email_subject, email_template, email_context,
+                        'pretix.event.order.email.download_reminder_sent',
+                        attach_tickets=True
+                    )
+
+                    if event.settings.mail_send_download_reminder_attendee:
+                        for p in positions:
+                            if p.subevent_id:
+                                reminder_date = (p.subevent.date_from - timedelta(days=event.reminder_days)).replace(
+                                    hour=0, minute=0, second=0, microsecond=0
+                                )
+                                if now() < reminder_date:
+                                    continue
+                            if p.addon_to_id is None and p.attendee_email and p.attendee_email != o.email:
+                                email_template = event.settings.mail_text_download_reminder_attendee
+                                email_subject = event.settings.mail_subject_download_reminder_attendee
+                                email_context = get_email_context(event=event, order=o, position=p)
+                                o.send_mail(
+                                    email_subject, email_template, email_context,
+                                    'pretix.event.order.email.download_reminder_sent',
+                                    attach_tickets=True, position=p
+                                )
 
 
 def notify_user_changed_order(order, user=None, auth=None, invoices=[]):

src/pretix/base/settings.py

@@ -2930,7 +2930,7 @@ Your {event} team"""))  # noqa: W291
     },
     'mail_days_download_reminder': {
         'type': int,
-        'default': None
+        'default': None  # when this default is changed, send_download_reminders needs to change
     },
     'mail_send_download_reminder_attendee': {
         'type': bool,

src/pretix/plugins/paypal2/payment.py

@@ -23,7 +23,7 @@ import json
 import logging
 import urllib.parse
 from collections import OrderedDict
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from decimal import Decimal
 
 from django import forms
@@ -55,7 +55,7 @@ from pretix.base.forms import SecretKeySettingsField
 from pretix.base.forms.questions import guess_country
 from pretix.base.models import Event, Order, OrderPayment, OrderRefund, Quota
 from pretix.base.payment import BasePaymentProvider, PaymentException
-from pretix.base.settings import SettingsSandbox
+from pretix.base.settings import GlobalSettingsObject, SettingsSandbox
 from pretix.helpers import OF_SELF
 from pretix.helpers.urls import mainreverse_absolute
 from pretix.multidomain.urlreverse import eventreverse, eventreverse_absolute
@@ -645,7 +645,7 @@ class PaypalMethod(BasePaymentProvider):
     def _execute_payment(self, request: HttpRequest, payment: OrderPayment):
         payment = OrderPayment.objects.select_for_update(of=OF_SELF).get(pk=payment.pk)
         if payment.state == OrderPayment.PAYMENT_STATE_CONFIRMED:
-            # payment is already confirmed; possible return-view/webhook race-condition
+            logger.warning('payment is already confirmed; possible return-view/webhook race-condition')
             return
 
         try:

src/pretix/plugins/paypal2/signals.py

@@ -99,6 +99,10 @@ def register_global_settings(sender, **kwargs):
                 ('sandbox', 'Sandbox'),
             ),
         )),
+        ('payment_paypal_abort_pending_payment_allowed_timeout', forms.IntegerField(
+            label=_('Paypal: Debounce timeout (minutes) after which it is possible to abort a PENDING_REVIEW payment.'),
+            initial=30,
+        )),
     ])
 
 

src/pretix/plugins/paypal2/views.py

@@ -471,8 +471,8 @@ def webhook(request, *args, **kwargs):
     elif payment.state in (OrderPayment.PAYMENT_STATE_PENDING, OrderPayment.PAYMENT_STATE_CREATED,
                            OrderPayment.PAYMENT_STATE_CANCELED, OrderPayment.PAYMENT_STATE_FAILED):
         if sale['status'] == 'COMPLETED':
-            any_captures = False
             all_captures_completed = True
+            any_pending_review = False
             for purchaseunit in sale['purchase_units']:
                 for capture in purchaseunit['payments']['captures']:
                     try:
@@ -483,9 +483,9 @@ def webhook(request, *args, **kwargs):
 
                     if capture['status'] not in ('COMPLETED', 'REFUNDED', 'PARTIALLY_REFUNDED'):
                         all_captures_completed = False
-                    else:
-                        any_captures = True
-            if any_captures and all_captures_completed:
+                        if capture['status_details']['reason'] == "PENDING_REVIEW":
+                            any_pending_review = True
+            if all_captures_completed:
                 try:
                     payment.info = json.dumps(sale.dict())
                     payment.save(update_fields=['info'])
@@ -493,6 +493,9 @@ def webhook(request, *args, **kwargs):
                     prov.log_payment_duration(payment)
                 except Quota.QuotaExceededException:
                     pass
+            if any_pending_review:
+                payment.state = OrderPayment.PAYMENT_STATE_PENDING
+                payment.save()
         elif sale['status'] == 'APPROVED':
             try:
                 request.session['payment_paypal_oid'] = payment.info_data['id']