Run exporters in repeatable read by default (Z#23173095) (#5500)

* Run exporters in repeatable read by default (Z#23173095)

* Update src/pretix/helpers/database.py

Co-authored-by: Richard Schreiber <schreiber@rami.io>

* Rename parameter, add test

* Do not run during tests

---------

Co-authored-by: Richard Schreiber <schreiber@rami.io>

src/pretix/base/exporter.py

@@ -105,6 +105,18 @@ class BaseExporter:
         """
         return False
 
+    @property
+    def repeatable_read(self) -> bool:
+        """
+        If ``True``, this exporter will be run in a REPEATABLE READ transaction. This ensures consistent results for
+        all queries performed by the exporter, but creates a performance burden on the database server. We recommend to
+        disable this for exporters that take very long to run and do not rely on this behavior, such as export of lists
+        to CSV files.
+
+        Defaults to ``True`` for now, but default may change in future versions.
+        """
+        return True
+
     @property
     def identifier(self) -> str:
         """

src/pretix/base/exporters/invoices.py

@@ -180,6 +180,7 @@ class InvoiceDataExporter(InvoiceExporterMixin, MultiSheetListExporter):
                     'includes two sheets, one with a line for every invoice, and one with a line for every position of '
                     'every invoice.')
     featured = True
+    repeatable_read = False
 
     @property
     def additional_form_fields(self):

src/pretix/base/exporters/orderlist.py

@@ -90,6 +90,7 @@ class OrderListExporter(MultiSheetListExporter):
                                'with a line for every order, one with a line for every order position, and one with '
                                'a line for every additional fee charged in an order.')
     featured = True
+    repeatable_read = False
 
     @cached_property
     def providers(self):
@@ -842,6 +843,7 @@ class TransactionListExporter(ListExporter):
     description = gettext_lazy('Download a spreadsheet of all substantial changes to orders, i.e. all changes to '
                                'products, prices or tax rates. The information is only accurate for changes made with '
                                'pretix versions released after October 2021.')
+    repeatable_read = False
 
     @cached_property
     def providers(self):
@@ -1020,6 +1022,7 @@ class PaymentListExporter(ListExporter):
     category = pgettext_lazy('export_category', 'Order data')
     description = gettext_lazy('Download a spreadsheet of all payments or refunds of every order.')
     featured = True
+    repeatable_read = False
 
     @property
     def additional_form_fields(self):
@@ -1159,7 +1162,7 @@ class QuotaListExporter(ListExporter):
         yield headers
 
         quotas = list(self.event.quotas.select_related('subevent'))
-        qa = QuotaAvailability(full_results=True)
+        qa = QuotaAvailability(full_results=True, allow_repeatable_read=False)
         qa.queue(*quotas)
         qa.compute()
 
@@ -1200,6 +1203,7 @@ class GiftcardTransactionListExporter(OrganizerLevelExportMixin, ListExporter):
     organizer_required_permission = 'can_manage_gift_cards'
     category = pgettext_lazy('export_category', 'Gift cards')
     description = gettext_lazy('Download a spreadsheet of all gift card transactions.')
+    repeatable_read = False
 
     @property
     def additional_form_fields(self):
@@ -1258,6 +1262,7 @@ class GiftcardRedemptionListExporter(ListExporter):
     verbose_name = gettext_lazy('Gift card redemptions')
     category = pgettext_lazy('export_category', 'Order data')
     description = gettext_lazy('Download a spreadsheet of all payments or refunds that involve gift cards.')
+    repeatable_read = False
 
     def iterate_list(self, form_data):
         payments = OrderPayment.objects.filter(

src/pretix/base/exporters/reusablemedia.py

@@ -34,6 +34,7 @@ class ReusableMediaExporter(OrganizerLevelExportMixin, ListExporter):
     verbose_name = _('Reusable media')
     category = pgettext_lazy('export_category', 'Reusable media')
     description = _('Download a spread sheet with the data of all reusable medias on your account.')
+    repeatable_read = False
 
     def iterate_list(self, form_data):
         media = ReusableMedium.objects.filter(

src/pretix/base/exporters/waitinglist.py

@@ -41,6 +41,7 @@ class WaitingListExporter(ListExporter):
     verbose_name = _('Waiting list')
     category = pgettext_lazy('export_category', 'Waiting list')
     description = _('Download a spread sheet with all your waiting list data.')
+    repeatable_read = False
 
     # map selected status to label and queryset-filter
     status_filters = [

src/pretix/base/services/export.py

@@ -49,7 +49,7 @@ from pretix.base.signals import (
     periodic_task, register_data_exporters, register_multievent_data_exporters,
 )
 from pretix.celery_app import app
-from pretix.helpers import OF_SELF
+from pretix.helpers import OF_SELF, repeatable_reads_transaction
 from pretix.helpers.urls import build_absolute_uri
 
 logger = logging.getLogger(__name__)
@@ -80,7 +80,12 @@ def export(self, event: Event, fileid: str, provider: str, form_data: Dict[str,
                 continue
             ex = response(event, event.organizer, set_progress)
             if ex.identifier == provider:
-                d = ex.render(form_data)
+                if ex.repeatable_read:
+                    with repeatable_reads_transaction():
+                        d = ex.render(form_data)
+                else:
+                    d = ex.render(form_data)
+
                 if d is None:
                     raise ExportError(
                         gettext('Your export did not contain any data.')
@@ -151,7 +156,11 @@ def multiexport(self, organizer: Organizer, user: User, device: int, token: int,
                         gettext('You do not have sufficient permission to perform this export.')
                     )
 
-                d = ex.render(form_data)
+                if ex.repeatable_read:
+                    with repeatable_reads_transaction():
+                        d = ex.render(form_data)
+                else:
+                    d = ex.render(form_data)
                 if d is None:
                     raise ExportError(
                         gettext('Your export did not contain any data.')
@@ -209,7 +218,11 @@ def _run_scheduled_export(schedule, context: Union[Event, Organizer], exporter,
         try:
             if not exporter:
                 raise ExportError("Export type not found.")
-            d = exporter.render(schedule.export_form_data)
+            if exporter.repeatable_read:
+                with repeatable_reads_transaction():
+                    d = exporter.render(schedule.export_form_data)
+            else:
+                d = exporter.render(schedule.export_form_data)
             if d is None:
                 raise ExportEmptyError(
                     gettext('Your export did not contain any data.')

src/pretix/base/services/quotas.py

@@ -26,7 +26,7 @@ from itertools import zip_longest
 
 import django_redis
 from django.conf import settings
-from django.db import models
+from django.db import connection, models
 from django.db.models import (
     Case, Count, F, Func, Max, OuterRef, Q, Subquery, Sum, Value, When,
     prefetch_related_objects,
@@ -64,7 +64,8 @@ class QuotaAvailability:
     * count_cart (dict mapping quotas to ints)
     """
 
-    def __init__(self, count_waitinglist=True, ignore_closed=False, full_results=False, early_out=True):
+    def __init__(self, count_waitinglist=True, ignore_closed=False, full_results=False, early_out=True,
+                 allow_repeatable_read=False):
         """
         Initialize a new quota availability calculator
 
@@ -86,6 +87,8 @@ class QuotaAvailability:
                           keep the database-level quota cache up to date so backend overviews render quickly. If you
                           do not care about keeping the cache up to date, you can set this to ``False`` for further
                           performance improvements.
+
+        :param allow_repeatable_read: Allow to run this even in REPEATABLE READ mode, generally not advised.
         """
         self._queue = []
         self._count_waitinglist = count_waitinglist
@@ -95,6 +98,7 @@ class QuotaAvailability:
         self._var_to_quotas = defaultdict(set)
         self._early_out = early_out
         self._quota_objects = {}
+        self._allow_repeatable_read = allow_repeatable_read
         self.results = {}
         self.count_paid_orders = defaultdict(int)
         self.count_pending_orders = defaultdict(int)
@@ -119,6 +123,10 @@ class QuotaAvailability:
         Compute the queued quotas. If ``allow_cache`` is set, results may also be taken from a cache that might
         be a few minutes outdated. In this case, you may not rely on the results in the ``count_*`` properties.
         """
+        if not self._allow_repeatable_read and getattr(connection, "tx_in_repeatable_read", False):
+            raise ValueError("You cannot compute quotas in REPEATABLE READ mode unless you explicitly opted in to "
+                             "do so.")
+
         now_dt = now_dt or now()
         quota_ids_set = {q.id for q in self._queue}
         if not quota_ids_set: