Optimize SQL queries in order list and order search

src/pretix/control/forms/filter.py

@@ -175,7 +175,7 @@ class OrderFilterForm(FilterForm):
 
 class EventOrderFilterForm(OrderFilterForm):
     orders = {'code': 'code', 'email': 'email', 'total': 'total',
-              'datetime': 'datetime', 'status': 'status', 'pcnt': 'pcnt'}
+              'datetime': 'datetime', 'status': 'status'}
 
     item = forms.ModelChoiceField(
         label=_('Products'),
@@ -275,16 +275,19 @@ class EventOrderFilterForm(OrderFilterForm):
                 qs = qs.annotate(has_answer=Exists(answers)).filter(has_answer=True)
 
         if fdata.get('status') == 'overpaid':
+            qs = Order.annotate_overpayments(qs, refunds=False, results=False, sums=True)
             qs = qs.filter(
                 Q(~Q(status=Order.STATUS_CANCELED) & Q(pending_sum_t__lt=0))
                 | Q(Q(status=Order.STATUS_CANCELED) & Q(pending_sum_rc__lt=0))
             )
         elif fdata.get('status') == 'pendingpaid':
+            qs = Order.annotate_overpayments(qs, refunds=False, results=False, sums=True)
             qs = qs.filter(
                 Q(status__in=(Order.STATUS_EXPIRED, Order.STATUS_PENDING)) & Q(pending_sum_t__lte=0)
                 & Q(require_approval=False)
             )
         elif fdata.get('status') == 'underpaid':
+            qs = Order.annotate_overpayments(qs, refunds=False, results=False, sums=True)
             qs = qs.filter(
                 status=Order.STATUS_PAID,
                 pending_sum_t__gt=0
@@ -300,7 +303,7 @@ class EventOrderFilterForm(OrderFilterForm):
 
 class OrderSearchFilterForm(OrderFilterForm):
     orders = {'code': 'code', 'email': 'email', 'total': 'total',
-              'datetime': 'datetime', 'status': 'status', 'pcnt': 'pcnt',
+              'datetime': 'datetime', 'status': 'status',
               'event': 'event'}
 
     organizer = forms.ModelChoiceField(

src/pretix/control/templates/pretixcontrol/orders/index.html

@@ -101,9 +101,7 @@
                     <th class="text-right">{% trans "Order total" %}
                         <a href="?{% url_replace request 'ordering' '-total' %}"><i class="fa fa-caret-down"></i></a>
                         <a href="?{% url_replace request 'ordering' 'total' %}"><i class="fa fa-caret-up"></i></a></th>
-                    <th class="text-right">{% trans "Positions" %}
-                        <a href="?{% url_replace request 'ordering' '-pcnt' %}"><i class="fa fa-caret-down"></i></a>
-                        <a href="?{% url_replace request 'ordering' 'pcnt' %}"><i class="fa fa-caret-up"></i></a></th>
+                    <th class="text-right">{% trans "Positions" %}</th>
                     <th class="text-right">{% trans "Status" %}
                         <a href="?{% url_replace request 'ordering' '-status' %}"><i class="fa fa-caret-down"></i></a>
                         <a href="?{% url_replace request 'ordering' 'status' %}"><i class="fa fa-caret-up"></i></a></th>

src/pretix/control/views/item.py

@@ -860,9 +860,7 @@ class ItemUpdateGeneral(ItemDetailMixin, EventPermissionRequiredMixin, UpdateVie
         messages.success(self.request, _('Your changes have been saved.'))
         if form.has_changed() or any(f.has_changed() for f in self.plugin_forms):
             data = {
-                k: (form.cleaned_data.get(k).name
-                    if isinstance(form.cleaned_data.get(k), File)
-                    else form.cleaned_data.get(k))
+                k: form.cleaned_data.get(k)
                 for k in form.changed_data
             }
             for f in self.plugin_forms:

src/pretix/control/views/orders.py

@@ -83,23 +83,46 @@ class OrderList(EventPermissionRequiredMixin, PaginationMixin, ListView):
     permission = 'can_view_orders'
 
     def get_queryset(self):
-        s = OrderPosition.objects.filter(
-            order=OuterRef('pk')
-        ).order_by().values('order').annotate(k=Count('id')).values('k')
         qs = Order.objects.filter(
             event=self.request.event
-        ).annotate(pcnt=Subquery(s, output_field=IntegerField())).select_related('invoice_address')
-
-        qs = Order.annotate_overpayments(qs)
+        ).select_related('invoice_address')
 
         if self.filter_form.is_valid():
             qs = self.filter_form.filter_qs(qs)
 
-        return qs.distinct()
+        return qs
 
     def get_context_data(self, **kwargs):
         ctx = super().get_context_data(**kwargs)
         ctx['filter_form'] = self.filter_form
+
+        # Only compute this annotations for this page (query optimization)
+        s = OrderPosition.objects.filter(
+            order=OuterRef('pk')
+        ).order_by().values('order').annotate(k=Count('id')).values('k')
+        annotated = {
+            o['pk']: o
+            for o in
+            Order.annotate_overpayments(Order.objects).filter(
+                pk__in=[o.pk for o in ctx['orders']]
+            ).annotate(
+                pcnt=Subquery(s, output_field=IntegerField())
+            ).values(
+                'pk', 'pcnt', 'is_overpaid', 'is_underpaid', 'is_pending_with_full_payment', 'has_external_refund',
+                'has_pending_refund'
+            )
+        }
+
+        for o in ctx['orders']:
+            if o.pk not in annotated:
+                continue
+            o.pcnt = annotated.get(o.pk)['pcnt']
+            o.is_overpaid = annotated.get(o.pk)['is_overpaid']
+            o.is_underpaid = annotated.get(o.pk)['is_underpaid']
+            o.is_pending_with_full_payment = annotated.get(o.pk)['is_pending_with_full_payment']
+            o.has_external_refund = annotated.get(o.pk)['has_external_refund']
+            o.has_pending_refund = annotated.get(o.pk)['has_pending_refund']
+
         return ctx
 
     @cached_property

src/pretix/control/views/search.py

@@ -20,16 +20,33 @@ class OrderSearch(PaginationMixin, ListView):
     def get_context_data(self, **kwargs):
         ctx = super().get_context_data()
         ctx['filter_form'] = self.filter_form
+
+        # Only compute this annotations for this page (query optimization)
+        s = OrderPosition.objects.filter(
+            order=OuterRef('pk')
+        ).order_by().values('order').annotate(k=Count('id')).values('k')
+        annotated = {
+            o['pk']: o
+            for o in
+            Order.objects.filter(
+                pk__in=[o.pk for o in ctx['orders']]
+            ).annotate(
+                pcnt=Subquery(s, output_field=IntegerField())
+            ).values(
+                'pk', 'pcnt',
+            )
+        }
+
+        for o in ctx['orders']:
+            if o.pk not in annotated:
+                continue
+            o.pcnt = annotated.get(o.pk)['pcnt']
+
         return ctx
 
     def get_queryset(self):
         qs = Order.objects.select_related('invoice_address')
 
-        s = OrderPosition.objects.filter(
-            order=OuterRef('pk')
-        ).order_by().values('order').annotate(k=Count('id')).values('k')
-        qs = qs.annotate(pcnt=Subquery(s, output_field=IntegerField()))
-
         if not self.request.user.has_active_staff_session(self.request.session.session_key):
             qs = qs.filter(
                 Q(event__organizer_id__in=self.request.user.teams.filter(