Optimize SQL queries in order list and order search

src/pretix/base/migrations/0107_auto_20190129_1337.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.1.5 on 2019-01-29 13:37
+
+from django.db import migrations, models
+import django.db.models.deletion
+import jsonfallback.fields
+import pretix.base.models.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pretixbase', '0106_auto_20190118_1527'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='order',
+            name='datetime',
+            field=models.DateTimeField(db_index=True, verbose_name='Date'),
+        ),
+    ]

src/pretix/base/models/orders.py

@@ -137,7 +137,7 @@ class Order(LockModel, LoggedModel):
     )
     secret = models.CharField(max_length=32, default=generate_secret)
     datetime = models.DateTimeField(
-        verbose_name=_("Date")
+        verbose_name=_("Date"), db_index=True
     )
     expires = models.DateTimeField(
         verbose_name=_("Expiration date")
@@ -240,7 +240,7 @@ class Order(LockModel, LoggedModel):
         return total - payment_sum + refund_sum
 
     @classmethod
-    def annotate_overpayments(cls, qs):
+    def annotate_overpayments(cls, qs, results=True, refunds=True, sums=False):
         payment_sum = OrderPayment.objects.filter(
             state__in=(OrderPayment.PAYMENT_STATE_CONFIRMED, OrderPayment.PAYMENT_STATE_REFUNDED),
             order=OuterRef('pk')
@@ -258,38 +258,47 @@ class Order(LockModel, LoggedModel):
             state__in=(OrderRefund.REFUND_STATE_CREATED, OrderRefund.REFUND_STATE_TRANSIT),
             order=OuterRef('pk')
         )
+        payment_sum_sq = Subquery(payment_sum, output_field=models.DecimalField(decimal_places=2, max_digits=10))
+        refund_sum_sq = Subquery(refund_sum, output_field=models.DecimalField(decimal_places=2, max_digits=10))
+        if sums:
+            qs = qs.annotate(
+                payment_sum=payment_sum_sq,
+                refund_sum=refund_sum_sq,
+            )
 
         qs = qs.annotate(
-            payment_sum=Subquery(payment_sum, output_field=models.DecimalField(decimal_places=2, max_digits=10)),
+            pending_sum_t=F('total') - Coalesce(payment_sum_sq, 0) + Coalesce(refund_sum_sq, 0),
-            refund_sum=Subquery(refund_sum, output_field=models.DecimalField(decimal_places=2, max_digits=10)),
+            pending_sum_rc=-1 * Coalesce(payment_sum_sq, 0) + Coalesce(refund_sum_sq, 0),
-            has_external_refund=Exists(external_refund),
-            has_pending_refund=Exists(pending_refund),
-        ).annotate(
-            pending_sum_t=F('total') - Coalesce(F('payment_sum'), 0) + Coalesce(F('refund_sum'), 0),
-            pending_sum_rc=-1 * Coalesce(F('payment_sum'), 0) + Coalesce(F('refund_sum'), 0),
-        ).annotate(
-            is_overpaid=Case(
-                When(~Q(status=Order.STATUS_CANCELED) & Q(pending_sum_t__lt=-1e-8),
-                     then=Value('1')),
-                When(Q(status=Order.STATUS_CANCELED) & Q(pending_sum_rc__lt=-1e-8),
-                     then=Value('1')),
-                default=Value('0'),
-                output_field=models.IntegerField()
-            ),
-            is_pending_with_full_payment=Case(
-                When(Q(status__in=(Order.STATUS_EXPIRED, Order.STATUS_PENDING)) & Q(pending_sum_t__lte=1e-8)
-                     & Q(require_approval=False),
-                     then=Value('1')),
-                default=Value('0'),
-                output_field=models.IntegerField()
-            ),
-            is_underpaid=Case(
-                When(Q(status=Order.STATUS_PAID) & Q(pending_sum_t__gt=1e-8),
-                     then=Value('1')),
-                default=Value('0'),
-                output_field=models.IntegerField()
-            )
         )
+        if refunds:
+            qs = qs.annotate(
+                has_external_refund=Exists(external_refund),
+                has_pending_refund=Exists(pending_refund),
+            )
+        if results:
+            qs = qs.annotate(
+                is_overpaid=Case(
+                    When(~Q(status=Order.STATUS_CANCELED) & Q(pending_sum_t__lt=-1e-8),
+                         then=Value('1')),
+                    When(Q(status=Order.STATUS_CANCELED) & Q(pending_sum_rc__lt=-1e-8),
+                         then=Value('1')),
+                    default=Value('0'),
+                    output_field=models.IntegerField()
+                ),
+                is_pending_with_full_payment=Case(
+                    When(Q(status__in=(Order.STATUS_EXPIRED, Order.STATUS_PENDING)) & Q(pending_sum_t__lte=1e-8)
+                         & Q(require_approval=False),
+                         then=Value('1')),
+                    default=Value('0'),
+                    output_field=models.IntegerField()
+                ),
+                is_underpaid=Case(
+                    When(Q(status=Order.STATUS_PAID) & Q(pending_sum_t__gt=1e-8),
+                         then=Value('1')),
+                    default=Value('0'),
+                    output_field=models.IntegerField()
+                )
+            )
         return qs
 
     @property

src/pretix/control/forms/filter.py

@@ -175,7 +175,7 @@ class OrderFilterForm(FilterForm):
 
 class EventOrderFilterForm(OrderFilterForm):
     orders = {'code': 'code', 'email': 'email', 'total': 'total',
-              'datetime': 'datetime', 'status': 'status', 'pcnt': 'pcnt'}
+              'datetime': 'datetime', 'status': 'status'}
 
     item = forms.ModelChoiceField(
         label=_('Products'),
@@ -275,16 +275,19 @@ class EventOrderFilterForm(OrderFilterForm):
                 qs = qs.annotate(has_answer=Exists(answers)).filter(has_answer=True)
 
         if fdata.get('status') == 'overpaid':
+            qs = Order.annotate_overpayments(qs, refunds=False, results=False, sums=True)
             qs = qs.filter(
                 Q(~Q(status=Order.STATUS_CANCELED) & Q(pending_sum_t__lt=0))
                 | Q(Q(status=Order.STATUS_CANCELED) & Q(pending_sum_rc__lt=0))
             )
         elif fdata.get('status') == 'pendingpaid':
+            qs = Order.annotate_overpayments(qs, refunds=False, results=False, sums=True)
             qs = qs.filter(
                 Q(status__in=(Order.STATUS_EXPIRED, Order.STATUS_PENDING)) & Q(pending_sum_t__lte=0)
                 & Q(require_approval=False)
             )
         elif fdata.get('status') == 'underpaid':
+            qs = Order.annotate_overpayments(qs, refunds=False, results=False, sums=True)
             qs = qs.filter(
                 status=Order.STATUS_PAID,
                 pending_sum_t__gt=0
@@ -300,7 +303,7 @@ class EventOrderFilterForm(OrderFilterForm):
 
 class OrderSearchFilterForm(OrderFilterForm):
     orders = {'code': 'code', 'email': 'email', 'total': 'total',
-              'datetime': 'datetime', 'status': 'status', 'pcnt': 'pcnt',
+              'datetime': 'datetime', 'status': 'status',
               'event': 'event'}
 
     organizer = forms.ModelChoiceField(

src/pretix/control/templates/pretixcontrol/orders/index.html

@@ -101,9 +101,7 @@
                     <th class="text-right">{% trans "Order total" %}
                         <a href="?{% url_replace request 'ordering' '-total' %}"><i class="fa fa-caret-down"></i></a>
                         <a href="?{% url_replace request 'ordering' 'total' %}"><i class="fa fa-caret-up"></i></a></th>
-                    <th class="text-right">{% trans "Positions" %}
+                    <th class="text-right">{% trans "Positions" %}</th>
-                        <a href="?{% url_replace request 'ordering' '-pcnt' %}"><i class="fa fa-caret-down"></i></a>
-                        <a href="?{% url_replace request 'ordering' 'pcnt' %}"><i class="fa fa-caret-up"></i></a></th>
                     <th class="text-right">{% trans "Status" %}
                         <a href="?{% url_replace request 'ordering' '-status' %}"><i class="fa fa-caret-down"></i></a>
                         <a href="?{% url_replace request 'ordering' 'status' %}"><i class="fa fa-caret-up"></i></a></th>

src/pretix/control/views/item.py

@@ -860,9 +860,7 @@ class ItemUpdateGeneral(ItemDetailMixin, EventPermissionRequiredMixin, UpdateVie
         messages.success(self.request, _('Your changes have been saved.'))
         if form.has_changed() or any(f.has_changed() for f in self.plugin_forms):
             data = {
-                k: (form.cleaned_data.get(k).name
+                k: form.cleaned_data.get(k)
-                    if isinstance(form.cleaned_data.get(k), File)
-                    else form.cleaned_data.get(k))
                 for k in form.changed_data
             }
             for f in self.plugin_forms:

src/pretix/control/views/orders.py

@@ -83,23 +83,46 @@ class OrderList(EventPermissionRequiredMixin, PaginationMixin, ListView):
     permission = 'can_view_orders'
 
     def get_queryset(self):
-        s = OrderPosition.objects.filter(
-            order=OuterRef('pk')
-        ).order_by().values('order').annotate(k=Count('id')).values('k')
         qs = Order.objects.filter(
             event=self.request.event
-        ).annotate(pcnt=Subquery(s, output_field=IntegerField())).select_related('invoice_address')
+        ).select_related('invoice_address')
-
-        qs = Order.annotate_overpayments(qs)
 
         if self.filter_form.is_valid():
             qs = self.filter_form.filter_qs(qs)
 
-        return qs.distinct()
+        return qs
 
     def get_context_data(self, **kwargs):
         ctx = super().get_context_data(**kwargs)
         ctx['filter_form'] = self.filter_form
+
+        # Only compute this annotations for this page (query optimization)
+        s = OrderPosition.objects.filter(
+            order=OuterRef('pk')
+        ).order_by().values('order').annotate(k=Count('id')).values('k')
+        annotated = {
+            o['pk']: o
+            for o in
+            Order.annotate_overpayments(Order.objects).filter(
+                pk__in=[o.pk for o in ctx['orders']]
+            ).annotate(
+                pcnt=Subquery(s, output_field=IntegerField())
+            ).values(
+                'pk', 'pcnt', 'is_overpaid', 'is_underpaid', 'is_pending_with_full_payment', 'has_external_refund',
+                'has_pending_refund'
+            )
+        }
+
+        for o in ctx['orders']:
+            if o.pk not in annotated:
+                continue
+            o.pcnt = annotated.get(o.pk)['pcnt']
+            o.is_overpaid = annotated.get(o.pk)['is_overpaid']
+            o.is_underpaid = annotated.get(o.pk)['is_underpaid']
+            o.is_pending_with_full_payment = annotated.get(o.pk)['is_pending_with_full_payment']
+            o.has_external_refund = annotated.get(o.pk)['has_external_refund']
+            o.has_pending_refund = annotated.get(o.pk)['has_pending_refund']
+
         return ctx
 
     @cached_property

src/pretix/control/views/search.py

@@ -20,16 +20,33 @@ class OrderSearch(PaginationMixin, ListView):
     def get_context_data(self, **kwargs):
         ctx = super().get_context_data()
         ctx['filter_form'] = self.filter_form
+
+        # Only compute this annotations for this page (query optimization)
+        s = OrderPosition.objects.filter(
+            order=OuterRef('pk')
+        ).order_by().values('order').annotate(k=Count('id')).values('k')
+        annotated = {
+            o['pk']: o
+            for o in
+            Order.objects.filter(
+                pk__in=[o.pk for o in ctx['orders']]
+            ).annotate(
+                pcnt=Subquery(s, output_field=IntegerField())
+            ).values(
+                'pk', 'pcnt',
+            )
+        }
+
+        for o in ctx['orders']:
+            if o.pk not in annotated:
+                continue
+            o.pcnt = annotated.get(o.pk)['pcnt']
+
         return ctx
 
     def get_queryset(self):
         qs = Order.objects.select_related('invoice_address')
 
-        s = OrderPosition.objects.filter(
-            order=OuterRef('pk')
-        ).order_by().values('order').annotate(k=Count('id')).values('k')
-        qs = qs.annotate(pcnt=Subquery(s, output_field=IntegerField()))
-
         if not self.request.user.has_active_staff_session(self.request.session.session_key):
             qs = qs.filter(
                 Q(event__organizer_id__in=self.request.user.teams.filter(