Improve performance of global order search

src/pretix/base/migrations/0079_auto_20180115_0855.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.8 on 2018-01-15 08:55
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+from django.db.models import F
+from django.db.models.functions import Concat
+
+
+def set_full_invoice_no(app, schema_editor):
+    Invoice = app.get_model('pretixbase', 'Invoice')
+    Invoice.objects.all().update(
+        full_invoice_no=Concat(F('prefix'), F('invoice_no'))
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pretixbase', '0078_auto_20171206_1603'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='invoice',
+            name='full_invoice_no',
+            field=models.CharField(db_index=True, default='', max_length=190),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='question',
+            name='type',
+            field=models.CharField(choices=[('N', 'Number'), ('S', 'Text (one line)'), ('T', 'Multiline text'), ('B', 'Yes/No'), ('C', 'Choose one from a list'), ('M', 'Choose multiple from a list'), ('F', 'File upload'), ('D', 'Date'), ('H', 'Time'), ('W', 'Date and time')], max_length=5, verbose_name='Question type'),
+        ),
+        migrations.RunPython(
+            set_full_invoice_no,
+            migrations.RunPython.noop
+        )
+    ]

src/pretix/base/models/invoices.py

@@ -41,6 +41,8 @@ class Invoice(models.Model):
     :type invoice_from: str
     :param invoice_to: The receiver address
     :type invoice_to: str
+    :param full_invoice_no: The full invoice number (for performance reasons only)
+    :type full_invoice_no: str
     :param date: The invoice date
     :type date: date
     :param locale: The locale in which the invoice should be printed
@@ -67,6 +69,7 @@ class Invoice(models.Model):
     event = models.ForeignKey('Event', related_name='invoices', db_index=True)
     prefix = models.CharField(max_length=160, db_index=True)
     invoice_no = models.CharField(max_length=19, db_index=True)
+    full_invoice_no = models.CharField(max_length=190, db_index=True)
     is_cancellation = models.BooleanField(default=False)
     refers = models.ForeignKey('Invoice', related_name='refered', null=True, blank=True)
     invoice_from = models.TextField()
@@ -122,6 +125,8 @@ class Invoice(models.Model):
                     # Suppress duplicate key errors and try again
                     if i == 9:
                         raise
+
+        self.full_invoice_no = self.prefix + self.invoice_no
         return super().save(*args, **kwargs)
 
     def delete(self, *args, **kwargs):

src/pretix/control/forms/filter.py

@@ -1,11 +1,13 @@
 from django import forms
 from django.apps import apps
-from django.db.models import F, Q
+from django.db.models import Exists, F, OuterRef, Q
-from django.db.models.functions import Coalesce, Concat
+from django.db.models.functions import Coalesce
 from django.utils.timezone import now
 from django.utils.translation import pgettext_lazy, ugettext_lazy as _
 
-from pretix.base.models import Event, Invoice, Item, Order, Organizer, SubEvent
+from pretix.base.models import (
+    Event, Invoice, Item, Order, OrderPosition, Organizer, SubEvent,
+)
 from pretix.base.signals import register_payment_providers
 from pretix.control.utils.i18n import i18ncomp
 from pretix.helpers.database import FixedOrderBy, rolledback_transaction
@@ -115,22 +117,25 @@ class OrderFilterForm(FilterForm):
             else:
                 code = Q(code__icontains=Order.normalize_code(u))
 
-            matching_invoices = Invoice.objects.annotate(
+            matching_invoices = Invoice.objects.filter(
-                inr=Concat('prefix', 'invoice_no')
-            ).filter(
                 Q(invoice_no__iexact=u)
                 | Q(invoice_no__iexact=u.zfill(5))
-                | Q(inr=u)
+                | Q(full_invoice_no__iexact=u)
             ).values_list('order_id', flat=True)
 
-            qs = qs.filter(
+            matching_positions = OrderPosition.objects.filter(
+                Q(order=OuterRef('pk')) & Q(
+                    Q(attendee_name__icontains=u) & Q(attendee_email__icontains=u)
+                )
+            ).values('id')
+
+            qs = qs.annotate(has_pos=Exists(matching_positions)).filter(
                 code
                 | Q(email__icontains=u)
-                | Q(positions__attendee_name__icontains=u)
-                | Q(positions__attendee_email__icontains=u)
                 | Q(invoice_address__name__icontains=u)
                 | Q(invoice_address__company__icontains=u)
                 | Q(pk__in=matching_invoices)
+                | Q(has_pos=True)
             )
 
         if fdata.get('status'):

src/pretix/control/templates/pretixcontrol/pagination_huge.html

@@ -0,0 +1,48 @@
+{% load i18n %}
+{% load urlreplace %}
+<nav class="text-center pagination-container">
+    <ul class="pagination">
+        {% if is_paginated %}
+            {% if page_obj.has_previous %}
+                <li>
+                    <a href="?{% url_replace request 'page' page_obj.previous_page_number %}">
+                        <span>&laquo;</span>
+                    </a>
+                </li>
+            {% endif %}
+            <li class="page-current"><a>
+                {% blocktrans trimmed with page=page_obj.number of=page_obj.paginator.num_pages count=page_obj.paginator.count %}
+                    Page {{ page }}
+                {% endblocktrans %}
+            </a></li>
+            {% if page_obj.has_next %}
+                <li>
+                        <a href="?{% url_replace request 'page' page_obj.next_page_number %}">
+                        <span>&raquo;</span>
+                    </a>
+                </li>
+            {% endif %}
+        {% else %}
+        {% if page_obj.paginator.count > 1 %}
+                <li class="page-current"><a>
+                    {% blocktrans trimmed with count=page_obj.paginator.count %}
+                        {{ count }} elements
+                    {% endblocktrans %}
+                </a></li>
+            {% endif %}
+        {% endif %}
+    </ul>
+    {% if page_size %}
+        <div class="clearfix">
+            <small>
+                {% trans "Show per page:" %}
+            </small>
+            <a href="?{% url_replace request "page_size" "25" "page" "1" %}">
+                {% if page_size == 25 %}<strong>{% endif %}25{% if page_size == 25 %}</strong>{% endif %}</a> |
+            <a href="?{% url_replace request "page_size" "50" "page" "1" %}">
+                {% if page_size == 50 %}<strong>{% endif %}50{% if page_size == 50 %}</strong>{% endif %}</a> |
+            <a href="?{% url_replace request "page_size" "100" "page" "1" %}">
+                {% if page_size == 100 %}<strong>{% endif %}100{% if page_size == 100 %}</strong>{% endif %}</a>
+        </div>
+    {% endif %}
+</nav>

src/pretix/control/templates/pretixcontrol/search/orders.html

@@ -47,9 +47,6 @@
                 <th class="text-right">{% trans "Order total" %}
                     <a href="?{% url_replace request 'ordering' '-total' %}"><i class="fa fa-caret-down"></i></a>
                     <a href="?{% url_replace request 'ordering' 'total' %}"><i class="fa fa-caret-up"></i></a></th>
-                <th class="text-right">{% trans "Positions" %}
-                    <a href="?{% url_replace request 'ordering' '-pcnt' %}"><i class="fa fa-caret-down"></i></a>
-                    <a href="?{% url_replace request 'ordering' 'pcnt' %}"><i class="fa fa-caret-up"></i></a></th>
                 <th class="text-right">{% trans "Status" %}
                     <a href="?{% url_replace request 'ordering' '-status' %}"><i class="fa fa-caret-down"></i></a>
                     <a href="?{% url_replace request 'ordering' 'status' %}"><i class="fa fa-caret-up"></i></a></th>
@@ -74,7 +71,6 @@
                     </td>
                     <td>{{ o.datetime|date:"SHORT_DATETIME_FORMAT" }}</td>
                     <td class="text-right">{{ o.total|floatformat:2 }} {{ o.event.currency }}</td>
-                    <td class="text-right">{{ o.pcnt }}</td>
                     <td class="text-right">{% include "pretixcontrol/orders/fragment_order_status.html" with order=o %}</td>
                 </tr>
             {% empty %}
@@ -87,5 +83,5 @@
             </tbody>
         </table>
     </div>
-    {% include "pretixcontrol/pagination.html" %}
+    {% include "pretixcontrol/pagination_huge.html" %}
 {% endblock %}

src/pretix/control/views/__init__.py

@@ -1,3 +1,10 @@
+import collections
+import warnings
+
+from django.core.paginator import (
+    EmptyPage, PageNotAnInteger, UnorderedObjectListWarning,
+)
+from django.utils.translation import ugettext_lazy as _
 from django.views.generic import edit
 
 
@@ -56,3 +63,122 @@ class PaginationMixin:
         ctx = super().get_context_data(**kwargs)
         ctx['page_size'] = self.get_paginate_by(None)
         return ctx
+
+
+class LargeResultSetPage(collections.Sequence):
+
+    def __init__(self, object_list, number, paginator):
+        self.object_list = object_list
+        self.number = number
+        self.paginator = paginator
+
+    def __repr__(self):
+        return '<Page %s>' % self.number
+
+    def __len__(self):
+        return len(self.object_list)
+
+    def __getitem__(self, index):
+        if not isinstance(index, (slice, int)):
+            raise TypeError
+        # The object_list is converted to a list so that if it was a QuerySet
+        # it won't be a database hit per __getitem__.
+        if not isinstance(self.object_list, list):
+            self.object_list = list(self.object_list)
+        return self.object_list[index]
+
+    def has_next(self):
+        try:
+            return self[self.paginator.per_page - 1]
+        except:
+            return False
+
+    def has_previous(self):
+        return self.number > 1
+
+    def has_other_pages(self):
+        return self.has_previous() or self.has_next()
+
+    def next_page_number(self):
+        return self.paginator.validate_number(self.number + 1)
+
+    def previous_page_number(self):
+        return self.paginator.validate_number(self.number - 1)
+
+    def start_index(self):
+        """
+        Returns the 1-based index of the first object on this page,
+        relative to total objects in the paginator.
+        """
+        # Special case, return zero if no items.
+        if self.paginator.count == 0:
+            return 0
+        return (self.paginator.per_page * (self.number - 1)) + 1
+
+    def end_index(self):
+        """
+        Returns the 1-based index of the last object on this page,
+        relative to total objects found (hits).
+        """
+        # Special case for the last page because there can be orphans.
+        if self.number == self.paginator.num_pages:
+            return self.paginator.count
+        return self.number * self.paginator.per_page
+
+
+class LargeResultSetPaginator(object):
+
+    def __init__(self, object_list, per_page, orphans=0,
+                 allow_empty_first_page=True):
+        self.object_list = object_list
+        self._check_object_list_is_ordered()
+        self.per_page = int(per_page)
+        self.orphans = int(orphans)
+
+    def validate_number(self, number):
+        """
+        Validates the given 1-based page number.
+        """
+        try:
+            number = int(number)
+        except (TypeError, ValueError):
+            raise PageNotAnInteger(_('That page number is not an integer'))
+        if number < 1:
+            raise EmptyPage(_('That page number is less than 1'))
+        return number
+
+    def page(self, number):
+        """
+        Returns a Page object for the given 1-based page number.
+        """
+        number = self.validate_number(number)
+        bottom = (number - 1) * self.per_page
+        top = bottom + self.per_page
+        return self._get_page(self.object_list[bottom:top], number, self)
+
+    def _get_page(self, *args, **kwargs):
+        """
+        Returns an instance of a single page.
+
+        This hook can be used by subclasses to use an alternative to the
+        standard :cls:`Page` object.
+        """
+        return LargeResultSetPage(*args, **kwargs)
+
+    def _check_object_list_is_ordered(self):
+        """
+        Warn if self.object_list is unordered (typically a QuerySet).
+        """
+        ordered = getattr(self.object_list, 'ordered', None)
+        if ordered is not None and not ordered:
+            obj_list_repr = (
+                '{} {}'.format(self.object_list.model, self.object_list.__class__.__name__)
+                if hasattr(self.object_list, 'model')
+                else '{!r}'.format(self.object_list)
+            )
+            warnings.warn(
+                'Pagination may yield inconsistent results with an unordered '
+                'object_list: {}.'.format(obj_list_repr),
+                UnorderedObjectListWarning,
+                stacklevel=3
+            )

src/pretix/control/views/search.py

@@ -1,14 +1,15 @@
-from django.db.models import Count, Q
+from django.db.models import Q
 from django.utils.functional import cached_property
 from django.views.generic import ListView
 
 from pretix.base.models import Order
 from pretix.control.forms.filter import OrderSearchFilterForm
-from pretix.control.views import PaginationMixin
+from pretix.control.views import LargeResultSetPaginator, PaginationMixin
 
 
 class OrderSearch(PaginationMixin, ListView):
     model = Order
+    paginator_class = LargeResultSetPaginator
     context_object_name = 'orders'
     template_name = 'pretixcontrol/search/orders.html'
 
@@ -22,7 +23,7 @@ class OrderSearch(PaginationMixin, ListView):
         return ctx
 
     def get_queryset(self):
-        qs = Order.objects.all().annotate(pcnt=Count('positions', distinct=True)).select_related('invoice_address')
+        qs = Order.objects.select_related('invoice_address')
         if not self.request.user.is_superuser:
             qs = qs.filter(
                 Q(event__organizer_id__in=self.request.user.teams.filter(
@@ -34,7 +35,7 @@ class OrderSearch(PaginationMixin, ListView):
         if self.filter_form.is_valid():
             qs = self.filter_form.filter_qs(qs)
 
-        return qs.distinct().only(
+        return qs.only(
             'id', 'invoice_address__name', 'code', 'event', 'email', 'datetime', 'total', 'status'
         ).prefetch_related(
             'event', 'event__organizer'