Search experiments

src/pretix/base/management/commands/reindex_orders.py

@@ -0,0 +1,69 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020  Raphael Michel and contributors
+# Copyright (C) 2020-today pretix GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+import time
+
+from django.core.management.base import BaseCommand
+from django_scopes import scopes_disabled
+from tqdm import tqdm
+
+from pretix.base.models import (
+    Invoice, InvoiceAddress, Order, OrderPayment, OrderPosition, OrderRefund,
+)
+
+
+class Command(BaseCommand):
+    help = "Recreate order search index"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--slowdown",
+            dest="interval",
+            type=int,
+            default=0,
+            help="Interval for staggered execution. If set to a value different then zero, we will "
+                 "wait this many milliseconds between every order we process.",
+        )
+
+    @scopes_disabled()
+    def handle(self, *args, **options):
+        querysets = [
+            Order.objects.all(),
+            OrderPosition.all.all(),
+            OrderPayment.objects.all(),
+            OrderRefund.objects.all(),
+            InvoiceAddress.objects.filter(order__isnull=False),
+            Invoice.objects.all(),
+        ]
+
+        for qs in querysets:
+            last_pk = 0
+            with tqdm(total=qs.count()) as pbar:
+                while True:
+                    batch = list(qs.filter(pk__gt=last_pk)[:5000])
+                    if not batch:
+                        break
+
+                    for o in batch:
+                        o._update_search_index()
+                        time.sleep(0)
+                        pbar.update(1)
+                    last_pk = batch[-1].pk

src/pretix/base/migrations/0300_ordersearchindex.py

@@ -0,0 +1,132 @@
+# Generated by Django 5.2.12 on 2026-05-22 12:40
+
+import django.contrib.postgres.indexes
+import django.contrib.postgres.search
+import django.db.models.deletion
+from django.contrib.postgres.operations import UnaccentExtension
+from django.db import migrations, models
+
+from pretix.helpers.migration_utils import PostgreSQLAndStateOnly
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("pretixbase", "0299_itemprogramtime_location"),
+    ]
+
+    operations = [
+        PostgreSQLAndStateOnly(
+            UnaccentExtension(),
+            migrations.RunSQL(
+                sql="""
+                CREATE TEXT SEARCH CONFIGURATION pretix_search( COPY = english );
+                ALTER TEXT SEARCH CONFIGURATION pretix_search
+                ALTER MAPPING FOR hword, hword_part, word
+                WITH unaccent, english_stem;
+                """,
+                reverse_sql="""
+                DROP TEXT SEARCH CONFIGURATION pretix_search;
+                """
+            ),
+            migrations.CreateModel(
+                name="OrderSearchIndex",
+                fields=[
+                    (
+                        "id",
+                        models.BigAutoField(
+                            auto_created=True, primary_key=True, serialize=False
+                        ),
+                    ),
+                    ("last_modified", models.DateTimeField(auto_now=True)),
+                    ("search_vector", django.contrib.postgres.search.SearchVectorField()),
+                    (
+                        "organizer",
+                        models.ForeignKey(
+                            on_delete=django.db.models.deletion.CASCADE,
+                            to="pretixbase.organizer",
+                        ),
+                    ),
+                    (
+                        "event",
+                        models.ForeignKey(
+                            on_delete=django.db.models.deletion.CASCADE,
+                            to="pretixbase.event",
+                        ),
+                    ),
+                    (
+                        "order",
+                        models.ForeignKey(
+                            on_delete=django.db.models.deletion.CASCADE,
+                            to="pretixbase.order",
+                        ),
+                    ),
+                    (
+                        "orderpayment",
+                        models.OneToOneField(
+                            null=True,
+                            on_delete=django.db.models.deletion.CASCADE,
+                            related_name="search_index",
+                            to="pretixbase.orderpayment",
+                        ),
+                    ),
+                    (
+                        "orderposition",
+                        models.OneToOneField(
+                            null=True,
+                            on_delete=django.db.models.deletion.CASCADE,
+                            related_name="search_index",
+                            to="pretixbase.orderposition",
+                        ),
+                    ),
+                    (
+                        "orderrefund",
+                        models.OneToOneField(
+                            null=True,
+                            on_delete=django.db.models.deletion.CASCADE,
+                            related_name="search_index",
+                            to="pretixbase.orderrefund",
+                        ),
+                    ),
+                    (
+                        "invoiceaddress",
+                        models.OneToOneField(
+                            null=True,
+                            on_delete=django.db.models.deletion.CASCADE,
+                            related_name="search_index",
+                            to="pretixbase.invoiceaddress",
+                        ),
+                    ),
+                    (
+                        "invoice",
+                        models.OneToOneField(
+                            null=True,
+                            on_delete=django.db.models.deletion.CASCADE,
+                            related_name="search_index",
+                            to="pretixbase.invoice",
+                        ),
+                    ),
+                ],
+                options={
+                    "indexes": [
+                        django.contrib.postgres.indexes.GinIndex(
+                            models.F("search_vector"), name="ordersearchindex_vector"
+                        )
+                    ],
+                    "constraints": [
+                        models.UniqueConstraint(
+                            condition=models.Q(
+                                ("orderpayment__isnull", True),
+                                ("orderposition__isnull", True),
+                                ("orderrefund__isnull", True),
+                                ("invoiceaddress__isnull", True),
+                                ("invoice__isnull", True),
+                            ),
+                            fields=("order",),
+                            name="ordersearchindex_one_per_order",
+                        ),
+                    ],
+                },
+            ),
+        ),
+    ]

src/pretix/base/models/__init__.py

@@ -56,6 +56,7 @@ from .organizer import (
     Organizer, Organizer_SettingsStore, SalesChannel, Team, TeamAPIToken,
     TeamInvite,
 )
+from .search import OrderSearchIndex
 from .seating import Seat, SeatCategoryMapping, SeatingPlan
 from .tax import TaxRule
 from .vouchers import Voucher

src/pretix/base/models/invoices.py

@@ -47,6 +47,7 @@ from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _, pgettext
 from django_scopes import ScopedManager
 
+from pretix.base.models.orders import SearchIndexModelMixin
 from pretix.base.settings import COUNTRIES_WITH_STATE_IN_ADDRESS
 from pretix.helpers.countries import FastCountryField
 
@@ -64,7 +65,7 @@ def today():
     return timezone.now().date()
 
 
-class Invoice(models.Model):
+class Invoice(SearchIndexModelMixin, models.Model):
     """
     Represents an invoice that is issued because of an order. Because invoices are legally required
     not to change, this object duplicates a lot of data (e.g. the invoice address).
@@ -206,6 +207,10 @@ class Invoice(models.Model):
     plugin_data = models.JSONField(default=dict)
 
     objects = ScopedManager(organizer='event__organizer')
+    search_index_fields = [
+        "invoice_no",
+        "full_invoice_no",
+    ]
 
     @staticmethod
     def _to_numeric_invoice_number(number, places):

src/pretix/base/models/orders.py

@@ -135,7 +135,33 @@ class OrderQuerySet(models.QuerySet):
         return order
 
 
-class Order(LockModel, LoggedModel):
+class SearchIndexModelMixin:
+    search_index_fields = []
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        if "update_fields" in kwargs:
+            update_fields = kwargs["update_fields"]
+            if any(f in update_fields for f in self.search_index_fields):
+                self._update_search_index()
+        else:
+            self._update_search_index()
+
+    def get_search_index_content(self):
+        for f in self.search_index_fields:
+            val = getattr(self, f)
+            if val is not None:
+                yield str(val)
+
+    def _update_search_index(self):
+        from .search import OrderSearchIndex
+
+        contents = list(self.get_search_index_content())
+        print(self, contents)
+        OrderSearchIndex.update_for(self, contents)
+
+
+class Order(LockModel, SearchIndexModelMixin, LoggedModel):
     """
     An order is created when a user clicks 'buy' on his cart. It holds
     several OrderPositions and is connected to a user. It has an
@@ -330,6 +356,12 @@ class Order(LockModel, LoggedModel):
     )
 
     objects = ScopedManager(OrderQuerySet.as_manager().__class__, organizer='event__organizer')
+    search_index_fields = [
+        "code",
+        "comment",
+        "email",
+        "phone",
+    ]
 
     class Meta:
         verbose_name = _("Order")
@@ -351,6 +383,10 @@ class Order(LockModel, LoggedModel):
         if 'require_approval' not in self.get_deferred_fields() and 'status' not in self.get_deferred_fields():
             self._transaction_key_reset()
 
+    def get_search_index_content(self):
+        yield from super().get_search_index_content()
+        yield self.event.slug.upper() + "-" + self.code
+
     def _transaction_key_reset(self):
         self.__initial_status_paid_or_pending = self.status in (Order.STATUS_PENDING, Order.STATUS_PAID) and not self.require_approval
 
@@ -1690,7 +1726,7 @@ class AbstractPosition(RoundingCorrectionMixin, models.Model):
         return False
 
 
-class OrderPayment(models.Model):
+class OrderPayment(SearchIndexModelMixin, models.Model):
     """
     Represents a payment or payment attempt for an order.
 
@@ -1774,6 +1810,9 @@ class OrderPayment(models.Model):
     )
 
     objects = ScopedManager(organizer='order__event__organizer')
+    search_index_fields = [
+        "info",  # only for triggering the right updates
+    ]
 
     class Meta:
         ordering = ('local_id',)
@@ -1781,6 +1820,14 @@ class OrderPayment(models.Model):
     def __str__(self):
         return self.full_id
 
+    def get_search_index_content(self):
+        try:
+            return [
+                self.payment_provider.matching_id(self)
+            ]
+        except:
+            pass
+
     @property
     def info_data(self):
         """
@@ -2093,7 +2140,7 @@ class OrderPayment(models.Model):
         return r
 
 
-class OrderRefund(models.Model):
+class OrderRefund(SearchIndexModelMixin, models.Model):
     """
     Represents a refund or refund attempt for an order.
 
@@ -2195,6 +2242,9 @@ class OrderRefund(models.Model):
     )
 
     objects = ScopedManager(organizer='order__event__organizer')
+    search_index_fields = [
+        "info",  # only for triggering the right updates
+    ]
 
     class Meta:
         ordering = ('local_id',)
@@ -2202,6 +2252,14 @@ class OrderRefund(models.Model):
     def __str__(self):
         return self.full_id
 
+    def get_search_index_content(self):
+        try:
+            return [
+                self.payment_provider.refund_matching_id(self)
+            ]
+        except:
+            pass
+
     @property
     def info_data(self):
         """
@@ -2472,7 +2530,7 @@ class OrderFee(RoundingCorrectionMixin, models.Model):
         self.value_includes_rounding_correction = value
 
 
-class OrderPosition(AbstractPosition):
+class OrderPosition(SearchIndexModelMixin, AbstractPosition):
     """
     An OrderPosition is one line of an order, representing one ordered item
     of a specified type (or variation). This has all properties of
@@ -2579,6 +2637,13 @@ class OrderPosition(AbstractPosition):
 
     all = ScopedManager(organizer='order__event__organizer')
     objects = ActivePositionManager()
+    search_index_fields = [
+        "attendee_name_cached",
+        "attendee_email",
+        "company",
+        "secret",
+        "pseudonymization_id",
+    ]
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -3335,7 +3400,7 @@ class CartPosition(AbstractPosition):
         return self.predicted_validity[1]
 
 
-class InvoiceAddress(models.Model):
+class InvoiceAddress(SearchIndexModelMixin, models.Model):
     last_modified = models.DateTimeField(auto_now=True)
     order = models.OneToOneField(Order, null=True, blank=True, related_name='invoice_address', on_delete=models.CASCADE)
     customer = models.ForeignKey(
@@ -3373,6 +3438,10 @@ class InvoiceAddress(models.Model):
 
     objects = ScopedManager(organizer='order__event__organizer')
     profiles = ScopedManager(organizer='customer__organizer')
+    search_index_fields = [
+        "name_cached",
+        "company",
+    ]
 
     def save(self, **kwargs):
         if self.order:

src/pretix/base/models/search.py

@@ -0,0 +1,158 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020  Raphael Michel and contributors
+# Copyright (C) 2020-today pretix GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+from typing import List, Union
+
+from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.search import (
+    SearchQuery, SearchVector, SearchVectorField,
+)
+from django.db import models
+from django.db.models import Value
+from django_scopes.manager import ScopedManager
+
+from pretix.base.models import (
+    Event, Invoice, InvoiceAddress, Order, OrderPayment, OrderPosition,
+    OrderRefund, Organizer,
+)
+
+
+class SearchIndexQuerySet(models.QuerySet):
+    def search_for(self, query: str):
+        sq = SearchQuery(query, config="pretix_search", search_type="raw")  # todo: probably use plain
+        if query.count(" ") < 1 and "@" not in query:
+            # Order code normalization, only apply for one-word search that is not an email address
+            sq |= SearchQuery(Order.normalize_code(query.rsplit("-", 1)[-1]), config="pretix_search", search_type="plain")
+
+            if query.isdigit():
+                for i in range(2, 12):
+                    sq |= SearchQuery(query.zfill(i), config="pretix_search", search_type="plain")
+
+        return self.filter(search_vector=sq)
+
+
+class OrderSearchIndex(models.Model):
+    organizer = models.ForeignKey(Organizer, on_delete=models.CASCADE)
+    event = models.ForeignKey(Event, on_delete=models.CASCADE)
+    order = models.ForeignKey(Order, on_delete=models.CASCADE)
+    orderposition = models.OneToOneField(
+        OrderPosition,
+        on_delete=models.CASCADE,
+        related_name="search_index",
+        null=True,
+        blank=True,
+    )
+    orderpayment = models.OneToOneField(
+        OrderPayment,
+        on_delete=models.CASCADE,
+        related_name="search_index",
+        null=True,
+        blank=True,
+    )
+    orderrefund = models.OneToOneField(
+        OrderRefund,
+        on_delete=models.CASCADE,
+        related_name="search_index",
+        null=True,
+        blank=True,
+    )
+    invoiceaddress = models.OneToOneField(
+        InvoiceAddress,
+        on_delete=models.CASCADE,
+        related_name="search_index",
+        null=True,
+        blank=True,
+    )
+    invoice = models.OneToOneField(
+        Invoice,
+        on_delete=models.CASCADE,
+        related_name="search_index",
+        null=True,
+        blank=True,
+    )
+    last_modified = models.DateTimeField(auto_now=True)
+    search_vector = SearchVectorField()
+
+    objects = ScopedManager(SearchIndexQuerySet.as_manager().__class__, organizer='organizer')
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                condition=models.Q(
+                    orderposition__isnull=True,
+                    orderpayment__isnull=True,
+                    orderrefund__isnull=True,
+                    invoiceaddress__isnull=True,
+                    invoice__isnull=True,
+                ),
+                fields=("order",),
+                name="ordersearchindex_one_per_order",
+            ),
+        ]
+        indexes = [
+            GinIndex("search_vector", name="ordersearchindex_vector"),
+        ]
+
+    def save(self, *args, **kwargs):
+        fields_filled = [
+            bool(self.orderposition_id),
+            bool(self.orderpayment_id),
+            bool(self.orderrefund_id),
+            bool(self.invoiceaddress_id),
+            bool(self.invoice_id),
+        ]
+        if fields_filled.count(True) > 1:
+            raise ValueError("A OrderSearchIndex may one relate to one other instance")
+        super().save(*args, **kwargs)
+
+    @classmethod
+    def update_for(cls, obj: Union[Order, OrderPayment, OrderPosition, OrderRefund], inputs: List[str]):
+        index_text = "\n".join([i for i in inputs if i])
+        if isinstance(obj, Order):
+            order = obj
+        else:
+            order = obj.order
+
+        kwargs = dict(
+            order=order,
+            orderpayment=obj if isinstance(obj, OrderPayment) else None,
+            orderrefund=obj if isinstance(obj, OrderRefund) else None,
+            orderposition=obj if isinstance(obj, OrderPosition) else None,
+            invoiceaddress=obj if isinstance(obj, InvoiceAddress) else None,
+            invoice=obj if isinstance(obj, Invoice) else None,
+        )
+
+        if not index_text.strip():
+            OrderSearchIndex.objects.filter(**kwargs).delete()
+        else:
+            OrderSearchIndex.objects.update_or_create(
+                **kwargs,
+                defaults=dict(
+                    search_vector=SearchVector(
+                        Value(index_text),
+                        config="pretix_search"
+                    ),
+                ),
+                create_defaults=dict(
+                    event_id=order.event_id,
+                    organizer_id=order.organizer_id,
+                ),
+            )

src/pretix/base/settings.py

@@ -3719,6 +3719,10 @@ Your {organizer} team"""))  # noqa: W291
         'default': '',
         'type': str,
     },
+    'use_fts_beta': {
+        'default': 'False',
+        'type': bool,
+    }
 }
 PERSON_NAME_TITLE_GROUPS = OrderedDict([
     ('english_common', (_('Most common English titles'), (

src/pretix/control/forms/filter.py

@@ -57,10 +57,11 @@ from pretix.base.forms.widgets import (
 from pretix.base.models import (
     Checkin, CheckinList, Device, Event, EventMetaProperty, EventMetaValue,
     Gate, Invoice, InvoiceAddress, Item, Order, OrderPayment, OrderPosition,
-    OrderRefund, Organizer, OutgoingMail, Question, QuestionAnswer, Quota,
+    OrderRefund, OrderSearchIndex, Organizer, OutgoingMail, Question,
-    SalesChannel, SubEvent, SubEventMetaValue, Team, TeamAPIToken, TeamInvite,
+    QuestionAnswer, Quota, SalesChannel, SubEvent, SubEventMetaValue, Team,
-    Voucher,
+    TeamAPIToken, TeamInvite, Voucher,
 )
+from pretix.base.settings import GlobalSettingsObject
 from pretix.base.signals import register_payment_providers
 from pretix.base.timeframes import (
     DateFrameField,
@@ -262,7 +263,16 @@ class OrderFilterForm(FilterForm):
 
         if fdata.get('query'):
             u = fdata.get('query')
-
+            fts_beta = (
+                self.event.settings.use_fts_beta
+                if hasattr(self, 'event')
+                else GlobalSettingsObject().settings.use_fts_beta
+            )
+            if fts_beta:
+                qs = qs.filter(
+                    pk__in=OrderSearchIndex.objects.search_for(u).values_list("order_id", flat=True)
+                )
+            else:
                 if "-" in u:
                     code = (Q(event__slug__icontains=u.rsplit("-", 1)[0])
                             & Q(code__icontains=Order.normalize_code(u.rsplit("-", 1)[1])))

src/pretix/control/forms/global_settings.py

@@ -110,7 +110,11 @@ class GlobalSettingsForm(SettingsForm):
                 required=False,
                 label=_("Vite widget origins"),
                 help_text=_("One origin per line (e.g. https://example.com). Requests from these origins will be served the new vite-based widget."),
-            ))
+            )),
+            ('use_fts_beta', forms.BooleanField(
+                required=False,
+                label=_("Use full-text search beta"),
+            )),
         ])
         responses = register_global_settings.send(self)
         for r, response in sorted(responses, key=lambda r: str(r[0])):

src/pretix/helpers/migration_utils.py

@@ -0,0 +1,37 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020  Raphael Michel and contributors
+# Copyright (C) 2020-today pretix GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+from django.db import migrations
+
+
+class PostgreSQLAndStateOnly(migrations.SeparateDatabaseAndState):
+    def __init__(self, *operations):
+        super().__init__(database_operations=operations, state_operations=operations)
+
+    def database_forwards(self, app_label, schema_editor, from_state, to_state):
+        if schema_editor.connection.vendor != "postgresql":
+            return
+        super().database_forwards(app_label, schema_editor, from_state, to_state)
+
+    def database_backwards(self, app_label, schema_editor, from_state, to_state):
+        if schema_editor.connection.vendor != "postgresql":
+            return
+        super().database_forwards(app_label, schema_editor, from_state, to_state)