Search experiments

src/pretix/base/management/commands/reindex_orders.py

@@ -0,0 +1,95 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020  Raphael Michel and contributors
+# Copyright (C) 2020-today pretix GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+import time
+
+from django.core.management.base import BaseCommand
+from django.db.models import F, Max, Q
+from django.utils.timezone import now
+from django_scopes import scopes_disabled
+from tqdm import tqdm
+
+from pretix.base.models import Order
+
+
+class Command(BaseCommand):
+    help = "Recreate order search index"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--slowdown",
+            dest="interval",
+            type=int,
+            default=0,
+            help="Interval for staggered execution. If set to a value different then zero, we will "
+                 "wait this many milliseconds between every order we process.",
+        )
+
+    @scopes_disabled()
+    def handle(self, *args, **options):
+        t = 0
+        qs = Order.objects.annotate(
+            last_transaction=Max('transactions__created')
+        ).filter(
+            Q(last_transaction__isnull=True) | Q(last_modified__gt=F('last_transaction')),
+            require_approval=False,
+        ).prefetch_related(
+            'all_positions', 'all_fees'
+        ).order_by(
+            'pk'
+        )
+        last_pk = 0
+        with tqdm(total=qs.count()) as pbar:
+            while True:
+                batch = list(qs.filter(pk__gt=last_pk)[:5000])
+                if not batch:
+                    break
+
+                for o in batch:
+                    if o.last_transaction is None:
+                        tn = o.create_transactions(
+                            positions=o.all_positions.all(),
+                            fees=o.all_fees.all(),
+                            dt_now=o.datetime,
+                            migrated=True,
+                            is_new=True,
+                            _backfill_before_cancellation=True,
+                        )
+                        o.create_transactions(
+                            positions=o.all_positions.all(),
+                            fees=o.all_fees.all(),
+                            dt_now=o.cancellation_date or (o.expires if o.status == Order.STATUS_EXPIRED else o.datetime),
+                            migrated=True,
+                        )
+                    else:
+                        tn = o.create_transactions(
+                            positions=o.all_positions.all(),
+                            fees=o.all_fees.all(),
+                            dt_now=now(),
+                            migrated=True,
+                        )
+                    if tn:
+                        t += 1
+                    time.sleep(0)
+                    pbar.update(1)
+                last_pk = batch[-1].pk
+
+        self.stderr.write(self.style.SUCCESS(f'Created transactions for {t} orders.'))

src/pretix/base/migrations/0300_ordersearchindex.py

@@ -0,0 +1,132 @@
+# Generated by Django 5.2.12 on 2026-05-22 12:40
+
+import django.contrib.postgres.indexes
+import django.contrib.postgres.search
+import django.db.models.deletion
+from django.contrib.postgres.operations import UnaccentExtension
+from django.db import migrations, models
+
+from pretix.helpers.migration_utils import PostgreSQLAndStateOnly
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("pretixbase", "0299_itemprogramtime_location"),
+    ]
+
+    operations = [
+        PostgreSQLAndStateOnly(
+            UnaccentExtension(),
+            migrations.RunSQL(
+                sql="""
+                CREATE TEXT SEARCH CONFIGURATION pretix_search( COPY = english );
+                ALTER TEXT SEARCH CONFIGURATION pretix_search
+                ALTER MAPPING FOR hword, hword_part, word
+                WITH unaccent, english_stem;
+                """,
+                reverse_sql="""
+                DROP TEXT SEARCH CONFIGURATION pretix_search;
+                """
+            ),
+            migrations.CreateModel(
+                name="OrderSearchIndex",
+                fields=[
+                    (
+                        "id",
+                        models.BigAutoField(
+                            auto_created=True, primary_key=True, serialize=False
+                        ),
+                    ),
+                    ("last_modified", models.DateTimeField(auto_now=True)),
+                    ("search_vector", django.contrib.postgres.search.SearchVectorField()),
+                    (
+                        "organizer",
+                        models.ForeignKey(
+                            on_delete=django.db.models.deletion.CASCADE,
+                            to="pretixbase.organizer",
+                        ),
+                    ),
+                    (
+                        "event",
+                        models.ForeignKey(
+                            on_delete=django.db.models.deletion.CASCADE,
+                            to="pretixbase.event",
+                        ),
+                    ),
+                    (
+                        "order",
+                        models.ForeignKey(
+                            on_delete=django.db.models.deletion.CASCADE,
+                            to="pretixbase.order",
+                        ),
+                    ),
+                    (
+                        "orderpayment",
+                        models.OneToOneField(
+                            null=True,
+                            on_delete=django.db.models.deletion.CASCADE,
+                            related_name="search_index",
+                            to="pretixbase.orderpayment",
+                        ),
+                    ),
+                    (
+                        "orderposition",
+                        models.OneToOneField(
+                            null=True,
+                            on_delete=django.db.models.deletion.CASCADE,
+                            related_name="search_index",
+                            to="pretixbase.orderposition",
+                        ),
+                    ),
+                    (
+                        "orderrefund",
+                        models.OneToOneField(
+                            null=True,
+                            on_delete=django.db.models.deletion.CASCADE,
+                            related_name="search_index",
+                            to="pretixbase.orderrefund",
+                        ),
+                    ),
+                    (
+                        "invoiceaddress",
+                        models.OneToOneField(
+                            null=True,
+                            on_delete=django.db.models.deletion.CASCADE,
+                            related_name="search_index",
+                            to="pretixbase.invoiceaddress",
+                        ),
+                    ),
+                    (
+                        "invoice",
+                        models.OneToOneField(
+                            null=True,
+                            on_delete=django.db.models.deletion.CASCADE,
+                            related_name="search_index",
+                            to="pretixbase.invoice",
+                        ),
+                    ),
+                ],
+                options={
+                    "indexes": [
+                        django.contrib.postgres.indexes.GinIndex(
+                            models.F("search_vector"), name="ordersearchindex_vector"
+                        )
+                    ],
+                    "constraints": [
+                        models.UniqueConstraint(
+                            condition=models.Q(
+                                ("orderpayment__isnull", True),
+                                ("orderposition__isnull", True),
+                                ("orderrefund__isnull", True),
+                                ("invoiceaddress__isnull", True),
+                                ("invoice__isnull", True),
+                            ),
+                            fields=("order",),
+                            name="ordersearchindex_one_per_order",
+                        ),
+                    ],
+                },
+            ),
+        ),
+    ]

src/pretix/base/models/__init__.py

@@ -56,6 +56,7 @@ from .organizer import (
     Organizer, Organizer_SettingsStore, SalesChannel, Team, TeamAPIToken,
     TeamInvite,
 )
+from .search import OrderSearchIndex
 from .seating import Seat, SeatCategoryMapping, SeatingPlan
 from .tax import TaxRule
 from .vouchers import Voucher

src/pretix/base/models/invoices.py

@@ -47,6 +47,7 @@ from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _, pgettext
 from django_scopes import ScopedManager
 
+from pretix.base.models.orders import SearchIndexModelMixin
 from pretix.base.settings import COUNTRIES_WITH_STATE_IN_ADDRESS
 from pretix.helpers.countries import FastCountryField
 
@@ -64,7 +65,7 @@ def today():
     return timezone.now().date()
 
 
-class Invoice(models.Model):
+class Invoice(SearchIndexModelMixin, models.Model):
     """
     Represents an invoice that is issued because of an order. Because invoices are legally required
     not to change, this object duplicates a lot of data (e.g. the invoice address).
@@ -206,6 +207,10 @@ class Invoice(models.Model):
     plugin_data = models.JSONField(default=dict)
 
     objects = ScopedManager(organizer='event__organizer')
+    search_index_fields = [
+        "invoice_no",
+        "full_invoice_no",
+    ]
 
     @staticmethod
     def _to_numeric_invoice_number(number, places):

src/pretix/base/models/orders.py

@@ -135,7 +135,30 @@ class OrderQuerySet(models.QuerySet):
         return order
 
 
-class Order(LockModel, LoggedModel):
+class SearchIndexModelMixin:
+    search_index_fields = []
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        if "update_fields" in kwargs:
+            update_fields = kwargs["update_fields"]
+            if any(f in update_fields for f in self.search_index_fields):
+                self._update_search_index()
+        else:
+            self._update_search_index()
+
+    def get_search_index_content(self):
+        for f in self.search_index_fields:
+            yield str(getattr(self, f))
+
+    def _update_search_index(self):
+        from .search import OrderSearchIndex
+
+        contents = self.get_search_index_content()
+        OrderSearchIndex.update_for(self, contents)
+
+
+class Order(LockModel, SearchIndexModelMixin, LoggedModel):
     """
     An order is created when a user clicks 'buy' on his cart. It holds
     several OrderPositions and is connected to a user. It has an
@@ -330,6 +353,12 @@ class Order(LockModel, LoggedModel):
     )
 
     objects = ScopedManager(OrderQuerySet.as_manager().__class__, organizer='event__organizer')
+    search_index_fields = [
+        "code",
+        "comment",
+        "email",
+        "phone",
+    ]
 
     class Meta:
         verbose_name = _("Order")
@@ -351,6 +380,10 @@ class Order(LockModel, LoggedModel):
         if 'require_approval' not in self.get_deferred_fields() and 'status' not in self.get_deferred_fields():
             self._transaction_key_reset()
 
+    def get_search_index_content(self):
+        yield from super().get_search_index_content()
+        yield self.event.slug.upper() + "-" + self.code
+
     def _transaction_key_reset(self):
         self.__initial_status_paid_or_pending = self.status in (Order.STATUS_PENDING, Order.STATUS_PAID) and not self.require_approval
 
@@ -1690,7 +1723,7 @@ class AbstractPosition(RoundingCorrectionMixin, models.Model):
         return False
 
 
-class OrderPayment(models.Model):
+class OrderPayment(SearchIndexModelMixin, models.Model):
     """
     Represents a payment or payment attempt for an order.
 
@@ -1774,6 +1807,9 @@ class OrderPayment(models.Model):
     )
 
     objects = ScopedManager(organizer='order__event__organizer')
+    search_index_fields = [
+        "info",  # only for triggering the right updates
+    ]
 
     class Meta:
         ordering = ('local_id',)
@@ -1781,6 +1817,14 @@ class OrderPayment(models.Model):
     def __str__(self):
         return self.full_id
 
+    def get_search_index_content(self):
+        try:
+            return [
+                self.payment_provider.matching_id(self)
+            ]
+        except:
+            pass
+
     @property
     def info_data(self):
         """
@@ -2093,7 +2137,7 @@ class OrderPayment(models.Model):
         return r
 
 
-class OrderRefund(models.Model):
+class OrderRefund(SearchIndexModelMixin, models.Model):
     """
     Represents a refund or refund attempt for an order.
 
@@ -2195,6 +2239,9 @@ class OrderRefund(models.Model):
     )
 
     objects = ScopedManager(organizer='order__event__organizer')
+    search_index_fields = [
+        "info",  # only for triggering the right updates
+    ]
 
     class Meta:
         ordering = ('local_id',)
@@ -2202,6 +2249,14 @@ class OrderRefund(models.Model):
     def __str__(self):
         return self.full_id
 
+    def get_search_index_content(self):
+        try:
+            return [
+                self.payment_provider.refund_matching_id(self)
+            ]
+        except:
+            pass
+
     @property
     def info_data(self):
         """
@@ -2472,7 +2527,7 @@ class OrderFee(RoundingCorrectionMixin, models.Model):
         self.value_includes_rounding_correction = value
 
 
-class OrderPosition(AbstractPosition):
+class OrderPosition(SearchIndexModelMixin, AbstractPosition):
     """
     An OrderPosition is one line of an order, representing one ordered item
     of a specified type (or variation). This has all properties of
@@ -2579,6 +2634,13 @@ class OrderPosition(AbstractPosition):
 
     all = ScopedManager(organizer='order__event__organizer')
     objects = ActivePositionManager()
+    search_index_fields = [
+        "attendee_name_cached",
+        "attendee_email",
+        "company",
+        "secret",
+        "pseudonymization_id",
+    ]
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -3335,7 +3397,7 @@ class CartPosition(AbstractPosition):
         return self.predicted_validity[1]
 
 
-class InvoiceAddress(models.Model):
+class InvoiceAddress(SearchIndexModelMixin, models.Model):
     last_modified = models.DateTimeField(auto_now=True)
     order = models.OneToOneField(Order, null=True, blank=True, related_name='invoice_address', on_delete=models.CASCADE)
     customer = models.ForeignKey(
@@ -3373,6 +3435,10 @@ class InvoiceAddress(models.Model):
 
     objects = ScopedManager(organizer='order__event__organizer')
     profiles = ScopedManager(organizer='customer__organizer')
+    search_index_fields = [
+        "name_cached",
+        "company",
+    ]
 
     def save(self, **kwargs):
         if self.order:

src/pretix/base/models/search.py

@@ -0,0 +1,129 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020  Raphael Michel and contributors
+# Copyright (C) 2020-today pretix GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+from typing import List, Union
+
+from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.search import SearchVector, SearchVectorField
+from django.db import models
+from django.db.models import Value
+
+from pretix.base.models import (
+    Event, Invoice, InvoiceAddress, Order, OrderPayment, OrderPosition,
+    OrderRefund, Organizer,
+)
+
+
+class OrderSearchIndex(models.Model):
+    organizer = models.ForeignKey(Organizer, on_delete=models.CASCADE)
+    event = models.ForeignKey(Event, on_delete=models.CASCADE)
+    order = models.ForeignKey(Order, on_delete=models.CASCADE)
+    orderposition = models.OneToOneField(
+        OrderPosition,
+        on_delete=models.CASCADE,
+        related_name="search_index",
+        null=True,
+        blank=True,
+    )
+    orderpayment = models.OneToOneField(
+        OrderPayment,
+        on_delete=models.CASCADE,
+        related_name="search_index",
+        null=True,
+        blank=True,
+    )
+    orderrefund = models.OneToOneField(
+        OrderRefund,
+        on_delete=models.CASCADE,
+        related_name="search_index",
+        null=True,
+        blank=True,
+    )
+    invoiceaddress = models.OneToOneField(
+        InvoiceAddress,
+        on_delete=models.CASCADE,
+        related_name="search_index",
+        null=True,
+        blank=True,
+    )
+    invoice = models.OneToOneField(
+        Invoice,
+        on_delete=models.CASCADE,
+        related_name="search_index",
+        null=True,
+        blank=True,
+    )
+    last_modified = models.DateTimeField(auto_now=True)
+    search_vector = SearchVectorField()
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                condition=models.Q(
+                    orderposition__isnull=True,
+                    orderpayment__isnull=True,
+                    orderrefund__isnull=True,
+                    invoiceaddress__isnull=True,
+                    invoice__isnull=True,
+                ),
+                fields=("order",),
+                name="ordersearchindex_one_per_order",
+            ),
+        ]
+        indexes = [
+            GinIndex("search_vector", name="ordersearchindex_vector"),
+        ]
+
+    def save(self, *args, **kwargs):
+        fields_filled = [
+            bool(self.orderposition_id),
+            bool(self.orderpayment_id),
+            bool(self.orderrefund_id),
+            bool(self.invoiceaddress_id),
+            bool(self.invoice_id),
+        ]
+        if fields_filled.count(True) > 1:
+            raise ValueError("A OrderSearchIndex may one relate to one other instance")
+        super().save(*args, **kwargs)
+
+    @classmethod
+    def update_for(cls, obj: Union[Order, OrderPayment, OrderPosition, OrderRefund], inputs: List[str]):
+        index_text = "\n".join([i for i in inputs if i])
+        if isinstance(obj, Order):
+            order = obj
+        else:
+            order = obj.order
+        OrderSearchIndex.objects.update_or_create(
+            order=order,
+            orderpayment=obj if isinstance(obj, OrderPayment) else None,
+            orderrefund=obj if isinstance(obj, OrderRefund) else None,
+            orderposition=obj if isinstance(obj, OrderPosition) else None,
+            defaults=dict(
+                search_vector=SearchVector(
+                    Value(index_text),
+                    config="pretix_search"
+                ),
+            ),
+            create_defaults=dict(
+                event_id=order.event_id,
+                organizer_id=order.organizer_id,
+            ),
+        )

src/pretix/helpers/migration_utils.py

@@ -0,0 +1,37 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020  Raphael Michel and contributors
+# Copyright (C) 2020-today pretix GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+from django.db import migrations
+
+
+class PostgreSQLAndStateOnly(migrations.SeparateDatabaseAndState):
+    def __init__(self, *operations):
+        super().__init__(database_operations=operations, state_operations=operations)
+
+    def database_forwards(self, app_label, schema_editor, from_state, to_state):
+        if schema_editor.connection != "postgresql":
+            return
+        super().database_forwards(app_label, schema_editor, from_state, to_state)
+
+    def database_backwards(self, app_label, schema_editor, from_state, to_state):
+        if schema_editor.connection != "postgresql":
+            return
+        super().database_forwards(app_label, schema_editor, from_state, to_state)