From eba9ab4430ef5b37a78983fc69df553151bdefcf Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Fri, 22 May 2026 15:19:40 +0200 Subject: [PATCH] Search experiments --- .../management/commands/reindex_orders.py | 95 +++++++++++++ .../base/migrations/0300_ordersearchindex.py | 132 ++++++++++++++++++ src/pretix/base/models/__init__.py | 1 + src/pretix/base/models/invoices.py | 7 +- src/pretix/base/models/orders.py | 76 +++++++++- src/pretix/base/models/search.py | 129 +++++++++++++++++ src/pretix/helpers/migration_utils.py | 37 +++++ 7 files changed, 471 insertions(+), 6 deletions(-) create mode 100644 src/pretix/base/management/commands/reindex_orders.py create mode 100644 src/pretix/base/migrations/0300_ordersearchindex.py create mode 100644 src/pretix/base/models/search.py create mode 100644 src/pretix/helpers/migration_utils.py diff --git a/src/pretix/base/management/commands/reindex_orders.py b/src/pretix/base/management/commands/reindex_orders.py new file mode 100644 index 0000000000..25a1757095 --- /dev/null +++ b/src/pretix/base/management/commands/reindex_orders.py @@ -0,0 +1,95 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-today pretix GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +import time + +from django.core.management.base import BaseCommand +from django.db.models import F, Max, Q +from django.utils.timezone import now +from django_scopes import scopes_disabled +from tqdm import tqdm + +from pretix.base.models import Order + + +class Command(BaseCommand): + help = "Recreate order search index" + + def add_arguments(self, parser): + parser.add_argument( + "--slowdown", + dest="interval", + type=int, + default=0, + help="Interval for staggered execution. If set to a value different then zero, we will " + "wait this many milliseconds between every order we process.", + ) + + @scopes_disabled() + def handle(self, *args, **options): + t = 0 + qs = Order.objects.annotate( + last_transaction=Max('transactions__created') + ).filter( + Q(last_transaction__isnull=True) | Q(last_modified__gt=F('last_transaction')), + require_approval=False, + ).prefetch_related( + 'all_positions', 'all_fees' + ).order_by( + 'pk' + ) + last_pk = 0 + with tqdm(total=qs.count()) as pbar: + while True: + batch = list(qs.filter(pk__gt=last_pk)[:5000]) + if not batch: + break + + for o in batch: + if o.last_transaction is None: + tn = o.create_transactions( + positions=o.all_positions.all(), + fees=o.all_fees.all(), + dt_now=o.datetime, + migrated=True, + is_new=True, + _backfill_before_cancellation=True, + ) + o.create_transactions( + positions=o.all_positions.all(), + fees=o.all_fees.all(), + dt_now=o.cancellation_date or (o.expires if o.status == Order.STATUS_EXPIRED else o.datetime), + migrated=True, + ) + else: + tn = o.create_transactions( + positions=o.all_positions.all(), + fees=o.all_fees.all(), + dt_now=now(), + migrated=True, + ) + if tn: + t += 1 + time.sleep(0) + pbar.update(1) + last_pk = batch[-1].pk + + self.stderr.write(self.style.SUCCESS(f'Created transactions for {t} orders.')) diff --git a/src/pretix/base/migrations/0300_ordersearchindex.py b/src/pretix/base/migrations/0300_ordersearchindex.py new file mode 100644 index 0000000000..6029c88529 --- /dev/null +++ b/src/pretix/base/migrations/0300_ordersearchindex.py @@ -0,0 +1,132 @@ +# Generated by Django 5.2.12 on 2026-05-22 12:40 + +import django.contrib.postgres.indexes +import django.contrib.postgres.search +import django.db.models.deletion +from django.contrib.postgres.operations import UnaccentExtension +from django.db import migrations, models + +from pretix.helpers.migration_utils import PostgreSQLAndStateOnly + + +class Migration(migrations.Migration): + + dependencies = [ + ("pretixbase", "0299_itemprogramtime_location"), + ] + + operations = [ + PostgreSQLAndStateOnly( + UnaccentExtension(), + migrations.RunSQL( + sql=""" + CREATE TEXT SEARCH CONFIGURATION pretix_search( COPY = english ); + ALTER TEXT SEARCH CONFIGURATION pretix_search + ALTER MAPPING FOR hword, hword_part, word + WITH unaccent, english_stem; + """, + reverse_sql=""" + DROP TEXT SEARCH CONFIGURATION pretix_search; + """ + ), + migrations.CreateModel( + name="OrderSearchIndex", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False + ), + ), + ("last_modified", models.DateTimeField(auto_now=True)), + ("search_vector", django.contrib.postgres.search.SearchVectorField()), + ( + "organizer", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="pretixbase.organizer", + ), + ), + ( + "event", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="pretixbase.event", + ), + ), + ( + "order", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="pretixbase.order", + ), + ), + ( + "orderpayment", + models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="search_index", + to="pretixbase.orderpayment", + ), + ), + ( + "orderposition", + models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="search_index", + to="pretixbase.orderposition", + ), + ), + ( + "orderrefund", + models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="search_index", + to="pretixbase.orderrefund", + ), + ), + ( + "invoiceaddress", + models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="search_index", + to="pretixbase.invoiceaddress", + ), + ), + ( + "invoice", + models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="search_index", + to="pretixbase.invoice", + ), + ), + ], + options={ + "indexes": [ + django.contrib.postgres.indexes.GinIndex( + models.F("search_vector"), name="ordersearchindex_vector" + ) + ], + "constraints": [ + models.UniqueConstraint( + condition=models.Q( + ("orderpayment__isnull", True), + ("orderposition__isnull", True), + ("orderrefund__isnull", True), + ("invoiceaddress__isnull", True), + ("invoice__isnull", True), + ), + fields=("order",), + name="ordersearchindex_one_per_order", + ), + ], + }, + ), + ), + ] diff --git a/src/pretix/base/models/__init__.py b/src/pretix/base/models/__init__.py index 3657cfb4c2..e710fac501 100644 --- a/src/pretix/base/models/__init__.py +++ b/src/pretix/base/models/__init__.py @@ -56,6 +56,7 @@ from .organizer import ( Organizer, Organizer_SettingsStore, SalesChannel, Team, TeamAPIToken, TeamInvite, ) +from .search import OrderSearchIndex from .seating import Seat, SeatCategoryMapping, SeatingPlan from .tax import TaxRule from .vouchers import Voucher diff --git a/src/pretix/base/models/invoices.py b/src/pretix/base/models/invoices.py index e4e9eaa6c2..e6732785b2 100644 --- a/src/pretix/base/models/invoices.py +++ b/src/pretix/base/models/invoices.py @@ -47,6 +47,7 @@ from django.utils.timezone import now from django.utils.translation import gettext_lazy as _, pgettext from django_scopes import ScopedManager +from pretix.base.models.orders import SearchIndexModelMixin from pretix.base.settings import COUNTRIES_WITH_STATE_IN_ADDRESS from pretix.helpers.countries import FastCountryField @@ -64,7 +65,7 @@ def today(): return timezone.now().date() -class Invoice(models.Model): +class Invoice(SearchIndexModelMixin, models.Model): """ Represents an invoice that is issued because of an order. Because invoices are legally required not to change, this object duplicates a lot of data (e.g. the invoice address). @@ -206,6 +207,10 @@ class Invoice(models.Model): plugin_data = models.JSONField(default=dict) objects = ScopedManager(organizer='event__organizer') + search_index_fields = [ + "invoice_no", + "full_invoice_no", + ] @staticmethod def _to_numeric_invoice_number(number, places): diff --git a/src/pretix/base/models/orders.py b/src/pretix/base/models/orders.py index e74e450e8c..fa99a5fbb4 100644 --- a/src/pretix/base/models/orders.py +++ b/src/pretix/base/models/orders.py @@ -135,7 +135,30 @@ class OrderQuerySet(models.QuerySet): return order -class Order(LockModel, LoggedModel): +class SearchIndexModelMixin: + search_index_fields = [] + + def save(self, *args, **kwargs): + super().save(*args, **kwargs) + if "update_fields" in kwargs: + update_fields = kwargs["update_fields"] + if any(f in update_fields for f in self.search_index_fields): + self._update_search_index() + else: + self._update_search_index() + + def get_search_index_content(self): + for f in self.search_index_fields: + yield str(getattr(self, f)) + + def _update_search_index(self): + from .search import OrderSearchIndex + + contents = self.get_search_index_content() + OrderSearchIndex.update_for(self, contents) + + +class Order(LockModel, SearchIndexModelMixin, LoggedModel): """ An order is created when a user clicks 'buy' on his cart. It holds several OrderPositions and is connected to a user. It has an @@ -330,6 +353,12 @@ class Order(LockModel, LoggedModel): ) objects = ScopedManager(OrderQuerySet.as_manager().__class__, organizer='event__organizer') + search_index_fields = [ + "code", + "comment", + "email", + "phone", + ] class Meta: verbose_name = _("Order") @@ -351,6 +380,10 @@ class Order(LockModel, LoggedModel): if 'require_approval' not in self.get_deferred_fields() and 'status' not in self.get_deferred_fields(): self._transaction_key_reset() + def get_search_index_content(self): + yield from super().get_search_index_content() + yield self.event.slug.upper() + "-" + self.code + def _transaction_key_reset(self): self.__initial_status_paid_or_pending = self.status in (Order.STATUS_PENDING, Order.STATUS_PAID) and not self.require_approval @@ -1690,7 +1723,7 @@ class AbstractPosition(RoundingCorrectionMixin, models.Model): return False -class OrderPayment(models.Model): +class OrderPayment(SearchIndexModelMixin, models.Model): """ Represents a payment or payment attempt for an order. @@ -1774,6 +1807,9 @@ class OrderPayment(models.Model): ) objects = ScopedManager(organizer='order__event__organizer') + search_index_fields = [ + "info", # only for triggering the right updates + ] class Meta: ordering = ('local_id',) @@ -1781,6 +1817,14 @@ class OrderPayment(models.Model): def __str__(self): return self.full_id + def get_search_index_content(self): + try: + return [ + self.payment_provider.matching_id(self) + ] + except: + pass + @property def info_data(self): """ @@ -2093,7 +2137,7 @@ class OrderPayment(models.Model): return r -class OrderRefund(models.Model): +class OrderRefund(SearchIndexModelMixin, models.Model): """ Represents a refund or refund attempt for an order. @@ -2195,6 +2239,9 @@ class OrderRefund(models.Model): ) objects = ScopedManager(organizer='order__event__organizer') + search_index_fields = [ + "info", # only for triggering the right updates + ] class Meta: ordering = ('local_id',) @@ -2202,6 +2249,14 @@ class OrderRefund(models.Model): def __str__(self): return self.full_id + def get_search_index_content(self): + try: + return [ + self.payment_provider.refund_matching_id(self) + ] + except: + pass + @property def info_data(self): """ @@ -2472,7 +2527,7 @@ class OrderFee(RoundingCorrectionMixin, models.Model): self.value_includes_rounding_correction = value -class OrderPosition(AbstractPosition): +class OrderPosition(SearchIndexModelMixin, AbstractPosition): """ An OrderPosition is one line of an order, representing one ordered item of a specified type (or variation). This has all properties of @@ -2579,6 +2634,13 @@ class OrderPosition(AbstractPosition): all = ScopedManager(organizer='order__event__organizer') objects = ActivePositionManager() + search_index_fields = [ + "attendee_name_cached", + "attendee_email", + "company", + "secret", + "pseudonymization_id", + ] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -3335,7 +3397,7 @@ class CartPosition(AbstractPosition): return self.predicted_validity[1] -class InvoiceAddress(models.Model): +class InvoiceAddress(SearchIndexModelMixin, models.Model): last_modified = models.DateTimeField(auto_now=True) order = models.OneToOneField(Order, null=True, blank=True, related_name='invoice_address', on_delete=models.CASCADE) customer = models.ForeignKey( @@ -3373,6 +3435,10 @@ class InvoiceAddress(models.Model): objects = ScopedManager(organizer='order__event__organizer') profiles = ScopedManager(organizer='customer__organizer') + search_index_fields = [ + "name_cached", + "company", + ] def save(self, **kwargs): if self.order: diff --git a/src/pretix/base/models/search.py b/src/pretix/base/models/search.py new file mode 100644 index 0000000000..243b6a6cfa --- /dev/null +++ b/src/pretix/base/models/search.py @@ -0,0 +1,129 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-today pretix GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +from typing import List, Union + +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector, SearchVectorField +from django.db import models +from django.db.models import Value + +from pretix.base.models import ( + Event, Invoice, InvoiceAddress, Order, OrderPayment, OrderPosition, + OrderRefund, Organizer, +) + + +class OrderSearchIndex(models.Model): + organizer = models.ForeignKey(Organizer, on_delete=models.CASCADE) + event = models.ForeignKey(Event, on_delete=models.CASCADE) + order = models.ForeignKey(Order, on_delete=models.CASCADE) + orderposition = models.OneToOneField( + OrderPosition, + on_delete=models.CASCADE, + related_name="search_index", + null=True, + blank=True, + ) + orderpayment = models.OneToOneField( + OrderPayment, + on_delete=models.CASCADE, + related_name="search_index", + null=True, + blank=True, + ) + orderrefund = models.OneToOneField( + OrderRefund, + on_delete=models.CASCADE, + related_name="search_index", + null=True, + blank=True, + ) + invoiceaddress = models.OneToOneField( + InvoiceAddress, + on_delete=models.CASCADE, + related_name="search_index", + null=True, + blank=True, + ) + invoice = models.OneToOneField( + Invoice, + on_delete=models.CASCADE, + related_name="search_index", + null=True, + blank=True, + ) + last_modified = models.DateTimeField(auto_now=True) + search_vector = SearchVectorField() + + class Meta: + constraints = [ + models.UniqueConstraint( + condition=models.Q( + orderposition__isnull=True, + orderpayment__isnull=True, + orderrefund__isnull=True, + invoiceaddress__isnull=True, + invoice__isnull=True, + ), + fields=("order",), + name="ordersearchindex_one_per_order", + ), + ] + indexes = [ + GinIndex("search_vector", name="ordersearchindex_vector"), + ] + + def save(self, *args, **kwargs): + fields_filled = [ + bool(self.orderposition_id), + bool(self.orderpayment_id), + bool(self.orderrefund_id), + bool(self.invoiceaddress_id), + bool(self.invoice_id), + ] + if fields_filled.count(True) > 1: + raise ValueError("A OrderSearchIndex may one relate to one other instance") + super().save(*args, **kwargs) + + @classmethod + def update_for(cls, obj: Union[Order, OrderPayment, OrderPosition, OrderRefund], inputs: List[str]): + index_text = "\n".join([i for i in inputs if i]) + if isinstance(obj, Order): + order = obj + else: + order = obj.order + OrderSearchIndex.objects.update_or_create( + order=order, + orderpayment=obj if isinstance(obj, OrderPayment) else None, + orderrefund=obj if isinstance(obj, OrderRefund) else None, + orderposition=obj if isinstance(obj, OrderPosition) else None, + defaults=dict( + search_vector=SearchVector( + Value(index_text), + config="pretix_search" + ), + ), + create_defaults=dict( + event_id=order.event_id, + organizer_id=order.organizer_id, + ), + ) diff --git a/src/pretix/helpers/migration_utils.py b/src/pretix/helpers/migration_utils.py new file mode 100644 index 0000000000..1f00306388 --- /dev/null +++ b/src/pretix/helpers/migration_utils.py @@ -0,0 +1,37 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-today pretix GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +from django.db import migrations + + +class PostgreSQLAndStateOnly(migrations.SeparateDatabaseAndState): + def __init__(self, *operations): + super().__init__(database_operations=operations, state_operations=operations) + + def database_forwards(self, app_label, schema_editor, from_state, to_state): + if schema_editor.connection != "postgresql": + return + super().database_forwards(app_label, schema_editor, from_state, to_state) + + def database_backwards(self, app_label, schema_editor, from_state, to_state): + if schema_editor.connection != "postgresql": + return + super().database_forwards(app_label, schema_editor, from_state, to_state)