Search experiments

This commit is contained in:
Raphael Michel
2026-05-22 15:19:40 +02:00
parent e4da2e5e03
commit 83b2e79083
10 changed files with 540 additions and 51 deletions

View File

@@ -0,0 +1,69 @@
#
# This file is part of pretix (Community Edition).
#
# Copyright (C) 2014-2020 Raphael Michel and contributors
# Copyright (C) 2020-today pretix GmbH and contributors
#
# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
# Public License as published by the Free Software Foundation in version 3 of the License.
#
# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
# this file, see <https://pretix.eu/about/en/license>.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
# <https://www.gnu.org/licenses/>.
#
import time
from django.core.management.base import BaseCommand
from django_scopes import scopes_disabled
from tqdm import tqdm
from pretix.base.models import (
Invoice, InvoiceAddress, Order, OrderPayment, OrderPosition, OrderRefund,
)
class Command(BaseCommand):
help = "Recreate order search index"
def add_arguments(self, parser):
parser.add_argument(
"--slowdown",
dest="interval",
type=int,
default=0,
help="Interval for staggered execution. If set to a value different then zero, we will "
"wait this many milliseconds between every order we process.",
)
@scopes_disabled()
def handle(self, *args, **options):
querysets = [
Order.objects.all(),
OrderPosition.all.all(),
OrderPayment.objects.all(),
OrderRefund.objects.all(),
InvoiceAddress.objects.filter(order__isnull=False),
Invoice.objects.all(),
]
for qs in querysets:
last_pk = 0
with tqdm(total=qs.count()) as pbar:
while True:
batch = list(qs.filter(pk__gt=last_pk)[:5000])
if not batch:
break
for o in batch:
o._update_search_index()
time.sleep(0)
pbar.update(1)
last_pk = batch[-1].pk

View File

@@ -0,0 +1,132 @@
# Generated by Django 5.2.12 on 2026-05-22 12:40
import django.contrib.postgres.indexes
import django.contrib.postgres.search
import django.db.models.deletion
from django.contrib.postgres.operations import UnaccentExtension
from django.db import migrations, models
from pretix.helpers.migration_utils import PostgreSQLAndStateOnly
class Migration(migrations.Migration):
dependencies = [
("pretixbase", "0299_itemprogramtime_location"),
]
operations = [
PostgreSQLAndStateOnly(
UnaccentExtension(),
migrations.RunSQL(
sql="""
CREATE TEXT SEARCH CONFIGURATION pretix_search( COPY = english );
ALTER TEXT SEARCH CONFIGURATION pretix_search
ALTER MAPPING FOR hword, hword_part, word
WITH unaccent, english_stem;
""",
reverse_sql="""
DROP TEXT SEARCH CONFIGURATION pretix_search;
"""
),
migrations.CreateModel(
name="OrderSearchIndex",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False
),
),
("last_modified", models.DateTimeField(auto_now=True)),
("search_vector", django.contrib.postgres.search.SearchVectorField()),
(
"organizer",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="pretixbase.organizer",
),
),
(
"event",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="pretixbase.event",
),
),
(
"order",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="pretixbase.order",
),
),
(
"orderpayment",
models.OneToOneField(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="search_index",
to="pretixbase.orderpayment",
),
),
(
"orderposition",
models.OneToOneField(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="search_index",
to="pretixbase.orderposition",
),
),
(
"orderrefund",
models.OneToOneField(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="search_index",
to="pretixbase.orderrefund",
),
),
(
"invoiceaddress",
models.OneToOneField(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="search_index",
to="pretixbase.invoiceaddress",
),
),
(
"invoice",
models.OneToOneField(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="search_index",
to="pretixbase.invoice",
),
),
],
options={
"indexes": [
django.contrib.postgres.indexes.GinIndex(
models.F("search_vector"), name="ordersearchindex_vector"
)
],
"constraints": [
models.UniqueConstraint(
condition=models.Q(
("orderpayment__isnull", True),
("orderposition__isnull", True),
("orderrefund__isnull", True),
("invoiceaddress__isnull", True),
("invoice__isnull", True),
),
fields=("order",),
name="ordersearchindex_one_per_order",
),
],
},
),
),
]

View File

@@ -56,6 +56,7 @@ from .organizer import (
Organizer, Organizer_SettingsStore, SalesChannel, Team, TeamAPIToken,
TeamInvite,
)
from .search import OrderSearchIndex
from .seating import Seat, SeatCategoryMapping, SeatingPlan
from .tax import TaxRule
from .vouchers import Voucher

View File

@@ -47,6 +47,7 @@ from django.utils.timezone import now
from django.utils.translation import gettext_lazy as _, pgettext
from django_scopes import ScopedManager
from pretix.base.models.orders import SearchIndexModelMixin
from pretix.base.settings import COUNTRIES_WITH_STATE_IN_ADDRESS
from pretix.helpers.countries import FastCountryField
@@ -64,7 +65,7 @@ def today():
return timezone.now().date()
class Invoice(models.Model):
class Invoice(SearchIndexModelMixin, models.Model):
"""
Represents an invoice that is issued because of an order. Because invoices are legally required
not to change, this object duplicates a lot of data (e.g. the invoice address).
@@ -206,6 +207,10 @@ class Invoice(models.Model):
plugin_data = models.JSONField(default=dict)
objects = ScopedManager(organizer='event__organizer')
search_index_fields = [
"invoice_no",
"full_invoice_no",
]
@staticmethod
def _to_numeric_invoice_number(number, places):

View File

@@ -135,7 +135,33 @@ class OrderQuerySet(models.QuerySet):
return order
class Order(LockModel, LoggedModel):
class SearchIndexModelMixin:
search_index_fields = []
def save(self, *args, **kwargs):
super().save(*args, **kwargs)
if "update_fields" in kwargs:
update_fields = kwargs["update_fields"]
if any(f in update_fields for f in self.search_index_fields):
self._update_search_index()
else:
self._update_search_index()
def get_search_index_content(self):
for f in self.search_index_fields:
val = getattr(self, f)
if val is not None:
yield str(val)
def _update_search_index(self):
from .search import OrderSearchIndex
contents = list(self.get_search_index_content())
print(self, contents)
OrderSearchIndex.update_for(self, contents)
class Order(LockModel, SearchIndexModelMixin, LoggedModel):
"""
An order is created when a user clicks 'buy' on his cart. It holds
several OrderPositions and is connected to a user. It has an
@@ -330,6 +356,12 @@ class Order(LockModel, LoggedModel):
)
objects = ScopedManager(OrderQuerySet.as_manager().__class__, organizer='event__organizer')
search_index_fields = [
"code",
"comment",
"email",
"phone",
]
class Meta:
verbose_name = _("Order")
@@ -351,6 +383,10 @@ class Order(LockModel, LoggedModel):
if 'require_approval' not in self.get_deferred_fields() and 'status' not in self.get_deferred_fields():
self._transaction_key_reset()
def get_search_index_content(self):
yield from super().get_search_index_content()
yield self.event.slug.upper() + "-" + self.code
def _transaction_key_reset(self):
self.__initial_status_paid_or_pending = self.status in (Order.STATUS_PENDING, Order.STATUS_PAID) and not self.require_approval
@@ -1690,7 +1726,7 @@ class AbstractPosition(RoundingCorrectionMixin, models.Model):
return False
class OrderPayment(models.Model):
class OrderPayment(SearchIndexModelMixin, models.Model):
"""
Represents a payment or payment attempt for an order.
@@ -1774,6 +1810,9 @@ class OrderPayment(models.Model):
)
objects = ScopedManager(organizer='order__event__organizer')
search_index_fields = [
"info", # only for triggering the right updates
]
class Meta:
ordering = ('local_id',)
@@ -1781,6 +1820,14 @@ class OrderPayment(models.Model):
def __str__(self):
return self.full_id
def get_search_index_content(self):
try:
return [
self.payment_provider.matching_id(self)
]
except:
pass
@property
def info_data(self):
"""
@@ -2093,7 +2140,7 @@ class OrderPayment(models.Model):
return r
class OrderRefund(models.Model):
class OrderRefund(SearchIndexModelMixin, models.Model):
"""
Represents a refund or refund attempt for an order.
@@ -2195,6 +2242,9 @@ class OrderRefund(models.Model):
)
objects = ScopedManager(organizer='order__event__organizer')
search_index_fields = [
"info", # only for triggering the right updates
]
class Meta:
ordering = ('local_id',)
@@ -2202,6 +2252,14 @@ class OrderRefund(models.Model):
def __str__(self):
return self.full_id
def get_search_index_content(self):
try:
return [
self.payment_provider.refund_matching_id(self)
]
except:
pass
@property
def info_data(self):
"""
@@ -2472,7 +2530,7 @@ class OrderFee(RoundingCorrectionMixin, models.Model):
self.value_includes_rounding_correction = value
class OrderPosition(AbstractPosition):
class OrderPosition(SearchIndexModelMixin, AbstractPosition):
"""
An OrderPosition is one line of an order, representing one ordered item
of a specified type (or variation). This has all properties of
@@ -2579,6 +2637,13 @@ class OrderPosition(AbstractPosition):
all = ScopedManager(organizer='order__event__organizer')
objects = ActivePositionManager()
search_index_fields = [
"attendee_name_cached",
"attendee_email",
"company",
"secret",
"pseudonymization_id",
]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -3335,7 +3400,7 @@ class CartPosition(AbstractPosition):
return self.predicted_validity[1]
class InvoiceAddress(models.Model):
class InvoiceAddress(SearchIndexModelMixin, models.Model):
last_modified = models.DateTimeField(auto_now=True)
order = models.OneToOneField(Order, null=True, blank=True, related_name='invoice_address', on_delete=models.CASCADE)
customer = models.ForeignKey(
@@ -3373,6 +3438,10 @@ class InvoiceAddress(models.Model):
objects = ScopedManager(organizer='order__event__organizer')
profiles = ScopedManager(organizer='customer__organizer')
search_index_fields = [
"name_cached",
"company",
]
def save(self, **kwargs):
if self.order:

View File

@@ -0,0 +1,158 @@
#
# This file is part of pretix (Community Edition).
#
# Copyright (C) 2014-2020 Raphael Michel and contributors
# Copyright (C) 2020-today pretix GmbH and contributors
#
# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
# Public License as published by the Free Software Foundation in version 3 of the License.
#
# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
# this file, see <https://pretix.eu/about/en/license>.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
# <https://www.gnu.org/licenses/>.
#
from typing import List, Union
from django.contrib.postgres.indexes import GinIndex
from django.contrib.postgres.search import (
SearchQuery, SearchVector, SearchVectorField,
)
from django.db import models
from django.db.models import Value
from django_scopes.manager import ScopedManager
from pretix.base.models import (
Event, Invoice, InvoiceAddress, Order, OrderPayment, OrderPosition,
OrderRefund, Organizer,
)
class SearchIndexQuerySet(models.QuerySet):
def search_for(self, query: str):
sq = SearchQuery(query, config="pretix_search", search_type="raw") # todo: probably use plain
if query.count(" ") < 1 and "@" not in query:
# Order code normalization, only apply for one-word search that is not an email address
sq |= SearchQuery(Order.normalize_code(query.rsplit("-", 1)[-1]), config="pretix_search", search_type="plain")
if query.isdigit():
for i in range(2, 12):
sq |= SearchQuery(query.zfill(i), config="pretix_search", search_type="plain")
return self.filter(search_vector=sq)
class OrderSearchIndex(models.Model):
organizer = models.ForeignKey(Organizer, on_delete=models.CASCADE)
event = models.ForeignKey(Event, on_delete=models.CASCADE)
order = models.ForeignKey(Order, on_delete=models.CASCADE)
orderposition = models.OneToOneField(
OrderPosition,
on_delete=models.CASCADE,
related_name="search_index",
null=True,
blank=True,
)
orderpayment = models.OneToOneField(
OrderPayment,
on_delete=models.CASCADE,
related_name="search_index",
null=True,
blank=True,
)
orderrefund = models.OneToOneField(
OrderRefund,
on_delete=models.CASCADE,
related_name="search_index",
null=True,
blank=True,
)
invoiceaddress = models.OneToOneField(
InvoiceAddress,
on_delete=models.CASCADE,
related_name="search_index",
null=True,
blank=True,
)
invoice = models.OneToOneField(
Invoice,
on_delete=models.CASCADE,
related_name="search_index",
null=True,
blank=True,
)
last_modified = models.DateTimeField(auto_now=True)
search_vector = SearchVectorField()
objects = ScopedManager(SearchIndexQuerySet.as_manager().__class__, organizer='organizer')
class Meta:
constraints = [
models.UniqueConstraint(
condition=models.Q(
orderposition__isnull=True,
orderpayment__isnull=True,
orderrefund__isnull=True,
invoiceaddress__isnull=True,
invoice__isnull=True,
),
fields=("order",),
name="ordersearchindex_one_per_order",
),
]
indexes = [
GinIndex("search_vector", name="ordersearchindex_vector"),
]
def save(self, *args, **kwargs):
fields_filled = [
bool(self.orderposition_id),
bool(self.orderpayment_id),
bool(self.orderrefund_id),
bool(self.invoiceaddress_id),
bool(self.invoice_id),
]
if fields_filled.count(True) > 1:
raise ValueError("A OrderSearchIndex may one relate to one other instance")
super().save(*args, **kwargs)
@classmethod
def update_for(cls, obj: Union[Order, OrderPayment, OrderPosition, OrderRefund], inputs: List[str]):
index_text = "\n".join([i for i in inputs if i])
if isinstance(obj, Order):
order = obj
else:
order = obj.order
kwargs = dict(
order=order,
orderpayment=obj if isinstance(obj, OrderPayment) else None,
orderrefund=obj if isinstance(obj, OrderRefund) else None,
orderposition=obj if isinstance(obj, OrderPosition) else None,
invoiceaddress=obj if isinstance(obj, InvoiceAddress) else None,
invoice=obj if isinstance(obj, Invoice) else None,
)
if not index_text.strip():
OrderSearchIndex.objects.filter(**kwargs).delete()
else:
OrderSearchIndex.objects.update_or_create(
**kwargs,
defaults=dict(
search_vector=SearchVector(
Value(index_text),
config="pretix_search"
),
),
create_defaults=dict(
event_id=order.event_id,
organizer_id=order.organizer_id,
),
)

View File

@@ -3719,6 +3719,10 @@ Your {organizer} team""")) # noqa: W291
'default': '',
'type': str,
},
'use_fts_beta': {
'default': 'False',
'type': bool,
}
}
PERSON_NAME_TITLE_GROUPS = OrderedDict([
('english_common', (_('Most common English titles'), (

View File

@@ -57,10 +57,11 @@ from pretix.base.forms.widgets import (
from pretix.base.models import (
Checkin, CheckinList, Device, Event, EventMetaProperty, EventMetaValue,
Gate, Invoice, InvoiceAddress, Item, Order, OrderPayment, OrderPosition,
OrderRefund, Organizer, OutgoingMail, Question, QuestionAnswer, Quota,
SalesChannel, SubEvent, SubEventMetaValue, Team, TeamAPIToken, TeamInvite,
Voucher,
OrderRefund, OrderSearchIndex, Organizer, OutgoingMail, Question,
QuestionAnswer, Quota, SalesChannel, SubEvent, SubEventMetaValue, Team,
TeamAPIToken, TeamInvite, Voucher,
)
from pretix.base.settings import GlobalSettingsObject
from pretix.base.signals import register_payment_providers
from pretix.base.timeframes import (
DateFrameField,
@@ -262,52 +263,61 @@ class OrderFilterForm(FilterForm):
if fdata.get('query'):
u = fdata.get('query')
if "-" in u:
code = (Q(event__slug__icontains=u.rsplit("-", 1)[0])
& Q(code__icontains=Order.normalize_code(u.rsplit("-", 1)[1])))
fts_beta = (
self.event.settings.use_fts_beta
if hasattr(self, 'event')
else GlobalSettingsObject().settings.use_fts_beta
)
if fts_beta:
qs = qs.filter(
pk__in=OrderSearchIndex.objects.search_for(u).values_list("order_id", flat=True)
)
else:
code = Q(code__icontains=Order.normalize_code(u))
if "-" in u:
code = (Q(event__slug__icontains=u.rsplit("-", 1)[0])
& Q(code__icontains=Order.normalize_code(u.rsplit("-", 1)[1])))
else:
code = Q(code__icontains=Order.normalize_code(u))
invoice_nos = {u, u.upper()}
if u.isdigit():
for i in range(2, 12):
invoice_nos.add(u.zfill(i))
invoice_nos = {u, u.upper()}
if u.isdigit():
for i in range(2, 12):
invoice_nos.add(u.zfill(i))
matching_invoices = Invoice.objects.filter(
Q(invoice_no__in=invoice_nos)
| Q(full_invoice_no__iexact=u)
).values_list('order_id', flat=True)
matching_positions = OrderPosition.all.filter(
Q(
Q(attendee_name_cached__icontains=u) | Q(attendee_email__icontains=u)
| Q(company__icontains=u)
| Q(secret__istartswith=u)
| Q(pseudonymization_id__istartswith=u)
matching_invoices = Invoice.objects.filter(
Q(invoice_no__in=invoice_nos)
| Q(full_invoice_no__iexact=u)
).values_list('order_id', flat=True)
matching_positions = OrderPosition.all.filter(
Q(
Q(attendee_name_cached__icontains=u) | Q(attendee_email__icontains=u)
| Q(company__icontains=u)
| Q(secret__istartswith=u)
| Q(pseudonymization_id__istartswith=u)
)
).values_list('order_id', flat=True)
matching_invoice_addresses = InvoiceAddress.objects.filter(
Q(
Q(name_cached__icontains=u) | Q(company__icontains=u)
)
).values_list('order_id', flat=True)
matching_orders = Order.objects.filter(
code
| Q(email__icontains=u)
| Q(comment__icontains=u)
).values_list('id', flat=True)
mainq = (
Q(pk__in=matching_orders)
| Q(pk__in=matching_invoices)
| Q(pk__in=matching_positions)
| Q(pk__in=matching_invoice_addresses)
)
).values_list('order_id', flat=True)
matching_invoice_addresses = InvoiceAddress.objects.filter(
Q(
Q(name_cached__icontains=u) | Q(company__icontains=u)
for recv, q in order_search_filter_q.send(sender=getattr(self, 'event', None), query=u):
mainq = mainq | q
qs = qs.filter(
mainq
)
).values_list('order_id', flat=True)
matching_orders = Order.objects.filter(
code
| Q(email__icontains=u)
| Q(comment__icontains=u)
).values_list('id', flat=True)
mainq = (
Q(pk__in=matching_orders)
| Q(pk__in=matching_invoices)
| Q(pk__in=matching_positions)
| Q(pk__in=matching_invoice_addresses)
)
for recv, q in order_search_filter_q.send(sender=getattr(self, 'event', None), query=u):
mainq = mainq | q
qs = qs.filter(
mainq
)
if fdata.get('status'):
s = fdata.get('status')

View File

@@ -110,7 +110,11 @@ class GlobalSettingsForm(SettingsForm):
required=False,
label=_("Vite widget origins"),
help_text=_("One origin per line (e.g. https://example.com). Requests from these origins will be served the new vite-based widget."),
))
)),
('use_fts_beta', forms.BooleanField(
required=False,
label=_("Use full-text search beta"),
)),
])
responses = register_global_settings.send(self)
for r, response in sorted(responses, key=lambda r: str(r[0])):

View File

@@ -0,0 +1,37 @@
#
# This file is part of pretix (Community Edition).
#
# Copyright (C) 2014-2020 Raphael Michel and contributors
# Copyright (C) 2020-today pretix GmbH and contributors
#
# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
# Public License as published by the Free Software Foundation in version 3 of the License.
#
# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
# this file, see <https://pretix.eu/about/en/license>.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
# <https://www.gnu.org/licenses/>.
#
from django.db import migrations
class PostgreSQLAndStateOnly(migrations.SeparateDatabaseAndState):
def __init__(self, *operations):
super().__init__(database_operations=operations, state_operations=operations)
def database_forwards(self, app_label, schema_editor, from_state, to_state):
if schema_editor.connection.vendor != "postgresql":
return
super().database_forwards(app_label, schema_editor, from_state, to_state)
def database_backwards(self, app_label, schema_editor, from_state, to_state):
if schema_editor.connection.vendor != "postgresql":
return
super().database_forwards(app_label, schema_editor, from_state, to_state)