From 173cf751da685fe52b8311f5e2d04e93b6ce0d7b Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Thu, 14 Feb 2019 18:11:25 +0100 Subject: [PATCH] Add search index model --- .../commands/rebuild_search_index.py | 30 ++++++++++++ .../migrations/0110_auto_20190214_1041.py | 49 +++++++++++++++++++ src/pretix/base/models/__init__.py | 1 + src/pretix/base/models/index.py | 11 +++++ src/pretix/base/models/orders.py | 39 +++++++++++++++ 5 files changed, 130 insertions(+) create mode 100644 src/pretix/base/management/commands/rebuild_search_index.py create mode 100644 src/pretix/base/migrations/0110_auto_20190214_1041.py create mode 100644 src/pretix/base/models/index.py diff --git a/src/pretix/base/management/commands/rebuild_search_index.py b/src/pretix/base/management/commands/rebuild_search_index.py new file mode 100644 index 000000000..eb001a1fa --- /dev/null +++ b/src/pretix/base/management/commands/rebuild_search_index.py @@ -0,0 +1,30 @@ +from django.core.management.base import BaseCommand +from django.core.paginator import Paginator + +from pretix.base.models import Order, OrderSearchIndex + + +class Command(BaseCommand): + help = "Rebuild search index" + + def add_arguments(self, parser): + parser.add_argument( + '--clean', action='store_true', dest='clean', + help="Clear search index before run.", + ) + + def iter_pages(self, qs): + paginator = Paginator(qs, 500) + for index in range(paginator.num_pages): + yield paginator.get_page(index + 1) + + def handle(self, *args, **options): + if options.get('clean'): + OrderSearchIndex.objects.all().delete() + qs = Order.objects.select_related('event', 'event__organizer', 'invoice_address').prefetch_related('all_positions', 'payments') + for page in self.iter_pages(qs): + if options.get('clean'): + OrderSearchIndex.objects.bulk_create([o.index() for o in page]) + else: + for o in page: + o.index() diff --git a/src/pretix/base/migrations/0110_auto_20190214_1041.py b/src/pretix/base/migrations/0110_auto_20190214_1041.py new file mode 100644 index 000000000..1b5e3622f --- /dev/null +++ b/src/pretix/base/migrations/0110_auto_20190214_1041.py @@ -0,0 +1,49 @@ +# Generated by Django 2.1 on 2019-02-14 10:41 + +import django.db.models.deletion +from django.db import migrations, models + + +def create_extension(apps, schema_editor): + if 'postgresql' in schema_editor.connection.settings_dict['ENGINE']: + schema_editor.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;") + + +def create_trigram_index(apps, schema_editor): + if 'postgresql' in schema_editor.connection.settings_dict['ENGINE']: + schema_editor.execute("CREATE INDEX pretixbase_ordersearchindex_s ON pretixbase_ordersearchindex USING gin (search_body gin_trgm_ops);") + schema_editor.execute("CREATE INDEX pretixbase_ordersearchindex_spp ON pretixbase_ordersearchindex USING gin (payment_providers gin_trgm_ops);") + + +class Migration(migrations.Migration): + dependencies = [ + ('pretixbase', '0109_auto_20190208_1432'), + ] + + operations = [ + migrations.RunPython(create_extension), + migrations.CreateModel( + name='OrderSearchIndex', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('search_body', models.TextField()), + ('payment_providers', models.TextField()), + ], + ), + migrations.AddField( + model_name='ordersearchindex', + name='event', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pretixbase.Event'), + ), + migrations.AddField( + model_name='ordersearchindex', + name='order', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pretixbase.Order', unique=True), + ), + migrations.AddField( + model_name='ordersearchindex', + name='organizer', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pretixbase.Organizer'), + ), + migrations.RunPython(create_trigram_index) + ] diff --git a/src/pretix/base/models/__init__.py b/src/pretix/base/models/__init__.py index b3614d5d3..53571d319 100644 --- a/src/pretix/base/models/__init__.py +++ b/src/pretix/base/models/__init__.py @@ -7,6 +7,7 @@ from .event import ( Event, Event_SettingsStore, EventLock, EventMetaProperty, EventMetaValue, RequiredAction, SubEvent, SubEventMetaValue, generate_invite_token, ) +from .index import OrderSearchIndex from .invoices import Invoice, InvoiceLine, invoice_filename from .items import ( Item, ItemAddOn, ItemCategory, ItemVariation, Question, QuestionOption, diff --git a/src/pretix/base/models/index.py b/src/pretix/base/models/index.py new file mode 100644 index 000000000..43e07dcde --- /dev/null +++ b/src/pretix/base/models/index.py @@ -0,0 +1,11 @@ +from django.db import models + +DELIMITER = "\x1F" + + +class OrderSearchIndex(models.Model): + order = models.ForeignKey('Order', unique=True, null=False, on_delete=models.CASCADE) + event = models.ForeignKey('Event', null=False, on_delete=models.CASCADE) + organizer = models.ForeignKey('Organizer', null=False, on_delete=models.CASCADE) + search_body = models.TextField() + payment_providers = models.TextField() diff --git a/src/pretix/base/models/orders.py b/src/pretix/base/models/orders.py index 59d9235a7..5db17fc6f 100644 --- a/src/pretix/base/models/orders.py +++ b/src/pretix/base/models/orders.py @@ -693,6 +693,45 @@ class Order(LockModel, LoggedModel): continue yield op + def index(self, save=True): + from .index import OrderSearchIndex + indexed_strings = [ + self.code, + self.full_code, + self.email, + self.comment, + ] + try: + indexed_strings.append(self.invoice_address.name_cached) + indexed_strings.append(self.invoice_address.company) + except InvoiceAddress.DoesNotExist: + pass + for p in self.all_positions.all(): + indexed_strings.append(p.attendee_name_cached) + indexed_strings.append(p.attendee_email) + indexed_strings.append(p.secret) + pprovs = set() + for p in self.payments.all(): + pprovs.add(p.provider) + if save: + return OrderSearchIndex.objects.update_or_create( + order=self, + defaults={ + 'event': self.event, + 'organizer': self.event.organizer, + 'search_body': '\x1E'.join([str(v) for v in indexed_strings if v]), + 'payment_providers': '\x1E' + '\x1E'.join([str(v) for v in pprovs if v]) + '\x1E', + } + )[0] + else: + return OrderSearchIndex( + order=self, + event=self.event, + organizer=self.event.organizer, + search_body='\x1E'.join([str(v) for v in indexed_strings if v]), + payment_providers='\x1E' + '\x1E'.join([str(v) for v in pprovs if v]) + '\x1E', + ) + def answerfile_name(instance, filename: str) -> str: secret = get_random_string(length=32, allowed_chars=string.ascii_letters + string.digits)