diff --git a/src/pretix/control/forms/filter.py b/src/pretix/control/forms/filter.py index 1ccd554ed4..74fed2b417 100644 --- a/src/pretix/control/forms/filter.py +++ b/src/pretix/control/forms/filter.py @@ -811,6 +811,213 @@ class OrderSearchFilterForm(OrderFilterForm): ) +class OrderPaymentSearchFilterForm(forms.Form): + orders = {'id': 'id', 'local_id': 'local_id', 'state': 'state', 'amount': 'amount', 'order': 'order', + 'created': 'created', 'payment_date': 'payment_date', 'provider': 'provider', 'info': 'info', + 'fee': 'fee'} + + query = forms.CharField( + label=_('Search for…'), + widget=forms.TextInput(attrs={ + 'placeholder': _('Search for…'), + 'autofocus': 'autofocus' + }), + required=False, + ) + event = forms.ModelChoiceField( + label=_('Event'), + queryset=Event.objects.none(), + required=False, + widget=Select2( + attrs={ + 'data-model-select2': 'event', + 'data-select2-url': reverse_lazy('control:events.typeahead'), + 'data-placeholder': _('All events') + } + ) + ) + organizer = forms.ModelChoiceField( + label=_('Organizer'), + queryset=Organizer.objects.none(), + required=False, + empty_label=_('All organizers'), + widget=Select2( + attrs={ + 'data-model-select2': 'generic', + 'data-select2-url': reverse_lazy('control:organizers.select2'), + 'data-placeholder': _('All organizers') + } + ), + ) + state = forms.ChoiceField( + label=_('Status'), + required=False, + choices=[('', _('All payments'))] + list(OrderPayment.PAYMENT_STATES), + ) + provider = forms.ChoiceField( + label=_('Payment provider'), + choices=[ + ('', _('All payment providers')), + ], + required=False, + ) + created_from = forms.DateField( + label=_('Payment created from'), + required=False, + widget=DatePickerWidget, + ) + created_until = forms.DateField( + label=_('Payment created until'), + required=False, + widget=DatePickerWidget, + ) + completed_from = forms.DateField( + label=_('Paid from'), + required=False, + widget=DatePickerWidget, + ) + completed_until = forms.DateField( + label=_('Paid until'), + required=False, + widget=DatePickerWidget, + ) + amount = forms.CharField( + label=_('Amount'), + required=False, + widget=forms.NumberInput(attrs={ + 'placeholder': _('Amount'), + }), + ) + + def __init__(self, *args, **kwargs): + self.request = kwargs.pop('request') + super().__init__(*args, **kwargs) + + self.fields['ordering'] = forms.ChoiceField( + choices=sum([ + [(a, a), ('-' + a, '-' + a)] + for a in self.orders.keys() + ], []), + required=False + ) + + if self.request.user.has_active_staff_session(self.request.session.session_key): + self.fields['organizer'].queryset = Organizer.objects.all() + self.fields['event'].queryset = Event.objects.all() + + else: + self.fields['organizer'].queryset = Organizer.objects.filter( + pk__in=self.request.user.teams.values_list('organizer', flat=True) + ) + self.fields['event'].queryset = self.request.user.get_events_with_permission('can_view_orders') + + self.fields['provider'].choices += get_all_payment_providers() + + def filter_qs(self, qs): + fdata = self.cleaned_data + + if fdata.get('created_from'): + date_start = make_aware(datetime.combine( + fdata.get('created_from'), + time(hour=0, minute=0, second=0, microsecond=0) + ), get_current_timezone()) + qs = qs.filter(created__gte=date_start) + + if fdata.get('created_until'): + date_end = make_aware(datetime.combine( + fdata.get('created_until') + timedelta(days=1), + time(hour=0, minute=0, second=0, microsecond=0) + ), get_current_timezone()) + qs = qs.filter(created__lt=date_end) + + if fdata.get('completed_from'): + date_start = make_aware(datetime.combine( + fdata.get('completed_from'), + time(hour=0, minute=0, second=0, microsecond=0) + ), get_current_timezone()) + qs = qs.filter(payment_date__gte=date_start) + + if fdata.get('completed_until'): + date_end = make_aware(datetime.combine( + fdata.get('completed_until') + timedelta(days=1), + time(hour=0, minute=0, second=0, microsecond=0) + ), get_current_timezone()) + qs = qs.filter(payment_date__lt=date_end) + + if fdata.get('event'): + qs = qs.filter(order__event=fdata.get('event')) + + if fdata.get('organizer'): + qs = qs.filter(order__event__organizer=fdata.get('organizer')) + + if fdata.get('state'): + qs = qs.filter(state=fdata.get('state')) + + if fdata.get('provider'): + qs = qs.filter(provider=fdata.get('provider')) + + if fdata.get('query'): + u = fdata.get('query') + + matching_invoices = Invoice.objects.filter( + Q(invoice_no__iexact=u) + | Q(invoice_no__iexact=u.zfill(5)) + | Q(full_invoice_no__iexact=u) + ).values_list('order_id', flat=True) + + matching_invoice_addresses = InvoiceAddress.objects.filter( + Q( + Q(name_cached__icontains=u) | Q(company__icontains=u) + ) + ).values_list('order_id', flat=True) + + if "-" in u: + code = (Q(event__slug__icontains=u.rsplit("-", 1)[0]) + & Q(code__icontains=Order.normalize_code(u.rsplit("-", 1)[1]))) + else: + code = Q(code__icontains=Order.normalize_code(u)) + + matching_orders = Order.objects.filter( + Q( + code + | Q(email__icontains=u) + | Q(comment__icontains=u) + ) + ).values_list('id', flat=True) + + mainq = ( + Q(order__id__in=matching_invoices) + | Q(order__id__in=matching_invoice_addresses) + | Q(order__id__in=matching_orders) + ) + + qs = qs.filter(mainq) + + if fdata.get('amount'): + amount = fdata.get('amount') + + def is_decimal(value): + result = True + parts = value.split('.', maxsplit=1) + for part in parts: + result = result & part.isdecimal() + return result + + if is_decimal(amount): + qs = qs.filter(amount=Decimal(amount)) + + if fdata.get('ordering'): + p = self.cleaned_data.get('ordering') + if p.startswith('-') and p not in self.orders: + qs = qs.order_by('-' + self.orders[p[1:]]) + else: + qs = qs.order_by(self.orders[p]) + else: + qs = qs.order_by('-created') + + return qs + + class SubEventFilterForm(FilterForm): orders = { 'date_from': 'date_from', diff --git a/src/pretix/control/navigation.py b/src/pretix/control/navigation.py index 9cd4160373..370e15c7f0 100644 --- a/src/pretix/control/navigation.py +++ b/src/pretix/control/navigation.py @@ -343,10 +343,24 @@ def get_global_navigation(request): 'icon': 'group', }, { - 'label': _('Order search'), + 'label': _('Search'), 'url': reverse('control:search.orders'), - 'active': 'search.orders' in url.url_name, + 'active': False, 'icon': 'search', + 'children': [ + { + 'label': _('Orders'), + 'url': reverse('control:search.orders'), + 'active': 'search.orders' in url.url_name, + 'icon': 'search', + }, + { + 'label': _('Payments'), + 'url': reverse('control:search.payments'), + 'active': 'search.payments' in url.url_name, + 'icon': 'search', + }, + ] }, { 'label': _('User settings'), diff --git a/src/pretix/control/templates/pretixcontrol/search/payments.html b/src/pretix/control/templates/pretixcontrol/search/payments.html new file mode 100644 index 0000000000..dae3f1c068 --- /dev/null +++ b/src/pretix/control/templates/pretixcontrol/search/payments.html @@ -0,0 +1,164 @@ +{% extends "pretixcontrol/base.html" %} +{% load i18n %} +{% load eventurl %} +{% load urlreplace %} +{% load money %} +{% load bootstrap3 %} +{% block title %}{% trans "Payment search" %}{% endblock %} +{% block content %} +

{% trans "Payment search" %}

+
+
+

+ {% trans "Filter" %} +

+
+
+
+
+
+
+ {% bootstrap_field filter_form.query %} +
+
+ {% bootstrap_field filter_form.completed_from %} +
+
+ {% bootstrap_field filter_form.completed_until %} +
+
+
+
+ {% bootstrap_field filter_form.amount %} +
+
+ {% bootstrap_field filter_form.provider %} +
+
+ {% bootstrap_field filter_form.state %} +
+
+
+
+
+
+ {% bootstrap_field filter_form.organizer %} +
+
+ {% bootstrap_field filter_form.event %} +
+
+
+
+ {% bootstrap_field filter_form.created_from %} +
+
+ {% bootstrap_field filter_form.created_until %} +
+
+
+
+
+ +
+
+
+ +
+ + + + + + + + + + + + + + {% for p in payments %} + + + + + + + + + + {% if staff_session %} + + + + + {% endif %} + {% empty %} + + + + {% endfor %} + +
+ {% trans "Payment ID" %} + + {% trans "Order" %} + + + + {% trans "Start date" %} + + + + {% trans "Confirmation date" %} + + + + {% trans "Payment provider" %} + + + + {% trans "Amount" %} + + + + {% trans "Status" %} + + +
{{ p.full_id }} + + + {{ p.order.full_code }} + + {% if p.order.testmode %} + {% trans "TEST MODE" %} + {% endif %} + + {% if p.migrated %} + + {% trans "MIGRATED" %} + + {% else %} + {{ p.created|date:"SHORT_DATETIME_FORMAT" }} + {% endif %} + {{ p.payment_date|date:"SHORT_DATETIME_FORMAT" }}{{ p.payment_provider.verbose_name }}{{ p.amount|money:p.order.event.currency }} + + {{ p.get_state_display }} + +
+ + + {% trans "Inspect" %} + +
+ {% trans "We couldn't find any payments that you have access to and that match your search query." %} +
+
+ + {% include "pretixcontrol/pagination_huge.html" %} +{% endblock %} diff --git a/src/pretix/control/urls.py b/src/pretix/control/urls.py index d05eb5d785..8042f121c1 100644 --- a/src/pretix/control/urls.py +++ b/src/pretix/control/urls.py @@ -194,6 +194,7 @@ urlpatterns = [ re_path(r'^events/typeahead/$', typeahead.event_list, name='events.typeahead'), re_path(r'^events/typeahead/meta/$', typeahead.meta_values, name='events.meta.typeahead'), re_path(r'^search/orders/$', search.OrderSearch.as_view(), name='search.orders'), + re_path(r'^search/payments/$', search.PaymentSearch.as_view(), name='search.payments'), re_path(r'^event/(?P[^/]+)/(?P[^/]+)/', include([ re_path(r'^$', dashboards.event_index, name='event.index'), re_path(r'^widgets.json$', dashboards.event_index_widgets_lazy, name='event.index.widgets'), diff --git a/src/pretix/control/views/search.py b/src/pretix/control/views/search.py index 5ac9f38a54..e6dbbd1770 100644 --- a/src/pretix/control/views/search.py +++ b/src/pretix/control/views/search.py @@ -25,8 +25,10 @@ from django.utils.functional import cached_property from django.views.generic import ListView from pretix.base.models import Order, OrderPosition -from pretix.base.models.orders import CancellationRequest -from pretix.control.forms.filter import OrderSearchFilterForm +from pretix.base.models.orders import CancellationRequest, OrderPayment +from pretix.control.forms.filter import ( + OrderPaymentSearchFilterForm, OrderSearchFilterForm, +) from pretix.control.views import LargeResultSetPaginator, PaginationMixin @@ -136,3 +138,73 @@ class OrderSearch(PaginationMixin, ListView): ).prefetch_related( 'event', 'event__organizer' ).select_related('invoice_address') + + +class PaymentSearch(PaginationMixin, ListView): + model = OrderPayment + paginator_class = LargeResultSetPaginator + context_object_name = 'payments' + template_name = 'pretixcontrol/search/payments.html' + + @cached_property + def filter_form(self): + return OrderPaymentSearchFilterForm(data=self.request.GET, request=self.request) + + def get_context_data(self, **kwargs): + ctx = super().get_context_data() + ctx['filter_form'] = self.filter_form + return ctx + + def get_queryset(self): + qs = OrderPayment.objects.using(settings.DATABASE_REPLICA) + + if not self.request.user.has_active_staff_session(self.request.session.session_key): + qs = qs.filter( + Q(order__event_id__in=self.request.user.get_events_with_permission('can_view_orders').values_list('id', flat=True)) + ) + + if self.filter_form.is_valid(): + qs = self.filter_form.filter_qs(qs) + + if self.filter_form.cleaned_data.get('query'): + """ + We need to work around a bug in PostgreSQL's (and likely MySQL's) query plan optimizer here. + The database lacks statistical data to predict how common our search filter is and therefore + assumes that it is cheaper to first ORDER *all* orders in the system (since we got an index on + datetime), then filter out with a full scan until OFFSET/LIMIT condition is fulfilled. If we + look for something rare (such as an email address used once within hundreds of thousands of + orders, this ends up to be pathologically slow. + + For some search queries on pretix.eu, we see search times of >30s, just due to the ORDER BY and + LIMIT clause. Without them. the query runs in roughly 0.6s. This heuristical approach tries to + detect these cases and rewrite the query as a nested subquery that strongly suggests sorting + before filtering. However, since even that fails in some cases because PostgreSQL thinks it knows + better, we literally force it by evaluating the subquery explicitly. We only do this for n<=200, + to avoid memory leaks – and problems with maximum parameter count on SQLite. In cases where the + search query yields lots of results, this will actually be slower since it requires two queries, + sorry. + + Phew. + """ + + page = self.kwargs.get(self.page_kwarg) or self.request.GET.get(self.page_kwarg) or 1 + limit = self.get_paginate_by(None) + try: + offset = (int(page) - 1) * limit + except ValueError: + offset = 0 + resultids = list(qs.order_by().values_list('id', flat=True)[:201]) + if len(resultids) <= 200 and len(resultids) <= offset + limit: + qs = OrderPayment.objects.using(settings.DATABASE_REPLICA).filter( + id__in=resultids + ) + + """ + We use prefetch_related here instead of select_related for a reason, even though select_related + would be the common choice for a foreign key and doesn't require an additional database query. + The problem is, that if our results contain the same event 25 times, select_related will create + 25 Django objects which will all try to pull their ownsettings cache to show the event properly, + leading to lots of unnecessary queries. Due to the way prefetch_related works differently, it + will only create one shared Django object. + """ + return qs.prefetch_related('order', 'order__event', 'order__event__organizer') diff --git a/src/tests/control/test_search.py b/src/tests/control/test_search.py index 7aa1106d04..d4fb21ea81 100644 --- a/src/tests/control/test_search.py +++ b/src/tests/control/test_search.py @@ -27,7 +27,8 @@ from django_scopes import scopes_disabled from tests.base import SoupTest from pretix.base.models import ( - Event, InvoiceAddress, Item, Order, OrderPosition, Organizer, Team, User, + Event, InvoiceAddress, Item, Order, OrderPayment, OrderPosition, Organizer, + Team, User, ) @@ -168,3 +169,243 @@ class OrderSearchTest(SoupTest): assert '30C3-FO1' not in resp resp = self.client.get('/control/search/orders/?query=FO2').content.decode() assert '30C3-FO1' not in resp + + +class PaymentSearchTest(SoupTest): + @scopes_disabled() + def setUp(self): + super().setUp() + self.user = User.objects.create_user('dummy@dummy.dummy', 'dummy') + self.orga1 = Organizer.objects.create(name='CCC', slug='ccc') + self.orga2 = Organizer.objects.create(name='NoOrga', slug='no') + self.event1 = Event.objects.create( + organizer=self.orga1, name='30C3', slug='30c3', + date_from=datetime.datetime(2013, 12, 26, tzinfo=datetime.timezone.utc), + plugins='pretix.plugins.banktransfer,tests.testdummy' + ) + self.event2 = Event.objects.create( + organizer=self.orga1, name='31C3', slug='31c3', + date_from=datetime.datetime(2014, 12, 26, tzinfo=datetime.timezone.utc), + ) + + o1 = Order.objects.create( + code='FO1A', event=self.event1, email='dummy1@dummy.test', + status=Order.STATUS_PENDING, + datetime=now(), expires=now() + datetime.timedelta(days=10), + total=14, locale='en' + ) + InvoiceAddress.objects.create(order=o1, company="Test Ltd.", name_parts={'full_name': "Peter Miller", "_scheme": "full"}) + ticket1 = Item.objects.create(event=self.event1, name='Early-bird ticket', + category=None, default_price=23, + admission=True) + OrderPosition.objects.create( + order=o1, + item=ticket1, + variation=None, + price=Decimal("14"), + attendee_name_parts={'full_name': "Peter", "_scheme": "full"}, + attendee_email="att@att.com" + ) + OrderPayment.objects.create( + local_id=1, + state=OrderPayment.PAYMENT_STATE_CONFIRMED, + amount=Decimal("14"), + order=o1, + provider="giftcard", + info="{test payment order 1}" + ) + OrderPayment.objects.create( + local_id=1, + state=OrderPayment.PAYMENT_STATE_REFUNDED, + amount=Decimal("14"), + order=o1, + provider="manual", + info="{refunded payment}" + ) + OrderPayment.objects.create( + local_id=1, + state=OrderPayment.PAYMENT_STATE_CANCELED, + amount=Decimal("14"), + order=o1, + provider="manual", + info="{canceled payment}" + ) + OrderPayment.objects.create( + local_id=1, + state=OrderPayment.PAYMENT_STATE_FAILED, + amount=Decimal("14"), + order=o1, + provider="manual", + info="{failed payment}" + ) + OrderPayment.objects.create( + local_id=1, + state=OrderPayment.PAYMENT_STATE_PENDING, + amount=Decimal("14"), + order=o1, + provider="manual", + info="{pending payment}" + ) + + o2 = Order.objects.create( + code='FO2', event=self.event2, email='dummy2@dummy.test', + status=Order.STATUS_PENDING, + datetime=now(), expires=now() + datetime.timedelta(days=10), + total=15, locale='en' + ) + ticket2 = Item.objects.create(event=self.event1, name='Early-bird ticket', + category=None, default_price=23, + admission=True) + OrderPosition.objects.create( + order=o2, + item=ticket2, + variation=None, + price=Decimal("15"), + attendee_name_parts={'full_name': "Mark", "_scheme": "full"} + ) + OrderPayment.objects.create( + local_id=1, + state=OrderPayment.PAYMENT_STATE_CREATED, + amount=Decimal("15"), + order=o2, + provider="manual", + info="{test payment order 2}" + ) + + self.team = Team.objects.create(organizer=self.orga1, can_view_orders=True) + self.team2 = Team.objects.create(organizer=self.orga2, can_view_orders=True) + self.team.members.add(self.user) + self.team.limit_events.add(self.event1) + + self.client.login(email='dummy@dummy.dummy', password='dummy') + + def test_team_limit_event(self): + resp = self.client.get('/control/search/payments/').content.decode() + assert 'FO1' in resp + assert 'FO2' not in resp + + def test_team_limit_event_wrong_permission(self): + self.team.can_view_orders = False + self.team.save() + resp = self.client.get('/control/search/payments/').content.decode() + assert 'FO1' not in resp + assert 'FO2' not in resp + + def test_team_all_events(self): + self.team.all_events = True + self.team.save() + resp = self.client.get('/control/search/payments/').content.decode() + assert 'FO1' in resp + assert 'FO2' in resp + + def test_team_all_events_wrong_permission(self): + self.team.all_events = True + self.team.can_view_orders = False + self.team.save() + resp = self.client.get('/control/search/payments/').content.decode() + assert 'FO1' not in resp + assert 'FO2' not in resp + + def test_team_none(self): + self.team.members.clear() + resp = self.client.get('/control/search/payments/').content.decode() + assert 'FO1' not in resp + assert 'FO2' not in resp + + def test_superuser(self): + self.user.is_staff = True + self.user.staffsession_set.create(date_start=now(), session_key=self.client.session.session_key) + self.user.save() + self.team.members.clear() + resp = self.client.get('/control/search/payments/').content.decode() + assert 'FO1' in resp + assert 'FO2' in resp + + def test_filter_email(self): + resp = self.client.get('/control/search/payments/?query=dummy1@dummy').content.decode() + assert 'FO1' in resp + resp = self.client.get('/control/search/payments/?query=dummynope').content.decode() + assert 'FO1' not in resp + + def test_filter_invoice_name(self): + resp = self.client.get('/control/search/payments/?query=Pete').content.decode() + assert 'FO1' in resp + resp = self.client.get('/control/search/payments/?query=Mark').content.decode() + assert 'FO1' not in resp + + def test_filter_invoice_address(self): + resp = self.client.get('/control/search/payments/?query=Ltd').content.decode() + assert 'FO1' in resp + resp = self.client.get('/control/search/payments/?query=Miller').content.decode() + assert 'FO1' in resp + resp = self.client.get('/control/search/payments/?query=Mark').content.decode() + assert 'FO1' not in resp + + def test_filter_code(self): + resp = self.client.get('/control/search/payments/?query=FO1').content.decode() + assert '30C3-FO1' in resp + resp = self.client.get('/control/search/payments/?query=30c3-FO1').content.decode() + assert '30C3-FO1' in resp + resp = self.client.get('/control/search/payments/?query=30C3-fO1A').content.decode() + assert '30C3-FO1' in resp + resp = self.client.get('/control/search/payments/?query=30C3-fo14').content.decode() + assert '30C3-FO1' in resp + resp = self.client.get('/control/search/payments/?query=31c3-FO1').content.decode() + assert '30C3-FO1' not in resp + resp = self.client.get('/control/search/payments/?query=FO2').content.decode() + assert '30C3-FO1' not in resp + + def test_filter_amount(self): + self.team.all_events = True + self.team.save() + resp = self.client.get('/control/search/payments/?amount=14').content.decode() + assert 'FO1' in resp + assert 'FO2' not in resp + resp = self.client.get('/control/search/payments/?amount=15.00').content.decode() + assert 'FO1' not in resp + assert 'FO2' in resp + + def test_filter_event(self): + self.team.all_events = True + self.team.save() + event_id = str(self.event1.pk) + resp = self.client.get('/control/search/payments/?event=' + event_id).content.decode() + assert "FO1" in resp + assert "FO2" not in resp + + def test_filter_organizer(self): + self.team2.members.add(self.user) + self.user.save() + + b = str(self.orga1.pk) + resp = self.client.get('/control/search/payments/?organizer=' + b).content.decode() + print(resp) + assert "FO1" in resp + + b = str(self.orga2.pk) + resp = self.client.get('/control/search/payments/?organizer=' + b).content.decode() + print(resp) + assert "FO1" not in resp + + def test_filter_state(self): + self.user.is_staff = True + self.user.staffsession_set.create(date_start=now(), session_key=self.client.session.session_key) + self.user.save() + + confirmed = OrderPayment.PAYMENT_STATE_CONFIRMED + resp = self.client.get('/control/search/payments/?state=' + confirmed).content.decode() + assert "P-1" in resp + assert "P-2" not in resp + assert "P-3" not in resp + assert "P-4" not in resp + assert "P-5" not in resp + assert "P-6" not in resp + + def test_filter_provider(self): + resp = self.client.get('/control/search/payments/?state=giftcard').content.decode() + assert "P-1" in resp + assert "P-2" not in resp + assert "P-3" not in resp + assert "P-4" not in resp + assert "P-5" not in resp + assert "P-6" not in resp