diff --git a/src/pretix/api/pagination.py b/src/pretix/api/pagination.py index f7f4298ed3..4f8a55579e 100644 --- a/src/pretix/api/pagination.py +++ b/src/pretix/api/pagination.py @@ -19,9 +19,19 @@ # You should have received a copy of the GNU Affero General Public License along with this program. If not, see # . # +from rest_framework.filters import OrderingFilter from rest_framework.pagination import PageNumberPagination +from pretix.helpers import get_deterministic_ordering + class Pagination(PageNumberPagination): page_size_query_param = 'page_size' max_page_size = 50 + + +class TotalOrderingFilter(OrderingFilter): + def get_ordering(self, request, queryset, view): + o = super().get_ordering(request, queryset, view) + o = get_deterministic_ordering(queryset.model, o) + return o diff --git a/src/pretix/api/views/__init__.py b/src/pretix/api/views/__init__.py index 53b11d084c..04728f638e 100644 --- a/src/pretix/api/views/__init__.py +++ b/src/pretix/api/views/__init__.py @@ -24,10 +24,11 @@ from calendar import timegm from django.db.models import Max from django.http import HttpResponse from django.utils.http import http_date, parse_http_date_safe -from rest_framework.filters import OrderingFilter + +from pretix.api.pagination import TotalOrderingFilter -class RichOrderingFilter(OrderingFilter): +class RichOrderingFilter(TotalOrderingFilter): def filter_queryset(self, request, queryset, view): ordering = self.get_ordering(request, queryset, view) diff --git a/src/pretix/api/views/cart.py b/src/pretix/api/views/cart.py index 0d1849a614..1c13c41ed7 100644 --- a/src/pretix/api/views/cart.py +++ b/src/pretix/api/views/cart.py @@ -29,11 +29,11 @@ from django.utils.translation import gettext as _ from rest_framework import status, viewsets from rest_framework.decorators import action from rest_framework.exceptions import ValidationError -from rest_framework.filters import OrderingFilter from rest_framework.mixins import CreateModelMixin, DestroyModelMixin from rest_framework.response import Response from rest_framework.serializers import as_serializer_error +from pretix.api.pagination import TotalOrderingFilter from pretix.api.serializers.cart import ( CartPositionCreateSerializer, CartPositionSerializer, ) @@ -47,7 +47,7 @@ from pretix.base.services.locking import NoLockManager class CartPositionViewSet(CreateModelMixin, DestroyModelMixin, viewsets.ReadOnlyModelViewSet): serializer_class = CartPositionSerializer queryset = CartPosition.objects.none() - filter_backends = (OrderingFilter,) + filter_backends = (TotalOrderingFilter,) ordering = ('datetime',) ordering_fields = ('datetime', 'cart_id') lookup_field = 'id' diff --git a/src/pretix/api/views/discount.py b/src/pretix/api/views/discount.py index 2f436e44eb..71be2a2799 100644 --- a/src/pretix/api/views/discount.py +++ b/src/pretix/api/views/discount.py @@ -36,8 +36,8 @@ from django_filters.rest_framework import DjangoFilterBackend, FilterSet from django_scopes import scopes_disabled from rest_framework import viewsets from rest_framework.exceptions import PermissionDenied -from rest_framework.filters import OrderingFilter +from pretix.api.pagination import TotalOrderingFilter from pretix.api.serializers.discount import DiscountSerializer from pretix.api.views import ConditionalListView from pretix.base.models import CartPosition, Discount @@ -52,7 +52,7 @@ with scopes_disabled(): class DiscountViewSet(ConditionalListView, viewsets.ModelViewSet): serializer_class = DiscountSerializer queryset = Discount.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) filterset_class = DiscountFilter ordering_fields = ('id', 'position') ordering = ('position', 'id') diff --git a/src/pretix/api/views/event.py b/src/pretix/api/views/event.py index e5b482d0d5..e564de0576 100644 --- a/src/pretix/api/views/event.py +++ b/src/pretix/api/views/event.py @@ -39,11 +39,12 @@ from django.db.models import Prefetch, ProtectedError, Q from django.utils.timezone import now from django_filters.rest_framework import DjangoFilterBackend, FilterSet from django_scopes import scopes_disabled -from rest_framework import filters, serializers, views, viewsets +from rest_framework import serializers, views, viewsets from rest_framework.exceptions import PermissionDenied, ValidationError from rest_framework.response import Response from pretix.api.auth.permission import EventCRUDPermission +from pretix.api.pagination import TotalOrderingFilter from pretix.api.serializers.event import ( CloneEventSerializer, DeviceEventSettingsSerializer, EventSerializer, EventSettingsSerializer, SubEventSerializer, TaxRuleSerializer, @@ -127,7 +128,7 @@ class EventViewSet(viewsets.ModelViewSet): lookup_url_kwarg = 'event' lookup_value_regex = '[^/]+' permission_classes = (EventCRUDPermission,) - filter_backends = (DjangoFilterBackend, filters.OrderingFilter) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) ordering = ('slug',) ordering_fields = ('date_from', 'slug') filterset_class = EventFilter @@ -379,7 +380,7 @@ class SubEventViewSet(ConditionalListView, viewsets.ModelViewSet): serializer_class = SubEventSerializer queryset = SubEvent.objects.none() write_permission = 'can_change_event_settings' - filter_backends = (DjangoFilterBackend, filters.OrderingFilter) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) filterset_class = SubEventFilter ordering = ('date_from',) ordering_fields = ('id', 'date_from', 'last_modified') diff --git a/src/pretix/api/views/item.py b/src/pretix/api/views/item.py index 3003aa7b96..2d24f4065c 100644 --- a/src/pretix/api/views/item.py +++ b/src/pretix/api/views/item.py @@ -41,9 +41,9 @@ from django_scopes import scopes_disabled from rest_framework import viewsets from rest_framework.decorators import action from rest_framework.exceptions import PermissionDenied -from rest_framework.filters import OrderingFilter from rest_framework.response import Response +from pretix.api.pagination import TotalOrderingFilter from pretix.api.serializers.item import ( ItemAddOnSerializer, ItemBundleSerializer, ItemCategorySerializer, ItemSerializer, ItemVariationSerializer, QuestionOptionSerializer, @@ -75,7 +75,7 @@ with scopes_disabled(): class ItemViewSet(ConditionalListView, viewsets.ModelViewSet): serializer_class = ItemSerializer queryset = Item.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) ordering_fields = ('id', 'position') ordering = ('position', 'id') filterset_class = ItemFilter @@ -138,7 +138,7 @@ class ItemViewSet(ConditionalListView, viewsets.ModelViewSet): class ItemVariationViewSet(viewsets.ModelViewSet): serializer_class = ItemVariationSerializer queryset = ItemVariation.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter,) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter,) ordering_fields = ('id', 'position') ordering = ('id',) permission = None @@ -208,7 +208,7 @@ class ItemVariationViewSet(viewsets.ModelViewSet): class ItemBundleViewSet(viewsets.ModelViewSet): serializer_class = ItemBundleSerializer queryset = ItemBundle.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter,) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter,) ordering_fields = ('id',) ordering = ('id',) permission = None @@ -260,7 +260,7 @@ class ItemBundleViewSet(viewsets.ModelViewSet): class ItemAddOnViewSet(viewsets.ModelViewSet): serializer_class = ItemAddOnSerializer queryset = ItemAddOn.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter,) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter,) ordering_fields = ('id', 'position') ordering = ('id',) permission = None @@ -318,7 +318,7 @@ class ItemCategoryFilter(FilterSet): class ItemCategoryViewSet(ConditionalListView, viewsets.ModelViewSet): serializer_class = ItemCategorySerializer queryset = ItemCategory.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) filterset_class = ItemCategoryFilter ordering_fields = ('id', 'position') ordering = ('position', 'id') @@ -373,7 +373,7 @@ with scopes_disabled(): class QuestionViewSet(ConditionalListView, viewsets.ModelViewSet): serializer_class = QuestionSerializer queryset = Question.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) filterset_class = QuestionFilter ordering_fields = ('id', 'position') ordering = ('position', 'id') @@ -418,7 +418,7 @@ class QuestionViewSet(ConditionalListView, viewsets.ModelViewSet): class QuestionOptionViewSet(viewsets.ModelViewSet): serializer_class = QuestionOptionSerializer queryset = QuestionOption.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter,) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter,) ordering_fields = ('id', 'position') ordering = ('position',) permission = None @@ -475,7 +475,7 @@ with scopes_disabled(): class QuotaViewSet(ConditionalListView, viewsets.ModelViewSet): serializer_class = QuotaSerializer queryset = Quota.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter,) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter,) filterset_class = QuotaFilter ordering_fields = ('id', 'size') ordering = ('id',) diff --git a/src/pretix/api/views/order.py b/src/pretix/api/views/order.py index 173811b4c2..63c7f45f35 100644 --- a/src/pretix/api/views/order.py +++ b/src/pretix/api/views/order.py @@ -43,11 +43,11 @@ from rest_framework.decorators import action from rest_framework.exceptions import ( APIException, NotFound, PermissionDenied, ValidationError, ) -from rest_framework.filters import OrderingFilter from rest_framework.mixins import CreateModelMixin from rest_framework.response import Response from pretix.api.models import OAuthAccessToken +from pretix.api.pagination import TotalOrderingFilter from pretix.api.serializers.order import ( BlockedTicketSecretSerializer, InvoiceSerializer, OrderCreateSerializer, OrderPaymentCreateSerializer, OrderPaymentSerializer, @@ -181,7 +181,7 @@ with scopes_disabled(): class OrderViewSet(viewsets.ModelViewSet): serializer_class = OrderSerializer queryset = Order.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) ordering = ('datetime',) ordering_fields = ('datetime', 'code', 'status', 'last_modified') filterset_class = OrderFilter @@ -1749,7 +1749,7 @@ class RetryException(APIException): class InvoiceViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = InvoiceSerializer queryset = Invoice.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) ordering = ('nr',) ordering_fields = ('nr', 'date') filterset_class = InvoiceFilter @@ -1842,7 +1842,7 @@ with scopes_disabled(): class RevokedSecretViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = RevokedTicketSecretSerializer queryset = RevokedTicketSecret.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) ordering = ('-created',) ordering_fields = ('created', 'secret') filterset_class = RevokedSecretFilter @@ -1865,7 +1865,7 @@ with scopes_disabled(): class BlockedSecretViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = BlockedTicketSecretSerializer queryset = BlockedTicketSecret.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) ordering = ('-updated', '-pk') filterset_class = BlockedSecretFilter permission = 'can_view_orders' diff --git a/src/pretix/api/views/organizer.py b/src/pretix/api/views/organizer.py index 949723c35b..a578a37761 100644 --- a/src/pretix/api/views/organizer.py +++ b/src/pretix/api/views/organizer.py @@ -28,9 +28,7 @@ from django.shortcuts import get_object_or_404 from django.utils.functional import cached_property from django_filters.rest_framework import DjangoFilterBackend, FilterSet from django_scopes import scopes_disabled -from rest_framework import ( - filters, mixins, serializers, status, views, viewsets, -) +from rest_framework import mixins, serializers, status, views, viewsets from rest_framework.decorators import action from rest_framework.exceptions import MethodNotAllowed, PermissionDenied from rest_framework.mixins import CreateModelMixin, DestroyModelMixin @@ -38,6 +36,7 @@ from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet from pretix.api.models import OAuthAccessToken +from pretix.api.pagination import TotalOrderingFilter from pretix.api.serializers.organizer import ( CustomerCreateSerializer, CustomerSerializer, DeviceSerializer, GiftCardSerializer, GiftCardTransactionSerializer, MembershipSerializer, @@ -62,7 +61,7 @@ class OrganizerViewSet(viewsets.ReadOnlyModelViewSet): lookup_field = 'slug' lookup_url_kwarg = 'organizer' lookup_value_regex = '[^/]+' - filter_backends = (filters.OrderingFilter,) + filter_backends = (TotalOrderingFilter,) ordering = ('slug',) ordering_fields = ('name', 'slug') diff --git a/src/pretix/api/views/voucher.py b/src/pretix/api/views/voucher.py index bea1e07fcb..8c119f74a2 100644 --- a/src/pretix/api/views/voucher.py +++ b/src/pretix/api/views/voucher.py @@ -31,9 +31,9 @@ from django_scopes import scopes_disabled from rest_framework import status, viewsets from rest_framework.decorators import action from rest_framework.exceptions import PermissionDenied -from rest_framework.filters import OrderingFilter from rest_framework.response import Response +from pretix.api.pagination import TotalOrderingFilter from pretix.api.serializers.voucher import VoucherSerializer from pretix.base.models import Voucher @@ -59,7 +59,7 @@ with scopes_disabled(): class VoucherViewSet(viewsets.ModelViewSet): serializer_class = VoucherSerializer queryset = Voucher.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) ordering = ('id',) ordering_fields = ('id', 'code', 'max_usages', 'valid_until', 'value') filterset_class = VoucherFilter diff --git a/src/pretix/api/views/waitinglist.py b/src/pretix/api/views/waitinglist.py index 430995a104..2683e496a1 100644 --- a/src/pretix/api/views/waitinglist.py +++ b/src/pretix/api/views/waitinglist.py @@ -25,9 +25,9 @@ from django_scopes import scopes_disabled from rest_framework import viewsets from rest_framework.decorators import action from rest_framework.exceptions import PermissionDenied, ValidationError -from rest_framework.filters import OrderingFilter from rest_framework.response import Response +from pretix.api.pagination import TotalOrderingFilter from pretix.api.serializers.waitinglist import WaitingListSerializer from pretix.base.models import WaitingListEntry from pretix.base.models.waitinglist import WaitingListException @@ -47,8 +47,8 @@ with scopes_disabled(): class WaitingListViewSet(viewsets.ModelViewSet): serializer_class = WaitingListSerializer queryset = WaitingListEntry.objects.none() - filter_backends = (DjangoFilterBackend, OrderingFilter) - ordering = ('created',) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) + ordering = ('created', 'pk',) ordering_fields = ('id', 'created', 'email', 'item') filterset_class = WaitingListFilter permission = 'can_view_orders' diff --git a/src/pretix/base/migrations/0234_total_ordering.py b/src/pretix/base/migrations/0234_total_ordering.py new file mode 100644 index 0000000000..dfbd53d53d --- /dev/null +++ b/src/pretix/base/migrations/0234_total_ordering.py @@ -0,0 +1,56 @@ +# Generated by Django 3.2.16 on 2023-02-01 10:58 + +import django.core.serializers.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('pretixbase', '0233_ignore_from_quota_while_blocked'), + ] + + operations = [ + migrations.AlterField( + model_name='logentry', + name='datetime', + field=models.DateTimeField(auto_now_add=True), + ), + migrations.AlterField( + model_name='order', + name='datetime', + field=models.DateTimeField(), + ), + migrations.AlterField( + model_name='order', + name='last_modified', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='scheduledeventexport', + name='export_form_data', + field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), + ), + migrations.AlterField( + model_name='scheduledorganizerexport', + name='export_form_data', + field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), + ), + migrations.AlterField( + model_name='transaction', + name='datetime', + field=models.DateTimeField(), + ), + migrations.AlterIndexTogether( + name='logentry', + index_together={('datetime', 'id')}, + ), + migrations.AlterIndexTogether( + name='order', + index_together={('datetime', 'id'), ('last_modified', 'id')}, + ), + migrations.AlterIndexTogether( + name='transaction', + index_together={('datetime', 'id')}, + ), + ] diff --git a/src/pretix/base/models/checkin.py b/src/pretix/base/models/checkin.py index 9206adabb4..2a66b2d3c5 100644 --- a/src/pretix/base/models/checkin.py +++ b/src/pretix/base/models/checkin.py @@ -97,7 +97,7 @@ class CheckinList(LoggedModel): objects = ScopedManager(organizer='event__organizer') class Meta: - ordering = ('subevent__date_from', 'name') + ordering = ('subevent__date_from', 'name', 'pk') def positions_query(self, ignore_status=False): from . import Order, OrderPosition diff --git a/src/pretix/base/models/event.py b/src/pretix/base/models/event.py index c52e63b90e..34a16ec851 100644 --- a/src/pretix/base/models/event.py +++ b/src/pretix/base/models/event.py @@ -612,7 +612,7 @@ class Event(EventMixin, LoggedModel): class Meta: verbose_name = _("Event") verbose_name_plural = _("Events") - ordering = ("date_from", "name") + ordering = ("date_from", "name", "slug") unique_together = (('organizer', 'slug'),) def __str__(self): diff --git a/src/pretix/base/models/log.py b/src/pretix/base/models/log.py index acb48b41e1..ec8ac74d9e 100644 --- a/src/pretix/base/models/log.py +++ b/src/pretix/base/models/log.py @@ -72,7 +72,7 @@ class LogEntry(models.Model): content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField(db_index=True) content_object = GenericForeignKey('content_type', 'object_id') - datetime = models.DateTimeField(auto_now_add=True, db_index=True) + datetime = models.DateTimeField(auto_now_add=True) user = models.ForeignKey('User', null=True, blank=True, on_delete=models.PROTECT) api_token = models.ForeignKey('TeamAPIToken', null=True, blank=True, on_delete=models.PROTECT) device = models.ForeignKey('Device', null=True, blank=True, on_delete=models.PROTECT) @@ -88,6 +88,9 @@ class LogEntry(models.Model): class Meta: ordering = ('-datetime', '-id') + index_together = [ + ['datetime', 'id'] + ] def display(self): from ..signals import logentry_display diff --git a/src/pretix/base/models/orders.py b/src/pretix/base/models/orders.py index 642d263f26..248bc2a50f 100644 --- a/src/pretix/base/models/orders.py +++ b/src/pretix/base/models/orders.py @@ -211,7 +211,7 @@ class Order(LockModel, LoggedModel): ) secret = models.CharField(max_length=32, default=generate_secret) datetime = models.DateTimeField( - verbose_name=_("Date"), db_index=True + verbose_name=_("Date"), db_index=False ) cancellation_date = models.DateTimeField( null=True, blank=True @@ -252,7 +252,7 @@ class Order(LockModel, LoggedModel): null=True, blank=True ) last_modified = models.DateTimeField( - auto_now=True, db_index=True + auto_now=True, db_index=False ) require_approval = models.BooleanField( default=False @@ -268,7 +268,11 @@ class Order(LockModel, LoggedModel): class Meta: verbose_name = _("Order") verbose_name_plural = _("Orders") - ordering = ("-datetime",) + ordering = ("-datetime", "-pk") + index_together = [ + ["datetime", "id"], + ["last_modified", "id"], + ] def __str__(self): return self.full_code @@ -2618,7 +2622,6 @@ class Transaction(models.Model): ) datetime = models.DateTimeField( verbose_name=_("Date"), - db_index=True, ) migrated = models.BooleanField( default=False @@ -2669,6 +2672,9 @@ class Transaction(models.Model): class Meta: ordering = 'datetime', 'pk' + index_together = [ + ['datetime', 'id'] + ] def save(self, *args, **kwargs): if not self.fee_type and not self.item: diff --git a/src/pretix/base/models/organizer.py b/src/pretix/base/models/organizer.py index 9568ec5bd5..2417577a91 100644 --- a/src/pretix/base/models/organizer.py +++ b/src/pretix/base/models/organizer.py @@ -93,7 +93,7 @@ class Organizer(LoggedModel): class Meta: verbose_name = _("Organizer") verbose_name_plural = _("Organizers") - ordering = ("name",) + ordering = ("name", "slug") def __str__(self) -> str: return self.name diff --git a/src/pretix/base/models/waitinglist.py b/src/pretix/base/models/waitinglist.py index 98571d39ba..5c650c378e 100644 --- a/src/pretix/base/models/waitinglist.py +++ b/src/pretix/base/models/waitinglist.py @@ -112,7 +112,7 @@ class WaitingListEntry(LoggedModel): class Meta: verbose_name = _("Waiting list entry") verbose_name_plural = _("Waiting list entries") - ordering = ('-priority', 'created') + ordering = ('-priority', 'created', 'pk') def __str__(self): return '%s waits for %s' % (str(self.email), str(self.item)) diff --git a/src/pretix/control/forms/filter.py b/src/pretix/control/forms/filter.py index cb775be981..c91449b03c 100644 --- a/src/pretix/control/forms/filter.py +++ b/src/pretix/control/forms/filter.py @@ -58,13 +58,15 @@ from pretix.base.models import ( Checkin, CheckinList, Device, Event, EventMetaProperty, EventMetaValue, Gate, Invoice, InvoiceAddress, Item, Order, OrderPayment, OrderPosition, OrderRefund, Organizer, Question, QuestionAnswer, SubEvent, - SubEventMetaValue, Team, TeamAPIToken, TeamInvite, + SubEventMetaValue, Team, TeamAPIToken, TeamInvite, Voucher, ) from pretix.base.signals import register_payment_providers from pretix.control.forms.widgets import Select2 from pretix.control.signals import order_search_filter_q from pretix.helpers.countries import CachedCountries -from pretix.helpers.database import rolledback_transaction +from pretix.helpers.database import ( + get_deterministic_ordering, rolledback_transaction, +) from pretix.helpers.dicts import move_to_end from pretix.helpers.i18n import i18ncomp @@ -380,7 +382,9 @@ class OrderFilterForm(FilterForm): ) if fdata.get('ordering'): - qs = qs.order_by(self.get_order_by()) + qs = qs.order_by(*get_deterministic_ordering(Order, self.get_order_by())) + else: + qs = qs.order_by('-datetime', '-pk') if fdata.get('provider'): qs = qs.annotate( @@ -1044,11 +1048,11 @@ class OrderPaymentSearchFilterForm(forms.Form): if fdata.get('ordering'): p = self.cleaned_data.get('ordering') if p.startswith('-') and p not in self.orders: - qs = qs.order_by('-' + self.orders[p[1:]]) + qs = qs.order_by(*get_deterministic_ordering(OrderPayment, '-' + self.orders[p[1:]])) else: - qs = qs.order_by(self.orders[p]) + qs = qs.order_by(*get_deterministic_ordering(OrderPayment, self.orders[p])) else: - qs = qs.order_by('-created') + qs = qs.order_by('-created', '-pk') return qs @@ -1226,9 +1230,9 @@ class SubEventFilterForm(FilterForm): qs = qs.filter(f) if fdata.get('ordering'): - qs = qs.order_by(self.get_order_by()) + qs = qs.order_by(*get_deterministic_ordering(SubEvent, self.get_order_by())) else: - qs = qs.order_by('-date_from') + qs = qs.order_by('-date_from', '-pk') return qs @@ -1462,9 +1466,7 @@ class TeamFilterForm(FilterForm): ) if fdata.get('ordering'): - qs = qs.order_by(self.get_order_by()) - else: - qs = qs.order_by('name') + qs = qs.order_by(*get_deterministic_ordering(Team, self.get_order_by())) return qs.distinct() @@ -1619,7 +1621,7 @@ class EventFilterForm(FilterForm): qs = qs.filter(f) if fdata.get('ordering'): - qs = qs.order_by(self.get_order_by()) + qs = qs.order_by(*get_deterministic_ordering(Event, self.get_order_by())) return qs @@ -1764,11 +1766,11 @@ class CheckinListAttendeeFilterForm(FilterForm): if isinstance(ob, dict): ob = dict(ob) o = ob.pop('_order') - qs = qs.annotate(**ob).order_by(o) + qs = qs.annotate(**ob).order_by(*get_deterministic_ordering(OrderPosition, [o])) elif isinstance(ob, (list, tuple)): - qs = qs.order_by(*ob) + qs = qs.order_by(*get_deterministic_ordering(OrderPosition, ob)) else: - qs = qs.order_by(ob) + qs = qs.order_by(*get_deterministic_ordering(OrderPosition, [ob])) if fdata.get('item'): qs = qs.filter(item=fdata.get('item')) @@ -2011,11 +2013,11 @@ class VoucherFilterForm(FilterForm): if isinstance(ob, dict): ob = dict(ob) o = ob.pop('_order') - qs = qs.annotate(**ob).order_by(o) + qs = qs.annotate(**ob).order_by(*get_deterministic_ordering(Voucher, o)) elif isinstance(ob, (list, tuple)): - qs = qs.order_by(*ob) + qs = qs.order_by(*get_deterministic_ordering(Voucher, ob)) else: - qs = qs.order_by(ob) + qs = qs.order_by(*get_deterministic_ordering(Voucher, ob)) return qs @@ -2098,9 +2100,7 @@ class RefundFilterForm(FilterForm): OrderRefund.REFUND_STATE_EXTERNAL]) if fdata.get('ordering'): - qs = qs.order_by(self.get_order_by()) - else: - qs = qs.order_by('-created') + qs = qs.order_by(*get_deterministic_ordering(OrderRefund, self.get_order_by())) return qs diff --git a/src/pretix/control/views/checkin.py b/src/pretix/control/views/checkin.py index 63a86bbcd9..2e891bab46 100644 --- a/src/pretix/control/views/checkin.py +++ b/src/pretix/control/views/checkin.py @@ -447,6 +447,7 @@ class CheckinListView(EventPermissionRequiredMixin, PaginationMixin, ListView): context_object_name = 'checkins' permission = 'can_view_orders' template_name = 'pretixcontrol/checkin/checkins.html' + ordering = ('-datetime', '-pk') def get_queryset(self): qs = Checkin.all.filter( diff --git a/src/pretix/control/views/event.py b/src/pretix/control/views/event.py index 1de62ba1e0..65f3efa8f1 100644 --- a/src/pretix/control/views/event.py +++ b/src/pretix/control/views/event.py @@ -1061,7 +1061,7 @@ class EventLog(EventPermissionRequiredMixin, PaginationMixin, ListView): def get_queryset(self): qs = self.request.event.logentry_set.all().select_related( 'user', 'content_type', 'api_token', 'oauth_application', 'device' - ).order_by('-datetime') + ).order_by('-datetime', '-pk') qs = qs.exclude(action_type__in=OVERVIEW_BANLIST) if not self.request.user.has_event_permission(self.request.organizer, self.request.event, 'can_view_orders', request=self.request): diff --git a/src/pretix/control/views/item.py b/src/pretix/control/views/item.py index fa214921fb..1504e0b808 100644 --- a/src/pretix/control/views/item.py +++ b/src/pretix/control/views/item.py @@ -816,16 +816,18 @@ class QuotaList(PaginationMixin, ListView): qs = qs.filter(subevent_id=s) valid_orders = { - '-date': ('-subevent__date_from', 'name'), - 'date': ('subevent__date_from', '-name'), - 'size': ('size', 'name'), - '-size': ('-size', '-name'), - 'name': ('name',), - '-name': ('-name',), + '-date': ('-subevent__date_from', 'name', 'pk'), + 'date': ('subevent__date_from', '-name', '-pk'), + 'size': ('size', 'name', 'pk'), + '-size': ('-size', '-name', '-pk'), + 'name': ('name', 'pk'), + '-name': ('-name', '-pk'), } if self.request.GET.get("ordering", "-date") in valid_orders: qs = qs.order_by(*valid_orders[self.request.GET.get("ordering", "-date")]) + else: + qs = qs.order_by('name', 'subevent__date_from', 'pk') return qs diff --git a/src/pretix/control/views/orders.py b/src/pretix/control/views/orders.py index 0a5747237b..fe6ad2aff2 100644 --- a/src/pretix/control/views/orders.py +++ b/src/pretix/control/views/orders.py @@ -2529,7 +2529,7 @@ class RefundList(EventPermissionRequiredMixin, PaginationMixin, ListView): def get_queryset(self): qs = OrderRefund.objects.filter( order__event=self.request.event - ).select_related('order') + ).select_related('order').order_by('-created', '-pk') if self.filter_form.is_valid(): qs = self.filter_form.filter_qs(qs) diff --git a/src/pretix/control/views/organizer.py b/src/pretix/control/views/organizer.py index b46f44233a..ae531e358c 100644 --- a/src/pretix/control/views/organizer.py +++ b/src/pretix/control/views/organizer.py @@ -556,7 +556,7 @@ class TeamListView(OrganizerDetailViewMixin, OrganizerPermissionRequiredMixin, P memcount=Count('members', distinct=True), eventcount=Count('limit_events', distinct=True), invcount=Count('invites', distinct=True) - ).all().order_by('name') + ).all().order_by('name', 'pk') if self.filter_form.is_valid(): qs = self.filter_form.filter_qs(qs) return qs @@ -2040,6 +2040,8 @@ class LogView(OrganizerPermissionRequiredMixin, PaginationMixin, ListView): context_object_name = 'logs' def get_queryset(self): + # technically, we'd also need to sort by pk since this is a paginated list, but in this case we just can't + # bear the performance cost qs = self.request.organizer.all_logentries().select_related( 'user', 'content_type', 'api_token', 'oauth_application', 'device' ).order_by('-datetime') @@ -2437,7 +2439,7 @@ class CustomerDetailView(OrganizerDetailViewMixin, OrganizerPermissionRequiredMi q |= Q(email__iexact=self.customer.email) qs = Order.objects.filter( q - ).select_related('event').order_by('-datetime') + ).select_related('event').order_by('-datetime', 'pk') return qs @cached_property diff --git a/src/pretix/helpers/database.py b/src/pretix/helpers/database.py index b9568a204d..c676ff98a7 100644 --- a/src/pretix/helpers/database.py +++ b/src/pretix/helpers/database.py @@ -21,8 +21,11 @@ # import contextlib +from django.core.exceptions import FieldDoesNotExist from django.db import connection, transaction -from django.db.models import Aggregate, Expression, Field, Lookup, Value +from django.db.models import ( + Aggregate, Expression, F, Field, Lookup, OrderBy, Value, +) from django.utils.functional import lazy @@ -148,3 +151,74 @@ class PostgresWindowFrame(Expression): # This is a short-hand for .select_for_update(of=("self,")), that falls back gracefully on databases that don't support # the SELECT FOR UPDATE OF ... query. OF_SELF = lazy(lambda: ("self",) if connection.features.has_select_for_update_of else (), tuple)() + + +def get_deterministic_ordering(model, ordering): + """ + Ensure a deterministic order across all database backends. Search for a + single field or unique together set of fields providing a total + ordering. If these are missing, augment the ordering with a descendant + primary key. + + This has mostly been vendored from + https://github.com/django/django/blob/d8e1442ce2c56282785dd806e5c1147975e8c857/django/contrib/admin/views/main.py#L390 + """ + if isinstance(ordering, str): + ordering = (ordering,) + ordering = list(ordering) + ordering_fields = set() + total_ordering_fields = {"pk"} | { + field.attname + for field in model._meta.fields + if field.unique and not field.null + } + for part in ordering: + # Search for single field providing a total ordering. + field_name = None + if isinstance(part, str): + field_name = part.lstrip("-") + elif isinstance(part, F): + field_name = part.name + elif isinstance(part, OrderBy) and isinstance(part.expression, F): + field_name = part.expression.name + if field_name: + # Normalize attname references by using get_field(). + try: + field = model._meta.get_field(field_name) + except FieldDoesNotExist: + # Could be "?" for random ordering or a related field + # lookup. Skip this part of introspection for now. + continue + # Ordering by a related field name orders by the referenced + # model's ordering. Skip this part of introspection for now. + if field.remote_field and field_name == field.name: + continue + if field.attname in total_ordering_fields: + break + ordering_fields.add(field.attname) + else: + # No single total ordering field, try unique_together and total + # unique constraints. + constraint_field_names = ( + *model._meta.unique_together, + *( + constraint.fields + for constraint in model._meta.total_unique_constraints + ), + ) + for field_names in constraint_field_names: + # Normalize attname references by using get_field(). + fields = [ + model._meta.get_field(field_name) for field_name in field_names + ] + # Composite unique constraints containing a nullable column + # cannot ensure total ordering. + if any(field.null for field in fields): + continue + if ordering_fields.issuperset(field.attname for field in fields): + break + else: + # If no set of unique fields is present in the ordering, rely + # on the primary key to provide total ordering. + ordering.append("-pk") + return ordering diff --git a/src/pretix/plugins/sendmail/api.py b/src/pretix/plugins/sendmail/api.py index 134f353d04..f9fbcb9005 100644 --- a/src/pretix/plugins/sendmail/api.py +++ b/src/pretix/plugins/sendmail/api.py @@ -23,8 +23,8 @@ from django.core.exceptions import ValidationError from django_filters.rest_framework import DjangoFilterBackend, FilterSet from django_scopes import scopes_disabled from rest_framework import viewsets -from rest_framework.filters import OrderingFilter +from pretix.api.pagination import TotalOrderingFilter from pretix.api.serializers.i18n import I18nAwareModelSerializer from pretix.plugins.sendmail.models import Rule @@ -73,7 +73,7 @@ with scopes_disabled(): class RuleViewSet(viewsets.ModelViewSet): queryset = Rule.objects.none() serializer_class = RuleSerializer - filter_backends = (DjangoFilterBackend, OrderingFilter) + filter_backends = (DjangoFilterBackend, TotalOrderingFilter) filterset_class = RuleFilter ordering = ('id',) ordering_fields = ('id',) diff --git a/src/pretix/plugins/sendmail/views.py b/src/pretix/plugins/sendmail/views.py index 79da63e5f5..f173d55473 100644 --- a/src/pretix/plugins/sendmail/views.py +++ b/src/pretix/plugins/sendmail/views.py @@ -700,7 +700,7 @@ class ListRules(EventPermissionRequiredMixin, PaginationMixin, ListView): ), ).prefetch_related( 'limit_products' - ) + ).order_by('-send_date', 'subject', 'pk') class DeleteRule(EventPermissionRequiredMixin, DeleteView): @@ -746,7 +746,7 @@ class ScheduleView(EventPermissionRequiredMixin, PaginationMixin, ListView): def get_queryset(self): return self.rule.scheduledmail_set.select_related('subevent').order_by( - '-computed_datetime' + '-computed_datetime', '-pk' ) def get_context_data(self, **kwargs):