Tests: Do not rely on actual DNS for peppol validation

src/pretix/base/invoicing/peppol.py

@@ -131,6 +131,27 @@ class PeppolIdValidator:
     def __init__(self, validate_online=False):
         self.validate_online = validate_online
 
+    def _validate_peppol_id_online(self, value):
+        base_hostnames = ['participant.sml.prod.tech.peppol.org', 'participant.sml.test.tech.peppol.org']
+        smp_id = base64.b32encode(hashlib.sha256(value.lower().encode()).digest()).decode().rstrip("=")
+        for base_hostname in base_hostnames:
+            smp_domain = f'{smp_id}.iso6523-actorid-upis.{base_hostname}'
+            resolver = dns.resolver.Resolver()
+            try:
+                answers = resolver.resolve(smp_domain, 'NAPTR', lifetime=1.0)
+                if answers:
+                    return value
+            except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+                # ID not registered, do not set found=True
+                pass
+            except Exception:  # noqa
+                # Error likely on our end or infrastructure is down, allow user to proceed
+                return value
+
+        raise ValidationError(
+            _("The Peppol participant ID is not registered on the Peppol network."),
+        )
+
     def __call__(self, value):
         if ":" not in value:
             raise ValidationError(_("A Peppol participant ID always starts with a prefix, followed by a colon (:)."))
@@ -146,25 +167,7 @@ class PeppolIdValidator:
                                   params={"number": prefix})
 
         if self.validate_online:
-            base_hostnames = ['participant.sml.prod.tech.peppol.org', 'participant.sml.test.tech.peppol.org']
-            smp_id = base64.b32encode(hashlib.sha256(value.lower().encode()).digest()).decode().rstrip("=")
-            for base_hostname in base_hostnames:
-                smp_domain = f'{smp_id}.iso6523-actorid-upis.{base_hostname}'
-                resolver = dns.resolver.Resolver()
-                try:
-                    answers = resolver.resolve(smp_domain, 'NAPTR', lifetime=1.0)
-                    if answers:
-                        return value
-                except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-                    # ID not registered, do not set found=True
-                    pass
-                except Exception:  # noqa
-                    # Error likely on our end or infrastructure is down, allow user to proceed
-                    return value
-
-            raise ValidationError(
-                _("The Peppol participant ID is not registered on the Peppol network."),
-            )
+            value = self._validate_peppol_id_online(value)
 
         return value
 

src/pretix/control/forms/subevents.py

@@ -153,7 +153,7 @@ class SubEventBulkEditForm(I18nModelForm):
             self.fields[k].widget.is_required = False
             self.fields[k].required = False
 
-        for k in ('date_from', 'date_to', 'date_admission', 'presale_start', 'presale_end'):
+        for k in ('date_from', 'date_to'):
             self.fields[k + '_day'] = forms.DateField(
                 label=self._meta.model._meta.get_field(k).verbose_name,
                 help_text=self._meta.model._meta.get_field(k).help_text,
@@ -167,6 +167,14 @@ class SubEventBulkEditForm(I18nModelForm):
                 required=False,
             )
 
+        for k in ('date_admission', 'presale_start', 'presale_end'):
+            self.fields[k + '_rel'] = RelativeDateTimeField(
+                label=self._meta.model._meta.get_field(k).verbose_name,
+                help_text=self._meta.model._meta.get_field(k).help_text,
+                required=False,
+                limit_choices=('event__date_from',),
+            )
+
     class Meta:
         model = SubEvent
         localized_fields = '__all__'
@@ -247,6 +255,18 @@ class SubEventBulkEditForm(I18nModelForm):
                     )
                     setattr(obj, k.replace('_time', ''), newval)
                 fields.add(k.replace('_time', ''))
+            elif k.endswith('_rel'):
+                target_key = k[:-4]
+                for obj in objs:
+                    cval = self.cleaned_data[k]
+                    if cval is None:
+                        newval = None
+                        if not self._meta.model._meta.get_field(target_key).null:
+                            continue
+                    else:
+                        newval = cval.datetime(obj)
+                    setattr(obj, target_key, newval)
+                fields.add(target_key)
             else:
                 fields.add(k)
                 for obj in objs:

src/pretix/control/templates/pretixcontrol/subevents/bulk_edit.html

@@ -116,39 +116,9 @@
                     {% bootstrap_field form.date_to_time layout="bulkedit_inline" form_group_class="" %}
                 </div>
             </div>
-            <div class="form-group">
-                <label class="col-md-3 control-label" for="{{ form.date_admission_day.id_for_label }}">
-                    {{ form.date_admission_day.label }}
-                </label>
-                <div class="col-md-5">
-                    {% bootstrap_field form.date_admission_day layout="bulkedit_inline" form_group_class="" %}
-                </div>
-                <div class="col-md-4">
-                    {% bootstrap_field form.date_admission_time layout="bulkedit_inline" form_group_class="" %}
-                </div>
-            </div>
-            <div class="form-group">
-                <label class="col-md-3 control-label" for="{{ form.presale_start_day.id_for_label }}">
-                    {{ form.presale_start_day.label }}
-                </label>
-                <div class="col-md-5">
-                    {% bootstrap_field form.presale_start_day layout="bulkedit_inline" form_group_class="" %}
-                </div>
-                <div class="col-md-4">
-                    {% bootstrap_field form.presale_start_time layout="bulkedit_inline" form_group_class="" %}
-                </div>
-            </div>
-            <div class="form-group">
-                <label class="col-md-3 control-label" for="{{ form.presale_end_day.id_for_label }}">
-                    {{ form.presale_end_day.label }}
-                </label>
-                <div class="col-md-5">
-                    {% bootstrap_field form.presale_end_day layout="bulkedit_inline" form_group_class="" %}
-                </div>
-                <div class="col-md-4">
-                    {% bootstrap_field form.presale_end_time layout="bulkedit_inline" form_group_class="" %}
-                </div>
-            </div>
+            {% bootstrap_field form.date_admission_rel layout="bulkedit" %}
+            {% bootstrap_field form.presale_start_rel layout="bulkedit" %}
+            {% bootstrap_field form.presale_end_rel layout="bulkedit" %}
         </fieldset>
         <fieldset>
             <legend>{% trans "Item prices" %}</legend>

src/pretix/control/views/subevents.py

@@ -85,6 +85,7 @@ from pretix.control.views import PaginationMixin
 from pretix.control.views.event import MetaDataEditorMixin
 from pretix.helpers import GroupConcat
 from pretix.helpers.compat import CompatDeleteView
+from pretix.helpers.database import UnixTimestamp
 from pretix.helpers.i18n import get_format_without_seconds
 from pretix.helpers.models import modelcopy
 
@@ -1583,21 +1584,6 @@ class SubEventBulkEdit(SubEventQueryMixin, EventPermissionRequiredMixin, FormVie
         mixed_values = set()
         qs = self.get_queryset()
 
-        qs = qs.annotate(
-            **{
-                # TODO: Once we're on Django 3.2, pass a tzinfo parameter
-                # Before Django 3.2, it uses the current timezone, which is hopefully fine
-                # as well in all cases we are concerned about
-                # See also: https://code.djangoproject.com/ticket/31948
-                k + '_day': TruncDate(k)
-                for k in ('date_from', 'date_to', 'date_admission', 'presale_start', 'presale_end')
-            },
-            **{
-                k + '_time': TruncTime(k)
-                for k in ('date_from', 'date_to', 'date_admission', 'presale_start', 'presale_end')
-            },
-        )
-
         fields = {
             'name',
             'location',
@@ -1611,21 +1597,75 @@ class SubEventBulkEdit(SubEventQueryMixin, EventPermissionRequiredMixin, FormVie
             'date_from_time',
             'date_to_day',
             'date_to_time',
-            'date_admission_day',
-            'date_admission_time',
-            'presale_start_day',
-            'presale_start_time',
-            'presale_end_day',
-            'presale_end_time',
         }
+        rel_fields = {'date_admission', 'presale_start', 'presale_end'}
+
+        qs = qs.annotate(
+            **{
+                k + '_day': TruncDate(k, tzinfo=self.request.event.timezone)
+                for k in ('date_from', 'date_to', 'date_admission')
+            },
+            **{
+                k + '_time': TruncTime(k, tzinfo=self.request.event.timezone)
+                for k in ('date_from', 'date_to', 'date_admission')
+            },
+            **{
+                k + '_rel_min': (UnixTimestamp('date_from') - UnixTimestamp(k)) / 60
+                for k in rel_fields
+            },
+            **{
+                k + '_rel_days': TruncDate('date_from', tzinfo=self.request.event.timezone) - TruncDate(k, tzinfo=self.request.event.timezone)
+                for k in rel_fields
+            }
+        )
+
         for k in fields:
-            existing_values = list(qs.order_by(k).values(k).annotate(c=Count('*')))
+            existing_values = list(qs.order_by(k).values(k).distinct())
             if len(existing_values) == 1:
                 initial[k] = existing_values[0][k]
             elif len(existing_values) > 1:
                 mixed_values.add(k)
                 initial[k] = None
 
+        for k in rel_fields:
+            existing_fixed_values = list(qs.order_by(k).values(k).distinct())
+            existing_rel_days_values = list(qs.order_by(f'{k}_rel_days').values(f'{k}_rel_days').distinct())
+            existing_rel_min_values = list(qs.order_by(f'{k}_rel_min').values(f'{k}_rel_min').distinct())
+
+            if len(existing_fixed_values) == 1:
+                if existing_fixed_values[0][k]:
+                    initial[f"{k}_rel"] = RelativeDateWrapper(existing_fixed_values[0][k])
+                else:
+                    initial[f"{k}_rel"] = None
+
+            elif len(existing_rel_min_values) == 1:
+                if rel_min := existing_rel_min_values[0][f'{k}_rel_min']:
+                    initial[f"{k}_rel"] = RelativeDateWrapper(RelativeDate(
+                        days=0,
+                        base_date_name='event__date_from',
+                        time=None,
+                        minutes=abs(rel_min),
+                        is_after=rel_min < 0,
+                    ))
+                else:
+                    initial[f"{k}_rel"] = None
+
+            elif len(existing_rel_days_values) == 1:
+                if rel_days := existing_rel_days_values[0][f'{k}_rel_days']:
+                    initial[f"{k}_rel"] = RelativeDateWrapper(RelativeDate(
+                        days=abs(rel_days.days),
+                        base_date_name='event__date_from',
+                        time=None,
+                        minutes=None,
+                        is_after=rel_days.days < 0,
+                    ))
+                else:
+                    initial[f"{k}_rel"] = None
+
+            elif len(existing_fixed_values) > 1 or len(existing_rel_days_values) > 1 or len(existing_rel_min_values) > 1:
+                mixed_values.add(k)
+                initial[k] = None
+
         kwargs = super().get_form_kwargs()
         kwargs['event'] = self.request.event
         kwargs['prefix'] = 'bulkedit'

src/pretix/helpers/database.py

@@ -26,7 +26,8 @@ from django.contrib.postgres.indexes import BrinIndex
 from django.core.exceptions import FieldDoesNotExist, ImproperlyConfigured
 from django.db import connection, transaction
 from django.db.models import (
-    Aggregate, Expression, F, Field, JSONField, Lookup, OrderBy, Value,
+    Aggregate, Expression, F, Field, Func, IntegerField, JSONField, Lookup,
+    OrderBy, Value,
 )
 from django.utils.functional import lazy
 
@@ -313,3 +314,13 @@ class IgnoreOnSQLiteMixin:
 
 class BrinIndexIgnoredOnSQLite(IgnoreOnSQLiteMixin, BrinIndex):
     pass
+
+
+class UnixTimestamp(Func):
+    output_field = IntegerField()
+
+    def as_postgresql(self, compiler, connection):
+        return self.as_sql(compiler, connection, template="EXTRACT(epoch from %(expressions)s)")
+
+    def as_sqlite(self, compiler, connection):
+        return self.as_sql(compiler, connection, template="CAST(strftime('%%%%s', datetime(%(expressions)s)) AS INT)")

src/pretix/presale/views/organizer.py

@@ -46,7 +46,7 @@ import isoweek
 from django.conf import settings
 from django.core.cache import caches
 from django.db.models import (
-    Case, Exists, F, Max, Min, OuterRef, Prefetch, Q, Value, When,
+    Case, Exists, F, Max, Min, OuterRef, Prefetch, Q, Subquery, Value, When,
 )
 from django.db.models.functions import Coalesce, Greatest
 from django.dispatch.dispatcher import NO_RECEIVERS
@@ -189,22 +189,27 @@ class EventListMixin:
     def _get_event_list_queryset(self):
         query = Q(is_public=True) & Q(live=True)
         qs = self.request.organizer.events.using(settings.DATABASE_REPLICA).filter(query)
-        qs = qs.filter(Q(all_sales_channels=True) | Q(limit_sales_channels=self.request.sales_channel))
+        qs = qs.filter(Q(all_sales_channels=True) | Q(id__in=self.request.sales_channel.event_set.values_list("pk")))
 
         show_old = "old" in self.request.GET
 
-        subevent_filter = Q(subevents__active=True, subevents__is_public=True)
+        subevent_filter = Q(active=True, is_public=True)
         if not show_old:
             subevent_filter &= Q(
-                Q(subevents__date_to__gte=now()) | Q(subevents__date_from__gte=now())
+                Q(date_to__gte=now()) | Q(date_from__gte=now())
             )
 
+        subevent_subquery_qs = SubEvent.objects.with_scopes_disabled().filter(
+            subevent_filter,
+            event_id=OuterRef('pk')
+        ).values('event').order_by()
         qs = qs.annotate(
-            min_from=Min('subevents__date_from', filter=subevent_filter),
-            min_to=Min('subevents__date_to', filter=subevent_filter),
-            max_from=Max('subevents__date_from', filter=subevent_filter),
-            max_to=Max('subevents__date_to', filter=subevent_filter),
-            max_fromto=Greatest(Max('subevents__date_to', filter=subevent_filter), Max('subevents__date_from', filter=subevent_filter)),
+            min_from=Subquery(subevent_subquery_qs.annotate(m=Min('date_from')).values('m')),
+            min_to=Subquery(subevent_subquery_qs.annotate(m=Min('date_to')).values('m')),
+            max_from=Subquery(subevent_subquery_qs.annotate(m=Max('date_from')).values('m')),
+            max_to=Subquery(subevent_subquery_qs.annotate(m=Max('date_to')).values('m')),
+        ).annotate(
+            max_fromto=Greatest(F("max_to"), F("max_from")),
         )
         if show_old:
             date_q = Q(date_to__lt=now()) | (Q(date_to__isnull=True) & Q(date_from__lt=now()))

src/tests/api/test_order_change.py

@@ -26,6 +26,7 @@ from unittest import mock
 
 import pytest
 from django.core import mail as djmail
+from django.core.exceptions import ValidationError
 from django.core.files.base import ContentFile
 from django.utils.timezone import now
 from django_countries.fields import Country
@@ -248,7 +249,14 @@ def test_order_update_state_validation(token_client, organizer, event, order):
 
 
 @pytest.mark.django_db
-def test_order_update_transmission_validation(token_client, organizer, event, order):
+def test_order_update_transmission_validation(token_client, organizer, event, order, monkeypatch):
+    def mock_validate(self, value):
+        if value == "9930:DE111111111":
+            return value
+        raise ValidationError("Invalid")
+
+    monkeypatch.setattr("pretix.base.invoicing.peppol.PeppolIdValidator._validate_peppol_id_online", mock_validate)
+
     resp = token_client.patch(
         '/api/v1/organizers/{}/events/{}/orders/{}/'.format(
             organizer.slug, event.slug, order.code
@@ -374,7 +382,7 @@ def test_order_update_transmission_validation(token_client, organizer, event, or
                 "vat_id": "",
                 "transmission_type": "peppol",
                 "transmission_info": {
-                    "transmission_peppol_participant_id": "9930:DE811569869",
+                    "transmission_peppol_participant_id": "9930:DE111111111",
                     "ignored": "parameter",
                 },
             }
@@ -384,7 +392,7 @@ def test_order_update_transmission_validation(token_client, organizer, event, or
     order.invoice_address.refresh_from_db()
     assert order.invoice_address.transmission_type == "peppol"
     assert order.invoice_address.transmission_info == {
-        "transmission_peppol_participant_id": "9930:DE811569869",
+        "transmission_peppol_participant_id": "9930:DE111111111",
     }
 
 

src/tests/control/test_subevents.py

@@ -1107,6 +1107,48 @@ class SubEventsTest(SoupTest):
             assert self.subevent1.date_to == datetime.datetime(2013, 12, 26, 17, 0, 0, tzinfo=datetime.timezone.utc)
             assert self.subevent2.date_to == datetime.datetime(2013, 12, 27, 17, 0, 0, tzinfo=datetime.timezone.utc)
 
+    def test_edit_bulk_rel(self):
+        with scopes_disabled():
+            self.subevent1.date_from = datetime.datetime(2013, 12, 26, 9, 0, 0, tzinfo=datetime.timezone.utc)
+            self.subevent1.presale_start = datetime.datetime(2013, 12, 16, 0, 0, 0, tzinfo=datetime.timezone.utc)
+            self.subevent1.presale_end = datetime.datetime(2013, 12, 25, 9, 0, 0, tzinfo=datetime.timezone.utc)
+            self.subevent1.save()
+            self.subevent2.date_from = datetime.datetime(2013, 10, 27, 11, 0, 0, tzinfo=datetime.timezone.utc)  # DST change is on 2013-10-27
+            self.subevent2.presale_start = datetime.datetime(2013, 10, 17, 0, 0, 0, tzinfo=datetime.timezone.utc)
+            self.subevent2.presale_end = datetime.datetime(2013, 10, 26, 11, 0, 0, tzinfo=datetime.timezone.utc)
+            self.subevent2.save()
+
+        doc = self.post_doc('/control/event/ccc/30c3/subevents/bulk_edit', {
+            '__ALL': 'on',
+        }, follow=True)
+        fields = extract_form_fields(doc)
+        expected = {
+            'bulkedit-presale_start_rel_0': 'relative',
+            'bulkedit-presale_start_rel_2': '10',
+            'bulkedit-presale_start_rel_6': 'event__date_from',
+            'bulkedit-presale_start_rel_8': 'before',
+            'bulkedit-presale_end_rel_0': 'relative_minutes',
+            'bulkedit-presale_end_rel_3': 'event__date_from',
+            'bulkedit-presale_end_rel_5': '1440',
+            'bulkedit-presale_end_rel_7': 'before',
+        }
+        for k, v in expected.items():
+            assert k in fields
+            assert fields[k] == v
+
+        assert fields.get('bulkedit-presale_start_rel_4') != '10:00'
+        fields.update({
+            '_bulk': ['bulkeditpresale_start_rel'],
+            'bulkedit-presale_start_rel_4': '10:00',
+        })
+        doc = self.post_doc('/control/event/ccc/30c3/subevents/bulk_edit', fields, follow=True)
+        assert doc.select(".alert-success")
+        with scopes_disabled():
+            self.subevent1.refresh_from_db()
+            self.subevent2.refresh_from_db()
+            assert self.subevent1.presale_start == datetime.datetime(2013, 12, 16, 10, 0, 0, tzinfo=datetime.timezone.utc)
+            assert self.subevent2.presale_start == datetime.datetime(2013, 10, 17, 10, 0, 0, tzinfo=datetime.timezone.utc)
+
     def test_edit_bulk_price(self):
         sei1 = SubEventItem.objects.create(subevent=self.subevent1, item=self.ticket, price=Decimal('4.00'))
         sei2 = SubEventItem.objects.create(subevent=self.subevent2, item=self.ticket, price=Decimal('4.00'))