refactoring

Currently translated at 73.7% (3982 of 5400 strings)

Translation: pretix/pretix
Translate-URL: https://translate.pretix.eu/projects/pretix/pretix/nl_Informal/

powered by weblate

doc/api/resources/sendmail_rules.rst

@@ -23,10 +23,14 @@ limit_products                        list of integers           List of product
 restrict_to_status                    list                       List of order states to restrict recipients to. Valid
                                                                  entries are ``p`` for paid, ``e`` for expired, ``c`` for canceled,
                                                                  ``n__pending_approval`` for pending approval,
-                                                                 ``n__not_pending_approval_and_not_valid_if_pending`` for payment pending,
-                                                                 ``n__valid_if_pending`` for payment pending but already confirmed,
+                                                                 ``n__not_pending_approval_and_not_valid_if_pending`` for payment
+                                                                 pending, ``n__valid_if_pending`` for payment pending but already confirmed,
                                                                  and ``n__pending_overdue`` for pending with payment overdue.
                                                                  The default is ``["p", "n__valid_if_pending"]``.
+checked_in_status                     string                     Check-in status to restrict recipients to. Valid strings are:
+                                                                 ``null`` for no filtering (default), ``checked_in`` for
+                                                                 limiting to attendees that are or have been checked in, and
+                                                                 ``no_checkin`` for limiting to attendees who have not checked in.
 date_is_absolute                      boolean                    If ``true``, the email is set at a specific point in time.
 send_date                             datetime                   If ``date_is_absolute`` is set: Date and time to send the email.
 send_offset_days                      integer                    If ``date_is_absolute`` is not set, this is the number of days
@@ -89,6 +93,7 @@ Endpoints
                 "n__not_pending_approval_and_not_valid_if_pending",
                 "n__valid_if_pending"
             ],
+            "checked_in_status": null,
             "send_date": null,
             "send_offset_days": 1,
             "send_offset_time": "18:00",
@@ -139,6 +144,7 @@ Endpoints
             "n__not_pending_approval_and_not_valid_if_pending",
             "n__valid_if_pending"
         ],
+        "checked_in_status": null,
         "send_date": null,
         "send_offset_days": 1,
         "send_offset_time": "18:00",
@@ -180,6 +186,7 @@ Endpoints
             "n__not_pending_approval_and_not_valid_if_pending",
             "n__valid_if_pending"
         ],
+        "checked_in_status": "checked_in",
         "send_date": null,
         "send_offset_days": 1,
         "send_offset_time": "18:00",
@@ -209,6 +216,7 @@ Endpoints
             "n__not_pending_approval_and_not_valid_if_pending",
             "n__valid_if_pending"
         ],
+        "checked_in_status": "checked_in",
         "send_date": null,
         "send_offset_days": 1,
         "send_offset_time": "18:00",
@@ -266,6 +274,7 @@ Endpoints
             "n__not_pending_approval_and_not_valid_if_pending",
             "n__valid_if_pending"
         ],
+        "checked_in_status": "checked_in",
         "send_date": null,
         "send_offset_days": 1,
         "send_offset_time": "18:00",

doc/development/setup.rst

@@ -96,6 +96,20 @@ http://localhost:8000/control/ for the admin view.
           port (for example because you develop on `pretixdroid`_), you can check
           `Django's documentation`_ for more options.
 
+When running the local development webserver, ensure Celery is not configured
+in ``pretix.cfg``. i.e., you should remove anything such as::
+
+    [celery]
+    backend=redis://redis:6379/2
+    broker=redis://redis:6379/2
+
+If you choose to use Celery for development, you must also start a Celery worker
+process::
+
+    celery -A pretix.celery_app worker -l info
+
+However, beware that code changes will not auto-reload within Celery.
+
 .. _`checksandtests`:
 
 Code checks and unit tests

pyproject.toml

@@ -90,7 +90,7 @@ dependencies = [
   "pytz-deprecation-shim==0.1.*",
   "pyuca",
   "qrcode==7.4.*",
-  "redis==4.5.*,>=4.5.4",
+  "redis==4.6.*",
   "reportlab==4.0.*",
   "requests==2.31.*",
   "sentry-sdk==1.15.*",
@@ -112,6 +112,7 @@ memcached = ["pylibmc"]
 dev = [
   "coverage",
   "coveralls",
+  "fakeredis==2.18.*",
   "flake8==6.0.*",
   "freezegun",
   "isort==5.12.*",

src/pretix/api/serializers/order.py

@@ -27,6 +27,7 @@ from decimal import Decimal
 import pycountry
 from django.conf import settings
 from django.core.files import File
+from django.db import models
 from django.db.models import F, Q
 from django.utils.encoding import force_str
 from django.utils.timezone import now
@@ -372,11 +373,15 @@ class PdfDataSerializer(serializers.Field):
                 self.context['vars_images'] = get_images(self.context['event'])
 
             for k, f in self.context['vars'].items():
-                try:
-                    res[k] = f['evaluate'](instance, instance.order, ev)
-                except:
-                    logger.exception('Evaluating PDF variable failed')
-                    res[k] = '(error)'
+                if 'evaluate_bulk' in f:
+                    # Will be evaluated later by our list serializers
+                    res[k] = (f['evaluate_bulk'], instance)
+                else:
+                    try:
+                        res[k] = f['evaluate'](instance, instance.order, ev)
+                    except:
+                        logger.exception('Evaluating PDF variable failed')
+                        res[k] = '(error)'
 
             if not hasattr(ev, '_cached_meta_data'):
                 ev._cached_meta_data = ev.meta_data
@@ -429,6 +434,38 @@ class PdfDataSerializer(serializers.Field):
             return res
 
 
+class OrderPositionListSerializer(serializers.ListSerializer):
+
+    def to_representation(self, data):
+        # We have a custom implementation of this method because PdfDataSerializer() might keep some elements unevaluated
+        # with a (callable, input) tuple. We'll loop over these entries and evaluate them bulk-wise to save on SQL queries.
+
+        if isinstance(self.parent, OrderSerializer) and isinstance(self.parent.parent, OrderListSerializer):
+            # Do not execute our custom code because it will be executed by OrderListSerializer later for the
+            # full result set.
+            return super().to_representation(data)
+
+        iterable = data.all() if isinstance(data, models.Manager) else data
+
+        data = []
+        evaluate_queue = defaultdict(list)
+
+        for item in iterable:
+            entry = self.child.to_representation(item)
+            if "pdf_data" in entry:
+                for k, v in entry["pdf_data"].items():
+                    if isinstance(v, tuple) and callable(v[0]):
+                        evaluate_queue[v[0]].append((v[1], entry, k))
+            data.append(entry)
+
+        for func, entries in evaluate_queue.items():
+            results = func([item for (item, entry, k) in entries])
+            for (item, entry, k), result in zip(entries, results):
+                entry["pdf_data"][k] = result
+
+        return data
+
+
 class OrderPositionSerializer(I18nAwareModelSerializer):
     checkins = CheckinSerializer(many=True, read_only=True)
     answers = AnswerSerializer(many=True)
@@ -440,6 +477,7 @@ class OrderPositionSerializer(I18nAwareModelSerializer):
     attendee_name = serializers.CharField(required=False)
 
     class Meta:
+        list_serializer_class = OrderPositionListSerializer
         model = OrderPosition
         fields = ('id', 'order', 'positionid', 'item', 'variation', 'price', 'attendee_name', 'attendee_name_parts',
                   'company', 'street', 'zipcode', 'city', 'country', 'state', 'discount',
@@ -468,6 +506,20 @@ class OrderPositionSerializer(I18nAwareModelSerializer):
     def validate(self, data):
         raise TypeError("this serializer is readonly")
 
+    def to_representation(self, data):
+        if isinstance(self.parent, (OrderListSerializer, OrderPositionListSerializer)):
+            # Do not execute our custom code because it will be executed by OrderListSerializer later for the
+            # full result set.
+            return super().to_representation(data)
+
+        entry = super().to_representation(data)
+        if "pdf_data" in entry:
+            for k, v in entry["pdf_data"].items():
+                if isinstance(v, tuple) and callable(v[0]):
+                    entry["pdf_data"][k] = v[0]([v[1]])[0]
+
+        return entry
+
 
 class RequireAttentionField(serializers.Field):
     def to_representation(self, instance: OrderPosition):
@@ -613,6 +665,34 @@ class OrderURLField(serializers.URLField):
         })
 
 
+class OrderListSerializer(serializers.ListSerializer):
+
+    def to_representation(self, data):
+        # We have a custom implementation of this method because PdfDataSerializer() might keep some elements
+        # unevaluated with a (callable, input) tuple. We'll loop over these entries and evaluate them bulk-wise to
+        # save on SQL queries.
+        iterable = data.all() if isinstance(data, models.Manager) else data
+
+        data = []
+        evaluate_queue = defaultdict(list)
+
+        for item in iterable:
+            entry = self.child.to_representation(item)
+            for p in entry.get("positions", []):
+                if "pdf_data" in p:
+                    for k, v in p["pdf_data"].items():
+                        if isinstance(v, tuple) and callable(v[0]):
+                            evaluate_queue[v[0]].append((v[1], p, k))
+            data.append(entry)
+
+        for func, entries in evaluate_queue.items():
+            results = func([item for (item, entry, k) in entries])
+            for (item, entry, k), result in zip(entries, results):
+                entry["pdf_data"][k] = result
+
+        return data
+
+
 class OrderSerializer(I18nAwareModelSerializer):
     invoice_address = InvoiceAddressSerializer(allow_null=True)
     positions = OrderPositionSerializer(many=True, read_only=True)
@@ -627,6 +707,7 @@ class OrderSerializer(I18nAwareModelSerializer):
 
     class Meta:
         model = Order
+        list_serializer_class = OrderListSerializer
         fields = (
             'code', 'status', 'testmode', 'secret', 'email', 'phone', 'locale', 'datetime', 'expires', 'payment_date',
             'payment_provider', 'fees', 'total', 'comment', 'custom_followup_at', 'invoice_address', 'positions', 'downloads',

src/pretix/api/views/event.py

@@ -415,6 +415,7 @@ class SubEventViewSet(ConditionalListView, viewsets.ModelViewSet):
             'subeventitem_set',
             'subeventitemvariation_set',
             'meta_values',
+            'meta_values__property',
             Prefetch(
                 'seat_category_mappings',
                 to_attr='_seat_category_mappings',

src/pretix/base/cache.py

@@ -62,27 +62,27 @@ class NamespacedCache:
             prefix = int(time.time())
             self.cache.set(self.prefixkey, prefix)
 
-    def set(self, key: str, value: str, timeout: int=300):
+    def set(self, key: str, value: any, timeout: int=300):
         return self.cache.set(self._prefix_key(key), value, timeout)
 
-    def get(self, key: str) -> str:
+    def get(self, key: str) -> any:
         return self.cache.get(self._prefix_key(key, known_prefix=self._last_prefix))
 
-    def get_or_set(self, key: str, default: Callable, timeout=300) -> str:
+    def get_or_set(self, key: str, default: Callable, timeout=300) -> any:
         return self.cache.get_or_set(
             self._prefix_key(key, known_prefix=self._last_prefix),
             default=default,
             timeout=timeout
         )
 
-    def get_many(self, keys: List[str]) -> Dict[str, str]:
+    def get_many(self, keys: List[str]) -> Dict[str, any]:
         values = self.cache.get_many([self._prefix_key(key) for key in keys])
         newvalues = {}
         for k, v in values.items():
             newvalues[self._strip_prefix(k)] = v
         return newvalues
 
-    def set_many(self, values: Dict[str, str], timeout=300):
+    def set_many(self, values: Dict[str, any], timeout=300):
         newvalues = {}
         for k, v in values.items():
             newvalues[self._prefix_key(k)] = v

src/pretix/base/exporters/orderlist.py

@@ -549,7 +549,9 @@ class OrderListExporter(MultiSheetListExporter):
             headers.append(_('End date'))
         headers += [
             _('Product'),
+            _('Product ID'),
             _('Variation'),
+            _('Variation ID'),
             _('Price'),
             _('Tax rate'),
             _('Tax rule'),
@@ -656,7 +658,9 @@ class OrderListExporter(MultiSheetListExporter):
                         row.append('')
                 row += [
                     str(op.item),
+                    str(op.item_id),
                     str(op.variation) if op.variation else '',
+                    str(op.variation_id) if op.variation_id else '',
                     op.price,
                     op.tax_rate,
                     str(op.tax_rule) if op.tax_rule else '',

src/pretix/base/models/items.py

@@ -43,6 +43,7 @@ from typing import Optional, Tuple
 from zoneinfo import ZoneInfo
 
 import dateutil.parser
+import django_redis
 from dateutil.tz import datetime_exists
 from django.conf import settings
 from django.core.exceptions import ValidationError
@@ -57,7 +58,6 @@ from django.utils.functional import cached_property
 from django.utils.timezone import is_naive, make_aware, now
 from django.utils.translation import gettext_lazy as _, pgettext_lazy
 from django_countries.fields import Country
-from django_redis import get_redis_connection
 from django_scopes import ScopedManager
 from i18nfield.fields import I18nCharField, I18nTextField
 
@@ -1910,8 +1910,13 @@ class Quota(LoggedModel):
 
     def rebuild_cache(self, now_dt=None):
         if settings.HAS_REDIS:
-            rc = get_redis_connection("redis")
-            rc.hdel(f'quotas:{self.event_id}:availabilitycache', str(self.pk))
+            rc = django_redis.get_redis_connection("redis")
+            p = rc.pipeline()
+            p.hdel(f'quotas:{self.event_id}:availabilitycache', str(self.pk))
+            p.hdel(f'quotas:{self.event_id}:availabilitycache:nocw', str(self.pk))
+            p.hdel(f'quotas:{self.event_id}:availabilitycache:igcl', str(self.pk))
+            p.hdel(f'quotas:{self.event_id}:availabilitycache:nocw:igcl', str(self.pk))
+            p.execute()
             self.availability(now_dt=now_dt)
 
     def availability(

src/pretix/base/models/tax.py

@@ -340,10 +340,17 @@ class TaxRule(LoggedModel):
         rules = self._custom_rules
         if invoice_address:
             for r in rules:
-                if r['country'] == 'EU' and not is_eu_country(invoice_address.country):
-                    continue
-                if r['country'] not in ('ZZ', 'EU') and r['country'] != str(invoice_address.country):
-                    continue
+                if r['country'] == 'ZZ':    # Rule: Any country
+                    pass
+                elif r['country'] == 'EU':  # Rule: Any EU country
+                    if not is_eu_country(invoice_address.country):
+                        continue
+                elif '-' in r['country']:   # Rule: Specific country and state
+                    if r['country'] != str(invoice_address.country) + '-' + str(invoice_address.state):
+                        continue
+                else:                       # Rule: Specific country
+                    if r['country'] != str(invoice_address.country):
+                        continue
                 if r['address_type'] == 'individual' and invoice_address.is_business:
                     continue
                 if r['address_type'] in ('business', 'business_vat_id') and not invoice_address.is_business:

src/pretix/base/payment.py

@@ -336,6 +336,12 @@ class BasePaymentProvider:
                  help_text=_('Users will not be able to choose this payment provider after the given date.'),
                  required=False,
              )),
+            ('_availability_start',
+             RelativeDateField(
+                 label=_('Available from'),
+                 help_text=_('Users will not be able to choose this payment provider before the given date.'),
+                 required=False,
+             )),
             ('_total_min',
              forms.DecimalField(
                  label=_('Minimum order total'),
@@ -539,40 +545,57 @@ class BasePaymentProvider:
 
         return form
 
-    def _is_still_available(self, now_dt=None, cart_id=None, order=None):
+    def _convert_availability_date_to_absolute(self, rel_date, cart_id=None, order=None):
+        if not rel_date:
+            return None
+        # In an event series, we use min() here, which makes it less restrictive than max() and thus makes
+        # it harder to put one self into a situation where no payment provider is available.
+        if self.event.has_subevents and cart_id:
+            dates = [
+                rel_date.datetime(se).date()
+                for se in self.event.subevents.filter(
+                    id__in=CartPosition.objects.filter(
+                        cart_id=cart_id, event=self.event
+                    ).values_list('subevent', flat=True)
+                )
+            ]
+            return min(dates) if dates else None
+        elif self.event.has_subevents and order:
+            dates = [
+                rel_date.datetime(se).date()
+                for se in self.event.subevents.filter(
+                    id__in=order.positions.values_list('subevent', flat=True)
+                )
+            ]
+            return min(dates) if dates else None
+        elif self.event.has_subevents:
+            raise NotImplementedError('Payment provider is not subevent-ready.')
+        else:
+            return rel_date.datetime(self.event).date()
+
+    def _is_available_by_time(self, now_dt=None, cart_id=None, order=None):
         now_dt = now_dt or now()
         tz = ZoneInfo(self.event.settings.timezone)
 
-        availability_date = self.settings.get('_availability_date', as_type=RelativeDateWrapper)
-        if availability_date:
-            if self.event.has_subevents and cart_id:
-                dates = [
-                    availability_date.datetime(se).date()
-                    for se in self.event.subevents.filter(
-                        id__in=CartPosition.objects.filter(
-                            cart_id=cart_id, event=self.event
-                        ).values_list('subevent', flat=True)
-                    )
-                ]
-                availability_date = min(dates) if dates else None
-            elif self.event.has_subevents and order:
-                dates = [
-                    availability_date.datetime(se).date()
-                    for se in self.event.subevents.filter(
-                        id__in=order.positions.values_list('subevent', flat=True)
-                    )
-                ]
-                availability_date = min(dates) if dates else None
-            elif self.event.has_subevents:
-                logger.error('Payment provider is not subevent-ready.')
-                return False
-            else:
-                availability_date = availability_date.datetime(self.event).date()
+        try:
+            availability_start = self._convert_availability_date_to_absolute(
+                self.settings.get('_availability_start', as_type=RelativeDateWrapper), cart_id, order)
 
-            if availability_date:
-                return availability_date >= now_dt.astimezone(tz).date()
+            if availability_start:
+                if availability_start > now_dt.astimezone(tz).date():
+                    return False
 
-        return True
+            availability_end = self._convert_availability_date_to_absolute(
+                self.settings.get('_availability_date', as_type=RelativeDateWrapper), cart_id, order)
+
+            if availability_end:
+                if availability_end < now_dt.astimezone(tz).date():
+                    return False
+
+            return True
+        except NotImplementedError:
+            logger.exception('Unable to check availability')
+            return False
 
     def is_allowed(self, request: HttpRequest, total: Decimal=None) -> bool:
         """
@@ -581,9 +604,9 @@ class BasePaymentProvider:
         user will not be able to select this payment method. This will only be called
         during checkout, not on retrying.
 
-        The default implementation checks for the _availability_date setting to be either unset or in the future
-        and for the _total_max and _total_min requirements to be met. It also checks the ``_restrict_countries``
-        and ``_restrict_to_sales_channels`` setting.
+        The default implementation checks for the ``_availability_date`` setting to be either unset or in the future
+        and for the ``_availability_from``, ``_total_max``, and ``_total_min`` requirements to be met. It also checks
+        the ``_restrict_countries`` and ``_restrict_to_sales_channels`` setting.
 
         :param total: The total value without the payment method fee, after taxes.
 
@@ -592,7 +615,7 @@ class BasePaymentProvider:
            The ``total`` parameter has been added. For backwards compatibility, this method is called again
            without this parameter if it raises a ``TypeError`` on first try.
         """
-        timing = self._is_still_available(cart_id=get_or_create_cart_id(request))
+        timing = self._is_available_by_time(cart_id=get_or_create_cart_id(request))
         pricing = True
 
         if (self.settings._total_max is not None or self.settings._total_min is not None) and total is None:
@@ -776,8 +799,8 @@ class BasePaymentProvider:
         Will be called to check whether it is allowed to change the payment method of
         an order to this one.
 
-        The default implementation checks for the _availability_date setting to be either unset or in the future,
-        as well as for the _total_max, _total_min and _restricted_countries settings.
+        The default implementation checks for the ``_availability_date`` setting to be either unset or in the future,
+        as well as for the ``_availabilty_from``, ``_total_max``, ``_total_min``, and ``_restricted_countries`` settings.
 
         :param order: The order object
         """
@@ -804,7 +827,7 @@ class BasePaymentProvider:
         if order.sales_channel not in self.settings.get('_restrict_to_sales_channels', as_type=list, default=['web']):
             return False
 
-        return self._is_still_available(order=order)
+        return self._is_available_by_time(order=order)
 
     def payment_prepare(self, request: HttpRequest, payment: OrderPayment) -> Union[bool, str]:
         """

src/pretix/base/pdf.py

@@ -108,7 +108,10 @@ DEFAULT_VARIABLES = OrderedDict((
     ("positionid", {
         "label": _("Order position number"),
         "editor_sample": "1",
-        "evaluate": lambda orderposition, order, event: str(orderposition.positionid)
+        "evaluate": lambda orderposition, order, event: str(orderposition.positionid),
+        # There is no performance gain in using evaluate_bulk here, but we want to make sure it is used somewhere
+        # in core to make sure we notice if the implementation of the API breaks.
+        "evaluate_bulk": lambda orderpositions: [str(p.positionid) for p in orderpositions],
     }),
     ("order_positionid", {
         "label": _("Order code and position number"),
@@ -699,10 +702,10 @@ def get_seat(op: OrderPosition):
 
 def generate_compressed_addon_list(op, order, event):
     itemcount = defaultdict(int)
-    addons = (
+    addons = [p for p in (
         op.addons.all() if 'addons' in getattr(op, '_prefetched_objects_cache', {})
         else op.addons.select_related('item', 'variation')
-    )
+    ) if not p.canceled]
     for pos in addons:
         itemcount[pos.item, pos.variation] += 1
 

src/pretix/base/services/cart.py

@@ -1078,6 +1078,7 @@ class CartManager:
         quotas_ok = _get_quota_availability(self._quota_diff, self.now_dt)
         err = None
         new_cart_positions = []
+        deleted_positions = set()
 
         err = err or self._check_min_max_per_product()
 
@@ -1089,7 +1090,10 @@ class CartManager:
                 if op.position.expires > self.now_dt:
                     for q in op.position.quotas:
                         quotas_ok[q] += 1
-                op.position.addons.all().delete()
+                addons = op.position.addons.all()
+                deleted_positions |= {a.pk for a in addons}
+                addons.delete()
+                deleted_positions.add(op.position.pk)
                 op.position.delete()
 
             elif isinstance(op, (self.AddOperation, self.ExtendOperation)):
@@ -1239,20 +1243,28 @@ class CartManager:
                     if op.seat and not op.seat.is_available(ignore_cart=op.position, sales_channel=self._sales_channel,
                                                             ignore_voucher_id=op.position.voucher_id):
                         err = err or error_messages['seat_unavailable']
-                        op.position.addons.all().delete()
+
+                        addons = op.position.addons.all()
+                        deleted_positions |= {a.pk for a in addons}
+                        deleted_positions.add(op.position.pk)
+                        addons.delete()
                         op.position.delete()
                     elif available_count == 1:
                         op.position.expires = self._expiry
                         op.position.listed_price = op.listed_price
                         op.position.price_after_voucher = op.price_after_voucher
                         # op.position.price will be updated by recompute_final_prices_and_taxes()
-                        try:
-                            op.position.save(force_update=True, update_fields=['expires', 'listed_price', 'price_after_voucher'])
-                        except DatabaseError:
-                            # Best effort... The position might have been deleted in the meantime!
-                            pass
+                        if op.position.pk not in deleted_positions:
+                            try:
+                                op.position.save(force_update=True, update_fields=['expires', 'listed_price', 'price_after_voucher'])
+                            except DatabaseError:
+                                # Best effort... The position might have been deleted in the meantime!
+                                pass
                     elif available_count == 0:
-                        op.position.addons.all().delete()
+                        addons = op.position.addons.all()
+                        deleted_positions |= {a.pk for a in addons}
+                        deleted_positions.add(op.position.pk)
+                        addons.delete()
                         op.position.delete()
                     else:
                         raise AssertionError("ExtendOperation cannot affect more than one item")

src/pretix/base/services/quotas.py

@@ -24,13 +24,13 @@ import time
 from collections import Counter, defaultdict
 from itertools import zip_longest
 
+import django_redis
 from django.conf import settings
 from django.db import models
 from django.db.models import (
     Case, Count, F, Func, Max, OuterRef, Q, Subquery, Sum, Value, When,
 )
 from django.utils.timezone import now
-from django_redis import get_redis_connection
 
 from pretix.base.models import (
     CartPosition, Checkin, Order, OrderPosition, Quota, Voucher,
@@ -102,6 +102,12 @@ class QuotaAvailability:
         self.count_waitinglist = defaultdict(int)
         self.count_cart = defaultdict(int)
 
+        self._cache_key_suffix = ""
+        if not self._count_waitinglist:
+            self._cache_key_suffix += ":nocw"
+        if self._ignore_closed:
+            self._cache_key_suffix += ":igcl"
+
         self.sizes = {}
 
     def queue(self, *quota):
@@ -121,17 +127,14 @@ class QuotaAvailability:
             if self._full_results:
                 raise ValueError("You cannot combine full_results and allow_cache.")
 
-            elif not self._count_waitinglist:
-                raise ValueError("If you set allow_cache, you need to set count_waitinglist.")
-
             elif settings.HAS_REDIS:
-                rc = get_redis_connection("redis")
+                rc = django_redis.get_redis_connection("redis")
                 quotas_by_event = defaultdict(list)
                 for q in [_q for _q in self._queue if _q.id in quota_ids_set]:
                     quotas_by_event[q.event_id].append(q)
 
                 for eventid, evquotas in quotas_by_event.items():
-                    d = rc.hmget(f'quotas:{eventid}:availabilitycache', [str(q.pk) for q in evquotas])
+                    d = rc.hmget(f'quotas:{eventid}:availabilitycache{self._cache_key_suffix}', [str(q.pk) for q in evquotas])
                     for redisval, q in zip(d, evquotas):
                         if redisval is not None:
                             data = [rv for rv in redisval.decode().split(',')]
@@ -164,12 +167,12 @@ class QuotaAvailability:
         if not settings.HAS_REDIS or not quotas:
             return
 
-        rc = get_redis_connection("redis")
+        rc = django_redis.get_redis_connection("redis")
         # We write the computed availability to redis in a per-event hash as
         #
         #   quota_id -> (availability_state, availability_number, timestamp).
         #
-        # We store this in a hash instead of inidividual values to avoid making two many redis requests
+        # We store this in a hash instead of individual values to avoid making too many redis requests
         # which would introduce latency.
 
         # The individual entries in the hash are "valid" for 120 seconds. This means in a typical peak scenario with
@@ -179,16 +182,16 @@ class QuotaAvailability:
         # these quotas. We choose 10 seconds since that should be well above the duration of a write.
 
         lock_name = '_'.join([str(p) for p in sorted([q.pk for q in quotas])])
-        if rc.exists(f'quotas:availabilitycachewrite:{lock_name}'):
+        if rc.exists(f'quotas:availabilitycachewrite:{lock_name}{self._cache_key_suffix}'):
             return
-        rc.setex(f'quotas:availabilitycachewrite:{lock_name}', '1', 10)
+        rc.setex(f'quotas:availabilitycachewrite:{lock_name}{self._cache_key_suffix}', '1', 10)
 
         update = defaultdict(list)
         for q in quotas:
             update[q.event_id].append(q)
 
         for eventid, quotas in update.items():
-            rc.hmset(f'quotas:{eventid}:availabilitycache', {
+            rc.hmset(f'quotas:{eventid}:availabilitycache{self._cache_key_suffix}', {
                 str(q.id): ",".join(
                     [str(i) for i in self.results[q]] +
                     [str(int(time.time()))]
@@ -197,7 +200,7 @@ class QuotaAvailability:
             # To make sure old events do not fill up our redis instance, we set an expiry on the cache. However, we set it
             # on 7 days even though we mostly ignore values older than 2 monites. The reasoning is that we have some places
             # where we set allow_cache_stale and use the old entries anyways to save on performance.
-            rc.expire(f'quotas:{eventid}:availabilitycache', 3600 * 24 * 7)
+            rc.expire(f'quotas:{eventid}:availabilitycache{self._cache_key_suffix}', 3600 * 24 * 7)
 
         # We used to also delete item_quota_cache:* from the event cache here, but as the cache
         # gets more complex, this does not seem worth it. The cache is only present for up to

src/pretix/base/services/waitinglist.py

@@ -22,13 +22,15 @@
 import sys
 from datetime import timedelta
 
-from django.db.models import Exists, F, OuterRef, Q, Sum
+from django.db.models import (
+    Exists, F, OuterRef, Prefetch, Q, Sum, prefetch_related_objects,
+)
 from django.dispatch import receiver
 from django.utils.timezone import now
 from django_scopes import scopes_disabled
 
 from pretix.base.models import (
-    Event, SeatCategoryMapping, User, WaitingListEntry,
+    Event, EventMetaValue, SeatCategoryMapping, User, WaitingListEntry,
 )
 from pretix.base.models.waitinglist import WaitingListException
 from pretix.base.services.tasks import EventTask
@@ -59,8 +61,21 @@ def assign_automatically(event: Event, user_id: int=None, subevent_id: int=None)
         ).aggregate(free=Sum(F('max_usages') - F('redeemed')))['free'] or 0
         seats_available[(m.product_id, m.subevent_id)] = num_free_seets_for_product - num_valid_vouchers_for_product
 
-    qs = WaitingListEntry.objects.filter(
-        event=event, voucher__isnull=True
+    prefetch_related_objects(
+        [event.organizer],
+        'meta_properties'
+    )
+    prefetch_related_objects(
+        [event],
+        Prefetch(
+            'meta_values',
+            EventMetaValue.objects.select_related('property'),
+            to_attr='meta_values_cached'
+        )
+    )
+
+    qs = event.waitinglistentries.filter(
+        voucher__isnull=True
     ).select_related('item', 'variation', 'subevent').prefetch_related(
         'item__quotas', 'variation__quotas'
     ).order_by('-priority', 'created')

src/pretix/base/shredder.py

@@ -210,6 +210,8 @@ def slow_delete(qs, batch_size=1000, sleep_time=.5, progress_callback=None, prog
             break
         if total_deleted >= 0.8 * batch_size:
             time.sleep(sleep_time)
+        if progress_callback and progress_total:
+            progress_callback((progress_offset + total_deleted) / progress_total)
     return total_deleted
 
 

src/pretix/base/signals.py

@@ -683,12 +683,16 @@ dictionaries as values that contain keys like in the following example::
         "product": {
             "label": _("Product name"),
             "editor_sample": _("Sample product"),
-            "evaluate": lambda orderposition, order, event: str(orderposition.item)
+            "evaluate": lambda orderposition, order, event: str(orderposition.item),
+            "evaluate_bulk": lambda orderpositions: [str(op.item) for op in orderpositions],
         }
     }
 
 The ``evaluate`` member will be called with the order position, order and event as arguments. The event might
 also be a subevent, if applicable.
+
+The ``evaluate_bulk`` member is optional but can significantly improve performance in some situations because you
+can perform database fetches in bulk instead of single queries for every position.
 """
 
 

src/pretix/control/forms/event.py

@@ -38,6 +38,7 @@ from decimal import Decimal
 from urllib.parse import urlencode, urlparse
 from zoneinfo import ZoneInfo
 
+import pycountry
 from django import forms
 from django.conf import settings
 from django.core.exceptions import NON_FIELD_ERRORS, ValidationError
@@ -65,7 +66,8 @@ from pretix.base.models import Event, Organizer, TaxRule, Team
 from pretix.base.models.event import EventFooterLink, EventMetaValue, SubEvent
 from pretix.base.reldate import RelativeDateField, RelativeDateTimeField
 from pretix.base.settings import (
-    PERSON_NAME_SCHEMES, PERSON_NAME_TITLE_GROUPS, validate_event_settings,
+    COUNTRIES_WITH_STATE_IN_ADDRESS, PERSON_NAME_SCHEMES,
+    PERSON_NAME_TITLE_GROUPS, validate_event_settings,
 )
 from pretix.base.validators import multimail_validate
 from pretix.control.forms import (
@@ -1428,9 +1430,20 @@ class CountriesAndEU(CachedCountries):
     cache_subkey = 'with_any_or_eu'
 
 
+class CountriesAndEUAndStates(CountriesAndEU):
+    def __iter__(self):
+        for country_code, country_name in super().__iter__():
+            yield country_code, country_name
+            if country_code in COUNTRIES_WITH_STATE_IN_ADDRESS:
+                types, form = COUNTRIES_WITH_STATE_IN_ADDRESS[country_code]
+                yield from sorted(((state.code, country_name + " - " + state.name)
+                                   for state in pycountry.subdivisions.get(country_code=country_code)
+                                   if state.type in types), key=lambda s: s[1])
+
+
 class TaxRuleLineForm(I18nForm):
     country = LazyTypedChoiceField(
-        choices=CountriesAndEU(),
+        choices=CountriesAndEUAndStates(),
         required=False
     )
     address_type = forms.ChoiceField(

src/pretix/control/forms/item.py

@@ -461,11 +461,6 @@ class ItemCreateForm(I18nModelForm):
                 )
 
         if self.cleaned_data.get('copy_from'):
-            for mv in self.cleaned_data['copy_from'].meta_values.all():
-                mv.pk = None
-                mv.item = instance
-                mv.save(force_insert=True)
-
             for question in self.cleaned_data['copy_from'].questions.all():
                 question.items.add(instance)
                 question.log_action('pretix.event.question.changed', user=self.user, data={

src/pretix/control/forms/vouchers.py

@@ -340,6 +340,9 @@ class VoucherBulkForm(VoucherForm):
 
     def clean_send_recipients(self):
         raw = self.cleaned_data['send_recipients']
+        if self.cleaned_data.get('send', None) is False:
+            # No need to validate addresses if the section was turned off
+            return []
         if not raw:
             return []
         r = raw.split('\n')

src/pretix/control/templates/pretixcontrol/vouchers/index.html

@@ -96,7 +96,9 @@
                     <tr>
                         <th>
                             {% if "can_change_vouchers" in request.eventpermset %}
-                                <input type="checkbox" data-toggle-table />
+                                <label aria-label="{% trans "select all rows for batch-operation" %}" class="batch-select-label">
+                                    <input type="checkbox" data-toggle-table />
+                                </label>
                             {% endif %}
                         </th>
                         <th>
@@ -139,7 +141,9 @@
                         <tr>
                             <td>
                                 {% if "can_change_vouchers" in request.eventpermset %}
-                                    <input type="checkbox" name="voucher" class="" value="{{ v.pk }}"/>
+                                    <label aria-label="{% trans "select row for batch-operation" %}" class="batch-select-label">
+                                        <input type="checkbox" name="voucher" class="batch-select-checkbox" value="{{ v.pk }}"/>
+                                    </label>
                                 {% endif %}
                             </td>
                             <td>
@@ -194,9 +198,12 @@
                 </table>
             </div>
             {% if "can_change_vouchers" in request.eventpermset %}
-                <button type="submit" class="btn btn-default btn-save" name="action" value="delete">
-                    {% trans "Delete selected" %}
-                </button>
+                <div class="batch-select-actions">
+                    <button type="submit" class="btn btn-danger" name="action" value="delete">
+                        <i class="fa fa-trash" aria-hidden="true"></i>
+                        {% trans "Delete selected" %}
+                    </button>
+                </div>
             {% endif %}
         </form>
         {% include "pretixcontrol/pagination.html" %}

src/pretix/control/views/dashboards.py

@@ -198,12 +198,12 @@ def waitinglist_widgets(sender, subevent=None, lazy=False, **kwargs):
                     else item.check_quotas(subevent=subevent, count_waitinglist=False, _cache=quota_cache)
                 )
                 if row[1] is None:
-                    happy += 1
+                    happy += wlt['cnt']
                 elif row[1] > 0:
-                    happy += 1
+                    happy += min(wlt['cnt'], row[1])
                     for q in quotas:
                         if q.size is not None:
-                            quota_cache[q.pk] = (quota_cache[q.pk][0], quota_cache[q.pk][1] - 1)
+                            quota_cache[q.pk] = (quota_cache[q.pk][0], quota_cache[q.pk][1] - min(wlt['cnt'], row[1]))
 
         widgets.append({
             'content': None if lazy else NUM_WIDGET.format(

src/pretix/control/views/event.py

@@ -785,8 +785,8 @@ class MailSettingsRendererPreview(MailSettingsPreview):
         return ctx
 
     def get(self, request, *args, **kwargs):
-        v = str(request.event.settings.mail_text_order_payment_failed)
-        v = format_map(v, self.placeholders('mail_text_order_payment_failed'))
+        v = str(request.event.settings.mail_text_order_placed)
+        v = format_map(v, self.placeholders('mail_text_order_placed'))
         renderers = request.event.get_html_mail_renderers()
         if request.GET.get('renderer') in renderers:
             with rolledback_transaction():

src/pretix/control/views/item.py

@@ -1188,30 +1188,46 @@ class MetaDataEditorMixin:
 
     @cached_property
     def meta_forms(self):
-        if hasattr(self, 'object') and self.object:
+        if getattr(self, 'object', None):
             val_instances = {
                 v.property_id: v for v in self.object.meta_values.all()
             }
         else:
             val_instances = {}
 
+        if getattr(self, 'copy_from', None):
+            defaults = {
+                v.property_id: v.value for v in self.copy_from.meta_values.all()
+            }
+        else:
+            defaults = {}
+
         formlist = []
 
         for p in self.request.event.item_meta_properties.all():
-            formlist.append(self._make_meta_form(p, val_instances))
+            formlist.append(self._make_meta_form(p, val_instances, defaults))
         return formlist
 
-    def _make_meta_form(self, p, val_instances):
+    def _make_meta_form(self, p, val_instances, defaults):
         return self.meta_form(
             prefix='prop-{}'.format(p.pk),
             property=p,
-            instance=val_instances.get(p.pk, self.meta_model(property=p, item=self.object)),
+            instance=val_instances.get(
+                p.pk,
+                self.meta_model(
+                    property=p,
+                    item=self.object if getattr(self, 'object', None) else None,
+                    value=defaults.get(p.pk, None)
+                )
+            ),
             data=(self.request.POST if self.request.method == "POST" else None)
         )
 
     def save_meta(self):
         for f in self.meta_forms:
             if f.cleaned_data.get('value'):
+                if not f.instance.item_id:
+                    f.instance.item = self.object
                 f.save()
             elif f.instance and f.instance.pk:
                 f.instance.delete()
@@ -1257,6 +1273,7 @@ class ItemCreate(EventPermissionRequiredMixin, MetaDataEditorMixin, CreateView):
         messages.success(self.request, _('Your changes have been saved.'))
 
         ret = super().form_valid(form)
+        self.save_meta()
         form.instance.log_action('pretix.event.item.added', user=self.request.user, data={
             k: (form.cleaned_data.get(k).name
                 if isinstance(form.cleaned_data.get(k), File)
@@ -1283,6 +1300,14 @@ class ItemCreate(EventPermissionRequiredMixin, MetaDataEditorMixin, CreateView):
         ctx['meta_forms'] = self.meta_forms
         return ctx
 
+    def post(self, request, *args, **kwargs):
+        self.object = None
+        form = self.get_form()
+        if form.is_valid() and all([f.is_valid() for f in self.meta_forms]):
+            return self.form_valid(form)
+        else:
+            return self.form_invalid(form)
+
 
 class ItemUpdateGeneral(ItemDetailMixin, EventPermissionRequiredMixin, MetaDataEditorMixin, UpdateView):
     form_class = ItemUpdateForm

src/pretix/control/views/organizer.py

@@ -1054,8 +1054,8 @@ class DeviceBulkUpdateView(DeviceQueryMixin, OrganizerDetailViewMixin, Organizer
             limit_events_list=Subquery(
                 Device.limit_events.through.objects.filter(
                     device_id=OuterRef('pk')
-                ).order_by('device_id', 'event_id').values('device_id').annotate(
-                    g=GroupConcat('event_id', separator=',')
+                ).order_by().values('device_id').annotate(
+                    g=GroupConcat('event_id', separator=',', ordered=True)
                 ).values('g')
             )
         )

src/pretix/helpers/database.py

@@ -66,18 +66,26 @@ class GroupConcat(Aggregate):
     function = 'group_concat'
     template = '%(function)s(%(field)s, "%(separator)s")'
 
-    def __init__(self, *expressions, **extra):
+    def __init__(self, *expressions, ordered=False, **extra):
+        self.ordered = ordered
         if 'separator' not in extra:
             # For PostgreSQL separator is an obligatory
             extra.update({'separator': ','})
         super().__init__(*expressions, **extra)
 
     def as_postgresql(self, compiler, connection):
-        return super().as_sql(
-            compiler, connection,
-            function='string_agg',
-            template="%(function)s(%(field)s::text, '%(separator)s')",
-        )
+        if self.ordered:
+            return super().as_sql(
+                compiler, connection,
+                function='string_agg',
+                template="%(function)s(%(field)s::text, '%(separator)s' ORDER BY %(field)s ASC)",
+            )
+        else:
+            return super().as_sql(
+                compiler, connection,
+                function='string_agg',
+                template="%(function)s(%(field)s::text, '%(separator)s')",
+            )
 
 
 class ReplicaRouter:

src/pretix/locale/de_Informal/LC_MESSAGES/django.po

@@ -8,8 +8,8 @@ msgstr ""
 "Project-Id-Version: 1\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2023-07-27 11:49+0000\n"
-"PO-Revision-Date: 2023-07-28 09:11+0000\n"
-"Last-Translator: Raphael Michel <michel@rami.io>\n"
+"PO-Revision-Date: 2023-08-16 22:00+0000\n"
+"Last-Translator: Felix Hartnagel <felix@fhcom.de>\n"
 "Language-Team: German (informal) <https://translate.pretix.eu/projects/"
 "pretix/pretix/de_Informal/>\n"
 "Language: de_Informal\n"
@@ -31019,7 +31019,7 @@ msgstr ""
 #: pretix/presale/templates/pretixpresale/event/order.html:43
 msgid "Please note that we still await your payment to complete the process."
 msgstr ""
-"Bitte beachten Sie, dass wir noch deine Zahlung erwarten, um den Prozess "
+"Bitte beachte, dass wir noch deine Zahlung erwarten, um den Prozess "
 "abzuschließen."
 
 #: pretix/presale/templates/pretixpresale/event/order.html:55

src/pretix/locale/fr/LC_MESSAGES/django.po

@@ -4,8 +4,8 @@ msgstr ""
 "Project-Id-Version: 1\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2023-07-27 11:49+0000\n"
-"PO-Revision-Date: 2023-08-09 03:00+0000\n"
-"Last-Translator: Ronan LE MEILLAT <ronan.le_meillat@highcanfly.club>\n"
+"PO-Revision-Date: 2023-08-16 22:00+0000\n"
+"Last-Translator: Maurice Kaag <maurice@kaag.me>\n"
 "Language-Team: French <https://translate.pretix.eu/projects/pretix/pretix/fr/"
 ">\n"
 "Language: fr\n"
@@ -247,9 +247,7 @@ msgstr ""
 
 #: pretix/api/serializers/item.py:185 pretix/control/forms/item.py:1084
 msgid "The bundled item must not have bundles on its own."
-msgstr ""
-"Un forfait ne doit pas contenir des produits, qui sont eux-mêmes des "
-"forfaits."
+msgstr "Un produit groupé ne doit pas contenir des produits groupés."
 
 #: pretix/api/serializers/item.py:262
 msgid ""
@@ -3097,7 +3095,7 @@ msgstr "Annulation"
 #: pretix/base/invoice.py:620 pretix/base/invoice.py:628
 msgctxt "invoice"
 msgid "Description"
-msgstr "Déscription"
+msgstr "Description"
 
 #: pretix/base/invoice.py:621 pretix/base/invoice.py:629
 msgctxt "invoice"
@@ -6805,16 +6803,12 @@ msgid "List of Add-Ons"
 msgstr "Liste des Addons"
 
 #: pretix/base/pdf.py:364
-#, fuzzy
-#| msgid ""
-#| "Add-on 1\n"
-#| "Add-on 2"
 msgid ""
 "Add-on 1\n"
 "2x Add-on 2"
 msgstr ""
 "Add-on 1\n"
-"Add-on 2"
+"2x Add-on 2"
 
 #: pretix/base/pdf.py:370 pretix/control/forms/filter.py:1275
 #: pretix/control/forms/filter.py:1277
@@ -11136,7 +11130,7 @@ msgstr "Degré (après le nom)"
 #: pretix/base/settings.py:3577
 msgctxt "person_name_sample"
 msgid "MA"
-msgstr ""
+msgstr "MA"
 
 #: pretix/base/settings.py:3684 pretix/control/forms/event.py:217
 msgid ""

src/pretix/locale/nl/LC_MESSAGES/django.po

@@ -7,16 +7,16 @@ msgstr ""
 "Project-Id-Version: 1\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2023-07-27 11:49+0000\n"
-"PO-Revision-Date: 2023-07-16 22:00+0000\n"
-"Last-Translator: Freek Engelbarts <freekengelbarts@gmail.com>\n"
-"Language-Team: Dutch <https://translate.pretix.eu/projects/pretix/pretix/nl/"
-">\n"
+"PO-Revision-Date: 2023-08-24 04:00+0000\n"
+"Last-Translator: Alain <alain@waag.org>\n"
+"Language-Team: Dutch <https://translate.pretix.eu/projects/pretix/pretix/nl/>"
+"\n"
 "Language: nl\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
-"X-Generator: Weblate 4.17\n"
+"X-Generator: Weblate 4.18.2\n"
 
 #: pretix/_base_settings.py:78
 msgid "English"
@@ -534,10 +534,8 @@ msgid "Waiting list entry deleted"
 msgstr "Wachtlijstitem verwijderd"
 
 #: pretix/api/webhooks.py:351
-#, fuzzy
-#| msgid "Waiting list entries"
 msgid "Waiting list entry received voucher"
-msgstr "Wachtlijstitems"
+msgstr "Wachtlijstitem heeft voucher ontvangen"
 
 #: pretix/base/addressvalidation.py:100 pretix/base/addressvalidation.py:103
 #: pretix/base/addressvalidation.py:108 pretix/base/forms/questions.py:938
@@ -553,11 +551,11 @@ msgstr "Dit veld is verplicht."
 
 #: pretix/base/addressvalidation.py:213
 msgid "Enter a postal code in the format XXX."
-msgstr "Voer een postcode in in het formaat XXX."
+msgstr "Postcode in het formaat XXX invoeren."
 
 #: pretix/base/addressvalidation.py:222 pretix/base/addressvalidation.py:224
 msgid "Enter a postal code in the format XXXX."
-msgstr "Voer een postcode in in het format XXXX."
+msgstr "Postcode in het format XXXX invoeren."
 
 #: pretix/base/auth.py:143
 #, python-brace-format
@@ -2311,7 +2309,7 @@ msgstr ""
 
 #: pretix/base/exporters/orderlist.py:887
 msgid "Converted from legacy version"
-msgstr ""
+msgstr "Vanuit oudere versie geconverteerd"
 
 #: pretix/base/exporters/orderlist.py:949
 msgid "Payments and refunds"
@@ -4380,7 +4378,7 @@ msgstr ""
 
 #: pretix/base/models/items.py:662
 msgid "Reusable media type"
-msgstr ""
+msgstr "Mediatype"
 
 #: pretix/base/models/items.py:664
 msgid ""
@@ -6146,7 +6144,7 @@ msgstr "Vul een geldige taalcode in."
 #: pretix/base/orderimport.py:669 pretix/base/orderimport.py:692
 #, python-brace-format
 msgid "Could not parse {value} as a date and time."
-msgstr ""
+msgstr "Kon {value} niet als datum en tijd herkennen."
 
 #: pretix/base/orderimport.py:711
 msgid "Please enter a valid sales channel."
@@ -6847,7 +6845,7 @@ msgstr "Geldig tot"
 
 #: pretix/base/pdf.py:457
 msgid "Reusable Medium ID"
-msgstr ""
+msgstr "Media-ID"
 
 #: pretix/base/pdf.py:462
 msgid "Seat: Full name"
@@ -7524,6 +7522,8 @@ msgstr "Geen toestemming"
 #: pretix/base/services/export.py:221
 msgid "Your exported data exceeded the size limit for scheduled exports."
 msgstr ""
+"De door u geëxporteerde data overschrijdt de grootte-limiet voor geplande "
+"exports."
 
 #: pretix/base/services/invoices.py:103
 #, python-brace-format
@@ -7766,10 +7766,7 @@ msgid "Your cart is empty."
 msgstr "Uw winkelwagen is leeg."
 
 #: pretix/base/services/orders.py:138
-#, fuzzy, python-format
-#| msgid ""
-#| "You cannot select more than %(max)s items of the product %(product)s. We "
-#| "removed the surplus items from your cart."
+#, python-format
 msgid ""
 "You cannot select more than %(max)s item of the product %(product)s. We "
 "removed the surplus items from your cart."
@@ -7777,11 +7774,11 @@ msgid_plural ""
 "You cannot select more than %(max)s items of the product %(product)s. We "
 "removed the surplus items from your cart."
 msgstr[0] ""
-"U kunt niet meer dan %(max)s kopieën van het product %(product)s kiezen. We "
-"hebben het overschot uit uw winkelwagen verwijderd."
+"U kunt van het product %(product)s niet meer dan %(max)s per bestelling "
+"kiezen. We hebben de overtallige producten uit uw winkelwagen verwijderd."
 msgstr[1] ""
-"U kunt niet meer dan %(max)s kopieën van het product %(product)s kiezen. We "
-"hebben het overschot uit uw winkelwagen verwijderd."
+"U kunt van het product %(product)s niet meer dan %(max)s per bestelling "
+"kiezen. We hebben de overtallige producten uit uw winkelwagen verwijderd."
 
 #: pretix/base/services/orders.py:147
 msgid "The booking period has ended."
@@ -7831,10 +7828,9 @@ msgstr ""
 "niet geldig voor dit item. We hebben dit item uit uw winkelwagen verwijderd."
 
 #: pretix/base/services/orders.py:168
-#, fuzzy
-#| msgid "You need a valid voucher code to order this product."
 msgid "You need a valid voucher code to order one of the products."
-msgstr "U heeft een geldige vouchercode nodig om dit product te bestellen."
+msgstr ""
+"U heeft een geldige vouchercode nodig om een van de producten te bestellen."
 
 #: pretix/base/services/orders.py:170
 msgid ""
@@ -7873,10 +7869,8 @@ msgstr ""
 "is besteld."
 
 #: pretix/base/services/orders.py:210
-#, fuzzy
-#| msgid "The order has been canceled."
 msgid "The order was not canceled."
-msgstr "De bestelling is geannuleerd."
+msgstr "De bestelling is niet geannuleerd."
 
 #: pretix/base/services/orders.py:265 pretix/control/forms/orders.py:120
 msgid "The new expiry date needs to be in the future."
@@ -7912,10 +7906,8 @@ msgstr ""
 "bestelling is betaald."
 
 #: pretix/base/services/orders.py:918
-#, fuzzy
-#| msgid "This payment method does not support automatic refunds."
 msgid "The selected payment methods do not cover the total balance."
-msgstr "Deze betalingsmethode ondersteunt geen automatische terugbetalingen."
+msgstr "Deze betalingsmethode dekt het volledige bedrag niet."
 
 #: pretix/base/services/orders.py:990
 msgid ""
@@ -8070,10 +8062,8 @@ msgid "Something happened in your event after the export, please try again."
 msgstr "Er is iets gebeurd in uw evenement na de export, probeer het opnieuw."
 
 #: pretix/base/services/shredder.py:177
-#, fuzzy
-#| msgid "Payment completed."
 msgid "Data shredding completed"
-msgstr "Betaling voltooid."
+msgstr "Verwijderen van data voltooid."
 
 #: pretix/base/services/stats.py:210
 msgid "Uncategorized"
@@ -10100,19 +10090,7 @@ msgid "Your order is pending payment: {code}"
 msgstr "Uw bestelling wacht op betaling: {code}"
 
 #: pretix/base/settings.py:2316
-#, fuzzy, python-brace-format
-#| msgid ""
-#| "Hello,\n"
-#| "\n"
-#| "we did not yet receive a full payment for your order for {event}.\n"
-#| "Please keep in mind that we only guarantee your order if we receive\n"
-#| "your payment before {expire_date}.\n"
-#| "\n"
-#| "You can view the payment information and the status of your order at\n"
-#| "{url}\n"
-#| "\n"
-#| "Best regards,  \n"
-#| "Your {event} team"
+#, python-brace-format
 msgid ""
 "Hello,\n"
 "\n"
@@ -10135,7 +10113,7 @@ msgstr ""
 "U kunt de betalingsinformatie en de status van uw bestelling inzien op\n"
 "{url}.\n"
 "\n"
-"Met vriendelijke groet,\n"
+"Met vriendelijke groet,  \n"
 "De organisatoren van {event}"
 
 #: pretix/base/settings.py:2329
@@ -10145,19 +10123,7 @@ msgid "Incomplete payment received: {code}"
 msgstr "Betaling ontvangen voor uw bestelling: {code}"
 
 #: pretix/base/settings.py:2333
-#, fuzzy, python-brace-format
-#| msgid ""
-#| "Hello,\n"
-#| "\n"
-#| "we did not yet receive a full payment for your order for {event}.\n"
-#| "Please keep in mind that we only guarantee your order if we receive\n"
-#| "your payment before {expire_date}.\n"
-#| "\n"
-#| "You can view the payment information and the status of your order at\n"
-#| "{url}\n"
-#| "\n"
-#| "Best regards,  \n"
-#| "Your {event} team"
+#, python-brace-format
 msgid ""
 "Hello,\n"
 "\n"
@@ -10175,10 +10141,11 @@ msgid ""
 msgstr ""
 "Hallo,\n"
 "\n"
-"We hebben nog geen volledige betaling ontvangen voor uw bestelling voor "
-"{event}.\n"
-"We kunnen uw bestelling alleen garanderen als we uw betaling ontvangen\n"
-"voor {expire_date}.\n"
+"We hebben een betaling ontvangen voor {event}\n"
+"\n"
+"Helaas is het ontvangen bedrag minder dan het volledige verschuldigde "
+"bedrag. Graag nog het bedrag van **{pending_sum}** voldoen om de bestelling "
+"te voltooien.\n"
 "\n"
 "U kunt de betalingsinformatie en de status van uw bestelling inzien op\n"
 "{url}.\n"
@@ -10367,17 +10334,7 @@ msgstr ""
 "Organisatie van {event}"
 
 #: pretix/base/settings.py:2446 pretix/base/settings.py:2483
-#, fuzzy, python-brace-format
-#| msgid ""
-#| "Hello {attendee_name},\n"
-#| "\n"
-#| "a ticket for {event} has been ordered for you.\n"
-#| "\n"
-#| "You can view the details and status of your ticket here:\n"
-#| "{url}\n"
-#| "\n"
-#| "Best regards,  \n"
-#| "Your {event} team"
+#, python-brace-format
 msgid ""
 "Hello,\n"
 "\n"
@@ -10389,9 +10346,9 @@ msgid ""
 "Best regards,  \n"
 "Your {event} team"
 msgstr ""
-"Beste {attendee_name},\n"
+"Beste,\n"
 "\n"
-"Er is een ticket voor {event} voor u besteld.\n"
+"Uw ticket voor {event} is geaccordeerd.\n"
 "\n"
 "U kunt de details en status van uw ticket hier bekijken:\n"
 "{url}\n"
@@ -17295,10 +17252,8 @@ msgid "Valid check-in"
 msgstr "Alle check-ins"
 
 #: pretix/control/templates/pretixcontrol/checkin/simulator.html:67
-#, fuzzy
-#| msgid "Additional information"
 msgid "Additional information required"
-msgstr "Extra informatie"
+msgstr "Extra informatie vereist"
 
 #: pretix/control/templates/pretixcontrol/checkin/simulator.html:69
 msgid ""

src/pretix/locale/nl/LC_MESSAGES/djangojs.po

@@ -7,8 +7,8 @@ msgstr ""
 "Project-Id-Version: 1\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2023-07-21 11:46+0000\n"
-"PO-Revision-Date: 2021-10-29 02:00+0000\n"
-"Last-Translator: Maarten van den Berg <maartenberg1@gmail.com>\n"
+"PO-Revision-Date: 2023-08-24 04:00+0000\n"
+"Last-Translator: Alain <alain@waag.org>\n"
 "Language-Team: Dutch <https://translate.pretix.eu/projects/pretix/pretix-js/"
 "nl/>\n"
 "Language: nl\n"
@@ -16,7 +16,7 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
-"X-Generator: Weblate 4.8\n"
+"X-Generator: Weblate 4.18.2\n"
 
 #: pretix/plugins/banktransfer/static/pretixplugins/banktransfer/ui.js:56
 #: pretix/plugins/banktransfer/static/pretixplugins/banktransfer/ui.js:62
@@ -252,7 +252,7 @@ msgstr "Dit ticket is nog niet betaald. Wilt u toch doorgaan?"
 
 #: pretix/plugins/webcheckin/static/pretixplugins/webcheckin/main.js:51
 msgid "Additional information required"
-msgstr "Extra informatie nodig"
+msgstr "Extra informatie vereist"
 
 #: pretix/plugins/webcheckin/static/pretixplugins/webcheckin/main.js:52
 msgid "Valid ticket"

src/pretix/locale/nl_Informal/LC_MESSAGES/django.po

@@ -8,8 +8,8 @@ msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2023-07-27 11:49+0000\n"
-"PO-Revision-Date: 2023-07-16 22:00+0000\n"
-"Last-Translator: Freek Engelbarts <freekengelbarts@gmail.com>\n"
+"PO-Revision-Date: 2023-08-24 04:00+0000\n"
+"Last-Translator: Alain <alain@waag.org>\n"
 "Language-Team: Dutch (informal) <https://translate.pretix.eu/projects/pretix/"
 "pretix/nl_Informal/>\n"
 "Language: nl_Informal\n"
@@ -17,7 +17,7 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
-"X-Generator: Weblate 4.17\n"
+"X-Generator: Weblate 4.18.2\n"
 
 #: pretix/_base_settings.py:78
 msgid "English"
@@ -555,10 +555,8 @@ msgid "Waiting list entry deleted"
 msgstr "Wachtlijstitem"
 
 #: pretix/api/webhooks.py:351
-#, fuzzy
-#| msgid "Waiting list entries"
 msgid "Waiting list entry received voucher"
-msgstr "Wachtlijstitems"
+msgstr "Wachtlijstitem heeft voucher ontvangen"
 
 #: pretix/base/addressvalidation.py:100 pretix/base/addressvalidation.py:103
 #: pretix/base/addressvalidation.py:108 pretix/base/forms/questions.py:938

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_assign.html

@@ -11,10 +11,10 @@
             <span class="icon icon-upload"></span> {% trans "Continue" %}
         </button>
         <div class="flipped-scroll-wrapper clearfix">
-            <table class="table table-condensed flipped-scroll-inner">
+            <table class="table table-condensed table-th-sticky-horizontal flipped-scroll-inner">
                 <thead>
                 <tr>
-                    <th>{% trans "Date" %}</th>
+                    <th scope="row">{% trans "Date" %}</th>
                     {% for col in rows.0 %}
                         <th>
                             <input type="radio" name="date" value="{{ forloop.counter0 }}"/>
@@ -22,7 +22,7 @@
                     {% endfor %}
                 </tr>
                 <tr>
-                    <th>{% trans "Amount" %}</th>
+                    <th scope="row">{% trans "Amount" %}</th>
                     {% for col in rows.0 %}
                         <th>
                             <input type="radio" name="amount" value="{{ forloop.counter0 }}" required="required"/>
@@ -30,7 +30,7 @@
                     {% endfor %}
                 </tr>
                 <tr>
-                    <th>{% trans "Reference" %}</th>
+                    <th scope="row">{% trans "Reference" %}</th>
                     {% for col in rows.0 %}
                         <th>
                             <input type="checkbox" name="reference" value="{{ forloop.counter0 }}"/>
@@ -38,7 +38,7 @@
                     {% endfor %}
                 </tr>
                 <tr>
-                    <th>{% trans "Payer" %}</th>
+                    <th scope="row">{% trans "Payer" %}</th>
                     {% for col in rows.0 %}
                         <th>
                             <input type="checkbox" name="payer" value="{{ forloop.counter0 }}"/>
@@ -46,7 +46,7 @@
                     {% endfor %}
                 </tr>
                 <tr>
-                    <th>
+                    <th scope="row">
                         {% trans "IBAN" %}
                         <label for="id_iban_clear">
                             <span class="btn btn-default btn-sm fa fa-close"></span>
@@ -62,7 +62,7 @@
                     {% endfor %}
                 </tr>
                 <tr>
-                    <th>
+                    <th scope="row">
                         {% trans "BIC" %}
                         <label for="id_bic_clear">
                             <span class="btn btn-default btn-sm fa fa-close"></span>

src/pretix/plugins/sendmail/api.py

@@ -34,7 +34,7 @@ class RuleSerializer(I18nAwareModelSerializer):
     class Meta:
         model = Rule
         fields = ['id', 'subject', 'template', 'all_products', 'limit_products', 'restrict_to_status',
-                  'send_date', 'send_offset_days', 'send_offset_time', 'date_is_absolute',
+                  'checked_in_status', 'send_date', 'send_offset_days', 'send_offset_time', 'date_is_absolute',
                   'offset_to_event_end', 'offset_is_after', 'send_to', 'enabled']
         read_only_fields = ['id']
 
@@ -88,6 +88,10 @@ class RuleSerializer(I18nAwareModelSerializer):
                 ]:
                     raise ValidationError(f'status {s} not allowed: restrict_to_status may only include valid states')
 
+        if full_data.get('checked_in_status') == "":
+            # even though "blank" is not allowed on this field, "" gets accepted without this check
+            raise ValidationError('empty string not allowed: use null to disable check-in based filtering')
+
         return full_data
 
     def save(self, **kwargs):

src/pretix/plugins/sendmail/forms.py

@@ -312,7 +312,7 @@ class RuleForm(FormPlaceholderMixin, I18nModelForm):
         fields = ['subject', 'template', 'attach_ical',
                   'send_date', 'send_offset_days', 'send_offset_time',
                   'all_products', 'limit_products', 'restrict_to_status',
-                  'send_to', 'enabled']
+                  'checked_in_status', 'send_to', 'enabled']
 
         field_classes = {
             'subevent': SafeModelMultipleChoiceField,
@@ -337,6 +337,7 @@ class RuleForm(FormPlaceholderMixin, I18nModelForm):
                        'data-inverse-dependency': '#id_all_products'},
             ),
             'send_to': forms.RadioSelect,
+            'checked_in_status': forms.RadioSelect,
         }
 
     def __init__(self, *args, **kwargs):

src/pretix/plugins/sendmail/migrations/0005_rule_checked_in_status.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.19 on 2023-08-09 11:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sendmail', '0004_rule_restrict_to_status'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='rule',
+            name='checked_in_status',
+            field=models.CharField(max_length=10, null=True),
+        ),
+    ]

src/pretix/plugins/sendmail/models.py

@@ -34,7 +34,8 @@ from i18nfield.fields import I18nCharField, I18nTextField
 from pretix.base.email import get_email_context
 from pretix.base.i18n import language
 from pretix.base.models import (
-    Event, InvoiceAddress, Item, Order, OrderPosition, SubEvent, fields,
+    Checkin, Event, InvoiceAddress, Item, Order, OrderPosition, SubEvent,
+    fields,
 )
 from pretix.base.models.base import LoggingMixin
 from pretix.base.services.mail import SendMailException
@@ -112,19 +113,30 @@ class ScheduledMail(models.Model):
         e = self.event
 
         orders = e.orders.all()
-        limit_products = self.rule.limit_products.values_list('pk', flat=True) if not self.rule.all_products else None
+
+        filter_orders_by_op = False
+        op_qs = OrderPosition.objects.filter(
+            order__event=self.event,
+            canceled=False,
+        )
 
         if self.subevent:
-            orders = orders.filter(
-                Exists(OrderPosition.objects.filter(order=OuterRef('pk'), subevent=self.subevent))
-            )
+            filter_orders_by_op = True
+            op_qs = op_qs.filter(subevent=self.subevent)
         elif e.has_subevents:
             return  # This rule should not even exist
 
         if not self.rule.all_products:
-            orders = orders.filter(
-                Exists(OrderPosition.objects.filter(order=OuterRef('pk'), item_id__in=limit_products))
-            )
+            filter_orders_by_op = True
+            limit_products = self.rule.limit_products.values_list('pk', flat=True)
+            op_qs = op_qs.filter(item_id__in=limit_products)
+
+        if self.rule.checked_in_status == "no_checkin":
+            filter_orders_by_op = True
+            op_qs = op_qs.filter(~Exists(Checkin.objects.filter(position_id=OuterRef('pk'))))
+        elif self.rule.checked_in_status == "checked_in":
+            filter_orders_by_op = True
+            op_qs = op_qs.filter(Exists(Checkin.objects.filter(position_id=OuterRef('pk'))))
 
         status_q = Q(status__in=self.rule.restrict_to_status)
         if 'n__pending_approval' in self.rule.restrict_to_status:
@@ -142,6 +154,8 @@ class ScheduledMail(models.Model):
                 pk__gt=self.last_successful_order_id
             )
 
+        if filter_orders_by_op:
+            orders = orders.filter(pk__in=op_qs.values_list('order_id', flat=True))
         orders = orders.filter(
             status_q,
         ).order_by('pk').select_related('invoice_address').prefetch_related('positions')
@@ -205,6 +219,12 @@ class Rule(models.Model, LoggingMixin):
         (BOTH, _('Both (all order contact addresses and all attendee email addresses)'))
     ]
 
+    CHECK_IN_STATUS_CHOICES = [
+        (None, _("Everyone")),
+        ("checked_in", _("Anyone who is or was checked in")),
+        ("no_checkin", _("Anyone who never checked in before"))
+    ]
+
     id = models.BigAutoField(primary_key=True)
     event = models.ForeignKey(Event, on_delete=models.CASCADE, related_name='sendmail_rules')
 
@@ -219,6 +239,15 @@ class Rule(models.Model, LoggingMixin):
         default=['p', 'n__valid_if_pending'],
     )
 
+    checked_in_status = models.CharField(
+        verbose_name=_("Restrict to check-in status"),
+        default=None,
+        choices=CHECK_IN_STATUS_CHOICES,
+        max_length=10,
+        null=True,
+        blank=True,
+    )
+
     attach_ical = models.BooleanField(
         default=False,
         verbose_name=_("Attach calendar files"),

src/pretix/plugins/sendmail/templates/pretixplugins/sendmail/rule_create.html

@@ -28,6 +28,8 @@
                 <legend>{% trans "Recipients" %}</legend>
                 {% bootstrap_field form.send_to layout='control' %}
                 {% bootstrap_field form.restrict_to_status layout='control' %}
+                {% bootstrap_field form.checked_in_status layout='control' %}
+            <hr>
                 {% bootstrap_field form.all_products layout='control' %}
                 {% bootstrap_field form.limit_products layout='horizontal' %}
             </fieldset>

src/pretix/plugins/sendmail/templates/pretixplugins/sendmail/rule_update.html

@@ -42,6 +42,8 @@
                 <legend>{% trans "Recipients" %}</legend>
                 {% bootstrap_field form.send_to layout='control' %}
                 {% bootstrap_field form.restrict_to_status layout='control' %}
+                {% bootstrap_field form.checked_in_status layout='control' %}
+            <hr>
                 {% bootstrap_field form.all_products layout='control' %}
                 {% bootstrap_field form.limit_products layout='horizontal' %}
             </fieldset>

src/pretix/plugins/stripe/payment.py

@@ -554,9 +554,6 @@ class StripeMethod(BasePaymentProvider):
         ctx = {'request': request, 'event': self.event, 'settings': self.settings, 'provider': self}
         return template.render(ctx)
 
-    def payment_can_retry(self, payment):
-        return self._is_still_available(order=payment.order)
-
     def _charge_source(self, request, source, payment):
         try:
             params = {}
@@ -1581,9 +1578,6 @@ class StripeSofort(StripeMethod):
             return True
         return False
 
-    def payment_can_retry(self, payment):
-        return payment.state != OrderPayment.PAYMENT_STATE_PENDING and self._is_still_available(order=payment.order)
-
     def payment_presale_render(self, payment: OrderPayment) -> str:
         pi = payment.info_data or {}
         try:

src/pretix/presale/checkoutflow.py

@@ -39,10 +39,13 @@ from decimal import Decimal
 
 from django.conf import settings
 from django.contrib import messages
+from django.core.cache import caches
 from django.core.exceptions import ImproperlyConfigured, ValidationError
 from django.core.signing import BadSignature, loads
 from django.core.validators import EmailValidator
-from django.db.models import F, Q
+from django.db import models
+from django.db.models import Count, F, Q, Sum
+from django.db.models.functions import Cast
 from django.http import HttpResponseNotAllowed, JsonResponse
 from django.shortcuts import redirect
 from django.utils import translation
@@ -62,12 +65,14 @@ from pretix.base.services.cart import (
 )
 from pretix.base.services.memberships import validate_memberships_in_order
 from pretix.base.services.orders import perform_order
+from pretix.base.services.tasks import EventTask
 from pretix.base.settings import PERSON_NAME_SCHEMES
 from pretix.base.signals import validate_cart_addons
 from pretix.base.templatetags.money import money_filter
 from pretix.base.templatetags.phone_format import phone_format
 from pretix.base.templatetags.rich_text import rich_text_snippet
 from pretix.base.views.tasks import AsyncAction
+from pretix.celery_app import app
 from pretix.multidomain.urlreverse import eventreverse
 from pretix.presale.forms.checkout import (
     ContactForm, InvoiceAddressForm, InvoiceNameForm, MembershipForm,
@@ -802,7 +807,9 @@ class QuestionsStep(QuestionsViewMixin, CartMixin, TemplateFlowStep):
     @cached_property
     def invoice_form(self):
         wd = self.cart_session.get('widget_data', {})
-        if not self.invoice_address.pk:
+        if self.invoice_address.pk:
+            wd_initial = {}
+        elif wd:
             wd_initial = {
                 'name_parts': {
                     k[21:].replace('-', '_'): v
@@ -817,7 +824,9 @@ class QuestionsStep(QuestionsViewMixin, CartMixin, TemplateFlowStep):
                 'country': wd.get('invoice-address-country', ''),
             }
         else:
-            wd_initial = {}
+            wd_initial = {
+                'is_business': self._get_is_business_heuristic(),
+            }
         initial = dict(wd_initial)
 
         if self.cart_customer:
@@ -1026,6 +1035,25 @@ class QuestionsStep(QuestionsViewMixin, CartMixin, TemplateFlowStep):
         ctx['cart_session'] = self.cart_session
         ctx['invoice_address_asked'] = self.address_asked
 
+        def reduce_initial(v):
+            if isinstance(v, dict):
+                # try to flatten objects such as name_parts to a single string to determine whether they have any value set
+                return ''.join([v for k, v in v.items() if not k.startswith('_')])
+            else:
+                return v
+
+        def is_form_filled(form, ignore_keys=()):
+            return any([reduce_initial(v) for k, v in form.initial.items() if k not in ignore_keys])
+
+        ctx['invoice_address_open'] = (
+            self.request.event.settings.invoice_address_required or
+            self.request.event.settings.invoice_name_required or
+            'invoice' in self.request.GET or
+            # Checking for self.invoice_address.pk is not enough as when an invoice_address has been added and later edited to be empty, it’s not None.
+            # So check initial values as invoice_form can receive pre-filled values from invoice_address, widget-data or overwrites from plug-ins.
+            is_form_filled(self.invoice_form, ignore_keys=('is_business', 'country'))
+        )
+
         if self.cart_customer:
             if self.address_asked:
                 addresses = self.cart_customer.stored_addresses.all()
@@ -1114,6 +1142,31 @@ class QuestionsStep(QuestionsViewMixin, CartMixin, TemplateFlowStep):
             ctx['profiles_data'] = profiles_list
         return ctx
 
+    def _get_is_business_heuristic(self):
+        key = 'checkout_heuristic_is_business:' + str(self.event.pk)
+        cached_result = caches['default'].get(key)
+        if cached_result is None:
+            if caches['default'].add(key, False, timeout=10):  # return False while query is running
+                QuestionsStep._update_is_business_heuristic.apply_async(args=(self.event.pk,))
+            return False
+        else:
+            return cached_result
+
+    @staticmethod
+    @app.task(base=EventTask)
+    def _update_is_business_heuristic(event):
+        result = InvoiceAddress.objects.filter(order__event=event).aggregate(
+            total=Count('*'), business=Sum(Cast('is_business', output_field=models.IntegerField())))
+        if result['total'] < 100:
+            result = InvoiceAddress.objects.filter(order__event__organizer=event.organizer).aggregate(
+                total=Count('*'), business=Sum(Cast('is_business', output_field=models.IntegerField())))
+        if result['business'] and result['total']:
+            is_business = result['business'] / result['total'] >= 0.6
+        else:
+            is_business = False
+        key = 'checkout_heuristic_is_business:' + str(event.pk)
+        caches['default'].set(key, is_business, timeout=12 * 3600)  # 12 hours
+
 
 class PaymentStep(CartMixin, TemplateFlowStep):
     priority = 200

src/pretix/presale/templates/pretixpresale/event/checkout_confirm.html

@@ -76,7 +76,7 @@
                         <div class="panel-heading">
                             <h3 class="panel-title">
                                 {% trans "Invoice information" %}
-                                <a href="{% eventurl request.event "presale:event.checkout" step="questions" cart_namespace=cart_namespace|default_if_none:"" %}?invoice=1" aria-label="{% trans "Modify invoice information" %}" class="h6">
+                                <a href="{% eventurl request.event "presale:event.checkout" step="questions" cart_namespace=cart_namespace|default_if_none:"" %}?invoice=1#invoice-details" aria-label="{% trans "Modify invoice information" %}" class="h6">
                                     <span class="fa fa-edit" aria-hidden="true"></span>{% trans "Modify" %}
                                 </a>
                             </h3>

src/pretix/presale/templates/pretixpresale/event/checkout_questions.html

@@ -34,7 +34,7 @@
                 </div>
             </details>
             {% if invoice_address_asked %}
-                <details class="panel panel-default" {% if event.settings.invoice_address_required or event.settings.invoice_name_required %}open{% endif %}>
+                <details class="panel panel-default" {% if invoice_address_open %}open{% endif %} id="invoice-details">
                     <summary class="panel-heading">
                         <h3 class="panel-title">
                             <strong>{% trans "Invoice information" %}{% if not event.settings.invoice_address_required and not event.settings.invoice_name_required %}

src/pretix/presale/views/order.py

@@ -496,7 +496,12 @@ class OrderPaymentConfirm(EventViewMixin, OrderDetailMixin, TemplateView):
         ctx['order'] = self.order
         ctx['payment'] = self.payment
         if 'order' in inspect.signature(self.payment.payment_provider.checkout_confirm_render).parameters:
-            ctx['payment_info'] = self.payment.payment_provider.checkout_confirm_render(self.request, order=self.order)
+            if 'info_data' in inspect.signature(self.payment.payment_provider.checkout_confirm_render).parameters:
+                ctx['payment_info'] = self.payment.payment_provider.checkout_confirm_render(
+                    self.request, order=self.order, info_data=self.payment.info_data
+                )
+            else:
+                ctx['payment_info'] = self.payment.payment_provider.checkout_confirm_render(self.request, order=self.order)
         else:
             ctx['payment_info'] = self.payment.payment_provider.checkout_confirm_render(self.request)
         ctx['payment_provider'] = self.payment.payment_provider

src/pretix/static/pretixcontrol/scss/main.scss

@@ -813,7 +813,11 @@ tbody[data-dnd-url] {
 tbody th {
     background: $table-bg-hover;
 }
-
+.table-th-sticky-horizontal th[scope=row] {
+    position: sticky;
+    left: 0;
+    background: linear-gradient(0deg, rgba(255,255,255,1) 0%, rgba(255,255,255,1) 96%, rgba(255,255,255,0) 100%);;
+}
 .large-link-group {
     a.list-group-item {
         &::before {

src/pretix/static/schema/tax-rules-custom.schema.json

@@ -8,8 +8,8 @@
     "description": "List of rules, executed in order until one matches",
     "properties": {
       "country": {
-        "description": "Country code to match. ZZ = any country, EU = any EU country.",
-        "enum": ["AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "EU", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW", "ZZ"]
+        "description": "Country code to match. ZZ = any country, EU = any EU country. For selected countries, a state can be matched (e.g. US-NY for New York).",
+        "enum": ["ZZ", "EU", "AF", "EG", "AX", "AL", "DZ", "AS", "VI", "AD", "AO", "AI", "AQ", "AG", "GQ", "AR", "AM", "AW", "AZ", "ET", "AU", "AU-ACT", "AU-NSW", "AU-NT", "AU-QLD", "AU-SA", "AU-TAS", "AU-VIC", "AU-WA", "BS", "BH", "BD", "BB", "BE", "BZ", "BJ", "BM", "BT", "BO", "BQ", "BA", "BW", "BV", "BR", "BR-AC", "BR-AL", "BR-AP", "BR-AM", "BR-BA", "BR-CE", "BR-ES", "BR-GO", "BR-MA", "BR-MT", "BR-MS", "BR-MG", "BR-PR", "BR-PB", "BR-PA", "BR-PE", "BR-PI", "BR-RN", "BR-RS", "BR-RJ", "BR-RO", "BR-RR", "BR-SC", "BR-SE", "BR-SP", "BR-TO", "VG", "IO", "BN", "BG", "BF", "BI", "CL", "CN", "MP", "CK", "CR", "CI", "CW", "DK", "DE", "DM", "DO", "DJ", "EC", "SV", "ER", "EE", "FK", "FO", "FJ", "FI", "FR", "GF", "PF", "TF", "GA", "GM", "GE", "GH", "GI", "GD", "GR", "GL", "GP", "GU", "GT", "GG", "GN", "GW", "GY", "HT", "HM", "HN", "HK", "IN", "ID", "IQ", "IR", "IE", "IS", "IM", "IL", "IT", "JM", "JP", "YE", "JE", "JO", "KY", "KH", "CM", "CA", "CA-AB", "CA-BC", "CA-MB", "CA-NB", "CA-NL", "CA-NT", "CA-NS", "CA-NU", "CA-ON", "CA-PE", "CA-QC", "CA-SK", "CA-YT", "CV", "KZ", "QA", "KE", "KG", "KI", "CC", "CO", "KM", "CG", "CD", "HR", "CU", "KW", "LA", "LS", "LV", "LB", "LR", "LY", "LI", "LT", "LU", "MO", "MG", "MW", "MY", "MY-01", "MY-02", "MY-03", "MY-04", "MY-05", "MY-06", "MY-08", "MY-09", "MY-07", "MY-12", "MY-13", "MY-10", "MY-11", "MV", "ML", "MT", "MA", "MH", "MQ", "MR", "MU", "YT", "MK", "MX", "MX-AGU", "MX-BCN", "MX-BCS", "MX-CAM", "MX-CHP", "MX-CHH", "MX-CMX", "MX-COA", "MX-COL", "MX-DUR", "MX-GUA", "MX-GRO", "MX-HID", "MX-JAL", "MX-MIC", "MX-MOR", "MX-MEX", "MX-NAY", "MX-NLE", "MX-OAX", "MX-PUE", "MX-QUE", "MX-ROO", "MX-SLP", "MX-SIN", "MX-SON", "MX-TAB", "MX-TAM", "MX-TLA", "MX-VER", "MX-YUC", "MX-ZAC", "FM", "MD", "MC", "MN", "ME", "MS", "MZ", "MM", "NA", "NR", "NP", "NC", "NZ", "NI", "NL", "NE", "NG", "NU", "KP", "NF", "NO", "OM", "AT", "TL", "PK", "PS", "PW", "PA", "PG", "PY", "PE", "PH", "PN", "PL", "PT", "PR", "RE", "RW", "RO", "RU", "BL", "PM", "SB", "ZM", "WS", "SM", "ST", "SA", "SE", "CH", "SN", "RS", "SC", "SL", "ZW", "SG", "SX", "SK", "SI", "SO", "ES", "SJ", "LK", "SH", "KN", "LC", "MF", "VC", "ZA", "SD", "GS", "KR", "SS", "SR", "SZ", "SY", "TJ", "TW", "TZ", "TH", "TG", "TK", "TO", "TT", "TD", "CZ", "TN", "TR", "TM", "TC", "TV", "UG", "UA", "HU", "UY", "UM", "UZ", "VU", "VA", "VE", "AE", "US", "US-AL", "US-AK", "US-AS", "US-AZ", "US-AR", "US-CA", "US-CO", "US-CT", "US-DE", "US-DC", "US-FL", "US-GA", "US-GU", "US-HI", "US-ID", "US-IL", "US-IN", "US-IA", "US-KS", "US-KY", "US-LA", "US-ME", "US-MD", "US-MA", "US-MI", "US-MN", "US-MS", "US-MO", "US-MT", "US-NE", "US-NV", "US-NH", "US-NJ", "US-NM", "US-NY", "US-NC", "US-ND", "US-MP", "US-OH", "US-OK", "US-OR", "US-PA", "US-PR", "US-RI", "US-SC", "US-SD", "US-TN", "US-TX", "US-UM", "US-UT", "US-VT", "US-VI", "US-VA", "US-WA", "US-WV", "US-WI", "US-WY", "GB", "VN", "WF", "CX", "BY", "EH", "CF", "CY"]
       },
       "address_type": {
         "description": "Type of customer, emtpy = any.",

src/pretix/testutils/mock.py

@@ -21,6 +21,7 @@
 #
 from contextlib import contextmanager
 
+import fakeredis
 from pytest_mock import MockFixture
 
 
@@ -34,3 +35,7 @@ def mocker_context():
     result = MockFixture(FakePytestConfig())
     yield result
     result.stopall()
+
+
+def get_redis_connection(alias="default", write=True):
+    return fakeredis.FakeStrictRedis(server=fakeredis.FakeServer.get_server("127.0.0.1:None:v(7, 0)", (7, 0)))

src/setup.cfg

@@ -37,6 +37,7 @@ filterwarnings =
     ignore::ResourceWarning
     ignore:django.contrib.staticfiles.templatetags.static:DeprecationWarning
     ignore::DeprecationWarning:compressor
+    ignore:.*FakeStrictRedis.hmset.*:DeprecationWarning:
     ignore:pkg_resources is deprecated as an API:
     ignore:.*pkg_resources.declare_namespace.*:
 

src/tests/api/test_orders.py

@@ -1794,6 +1794,8 @@ def test_pdf_data(token_client, organizer, event, order, django_assert_max_num_q
     ))
     assert resp.status_code == 200
     assert resp.data['positions'][0].get('pdf_data')
+    assert resp.data['positions'][0]['pdf_data']['positionid'] == '1'
+    assert resp.data['positions'][0]['pdf_data']['order'] == order.code
     resp = token_client.get('/api/v1/organizers/{}/events/{}/orders/{}/'.format(
         organizer.slug, event.slug, order.code
     ))
@@ -1807,6 +1809,8 @@ def test_pdf_data(token_client, organizer, event, order, django_assert_max_num_q
         ))
     assert resp.status_code == 200
     assert resp.data['results'][0]['positions'][0].get('pdf_data')
+    assert resp.data['results'][0]['positions'][0]['pdf_data']['positionid'] == '1'
+    assert resp.data['results'][0]['positions'][0]['pdf_data']['order'] == order.code
     resp = token_client.get('/api/v1/organizers/{}/events/{}/orders/'.format(
         organizer.slug, event.slug
     ))
@@ -1820,6 +1824,8 @@ def test_pdf_data(token_client, organizer, event, order, django_assert_max_num_q
         ))
     assert resp.status_code == 200
     assert resp.data['results'][0].get('pdf_data')
+    assert resp.data['results'][0]['pdf_data']['positionid'] == '1'
+    assert resp.data['results'][0]['pdf_data']['order'] == order.code
     resp = token_client.get('/api/v1/organizers/{}/events/{}/orderpositions/'.format(
         organizer.slug, event.slug
     ))
@@ -1834,6 +1840,8 @@ def test_pdf_data(token_client, organizer, event, order, django_assert_max_num_q
     ))
     assert resp.status_code == 200
     assert resp.data.get('pdf_data')
+    assert resp.data['pdf_data']['positionid'] == '1'
+    assert resp.data['pdf_data']['order'] == order.code
     resp = token_client.get('/api/v1/organizers/{}/events/{}/orderpositions/{}/'.format(
         organizer.slug, event.slug, posid
     ))

src/tests/api/test_sendmail.py

@@ -39,7 +39,8 @@ TEST_RULE_RES = {
     'template': {'en': 'foo'},
     'all_products': True,
     'limit_products': [],
-    "restrict_to_status": ['p', 'n__valid_if_pending'],
+    'restrict_to_status': ['p', 'n__valid_if_pending'],
+    'checked_in_status': None,
     'send_date': '2021-07-08T00:00:00Z',
     'send_offset_days': None,
     'send_offset_time': None,
@@ -160,7 +161,8 @@ def test_sendmail_rule_create_full(token_client, organizer, event, item):
             'template': {'en': 'foobar'},
             'all_products': False,
             'limit_products': [event.items.first().pk],
-            "restrict_to_status": ['p', 'n__not_pending_approval_and_not_valid_if_pending', 'n__valid_if_pending'],
+            'restrict_to_status': ['p', 'n__not_pending_approval_and_not_valid_if_pending', 'n__valid_if_pending'],
+            'checked_in_status': None,
             'send_offset_days': 3,
             'send_offset_time': '09:30',
             'date_is_absolute': False,
@@ -174,6 +176,7 @@ def test_sendmail_rule_create_full(token_client, organizer, event, item):
     assert r.all_products is False
     assert [i.pk for i in r.limit_products.all()] == [event.items.first().pk]
     assert r.restrict_to_status == ['p', 'n__not_pending_approval_and_not_valid_if_pending', 'n__valid_if_pending']
+    assert r.checked_in_status is None
     assert r.send_offset_days == 3
     assert r.send_offset_time == datetime.time(9, 30)
     assert r.date_is_absolute is False
@@ -348,6 +351,49 @@ def test_sendmail_rule_restrict_recipients(token_client, organizer, event, rule)
     )
 
 
+@scopes_disabled()
+@pytest.mark.django_db
+def test_sendmail_rule_checkin(token_client, organizer, event, rule):
+    valid_states = [None, 'checked_in', 'no_checkin', ]
+    invalid_states = ['', 'foo']
+
+    for s in valid_states:
+        result = create_rule(
+            token_client, organizer, event,
+            data={
+                'subject': {'en': 'meow'},
+                'template': {'en': 'creative text here'},
+                'send_date': '2018-03-17T13:31Z',
+                'checked_in_status': s,
+            },
+            expected_failure=False
+        )
+        assert result.checked_in_status == s
+
+    for s in invalid_states:
+        create_rule(
+            token_client, organizer, event,
+            data={
+                'subject': {'en': 'meow'},
+                'template': {'en': 'creative text here'},
+                'send_date': '2018-03-17T13:31Z',
+                'checked_in_status': s,
+            },
+            expected_failure=True
+        )
+
+    result = create_rule(
+        token_client, organizer, event,
+        data={
+            'subject': {'en': 'meow'},
+            'template': {'en': 'creative text here'},
+            'send_date': '2018-03-17T13:31Z',
+        },
+        expected_failure=False
+    )
+    assert result.checked_in_status is None
+
+
 @scopes_disabled()
 @pytest.mark.django_db
 def test_sendmail_rule_change(token_client, organizer, event, rule):

src/tests/base/test_models.py

@@ -98,6 +98,7 @@ class BaseQuotaTestCase(TestCase):
         self.var3 = ItemVariation.objects.create(item=self.item3, value='Fancy')
 
 
+@pytest.mark.usefixtures("fakeredis_client")
 class QuotaTestCase(BaseQuotaTestCase):
     @classscope(attr='o')
     def test_available(self):
@@ -434,6 +435,62 @@ class QuotaTestCase(BaseQuotaTestCase):
         self.assertEqual(self.var1.check_quotas(), (Quota.AVAILABILITY_ORDERED, 0))
         self.assertEqual(self.var1.check_quotas(count_waitinglist=False), (Quota.AVAILABILITY_OK, 1))
 
+    @classscope(attr='o')
+    def test_waitinglist_cache_separation(self):
+        self.quota.items.add(self.item1)
+        self.quota.size = 1
+        self.quota.save()
+        WaitingListEntry.objects.create(
+            event=self.event, item=self.item1, email='foo@bar.com'
+        )
+
+        # Check that there is no "cache mixup" even across multiple runs
+        qa = QuotaAvailability(count_waitinglist=False)
+        qa.queue(self.quota)
+        qa.compute()
+        assert qa.results[self.quota] == (Quota.AVAILABILITY_OK, 1)
+
+        qa = QuotaAvailability(count_waitinglist=True)
+        qa.queue(self.quota)
+        qa.compute(allow_cache=True)
+        assert qa.results[self.quota] == (Quota.AVAILABILITY_ORDERED, 0)
+
+        qa = QuotaAvailability(count_waitinglist=True)
+        qa.queue(self.quota)
+        qa.compute(allow_cache=True)
+        assert qa.results[self.quota] == (Quota.AVAILABILITY_ORDERED, 0)
+
+        qa = QuotaAvailability(count_waitinglist=False)
+        qa.queue(self.quota)
+        qa.compute()
+        assert qa.results[self.quota] == (Quota.AVAILABILITY_OK, 1)
+
+        # Rebuild cache required
+        self.quota.size = 5
+        self.quota.save()
+
+        qa = QuotaAvailability(count_waitinglist=True)
+        qa.queue(self.quota)
+        qa.compute(allow_cache=True)
+        assert qa.results[self.quota] == (Quota.AVAILABILITY_ORDERED, 0)
+
+        qa = QuotaAvailability(count_waitinglist=False)
+        qa.queue(self.quota)
+        qa.compute(allow_cache=True)
+        assert qa.results[self.quota] == (Quota.AVAILABILITY_OK, 1)
+
+        self.quota.rebuild_cache()
+
+        qa = QuotaAvailability(count_waitinglist=True)
+        qa.queue(self.quota)
+        qa.compute(allow_cache=True)
+        assert qa.results[self.quota] == (Quota.AVAILABILITY_OK, 4)
+
+        qa = QuotaAvailability(count_waitinglist=False)
+        qa.queue(self.quota)
+        qa.compute(allow_cache=True)
+        assert qa.results[self.quota] == (Quota.AVAILABILITY_OK, 5)
+
     @classscope(attr='o')
     def test_waitinglist_variation_fulfilled(self):
         self.quota.variations.add(self.var1)

src/tests/base/test_payment.py

@@ -97,7 +97,15 @@ def test_payment_fee_reverse_percent_and_abs_default(event):
 def test_availability_date_available(event):
     prov = DummyPaymentProvider(event)
     prov.settings.set('_availability_date', datetime.date.today() + datetime.timedelta(days=1))
-    result = prov._is_still_available()
+    result = prov._is_available_by_time()
+    assert result
+
+
+@pytest.mark.django_db
+def test_availability_start_available(event):
+    prov = DummyPaymentProvider(event)
+    prov.settings.set('_availability_start', datetime.date.today() - datetime.timedelta(days=1))
+    result = prov._is_available_by_time()
     assert result
 
 
@@ -105,7 +113,15 @@ def test_availability_date_available(event):
 def test_availability_date_not_available(event):
     prov = DummyPaymentProvider(event)
     prov.settings.set('_availability_date', datetime.date.today() - datetime.timedelta(days=1))
-    result = prov._is_still_available()
+    result = prov._is_available_by_time()
+    assert not result
+
+
+@pytest.mark.django_db
+def test_availability_start_not_available(event):
+    prov = DummyPaymentProvider(event)
+    prov.settings.set('_availability_start', datetime.date.today() + datetime.timedelta(days=1))
+    result = prov._is_available_by_time()
     assert not result
 
 
@@ -121,9 +137,26 @@ def test_availability_date_relative(event):
     ))
 
     utc = datetime.timezone.utc
-    assert prov._is_still_available(datetime.datetime(2016, 11, 30, 23, 0, 0, tzinfo=tz).astimezone(utc))
-    assert prov._is_still_available(datetime.datetime(2016, 12, 1, 23, 59, 0, tzinfo=tz).astimezone(utc))
-    assert not prov._is_still_available(datetime.datetime(2016, 12, 2, 0, 0, 1, tzinfo=tz).astimezone(utc))
+    assert prov._is_available_by_time(datetime.datetime(2016, 11, 30, 23, 0, 0, tzinfo=tz).astimezone(utc))
+    assert prov._is_available_by_time(datetime.datetime(2016, 12, 1, 23, 59, 0, tzinfo=tz).astimezone(utc))
+    assert not prov._is_available_by_time(datetime.datetime(2016, 12, 2, 0, 0, 1, tzinfo=tz).astimezone(utc))
+
+
+@pytest.mark.django_db
+def test_availability_start_relative(event):
+    event.settings.set('timezone', 'US/Pacific')
+    tz = ZoneInfo('US/Pacific')
+    event.date_from = datetime.datetime(2016, 12, 3, 12, 0, 0, tzinfo=tz)
+    event.save()
+    prov = DummyPaymentProvider(event)
+    prov.settings.set('_availability_start', RelativeDateWrapper(
+        RelativeDate(days_before=2, time=datetime.time(12, 0), base_date_name='date_from', minutes_before=None)
+    ))
+
+    utc = datetime.timezone.utc
+    assert not prov._is_available_by_time(datetime.datetime(2016, 11, 30, 23, 0, 0, tzinfo=tz).astimezone(utc))
+    assert prov._is_available_by_time(datetime.datetime(2016, 12, 1, 0, 0, tzinfo=tz).astimezone(utc))
+    assert prov._is_available_by_time(datetime.datetime(2016, 12, 2, 0, 0, 1, tzinfo=tz).astimezone(utc))
 
 
 @pytest.mark.django_db
@@ -134,9 +167,9 @@ def test_availability_date_timezones(event):
 
     tz = ZoneInfo('US/Pacific')
     utc = ZoneInfo('UTC')
-    assert prov._is_still_available(datetime.datetime(2016, 11, 30, 23, 0, 0, tzinfo=tz).astimezone(utc))
-    assert prov._is_still_available(datetime.datetime(2016, 12, 1, 23, 59, 0, tzinfo=tz).astimezone(utc))
-    assert not prov._is_still_available(datetime.datetime(2016, 12, 2, 0, 0, 1, tzinfo=tz).astimezone(utc))
+    assert prov._is_available_by_time(datetime.datetime(2016, 11, 30, 23, 0, 0, tzinfo=tz).astimezone(utc))
+    assert prov._is_available_by_time(datetime.datetime(2016, 12, 1, 23, 59, 0, tzinfo=tz).astimezone(utc))
+    assert not prov._is_available_by_time(datetime.datetime(2016, 12, 2, 0, 0, 1, tzinfo=tz).astimezone(utc))
 
 
 @pytest.mark.django_db
@@ -162,12 +195,12 @@ def test_availability_date_cart_relative_subevents(event):
     prov.settings.set('_availability_date', RelativeDateWrapper(
         RelativeDate(days_before=3, time=None, base_date_name='date_from', minutes_before=None)
     ))
-    assert prov._is_still_available(cart_id="123")
+    assert prov._is_available_by_time(cart_id="123")
 
     prov.settings.set('_availability_date', RelativeDateWrapper(
         RelativeDate(days_before=4, time=None, base_date_name='date_from', minutes_before=None)
     ))
-    assert not prov._is_still_available(cart_id="123")
+    assert not prov._is_available_by_time(cart_id="123")
 
 
 @pytest.mark.django_db
@@ -201,9 +234,9 @@ def test_availability_date_order_relative_subevents(event):
     prov.settings.set('_availability_date', RelativeDateWrapper(
         RelativeDate(days_before=3, time=None, base_date_name='date_from', minutes_before=None)
     ))
-    assert prov._is_still_available(order=order)
+    assert prov._is_available_by_time(order=order)
 
     prov.settings.set('_availability_date', RelativeDateWrapper(
         RelativeDate(days_before=4, time=None, base_date_name='date_from', minutes_before=None)
     ))
-    assert not prov._is_still_available(order=order)
+    assert not prov._is_available_by_time(order=order)

src/tests/base/test_taxrules.py

@@ -477,6 +477,81 @@ def test_custom_rules_specific_country(event):
     )
 
 
+@pytest.mark.django_db
+def test_custom_rules_specific_state(event):
+    tr = TaxRule(
+        event=event,
+        rate=Decimal('10.00'), price_includes_tax=False,
+        custom_rules=json.dumps([
+            {'country': 'US-NY', 'address_type': '', 'action': 'vat', 'rate': '20.00'},
+            {'country': 'US-DE', 'address_type': '', 'action': 'no'},
+            {'country': 'US', 'address_type': '', 'action': 'vat', 'rate': '30.00'},
+        ])
+    )
+    ia = InvoiceAddress(
+        is_business=True,
+        country=Country('DE')
+    )
+    assert not tr.is_reverse_charge(ia)
+    assert tr._tax_applicable(ia)
+    assert tr.tax_rate_for(ia) == Decimal('10.00')
+    assert tr.tax(Decimal('100.00'), invoice_address=ia) == TaxedPrice(
+        gross=Decimal('110.00'),
+        net=Decimal('100.00'),
+        tax=Decimal('10.00'),
+        rate=Decimal('10.00'),
+        name='',
+    )
+
+    ia = InvoiceAddress(
+        is_business=True,
+        country=Country('US'),
+        state='NC'
+    )
+    assert not tr.is_reverse_charge(ia)
+    assert tr._tax_applicable(ia)
+    assert tr.tax_rate_for(ia) == Decimal('30.00')
+    assert tr.tax(Decimal('100.00'), invoice_address=ia) == TaxedPrice(
+        gross=Decimal('130.00'),
+        net=Decimal('100.00'),
+        tax=Decimal('30.00'),
+        rate=Decimal('30.00'),
+        name='',
+    )
+
+    ia = InvoiceAddress(
+        is_business=True,
+        country=Country('US'),
+        state='NY'
+    )
+    assert not tr.is_reverse_charge(ia)
+    assert tr._tax_applicable(ia)
+    assert tr.tax_rate_for(ia) == Decimal('20.00')
+    assert tr.tax(Decimal('100.00'), invoice_address=ia) == TaxedPrice(
+        gross=Decimal('120.00'),
+        net=Decimal('100.00'),
+        tax=Decimal('20.00'),
+        rate=Decimal('20.00'),
+        name='',
+    )
+
+    ia = InvoiceAddress(
+        is_business=True,
+        country=Country('US'),
+        state='DE'
+    )
+    assert not tr.is_reverse_charge(ia)
+    assert not tr._tax_applicable(ia)
+    assert tr.tax_rate_for(ia) == Decimal('0.00')
+    assert tr.tax(Decimal('100.00'), invoice_address=ia) == TaxedPrice(
+        gross=Decimal('100.00'),
+        net=Decimal('100.00'),
+        tax=Decimal('0.00'),
+        rate=Decimal('0.00'),
+        name='',
+    )
+
+
 @pytest.mark.django_db
 def test_custom_rules_individual(event):
     tr = TaxRule(

src/tests/conftest.py

@@ -22,10 +22,14 @@
 import inspect
 
 import pytest
+from django.test import override_settings
 from django.utils import translation
 from django_scopes import scopes_disabled
+from fakeredis import FakeConnection
 from xdist.dsession import DSession
 
+from pretix.testutils.mock import get_redis_connection
+
 CRASHED_ITEMS = set()
 
 
@@ -74,3 +78,38 @@ def pytest_fixture_setup(fixturedef, request):
 @pytest.fixture(autouse=True)
 def reset_locale():
     translation.activate("en")
+
+
+@pytest.fixture
+def fakeredis_client(monkeypatch):
+    with override_settings(
+        HAS_REDIS=True,
+        REAL_CACHE_USED=True,
+        CACHES={
+            'redis': {
+                'BACKEND': 'django.core.cache.backends.redis.RedisCache',
+                'LOCATION': 'redis://127.0.0.1',
+                'OPTIONS': {
+                    'connection_class': FakeConnection
+                }
+            },
+            'redis_session': {
+                'BACKEND': 'django.core.cache.backends.redis.RedisCache',
+                'LOCATION': 'redis://127.0.0.1',
+                'OPTIONS': {
+                    'connection_class': FakeConnection
+                }
+            },
+            'default': {
+                'BACKEND': 'django.core.cache.backends.redis.RedisCache',
+                'LOCATION': 'redis://127.0.0.1',
+                'OPTIONS': {
+                    'connection_class': FakeConnection
+                }
+            },
+        }
+    ):
+        redis = get_redis_connection("default", True)
+        redis.flushall()
+        monkeypatch.setattr('django_redis.get_redis_connection', get_redis_connection, raising=False)
+        yield redis

src/tests/control/test_items.py

@@ -640,7 +640,10 @@ class ItemsTest(ItemFormTest):
         prop = self.event1.item_meta_properties.create(name="Foo")
         self.item2.meta_values.create(property=prop, value="Bar")
 
+        doc = self.get_doc('/control/event/%s/%s/items/add?copy_from=%d' % (self.orga1.slug, self.event1.slug, self.item2.pk))
+        data = extract_form_fields(doc.select("form")[0])
         self.client.post('/control/event/%s/%s/items/add' % (self.orga1.slug, self.event1.slug), {
+            **data,
             'name_0': 'Intermediate',
             'default_price': '23.00',
             'tax_rate': '19.00',

src/tests/plugins/sendmail/test_rules.py

@@ -29,6 +29,7 @@ from django.utils.timezone import now
 from django_scopes import scopes_disabled
 
 from pretix.base.models import InvoiceAddress, Order
+from pretix.base.services.checkin import perform_checkin
 from pretix.plugins.sendmail.models import Rule, ScheduledMail
 from pretix.plugins.sendmail.signals import sendmail_run_rules
 
@@ -276,6 +277,100 @@ def test_sendmail_rule_send_correct_products(event, order, item, item2):
     assert djmail.outbox[0].to[0] == p1.attendee_email
 
 
+@pytest.mark.django_db
+@scopes_disabled()
+def test_sendmail_rule_not_checked_in_all_get_mail(event, order, item):
+    order.status = Order.STATUS_PAID
+    order.save()
+    event.sendmail_rules.create(send_date=dt_now - datetime.timedelta(hours=1), checked_in_status="all",
+                                subject='meow', template='meow meow meow')
+
+    djmail.outbox = []
+    sendmail_run_rules(None)
+    assert len(djmail.outbox) == 1, "email not sent"
+
+
+@pytest.mark.django_db
+@scopes_disabled()
+def test_sendmail_rule_checked_in_all_get_mail(event, order, item):
+    order.status = Order.STATUS_PAID
+    order.save()
+    p1 = order.all_positions.create(item=item, price=13, attendee_email='item1@dummy.test')
+    clist = event.checkin_lists.create(name="Default", all_products=True)
+    perform_checkin(p1, clist, {})
+    event.sendmail_rules.create(send_date=dt_now - datetime.timedelta(hours=1), checked_in_status="all",
+                                subject='meow', template='meow meow meow')
+
+    djmail.outbox = []
+    sendmail_run_rules(None)
+    assert len(djmail.outbox) == 1, "email not sent"
+
+
+@pytest.mark.django_db
+@scopes_disabled()
+def test_sendmail_rule_not_checked_in_no_mail(event, order, item):
+    order.status = Order.STATUS_PAID
+    order.save()
+    event.sendmail_rules.create(send_date=dt_now - datetime.timedelta(hours=1), checked_in_status="checked_in",
+                                subject='meow', template='meow meow meow')
+
+    # receives no mail when not checked in
+    djmail.outbox = []
+    sendmail_run_rules(None)
+    assert len(djmail.outbox) == 0, "email sent unexpectedly"
+
+
+@pytest.mark.django_db
+@scopes_disabled()
+def test_sendmail_rule_not_checked_in_get_mail(event, order, item):
+    order.status = Order.STATUS_PAID
+    order.save()
+    order.all_positions.create(item=item, price=13, attendee_email='item1@dummy.test')
+    event.sendmail_rules.create(send_date=dt_now - datetime.timedelta(hours=1), checked_in_status="no_checkin",
+                                subject='meow', template='meow meow meow')
+
+    # receives mail when not checked in
+    djmail.outbox = []
+    sendmail_run_rules(None)
+    assert len(djmail.outbox) == 1, "email not sent"
+
+
+@pytest.mark.django_db
+@scopes_disabled()
+def test_sendmail_rule_checked_in_no_mail(event, order, item):
+    order.status = Order.STATUS_PAID
+    order.save()
+    p1 = order.all_positions.create(item=item, price=13, attendee_email='item1@dummy.test')
+    clist = event.checkin_lists.create(name="Default", all_products=True)
+
+    # receives no mail when checked in
+    djmail.outbox = []
+    perform_checkin(p1, clist, {})
+    assert clist.checkin_count == 1
+    event.sendmail_rules.create(send_date=dt_now - datetime.timedelta(hours=1), checked_in_status="no_checkin",
+                                subject='meow', template='meow meow meow')
+    sendmail_run_rules(None)
+    assert len(djmail.outbox) == 0, "email sent unexpectedly"
+
+
+@pytest.mark.django_db
+@scopes_disabled()
+def test_sendmail_rule_checked_in_get_mail(event, order, item):
+    order.status = Order.STATUS_PAID
+    order.save()
+    p1 = order.all_positions.create(item=item, price=13, attendee_email='item1@dummy.test')
+    clist = event.checkin_lists.create(name="Default", all_products=True)
+
+    # receives mail when checked in
+    djmail.outbox = []
+    perform_checkin(p1, clist, {})
+    assert clist.checkin_count == 1
+    event.sendmail_rules.create(send_date=dt_now - datetime.timedelta(hours=1), checked_in_status="checked_in",
+                                subject='meow', template='meow meow meow')
+    sendmail_run_rules(None)
+    assert len(djmail.outbox) == 1, "email not sent"
+
+
 @pytest.mark.django_db
 @scopes_disabled()
 def run_restriction_test(event, order, restrictions_pass=[], restrictions_fail=[]):

src/tests/presale/test_cart.py

@@ -2414,6 +2414,25 @@ class CartAddonTest(CartTestMixin, TestCase):
         assert cp2.item == self.workshop1
         assert cp2.price == 0
 
+    @classscope(attr='orga')
+    def test_extend_included_addon_no_longer_available(self):
+        self.addon1.price_included = True
+        self.addon1.save()
+        self.quota_tickets.size = 0
+        self.quota_tickets.save()
+        cp1 = CartPosition.objects.create(
+            expires=now() - timedelta(minutes=10), item=self.ticket, price=Decimal('23.00'),
+            event=self.event, cart_id=self.session_key
+        )
+        CartPosition.objects.create(
+            expires=now() - timedelta(minutes=10), item=self.workshop1, price=Decimal('0.00'),
+            event=self.event, cart_id=self.session_key, addon_to=cp1
+        )
+        self.cm.extend_expired_positions()
+        with self.assertRaises(CartError):
+            self.cm.commit()
+        assert CartPosition.objects.count() == 0
+
     @classscope(attr='orga')
     def test_cart_addon_remove_parent(self):
         self.addon1.price_included = True