Bump celery to 5.3

also fixes #3070

doc/api/resources/taxrules.rst

@@ -20,16 +20,11 @@ internal_name                         string                     An optional nam
 rate                                  decimal (string)           Tax rate in percent
 price_includes_tax                    boolean                    If ``true`` (default), tax is assumed to be included in
                                                                  the specified product price
-eu_reverse_charge                     boolean                    If ``true``, EU reverse charge rules are applied. Will
-                                                                 be ignored if custom rules are set.
+eu_reverse_charge                     boolean                    If ``true``, EU reverse charge rules are applied
 home_country                          string                     Merchant country (required for reverse charge), can be
                                                                  ``null`` or empty string
 keep_gross_if_rate_changes            boolean                    If ``true``, changes of the tax rate based on custom
                                                                  rules keep the gross price constant (default is ``false``)
-custom_rules                          object                     Dynamic rules specification. Each list element
-                                                                 corresponds to one rule that will be processed in order.
-                                                                 The current version of the schema in use can be found
-                                                                 `here`_.
 ===================================== ========================== =======================================================
 
 
@@ -37,10 +32,6 @@ custom_rules                          object                     Dynamic rules s
 
     The ``internal_name`` and ``keep_gross_if_rate_changes`` attributes have been added.
 
-.. versionchanged:: 2023.6
-
-    The ``custom_rules`` attribute has been added.
-
 Endpoints
 ---------
 
@@ -77,7 +68,6 @@ Endpoints
             "price_includes_tax": true,
             "eu_reverse_charge": false,
             "keep_gross_if_rate_changes": false,
-            "custom_rules": null,
             "home_country": "DE"
           }
         ]
@@ -118,7 +108,6 @@ Endpoints
         "price_includes_tax": true,
         "eu_reverse_charge": false,
         "keep_gross_if_rate_changes": false,
-        "custom_rules": null,
         "home_country": "DE"
       }
 
@@ -167,7 +156,6 @@ Endpoints
         "price_includes_tax": true,
         "eu_reverse_charge": false,
         "keep_gross_if_rate_changes": false,
-        "custom_rules": null,
         "home_country": "DE"
       }
 
@@ -215,7 +203,6 @@ Endpoints
         "price_includes_tax": true,
         "eu_reverse_charge": false,
         "keep_gross_if_rate_changes": false,
-        "custom_rules": null,
         "home_country": "DE"
       }
 
@@ -255,5 +242,3 @@ Endpoints
    :statuscode 204: no error
    :statuscode 401: Authentication failure
    :statuscode 403: The requested organizer/event/rule does not exist **or** you have no permission to change it **or** this tax rule cannot be deleted since it is currently in use.
-
-.. _here: https://github.com/pretix/pretix/blob/master/src/pretix/static/schema/tax-rules-custom.schema.json

pyproject.toml

@@ -30,7 +30,7 @@ dependencies = [
   "babel",
   "BeautifulSoup4==4.12.*",
   "bleach==5.0.*",
-  "celery==5.2.*",
+  "celery==5.3.*",
   "chardet==5.1.*",
   "cryptography>=3.4.2",
   "css-inline==0.8.*",
@@ -62,7 +62,7 @@ dependencies = [
   "importlib_metadata==6.6.*",  # Polyfill, we can probably drop this once we require Python 3.10+
   "isoweek",
   "jsonschema",
-  "kombu==5.2.*",
+  "kombu==5.3.*",
   "libsass==0.22.*",
   "lxml",
   "markdown==3.4.3",  # 3.3.5 requires importlib-metadata>=4.4, but django-bootstrap3 requires importlib-metadata<3.

src/pretix/api/serializers/__init__.py

@@ -19,8 +19,6 @@
 # You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
 # <https://www.gnu.org/licenses/>.
 #
-import json
-
 from rest_framework import serializers
 
 
@@ -48,16 +46,3 @@ class AsymmetricField(serializers.Field):
 
     def run_validation(self, data=serializers.empty):
         return self.write.run_validation(data)
-
-
-class CompatibleJSONField(serializers.JSONField):
-    def to_internal_value(self, data):
-        try:
-            return json.dumps(data)
-        except (TypeError, ValueError):
-            self.fail('invalid')
-
-    def to_representation(self, value):
-        if value:
-            return json.loads(value)
-        return value

src/pretix/api/serializers/event.py

@@ -46,7 +46,6 @@ from rest_framework import serializers
 from rest_framework.fields import ChoiceField, Field
 from rest_framework.relations import SlugRelatedField
 
-from pretix.api.serializers import CompatibleJSONField
 from pretix.api.serializers.i18n import I18nAwareModelSerializer
 from pretix.api.serializers.settings import SettingsSerializer
 from pretix.base.models import Device, Event, TaxRule, TeamAPIToken
@@ -54,7 +53,6 @@ from pretix.base.models.event import SubEvent
 from pretix.base.models.items import (
     ItemMetaProperty, SubEventItem, SubEventItemVariation,
 )
-from pretix.base.models.tax import CustomRulesValidator
 from pretix.base.services.seating import (
     SeatProtected, generate_seats, validate_plan_change,
 )
@@ -652,16 +650,9 @@ class SubEventSerializer(I18nAwareModelSerializer):
 
 
 class TaxRuleSerializer(CountryFieldMixin, I18nAwareModelSerializer):
-    custom_rules = CompatibleJSONField(
-        validators=[CustomRulesValidator()],
-        required=False,
-        allow_null=True,
-    )
-
     class Meta:
         model = TaxRule
-        fields = ('id', 'name', 'rate', 'price_includes_tax', 'eu_reverse_charge', 'home_country', 'internal_name',
-                  'keep_gross_if_rate_changes', 'custom_rules')
+        fields = ('id', 'name', 'rate', 'price_includes_tax', 'eu_reverse_charge', 'home_country', 'internal_name', 'keep_gross_if_rate_changes')
 
 
 class EventSettingsSerializer(SettingsSerializer):

src/pretix/api/serializers/order.py

@@ -19,6 +19,7 @@
 # You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
 # <https://www.gnu.org/licenses/>.
 #
+import json
 import logging
 import os
 from collections import Counter, defaultdict
@@ -38,7 +39,6 @@ from rest_framework.exceptions import ValidationError
 from rest_framework.relations import SlugRelatedField
 from rest_framework.reverse import reverse
 
-from pretix.api.serializers import CompatibleJSONField
 from pretix.api.serializers.event import SubEventSerializer
 from pretix.api.serializers.i18n import I18nAwareModelSerializer
 from pretix.api.serializers.item import (
@@ -896,6 +896,19 @@ class OrderPositionCreateSerializer(I18nAwareModelSerializer):
         return data
 
 
+class CompatibleJSONField(serializers.JSONField):
+    def to_internal_value(self, data):
+        try:
+            return json.dumps(data)
+        except (TypeError, ValueError):
+            self.fail('invalid')
+
+    def to_representation(self, value):
+        if value:
+            return json.loads(value)
+        return value
+
+
 class WrappedList:
     def __init__(self, data):
         self._data = data

src/pretix/base/models/event.py

@@ -1277,6 +1277,9 @@ class Event(EventMixin, LoggedModel):
         return not self.orders.exists() and not self.invoices.exists()
 
     def delete_sub_objects(self):
+        from .checkin import Checkin
+
+        Checkin.all.filter(successful=False, list__event=self).delete()
         self.cartposition_set.filter(addon_to__isnull=False).delete()
         self.cartposition_set.all().delete()
         self.vouchers.all().delete()

src/pretix/base/models/tax.py

@@ -22,12 +22,9 @@
 import json
 from decimal import Decimal
 
-import jsonschema
-from django.contrib.staticfiles import finders
 from django.core.exceptions import ValidationError
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
-from django.utils.deconstruct import deconstructible
 from django.utils.formats import localize
 from django.utils.translation import gettext_lazy as _, pgettext
 from i18nfield.fields import I18nCharField
@@ -138,25 +135,6 @@ def cc_to_vat_prefix(country_code):
     return country_code
 
 
-@deconstructible
-class CustomRulesValidator:
-    def __call__(self, value):
-        if not isinstance(value, dict):
-            try:
-                val = json.loads(value)
-            except ValueError:
-                raise ValidationError(_('Your layout file is not a valid JSON file.'))
-        else:
-            val = value
-        with open(finders.find('schema/tax-rules-custom.schema.json'), 'r') as f:
-            schema = json.loads(f.read())
-        try:
-            jsonschema.validate(val, schema)
-        except jsonschema.ValidationError as e:
-            e = str(e).replace('%', '%%')
-            raise ValidationError(_('Your set of rules is not valid. Error message: {}').format(e))
-
-
 class TaxRule(LoggedModel):
     event = models.ForeignKey('Event', related_name='tax_rules', on_delete=models.CASCADE)
     internal_name = models.CharField(

src/pretix/base/payment.py

@@ -60,7 +60,7 @@ from pretix.base.channels import get_all_sales_channels
 from pretix.base.forms import PlaceholderValidator
 from pretix.base.models import (
     CartPosition, Event, GiftCard, InvoiceAddress, Order, OrderPayment,
-    OrderRefund, Quota,
+    OrderRefund, Quota, TaxRule,
 )
 from pretix.base.reldate import RelativeDateField, RelativeDateWrapper
 from pretix.base.settings import SettingsSandbox
@@ -1015,7 +1015,11 @@ class FreeOrderProvider(BasePaymentProvider):
 
         cart = get_cart(request)
         total = get_cart_total(request)
-        total += sum([f.value for f in get_fees(self.event, request, total, None, None, cart)])
+        try:
+            total += sum([f.value for f in get_fees(self.event, request, total, None, None, cart)])
+        except TaxRule.SaleNotAllowed:
+            # ignore for now, will fail on order creation
+            pass
         return total == 0
 
     def order_change_allowed(self, order: Order) -> bool:

src/pretix/base/pdf.py

@@ -860,22 +860,32 @@ class Renderer:
                 image_file = None
 
         if image_file:
-            ir = ThumbnailingImageReader(image_file)
             try:
+                ir = ThumbnailingImageReader(image_file)
                 ir.resize(float(o['width']) * mm, float(o['height']) * mm, 300)
+                canvas.drawImage(
+                    image=ir,
+                    x=float(o['left']) * mm,
+                    y=float(o['bottom']) * mm,
+                    width=float(o['width']) * mm,
+                    height=float(o['height']) * mm,
+                    preserveAspectRatio=True,
+                    anchor='c',  # centered in frame
+                    mask='auto'
+                )
             except:
-                logger.exception("Can not resize image")
-                pass
-            canvas.drawImage(
-                image=ir,
-                x=float(o['left']) * mm,
-                y=float(o['bottom']) * mm,
-                width=float(o['width']) * mm,
-                height=float(o['height']) * mm,
-                preserveAspectRatio=True,
-                anchor='c',  # centered in frame
-                mask='auto'
-            )
+                logger.exception("Can not load or resize image")
+                canvas.saveState()
+                canvas.setFillColorRGB(.8, .8, .8, alpha=1)
+                canvas.rect(
+                    x=float(o['left']) * mm,
+                    y=float(o['bottom']) * mm,
+                    width=float(o['width']) * mm,
+                    height=float(o['height']) * mm,
+                    stroke=0,
+                    fill=1,
+                )
+                canvas.restoreState()
         else:
             canvas.saveState()
             canvas.setFillColorRGB(.8, .8, .8, alpha=1)

src/pretix/base/services/cart.py

@@ -317,6 +317,9 @@ class CartManager:
     def _delete_out_of_timeframe(self):
         err = None
         for cp in self.positions:
+            if not cp.pk:
+                continue
+
             if cp.subevent and cp.subevent.presale_start and self.now_dt < cp.subevent.presale_start:
                 err = error_messages['some_subevent_not_started']
                 cp.addons.all().delete()

src/pretix/base/services/orders.py

@@ -935,7 +935,10 @@ def _create_order(event: Event, email: str, positions: List[CartPosition], now_d
             raise OrderError(e.message)
 
         require_approval = any(p.requires_approval(invoice_address=address) for p in positions)
-        fees = _get_fees(positions, payment_requests, address, meta_info, event, require_approval=require_approval)
+        try:
+            fees = _get_fees(positions, payment_requests, address, meta_info, event, require_approval=require_approval)
+        except TaxRule.SaleNotAllowed:
+            raise OrderError(error_messages['country_blocked'])
         total = pending_sum = sum([c.price for c in positions]) + sum([c.value for c in fees])
 
         order = Order(
@@ -968,7 +971,10 @@ def _create_order(event: Event, email: str, positions: List[CartPosition], now_d
 
         for fee in fees:
             fee.order = order
-            fee._calculate_tax()
+            try:
+                fee._calculate_tax()
+            except TaxRule.SaleNotAllowed:
+                raise OrderError(error_messages['country_blocked'])
             if fee.tax_rule and not fee.tax_rule.pk:
                 fee.tax_rule = None  # TODO: deprecate
             fee.save()

src/pretix/base/services/shredder.py

@@ -20,6 +20,7 @@
 # <https://www.gnu.org/licenses/>.
 #
 
+
 # This file is based on an earlier version of pretix which was released under the Apache License 2.0. The full text of
 # the Apache License 2.0 can be obtained at <http://www.apache.org/licenses/LICENSE-2.0>.
 #
@@ -31,7 +32,7 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the Apache License 2.0 is
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under the License.
-
+import inspect
 import json
 from datetime import timedelta
 from tempfile import NamedTemporaryFile
@@ -41,10 +42,13 @@ from zipfile import ZipFile
 from dateutil.parser import parse
 from django.conf import settings
 from django.utils.crypto import get_random_string
+from django.utils.formats import date_format
 from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _
 
-from pretix.base.models import CachedFile, Event, cachedfile_name
+from pretix.base.i18n import language
+from pretix.base.models import CachedFile, Event, User, cachedfile_name
+from pretix.base.services.mail import SendMailException, mail
 from pretix.base.services.tasks import ProfiledEventTask
 from pretix.base.shredder import ShredError
 from pretix.celery_app import app
@@ -101,8 +105,17 @@ def export(event: Event, shredders: List[str], session_key=None, cfid=None) -> N
     return cf.pk
 
 
-@app.task(base=ProfiledEventTask, throws=(ShredError,))
-def shred(event: Event, fileid: str, confirm_code: str) -> None:
+@app.task(base=ProfiledEventTask, throws=(ShredError,), bind=True)
+def shred(self, event: Event, fileid: str, confirm_code: str, user: int=None, locale: str='en') -> None:
+    steps = []
+
+    def set_progress(val):
+        if not self.request.called_directly:
+            self.update_state(
+                state='PROGRESS',
+                meta={'value': val, 'steps': steps}
+            )
+
     known_shredders = event.get_data_shredders()
     try:
         cf = CachedFile.objects.get(pk=fileid)
@@ -124,8 +137,41 @@ def shred(event: Event, fileid: str, confirm_code: str) -> None:
     if event.logentry_set.filter(datetime__gte=parse(indexdata['time'])):
         raise ShredError(_("Something happened in your event after the export, please try again."))
 
-    for shredder in shredders:
-        shredder.shred_data()
+    for i, shredder in enumerate(shredders):
+        with language(locale):
+            steps.append({'label': str(shredder.verbose_name), 'done': False})
+        set_progress(i * 100 / len(shredders))
+        if 'progress_callback' in inspect.signature(shredder.shred_data).parameters:
+            shredder.shred_data(
+                progress_callback=lambda y: set_progress(
+                    i * 100 / len(shredders) + min(max(y, 0), 100) / 100 * 100 / len(shredders)
+                )
+            )
+        else:
+            shredder.shred_data()
+        steps[-1]['done'] = True
 
     cf.file.delete(save=False)
     cf.delete()
+
+    if user:
+        user = User.objects.get(pk=user)
+        with language(user.locale):
+            try:
+                mail(
+                    user.email,
+                    _('Data shredding completed'),
+                    'pretixbase/email/shred_completed.txt',
+                    {
+                        'user': user,
+                        'organizer': event.organizer.name,
+                        'event': str(event.name),
+                        'start_time': date_format(parse(indexdata['time']).astimezone(event.timezone), 'SHORT_DATETIME_FORMAT'),
+                        'shredders': ', '.join([str(s.verbose_name) for s in shredders])
+                    },
+                    event=None,
+                    user=user,
+                    locale=user.locale,
+                )
+            except SendMailException:
+                pass  # Already logged

src/pretix/base/shredder.py

@@ -32,11 +32,12 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under the License.
 
+import copy
 import json
 import os
+import time
 from typing import List, Tuple
 
-from django.db import transaction
 from django.db.models import Max, Q
 from django.db.models.functions import Greatest
 from django.dispatch import receiver
@@ -99,11 +100,13 @@ class BaseDataShredder:
         """
         raise NotImplementedError()  # NOQA
 
-    def shred_data(self):
+    def shred_data(self, progress_callback=None):
         """
         This method is called to actually remove the data from the system. You should remove any database objects
         here.
 
+        You can call ``progress_callback`` with an integer value between 0 and 100 to communicate back your progress.
+
         You should never delete ``LogEntry`` objects, but you might modify them to remove personal data. In this
         case, set the ``LogEntry.shredded`` attribute to ``True`` to show that this is no longer original log data.
         """
@@ -151,6 +154,7 @@ class BaseDataShredder:
 
 def shred_log_fields(logentry, banlist=None, whitelist=None):
     d = logentry.parsed_data
+    initial_data = copy.copy(d)
     shredded = False
     if whitelist:
         for k, v in d.items():
@@ -162,9 +166,61 @@ def shred_log_fields(logentry, banlist=None, whitelist=None):
             if f in d:
                 d[f] = '█'
                 shredded = True
-    logentry.data = json.dumps(d)
-    logentry.shredded = logentry.shredded or shredded
-    logentry.save(update_fields=['data', 'shredded'])
+    if d != initial_data:
+        logentry.data = json.dumps(d)
+        logentry.shredded = logentry.shredded or shredded
+        logentry.save(update_fields=['data', 'shredded'])
+
+
+def slow_update(qs, batch_size=1000, sleep_time=.5, progress_callback=None, progress_offset=0, progress_total=None, **update):
+    """
+    Doing UPDATE queries on hundreds of thousands of rows can cause outages due to high write load on the database.
+    This provides a throttled way to update rows. The condition for this to work properly is that the queryset has a
+    filter condition that no longer applies after the update!
+    Otherwise, this will be an endless loop!
+    """
+    total_updated = 0
+    while True:
+        updated = qs.order_by().filter(
+            pk__in=qs.order_by().values_list('pk', flat=True)[:batch_size]
+        ).update(**update)
+        total_updated += updated
+        if not updated:
+            break
+        if total_updated >= 0.8 * batch_size:
+            time.sleep(sleep_time)
+        if progress_callback and progress_total:
+            progress_callback((progress_offset + total_updated) / progress_total)
+
+    return total_updated
+
+
+def slow_delete(qs, batch_size=1000, sleep_time=.5, progress_callback=None, progress_offset=0, progress_total=None):
+    """
+    Doing DELETE queries on hundreds of thousands of rows can cause outages due to high write load on the database.
+    This provides a throttled way to update rows.
+    """
+    total_deleted = 0
+    while True:
+        deleted = qs.order_by().filter(
+            pk__in=qs.order_by().values_list('pk', flat=True)[:batch_size]
+        ).delete()[0]
+        total_deleted += deleted
+        if not deleted:
+            break
+        if total_deleted >= 0.8 * batch_size:
+            time.sleep(sleep_time)
+    return total_deleted
+
+
+def _progress_helper(queryset, progress_callback, offset, total):
+    if not progress_callback:
+        yield from queryset
+    else:
+        for i, o in enumerate(queryset):
+            yield o
+            if i % 10 == 0:
+                progress_callback((i + offset) / total * 100)
 
 
 class PhoneNumberShredder(BaseDataShredder):
@@ -177,18 +233,26 @@ class PhoneNumberShredder(BaseDataShredder):
             o.code: o.phone for o in self.event.orders.filter(phone__isnull=False)
         }, cls=CustomJSONEncoder, indent=4)
 
-    @transaction.atomic
-    def shred_data(self):
-        for o in self.event.orders.all():
+    def shred_data(self, progress_callback=None):
+        qs_orders = self.event.orders.all()
+        qs_orders_cnt = qs_orders.count()
+        qs_le = self.event.logentry_set.filter(action_type="pretix.event.order.phone.changed")
+        qs_le_cnt = qs_le.count()
+        total = qs_le_cnt + qs_orders_cnt
+
+        for o in _progress_helper(qs_orders, progress_callback, 0, total):
+            changed = bool(o.phone)
             o.phone = None
             d = o.meta_info_data
             if d:
                 if 'contact_form_data' in d and 'phone' in d['contact_form_data']:
+                    changed = True
                     del d['contact_form_data']['phone']
-                o.meta_info = json.dumps(d)
-            o.save(update_fields=['meta_info', 'phone'])
+                    o.meta_info = json.dumps(d)
+            if changed:
+                o.save(update_fields=['meta_info', 'phone'])
 
-        for le in self.event.logentry_set.filter(action_type="pretix.event.order.phone.changed"):
+        for le in _progress_helper(qs_le, progress_callback, qs_orders_cnt, total):
             shred_log_fields(le, banlist=['old_phone', 'new_phone'])
 
 
@@ -207,37 +271,66 @@ class EmailAddressShredder(BaseDataShredder):
             for op in OrderPosition.all.filter(order__event=self.event, attendee_email__isnull=False)
         }, indent=4)
 
-    @transaction.atomic
-    def shred_data(self):
-        OrderPosition.all.filter(order__event=self.event, attendee_email__isnull=False).update(attendee_email=None)
+    def shred_data(self, progress_callback=None):
+        qs_op = OrderPosition.all.filter(order__event=self.event, attendee_email__isnull=False)
+        qs_op_cnt = qs_op.count()
 
-        for o in self.event.orders.all():
+        qs_orders = self.event.orders.all()
+        qs_orders_cnt = qs_orders.count()
+
+        qs_le = self.event.logentry_set.filter(
+            Q(action_type__contains="order.email") | Q(action_type__contains="position.email") |
+            Q(action_type="pretix.event.order.contact.changed") |
+            Q(action_type="pretix.event.order.modified")
+        ).exclude(data="")
+        qs_le_cnt = qs_le.count()
+
+        total = qs_op_cnt + qs_orders_cnt + qs_le_cnt
+
+        slow_update(
+            qs_op,
+            attendee_email=None,
+            progress_callback=progress_callback,
+            progress_offset=0,
+            progress_total=total,
+            # Updates to order position table are slow, since PostgreSQL needs to update many indexes, so let's
+            # take them really slowly to not overwhelm the database.
+            batch_size=100,
+            sleep_time=2,
+        )
+
+        for o in _progress_helper(qs_orders, progress_callback, qs_op_cnt, total):
+            changed = bool(o.email) or bool(o.customer)
             o.email = None
             o.customer = None
             d = o.meta_info_data
             if d:
                 if 'contact_form_data' in d and 'email' in d['contact_form_data']:
                     del d['contact_form_data']['email']
-                o.meta_info = json.dumps(d)
-            o.save(update_fields=['meta_info', 'email', 'customer'])
+                    changed = True
+                    o.meta_info = json.dumps(d)
+                if 'contact_form_data' in d and 'email_repeat' in d['contact_form_data']:
+                    del d['contact_form_data']['email_repeat']
+                    changed = True
+            if changed:
+                if d:
+                    o.meta_info = json.dumps(d)
+                o.save(update_fields=['meta_info', 'email', 'customer'])
 
-        for le in self.event.logentry_set.filter(
-            Q(action_type__contains="order.email") | Q(action_type__contains="position.email"),
-        ):
-            shred_log_fields(le, banlist=['recipient', 'message', 'subject', 'full_mail'])
-
-        for le in self.event.logentry_set.filter(action_type="pretix.event.order.contact.changed"):
-            shred_log_fields(le, banlist=['old_email', 'new_email'])
-
-        for le in self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data=""):
-            d = le.parsed_data
-            if 'data' in d:
-                for row in d['data']:
-                    if 'attendee_email' in row:
-                        row['attendee_email'] = '█'
-                le.data = json.dumps(d)
-                le.shredded = True
-                le.save(update_fields=['data', 'shredded'])
+        for le in _progress_helper(qs_le, progress_callback, qs_op_cnt + qs_orders_cnt, total):
+            if le.action_type == "pretix.event.order.modified":
+                d = le.parsed_data
+                if 'data' in d:
+                    for row in d['data']:
+                        if 'attendee_email' in row:
+                            row['attendee_email'] = '█'
+                    le.data = json.dumps(d)
+                    le.shredded = True
+                    le.save(update_fields=['data', 'shredded'])
+            else:
+                shred_log_fields(le, banlist=[
+                    'recipient', 'message', 'subject', 'full_mail', 'old_email', 'new_email'
+                ])
 
 
 class WaitingListShredder(BaseDataShredder):
@@ -251,16 +344,35 @@ class WaitingListShredder(BaseDataShredder):
             for wle in self.event.waitinglistentries.all()
         ], indent=4)
 
-    @transaction.atomic
-    def shred_data(self):
-        self.event.waitinglistentries.update(name_cached=None, name_parts={'_shredded': True}, email='█', phone='█')
+    def shred_data(self, progress_callback=None):
+        qs_wle = self.event.waitinglistentries.exclude(email='█')
+        qs_wle_cnt = qs_wle.count()
 
-        for wle in self.event.waitinglistentries.select_related('voucher').filter(voucher__isnull=False):
+        qs_voucher = self.event.waitinglistentries.select_related('voucher').filter(voucher__isnull=False)
+        qs_voucher_cnt = qs_voucher.count()
+
+        qs_le = self.event.logentry_set.filter(action_type="pretix.voucher.added.waitinglist").exclude(data="")
+        qs_le_cnt = qs_le.count()
+
+        total = qs_voucher_cnt + qs_wle_cnt + qs_le_cnt
+
+        slow_update(
+            qs_wle,
+            name_cached=None,
+            name_parts={'_shredded': True},
+            email='█',
+            phone='█',
+            progress_callback=progress_callback,
+            progress_offset=0,
+            progress_total=total,
+        )
+
+        for wle in _progress_helper(qs_voucher, progress_callback, qs_wle_cnt, total):
             if '@' in wle.voucher.comment:
                 wle.voucher.comment = '█'
                 wle.voucher.save(update_fields=['comment'])
 
-        for le in self.event.logentry_set.filter(action_type="pretix.voucher.added.waitinglist").exclude(data=""):
+        for le in _progress_helper(qs_le, progress_callback, qs_wle_cnt + qs_voucher_cnt, total):
             d = le.parsed_data
             if 'name' in d:
                 d['name'] = '█'
@@ -298,17 +410,41 @@ class AttendeeInfoShredder(BaseDataShredder):
             )
         }, indent=4)
 
-    @transaction.atomic
-    def shred_data(self):
-        OrderPosition.all.filter(
+    def shred_data(self, progress_callback=None):
+        qs_op = OrderPosition.all.filter(
             order__event=self.event
         ).filter(
-            Q(attendee_name_cached__isnull=False) | Q(attendee_name_parts__isnull=False) |
-            Q(company__isnull=False) | Q(street__isnull=False) | Q(zipcode__isnull=False) | Q(city__isnull=False)
-        ).update(attendee_name_cached=None, attendee_name_parts={'_shredded': True}, company=None, street=None,
-                 zipcode=None, city=None)
+            Q(attendee_name_cached__isnull=False) |
+            Q(company__isnull=False) |
+            Q(street__isnull=False) |
+            Q(zipcode__isnull=False) |
+            Q(city__isnull=False)
+        )
+        qs_op_cnt = qs_op.count()
 
-        for le in self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data=""):
+        qs_le = self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data="")
+        qs_le_cnt = qs_le.count()
+
+        total = qs_op_cnt + qs_le_cnt
+
+        slow_update(
+            qs_op,
+            attendee_name_cached=None,
+            attendee_name_parts={'_shredded': True},
+            company=None,
+            street=None,
+            zipcode=None,
+            city=None,
+            progress_callback=progress_callback,
+            progress_total=total,
+            progress_offset=0,
+            # Updates to order position table are slow, since PostgreSQL needs to update many indexes, so let's
+            # take them really slowly to not overwhelm the database.
+            batch_size=100,
+            sleep_time=2,
+        )
+
+        for le in _progress_helper(qs_le, progress_callback, qs_op_cnt, total):
             d = le.parsed_data
             if 'data' in d:
                 for i, row in enumerate(d['data']):
@@ -343,11 +479,18 @@ class InvoiceAddressShredder(BaseDataShredder):
             for ia in InvoiceAddress.objects.filter(order__event=self.event)
         }, indent=4)
 
-    @transaction.atomic
-    def shred_data(self):
-        InvoiceAddress.objects.filter(order__event=self.event).delete()
+    def shred_data(self, progress_callback=None):
+        qs_ia = InvoiceAddress.objects.filter(order__event=self.event)
+        qs_ia_cnt = qs_ia.count()
 
-        for le in self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data=""):
+        qs_le = self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data="")
+        qs_le_cnt = qs_le.count()
+
+        total = qs_ia_cnt + qs_le_cnt
+
+        slow_delete(qs_ia, progress_callback=progress_callback, progress_total=total, progress_offset=0)
+
+        for le in _progress_helper(qs_le, progress_callback, qs_ia_cnt, total):
             d = le.parsed_data
             if 'invoice_data' in d and not isinstance(d['invoice_data'], bool):
                 for field in d['invoice_data']:
@@ -375,11 +518,18 @@ class QuestionAnswerShredder(BaseDataShredder):
             ).data
         yield 'question-answers.json', 'application/json', json.dumps(d, indent=4)
 
-    @transaction.atomic
-    def shred_data(self):
-        QuestionAnswer.objects.filter(orderposition__order__event=self.event).delete()
+    def shred_data(self, progress_callback=None):
+        qs_qa = QuestionAnswer.objects.filter(orderposition__order__event=self.event)
+        qs_qa_cnt = qs_qa.count()
 
-        for le in self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data=""):
+        qs_le = self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data="")
+        qs_le_cnt = qs_le.count()
+
+        total = qs_qa_cnt + qs_le_cnt
+
+        slow_delete(qs_qa, progress_callback=progress_callback, progress_total=total, progress_offset=0)
+
+        for le in _progress_helper(qs_le, progress_callback, qs_qa_cnt, total):
             d = le.parsed_data
             if 'data' in d:
                 for i, row in enumerate(d['data']):
@@ -408,9 +558,11 @@ class InvoiceShredder(BaseDataShredder):
             yield 'invoices/{}.pdf'.format(i.number), 'application/pdf', i.file.read()
             i.file.close()
 
-    @transaction.atomic
-    def shred_data(self):
-        for i in self.event.invoices.filter(shredded=False):
+    def shred_data(self, progress_callback=None):
+        qs_i = self.event.invoices.filter(shredded=False)
+        total = qs_i.count()
+
+        for i in _progress_helper(qs_i, progress_callback, 0, total):
             if i.file:
                 i.file.delete()
                 i.shredded = True
@@ -430,10 +582,17 @@ class CachedTicketShredder(BaseDataShredder):
     def generate_files(self) -> List[Tuple[str, str, str]]:
         pass
 
-    @transaction.atomic
-    def shred_data(self):
-        CachedTicket.objects.filter(order_position__order__event=self.event).delete()
-        CachedCombinedTicket.objects.filter(order__event=self.event).delete()
+    def shred_data(self, progress_callback=None):
+        qs_1 = CachedTicket.objects.filter(order_position__order__event=self.event)
+        qs_1_cnt = qs_1.count()
+
+        qs_2 = CachedCombinedTicket.objects.filter(order__event=self.event)
+        qs_2_cnt = qs_2.count()
+
+        total = qs_1_cnt + qs_2_cnt
+
+        slow_delete(qs_1, progress_callback=progress_callback, progress_total=total, progress_offset=0)
+        slow_delete(qs_2, progress_callback=progress_callback, progress_total=total, progress_offset=qs_1_cnt)
 
 
 class PaymentInfoShredder(BaseDataShredder):
@@ -446,14 +605,21 @@ class PaymentInfoShredder(BaseDataShredder):
     def generate_files(self) -> List[Tuple[str, str, str]]:
         pass
 
-    @transaction.atomic
-    def shred_data(self):
+    def shred_data(self, progress_callback=None):
+        qs_p = OrderPayment.objects.filter(order__event=self.event)
+        qs_p_count = qs_p.count()
+        qs_r = OrderRefund.objects.filter(order__event=self.event)
+        qs_r_count = qs_r.count()
+
+        total = qs_p_count + qs_r_count
+
         provs = self.event.get_payment_providers()
-        for obj in OrderPayment.objects.filter(order__event=self.event):
+        for obj in _progress_helper(qs_p, progress_callback, 0, total):
             pprov = provs.get(obj.provider)
             if pprov:
                 pprov.shred_payment_info(obj)
-        for obj in OrderRefund.objects.filter(order__event=self.event):
+
+        for obj in _progress_helper(qs_r, progress_callback, qs_p_count, total):
             pprov = provs.get(obj.provider)
             if pprov:
                 pprov.shred_payment_info(obj)

src/pretix/base/templates/pretixbase/email/shred_completed.txt

@@ -0,0 +1,17 @@
+{% load i18n %}
+{% load i18n %}{% blocktrans with url=url|safe %}Hello,
+
+we hereby confirm that the following data shredding job has been completed:
+
+Organizer: {{ organizer }}
+
+Event: {{ event }}
+
+Data selection: {{ shredders }}
+
+Start time: {{ start_time }} (new data added after this time might not have been deleted)
+
+Best regards,
+
+Your pretix team
+{% endblocktrans %}

src/pretix/base/views/tasks.py

@@ -115,7 +115,8 @@ class AsyncMixin:
         elif state == 'PROGRESS':
             data.update({
                 'started': True,
-                'percentage': info.get('value', 0) if isinstance(info, dict) else 0
+                'percentage': info.get('value', 0) if isinstance(info, dict) else 0,
+                'steps': info.get('steps', []) if isinstance(info, dict) else None,
             })
         elif state == 'STARTED':
             data.update({

src/pretix/control/templates/pretixcontrol/base.html

@@ -470,6 +470,8 @@
                         <div class="progress-bar progress-bar-success">
                         </div>
                     </div>
+                    <div class="steps">
+                    </div>
                 </div>
             </div>
         </div>

src/pretix/control/templates/pretixcontrol/shredder/download.html

@@ -8,7 +8,7 @@
         {% trans "Data shredder" %}
     </h1>
     <form action="{% url "control:event.shredder.shred" event=request.event.slug organizer=request.organizer.slug %}"
-          method="post" class="form-horizontal" data-asynctask>
+          method="post" class="form-horizontal" data-asynctask data-asynctask-long>
         {% csrf_token %}
         <fieldset>
             {% if download_on_shred %}
@@ -55,6 +55,12 @@
             </fieldset>
         {% endif %}
         <input type="hidden" name="file" value="{{ file.pk }}">
+        <div class="alert alert-info">
+            {% blocktrans trimmed %}
+                Depending on the amount of data in your event, the following step may take a while to complete.
+                We will inform you via email once it has been completed.
+            {% endblocktrans %}
+        </div>
         <div class="form-group submit-group">
             <button type="submit" class="btn btn-primary btn-save">
                 {% trans "Continue" %}

src/pretix/control/templates/pretixcontrol/shredder/index.html

@@ -42,7 +42,7 @@
         </div>
     {% else %}
         <form action="{% url "control:event.shredder.export" event=request.event.slug organizer=request.organizer.slug %}"
-                method="post" class="form-horizontal" data-asynctask>
+                method="post" class="form-horizontal" data-asynctask data-asynctask-long>
             <legend>{% trans "Data selection" %}</legend>
             {% csrf_token %}
             <div class="panel-group" id="payment_accordion">

src/pretix/control/views/organizer.py

@@ -396,6 +396,7 @@ class OrganizerDelete(AdministratorPermissionRequiredMixin, FormView):
                 )
                 self.request.organizer.delete_sub_objects()
                 self.request.organizer.delete()
+                self.request.organizer = None
             messages.success(self.request, _('The organizer has been deleted.'))
             return redirect(self.get_success_url())
         except ProtectedError as e:

src/pretix/control/views/shredder.py

@@ -37,10 +37,11 @@ import logging
 from collections import OrderedDict
 from zipfile import ZipFile
 
-from django.shortcuts import get_object_or_404
+from django.contrib import messages
+from django.shortcuts import get_object_or_404, redirect
 from django.urls import reverse
 from django.utils.functional import cached_property
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import get_language, gettext_lazy as _
 from django.views import View
 from django.views.generic import TemplateView
 
@@ -62,6 +63,16 @@ class ShredderMixin:
             sorted(self.request.event.get_data_shredders().items(), key=lambda s: s[1].verbose_name)
         )
 
+    def dispatch(self, request, *args, **kwargs):
+        try:
+            return super().dispatch(request, *args, **kwargs)
+        except ShredError as e:
+            messages.error(request, str(e))
+            return redirect(reverse('control:event.shredder.start', kwargs={
+                'event': self.request.event.slug,
+                'organizer': self.request.event.organizer.slug
+            }))
+
 
 class StartShredView(RecentAuthenticationRequiredMixin, EventPermissionRequiredMixin, ShredderMixin, TemplateView):
     permission = 'can_change_orders'
@@ -167,4 +178,5 @@ class ShredDoView(RecentAuthenticationRequiredMixin, EventPermissionRequiredMixi
         if request.event.slug != request.POST.get("slug"):
             return self.error(ShredError(_("The slug you entered was not correct.")))
 
-        return self.do(self.request.event.id, request.POST.get("file"), request.POST.get("confirm_code"))
+        return self.do(self.request.event.id, request.POST.get("file"), request.POST.get("confirm_code"),
+                       self.request.user.pk, get_language())

src/pretix/locale/zh_Hant/LC_MESSAGES/djangojs.po

@@ -8,7 +8,7 @@ msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2023-06-16 14:35+0000\n"
-"PO-Revision-Date: 2023-05-18 01:00+0000\n"
+"PO-Revision-Date: 2023-06-19 19:00+0000\n"
 "Last-Translator: Yucheng Lin <yuchenglinedu@gmail.com>\n"
 "Language-Team: Chinese (Traditional) <https://translate.pretix.eu/projects/"
 "pretix/pretix-js/zh_Hant/>\n"
@@ -250,7 +250,7 @@ msgstr "這張票券還沒有付款，你想要繼續嗎？"
 
 #: pretix/plugins/webcheckin/static/pretixplugins/webcheckin/main.js:51
 msgid "Additional information required"
-msgstr "需要額外的資訊"
+msgstr "需要額外資訊"
 
 #: pretix/plugins/webcheckin/static/pretixplugins/webcheckin/main.js:52
 msgid "Valid ticket"

src/pretix/plugins/paypal2/views.py

@@ -56,7 +56,9 @@ from django.views.generic import TemplateView
 from django_scopes import scopes_disabled
 from paypalcheckoutsdk import orders as pp_orders, payments as pp_payments
 
-from pretix.base.models import Event, Order, OrderPayment, OrderRefund, Quota
+from pretix.base.models import (
+    Event, Order, OrderPayment, OrderRefund, Quota, TaxRule,
+)
 from pretix.base.payment import PaymentException
 from pretix.base.services.cart import add_payment_to_cart, get_fees
 from pretix.base.settings import GlobalSettingsObject
@@ -158,8 +160,12 @@ class XHRView(View):
                 'info_data': {},
             }]
 
-            for fee in get_fees(request.event, request, cart_total, None, simulated_payments, get_cart(request)):
-                cart_total += fee.value
+            try:
+                for fee in get_fees(request.event, request, cart_total, None, simulated_payments, get_cart(request)):
+                    cart_total += fee.value
+            except TaxRule.SaleNotAllowed:
+                # ignore for now, will fail on order creation
+                pass
 
             total_remaining = cart_total
             for p in multi_use_cart_payments:

src/pretix/presale/checkoutflow.py

@@ -1126,13 +1126,17 @@ class PaymentStep(CartMixin, TemplateFlowStep):
     def _total_order_value(self):
         cart = get_cart(self.request)
         total = get_cart_total(self.request)
-        total += sum([
-            f.value for f in get_fees(
-                self.request.event, self.request, total, self.invoice_address,
-                [p for p in self.cart_session.get('payments', []) if p.get('multi_use_supported')],
-                cart,
-            )
-        ])
+        try:
+            total += sum([
+                f.value for f in get_fees(
+                    self.request.event, self.request, total, self.invoice_address,
+                    [p for p in self.cart_session.get('payments', []) if p.get('multi_use_supported')],
+                    cart,
+                )
+            ])
+        except TaxRule.SaleNotAllowed:
+            # ignore for now, will fail on order creation
+            pass
         return Decimal(total)
 
     @cached_property
@@ -1201,6 +1205,7 @@ class PaymentStep(CartMixin, TemplateFlowStep):
                     cart = self.get_cart(payments=simulated_payments)
                 else:
                     cart = self.get_cart()
+
                 resp = pprov.checkout_prepare(
                     request,
                     cart,
@@ -1283,8 +1288,12 @@ class PaymentStep(CartMixin, TemplateFlowStep):
 
         cart = get_cart(self.request)
         total = get_cart_total(self.request)
-        total += sum([f.value for f in get_fees(self.request.event, self.request, total, self.invoice_address,
-                                                self.cart_session.get('payments', []), cart)])
+        try:
+            total += sum([f.value for f in get_fees(self.request.event, self.request, total, self.invoice_address,
+                                                    self.cart_session.get('payments', []), cart)])
+        except TaxRule.SaleNotAllowed:
+            # ignore for now, will fail on order creation
+            pass
         selected = self.current_selected_payments(total, warn=warn, total_includes_payment_fees=True)
         if sum(p['payment_amount'] for p in selected) != total:
             if warn:

src/pretix/presale/views/__init__.py

@@ -49,7 +49,7 @@ from django_scopes import scopes_disabled
 from pretix.base.i18n import language
 from pretix.base.models import (
     CartPosition, Customer, InvoiceAddress, ItemAddOn, Question,
-    QuestionAnswer, QuestionOption,
+    QuestionAnswer, QuestionOption, TaxRule,
 )
 from pretix.base.services.cart import get_fees
 from pretix.base.templatetags.money import money_filter
@@ -198,11 +198,15 @@ class CartMixin:
         if order:
             fees = order.fees.all()
         elif positions:
-            fees = get_fees(
-                self.request.event, self.request, total, self.invoice_address,
-                payments if payments is not None else self.cart_session.get('payments', []),
-                cartpos
-            )
+            try:
+                fees = get_fees(
+                    self.request.event, self.request, total, self.invoice_address,
+                    payments if payments is not None else self.cart_session.get('payments', []),
+                    cartpos
+                )
+            except TaxRule.SaleNotAllowed:
+                # ignore for now, will fail on order creation
+                fees = []
         else:
             fees = []
 
@@ -389,7 +393,11 @@ def get_cart_is_free(request):
         pos = get_cart(request)
         ia = get_cart_invoice_address(request)
         total = get_cart_total(request)
-        fees = get_fees(request.event, request, total, ia, cs.get('payments', []), pos)
+        try:
+            fees = get_fees(request.event, request, total, ia, cs.get('payments', []), pos)
+        except TaxRule.SaleNotAllowed:
+            # ignore for now, will fail on order creation
+            fees = []
         request._cart_free_cache = total + sum(f.value for f in fees) == Decimal('0.00')
     return request._cart_free_cache
 

src/pretix/static/pretixbase/js/asynctask.js

@@ -34,6 +34,21 @@ function async_task_check_callback(data, textStatus, jqXHR) {
     } else if (typeof data.percentage === "number") {
         $("#loadingmodal .progress").show();
         $("#loadingmodal .progress .progress-bar").css("width", data.percentage + "%");
+        if (typeof data.steps === "object" && Array.isArray(data.steps)) {
+            var $steps = $("#loadingmodal .steps");
+            $steps.html("").show()
+            for (var step of data.steps) {
+                $steps.append(
+                    $("<span>").addClass("fa fa-fw")
+                        .toggleClass("fa-check text-success", step.done)
+                        .toggleClass("fa-cog fa-spin text-muted", !step.done)
+                ).append(
+                    $("<span>").text(step.label)
+                ).append(
+                    $("<br>")
+                )
+            }
+        }
     }
     async_task_timeout = window.setTimeout(async_task_check, 250);
 
@@ -267,6 +282,7 @@ var waitingDialog = {
         "use strict";
         $("#loadingmodal h3").html(message);
         $("#loadingmodal .progress").hide();
+        $("#loadingmodal .steps").hide();
         $("body").addClass("loading");
         $("#loadingmodal").removeAttr("hidden");
     },

src/pretix/static/schema/tax-rules-custom.schema.json

@@ -1,49 +0,0 @@
-{
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "title": "Tax rules",
-  "description": "Dynamic taxation rules",
-  "type": "array",
-  "items": {
-    "type": "object",
-    "description": "List of rules, executed in order until one matches",
-    "properties": {
-      "country": {
-        "description": "Country code to match. ZZ = any country, EU = any EU country.",
-        "enum": ["AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "EU", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW", "ZZ"]
-      },
-      "address_type": {
-        "description": "Type of customer, emtpy = any.",
-        "enum": ["", "individual", "business", "business_vat_id"]
-      },
-      "action": {
-        "description": "Action to take.",
-        "enum": ["vat", "reverse", "no", "block", "require_approval"]
-      },
-      "rate": {
-        "description": "Tax rate in case of action=vat or action=require_approval (or null for default)",
-        "type": ["string", "null"],
-        "pattern": "^[0-9]+(\\.[0-9]+)?$"
-      },
-      "invoice_text": {
-        "description": "Text on invoice (localized)",
-        "type": ["object", "null"],
-        "patternProperties": {
-          "[a-zA-Z-]+": {
-            "type": "string"
-          }
-        },
-        "additionalProperties": false
-      },
-      "ORDER": {
-        "description": "Internal, for backwards-compatibility, will be ignored.",
-        "type": "number"
-      },
-      "DELETE": {
-        "description": "Internal, for backwards-compatibility, will be ignored.",
-        "type": "boolean"
-      }
-    },
-    "required": ["country", "address_type", "action"],
-    "additionalProperties": false
-  }
-}

src/tests/api/test_taxrules.py

@@ -33,8 +33,7 @@ TEST_TAXRULE_RES = {
     'rate': '19.00',
     'price_includes_tax': True,
     'eu_reverse_charge': False,
-    'home_country': '',
-    'custom_rules': None,
+    'home_country': ''
 }
 
 
@@ -85,10 +84,6 @@ def test_rule_update(token_client, organizer, event, taxrule):
         '/api/v1/organizers/{}/events/{}/taxrules/{}/'.format(organizer.slug, event.slug, taxrule.pk),
         {
             "rate": "20.00",
-            "custom_rules": [
-                {"country": "AT", "address_type": "", "action": "vat", "rate": "19.00",
-                 "invoice_text": {"en": "Austrian VAT applies"}}
-            ]
         },
         format='json'
     )
@@ -116,20 +111,3 @@ def test_rule_delete_forbidden(token_client, organizer, event, taxrule):
     )
     assert resp.status_code == 403
     assert event.tax_rules.exists()
-
-
-@pytest.mark.django_db
-def test_rule_update_invalid_rules(token_client, organizer, event, taxrule):
-    resp = token_client.patch(
-        '/api/v1/organizers/{}/events/{}/taxrules/{}/'.format(organizer.slug, event.slug, taxrule.pk),
-        {
-            "custom_rules": [
-                {"foo": "bar"}
-            ]
-        },
-        format='json'
-    )
-    assert resp.status_code == 400
-    assert resp.data["custom_rules"][0].startswith(
-        "Your set of rules is not valid. Error message: 'country' is a required property"
-    )

src/tests/presale/test_checkout.py

@@ -456,6 +456,59 @@ class CheckoutTestCase(BaseCheckoutTestCase, TestCase):
         cr1.refresh_from_db()
         assert cr1.price == Decimal('23.00')
 
+    def test_custom_tax_rules_blocked_on_fee(self):
+        self.tr7 = self.event.tax_rules.create(rate=7)
+        self.tr7.custom_rules = json.dumps([
+            {'country': 'AT', 'address_type': 'business_vat_id', 'action': 'reverse'},
+            {'country': 'ZZ', 'address_type': '', 'action': 'block'},
+        ])
+        self.tr7.save()
+        self.event.settings.set('payment_banktransfer__enabled', True)
+        self.event.settings.set('payment_banktransfer__fee_percent', 20)
+        self.event.settings.set('payment_banktransfer__fee_reverse_calc', False)
+        self.event.settings.set('tax_rate_default', self.tr7)
+        self.event.settings.invoice_address_vatid = True
+
+        with scopes_disabled():
+            CartPosition.objects.create(
+                event=self.event, cart_id=self.session_key, item=self.ticket,
+                price=23, expires=now() + timedelta(minutes=10)
+            )
+
+        self.client.post('/%s/%s/checkout/questions/' % (self.orga.slug, self.event.slug), {
+            'is_business': 'business',
+            'company': 'Foo',
+            'name': 'Bar',
+            'street': 'Baz',
+            'zipcode': '12345',
+            'city': 'Here',
+            'country': 'DE',
+            'email': 'admin@localhost'
+        }, follow=True)
+
+        with mock.patch('pretix.base.services.tax._validate_vat_id_EU') as mock_validate:
+            mock_validate.return_value = 'AT123456'
+            self.client.post('/%s/%s/checkout/questions/' % (self.orga.slug, self.event.slug), {
+                'is_business': 'business',
+                'company': 'Foo',
+                'name': 'Bar',
+                'street': 'Baz',
+                'zipcode': '1234',
+                'city': 'Here',
+                'country': 'AT',
+                'vat_id': 'AT123456',
+                'email': 'admin@localhost'
+            }, follow=True)
+
+        self.client.post('/%s/%s/checkout/payment/' % (self.orga.slug, self.event.slug), {
+            'payment': 'banktransfer'
+        }, follow=True)
+
+        r = self.client.post('/%s/%s/checkout/confirm/' % (self.orga.slug, self.event.slug), follow=True)
+        doc = BeautifulSoup(r.content.decode(), "lxml")
+        assert doc.select(".alert-danger")
+        assert "not available in the selected country" in doc.select(".alert-danger")[0].text
+
     def test_custom_tax_rules_blocked(self):
         self.tr19.custom_rules = json.dumps([
             {'country': 'AT', 'address_type': 'business_vat_id', 'action': 'reverse'},