Fix #678 -- Data shredders for personally identifiable information (#817)

* Add data shredders for PII

* First working shredder

* Add more shredders

* Add new shredders and download confirmation

* tmp

* PayPal, Stripe, banktransfer

* Add icon to logs

* Untested payment log shredders

* Add waiting list shredder

* First tests

* Add tests for shredders

* Improve templats, link to shredder

* Test payment info shredders

* More tests

* Documentation

* Fix enabled flag in payment provider overview

* Fix minor issues

src/pretix/api/views/order.py

@@ -345,6 +345,9 @@ class InvoiceViewSet(viewsets.ReadOnlyModelViewSet):
             invoice_pdf(invoice.pk)
             invoice.refresh_from_db()
 
+        if invoice.shredded:
+            raise PermissionDenied('The invoice file is no longer stored on the server.')
+
         if not invoice.file:
             raise RetryException()
 
@@ -357,6 +360,8 @@ class InvoiceViewSet(viewsets.ReadOnlyModelViewSet):
         inv = self.get_object()
         if inv.canceled:
             raise ValidationError('The invoice has already been canceled.')
+        elif inv.shredded:
+            raise PermissionDenied('The invoice file is no longer stored on the server.')
         else:
             inv = regenerate_invoice(inv)
             inv.order.log_action(
@@ -374,6 +379,8 @@ class InvoiceViewSet(viewsets.ReadOnlyModelViewSet):
         inv = self.get_object()
         if inv.canceled:
             raise ValidationError('The invoice has already been canceled.')
+        elif inv.shredded:
+            raise PermissionDenied('The invoice file is no longer stored on the server.')
         else:
             c = generate_cancellation(inv)
             if inv.order.status not in (Order.STATUS_CANCELED, Order.STATUS_REFUNDED):

src/pretix/base/exporters/invoices.py

@@ -18,7 +18,7 @@ class InvoiceExporter(BaseExporter):
     verbose_name = _('All invoices')
 
     def render(self, form_data: dict):
-        qs = self.event.invoices.all()
+        qs = self.event.invoices.filter(shredded=False)
 
         if form_data.get('payment_provider'):
             qs = qs.filter(order__payment_provider=form_data.get('payment_provider'))

src/pretix/base/migrations/0089_auto_20180315_1322.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.10 on 2018-03-15 13:22
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pretixbase', '0088_auto_20180328_1217'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='logentry',
+            name='shredded',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='invoice',
+            name='shredded',
+            field=models.BooleanField(default=False),
+        ),
+    ]

src/pretix/base/models/event.py

@@ -495,6 +495,22 @@ class Event(EventMixin, LoggedModel):
                 renderers[pp.identifier] = pp
         return renderers
 
+    def get_data_shredders(self) -> dict:
+        """
+        Returns a dictionary of initialized data shredders mapped by their identifiers.
+        """
+        from ..signals import register_data_shredders
+
+        responses = register_data_shredders.send(self)
+        renderers = {}
+        for receiver, response in responses:
+            if not isinstance(response, list):
+                response = [response]
+            for p in response:
+                pp = p(self)
+                renderers[pp.identifier] = pp
+        return renderers
+
     @property
     def invoice_renderer(self):
         """

src/pretix/base/models/invoices.py

@@ -83,6 +83,7 @@ class Invoice(models.Model):
     foreign_currency_display = models.CharField(max_length=50, null=True, blank=True)
     foreign_currency_rate = models.DecimalField(decimal_places=4, max_digits=10, null=True, blank=True)
     foreign_currency_rate_date = models.DateField(null=True, blank=True)
+    shredded = models.BooleanField(default=False)
 
     file = models.FileField(null=True, blank=True, upload_to=invoice_filename, max_length=255)
     internal_reference = models.TextField(blank=True)

src/pretix/base/models/log.py

@@ -45,6 +45,7 @@ class LogEntry(models.Model):
     action_type = models.CharField(max_length=255)
     data = models.TextField(default='{}')
     visible = models.BooleanField(default=True)
+    shredded = models.BooleanField(default=False)
 
     objects = VisibleOnlyManager()
     all = models.Manager()

src/pretix/base/models/orders.py

@@ -191,7 +191,10 @@ class Order(LoggedModel):
 
     @cached_property
     def meta_info_data(self):
-        return json.loads(self.meta_info)
+        try:
+            return json.loads(self.meta_info)
+        except TypeError:
+            return None
 
     @property
     def full_code(self):

src/pretix/base/models/waitinglist.py

@@ -87,6 +87,8 @@ class WaitingListEntry(LoggedModel):
             raise WaitingListException(_('This product is currently not available.'))
         if self.voucher:
             raise WaitingListException(_('A voucher has already been sent to this person.'))
+        if '@' not in self.email:
+            raise WaitingListException(_('This entry is anonymized and can no longer be used.'))
 
         with transaction.atomic():
             v = Voucher.objects.create(

src/pretix/base/payment.py

@@ -566,6 +566,19 @@ class BasePaymentProvider:
         messages.success(request, _('The order has been marked as refunded. Please transfer the money '
                                     'back to the buyer manually.'))
 
+    def shred_payment_info(self, order: Order):
+        """
+        When personal data is removed from an event, this method is called to scrub payment-related data
+        from an order. By default, it removes all info from the ``payment_info`` attribute. You can override
+        this behavior if you want to retain attributes that are not personal data on their own, i.e. a
+        reference to a transaction in an external system. You can also override this to scrub more data, e.g.
+        data from external sources that is saved in LogEntry objects or other places.
+
+        :param order: An order
+        """
+        order.payment_info = None
+        order.save(update_fields=['payment_info'])
+
 
 class PaymentException(Exception):
     pass

src/pretix/base/services/invoices.py

@@ -180,6 +180,8 @@ def generate_cancellation(invoice: Invoice, trigger_pdf=True):
 
 
 def regenerate_invoice(invoice: Invoice):
+    if invoice.shredded:
+        return invoice
     if invoice.is_cancellation:
         invoice = build_cancellation(invoice)
     else:
@@ -214,6 +216,10 @@ def generate_invoice(order: Order, trigger_pdf=True):
 @app.task(base=TransactionAwareTask)
 def invoice_pdf_task(invoice: int):
     i = Invoice.objects.get(pk=invoice)
+    if i.shredded:
+        return None
+    if i.file:
+        i.file.delete()
     with language(i.locale):
         fname, ftype, fcontent = i.event.invoice_renderer.generate(i)
         i.file.save(fname, ContentFile(fcontent))

src/pretix/base/services/shredder.py

@@ -0,0 +1,91 @@
+import json
+from datetime import timedelta
+from tempfile import NamedTemporaryFile
+from typing import List
+from zipfile import ZipFile
+
+from dateutil.parser import parse
+from django.conf import settings
+from django.utils.crypto import get_random_string
+from django.utils.timezone import now
+from django.utils.translation import ugettext_lazy as _
+
+from pretix.base.models import CachedFile, Event, cachedfile_name
+from pretix.base.services.async import ProfiledTask
+from pretix.base.shredder import ShredError
+from pretix.celery_app import app
+
+
+@app.task(base=ProfiledTask)
+def export(event: str, shredders: List[str]) -> None:
+    event = Event.objects.get(id=event)
+    known_shredders = event.get_data_shredders()
+
+    with NamedTemporaryFile() as rawfile:
+        with ZipFile(rawfile, 'w') as zipfile:
+            ccode = get_random_string(6)
+            zipfile.writestr(
+                'CONFIRM_CODE.txt',
+                ccode,
+            )
+            zipfile.writestr(
+                'index.json',
+                json.dumps({
+                    'instance': settings.SITE_URL,
+                    'organizer': event.organizer.slug,
+                    'event': event.slug,
+                    'time': now().isoformat(),
+                    'shredders': shredders,
+                    'confirm_code': ccode
+                }, indent=4)
+            )
+            for s in shredders:
+                shredder = known_shredders.get(s)
+                if not shredder:
+                    continue
+
+                it = shredder.generate_files()
+                if not it:
+                    continue
+                for fname, ftype, content in it:
+                    zipfile.writestr(fname, content)
+
+        rawfile.seek(0)
+
+        cf = CachedFile()
+        cf.date = now()
+        cf.filename = event.slug + '.zip'
+        cf.type = 'application/pdf'
+        cf.expires = now() + timedelta(hours=1)
+        cf.save()
+        cf.file.save(cachedfile_name(cf, cf.filename), rawfile)
+
+    return cf.pk
+
+
+@app.task(base=ProfiledTask, throws=(ShredError,))
+def shred(event: str, fileid: str, confirm_code: str) -> None:
+    event = Event.objects.get(id=event)
+    known_shredders = event.get_data_shredders()
+    try:
+        cf = CachedFile.objects.get(pk=fileid)
+    except CachedFile.DoesNotExist:
+        raise ShredError(_("The download file could no longer be found on the server, please try to start again."))
+    with ZipFile(cf.file.file, 'r') as zipfile:
+        indexdata = json.loads(zipfile.read('index.json').decode())
+    if indexdata['organizer'] != event.organizer.slug or indexdata['event'] != event.slug:
+        raise ShredError(_("This file is from a different event."))
+    if indexdata['confirm_code'] != confirm_code:
+        raise ShredError(_("The confirm code you entered was incorrect."))
+    if event.logentry_set.filter(datetime__gte=parse(indexdata['time'])):
+        raise ShredError(_("Something happened in your event after the export, please try again."))
+
+    for s in indexdata['shredders']:
+        shredder = known_shredders.get(s)
+        if not shredder:
+            continue
+
+        shredder.shred_data()
+
+    cf.file.delete(save=False)
+    cf.delete()

src/pretix/base/shredder.py

@@ -0,0 +1,341 @@
+import json
+from datetime import timedelta
+from typing import List, Tuple
+
+from django.db import transaction
+from django.db.models import Max
+from django.db.models.functions import Greatest
+from django.dispatch import receiver
+from django.utils.timezone import now
+from django.utils.translation import ugettext_lazy as _
+
+from pretix.api.serializers.order import (
+    AnswerSerializer, InvoiceAddressSerializer,
+)
+from pretix.api.serializers.waitinglist import WaitingListSerializer
+from pretix.base.i18n import LazyLocaleException
+from pretix.base.models import (
+    CachedCombinedTicket, CachedTicket, Event, InvoiceAddress, OrderPosition,
+    QuestionAnswer,
+)
+from pretix.base.services.invoices import invoice_pdf_task
+from pretix.base.signals import register_data_shredders
+
+
+class ShredError(LazyLocaleException):
+    pass
+
+
+def shred_constraints(event: Event):
+    if event.has_subevents:
+        max_date = event.subevents.aggregate(
+            max_from=Max('date_from'),
+            max_to=Max('date_to'),
+            max_fromto=Greatest(Max('date_to'), Max('date_from'))
+        )
+        max_date = max_date['max_fromto'] or max_date['max_to'] or max_date['max_From']
+        if max_date > now() - timedelta(days=60):
+            return _('Your event needs to be over for at least 60 days to use this feature.')
+    else:
+        if (event.date_to or event.date_from) > now() - timedelta(days=60):
+            return _('Your event needs to be over for at least 60 days to use this feature.')
+    if event.live:
+        return _('Your ticket shop needs to be offline to use this feature.')
+    return None
+
+
+class BaseDataShredder:
+    """
+    This is the base class for all data shredders.
+    """
+
+    def __init__(self, event: Event):
+        self.event = event
+
+    def __str__(self):
+        return self.identifier
+
+    def generate_files(self) -> List[Tuple[str, str, str]]:
+        """
+        This method is called to export the data that is about to be shred and return a list of tuples consisting of a
+        filename, a file type and file content.
+
+        You can also implement this as a generator and ``yield`` those tuples instead of returning a list of them.
+        """
+        raise NotImplementedError()  # NOQA
+
+    def shred_data(self):
+        """
+        This method is called to actually remove the data from the system. You should remove any database objects
+        here.
+
+        You should never delete ``LogEntry`` objects, but you might modify them to remove personal data. In this
+        case, set the ``LogEntry.shredded`` attribute to ``True`` to show that this is no longer original log data.
+        """
+        raise NotImplementedError()  # NOQA
+
+    @property
+    def verbose_name(self) -> str:
+        """
+        A human-readable name for what this shredder removes. This should be short but self-explanatory.
+        Good examples include 'E-Mail addresses' or 'Invoices'.
+        """
+        raise NotImplementedError()  # NOQA
+
+    @property
+    def identifier(self) -> str:
+        """
+        A short and unique identifier for this shredder.
+        This should only contain lowercase letters and in most
+        cases will be the same as your package name.
+        """
+        raise NotImplementedError()  # NOQA
+
+    @property
+    def description(self) -> str:
+        """
+        A more detailed description of what this shredder does. Can contain HTML.
+        """
+        raise NotImplementedError()  # NOQA
+
+
+def shred_log_fields(logentry, blacklist=None, whitelist=None):
+    d = logentry.parsed_data
+    if whitelist:
+        for k, v in d.items():
+            if k not in whitelist:
+                d[k] = '█'
+    elif blacklist:
+        for f in blacklist:
+            if f in d:
+                d[f] = '█'
+    logentry.data = json.dumps(d)
+    logentry.shredded = True
+    logentry.save(update_fields=['data', 'shredded'])
+
+
+class EmailAddressShredder(BaseDataShredder):
+    verbose_name = _('E-mails')
+    identifier = 'order_emails'
+    description = _('This will remove all e-mail addresses from orders and attendees, as well as logged email '
+                    'contents.')
+
+    def generate_files(self) -> List[Tuple[str, str, str]]:
+        yield 'emails-by-order.json', 'application/json', json.dumps({
+            o.code: o.email for o in self.event.orders.filter(email__isnull=False)
+        }, indent=4)
+        yield 'emails-by-attendee.json', 'application/json', json.dumps({
+            '{}-{}'.format(op.order.code, op.positionid): op.attendee_email
+            for op in OrderPosition.objects.filter(order__event=self.event, attendee_email__isnull=False)
+        }, indent=4)
+
+    @transaction.atomic
+    def shred_data(self):
+        OrderPosition.objects.filter(order__event=self.event, attendee_email__isnull=False).update(attendee_email=None)
+
+        for o in self.event.orders.all():
+            o.email = None
+            d = o.meta_info_data
+            if d:
+                if 'contact_form_data' in d and 'email' in d['contact_form_data']:
+                    del d['contact_form_data']['email']
+                o.meta_info = json.dumps(d)
+            o.save(update_fields=['meta_info', 'email'])
+
+        for le in self.event.logentry_set.filter(action_type__contains="order.email"):
+            shred_log_fields(le, blacklist=['recipient', 'message', 'subject'])
+
+        for le in self.event.logentry_set.filter(action_type="pretix.event.order.contact.changed"):
+            shred_log_fields(le, blacklist=['old_email', 'new_email'])
+
+        for le in self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data=""):
+            d = le.parsed_data
+            if 'data' in d:
+                for row in d['data']:
+                    if 'attendee_email' in row:
+                        row['attendee_email'] = '█'
+                le.data = json.dumps(d)
+                le.shredded = True
+                le.save(update_fields=['data', 'shredded'])
+
+
+class WaitingListShredder(BaseDataShredder):
+    verbose_name = _('Waiting list')
+    identifier = 'waiting_list'
+    description = _('This will remove all email addresses from the waiting list.')
+
+    def generate_files(self) -> List[Tuple[str, str, str]]:
+        yield 'waiting-list.json', 'application/json', json.dumps([
+            WaitingListSerializer(wle).data
+            for wle in self.event.waitinglistentries.all()
+        ], indent=4)
+
+    @transaction.atomic
+    def shred_data(self):
+        self.event.waitinglistentries.update(email='█')
+
+        for wle in self.event.waitinglistentries.select_related('voucher').filter(voucher__isnull=False):
+            if '@' in wle.voucher.comment:
+                wle.voucher.comment = '█'
+                wle.voucher.save(update_fields=['comment'])
+
+        for le in self.event.logentry_set.filter(action_type="pretix.voucher.added.waitinglist").exclude(data=""):
+            d = le.parsed_data
+            d['email'] = '█'
+            le.data = json.dumps(d)
+            le.shredded = True
+            le.save(update_fields=['data', 'shredded'])
+
+
+class AttendeeNameShredder(BaseDataShredder):
+    verbose_name = _('Attendee names')
+    identifier = 'attendee_names'
+    description = _('This will remove all attendee names from order positions, as well as logged changes to them.')
+
+    def generate_files(self) -> List[Tuple[str, str, str]]:
+        yield 'attendee-names.json', 'application/json', json.dumps({
+            '{}-{}'.format(op.order.code, op.positionid): op.attendee_name
+            for op in OrderPosition.objects.filter(order__event=self.event, attendee_name__isnull=False)
+        }, indent=4)
+
+    @transaction.atomic
+    def shred_data(self):
+        OrderPosition.objects.filter(order__event=self.event, attendee_name__isnull=False).update(attendee_name=None)
+
+        for le in self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data=""):
+            d = le.parsed_data
+            if 'data' in d:
+                for i, row in enumerate(d['data']):
+                    if 'attendee_name' in row:
+                        d['data'][i]['attendee_name'] = '█'
+                le.data = json.dumps(d)
+                le.shredded = True
+                le.save(update_fields=['data', 'shredded'])
+
+
+class InvoiceAddressShredder(BaseDataShredder):
+    verbose_name = _('Invoice addresses')
+    identifier = 'invoice_addresses'
+    description = _('This will remove all invoice addresses from orders, as well as logged changes to them.')
+
+    def generate_files(self) -> List[Tuple[str, str, str]]:
+        yield 'invoice-addresses.json', 'application/json', json.dumps({
+            ia.order.code: InvoiceAddressSerializer(ia).data
+            for ia in InvoiceAddress.objects.filter(order__event=self.event)
+        }, indent=4)
+
+    @transaction.atomic
+    def shred_data(self):
+        InvoiceAddress.objects.filter(order__event=self.event).delete()
+
+        for le in self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data=""):
+            d = le.parsed_data
+            if 'invoice_data' in d and not isinstance(d['invoice_data'], bool):
+                for field in d['invoice_data']:
+                    if d['invoice_data'][field]:
+                        d['invoice_data'][field] = '█'
+                le.data = json.dumps(d)
+                le.shredded = True
+                le.save(update_fields=['data', 'shredded'])
+
+
+class QuestionAnswerShredder(BaseDataShredder):
+    verbose_name = _('Question answers')
+    identifier = 'question_answers'
+    description = _('This will remove all answers to questions, as well as logged changes to them.')
+
+    def generate_files(self) -> List[Tuple[str, str, str]]:
+        yield 'question-answers.json', 'application/json', json.dumps({
+            '{}-{}'.format(op.order.code, op.positionid): AnswerSerializer(op.answers.all(), many=True).data
+            for op in OrderPosition.objects.filter(order__event=self.event).prefetch_related('answers')
+        }, indent=4)
+
+    @transaction.atomic
+    def shred_data(self):
+        QuestionAnswer.objects.filter(orderposition__order__event=self.event).delete()
+
+        for le in self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data=""):
+            d = le.parsed_data
+            if 'data' in d:
+                for i, row in enumerate(d['data']):
+                    for f in row:
+                        if f not in ('attendee_name', 'attendee_email'):
+                            d['data'][i][f] = '█'
+                le.data = json.dumps(d)
+                le.shredded = True
+                le.save(update_fields=['data', 'shredded'])
+
+
+class InvoiceShredder(BaseDataShredder):
+    verbose_name = _('Invoices')
+    identifier = 'invoices'
+    description = _('This will remove all invoice PDFs, as well as any of their text content that might contain '
+                    'personal data from the database. Invoice numbers and totals will be conserved.')
+
+    def generate_files(self) -> List[Tuple[str, str, str]]:
+        for i in self.event.invoices.filter(shredded=False):
+            if not i.file:
+                invoice_pdf_task.apply(args=(i.pk,))
+                i.refresh_from_db()
+            i.file.open('rb')
+            yield 'invoices/{}.pdf'.format(i.number), 'application/pdf', i.file.read()
+            i.file.close()
+
+    @transaction.atomic
+    def shred_data(self):
+        for i in self.event.invoices.filter(shredded=False):
+            if i.file:
+                i.file.delete()
+                i.shredded = True
+                i.introductory_text = "█"
+                i.additional_text = "█"
+                i.invoice_to = "█"
+                i.payment_provider_text = "█"
+                i.save()
+                i.lines.update(description="█")
+
+
+class CachedTicketShredder(BaseDataShredder):
+    verbose_name = _('Cached ticket files')
+    identifier = 'cachedtickets'
+    description = _('This will remove all cached ticket files. No download will be offered.')
+
+    def generate_files(self) -> List[Tuple[str, str, str]]:
+        pass
+
+    @transaction.atomic
+    def shred_data(self):
+        CachedTicket.objects.filter(order_position__order__event=self.event).delete()
+        CachedCombinedTicket.objects.filter(order__event=self.event).delete()
+
+
+class PaymentInfoShredder(BaseDataShredder):
+    verbose_name = _('Payment information')
+    identifier = 'payment_info'
+    description = _('This will remove payment-related information. Depending on the payment method, all data will be '
+                    'removed or personal data only. No download will be offered.')
+
+    def generate_files(self) -> List[Tuple[str, str, str]]:
+        pass
+
+    @transaction.atomic
+    def shred_data(self):
+        provs = self.event.get_payment_providers()
+        for o in self.event.orders.all():
+            pprov = provs.get(o.payment_provider)
+            if pprov:
+                pprov.shred_payment_info(o)
+
+
+@receiver(register_data_shredders, dispatch_uid="shredders_builtin")
+def register_payment_provider(sender, **kwargs):
+    return [
+        EmailAddressShredder,
+        AttendeeNameShredder,
+        InvoiceAddressShredder,
+        QuestionAnswerShredder,
+        InvoiceShredder,
+        CachedTicketShredder,
+        PaymentInfoShredder,
+        WaitingListShredder
+    ]

src/pretix/base/signals.py

@@ -119,7 +119,7 @@ register_payment_providers = EventPluginSignal(
 )
 """
 This signal is sent out to get all known payment providers. Receivers should return a
-subclass of pretix.base.payment.BasePaymentProvider
+subclass of pretix.base.payment.BasePaymentProvider or a list of these
 
 As with all event-plugin signals, the ``sender`` keyword argument will contain the event.
 """
@@ -129,7 +129,17 @@ register_invoice_renderers = EventPluginSignal(
 )
 """
 This signal is sent out to get all known invoice renderers. Receivers should return a
-subclass of pretix.base.invoice.BaseInvoiceRenderer
+subclass of pretix.base.invoice.BaseInvoiceRenderer or a list of these
+
+As with all event-plugin signals, the ``sender`` keyword argument will contain the event.
+"""
+
+register_data_shredders = EventPluginSignal(
+    providing_args=[]
+)
+"""
+This signal is sent out to get all known data shredders. Receivers should return a
+subclass of pretix.base.shredder.BaseDataShredder or a list of these
 
 As with all event-plugin signals, the ``sender`` keyword argument will contain the event.
 """

src/pretix/control/templates/pretixcontrol/event/delete.html

@@ -51,12 +51,18 @@
             <p>
                 {% trans "You can instead take your shop offline. This will hide it from everyone except from the organizer teams you configured to have access to the event." %}
             </p>
-            <form action="" method="post">
+            <form action="{% url "control:event.live" event=request.event.slug organizer=request.organizer.slug %}"
+                    method="post">
                 {% csrf_token %}
                 <input type="hidden" name="live" value="false">
 
                 <div class="form-group submit-group">
+                    <a href="{% url "control:event.shredder.start" event=request.event.slug organizer=request.organizer.slug %}" class="btn btn-danger btn-save">
+                        <span class="fa fa-eraser"></span>
+                        {% trans "Delete personal data" %}
+                    </a>
                     <button type="submit" class="btn btn-primary btn-save">
+                        <span class="fa fa-power-off"></span>
                         {% trans "Go offline" %}
                     </button>
                 </div>
@@ -64,6 +70,12 @@
         {% else %}
             <p>
                 {% trans "However, since your shop is offline, it is only visible to the organizing team according to the permissions you configured." %}
+                <div class="form-group submit-group">
+                    <a href="{% url "control:event.shredder.start" event=request.event.slug organizer=request.organizer.slug %}" class="btn btn-danger btn-save">
+                        <span class="fa fa-eraser"></span>
+                        {% trans "Delete personal data" %}
+                    </a>
+                </div>
             </p>
         {% endif %}
     {% endif %}

src/pretix/control/templates/pretixcontrol/event/index.html

@@ -106,6 +106,12 @@
                     <div class="row">
                         <div class="col-lg-2 col-sm-6 col-xs-12">
                             <span class="fa fa-clock-o"></span> {{ log.datetime|date:"SHORT_DATETIME_FORMAT" }}
+                            {% if log.shredded %}
+                                <span class="fa fa-eraser fa-danger fa-fw"
+                                      data-toggle="tooltip"
+                                      title="{% trans "Personal data was cleared from this log entry." %}">
+                                </span>
+                            {% endif %}
                         </div>
                         <div class="col-lg-2 col-sm-6 col-xs-12">
                             {% if log.user %}

src/pretix/control/templates/pretixcontrol/event/logs.html

@@ -31,6 +31,12 @@
                 <div class="row">
                     <div class="col-lg-2 col-sm-6 col-xs-12">
                         <span class="fa fa-clock-o"></span> {{ log.datetime|date:"SHORT_DATETIME_FORMAT" }}
+                        {% if log.shredded %}
+                            <span class="fa fa-eraser fa-danger fa-fw"
+                                  data-toggle="tooltip"
+                                  title="{% trans "Personal data was cleared from this log entry." %}">
+                            </span>
+                        {% endif %}
                     </div>
                     <div class="col-lg-2 col-sm-6 col-xs-12">
                         {% if log.user %}

src/pretix/control/templates/pretixcontrol/event/payment.html

@@ -14,7 +14,7 @@
                             <strong>{{ provider.verbose_name }}</strong>
                         </td>
                         <td>
-                            {% if provider.is_enabled %}
+                            {% if provider.show_enabled %}
                                 <span class="text-success">
                                     <span class="fa fa-check"></span>
                                     {% trans "Enabled" %}

src/pretix/control/templates/pretixcontrol/event/settings.html

@@ -78,10 +78,18 @@
             <button type="submit" class="btn btn-primary btn-save">
                 {% trans "Save" %}
             </button>
-            <a href="{% url "control:event.delete" organizer=request.organizer.slug event=request.event.slug %}"
-                    class="btn {% if request.event.allow_delete %}{% endif %} btn-danger btn-lg pull-left">
-                {% trans "Delete event" %}
-            </a>
+            <div class="pull-left">
+                <a href="{% url "control:event.delete" organizer=request.organizer.slug event=request.event.slug %}"
+                        class="btn {% if request.event.allow_delete %}{% endif %} btn-danger btn-lg">
+                    <span class="fa fa-trash"></span>
+                    {% trans "Delete event" %}
+                </a>
+                <a href="{% url "control:event.shredder.start" organizer=request.organizer.slug event=request.event.slug %}"
+                        class="btn btn-danger btn-lg">
+                    <span class="fa fa-eraser"></span>
+                    {% trans "Delete personal data" %}
+                </a>
+            </div>
         </div>
     </form>
 {% endblock %}

src/pretix/control/templates/pretixcontrol/includes/logs.html

@@ -10,12 +10,19 @@
                         <span class="fa fa-id-card fa-danger fa-fw"
                                 data-toggle="tooltip"
                                 title="{% trans "This change was performed by a pretix administrator." %}">
-                            </span>
+                        </span>
                     {% else %}
                         <span class="fa fa-user fa-fw"></span>
                     {% endif %}
                     {{ log.user.get_full_name }}
                 {% endif %}
+                {% if log.shredded %}
+                    <span class="fa fa-eraser fa-danger fa-fw"
+                          data-toggle="tooltip"
+                          title="{% trans "Personal data was cleared from this log entry." %}">
+
+                    </span>
+                {% endif %}
             </p>
 
             <p>

src/pretix/control/templates/pretixcontrol/order/mail_history.html

@@ -24,6 +24,9 @@
                         {% if log.display %}
                             <br/><span class="fa fa-fw fa-comment-o"></span> {{ log.display }}
                         {% endif %}
+                        {% if log.parsed_data.recipient %}
+                            <br/><span class="fa fa-fw fa-envelope-o"></span> {{ log.parsed_data.recipient }}
+                        {% endif %}
                     </p>
                     {% if log.parsed_data.subject.items %}
                         <div class="alert alert-info">

src/pretix/control/templates/pretixcontrol/shredder/download.html

@@ -0,0 +1,57 @@
+{% extends "pretixcontrol/event/base.html" %}
+{% load i18n %}
+{% load bootstrap3 %}
+{% load order_overview %}
+{% block title %}{% trans "Data shredder" %}{% endblock %}
+{% block content %}
+    <h1>
+        {% trans "Data shredder" %}
+    </h1>
+    <form action="{% url "control:event.shredder.shred" event=request.event.slug organizer=request.organizer.slug %}"
+          method="post" class="form-horizontal" data-asynctask>
+        {% csrf_token %}
+        <fieldset>
+            <legend>{% trans "Step 1: Download data" %}</legend>
+            <p>
+                {% blocktrans trimmed %}
+                    You are about to permamanently delete data from the server, even though you might be required to
+                    keep
+                    some of this data on file. You should therefore download the following file and store it in a safe
+                    place:
+                {% endblocktrans %}
+            </p>
+            <p>
+                <a href="{% url "cachedfile.download" id=file.pk %}" class="btn btn-primary btn-lg">
+                    {% trans "Download data" %}
+                </a>
+            </p>
+        </fieldset>
+        <fieldset>
+            <legend>{% trans "Step 2: Confirm download" %}</legend>
+            <p>
+                {% blocktrans trimmed %}
+                    In the downloaded file, there is a text file named "CONFIRM_CODE.txt" with a six-character code.
+                    Please enter this code here to confirm that you successfully downloaded the file.
+                {% endblocktrans %}
+            </p>
+            <input type="text" class="form-control" name="confirm_code" required placeholder="{% trans "Confirmation code" %}">
+            <br>
+        </fieldset>
+        <fieldset>
+            <legend>{% trans "Step 3: Confirm deletion" %}</legend>
+            <p>
+                {% blocktrans trimmed with event=request.event.name %}
+                Please re-check that you are fully certain that you want to delete the selected categories of data from the event <strong>{{ event }}</strong>.
+                    In this case, please enter your user password here:
+                {% endblocktrans %}
+            </p>
+            <input type="password" class="form-control" name="password" required placeholder="{% trans "Your password" %}">
+        </fieldset>
+        <input type="hidden" name="file" value="{{ file.pk }}">
+        <div class="form-group submit-group">
+            <button type="submit" class="btn btn-primary btn-save">
+                {% trans "Continue" %}
+            </button>
+        </div>
+    </form>
+{% endblock %}

src/pretix/control/templates/pretixcontrol/shredder/index.html

@@ -0,0 +1,73 @@
+{% extends "pretixcontrol/event/base.html" %}
+{% load i18n %}
+{% load bootstrap3 %}
+{% load order_overview %}
+{% block title %}{% trans "Data shredder" %}{% endblock %}
+{% block content %}
+    <h1>
+        {% trans "Data shredder" %}
+    </h1>
+    <p>
+        {% blocktrans trimmed %}
+            This feature allows you to remove personal data from this event. You will first select what kind of data
+            you want to shred, then you are able to download the affected data and after you confirmed the download,
+            the data will be removed from the server's database. The data might still exist in backups for a limited
+            period of time.
+        {% endblocktrans %}
+        <strong>
+            {% blocktrans trimmed %}
+                Using this will not remove the orders for your event, it just scrubs them of data that can be linked
+                to individual persons.
+            {% endblocktrans %}
+        </strong>
+    </p>
+    <div class="alert alert-legal">
+        <strong>
+            {% blocktrans trimmed %}
+                It is within your own responsibility to check if you are allowed to delete the affected data in your
+                legislation, e.g. for reasons of taxation.
+            {% endblocktrans %}
+        </strong>
+        {% blocktrans trimmed %}
+            For most categories of data, you will be able to partially download the data to store it offline. Some
+            kinds of data (such as some payment information) as well as historical log data cannot be downloaded at
+            the moment.
+        {% endblocktrans %}
+        <div class="clear"></div>
+    </div>
+    {% if constraints %}
+        <div class="alert alert-danger">
+            {{ constraints }}
+        </div>
+    {% else %}
+        <form action="{% url "control:event.shredder.export" event=request.event.slug organizer=request.organizer.slug %}"
+                method="post" class="form-horizontal" data-asynctask>
+            <legend>{% trans "Data selection" %}</legend>
+            {% csrf_token %}
+            <div class="panel-group" id="payment_accordion">
+                {% for ident, shredder in shredders.items %}
+                    <div class="panel panel-default">
+                        <label class="accordion-radio">
+                            <div class="panel-heading">
+                                <h4 class="panel-title">
+                                    <input type="checkbox" name="shredder" value="{{ shredder.identifier }}">
+                                    <strong>{{ shredder.verbose_name }}</strong>
+                                </h4>
+                            </div>
+                        </label>
+                        <div id="payment_{{ p.provider.identifier }}" class="panel-collapse in">
+                            <div class="panel-body">
+                                {{ shredder.description|safe }}
+                            </div>
+                        </div>
+                    </div>
+                {% endfor %}
+            </div>
+            <div class="form-group submit-group">
+                <button type="submit" class="btn btn-primary btn-save">
+                    {% trans "Continue" %}
+                </button>
+            </div>
+        </form>
+    {% endif %}
+{% endblock %}

src/pretix/control/urls.py

@@ -2,8 +2,8 @@ from django.conf.urls import include, url
 
 from pretix.control.views import (
     auth, checkin, dashboards, event, global_settings, item, main, orders,
-    organizer, pdf, search, subevents, typeahead, user, users, vouchers,
-    waitinglist,
+    organizer, pdf, search, shredder, subevents, typeahead, user, users,
+    vouchers, waitinglist,
 )
 
 urlpatterns = [
@@ -190,6 +190,10 @@ urlpatterns = [
         url(r'^orders/export/do$', orders.ExportDoView.as_view(), name='event.orders.export.do'),
         url(r'^orders/go$', orders.OrderGo.as_view(), name='event.orders.go'),
         url(r'^orders/$', orders.OrderList.as_view(), name='event.orders'),
+        url(r'^shredder/$', shredder.StartShredView.as_view(), name='event.shredder.start'),
+        url(r'^shredder/export$', shredder.ShredExportView.as_view(), name='event.shredder.export'),
+        url(r'^shredder/download/(?P<file>[^/]+)/$', shredder.ShredDownloadView.as_view(), name='event.shredder.download'),
+        url(r'^shredder/shred', shredder.ShredDoView.as_view(), name='event.shredder.shred'),
         url(r'^waitinglist/$', waitinglist.WaitingListView.as_view(), name='event.orders.waitinglist'),
         url(r'^waitinglist/auto_assign$', waitinglist.AutoAssign.as_view(), name='event.orders.waitinglist.auto'),
         url(r'^waitinglist/(?P<entry>\d+)/delete$', waitinglist.EntryDelete.as_view(),

src/pretix/control/views/event.py

@@ -369,8 +369,9 @@ class PaymentSettings(EventSettingsViewMixin, EventSettingsFormView):
             key=lambda s: s.verbose_name
         )
         for p in context['providers']:
-            if not p.is_enabled and p.is_meta and p.settings._enabled:
-                p.is_enabled = True
+            p.show_enabled = p.is_enabled
+            if p.is_meta:
+                p.show_enabled = p.settings._enabled in (True, 'True')
         return context
 
 

src/pretix/control/views/orders.py

@@ -347,6 +347,8 @@ class OrderInvoiceRegenerate(OrderView):
         else:
             if inv.canceled:
                 messages.error(self.request, _('The invoice has already been canceled.'))
+            elif inv.shredded:
+                messages.error(self.request, _('The invoice has been cleaned of personal data.'))
             else:
                 inv = regenerate_invoice(inv)
                 self.order.log_action('pretix.event.order.invoice.regenerated', user=self.request.user, data={
@@ -370,6 +372,8 @@ class OrderInvoiceReissue(OrderView):
         else:
             if inv.canceled:
                 messages.error(self.request, _('The invoice has already been canceled.'))
+            elif inv.shredded:
+                messages.error(self.request, _('The invoice has been cleaned of personal data.'))
             else:
                 c = generate_cancellation(inv)
                 if self.order.status not in (Order.STATUS_CANCELED, Order.STATUS_REFUNDED):
@@ -447,6 +451,10 @@ class InvoiceDownload(EventPermissionRequiredMixin, View):
             invoice_pdf(self.invoice.pk)
             self.invoice = Invoice.objects.get(pk=self.invoice.pk)
 
+        if self.invoice.shredded:
+            messages.error(request, _('The invoice file is no longer stored on the server.'))
+            return redirect(self.get_order_url())
+
         if not self.invoice.file:
             # This happens if we have celery installed and the file will be generated in the background
             messages.warning(request, _('The invoice file has not yet been generated, we will generate it for you '
@@ -648,7 +656,10 @@ class OrderModifyInformation(OrderQuestionsViewMixin, OrderView):
                            _("We had difficulties processing your input. Please review the errors below."))
             return self.get(request, *args, **kwargs)
         self.invoice_form.save()
-        self.order.log_action('pretix.event.order.modified', user=request.user)
+        self.order.log_action('pretix.event.order.modified', {
+            'invoice_data': self.invoice_form.cleaned_data,
+            'data': [f.cleaned_data for f in self.forms]
+        }, user=request.user)
         if self.invoice_form.has_changed():
             success_message = ('The invoice address has been updated. If you want to generate a new invoice, '
                                'you need to do this manually.')

src/pretix/control/views/shredder.py

@@ -0,0 +1,109 @@
+import logging
+from collections import OrderedDict
+
+from django.shortcuts import get_object_or_404
+from django.urls import reverse
+from django.utils.functional import cached_property
+from django.utils.translation import ugettext_lazy as _
+from django.views import View
+from django.views.generic import TemplateView
+
+from pretix.base.models import CachedFile
+from pretix.base.services.shredder import export, shred
+from pretix.base.shredder import ShredError, shred_constraints
+from pretix.base.views.async import AsyncAction
+from pretix.control.permissions import EventPermissionRequiredMixin
+
+logger = logging.getLogger(__name__)
+
+
+class ShredderMixin:
+
+    @cached_property
+    def shredders(self):
+        return OrderedDict(
+            sorted(self.request.event.get_data_shredders().items(), key=lambda s: s[1].verbose_name)
+        )
+
+
+class StartShredView(EventPermissionRequiredMixin, ShredderMixin, TemplateView):
+    permission = 'can_change_orders'
+    template_name = 'pretixcontrol/shredder/index.html'
+
+    def get_context_data(self, **kwargs):
+        ctx = super().get_context_data(**kwargs)
+        ctx['shredders'] = self.shredders
+        ctx['constraints'] = shred_constraints(self.request.event)
+        return ctx
+
+
+class ShredDownloadView(EventPermissionRequiredMixin, ShredderMixin, TemplateView):
+    permission = 'can_change_orders'
+    template_name = 'pretixcontrol/shredder/download.html'
+
+    def get_context_data(self, **kwargs):
+        ctx = super().get_context_data(**kwargs)
+        ctx['shredders'] = self.shredders
+        ctx['file'] = get_object_or_404(CachedFile, pk=kwargs.get("file"))
+        return ctx
+
+
+class ShredExportView(EventPermissionRequiredMixin, ShredderMixin, AsyncAction, View):
+    permission = 'can_change_orders'
+    task = export
+    known_errortypes = ['ShredError']
+
+    def get_success_message(self, value):
+        return None
+
+    def get_success_url(self, value):
+        return reverse('control:event.shredder.download', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+            'file': str(value)
+        })
+
+    def get_error_url(self):
+        return reverse('control:event.shredder.start', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug
+        })
+
+    def post(self, request, *args, **kwargs):
+        constr = shred_constraints(self.request.event)
+        if constr:
+            return self.error(ShredError(self.get_error_url()))
+
+        return self.do(self.request.event.id, request.POST.getlist("shredder"))
+
+
+class ShredDoView(EventPermissionRequiredMixin, ShredderMixin, AsyncAction, View):
+    permission = 'can_change_orders'
+    task = shred
+    known_errortypes = ['ShredError']
+
+    def get_success_url(self, value):
+        return reverse('control:event.shredder.start', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+        })
+
+    def get_success_message(self, value):
+        return _('The selected data was deleted successfully.')
+
+    def get_error_url(self):
+        return reverse('control:event.shredder.download', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+            'file': self.request.POST.get("file")
+        })
+
+    def post(self, request, *args, **kwargs):
+        constr = shred_constraints(self.request.event)
+        if constr:
+            return self.error(ShredError(self.get_error_url()))
+
+        if not self.request.user.check_password(request.POST.get("password")):
+            return self.error(ShredError(_("The current password you entered was not correct.")))
+
+        return self.do(self.request.event.id, request.POST.get("file"), request.POST.get("confirm_code"))

src/pretix/plugins/banktransfer/payment.py

@@ -7,6 +7,7 @@ from django.utils.translation import ugettext_lazy as _
 from i18nfield.fields import I18nFormField, I18nTextarea
 from i18nfield.strings import LazyI18nString
 
+from pretix.base.models import Order
 from pretix.base.payment import BasePaymentProvider
 
 
@@ -88,3 +89,13 @@ class BankTransfer(BasePaymentProvider):
         ctx = {'request': request, 'event': self.event,
                'payment_info': payment_info, 'order': order}
         return template.render(ctx)
+
+    def shred_payment_info(self, order: Order):
+        if not order.payment_info:
+            return
+        d = json.loads(order.payment_info)
+        d['reference'] = '█'
+        d['payer'] = '█'
+        d['_shredded'] = True
+        order.payment_info = json.dumps(d)
+        order.save(update_fields=['payment_info'])

src/pretix/plugins/paypal/payment.py

@@ -398,3 +398,23 @@ class Paypal(BasePaymentProvider):
         })
         request.session['payment_paypal_order'] = order.pk
         return self._create_payment(request, payment)
+
+    def shred_payment_info(self, order: Order):
+        d = json.loads(order.payment_info)
+        new = {
+            'id': d.get('id'),
+            'payer': {
+                'payer_info': {
+                    'email': '█'
+                }
+            },
+            'update_time': d.get('update_time'),
+            'transactions': [
+                {
+                    'amount': t.get('amount')
+                } for t in d.get('transactions', [])
+            ],
+            '_shredded': True
+        }
+        order.payment_info = json.dumps(new)
+        order.save(update_fields=['payment_info'])

src/pretix/plugins/paypal/signals.py

@@ -4,8 +4,10 @@ from django.dispatch import receiver
 from django.template.loader import get_template
 from django.utils.translation import ugettext_lazy as _
 
+from pretix.base.shredder import BaseDataShredder
 from pretix.base.signals import (
-    logentry_display, register_payment_providers, requiredaction_display,
+    logentry_display, register_data_shredders, register_payment_providers,
+    requiredaction_display,
 )
 
 
@@ -53,3 +55,32 @@ def pretixcontrol_action_display(sender, action, request, **kwargs):
 
     ctx = {'data': data, 'event': sender, 'action': action}
     return template.render(ctx, request)
+
+
+class PaymentLogsShredder(BaseDataShredder):
+    verbose_name = _('PayPal payment history')
+    identifier = 'paypal_logs'
+    description = _('This will remove payment-related history information. No download will be offered.')
+
+    def generate_files(self):
+        pass
+
+    def shred_data(self):
+        for le in self.event.logentry_set.filter(action_type="pretix.plugins.paypal.event").exclude(data=""):
+            d = le.parsed_data
+            if 'resource' in d:
+                d['resource'] = {
+                    'id': d['resource'].get('id'),
+                    'sale_id': d['resource'].get('sale_id'),
+                    'parent_payment': d['resource'].get('parent_payment'),
+                }
+            le.data = json.dumps(d)
+            le.shredded = True
+            le.save(update_fields=['data', 'shredded'])
+
+
+@receiver(register_data_shredders, dispatch_uid="paypal_shredders")
+def register_shredder(sender, **kwargs):
+    return [
+        PaymentLogsShredder,
+    ]

src/pretix/plugins/stripe/payment.py

@@ -16,7 +16,7 @@ from django.utils.http import urlquote
 from django.utils.translation import pgettext, ugettext, ugettext_lazy as _
 
 from pretix import __version__
-from pretix.base.models import Event, Quota, RequiredAction
+from pretix.base.models import Event, Order, Quota, RequiredAction
 from pretix.base.payment import BasePaymentProvider, PaymentException
 from pretix.base.services.mail import SendMailException
 from pretix.base.services.orders import mark_order_paid, mark_order_refunded
@@ -480,6 +480,34 @@ class StripeMethod(BasePaymentProvider):
         else:
             return str(url)
 
+    def shred_payment_info(self, order: Order):
+        if not order.payment_info:
+            return
+        d = json.loads(order.payment_info)
+        new = {}
+        if 'source' in d:
+            new['source'] = {
+                'id': d['source'].get('id'),
+                'type': d['source'].get('type'),
+                'brand': d['source'].get('brand'),
+                'last4': d['source'].get('last4'),
+                'bank_name': d['source'].get('bank_name'),
+                'bank': d['source'].get('bank'),
+                'bic': d['source'].get('bic'),
+                'card': {
+                    'brand': d['source'].get('card', {}).get('brand'),
+                    'country': d['source'].get('card', {}).get('cuntry'),
+                    'last4': d['source'].get('card', {}).get('last4'),
+                }
+            }
+            new['amount'] = d['amount']
+            new['currency'] = d['currency']
+            new['status'] = d['status']
+            new['id'] = d['id']
+            new['_shredded'] = True
+        order.payment_info = json.dumps(new)
+        order.save(update_fields=['payment_info'])
+
 
 class StripeCC(StripeMethod):
     identifier = 'stripe'

src/pretix/plugins/stripe/signals.py

@@ -8,9 +8,10 @@ from django.template.loader import get_template
 from django.utils.translation import ugettext_lazy as _
 
 from pretix.base.settings import settings_hierarkey
+from pretix.base.shredder import BaseDataShredder
 from pretix.base.signals import (
-    logentry_display, register_global_settings, register_payment_providers,
-    requiredaction_display,
+    logentry_display, register_data_shredders, register_global_settings,
+    register_payment_providers, requiredaction_display,
 )
 from pretix.plugins.stripe.forms import StripeKeyValidator
 from pretix.presale.signals import html_head
@@ -134,3 +135,30 @@ def register_global_settings(sender, **kwargs):
             ),
         )),
     ])
+
+
+class PaymentLogsShredder(BaseDataShredder):
+    verbose_name = _('Stripe payment history')
+    identifier = 'stripe_logs'
+    description = _('This will remove payment-related history information. No download will be offered.')
+
+    def generate_files(self):
+        pass
+
+    def shred_data(self):
+        for le in self.event.logentry_set.filter(action_type="pretix.plugins.stripe.event").exclude(data=""):
+            d = le.parsed_data
+            if 'data' in d:
+                for k, v in list(d['data']['object'].items()):
+                    if v not in ('reason', 'status', 'failure_message', 'object', 'id'):
+                        d['data']['object'][k] = '█'
+                le.data = json.dumps(d)
+                le.shredded = True
+                le.save(update_fields=['data', 'shredded'])
+
+
+@receiver(register_data_shredders, dispatch_uid="stripe_shredders")
+def register_shredder(sender, **kwargs):
+    return [
+        PaymentLogsShredder,
+    ]

src/pretix/presale/views/order.py

@@ -447,7 +447,10 @@ class OrderModify(EventViewMixin, OrderDetailMixin, OrderQuestionsViewMixin, Tem
                            _("We had difficulties processing your input. Please review the errors below."))
             return self.get(request, *args, **kwargs)
         self.invoice_form.save()
-        self.order.log_action('pretix.event.order.modified')
+        self.order.log_action('pretix.event.order.modified', {
+            'invoice_data': self.invoice_form.cleaned_data,
+            'data': [f.cleaned_data for f in self.forms]
+        })
         if self.invoice_form.has_changed():
             success_message = ('Your invoice address has been updated. Please contact us if you need us '
                                'to regenerate your invoice.')
@@ -660,6 +663,10 @@ class InvoiceDownload(EventViewMixin, OrderDetailMixin, View):
             invoice_pdf(invoice.pk)
             invoice = Invoice.objects.get(pk=invoice.pk)
 
+        if invoice.shredded:
+            messages.error(request, _('The invoice file is no longer stored on the server.'))
+            return redirect(self.get_order_url())
+
         if not invoice.file:
             # This happens if we have celery installed and the file will be generated in the background
             messages.warning(request, _('The invoice file has not yet been generated, we will generate it for you '

src/pretix/static/pretixcontrol/scss/_forms.scss

@@ -337,3 +337,19 @@ table td > .checkbox input[type="checkbox"] {
     line-height: 40px;
   }
 }
+.accordion-radio {
+  display: block;
+  margin: 0;
+}
+.panel-default>.accordion-radio>.panel-heading {
+  color: #333;
+  background-color: #f5f5f5;
+  padding: 12px 15px;
+
+  input[type=checkbox] {
+    margin-top: 0;
+    margin-right: 5px;
+    position: relative;
+    top: 2px;
+  }
+}

src/tests/base/test_shredders.py

@@ -0,0 +1,355 @@
+import json
+import os
+from datetime import timedelta
+from decimal import Decimal
+
+import pytest
+from django.core.files.base import ContentFile
+from django.utils.timezone import now
+
+from pretix.base.models import (
+    CachedCombinedTicket, CachedTicket, Event, InvoiceAddress, Order,
+    OrderPosition, Organizer, QuestionAnswer,
+)
+from pretix.base.services.invoices import generate_invoice, invoice_pdf_task
+from pretix.base.services.tickets import generate, generate_order
+from pretix.base.shredder import (
+    AttendeeNameShredder, CachedTicketShredder, EmailAddressShredder,
+    InvoiceAddressShredder, InvoiceShredder, PaymentInfoShredder,
+    QuestionAnswerShredder, WaitingListShredder, shred_constraints,
+)
+
+
+@pytest.fixture
+def event():
+    o = Organizer.objects.create(name='Dummy', slug='dummy')
+    event = Event.objects.create(
+        organizer=o, name='Dummy', slug='dummy',
+        date_from=now(), plugins='pretix.plugins.banktransfer,pretix.plugins.ticketoutputpdf'
+    )
+    return event
+
+
+@pytest.fixture
+def item(event):
+    return event.items.create(
+        name='Early-bird ticket',
+        category=None, default_price=23,
+        admission=True
+    )
+
+
+@pytest.fixture
+def order(event, item):
+    o = Order.objects.create(
+        code='FOO', event=event, email='dummy@dummy.test',
+        status=Order.STATUS_PENDING,
+        datetime=now(), expires=now() + timedelta(days=10),
+        total=14, payment_provider='banktransfer', locale='en'
+    )
+    event.settings.set('attendee_names_asked', True)
+    event.settings.set('locales', ['en', 'de'])
+    OrderPosition.objects.create(
+        order=o,
+        item=item,
+        variation=None,
+        price=Decimal("14"),
+        attendee_name="Peter",
+        attendee_email="foo@example.org"
+    )
+    return o
+
+
+@pytest.fixture
+def question(event, item):
+    q = event.questions.create(question="T-Shirt size", type="C", identifier="ABC")
+    q.items.add(item)
+    q.options.create(answer="XL", identifier="LVETRWVU")
+    return q
+
+
+@pytest.mark.django_db
+def test_email_shredder(event, order):
+    l1 = order.log_action(
+        'pretix.event.order.email.expired',
+        data={
+            'recipient': 'dummy@dummy.test',
+            'message': 'Hello Peter@,',
+            'subject': 'Foo'
+        }
+    )
+    l2 = order.log_action(
+        'pretix.event.order.contact.changed',
+        data={
+            'old_email': 'dummy@dummy.test',
+            'new_email': 'foo@bar.com',
+        }
+    )
+
+    s = EmailAddressShredder(event)
+    f = list(s.generate_files())
+    assert json.loads(f[0][2]) == {
+        order.code: 'dummy@dummy.test'
+    }
+    assert json.loads(f[1][2]) == {
+        '{}-{}'.format(order.code, 1): 'foo@example.org'
+    }
+    s.shred_data()
+    order.refresh_from_db()
+    assert order.email is None
+    assert order.positions.first().attendee_email is None
+    l1.refresh_from_db()
+    assert '@' not in l1.data
+    assert 'Foo' not in l1.data
+    l2.refresh_from_db()
+    assert '@' not in l2.data
+
+
+@pytest.mark.django_db
+def test_waitinglist_shredder(event, item):
+    q = event.quotas.create(size=5)
+    q.items.add(item)
+    wle = event.waitinglistentries.create(
+        item=item, email='foo@example.org'
+    )
+    wle.send_voucher()
+    assert '@' in wle.voucher.comment
+    assert '@' in wle.voucher.all_logentries().last().data
+    s = WaitingListShredder(event)
+    f = list(s.generate_files())
+    assert json.loads(f[0][2]) == [
+        {
+            'id': wle.pk,
+            'item': item.pk,
+            'variation': None,
+            'subevent': None,
+            'voucher': wle.voucher.pk,
+            'created': wle.created.isoformat().replace('+00:00', 'Z'),
+            'locale': 'en',
+            'email': 'foo@example.org'
+        }
+    ]
+    s.shred_data()
+    wle.refresh_from_db()
+    wle.voucher.refresh_from_db()
+    assert '@' not in wle.email
+    assert '@' not in wle.voucher.comment
+    assert '@' not in wle.voucher.all_logentries().last().data
+
+
+@pytest.mark.django_db
+def test_attendee_name_shredder(event, order):
+    l1 = order.log_action(
+        'pretix.event.order.modified',
+        data={
+            "data": [{"attendee_name": "Hans", "question_1": "Test"}],
+            "invoice_data": {"name": "Foo"}
+        }
+    )
+
+    s = AttendeeNameShredder(event)
+    f = list(s.generate_files())
+    assert json.loads(f[0][2]) == {
+        '{}-{}'.format(order.code, 1): 'Peter'
+    }
+    s.shred_data()
+    order.refresh_from_db()
+    assert order.positions.first().attendee_name is None
+    l1.refresh_from_db()
+    assert 'Hans' not in l1.data
+    assert 'Foo' in l1.data
+    assert 'Test' in l1.data
+
+
+@pytest.mark.django_db
+def test_invoice_address_shredder(event, order):
+    l1 = order.log_action(
+        'pretix.event.order.modified',
+        data={
+            "data": [{"attendee_name": "Hans", "question_1": "Test"}],
+            "invoice_data": {"name": "Peter", "country": "DE", "is_business": False, "internal_reference": "",
+                             "company": "ACME", "street": "Sesam Street", "city": "Sample City", "zipcode": "12345"}
+        }
+    )
+    ia = InvoiceAddress.objects.create(company='Acme Company', street='221B Baker Street',
+                                       zipcode='12345', city='London', country='UK',
+                                       order=order)
+    s = InvoiceAddressShredder(event)
+    f = list(s.generate_files())
+    assert json.loads(f[0][2]) == {
+        order.code: {
+            'city': 'London',
+            'company': 'Acme Company',
+            'country': 'UK',
+            'internal_reference': '',
+            'is_business': False,
+            'last_modified': ia.last_modified.isoformat().replace('+00:00', 'Z'),
+            'name': '',
+            'street': '221B Baker Street',
+            'vat_id': '',
+            'vat_id_validated': False,
+            'zipcode': '12345'
+        }
+    }
+    s.shred_data()
+    order.refresh_from_db()
+    assert not InvoiceAddress.objects.filter(order=order).exists()
+    l1.refresh_from_db()
+    assert l1.parsed_data == {
+        "data": [{"attendee_name": "Hans", "question_1": "Test"}],
+        "invoice_data": {"name": "█", "country": "█", "is_business": False, "internal_reference": "", "company": "█",
+                         "street": "█", "city": "█", "zipcode": "█"}
+    }
+
+
+@pytest.mark.django_db
+def test_question_answer_shredder(event, order, question):
+    opt = question.options.first()
+    l1 = order.log_action(
+        'pretix.event.order.modified',
+        data={
+            "data": [
+                {
+                    "attendee_name": "Hans",
+                    "question_%d" % question.pk: [{"id": opt.pk, "type": "QuestionOption"}]
+                }
+            ],
+        }
+    )
+    qa = QuestionAnswer.objects.create(
+        orderposition=order.positions.first(),
+        question=question,
+        answer='S'
+    )
+    qa.file.save('foo.pdf', ContentFile('foo'))
+    fname = qa.file.path
+    assert os.path.exists(fname)
+    qa.options.add(opt)
+    s = QuestionAnswerShredder(event)
+    f = list(s.generate_files())
+    assert json.loads(f[0][2]) == {
+        '{}-1'.format(order.code): [{
+            'question': question.pk,
+            'answer': 'S',
+            'question_identifier': question.identifier,
+            'options': [opt.pk],
+            'option_identifiers': [opt.identifier],
+        }]
+    }
+    s.shred_data()
+    order.refresh_from_db()
+    assert not os.path.exists(fname)
+    assert not QuestionAnswer.objects.filter(pk=qa.pk).exists()
+    l1.refresh_from_db()
+    assert l1.parsed_data == {
+        "data": [{"attendee_name": "Hans", "question_%d" % question.pk: "█"}],
+    }
+
+
+@pytest.mark.django_db
+def test_invoice_shredder(event, order):
+    InvoiceAddress.objects.create(company='Acme Company', street='221B Baker Street',
+                                  zipcode='12345', city='London', country='UK',
+                                  order=order)
+    inv = generate_invoice(order)
+    invoice_pdf_task.apply(args=(inv.pk,))
+    inv.refresh_from_db()
+    assert inv.invoice_to == "Acme Company\n\n221B Baker Street\n12345 London"
+    assert inv.file
+    fname = inv.file.path
+    assert os.path.exists(fname)
+    s = InvoiceShredder(event)
+    f = list(s.generate_files())
+    assert len(f) == 1
+    s.shred_data()
+    inv.refresh_from_db()
+
+    assert "Acme" not in inv.invoice_to
+    assert "icket" not in inv.lines.first().description
+    assert not inv.file
+    assert not os.path.exists(fname)
+
+
+@pytest.mark.django_db
+def test_cached_tickets(event, order):
+    generate(order.positions.first().pk, 'pdf')
+    generate_order(order.pk, 'pdf')
+
+    ct = CachedTicket.objects.get(order_position=order.positions.first(), provider='pdf')
+    cct = CachedCombinedTicket.objects.get(order=order, provider='pdf')
+    assert ct.file
+    assert cct.file
+    ct_fname = ct.file.path
+    cct_fname = cct.file.path
+    assert os.path.exists(ct_fname)
+    assert os.path.exists(cct_fname)
+    s = CachedTicketShredder(event)
+    assert s.generate_files() is None
+    s.shred_data()
+
+    assert not CachedTicket.objects.filter(order_position=order.positions.first(), provider='pdf').exists()
+    assert not CachedCombinedTicket.objects.filter(order=order, provider='pdf').exists()
+    assert not os.path.exists(ct_fname)
+    assert not os.path.exists(cct_fname)
+
+
+@pytest.mark.django_db
+def test_payment_info_shredder(event, order):
+    order.payment_info = json.dumps({
+        'reference': 'Verwendungszweck 1',
+        'date': '2018-05-01',
+        'payer': 'Hans',
+        'trans_id': 12
+    })
+    order.save()
+
+    s = PaymentInfoShredder(event)
+    assert s.generate_files() is None
+    s.shred_data()
+
+    order.refresh_from_db()
+    assert json.loads(order.payment_info) == {
+        '_shredded': True,
+        'reference': '█',
+        'date': '2018-05-01',
+        'payer': '█',
+        'trans_id': 12
+    }
+
+
+@pytest.mark.django_db
+def test_shred_constraint_offline(event):
+    event.live = True
+    event.date_from = now() - timedelta(days=365)
+    assert shred_constraints(event)
+
+
+@pytest.mark.django_db
+def test_shred_constraint_60_days(event):
+    event.live = False
+    event.date_from = now() - timedelta(days=62)
+    event.date_to = now() - timedelta(days=62)
+    assert shred_constraints(event) is None
+    event.date_from = now() - timedelta(days=52)
+    event.date_to = now() - timedelta(days=52)
+    assert shred_constraints(event)
+    event.date_from = now() - timedelta(days=62)
+    event.date_to = now() - timedelta(days=52)
+    assert shred_constraints(event)
+
+
+@pytest.mark.django_db
+def test_shred_constraint_60_days_subevents(event):
+    event.has_subevents = True
+    event.live = False
+
+    event.subevents.create(
+        date_from=now() - timedelta(days=62),
+        date_to=now() - timedelta(days=62)
+    )
+    assert shred_constraints(event) is None
+    event.subevents.create(
+        date_from=now() - timedelta(days=62),
+        date_to=now() - timedelta(days=52)
+    )
+    assert shred_constraints(event)

src/tests/control/test_shredders.py

@@ -0,0 +1,166 @@
+import datetime
+import json
+from io import BytesIO
+from zipfile import ZipFile
+
+from django.utils.timezone import now
+from tests.base import SoupTest
+
+from pretix.base.models import Event, Order, Organizer, Team, User
+
+
+class EventShredderTest(SoupTest):
+    def setUp(self):
+        super().setUp()
+        self.user = User.objects.create_user('dummy@dummy.dummy', 'dummy')
+        self.orga1 = Organizer.objects.create(name='CCC', slug='ccc')
+        self.orga2 = Organizer.objects.create(name='MRM', slug='mrm')
+        self.event1 = Event.objects.create(
+            organizer=self.orga1, name='30C3', slug='30c3',
+            date_from=datetime.datetime(2013, 12, 26, tzinfo=datetime.timezone.utc),
+            plugins='pretix.plugins.banktransfer,pretix.plugins.stripe,tests.testdummy'
+        )
+
+        t = Team.objects.create(organizer=self.orga1, can_create_events=True, can_change_event_settings=True,
+                                can_change_items=True, can_change_orders=True)
+        t.members.add(self.user)
+        t.limit_events.add(self.event1)
+        self.order = Order.objects.create(
+            code='FOO', event=self.event1, email='dummy@dummy.test',
+            status=Order.STATUS_PENDING,
+            datetime=now(), expires=now(),
+            total=14, payment_provider='banktransfer', locale='en'
+        )
+
+        self.client.login(email='dummy@dummy.dummy', password='dummy')
+
+    def test_shred_simple(self):
+        doc = self.get_doc('/control/event/%s/%s/shredder/' % (self.orga1.slug, self.event1.slug))
+        assert doc.select("input[value=order_emails]")
+        assert doc.select("input[value=stripe_logs]")
+        doc = self.post_doc('/control/event/%s/%s/shredder/export' % (self.orga1.slug, self.event1.slug), {
+            'shredder': 'order_emails'
+        })
+        assert doc.select("a.btn-primary")[0].text.strip() == "Download data"
+        dlink = doc.select("a.btn-primary")[0].attrs['href']
+        zipfiler = self.client.get(dlink)
+        with ZipFile(BytesIO(zipfiler.getvalue()), 'r') as zipfile:
+            indexdata = json.loads(zipfile.read('index.json').decode())
+            assert indexdata['shredders'] == ['order_emails']
+            assert indexdata['organizer'] == 'ccc'
+            assert indexdata['event'] == '30c3'
+            assert zipfile.read('CONFIRM_CODE.txt').decode() == indexdata['confirm_code']
+
+            maildata = json.loads(zipfile.read('emails-by-order.json').decode())
+            assert maildata == {
+                'FOO': 'dummy@dummy.test'
+            }
+        doc = self.post_doc('/control/event/%s/%s/shredder/shred' % (self.orga1.slug, self.event1.slug), {
+            'confirm_code': indexdata['confirm_code'],
+            'file': doc.select("input[name=file]")[0].attrs['value'],
+            'password': 'dummy'
+        })
+        assert doc.select('.alert-success')
+        self.order.refresh_from_db()
+        assert not self.order.email
+
+    def test_shred_password_wrong(self):
+        doc = self.get_doc('/control/event/%s/%s/shredder/' % (self.orga1.slug, self.event1.slug))
+        assert doc.select("input[value=order_emails]")
+        assert doc.select("input[value=stripe_logs]")
+        doc = self.post_doc('/control/event/%s/%s/shredder/export' % (self.orga1.slug, self.event1.slug), {
+            'shredder': 'order_emails'
+        })
+        assert doc.select("a.btn-primary")[0].text.strip() == "Download data"
+        dlink = doc.select("a.btn-primary")[0].attrs['href']
+        zipfiler = self.client.get(dlink)
+        with ZipFile(BytesIO(zipfiler.getvalue()), 'r') as zipfile:
+            indexdata = json.loads(zipfile.read('index.json').decode())
+            assert indexdata['shredders'] == ['order_emails']
+            assert indexdata['organizer'] == 'ccc'
+            assert indexdata['event'] == '30c3'
+            assert zipfile.read('CONFIRM_CODE.txt').decode() == indexdata['confirm_code']
+
+            maildata = json.loads(zipfile.read('emails-by-order.json').decode())
+            assert maildata == {
+                'FOO': 'dummy@dummy.test'
+            }
+        doc = self.post_doc('/control/event/%s/%s/shredder/shred' % (self.orga1.slug, self.event1.slug), {
+            'confirm_code': indexdata['confirm_code'],
+            'file': doc.select("input[name=file]")[0].attrs['value'],
+            'password': 'test'
+        })
+        assert doc.select('.alert-danger')
+        self.order.refresh_from_db()
+        assert self.order.email
+
+    def test_shred_confirm_code_wrong(self):
+        doc = self.get_doc('/control/event/%s/%s/shredder/' % (self.orga1.slug, self.event1.slug))
+        assert doc.select("input[value=order_emails]")
+        assert doc.select("input[value=stripe_logs]")
+        doc = self.post_doc('/control/event/%s/%s/shredder/export' % (self.orga1.slug, self.event1.slug), {
+            'shredder': 'order_emails'
+        })
+        assert doc.select("a.btn-primary")[0].text.strip() == "Download data"
+        dlink = doc.select("a.btn-primary")[0].attrs['href']
+        zipfiler = self.client.get(dlink)
+        with ZipFile(BytesIO(zipfiler.getvalue()), 'r') as zipfile:
+            indexdata = json.loads(zipfile.read('index.json').decode())
+            assert indexdata['shredders'] == ['order_emails']
+            assert indexdata['organizer'] == 'ccc'
+            assert indexdata['event'] == '30c3'
+            assert zipfile.read('CONFIRM_CODE.txt').decode() == indexdata['confirm_code']
+
+            maildata = json.loads(zipfile.read('emails-by-order.json').decode())
+            assert maildata == {
+                'FOO': 'dummy@dummy.test'
+            }
+        doc = self.post_doc('/control/event/%s/%s/shredder/shred' % (self.orga1.slug, self.event1.slug), {
+            'confirm_code': indexdata['confirm_code'][::-1] + 'A',
+            'file': doc.select("input[name=file]")[0].attrs['value'],
+            'password': 'dummy'
+        })
+        assert doc.select('.alert-danger')
+        self.order.refresh_from_db()
+        assert self.order.email
+
+    def test_shred_constraints(self):
+        self.event1.live = True
+        self.event1.save()
+        doc = self.get_doc('/control/event/%s/%s/shredder/' % (self.orga1.slug, self.event1.slug))
+        assert not doc.select("input[value=order_emails]")
+        doc = self.post_doc('/control/event/%s/%s/shredder/export' % (self.orga1.slug, self.event1.slug), {
+            'shredder': 'order_emails'
+        })
+        assert doc.select('.alert-danger')
+
+    def test_shred_something_happened(self):
+        doc = self.get_doc('/control/event/%s/%s/shredder/' % (self.orga1.slug, self.event1.slug))
+        assert doc.select("input[value=order_emails]")
+        assert doc.select("input[value=stripe_logs]")
+        doc = self.post_doc('/control/event/%s/%s/shredder/export' % (self.orga1.slug, self.event1.slug), {
+            'shredder': 'order_emails'
+        })
+        assert doc.select("a.btn-primary")[0].text.strip() == "Download data"
+        dlink = doc.select("a.btn-primary")[0].attrs['href']
+        zipfiler = self.client.get(dlink)
+        with ZipFile(BytesIO(zipfiler.getvalue()), 'r') as zipfile:
+            indexdata = json.loads(zipfile.read('index.json').decode())
+            assert indexdata['shredders'] == ['order_emails']
+            assert indexdata['organizer'] == 'ccc'
+            assert indexdata['event'] == '30c3'
+            assert zipfile.read('CONFIRM_CODE.txt').decode() == indexdata['confirm_code']
+
+            maildata = json.loads(zipfile.read('emails-by-order.json').decode())
+            assert maildata == {
+                'FOO': 'dummy@dummy.test'
+            }
+        self.order.log_action('dummy')
+        doc = self.post_doc('/control/event/%s/%s/shredder/shred' % (self.orga1.slug, self.event1.slug), {
+            'confirm_code': indexdata['confirm_code'],
+            'file': doc.select("input[name=file]")[0].attrs['value'],
+            'password': 'dummy'
+        })
+        assert doc.select('.alert-danger')
+        self.order.refresh_from_db()
+        assert self.order.email