Fix #678 -- Data shredders for personally identifiable information (#817)

* Add data shredders for PII

* First working shredder

* Add more shredders

* Add new shredders and download confirmation

* tmp

* PayPal, Stripe, banktransfer

* Add icon to logs

* Untested payment log shredders

* Add waiting list shredder

* First tests

* Add tests for shredders

* Improve templats, link to shredder

* Test payment info shredders

* More tests

* Documentation

* Fix enabled flag in payment provider overview

* Fix minor issues

src/pretix/control/templates/pretixcontrol/event/delete.html

@@ -51,12 +51,18 @@
             <p>
                 {% trans "You can instead take your shop offline. This will hide it from everyone except from the organizer teams you configured to have access to the event." %}
             </p>
-            <form action="" method="post">
+            <form action="{% url "control:event.live" event=request.event.slug organizer=request.organizer.slug %}"
+                    method="post">
                 {% csrf_token %}
                 <input type="hidden" name="live" value="false">
 
                 <div class="form-group submit-group">
+                    <a href="{% url "control:event.shredder.start" event=request.event.slug organizer=request.organizer.slug %}" class="btn btn-danger btn-save">
+                        <span class="fa fa-eraser"></span>
+                        {% trans "Delete personal data" %}
+                    </a>
                     <button type="submit" class="btn btn-primary btn-save">
+                        <span class="fa fa-power-off"></span>
                         {% trans "Go offline" %}
                     </button>
                 </div>
@@ -64,6 +70,12 @@
         {% else %}
             <p>
                 {% trans "However, since your shop is offline, it is only visible to the organizing team according to the permissions you configured." %}
+                <div class="form-group submit-group">
+                    <a href="{% url "control:event.shredder.start" event=request.event.slug organizer=request.organizer.slug %}" class="btn btn-danger btn-save">
+                        <span class="fa fa-eraser"></span>
+                        {% trans "Delete personal data" %}
+                    </a>
+                </div>
             </p>
         {% endif %}
     {% endif %}

src/pretix/control/templates/pretixcontrol/event/index.html

@@ -106,6 +106,12 @@
                     <div class="row">
                         <div class="col-lg-2 col-sm-6 col-xs-12">
                             <span class="fa fa-clock-o"></span> {{ log.datetime|date:"SHORT_DATETIME_FORMAT" }}
+                            {% if log.shredded %}
+                                <span class="fa fa-eraser fa-danger fa-fw"
+                                      data-toggle="tooltip"
+                                      title="{% trans "Personal data was cleared from this log entry." %}">
+                                </span>
+                            {% endif %}
                         </div>
                         <div class="col-lg-2 col-sm-6 col-xs-12">
                             {% if log.user %}

src/pretix/control/templates/pretixcontrol/event/logs.html

@@ -31,6 +31,12 @@
                 <div class="row">
                     <div class="col-lg-2 col-sm-6 col-xs-12">
                         <span class="fa fa-clock-o"></span> {{ log.datetime|date:"SHORT_DATETIME_FORMAT" }}
+                        {% if log.shredded %}
+                            <span class="fa fa-eraser fa-danger fa-fw"
+                                  data-toggle="tooltip"
+                                  title="{% trans "Personal data was cleared from this log entry." %}">
+                            </span>
+                        {% endif %}
                     </div>
                     <div class="col-lg-2 col-sm-6 col-xs-12">
                         {% if log.user %}

src/pretix/control/templates/pretixcontrol/event/payment.html

@@ -14,7 +14,7 @@
                             <strong>{{ provider.verbose_name }}</strong>
                         </td>
                         <td>
-                            {% if provider.is_enabled %}
+                            {% if provider.show_enabled %}
                                 <span class="text-success">
                                     <span class="fa fa-check"></span>
                                     {% trans "Enabled" %}

src/pretix/control/templates/pretixcontrol/event/settings.html

@@ -78,10 +78,18 @@
             <button type="submit" class="btn btn-primary btn-save">
                 {% trans "Save" %}
             </button>
-            <a href="{% url "control:event.delete" organizer=request.organizer.slug event=request.event.slug %}"
-                    class="btn {% if request.event.allow_delete %}{% endif %} btn-danger btn-lg pull-left">
-                {% trans "Delete event" %}
-            </a>
+            <div class="pull-left">
+                <a href="{% url "control:event.delete" organizer=request.organizer.slug event=request.event.slug %}"
+                        class="btn {% if request.event.allow_delete %}{% endif %} btn-danger btn-lg">
+                    <span class="fa fa-trash"></span>
+                    {% trans "Delete event" %}
+                </a>
+                <a href="{% url "control:event.shredder.start" organizer=request.organizer.slug event=request.event.slug %}"
+                        class="btn btn-danger btn-lg">
+                    <span class="fa fa-eraser"></span>
+                    {% trans "Delete personal data" %}
+                </a>
+            </div>
         </div>
     </form>
 {% endblock %}

src/pretix/control/templates/pretixcontrol/includes/logs.html

@@ -10,12 +10,19 @@
                         <span class="fa fa-id-card fa-danger fa-fw"
                                 data-toggle="tooltip"
                                 title="{% trans "This change was performed by a pretix administrator." %}">
-                            </span>
+                        </span>
                     {% else %}
                         <span class="fa fa-user fa-fw"></span>
                     {% endif %}
                     {{ log.user.get_full_name }}
                 {% endif %}
+                {% if log.shredded %}
+                    <span class="fa fa-eraser fa-danger fa-fw"
+                          data-toggle="tooltip"
+                          title="{% trans "Personal data was cleared from this log entry." %}">
+
+                    </span>
+                {% endif %}
             </p>
 
             <p>

src/pretix/control/templates/pretixcontrol/order/mail_history.html

@@ -24,6 +24,9 @@
                         {% if log.display %}
                             <br/><span class="fa fa-fw fa-comment-o"></span> {{ log.display }}
                         {% endif %}
+                        {% if log.parsed_data.recipient %}
+                            <br/><span class="fa fa-fw fa-envelope-o"></span> {{ log.parsed_data.recipient }}
+                        {% endif %}
                     </p>
                     {% if log.parsed_data.subject.items %}
                         <div class="alert alert-info">

src/pretix/control/templates/pretixcontrol/shredder/download.html

@@ -0,0 +1,57 @@
+{% extends "pretixcontrol/event/base.html" %}
+{% load i18n %}
+{% load bootstrap3 %}
+{% load order_overview %}
+{% block title %}{% trans "Data shredder" %}{% endblock %}
+{% block content %}
+    <h1>
+        {% trans "Data shredder" %}
+    </h1>
+    <form action="{% url "control:event.shredder.shred" event=request.event.slug organizer=request.organizer.slug %}"
+          method="post" class="form-horizontal" data-asynctask>
+        {% csrf_token %}
+        <fieldset>
+            <legend>{% trans "Step 1: Download data" %}</legend>
+            <p>
+                {% blocktrans trimmed %}
+                    You are about to permamanently delete data from the server, even though you might be required to
+                    keep
+                    some of this data on file. You should therefore download the following file and store it in a safe
+                    place:
+                {% endblocktrans %}
+            </p>
+            <p>
+                <a href="{% url "cachedfile.download" id=file.pk %}" class="btn btn-primary btn-lg">
+                    {% trans "Download data" %}
+                </a>
+            </p>
+        </fieldset>
+        <fieldset>
+            <legend>{% trans "Step 2: Confirm download" %}</legend>
+            <p>
+                {% blocktrans trimmed %}
+                    In the downloaded file, there is a text file named "CONFIRM_CODE.txt" with a six-character code.
+                    Please enter this code here to confirm that you successfully downloaded the file.
+                {% endblocktrans %}
+            </p>
+            <input type="text" class="form-control" name="confirm_code" required placeholder="{% trans "Confirmation code" %}">
+            <br>
+        </fieldset>
+        <fieldset>
+            <legend>{% trans "Step 3: Confirm deletion" %}</legend>
+            <p>
+                {% blocktrans trimmed with event=request.event.name %}
+                Please re-check that you are fully certain that you want to delete the selected categories of data from the event <strong>{{ event }}</strong>.
+                    In this case, please enter your user password here:
+                {% endblocktrans %}
+            </p>
+            <input type="password" class="form-control" name="password" required placeholder="{% trans "Your password" %}">
+        </fieldset>
+        <input type="hidden" name="file" value="{{ file.pk }}">
+        <div class="form-group submit-group">
+            <button type="submit" class="btn btn-primary btn-save">
+                {% trans "Continue" %}
+            </button>
+        </div>
+    </form>
+{% endblock %}

src/pretix/control/templates/pretixcontrol/shredder/index.html

@@ -0,0 +1,73 @@
+{% extends "pretixcontrol/event/base.html" %}
+{% load i18n %}
+{% load bootstrap3 %}
+{% load order_overview %}
+{% block title %}{% trans "Data shredder" %}{% endblock %}
+{% block content %}
+    <h1>
+        {% trans "Data shredder" %}
+    </h1>
+    <p>
+        {% blocktrans trimmed %}
+            This feature allows you to remove personal data from this event. You will first select what kind of data
+            you want to shred, then you are able to download the affected data and after you confirmed the download,
+            the data will be removed from the server's database. The data might still exist in backups for a limited
+            period of time.
+        {% endblocktrans %}
+        <strong>
+            {% blocktrans trimmed %}
+                Using this will not remove the orders for your event, it just scrubs them of data that can be linked
+                to individual persons.
+            {% endblocktrans %}
+        </strong>
+    </p>
+    <div class="alert alert-legal">
+        <strong>
+            {% blocktrans trimmed %}
+                It is within your own responsibility to check if you are allowed to delete the affected data in your
+                legislation, e.g. for reasons of taxation.
+            {% endblocktrans %}
+        </strong>
+        {% blocktrans trimmed %}
+            For most categories of data, you will be able to partially download the data to store it offline. Some
+            kinds of data (such as some payment information) as well as historical log data cannot be downloaded at
+            the moment.
+        {% endblocktrans %}
+        <div class="clear"></div>
+    </div>
+    {% if constraints %}
+        <div class="alert alert-danger">
+            {{ constraints }}
+        </div>
+    {% else %}
+        <form action="{% url "control:event.shredder.export" event=request.event.slug organizer=request.organizer.slug %}"
+                method="post" class="form-horizontal" data-asynctask>
+            <legend>{% trans "Data selection" %}</legend>
+            {% csrf_token %}
+            <div class="panel-group" id="payment_accordion">
+                {% for ident, shredder in shredders.items %}
+                    <div class="panel panel-default">
+                        <label class="accordion-radio">
+                            <div class="panel-heading">
+                                <h4 class="panel-title">
+                                    <input type="checkbox" name="shredder" value="{{ shredder.identifier }}">
+                                    <strong>{{ shredder.verbose_name }}</strong>
+                                </h4>
+                            </div>
+                        </label>
+                        <div id="payment_{{ p.provider.identifier }}" class="panel-collapse in">
+                            <div class="panel-body">
+                                {{ shredder.description|safe }}
+                            </div>
+                        </div>
+                    </div>
+                {% endfor %}
+            </div>
+            <div class="form-group submit-group">
+                <button type="submit" class="btn btn-primary btn-save">
+                    {% trans "Continue" %}
+                </button>
+            </div>
+        </form>
+    {% endif %}
+{% endblock %}

src/pretix/control/urls.py

@@ -2,8 +2,8 @@ from django.conf.urls import include, url
 
 from pretix.control.views import (
     auth, checkin, dashboards, event, global_settings, item, main, orders,
-    organizer, pdf, search, subevents, typeahead, user, users, vouchers,
-    waitinglist,
+    organizer, pdf, search, shredder, subevents, typeahead, user, users,
+    vouchers, waitinglist,
 )
 
 urlpatterns = [
@@ -190,6 +190,10 @@ urlpatterns = [
         url(r'^orders/export/do$', orders.ExportDoView.as_view(), name='event.orders.export.do'),
         url(r'^orders/go$', orders.OrderGo.as_view(), name='event.orders.go'),
         url(r'^orders/$', orders.OrderList.as_view(), name='event.orders'),
+        url(r'^shredder/$', shredder.StartShredView.as_view(), name='event.shredder.start'),
+        url(r'^shredder/export$', shredder.ShredExportView.as_view(), name='event.shredder.export'),
+        url(r'^shredder/download/(?P<file>[^/]+)/$', shredder.ShredDownloadView.as_view(), name='event.shredder.download'),
+        url(r'^shredder/shred', shredder.ShredDoView.as_view(), name='event.shredder.shred'),
         url(r'^waitinglist/$', waitinglist.WaitingListView.as_view(), name='event.orders.waitinglist'),
         url(r'^waitinglist/auto_assign$', waitinglist.AutoAssign.as_view(), name='event.orders.waitinglist.auto'),
         url(r'^waitinglist/(?P<entry>\d+)/delete$', waitinglist.EntryDelete.as_view(),

src/pretix/control/views/event.py

@@ -369,8 +369,9 @@ class PaymentSettings(EventSettingsViewMixin, EventSettingsFormView):
             key=lambda s: s.verbose_name
         )
         for p in context['providers']:
-            if not p.is_enabled and p.is_meta and p.settings._enabled:
-                p.is_enabled = True
+            p.show_enabled = p.is_enabled
+            if p.is_meta:
+                p.show_enabled = p.settings._enabled in (True, 'True')
         return context
 
 

src/pretix/control/views/orders.py

@@ -347,6 +347,8 @@ class OrderInvoiceRegenerate(OrderView):
         else:
             if inv.canceled:
                 messages.error(self.request, _('The invoice has already been canceled.'))
+            elif inv.shredded:
+                messages.error(self.request, _('The invoice has been cleaned of personal data.'))
             else:
                 inv = regenerate_invoice(inv)
                 self.order.log_action('pretix.event.order.invoice.regenerated', user=self.request.user, data={
@@ -370,6 +372,8 @@ class OrderInvoiceReissue(OrderView):
         else:
             if inv.canceled:
                 messages.error(self.request, _('The invoice has already been canceled.'))
+            elif inv.shredded:
+                messages.error(self.request, _('The invoice has been cleaned of personal data.'))
             else:
                 c = generate_cancellation(inv)
                 if self.order.status not in (Order.STATUS_CANCELED, Order.STATUS_REFUNDED):
@@ -447,6 +451,10 @@ class InvoiceDownload(EventPermissionRequiredMixin, View):
             invoice_pdf(self.invoice.pk)
             self.invoice = Invoice.objects.get(pk=self.invoice.pk)
 
+        if self.invoice.shredded:
+            messages.error(request, _('The invoice file is no longer stored on the server.'))
+            return redirect(self.get_order_url())
+
         if not self.invoice.file:
             # This happens if we have celery installed and the file will be generated in the background
             messages.warning(request, _('The invoice file has not yet been generated, we will generate it for you '
@@ -648,7 +656,10 @@ class OrderModifyInformation(OrderQuestionsViewMixin, OrderView):
                            _("We had difficulties processing your input. Please review the errors below."))
             return self.get(request, *args, **kwargs)
         self.invoice_form.save()
-        self.order.log_action('pretix.event.order.modified', user=request.user)
+        self.order.log_action('pretix.event.order.modified', {
+            'invoice_data': self.invoice_form.cleaned_data,
+            'data': [f.cleaned_data for f in self.forms]
+        }, user=request.user)
         if self.invoice_form.has_changed():
             success_message = ('The invoice address has been updated. If you want to generate a new invoice, '
                                'you need to do this manually.')

src/pretix/control/views/shredder.py

@@ -0,0 +1,109 @@
+import logging
+from collections import OrderedDict
+
+from django.shortcuts import get_object_or_404
+from django.urls import reverse
+from django.utils.functional import cached_property
+from django.utils.translation import ugettext_lazy as _
+from django.views import View
+from django.views.generic import TemplateView
+
+from pretix.base.models import CachedFile
+from pretix.base.services.shredder import export, shred
+from pretix.base.shredder import ShredError, shred_constraints
+from pretix.base.views.async import AsyncAction
+from pretix.control.permissions import EventPermissionRequiredMixin
+
+logger = logging.getLogger(__name__)
+
+
+class ShredderMixin:
+
+    @cached_property
+    def shredders(self):
+        return OrderedDict(
+            sorted(self.request.event.get_data_shredders().items(), key=lambda s: s[1].verbose_name)
+        )
+
+
+class StartShredView(EventPermissionRequiredMixin, ShredderMixin, TemplateView):
+    permission = 'can_change_orders'
+    template_name = 'pretixcontrol/shredder/index.html'
+
+    def get_context_data(self, **kwargs):
+        ctx = super().get_context_data(**kwargs)
+        ctx['shredders'] = self.shredders
+        ctx['constraints'] = shred_constraints(self.request.event)
+        return ctx
+
+
+class ShredDownloadView(EventPermissionRequiredMixin, ShredderMixin, TemplateView):
+    permission = 'can_change_orders'
+    template_name = 'pretixcontrol/shredder/download.html'
+
+    def get_context_data(self, **kwargs):
+        ctx = super().get_context_data(**kwargs)
+        ctx['shredders'] = self.shredders
+        ctx['file'] = get_object_or_404(CachedFile, pk=kwargs.get("file"))
+        return ctx
+
+
+class ShredExportView(EventPermissionRequiredMixin, ShredderMixin, AsyncAction, View):
+    permission = 'can_change_orders'
+    task = export
+    known_errortypes = ['ShredError']
+
+    def get_success_message(self, value):
+        return None
+
+    def get_success_url(self, value):
+        return reverse('control:event.shredder.download', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+            'file': str(value)
+        })
+
+    def get_error_url(self):
+        return reverse('control:event.shredder.start', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug
+        })
+
+    def post(self, request, *args, **kwargs):
+        constr = shred_constraints(self.request.event)
+        if constr:
+            return self.error(ShredError(self.get_error_url()))
+
+        return self.do(self.request.event.id, request.POST.getlist("shredder"))
+
+
+class ShredDoView(EventPermissionRequiredMixin, ShredderMixin, AsyncAction, View):
+    permission = 'can_change_orders'
+    task = shred
+    known_errortypes = ['ShredError']
+
+    def get_success_url(self, value):
+        return reverse('control:event.shredder.start', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+        })
+
+    def get_success_message(self, value):
+        return _('The selected data was deleted successfully.')
+
+    def get_error_url(self):
+        return reverse('control:event.shredder.download', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+            'file': self.request.POST.get("file")
+        })
+
+    def post(self, request, *args, **kwargs):
+        constr = shred_constraints(self.request.event)
+        if constr:
+            return self.error(ShredError(self.get_error_url()))
+
+        if not self.request.user.check_password(request.POST.get("password")):
+            return self.error(ShredError(_("The current password you entered was not correct.")))
+
+        return self.do(self.request.event.id, request.POST.get("file"), request.POST.get("confirm_code"))