mirror of
https://github.com/pretix/pretix.git
synced 2026-05-06 15:24:02 +00:00
* Add data shredders for PII * First working shredder * Add more shredders * Add new shredders and download confirmation * tmp * PayPal, Stripe, banktransfer * Add icon to logs * Untested payment log shredders * Add waiting list shredder * First tests * Add tests for shredders * Improve templats, link to shredder * Test payment info shredders * More tests * Documentation * Fix enabled flag in payment provider overview * Fix minor issues
This commit is contained in:
Raphael Michel
341
src/pretix/base/shredder.py
Normal file
341
src/pretix/base/shredder.py
Normal file
@@ -0,0 +1,341 @@
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from typing import List, Tuple
|
||||
|
||||
from django.db import transaction
|
||||
from django.db.models import Max
|
||||
from django.db.models.functions import Greatest
|
||||
from django.dispatch import receiver
|
||||
from django.utils.timezone import now
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from pretix.api.serializers.order import (
|
||||
AnswerSerializer, InvoiceAddressSerializer,
|
||||
)
|
||||
from pretix.api.serializers.waitinglist import WaitingListSerializer
|
||||
from pretix.base.i18n import LazyLocaleException
|
||||
from pretix.base.models import (
|
||||
CachedCombinedTicket, CachedTicket, Event, InvoiceAddress, OrderPosition,
|
||||
QuestionAnswer,
|
||||
)
|
||||
from pretix.base.services.invoices import invoice_pdf_task
|
||||
from pretix.base.signals import register_data_shredders
|
||||
|
||||
|
||||
class ShredError(LazyLocaleException):
|
||||
pass
|
||||
|
||||
|
||||
def shred_constraints(event: Event):
|
||||
if event.has_subevents:
|
||||
max_date = event.subevents.aggregate(
|
||||
max_from=Max('date_from'),
|
||||
max_to=Max('date_to'),
|
||||
max_fromto=Greatest(Max('date_to'), Max('date_from'))
|
||||
)
|
||||
max_date = max_date['max_fromto'] or max_date['max_to'] or max_date['max_From']
|
||||
if max_date > now() - timedelta(days=60):
|
||||
return _('Your event needs to be over for at least 60 days to use this feature.')
|
||||
else:
|
||||
if (event.date_to or event.date_from) > now() - timedelta(days=60):
|
||||
return _('Your event needs to be over for at least 60 days to use this feature.')
|
||||
if event.live:
|
||||
return _('Your ticket shop needs to be offline to use this feature.')
|
||||
return None
|
||||
|
||||
|
||||
class BaseDataShredder:
|
||||
"""
|
||||
This is the base class for all data shredders.
|
||||
"""
|
||||
|
||||
def __init__(self, event: Event):
|
||||
self.event = event
|
||||
|
||||
def __str__(self):
|
||||
return self.identifier
|
||||
|
||||
def generate_files(self) -> List[Tuple[str, str, str]]:
|
||||
"""
|
||||
This method is called to export the data that is about to be shred and return a list of tuples consisting of a
|
||||
filename, a file type and file content.
|
||||
|
||||
You can also implement this as a generator and ``yield`` those tuples instead of returning a list of them.
|
||||
"""
|
||||
raise NotImplementedError() # NOQA
|
||||
|
||||
def shred_data(self):
|
||||
"""
|
||||
This method is called to actually remove the data from the system. You should remove any database objects
|
||||
here.
|
||||
|
||||
You should never delete ``LogEntry`` objects, but you might modify them to remove personal data. In this
|
||||
case, set the ``LogEntry.shredded`` attribute to ``True`` to show that this is no longer original log data.
|
||||
"""
|
||||
raise NotImplementedError() # NOQA
|
||||
|
||||
@property
|
||||
def verbose_name(self) -> str:
|
||||
"""
|
||||
A human-readable name for what this shredder removes. This should be short but self-explanatory.
|
||||
Good examples include 'E-Mail addresses' or 'Invoices'.
|
||||
"""
|
||||
raise NotImplementedError() # NOQA
|
||||
|
||||
@property
|
||||
def identifier(self) -> str:
|
||||
"""
|
||||
A short and unique identifier for this shredder.
|
||||
This should only contain lowercase letters and in most
|
||||
cases will be the same as your package name.
|
||||
"""
|
||||
raise NotImplementedError() # NOQA
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
"""
|
||||
A more detailed description of what this shredder does. Can contain HTML.
|
||||
"""
|
||||
raise NotImplementedError() # NOQA
|
||||
|
||||
|
||||
def shred_log_fields(logentry, blacklist=None, whitelist=None):
|
||||
d = logentry.parsed_data
|
||||
if whitelist:
|
||||
for k, v in d.items():
|
||||
if k not in whitelist:
|
||||
d[k] = '█'
|
||||
elif blacklist:
|
||||
for f in blacklist:
|
||||
if f in d:
|
||||
d[f] = '█'
|
||||
logentry.data = json.dumps(d)
|
||||
logentry.shredded = True
|
||||
logentry.save(update_fields=['data', 'shredded'])
|
||||
|
||||
|
||||
class EmailAddressShredder(BaseDataShredder):
|
||||
verbose_name = _('E-mails')
|
||||
identifier = 'order_emails'
|
||||
description = _('This will remove all e-mail addresses from orders and attendees, as well as logged email '
|
||||
'contents.')
|
||||
|
||||
def generate_files(self) -> List[Tuple[str, str, str]]:
|
||||
yield 'emails-by-order.json', 'application/json', json.dumps({
|
||||
o.code: o.email for o in self.event.orders.filter(email__isnull=False)
|
||||
}, indent=4)
|
||||
yield 'emails-by-attendee.json', 'application/json', json.dumps({
|
||||
'{}-{}'.format(op.order.code, op.positionid): op.attendee_email
|
||||
for op in OrderPosition.objects.filter(order__event=self.event, attendee_email__isnull=False)
|
||||
}, indent=4)
|
||||
|
||||
@transaction.atomic
|
||||
def shred_data(self):
|
||||
OrderPosition.objects.filter(order__event=self.event, attendee_email__isnull=False).update(attendee_email=None)
|
||||
|
||||
for o in self.event.orders.all():
|
||||
o.email = None
|
||||
d = o.meta_info_data
|
||||
if d:
|
||||
if 'contact_form_data' in d and 'email' in d['contact_form_data']:
|
||||
del d['contact_form_data']['email']
|
||||
o.meta_info = json.dumps(d)
|
||||
o.save(update_fields=['meta_info', 'email'])
|
||||
|
||||
for le in self.event.logentry_set.filter(action_type__contains="order.email"):
|
||||
shred_log_fields(le, blacklist=['recipient', 'message', 'subject'])
|
||||
|
||||
for le in self.event.logentry_set.filter(action_type="pretix.event.order.contact.changed"):
|
||||
shred_log_fields(le, blacklist=['old_email', 'new_email'])
|
||||
|
||||
for le in self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data=""):
|
||||
d = le.parsed_data
|
||||
if 'data' in d:
|
||||
for row in d['data']:
|
||||
if 'attendee_email' in row:
|
||||
row['attendee_email'] = '█'
|
||||
le.data = json.dumps(d)
|
||||
le.shredded = True
|
||||
le.save(update_fields=['data', 'shredded'])
|
||||
|
||||
|
||||
class WaitingListShredder(BaseDataShredder):
|
||||
verbose_name = _('Waiting list')
|
||||
identifier = 'waiting_list'
|
||||
description = _('This will remove all email addresses from the waiting list.')
|
||||
|
||||
def generate_files(self) -> List[Tuple[str, str, str]]:
|
||||
yield 'waiting-list.json', 'application/json', json.dumps([
|
||||
WaitingListSerializer(wle).data
|
||||
for wle in self.event.waitinglistentries.all()
|
||||
], indent=4)
|
||||
|
||||
@transaction.atomic
|
||||
def shred_data(self):
|
||||
self.event.waitinglistentries.update(email='█')
|
||||
|
||||
for wle in self.event.waitinglistentries.select_related('voucher').filter(voucher__isnull=False):
|
||||
if '@' in wle.voucher.comment:
|
||||
wle.voucher.comment = '█'
|
||||
wle.voucher.save(update_fields=['comment'])
|
||||
|
||||
for le in self.event.logentry_set.filter(action_type="pretix.voucher.added.waitinglist").exclude(data=""):
|
||||
d = le.parsed_data
|
||||
d['email'] = '█'
|
||||
le.data = json.dumps(d)
|
||||
le.shredded = True
|
||||
le.save(update_fields=['data', 'shredded'])
|
||||
|
||||
|
||||
class AttendeeNameShredder(BaseDataShredder):
|
||||
verbose_name = _('Attendee names')
|
||||
identifier = 'attendee_names'
|
||||
description = _('This will remove all attendee names from order positions, as well as logged changes to them.')
|
||||
|
||||
def generate_files(self) -> List[Tuple[str, str, str]]:
|
||||
yield 'attendee-names.json', 'application/json', json.dumps({
|
||||
'{}-{}'.format(op.order.code, op.positionid): op.attendee_name
|
||||
for op in OrderPosition.objects.filter(order__event=self.event, attendee_name__isnull=False)
|
||||
}, indent=4)
|
||||
|
||||
@transaction.atomic
|
||||
def shred_data(self):
|
||||
OrderPosition.objects.filter(order__event=self.event, attendee_name__isnull=False).update(attendee_name=None)
|
||||
|
||||
for le in self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data=""):
|
||||
d = le.parsed_data
|
||||
if 'data' in d:
|
||||
for i, row in enumerate(d['data']):
|
||||
if 'attendee_name' in row:
|
||||
d['data'][i]['attendee_name'] = '█'
|
||||
le.data = json.dumps(d)
|
||||
le.shredded = True
|
||||
le.save(update_fields=['data', 'shredded'])
|
||||
|
||||
|
||||
class InvoiceAddressShredder(BaseDataShredder):
|
||||
verbose_name = _('Invoice addresses')
|
||||
identifier = 'invoice_addresses'
|
||||
description = _('This will remove all invoice addresses from orders, as well as logged changes to them.')
|
||||
|
||||
def generate_files(self) -> List[Tuple[str, str, str]]:
|
||||
yield 'invoice-addresses.json', 'application/json', json.dumps({
|
||||
ia.order.code: InvoiceAddressSerializer(ia).data
|
||||
for ia in InvoiceAddress.objects.filter(order__event=self.event)
|
||||
}, indent=4)
|
||||
|
||||
@transaction.atomic
|
||||
def shred_data(self):
|
||||
InvoiceAddress.objects.filter(order__event=self.event).delete()
|
||||
|
||||
for le in self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data=""):
|
||||
d = le.parsed_data
|
||||
if 'invoice_data' in d and not isinstance(d['invoice_data'], bool):
|
||||
for field in d['invoice_data']:
|
||||
if d['invoice_data'][field]:
|
||||
d['invoice_data'][field] = '█'
|
||||
le.data = json.dumps(d)
|
||||
le.shredded = True
|
||||
le.save(update_fields=['data', 'shredded'])
|
||||
|
||||
|
||||
class QuestionAnswerShredder(BaseDataShredder):
|
||||
verbose_name = _('Question answers')
|
||||
identifier = 'question_answers'
|
||||
description = _('This will remove all answers to questions, as well as logged changes to them.')
|
||||
|
||||
def generate_files(self) -> List[Tuple[str, str, str]]:
|
||||
yield 'question-answers.json', 'application/json', json.dumps({
|
||||
'{}-{}'.format(op.order.code, op.positionid): AnswerSerializer(op.answers.all(), many=True).data
|
||||
for op in OrderPosition.objects.filter(order__event=self.event).prefetch_related('answers')
|
||||
}, indent=4)
|
||||
|
||||
@transaction.atomic
|
||||
def shred_data(self):
|
||||
QuestionAnswer.objects.filter(orderposition__order__event=self.event).delete()
|
||||
|
||||
for le in self.event.logentry_set.filter(action_type="pretix.event.order.modified").exclude(data=""):
|
||||
d = le.parsed_data
|
||||
if 'data' in d:
|
||||
for i, row in enumerate(d['data']):
|
||||
for f in row:
|
||||
if f not in ('attendee_name', 'attendee_email'):
|
||||
d['data'][i][f] = '█'
|
||||
le.data = json.dumps(d)
|
||||
le.shredded = True
|
||||
le.save(update_fields=['data', 'shredded'])
|
||||
|
||||
|
||||
class InvoiceShredder(BaseDataShredder):
|
||||
verbose_name = _('Invoices')
|
||||
identifier = 'invoices'
|
||||
description = _('This will remove all invoice PDFs, as well as any of their text content that might contain '
|
||||
'personal data from the database. Invoice numbers and totals will be conserved.')
|
||||
|
||||
def generate_files(self) -> List[Tuple[str, str, str]]:
|
||||
for i in self.event.invoices.filter(shredded=False):
|
||||
if not i.file:
|
||||
invoice_pdf_task.apply(args=(i.pk,))
|
||||
i.refresh_from_db()
|
||||
i.file.open('rb')
|
||||
yield 'invoices/{}.pdf'.format(i.number), 'application/pdf', i.file.read()
|
||||
i.file.close()
|
||||
|
||||
@transaction.atomic
|
||||
def shred_data(self):
|
||||
for i in self.event.invoices.filter(shredded=False):
|
||||
if i.file:
|
||||
i.file.delete()
|
||||
i.shredded = True
|
||||
i.introductory_text = "█"
|
||||
i.additional_text = "█"
|
||||
i.invoice_to = "█"
|
||||
i.payment_provider_text = "█"
|
||||
i.save()
|
||||
i.lines.update(description="█")
|
||||
|
||||
|
||||
class CachedTicketShredder(BaseDataShredder):
|
||||
verbose_name = _('Cached ticket files')
|
||||
identifier = 'cachedtickets'
|
||||
description = _('This will remove all cached ticket files. No download will be offered.')
|
||||
|
||||
def generate_files(self) -> List[Tuple[str, str, str]]:
|
||||
pass
|
||||
|
||||
@transaction.atomic
|
||||
def shred_data(self):
|
||||
CachedTicket.objects.filter(order_position__order__event=self.event).delete()
|
||||
CachedCombinedTicket.objects.filter(order__event=self.event).delete()
|
||||
|
||||
|
||||
class PaymentInfoShredder(BaseDataShredder):
|
||||
verbose_name = _('Payment information')
|
||||
identifier = 'payment_info'
|
||||
description = _('This will remove payment-related information. Depending on the payment method, all data will be '
|
||||
'removed or personal data only. No download will be offered.')
|
||||
|
||||
def generate_files(self) -> List[Tuple[str, str, str]]:
|
||||
pass
|
||||
|
||||
@transaction.atomic
|
||||
def shred_data(self):
|
||||
provs = self.event.get_payment_providers()
|
||||
for o in self.event.orders.all():
|
||||
pprov = provs.get(o.payment_provider)
|
||||
if pprov:
|
||||
pprov.shred_payment_info(o)
|
||||
|
||||
|
||||
@receiver(register_data_shredders, dispatch_uid="shredders_builtin")
|
||||
def register_payment_provider(sender, **kwargs):
|
||||
return [
|
||||
EmailAddressShredder,
|
||||
AttendeeNameShredder,
|
||||
InvoiceAddressShredder,
|
||||
QuestionAnswerShredder,
|
||||
InvoiceShredder,
|
||||
CachedTicketShredder,
|
||||
PaymentInfoShredder,
|
||||
WaitingListShredder
|
||||
]
|
||||
Reference in New Issue
Block a user