Scheduled exports (#3033)

This commit is contained in:
Raphael Michel
2023-01-19 11:46:30 +01:00
committed by GitHub
parent 0bb5af191b
commit 19d1a8de71
36 changed files with 2461 additions and 293 deletions

View File

@@ -66,7 +66,7 @@ from django.utils.http import url_has_allowed_host_and_scheme
from django.utils.timezone import make_aware, now
from django.utils.translation import gettext, gettext_lazy as _, ngettext
from django.views.generic import (
DetailView, FormView, ListView, TemplateView, View,
DeleteView, DetailView, FormView, ListView, TemplateView, View,
)
from i18nfield.strings import LazyI18nString
@@ -77,7 +77,7 @@ from pretix.base.i18n import language
from pretix.base.models import (
CachedCombinedTicket, CachedFile, CachedTicket, Checkin, Invoice,
InvoiceAddress, Item, ItemVariation, LogEntry, Order, QuestionAnswer,
Quota, generate_secret,
Quota, ScheduledEventExport, generate_secret,
)
from pretix.base.models.orders import (
CancellationRequest, OrderFee, OrderPayment, OrderPosition, OrderRefund,
@@ -87,7 +87,7 @@ from pretix.base.payment import PaymentException
from pretix.base.secrets import assign_ticket_secret
from pretix.base.services import tickets
from pretix.base.services.cancelevent import cancel_event
from pretix.base.services.export import export
from pretix.base.services.export import export, scheduled_event_export
from pretix.base.services.invoices import (
generate_cancellation, generate_invoice, invoice_pdf, invoice_pdf_task,
invoice_qualified, regenerate_invoice,
@@ -111,6 +111,7 @@ from pretix.base.templatetags.money import money_filter
from pretix.base.templatetags.rich_text import markdown_compile_email
from pretix.base.views.mixins import OrderQuestionsViewMixin
from pretix.base.views.tasks import AsyncAction
from pretix.control.forms.exports import ScheduledEventExportForm
from pretix.control.forms.filter import (
EventOrderExpertFilterForm, EventOrderFilterForm, OverviewFilterForm,
RefundFilterForm,
@@ -122,6 +123,7 @@ from pretix.control.forms.orders import (
OrderPositionAddFormset, OrderPositionChangeForm, OrderPositionMailForm,
OrderRefundForm, OtherOperationsForm, ReactivateOrderForm,
)
from pretix.control.forms.rrule import RRuleForm
from pretix.control.permissions import EventPermissionRequiredMixin
from pretix.control.signals import order_search_forms
from pretix.control.views import PaginationMixin
@@ -2252,13 +2254,16 @@ class ExportMixin:
if id != ex.identifier:
continue
# Use form parse cycle to generate useful defaults
test_form = ExporterForm(data=self.request.GET, prefix=ex.identifier)
test_form.fields = ex.export_form_fields
test_form.is_valid()
initial = {
k: v for k, v in test_form.cleaned_data.items() if ex.identifier + "-" + k in self.request.GET
}
if self.scheduled:
initial = self.scheduled.export_form_data
else:
# Use form parse cycle to generate useful defaults
test_form = ExporterForm(data=self.request.GET, prefix=ex.identifier)
test_form.fields = ex.export_form_fields
test_form.is_valid()
initial = {
k: v for k, v in test_form.cleaned_data.items() if ex.identifier + "-" + k in self.request.GET
}
ex.form = ExporterForm(
data=(self.request.POST if self.request.method == 'POST' else None),
@@ -2268,6 +2273,21 @@ class ExportMixin:
ex.form.fields = ex.export_form_fields
return ex
def get_scheduled_queryset(self):
if not self.request.user.has_event_permission(self.request.organizer, self.request.event, 'can_change_event_settings',
request=self.request):
qs = self.request.event.scheduled_exports.filter(owner=self.request.user)
else:
qs = self.request.event.scheduled_exports
return qs.select_related('owner').order_by('export_identifier', 'schedule_next_run')
@cached_property
def scheduled(self):
if "scheduled" in self.request.POST:
return get_object_or_404(self.get_scheduled_queryset(), pk=self.request.POST.get("scheduled"))
elif "scheduled" in self.request.GET:
return get_object_or_404(self.get_scheduled_queryset(), pk=self.request.GET.get("scheduled"))
def get_context_data(self, **kwargs):
ctx = super().get_context_data(**kwargs)
ctx['exporters'] = self.exporters
@@ -2309,25 +2329,165 @@ class ExportDoView(EventPermissionRequiredMixin, ExportMixin, AsyncAction, Templ
'organizer': self.request.event.organizer.slug
}))
if not self.exporter.form.is_valid():
messages.error(self.request, _('There was a problem processing your input. See below for error details.'))
return self.get(request, *args, **kwargs)
if self.scheduled:
data = self.scheduled.export_form_data
else:
if not self.exporter.form.is_valid():
messages.error(self.request, _('There was a problem processing your input. See below for error details.'))
return self.get(request, *args, **kwargs)
data = self.exporter.form.cleaned_data
cf = CachedFile(web_download=True, session_key=request.session.session_key)
cf.date = now()
cf.expires = now() + timedelta(hours=24)
cf.save()
return self.do(self.request.event.id, str(cf.id), self.exporter.identifier, self.exporter.form.cleaned_data)
return self.do(self.request.event.id, str(cf.id), self.exporter.identifier, data)
class ExportView(EventPermissionRequiredMixin, ExportMixin, TemplateView):
class ExportView(EventPermissionRequiredMixin, ExportMixin, ListView):
permission = 'can_view_orders'
paginate_by = 25
context_object_name = 'scheduled'
def get_template_names(self):
if self.exporter:
return ['pretixcontrol/orders/export_form.html']
return ['pretixcontrol/orders/export.html']
@transaction.atomic()
def post(self, request, *args, **kwargs):
if request.POST.get("schedule") == "save":
if self.exporter.form.is_valid() and self.rrule_form.is_valid() and self.schedule_form.is_valid():
self.schedule_form.instance.export_identifier = self.exporter.identifier
self.schedule_form.instance.export_form_data = self.exporter.form.cleaned_data
self.schedule_form.instance.schedule_rrule = str(self.rrule_form.to_rrule())
self.schedule_form.instance.error_counter = 0
self.schedule_form.instance.error_last_message = None
self.schedule_form.instance.compute_next_run()
self.schedule_form.instance.save()
if self.schedule_form.instance.schedule_next_run:
messages.success(
request,
_('Your export schedule has been saved. The next export will start around {datetime}.').format(
datetime=date_format(self.schedule_form.instance.schedule_next_run, 'SHORT_DATETIME_FORMAT')
)
)
else:
messages.warning(request, _('Your export schedule has been saved, but no next export is planned.'))
self.request.event.log_action(
'pretix.event.export.schedule.changed' if self.scheduled else 'pretix.event.export.schedule.added',
user=self.request.user, data={
'id': self.schedule_form.instance.id,
'export_identifier': self.exporter.identifier,
'export_form_data': self.exporter.form.cleaned_data,
'schedule_rrule': self.schedule_form.instance.schedule_rrule,
**self.schedule_form.cleaned_data,
}
)
return redirect(reverse('control:event.orders.export', kwargs={
'event': self.request.event.slug,
'organizer': self.request.event.organizer.slug
}))
else:
return super().get(request, *args, **kwargs)
return super().get(request, *args, **kwargs)
@cached_property
def rrule_form(self):
if self.scheduled:
initial = RRuleForm.initial_from_rrule(self.scheduled.schedule_rrule)
else:
initial = {}
return RRuleForm(
data=self.request.POST if self.request.method == 'POST' and self.request.POST.get("schedule") == "save" else None,
prefix="rrule",
initial=initial
)
@cached_property
def schedule_form(self):
instance = self.scheduled or ScheduledEventExport(
event=self.request.event,
owner=self.request.user,
)
if not self.scheduled:
initial = {
"mail_subject": gettext("Export: {title}").format(title=self.exporter.verbose_name),
"mail_template": gettext("Hello,\n\nattached to this email, you can find a new scheduled report for {name}.").format(
name=str(self.request.event.name)
),
"schedule_rrule_time": time(4, 0, 0),
}
else:
initial = {}
return ScheduledEventExportForm(
data=self.request.POST if self.request.method == 'POST' and self.request.POST.get("schedule") == "save" else None,
prefix="schedule",
instance=instance,
initial=initial,
)
def get_queryset(self):
return self.get_scheduled_queryset()
def get_context_data(self, **kwargs):
ctx = super().get_context_data(**kwargs)
if "schedule" in self.request.POST or self.scheduled:
ctx['schedule_form'] = self.schedule_form
ctx['rrule_form'] = self.rrule_form
elif not self.exporter:
for s in ctx['scheduled']:
try:
s.export_verbose_name = [e for e in self.exporters if e.identifier == s.export_identifier][0].verbose_name
except IndexError:
s.export_verbose_name = "?"
return ctx
class DeleteScheduledExportView(EventPermissionRequiredMixin, ExportMixin, DeleteView):
permission = 'can_view_orders'
template_name = 'pretixcontrol/orders/export_delete.html'
context_object_name = 'export'
def get_queryset(self):
return self.get_scheduled_queryset()
def get_success_url(self):
return reverse('control:event.orders.export', kwargs={
'event': self.request.event.slug,
'organizer': self.request.event.organizer.slug
})
@transaction.atomic()
def delete(self, request, *args, **kwargs):
self.object = self.get_object()
self.object.delete()
self.request.event.log_action('pretix.event.export.schedule.deleted', user=self.request.user, data={
'id': self.object.id,
})
return redirect(self.get_success_url())
class RunScheduledExportView(EventPermissionRequiredMixin, ExportMixin, View):
def post(self, request, *args, **kwargs):
s = get_object_or_404(self.get_scheduled_queryset(), pk=kwargs.get('pk'))
scheduled_event_export.apply_async(
kwargs={
'event': s.event_id,
'schedule': s.pk,
},
# Scheduled exports usually run on the low-prio queue "background" but if they're manually triggered,
# we run them with normal priority
queue='default',
)
messages.success(self.request, _('Your export is queued to start soon. The results will be send via email. '
'Depending on system load and type and size of export, this may take a few '
'minutes.'))
return redirect(reverse('control:organizer.export', kwargs={
'organizer': self.request.organizer.slug
}))
class RefundList(EventPermissionRequiredMixin, PaginationMixin, ListView):
model = OrderRefund

View File

@@ -34,7 +34,7 @@
import json
import re
from datetime import timedelta
from datetime import time, timedelta
from decimal import Decimal
import bleach
@@ -53,9 +53,10 @@ from django.forms import DecimalField
from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse
from django.shortcuts import get_object_or_404, redirect
from django.urls import reverse
from django.utils.formats import date_format
from django.utils.functional import cached_property
from django.utils.timezone import now
from django.utils.translation import gettext_lazy as _
from django.utils.timezone import get_current_timezone, now
from django.utils.translation import gettext, gettext_lazy as _
from django.views import View
from django.views.generic import (
CreateView, DeleteView, DetailView, FormView, ListView, TemplateView,
@@ -71,7 +72,7 @@ from pretix.base.i18n import language
from pretix.base.models import (
CachedFile, Customer, Device, Gate, GiftCard, Invoice, LogEntry,
Membership, MembershipType, Order, OrderPayment, OrderPosition, Organizer,
Team, TeamInvite, User,
ScheduledOrganizerExport, Team, TeamInvite, User,
)
from pretix.base.models.customers import CustomerSSOClient, CustomerSSOProvider
from pretix.base.models.event import Event, EventMetaProperty, EventMetaValue
@@ -81,12 +82,13 @@ from pretix.base.models.giftcards import (
from pretix.base.models.orders import CancellationRequest
from pretix.base.models.organizer import TeamAPIToken
from pretix.base.payment import PaymentException
from pretix.base.services.export import multiexport
from pretix.base.services.export import multiexport, scheduled_organizer_export
from pretix.base.services.mail import SendMailException, mail
from pretix.base.settings import SETTINGS_AFFECTING_CSS
from pretix.base.signals import register_multievent_data_exporters
from pretix.base.templatetags.rich_text import markdown_compile_email
from pretix.base.views.tasks import AsyncAction
from pretix.control.forms.exports import ScheduledOrganizerExportForm
from pretix.control.forms.filter import (
CustomerFilterForm, DeviceFilterForm, EventFilterForm, GiftCardFilterForm,
OrganizerFilterForm, TeamFilterForm,
@@ -100,6 +102,7 @@ from pretix.control.forms.organizer import (
OrganizerSettingsForm, OrganizerUpdateForm, SSOClientForm, SSOProviderForm,
TeamForm, WebHookForm,
)
from pretix.control.forms.rrule import RRuleForm
from pretix.control.logdisplay import OVERVIEW_BANLIST
from pretix.control.permissions import (
AdministratorPermissionRequiredMixin, OrganizerPermissionRequiredMixin,
@@ -1521,13 +1524,18 @@ class ExportMixin:
for ex in self.exporters:
if id != ex.identifier:
continue
# Use form parse cycle to generate useful defaults
test_form = ExporterForm(data=self.request.GET, prefix=ex.identifier)
test_form.fields = ex.export_form_fields
test_form.is_valid()
initial = {
k: v for k, v in test_form.cleaned_data.items() if ex.identifier + "-" + k in self.request.GET
}
if self.scheduled:
initial = self.scheduled.export_form_data
else:
# Use form parse cycle to generate useful defaults
test_form = ExporterForm(data=self.request.GET, prefix=ex.identifier)
test_form.fields = ex.export_form_fields
test_form.is_valid()
initial = {
k: v for k, v in test_form.cleaned_data.items() if ex.identifier + "-" + k in self.request.GET
}
if 'events' not in initial:
initial.setdefault('all_events', True)
ex.form = ExporterForm(
data=(self.request.POST if self.request.method == 'POST' else None),
@@ -1537,15 +1545,22 @@ class ExportMixin:
ex.form.fields = ex.export_form_fields
if not isinstance(ex, OrganizerLevelExportMixin):
ex.form.fields.update([
('all_events',
forms.BooleanField(
label=_("All events (that I have access to)"),
required=False
)),
('events',
forms.ModelMultipleChoiceField(
queryset=self.events,
initial=self.events,
widget=forms.CheckboxSelectMultiple(
attrs={'class': 'scrolling-multiple-choice'}
attrs={
'class': 'scrolling-multiple-choice',
'data-inverse-dependency': f'#id_{ex.identifier}-all_events',
}
),
label=_('Events'),
required=True
required=False
)),
])
return ex
@@ -1582,6 +1597,21 @@ class ExportMixin:
ctx['exporters'] = self.exporters
return ctx
def get_scheduled_queryset(self):
if not self.request.user.has_organizer_permission(self.request.organizer, 'can_change_organizer_settings',
request=self.request):
qs = self.request.organizer.scheduled_exports.filter(owner=self.request.user)
else:
qs = self.request.organizer.scheduled_exports
return qs.select_related('owner').order_by('export_identifier', 'schedule_next_run')
@cached_property
def scheduled(self):
if "scheduled" in self.request.POST:
return get_object_or_404(self.get_scheduled_queryset(), pk=self.request.POST.get("scheduled"))
elif "scheduled" in self.request.GET:
return get_object_or_404(self.get_scheduled_queryset(), pk=self.request.GET.get("scheduled"))
class ExportDoView(OrganizerPermissionRequiredMixin, ExportMixin, AsyncAction, TemplateView):
known_errortypes = ['ExportError']
@@ -1611,9 +1641,13 @@ class ExportDoView(OrganizerPermissionRequiredMixin, ExportMixin, AsyncAction, T
'organizer': self.request.organizer.slug
})
if not self.exporter.form.is_valid():
messages.error(self.request, _('There was a problem processing your input. See below for error details.'))
return self.get(request, *args, **kwargs)
if self.scheduled:
data = self.scheduled.export_form_data
else:
if not self.exporter.form.is_valid():
messages.error(self.request, _('There was a problem processing your input. See below for error details.'))
return self.get(request, *args, **kwargs)
data = self.exporter.form.cleaned_data
cf = CachedFile(web_download=True, session_key=request.session.session_key)
cf.date = now()
@@ -1626,17 +1660,152 @@ class ExportDoView(OrganizerPermissionRequiredMixin, ExportMixin, AsyncAction, T
provider=self.exporter.identifier,
device=None,
token=None,
form_data=self.exporter.form.cleaned_data,
form_data=data,
staff_session=self.request.user.has_active_staff_session(self.request.session.session_key)
)
class ExportView(OrganizerPermissionRequiredMixin, ExportMixin, TemplateView):
class ExportView(OrganizerPermissionRequiredMixin, ExportMixin, ListView):
paginate_by = 25
context_object_name = 'scheduled'
def get_template_names(self):
if self.exporter:
return ['pretixcontrol/organizers/export_form.html']
return ['pretixcontrol/organizers/export.html']
@transaction.atomic()
def post(self, request, *args, **kwargs):
if request.POST.get("schedule") == "save":
if self.exporter.form.is_valid() and self.rrule_form.is_valid() and self.schedule_form.is_valid():
self.schedule_form.instance.export_identifier = self.exporter.identifier
self.schedule_form.instance.export_form_data = self.exporter.form.cleaned_data
self.schedule_form.instance.schedule_rrule = str(self.rrule_form.to_rrule())
self.schedule_form.instance.error_counter = 0
self.schedule_form.instance.error_last_message = None
self.schedule_form.instance.compute_next_run()
self.schedule_form.instance.save()
if self.schedule_form.instance.schedule_next_run:
messages.success(
request,
_('Your export schedule has been saved. The next export will start around {datetime}.').format(
datetime=date_format(self.schedule_form.instance.schedule_next_run, 'SHORT_DATETIME_FORMAT')
)
)
else:
messages.warning(request, _('Your export schedule has been saved, but no next export is planned.'))
self.request.organizer.log_action(
'pretix.organizer.export.schedule.changed' if self.scheduled else 'pretix.organizer.export.schedule.added',
user=self.request.user, data={
'id': self.schedule_form.instance.id,
'export_identifier': self.exporter.identifier,
'export_form_data': self.exporter.form.cleaned_data,
'schedule_rrule': self.schedule_form.instance.schedule_rrule,
**self.schedule_form.cleaned_data,
}
)
return redirect(reverse('control:organizer.export', kwargs={
'organizer': self.request.organizer.slug
}))
else:
return super().get(request, *args, **kwargs)
return super().get(request, *args, **kwargs)
@cached_property
def rrule_form(self):
if self.scheduled:
initial = RRuleForm.initial_from_rrule(self.scheduled.schedule_rrule)
else:
initial = {}
return RRuleForm(
data=self.request.POST if self.request.method == 'POST' and self.request.POST.get("schedule") == "save" else None,
prefix="rrule",
initial=initial
)
@cached_property
def schedule_form(self):
instance = self.scheduled or ScheduledOrganizerExport(
organizer=self.request.organizer,
owner=self.request.user,
timezone=get_current_timezone().zone,
)
if not self.scheduled:
initial = {
"mail_subject": gettext("Export: {title}").format(title=self.exporter.verbose_name),
"mail_template": gettext("Hello,\n\nattached to this email, you can find a new scheduled report for {name}.").format(
name=str(self.request.organizer.name)
),
"schedule_rrule_time": time(4, 0, 0),
}
else:
initial = {}
return ScheduledOrganizerExportForm(
data=self.request.POST if self.request.method == 'POST' and self.request.POST.get("schedule") == "save" else None,
prefix="schedule",
instance=instance,
initial=initial,
)
def get_queryset(self):
return self.get_scheduled_queryset()
def get_context_data(self, **kwargs):
ctx = super().get_context_data(**kwargs)
if "schedule" in self.request.POST or self.scheduled:
ctx['schedule_form'] = self.schedule_form
ctx['rrule_form'] = self.rrule_form
elif not self.exporter:
for s in ctx['scheduled']:
try:
s.export_verbose_name = [e for e in self.exporters if e.identifier == s.export_identifier][0].verbose_name
except IndexError:
s.export_verbose_name = "?"
return ctx
class DeleteScheduledExportView(OrganizerPermissionRequiredMixin, ExportMixin, DeleteView):
template_name = 'pretixcontrol/organizers/export_delete.html'
context_object_name = 'export'
def get_queryset(self):
return self.get_scheduled_queryset()
def get_success_url(self):
return reverse('control:organizer.export', kwargs={
'organizer': self.request.organizer.slug
})
@transaction.atomic()
def delete(self, request, *args, **kwargs):
self.object = self.get_object()
self.object.delete()
self.request.organizer.log_action('pretix.organizer.export.schedule.deleted', user=self.request.user, data={
'id': self.object.id,
})
return redirect(self.get_success_url())
class RunScheduledExportView(OrganizerPermissionRequiredMixin, ExportMixin, View):
def post(self, request, *args, **kwargs):
s = get_object_or_404(self.get_scheduled_queryset(), pk=kwargs.get('pk'))
scheduled_organizer_export.apply_async(
kwargs={
'organizer': s.organizer_id,
'schedule': s.pk,
},
# Scheduled exports usually run on the low-prio queue "background" but if they're manually triggered,
# we run them with normal priority
queue='default',
)
messages.success(self.request, _('Your export is queued to start soon. The results will be send via email. '
'Depending on system load and type and size of export, this may take a few '
'minutes.'))
return redirect(reverse('control:organizer.export', kwargs={
'organizer': self.request.organizer.slug
}))
class GateListView(OrganizerDetailViewMixin, OrganizerPermissionRequiredMixin, ListView):
model = Gate

View File

@@ -36,7 +36,7 @@ import copy
from collections import defaultdict
from datetime import datetime, time, timedelta
from dateutil.rrule import DAILY, MONTHLY, WEEKLY, YEARLY, rrule, rruleset
from dateutil.rrule import rruleset
from django.contrib import messages
from django.core.exceptions import ValidationError
from django.core.files import File
@@ -789,41 +789,10 @@ class SubEventBulkCreate(SubEventEditorMixin, EventPermissionRequiredMixin, Asyn
if f in self.rrule_formset.deleted_forms:
continue
rule_kwargs = {}
rule_kwargs['dtstart'] = f.cleaned_data['dtstart']
rule_kwargs['interval'] = f.cleaned_data['interval']
if f.cleaned_data['freq'] == 'yearly':
freq = YEARLY
if f.cleaned_data['yearly_same'] == "off":
rule_kwargs['bysetpos'] = int(f.cleaned_data['yearly_bysetpos'])
rule_kwargs['byweekday'] = f.parse_weekdays(f.cleaned_data['yearly_byweekday'])
rule_kwargs['bymonth'] = int(f.cleaned_data['yearly_bymonth'])
elif f.cleaned_data['freq'] == 'monthly':
freq = MONTHLY
if f.cleaned_data['monthly_same'] == "off":
rule_kwargs['bysetpos'] = int(f.cleaned_data['monthly_bysetpos'])
rule_kwargs['byweekday'] = f.parse_weekdays(f.cleaned_data['monthly_byweekday'])
elif f.cleaned_data['freq'] == 'weekly':
freq = WEEKLY
if f.cleaned_data['weekly_byweekday']:
rule_kwargs['byweekday'] = [f.parse_weekdays(a) for a in f.cleaned_data['weekly_byweekday']]
elif f.cleaned_data['freq'] == 'daily':
freq = DAILY
if f.cleaned_data['end'] == 'count':
rule_kwargs['count'] = f.cleaned_data['count']
else:
rule_kwargs['until'] = f.cleaned_data['until']
if f.cleaned_data['exclude']:
s.exrule(rrule(freq, **rule_kwargs))
s.exrule(f.to_rrule())
else:
s.rrule(rrule(freq, **rule_kwargs))
s.rrule(f.to_rrule())
return s