From e510a2c1215e1aab7a2b737baac7cf0587aec0da Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Tue, 25 May 2021 19:24:43 +0200 Subject: [PATCH] Fix creating large numbers of subevents (introduces async task) (#2091) --- src/pretix/base/views/tasks.py | 32 +++++--- .../pretixcontrol/subevents/bulk.html | 2 +- src/pretix/control/views/subevents.py | 82 ++++++++++++++++--- src/pretix/static/pretixbase/js/asynctask.js | 6 ++ src/pretix/static/pretixcontrol/js/ui/geo.js | 2 +- src/pretix/static/pretixcontrol/js/ui/main.js | 60 +++++++------- .../static/pretixcontrol/js/ui/subevent.js | 2 +- 7 files changed, 131 insertions(+), 55 deletions(-) diff --git a/src/pretix/base/views/tasks.py b/src/pretix/base/views/tasks.py index c34d2c05d0..fa26e3a870 100644 --- a/src/pretix/base/views/tasks.py +++ b/src/pretix/base/views/tasks.py @@ -27,7 +27,7 @@ from celery.result import AsyncResult from django.conf import settings from django.contrib import messages from django.core.exceptions import ValidationError -from django.http import JsonResponse +from django.http import JsonResponse, QueryDict from django.shortcuts import redirect, render from django.test import RequestFactory from django.utils import timezone, translation @@ -206,22 +206,30 @@ class AsyncFormView(AsyncMixin, FormView): def __init_subclass__(cls): def async_execute(self, *, request_path, form_kwargs, locale, tz, organizer=None, event=None, user=None): view_instance = cls() - view_instance.request = RequestFactory().post(request_path) - if organizer: + d = QueryDict(mutable=True) + d.update(form_kwargs['data']) + req = RequestFactory().post( + request_path, + data=d.urlencode(), + content_type='application/x-www-form-urlencoded' + ) + view_instance.request = req + if event: view_instance.request.event = event - if organizer: + view_instance.request.organizer = event.organizer + elif organizer: view_instance.request.organizer = organizer if user: view_instance.request.user = User.objects.get(pk=user) - form_class = view_instance.get_form_class() - if form_kwargs.get('instance'): - cls.model.objects.get(pk=form_kwargs['instance']) - - form_kwargs = view_instance.get_async_form_kwargs(form_kwargs, organizer, event) - - form = form_class(**form_kwargs) with translation.override(locale), timezone.override(pytz.timezone(tz)): + form_class = view_instance.get_form_class() + if form_kwargs.get('instance'): + cls.model.objects.get(pk=form_kwargs['instance']) + + form_kwargs = view_instance.get_async_form_kwargs(form_kwargs, organizer, event) + form = form_class(**form_kwargs) + form.is_valid() return view_instance.async_form_valid(self, form) cls.async_execute = app.task( @@ -254,6 +262,8 @@ class AsyncFormView(AsyncMixin, FormView): else: form_kwargs['instance'] = None form_kwargs.setdefault('data', {}) + form_kwargs['initial'] = {} + form_kwargs.pop('event', None) kwargs = { 'request_path': self.request.path, 'form_kwargs': form_kwargs, diff --git a/src/pretix/control/templates/pretixcontrol/subevents/bulk.html b/src/pretix/control/templates/pretixcontrol/subevents/bulk.html index 98f2e6dbf6..1b581d1d8d 100644 --- a/src/pretix/control/templates/pretixcontrol/subevents/bulk.html +++ b/src/pretix/control/templates/pretixcontrol/subevents/bulk.html @@ -8,7 +8,7 @@ {% block title %}{% trans "Date" context "subevent" %}{% endblock %} {% block content %}

{% trans "Create multiple dates" context "subevent" %}

-
+ {% csrf_token %} {% bootstrap_form_errors form %} {% for f in itemvar_forms %} diff --git a/src/pretix/control/views/subevents.py b/src/pretix/control/views/subevents.py index 5264a88e9f..aa49b45353 100644 --- a/src/pretix/control/views/subevents.py +++ b/src/pretix/control/views/subevents.py @@ -38,6 +38,7 @@ from datetime import datetime, time, timedelta from dateutil.rrule import DAILY, MONTHLY, WEEKLY, YEARLY, rrule, rruleset from django.contrib import messages +from django.core.exceptions import ValidationError from django.core.files import File from django.db import connections, transaction from django.db.models import Count, F, Prefetch @@ -64,6 +65,7 @@ from pretix.base.models.items import ( from pretix.base.reldate import RelativeDate, RelativeDateWrapper from pretix.base.services import tickets from pretix.base.services.quotas import QuotaAvailability +from pretix.base.views.tasks import AsyncFormView from pretix.control.forms.checkin import SimpleCheckinListForm from pretix.control.forms.filter import SubEventFilterForm from pretix.control.forms.item import QuotaForm @@ -659,7 +661,7 @@ class SubEventBulkAction(SubEventQueryMixin, EventPermissionRequiredMixin, View) }) -class SubEventBulkCreate(SubEventEditorMixin, EventPermissionRequiredMixin, CreateView): +class SubEventBulkCreate(SubEventEditorMixin, EventPermissionRequiredMixin, AsyncFormView): model = SubEvent template_name = 'pretixcontrol/subevents/bulk.html' permission = 'can_change_settings' @@ -668,10 +670,20 @@ class SubEventBulkCreate(SubEventEditorMixin, EventPermissionRequiredMixin, Crea itemformclass = BulkSubEventItemForm itemvarformclass = BulkSubEventItemVariationForm + def dispatch(self, request, *args, **kwargs): + self.object = SubEvent(event=self.request.event) + return super().dispatch(request, *args, **kwargs) + def is_valid(self, form): return self.rrule_formset.is_valid() and self.time_formset.is_valid() and super().is_valid(form) - def get_success_url(self) -> str: + def get_success_url(self, value) -> str: + return reverse('control:event.subevents', kwargs={ + 'organizer': self.request.event.organizer.slug, + 'event': self.request.event.slug, + }) + + def get_error_url(self) -> str: return reverse('control:event.subevents', kwargs={ 'organizer': self.request.event.organizer.slug, 'event': self.request.event.slug, @@ -755,6 +767,11 @@ class SubEventBulkCreate(SubEventEditorMixin, EventPermissionRequiredMixin, Crea kwargs['initial'] = initial return kwargs + def get_async_form_kwargs(self, form_kwargs, organizer=None, event=None): + form_kwargs['event'] = event + form_kwargs['instance'] = SubEvent(event=event) + return form_kwargs + def get_times(self): times = [] for f in self.time_formset: @@ -808,7 +825,31 @@ class SubEventBulkCreate(SubEventEditorMixin, EventPermissionRequiredMixin, Crea return s @transaction.atomic - def form_valid(self, form): + def async_form_valid(self, task, form): + self.object = SubEvent(event=self.request.event) + if not self.is_valid(form): + print([ + self.rrule_formset.is_valid(), + self.time_formset.is_valid(), + form.is_valid(), + [f.is_valid() for f in self.itemvar_forms], + self.formset.is_valid(), + [f.is_valid() for f in self.meta_forms], + self.cl_formset.is_valid(), + [f.is_valid() for f in self.plugin_forms] + ]) + print(form.errors) + raise ValidationError('Invalid submission') + + def set_progress(percent): + if not task.request.called_directly: + task.update_state( + state='PROGRESS', + meta={'value': percent} + ) + + set_progress(0) + tz = self.request.event.timezone subevents = [] for rdate in self.get_rrule_set(): @@ -840,9 +881,15 @@ class SubEventBulkCreate(SubEventEditorMixin, EventPermissionRequiredMixin, Crea if form.cleaned_data.get('rel_presale_end') else None ) - se.save(clear_cache=False) subevents.append(se) + for i, se in enumerate(subevents): + se.save(clear_cache=False) + if i % 100 == 0: + set_progress(10 * i / len(subevents)) + + set_progress(10) + data = dict(form.cleaned_data) for f in self.plugin_forms: data.update({ @@ -865,10 +912,12 @@ class SubEventBulkCreate(SubEventEditorMixin, EventPermissionRequiredMixin, Crea to_save.append(i) SubEventMetaValue.objects.bulk_create(to_save) + set_progress(20) + to_save_items = [] to_save_variations = [] for f in self.itemvar_forms: - for se in subevents: + for i, se in enumerate(subevents): i = copy.copy(f.instance) i.pk = None i.subevent = se @@ -888,17 +937,20 @@ class SubEventBulkCreate(SubEventEditorMixin, EventPermissionRequiredMixin, Crea to_save_items.append(i) else: to_save_variations.append(i) + SubEventItem.objects.bulk_create(to_save_items) + set_progress(30) SubEventItemVariation.objects.bulk_create(to_save_variations) + set_progress(40) to_save_items = [] to_save_variations = [] - for f in self.formset.forms: + for k, f in enumerate(self.formset.forms): if self.formset._should_delete_form(f) or not f.has_changed(): continue change_data = {k: f.cleaned_data.get(k) for k in f.changed_data} - for se in subevents: + for j, se in enumerate(subevents): i = copy.copy(f.instance) i.pk = None i.subevent = se @@ -923,8 +975,13 @@ class SubEventBulkCreate(SubEventEditorMixin, EventPermissionRequiredMixin, Crea log_entries.append( se.log_action('pretix.subevent.quota.added', user=self.request.user, data=change_data, save=False) ) + + if j % 100 == 0: + set_progress(50 + 10 * (j + k * len(subevents)) / (len(self.formset.forms) + len(subevents))) Quota.items.through.objects.bulk_create(to_save_items) + set_progress(60) Quota.variations.through.objects.bulk_create(to_save_variations) + set_progress(70) to_save_products = [] for f in self.cl_formset.forms: @@ -945,12 +1002,14 @@ class SubEventBulkCreate(SubEventEditorMixin, EventPermissionRequiredMixin, Crea save=False) ) CheckinList.limit_products.through.objects.bulk_create(to_save_products) + set_progress(80) for f in self.plugin_forms: f.is_valid() for se in subevents: f.subevent = se f.save() + set_progress(90) if connections['default'].features.can_return_rows_from_bulk_insert: LogEntry.objects.bulk_create(log_entries) @@ -961,11 +1020,10 @@ class SubEventBulkCreate(SubEventEditorMixin, EventPermissionRequiredMixin, Crea LogEntry.bulk_postprocess(log_entries) self.request.event.cache.clear() - messages.success(self.request, pgettext_lazy('subevent', '{} new dates have been created.').format(len(subevents))) - return redirect(reverse('control:event.subevents', kwargs={ - 'organizer': self.request.event.organizer.slug, - 'event': self.request.event.slug, - })) + return len(subevents) + + def get_success_message(self, value): + return pgettext_lazy('subevent', '{} new dates have been created.').format(value) def post(self, request, *args, **kwargs): form = self.get_form() diff --git a/src/pretix/static/pretixbase/js/asynctask.js b/src/pretix/static/pretixbase/js/asynctask.js index 34c630f083..a162de69e4 100644 --- a/src/pretix/static/pretixbase/js/asynctask.js +++ b/src/pretix/static/pretixbase/js/asynctask.js @@ -72,6 +72,8 @@ function async_task_check_error(jqXHR, textStatus, errorThrown) { )); form_handlers($("body")); setup_collapsible_details($("body")); + window.setTimeout(function () { $(window).scrollTop(0) }, 200) + $(document).trigger("pretix:bind-forms"); } else if (c.length > 0) { // This is some kind of 500/404/403 page, show it in an overlay $("body").data('ajaxing', false); @@ -152,6 +154,8 @@ function async_task_error(jqXHR, textStatus, errorThrown) { $("#page-wrapper").html(respdom.find("#page-wrapper").html()); form_handlers($("#page-wrapper")); setup_collapsible_details($("#page-wrapper")); + $(document).trigger("pretix:bind-forms"); + window.setTimeout(function () { $(window).scrollTop(0) }, 200) } else { $("body").html(jqXHR.responseText.substring( jqXHR.responseText.indexOf(" 0) { diff --git a/src/pretix/static/pretixcontrol/js/ui/geo.js b/src/pretix/static/pretixcontrol/js/ui/geo.js index 158eb027c0..bdc6446ee2 100644 --- a/src/pretix/static/pretixcontrol/js/ui/geo.js +++ b/src/pretix/static/pretixcontrol/js/ui/geo.js @@ -1,6 +1,6 @@ /*globals $*/ -$(function () { +$(document).on("pretix:bind-forms", function () { function cleanup(l) { return $.trim(l.replace(/\n/g, ", ")); } diff --git a/src/pretix/static/pretixcontrol/js/ui/main.js b/src/pretix/static/pretixcontrol/js/ui/main.js index c0a7048985..e78f8c06d8 100644 --- a/src/pretix/static/pretixcontrol/js/ui/main.js +++ b/src/pretix/static/pretixcontrol/js/ui/main.js @@ -72,6 +72,36 @@ $(document).ajaxError(function (event, jqXHR, settings, thrownError) { }); var form_handlers = function (el) { + el.find("[data-formset]").formset( + { + animateForms: true, + reorderMode: 'animate' + } + ); + el.find("[data-formset]").on("formAdded", "div", function (event) { + form_handlers($(event.target)); + }); + + // Vouchers + el.find("#voucher-bulk-codes-generate").click(function () { + var num = $("#voucher-bulk-codes-num").val(); + var prefix = $('#voucher-bulk-codes-prefix').val(); + if (num != "") { + var url = $(this).attr("data-rng-url"); + $("#id_codes").html("Generating..."); + $(".form-group:has(#voucher-bulk-codes-num)").removeClass("has-error"); + $.getJSON(url + '?num=' + num + '&prefix=' + escape(prefix), function (data) { + $("#id_codes").val(data.codes.join("\n")); + }); + } else { + $(".form-group:has(#voucher-bulk-codes-num)").addClass("has-error"); + $("#voucher-bulk-codes-num").focus(); + setTimeout(function () { + $(".form-group:has(#voucher-bulk-codes-num)").removeClass("has-error"); + }, 3000); + } + }); + el.find(".datetimepicker").each(function () { $(this).datetimepicker({ format: $("body").attr("data-datetimeformat"), @@ -602,15 +632,6 @@ $(function () { $("body").removeClass("nojs"); lightbox.init(); - $("[data-formset]").formset( - { - animateForms: true, - reorderMode: 'animate' - } - ); - $("[data-formset]").on("formAdded", "div", function (event) { - form_handlers($(event.target)); - }); $(document).on("click", ".variations .variations-select-all", function (e) { $(this).parent().parent().find("input[type=checkbox]").prop("checked", true).change(); e.stopPropagation(); @@ -675,27 +696,8 @@ $(function () { }); }); - // Vouchers - $("#voucher-bulk-codes-generate").click(function () { - var num = $("#voucher-bulk-codes-num").val(); - var prefix = $('#voucher-bulk-codes-prefix').val(); - if (num != "") { - var url = $(this).attr("data-rng-url"); - $("#id_codes").html("Generating..."); - $(".form-group:has(#voucher-bulk-codes-num)").removeClass("has-error"); - $.getJSON(url + '?num=' + num + '&prefix=' + escape(prefix), function (data) { - $("#id_codes").val(data.codes.join("\n")); - }); - } else { - $(".form-group:has(#voucher-bulk-codes-num)").addClass("has-error"); - $("#voucher-bulk-codes-num").focus(); - setTimeout(function () { - $(".form-group:has(#voucher-bulk-codes-num)").removeClass("has-error"); - }, 3000); - } - }); - form_handlers($("body")); + $(document).trigger("pretix:bind-forms"); $(".qrcode-canvas").each(function () { $(this).qrcode( diff --git a/src/pretix/static/pretixcontrol/js/ui/subevent.js b/src/pretix/static/pretixcontrol/js/ui/subevent.js index b35df37be3..85e4a8ca8b 100644 --- a/src/pretix/static/pretixcontrol/js/ui/subevent.js +++ b/src/pretix/static/pretixcontrol/js/ui/subevent.js @@ -1,6 +1,6 @@ /*globals $, Morris, gettext, RRule, RRuleSet*/ -$(function () { +$(document).on("pretix:bind-forms", function () { if (!$("div[data-formset-prefix=checkinlist_set]").length) { return; }