diff --git a/doc/development/api/import.rst b/doc/development/api/import.rst index 3eb893118c..4a2388ac31 100644 --- a/doc/development/api/import.rst +++ b/doc/development/api/import.rst @@ -3,11 +3,12 @@ .. _`importcol`: -Extending the order import process -================================== +Extending the import process +============================ -It's possible through the backend to import orders into pretix, for example from a legacy ticketing system. If your -plugins defines additional data structures around orders, it might be useful to make it possible to import them as well. +It's possible through the backend to import objects into pretix, for example orders from a legacy ticketing system. If +your plugin defines additional data structures around those objects, it might be useful to make it possible to import +them as well. Import process -------------- @@ -40,7 +41,7 @@ Column registration The import API does not make a lot of usage from signals, however, it does use a signal to get a list of all available import columns. Your plugin -should listen for this signal and return the subclass of ``pretix.base.orderimport.ImportColumn`` +should listen for this signal and return the subclass of ``pretix.base.modelimport.ImportColumn`` that we'll provide in this plugin: .. sourcecode:: python @@ -56,10 +57,16 @@ that we'll provide in this plugin: EmailColumn(sender), ] +Similar signals exist for other objects: + +.. automodule:: pretix.base.signals + :members: voucher_import_columns + + The column class API -------------------- -.. class:: pretix.base.orderimport.ImportColumn +.. class:: pretix.base.modelimport.ImportColumn The central object of each import extension is the subclass of ``ImportColumn``. diff --git a/src/pretix/base/apps.py b/src/pretix/base/apps.py index 6a0d56a466..dd5d08f705 100644 --- a/src/pretix/base/apps.py +++ b/src/pretix/base/apps.py @@ -46,7 +46,7 @@ class PretixBaseConfig(AppConfig): from . import invoice # NOQA from . import notifications # NOQA from . import email # NOQA - from .services import auth, checkin, currencies, export, mail, tickets, cart, orderimport, orders, invoices, cleanup, update_check, quotas, notifications, vouchers # NOQA + from .services import auth, checkin, currencies, export, mail, tickets, cart, modelimport, orders, invoices, cleanup, update_check, quotas, notifications, vouchers # NOQA from .models import _transactions # NOQA from django.conf import settings diff --git a/src/pretix/base/modelimport.py b/src/pretix/base/modelimport.py new file mode 100644 index 0000000000..7f2e38017d --- /dev/null +++ b/src/pretix/base/modelimport.py @@ -0,0 +1,287 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +import csv +import datetime +import io +import re +from decimal import Decimal, DecimalException + +from django.core.exceptions import ValidationError +from django.core.validators import validate_integer +from django.utils import formats +from django.utils.functional import cached_property +from django.utils.translation import gettext as _, gettext_lazy, pgettext + +from pretix.base.i18n import LazyLocaleException +from pretix.base.models import SubEvent + + +class DataImportError(LazyLocaleException): + def __init__(self, *args): + msg = args[0] + msgargs = args[1] if len(args) > 1 else None + self.args = args + if msgargs: + msg = _(msg) % msgargs + else: + msg = _(msg) + super().__init__(msg) + + +def parse_csv(file, length=None, mode="strict", charset=None): + file.seek(0) + data = file.read(length) + if not charset: + try: + import chardet + charset = chardet.detect(data)['encoding'] + except ImportError: + charset = file.charset + data = data.decode(charset or "utf-8", mode) + # If the file was modified on a Mac, it only contains \r as line breaks + if '\r' in data and '\n' not in data: + data = data.replace('\r', '\n') + + try: + dialect = csv.Sniffer().sniff(data.split("\n")[0], delimiters=";,.#:") + except csv.Error: + return None + + if dialect is None: + return None + + reader = csv.DictReader(io.StringIO(data), dialect=dialect) + return reader + + +class ImportColumn: + + @property + def identifier(self): + """ + Unique, internal name of the column. + """ + raise NotImplementedError + + @property + def verbose_name(self): + """ + Human-readable description of the column + """ + raise NotImplementedError + + @property + def initial(self): + """ + Initial value for the form component + """ + return None + + @property + def default_value(self): + """ + Internal default value for the assignment of this column. Defaults to ``empty``. Return ``None`` to disable this + option. + """ + return 'empty' + + @property + def default_label(self): + """ + Human-readable description of the default assignment of this column, defaults to "Keep empty". + """ + return gettext_lazy('Keep empty') + + def __init__(self, event): + self.event = event + + def static_choices(self): + """ + This will be called when rendering the form component and allows you to return a list of values that can be + selected by the user statically during import. + + :return: list of 2-tuples of strings + """ + return [] + + def resolve(self, settings, record): + """ + This method will be called to get the raw value for this field, usually by either using a static value or + inspecting the CSV file for the assigned header. You usually do not need to implement this on your own, + the default should be fine. + """ + k = settings.get(self.identifier, self.default_value) + if k == self.default_value: + return None + elif k.startswith('csv:'): + return record.get(k[4:], None) or None + elif k.startswith('static:'): + return k[7:] + raise ValidationError(_('Invalid setting for column "{header}".').format(header=self.verbose_name)) + + def clean(self, value, previous_values): + """ + Allows you to validate the raw input value for your column. Raise ``ValidationError`` if the value is invalid. + You do not need to include the column or row name or value in the error message as it will automatically be + included. + + :param value: Contains the raw value of your column as returned by ``resolve``. This can usually be ``None``, + e.g. if the column is empty or does not exist in this row. + :param previous_values: Dictionary containing the validated values of all columns that have already been validated. + """ + return value + + def assign(self, value, obj, **kwargs): + """ + This will be called to perform the actual import. You are supposed to set attributes on the ``obj`` or other + related objects that get passed in based on the input ``value``. This is called *before* the actual database + transaction, so the input objects do not yet have a primary key. If you want to create related objects, you + need to place them into some sort of internal queue and persist them when ``save`` is called. + """ + pass + + def save(self, obj): + """ + This will be called to perform the actual import. This is called inside the actual database transaction and the + input object ``obj`` has already been saved to the database. + """ + pass + + @property + def timezone(self): + return self.event.timezone + + +def i18n_flat(l): + if isinstance(l.data, dict): + return l.data.values() + return [l.data] + + +class BooleanColumnMixin: + default_value = None + initial = "static:false" + + def static_choices(self): + return ( + ("false", _("No")), + ("true", _("Yes")), + ) + + def clean(self, value, previous_values): + if not value: + return False + + if value.lower() in ("true", "1", "yes", _("Yes").lower()): + return True + elif value.lower() in ("false", "0", "no", _("No").lower()): + return False + else: + raise ValidationError(_("Could not parse {value} as a yes/no value.").format(value=value)) + + +class DatetimeColumnMixin: + def clean(self, value, previous_values): + if not value: + return + + input_formats = formats.get_format('DATETIME_INPUT_FORMATS', use_l10n=True) + for format in input_formats: + try: + d = datetime.datetime.strptime(value, format) + d = d.replace(tzinfo=self.timezone) + return d + except (ValueError, TypeError): + pass + else: + raise ValidationError(_("Could not parse {value} as a date and time.").format(value=value)) + + +class DecimalColumnMixin: + def clean(self, value, previous_values): + if value not in (None, ''): + value = formats.sanitize_separators(re.sub(r'[^0-9.,-]', '', value)) + try: + value = Decimal(value) + except (DecimalException, TypeError): + raise ValidationError(_('You entered an invalid number.')) + return value + + +class IntegerColumnMixin: + def clean(self, value, previous_values): + if value is not None: + validate_integer(value) + return int(value) + + +class SubeventColumnMixin: + + def __init__(self, *args, **kwargs): + self._subevent_cache = {} + super().__init__(*args, **kwargs) + + @cached_property + def subevents(self): + return list(self.event.subevents.filter(active=True).order_by('date_from')) + + def static_choices(self): + return [ + (str(p.pk), str(p)) for p in self.subevents + ] + + def clean(self, value, previous_values): + if value in self._subevent_cache: + return self._subevent_cache[value] + + input_formats = formats.get_format('DATETIME_INPUT_FORMATS', use_l10n=True) + for format in input_formats: + try: + d = datetime.datetime.strptime(value, format) + d = d.replace(tzinfo=self.event.timezone) + try: + se = self.event.subevents.get( + active=True, + date_from__gt=d - datetime.timedelta(seconds=1), + date_from__lt=d + datetime.timedelta(seconds=1), + ) + self._subevent_cache[value] = se + return se + except SubEvent.DoesNotExist: + raise ValidationError(pgettext("subevent", "No matching date was found.")) + except SubEvent.MultipleObjectsReturned: + raise ValidationError(pgettext("subevent", "Multiple matching dates were found.")) + except (ValueError, TypeError): + continue + + matches = [ + p for p in self.subevents + if str(p.pk) == value or any( + (v and v == value) for v in i18n_flat(p.name)) or p.date_from.isoformat() == value + ] + if len(matches) == 0: + raise ValidationError(pgettext("subevent", "No matching date was found.")) + if len(matches) > 1: + raise ValidationError(pgettext("subevent", "Multiple matching dates were found.")) + + self._subevent_cache[value] = matches[0] + return matches[0] diff --git a/src/pretix/base/orderimport.py b/src/pretix/base/modelimport_orders.py similarity index 76% rename from src/pretix/base/orderimport.py rename to src/pretix/base/modelimport_orders.py index f272c52c29..7ed48acb26 100644 --- a/src/pretix/base/orderimport.py +++ b/src/pretix/base/modelimport_orders.py @@ -19,17 +19,13 @@ # You should have received a copy of the GNU Affero General Public License along with this program. If not, see # . # -import datetime -import re from collections import defaultdict -from decimal import Decimal, DecimalException import pycountry from django.conf import settings from django.core.exceptions import ValidationError from django.core.validators import EmailValidator from django.db.models import Q -from django.utils import formats from django.utils.functional import cached_property from django.utils.translation import ( gettext as _, gettext_lazy, pgettext, pgettext_lazy, @@ -42,9 +38,13 @@ from phonenumbers import SUPPORTED_REGIONS from pretix.base.channels import get_all_sales_channels from pretix.base.forms.questions import guess_country +from pretix.base.modelimport import ( + DatetimeColumnMixin, DecimalColumnMixin, ImportColumn, SubeventColumnMixin, + i18n_flat, +) from pretix.base.models import ( Customer, ItemVariation, OrderPosition, Question, QuestionAnswer, - QuestionOption, Seat, SubEvent, + QuestionOption, Seat, ) from pretix.base.services.pricing import get_price from pretix.base.settings import ( @@ -53,99 +53,6 @@ from pretix.base.settings import ( from pretix.base.signals import order_import_columns -class ImportColumn: - @property - def identifier(self): - """ - Unique, internal name of the column. - """ - raise NotImplementedError - - @property - def verbose_name(self): - """ - Human-readable description of the column - """ - raise NotImplementedError - - @property - def initial(self): - """ - Initial value for the form component - """ - return None - - @property - def default_value(self): - """ - Internal default value for the assignment of this column. Defaults to ``empty``. Return ``None`` to disable this - option. - """ - return 'empty' - - @property - def default_label(self): - """ - Human-readable description of the default assignment of this column, defaults to "Keep empty". - """ - return gettext_lazy('Keep empty') - - def __init__(self, event): - self.event = event - - def static_choices(self): - """ - This will be called when rendering the form component and allows you to return a list of values that can be - selected by the user statically during import. - - :return: list of 2-tuples of strings - """ - return [] - - def resolve(self, settings, record): - """ - This method will be called to get the raw value for this field, usually by either using a static value or - inspecting the CSV file for the assigned header. You usually do not need to implement this on your own, - the default should be fine. - """ - k = settings.get(self.identifier, self.default_value) - if k == self.default_value: - return None - elif k.startswith('csv:'): - return record.get(k[4:], None) or None - elif k.startswith('static:'): - return k[7:] - raise ValidationError(_('Invalid setting for column "{header}".').format(header=self.verbose_name)) - - def clean(self, value, previous_values): - """ - Allows you to validate the raw input value for your column. Raise ``ValidationError`` if the value is invalid. - You do not need to include the column or row name or value in the error message as it will automatically be - included. - - :param value: Contains the raw value of your column as returned by ``resolve``. This can usually be ``None``, - e.g. if the column is empty or does not exist in this row. - :param previous_values: Dictionary containing the validated values of all columns that have already been validated. - """ - return value - - def assign(self, value, order, position, invoice_address, **kwargs): - """ - This will be called to perform the actual import. You are supposed to set attributes on the ``order``, ``position``, - or ``invoice_address`` objects based on the input ``value``. This is called *before* the actual database - transaction, so these three objects do not yet have a primary key. If you want to create related objects, you - need to place them into some sort of internal queue and persist them when ``save`` is called. - """ - pass - - def save(self, order): - """ - This will be called to perform the actual import. This is called inside the actual database transaction and the - input object ``order`` has already been saved to the database. - """ - pass - - class EmailColumn(ImportColumn): identifier = 'email' verbose_name = gettext_lazy('E-mail address') @@ -182,74 +89,20 @@ class PhoneColumn(ImportColumn): order.phone = value -class SubeventColumn(ImportColumn): +class SubeventColumn(SubeventColumnMixin, ImportColumn): identifier = 'subevent' verbose_name = pgettext_lazy('subevents', 'Date') default_value = None - def __init__(self, *args, **kwargs): - self._subevent_cache = {} - super().__init__(*args, **kwargs) - - @cached_property - def subevents(self): - return list(self.event.subevents.filter(active=True).order_by('date_from')) - - def static_choices(self): - return [ - (str(p.pk), str(p)) for p in self.subevents - ] - def clean(self, value, previous_values): if not value: raise ValidationError(pgettext("subevent", "You need to select a date.")) - - if value in self._subevent_cache: - return self._subevent_cache[value] - - input_formats = formats.get_format('DATETIME_INPUT_FORMATS', use_l10n=True) - for format in input_formats: - try: - d = datetime.datetime.strptime(value, format) - d = d.replace(tzinfo=self.event.timezone) - try: - se = self.event.subevents.get( - active=True, - date_from__gt=d - datetime.timedelta(seconds=1), - date_from__lt=d + datetime.timedelta(seconds=1), - ) - self._subevent_cache[value] = se - return se - except SubEvent.DoesNotExist: - raise ValidationError(pgettext("subevent", "No matching date was found.")) - except SubEvent.MultipleObjectsReturned: - raise ValidationError(pgettext("subevent", "Multiple matching dates were found.")) - except (ValueError, TypeError): - continue - - matches = [ - p for p in self.subevents - if str(p.pk) == value or any( - (v and v == value) for v in i18n_flat(p.name)) or p.date_from.isoformat() == value - ] - if len(matches) == 0: - raise ValidationError(pgettext("subevent", "No matching date was found.")) - if len(matches) > 1: - raise ValidationError(pgettext("subevent", "Multiple matching dates were found.")) - - self._subevent_cache[value] = matches[0] - return matches[0] + return super().clean(value, previous_values) def assign(self, value, order, position, invoice_address, **kwargs): position.subevent = value -def i18n_flat(l): - if isinstance(l.data, dict): - return l.data.values() - return [l.data] - - class ItemColumn(ImportColumn): identifier = 'item' verbose_name = gettext_lazy('Product') @@ -572,20 +425,11 @@ class AttendeeState(ImportColumn): position.state = value or '' -class Price(ImportColumn): +class Price(DecimalColumnMixin, ImportColumn): identifier = 'price' verbose_name = gettext_lazy('Price') default_label = gettext_lazy('Calculate from product') - def clean(self, value, previous_values): - if value not in (None, ''): - value = formats.sanitize_separators(re.sub(r'[^0-9.,-]', '', value)) - try: - value = Decimal(value) - except (DecimalException, TypeError): - raise ValidationError(_('You entered an invalid number.')) - return value - def assign(self, value, order, position, invoice_address, **kwargs): if value is None: p = get_price(position.item, position.variation, position.voucher, subevent=position.subevent, @@ -649,48 +493,18 @@ class Locale(ImportColumn): order.locale = value -class ValidFrom(ImportColumn): +class ValidFrom(DatetimeColumnMixin, ImportColumn): identifier = 'valid_from' verbose_name = gettext_lazy('Valid from') - def clean(self, value, previous_values): - if not value: - return - - input_formats = formats.get_format('DATETIME_INPUT_FORMATS', use_l10n=True) - for format in input_formats: - try: - d = datetime.datetime.strptime(value, format) - d = d.replace(tzinfo=self.event.timezone) - return d - except (ValueError, TypeError): - pass - else: - raise ValidationError(_("Could not parse {value} as a date and time.").format(value=value)) - def assign(self, value, order, position, invoice_address, **kwargs): position.valid_from = value -class ValidUntil(ImportColumn): +class ValidUntil(DatetimeColumnMixin, ImportColumn): identifier = 'valid_until' verbose_name = gettext_lazy('Valid until') - def clean(self, value, previous_values): - if not value: - return - - input_formats = formats.get_format('DATETIME_INPUT_FORMATS', use_l10n=True) - for format in input_formats: - try: - d = datetime.datetime.strptime(value, format) - d = d.replace(tzinfo=self.event.timezone) - return d - except (ValueError, TypeError): - pass - else: - raise ValidationError(_("Could not parse {value} as a date and time.").format(value=value)) - def assign(self, value, order, position, invoice_address, **kwargs): position.valid_until = value @@ -849,7 +663,7 @@ class CustomerColumn(ImportColumn): order.customer = value -def get_all_columns(event): +def get_order_import_columns(event): default = [] if event.has_subevents: default.append(SubeventColumn(event)) diff --git a/src/pretix/base/modelimport_vouchers.py b/src/pretix/base/modelimport_vouchers.py new file mode 100644 index 0000000000..6c3c771209 --- /dev/null +++ b/src/pretix/base/modelimport_vouchers.py @@ -0,0 +1,378 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +from decimal import Decimal + +from django.core.exceptions import ValidationError +from django.core.validators import MinLengthValidator +from django.utils.functional import cached_property +from django.utils.translation import gettext as _, gettext_lazy, pgettext_lazy + +from pretix.base.modelimport import ( + BooleanColumnMixin, DatetimeColumnMixin, DecimalColumnMixin, ImportColumn, + IntegerColumnMixin, i18n_flat, +) +from pretix.base.models import ItemVariation, Quota, Seat, Voucher +from pretix.base.signals import voucher_import_columns + + +class CodeColumn(ImportColumn): + identifier = 'code' + verbose_name = gettext_lazy('Voucher code') + default_value = None + + def __init__(self, *args): + self._cached = set() + super().__init__(*args) + + def clean(self, value, previous_values): + if value: + MinLengthValidator(5)(value) + if value and (value in self._cached or Voucher.objects.filter(event=self.event, code=value).exists()): + raise ValidationError(_('A voucher with this code already exists.')) + self._cached.add(value) + return value + + def assign(self, value, obj: Voucher, **kwargs): + obj.code = value + + +class SubeventColumn(ImportColumn): + identifier = 'subevent' + verbose_name = pgettext_lazy('subevents', 'Date') + + def assign(self, value, obj: Voucher, **kwargs): + obj.subevent = value + + +class MaxUsagesColumn(IntegerColumnMixin, ImportColumn): + identifier = 'max_usages' + verbose_name = gettext_lazy('Maximum usages') + initial = "static:1" + + def static_choices(self): + return [ + ("1", "1") + ] + + def assign(self, value, obj: Voucher, **kwargs): + obj.max_usages = value if value is not None else 1 + + +class MinUsagesColumn(IntegerColumnMixin, ImportColumn): + identifier = 'min_usages' + verbose_name = gettext_lazy('Minimum usages') + initial = "static:1" + + def static_choices(self): + return [ + ("1", "1") + ] + + def assign(self, value, obj: Voucher, **kwargs): + obj.min_usages = value if value is not None else 1 + + +class BudgetColumn(DecimalColumnMixin, ImportColumn): + identifier = 'budget' + verbose_name = gettext_lazy('Maximum discount budget') + + def assign(self, value, obj: Voucher, **kwargs): + obj.budget = value + + +class ValidUntilColumn(DatetimeColumnMixin, ImportColumn): + identifier = 'valid_until' + verbose_name = gettext_lazy('Valid until') + + def assign(self, value, obj: Voucher, **kwargs): + obj.valid_until = value + + +class BlockQuotaColumn(BooleanColumnMixin, ImportColumn): + identifier = 'block_quota' + verbose_name = gettext_lazy('Reserve ticket from quota') + + def assign(self, value, obj: Voucher, **kwargs): + obj.block_quota = value + + +class AllowIgnoreQuotaColumn(BooleanColumnMixin, ImportColumn): + identifier = 'allow_ignore_quota' + verbose_name = gettext_lazy('Allow to bypass quota') + + def assign(self, value, obj: Voucher, **kwargs): + obj.allow_ignore_quota = value + + +class PriceModeColumn(ImportColumn): + identifier = 'price_mode' + verbose_name = gettext_lazy('Price mode') + default_value = None + initial = 'static:none' + + def static_choices(self): + return Voucher.PRICE_MODES + + def clean(self, value, previous_values): + d = dict(Voucher.PRICE_MODES) + reverse = {v: k for k, v in Voucher.PRICE_MODES} + if value in d: + return value + elif value in reverse: + return reverse[value] + else: + raise ValidationError(_("Could not parse {value} as a price mode, use one of {options}.").format( + value=value, options=', '.join(d.keys()) + )) + + def assign(self, value, voucher: Voucher, **kwargs): + voucher.price_mode = value + + +class ValueColumn(DecimalColumnMixin, ImportColumn): + identifier = 'value' + verbose_name = gettext_lazy('Voucher value') + + def clean(self, value, previous_values): + value = super().clean(value, previous_values) + if value and previous_values.get("price_mode") == "none": + raise ValidationError(_("It is pointless to set a value without a price mode.")) + return value + + def assign(self, value, obj: Voucher, **kwargs): + obj.value = value or Decimal("0.00") + + +class ItemColumn(ImportColumn): + identifier = 'item' + verbose_name = gettext_lazy('Product') + + @cached_property + def items(self): + return list(self.event.items.filter(active=True)) + + def static_choices(self): + return [ + (str(p.pk), str(p)) for p in self.items + ] + + def clean(self, value, previous_values): + if not value: + return + matches = [ + p for p in self.items + if str(p.pk) == value or (p.internal_name and p.internal_name == value) or any( + (v and v == value) for v in i18n_flat(p.name)) + ] + if len(matches) == 0: + raise ValidationError(_("No matching product was found.")) + if len(matches) > 1: + raise ValidationError(_("Multiple matching products were found.")) + return matches[0] + + def assign(self, value, voucher, **kwargs): + voucher.item = value + + +class VariationColumn(ImportColumn): + identifier = 'variation' + verbose_name = gettext_lazy('Product variation') + + @cached_property + def items(self): + return list(ItemVariation.objects.filter( + active=True, item__active=True, item__event=self.event + ).select_related('item')) + + def static_choices(self): + return [ + (str(p.pk), '{} – {}'.format(p.item, p.value)) for p in self.items + ] + + def clean(self, value, previous_values): + if value: + matches = [ + p for p in self.items + if (str(p.pk) == value or any((v and v == value) for v in i18n_flat(p.value))) and p.item_id == previous_values['item'].pk + ] + if len(matches) == 0: + raise ValidationError(_("No matching variation was found.")) + if len(matches) > 1: + raise ValidationError(_("Multiple matching variations were found.")) + return matches[0] + return value + + def assign(self, value, voucher: Voucher, **kwargs): + voucher.variation = value + + +class QuotaColumn(ImportColumn): + identifier = 'quota' + verbose_name = gettext_lazy('Quota') + + @cached_property + def quotas(self): + return list(Quota.objects.filter( + event=self.event + )) + + def static_choices(self): + return [ + (str(q.pk), q.name) for q in self.quotas + ] + + def clean(self, value, previous_values): + if value: + if previous_values.get('item'): + raise ValidationError(_("You cannot specify a quota if you specified a product.")) + matches = [ + q for q in self.quotas + if str(q.pk) == value or any((v and v == value) for v in i18n_flat(q.name)) + ] + if len(matches) == 0: + raise ValidationError(_("No matching variation was found.")) + if len(matches) > 1: + raise ValidationError(_("Multiple matching variations were found.")) + + return matches[0] + return value + + def assign(self, value, voucher: Voucher, **kwargs): + voucher.quota = value + + +class SeatColumn(ImportColumn): + identifier = 'seat' + verbose_name = gettext_lazy('Seat ID') + + def __init__(self, *args): + self._cached = set() + super().__init__(*args) + + def clean(self, value, previous_values): + if value: + if self.event.has_subevents: + if not previous_values.get('subevent'): + raise ValidationError(_('You need to choose a date if you select a seat.')) + + try: + value = Seat.objects.get( + event=self.event, + seat_guid=value, + subevent=previous_values.get('subevent') + ) + except Seat.MultipleObjectsReturned: + raise ValidationError(_('Multiple matching seats were found.')) + except Seat.DoesNotExist: + raise ValidationError(_('No matching seat was found.')) + if not value.is_available() or value in self._cached: + raise ValidationError( + _('The seat you selected has already been taken. Please select a different seat.')) + + if previous_values.get("quota"): + raise ValidationError(_('You need to choose a specific product if you select a seat.')) + + if previous_values.get('max_usages', 1) > 1 or previous_values.get('min_usages', 1) > 1: + raise ValidationError(_('Seat-specific vouchers can only be used once.')) + + if previous_values.get("item") and value.product != previous_values.get("item"): + raise ValidationError( + _('You need to choose the product "{prod}" for this seat.').format(prod=value.product) + ) + + self._cached.add(value) + return value + + def assign(self, value, voucher: Voucher, **kwargs): + voucher.seat = value + + +class TagColumn(ImportColumn): + identifier = 'tag' + verbose_name = gettext_lazy('Tag') + + def assign(self, value, voucher: Voucher, **kwargs): + voucher.tag = value or '' + + +class CommentColumn(ImportColumn): + identifier = 'comment' + verbose_name = gettext_lazy('Comment') + + def assign(self, value, voucher: Voucher, **kwargs): + voucher.comment = value or '' + + +class ShowHiddenItemsColumn(BooleanColumnMixin, ImportColumn): + identifier = 'show_hidden_items' + verbose_name = gettext_lazy('Shows hidden products that match this voucher') + initial = "static:true" + + def assign(self, value, obj: Voucher, **kwargs): + obj.show_hidden_items = value + + +class AllAddonsIncludedColumn(BooleanColumnMixin, ImportColumn): + identifier = 'all_addons_included' + verbose_name = gettext_lazy('Offer all add-on products for free when redeeming this voucher') + + def assign(self, value, obj: Voucher, **kwargs): + obj.all_addons_included = value + + +class AllBundlesIncludedColumn(BooleanColumnMixin, ImportColumn): + identifier = 'all_bundles_included' + verbose_name = gettext_lazy('Include all bundled products without a designated price when redeeming this voucher') + + def assign(self, value, obj: Voucher, **kwargs): + obj.all_bundles_included = value + + +def get_voucher_import_columns(event): + default = [] + if event.has_subevents: + default.append(SubeventColumn(event)) + default += [ + CodeColumn(event), + MaxUsagesColumn(event), + MinUsagesColumn(event), + BudgetColumn(event), + ValidUntilColumn(event), + BlockQuotaColumn(event), + AllowIgnoreQuotaColumn(event), + PriceModeColumn(event), + ValueColumn(event), + ItemColumn(event), + VariationColumn(event), + QuotaColumn(event), + SeatColumn(event), + TagColumn(event), + CommentColumn(event), + ShowHiddenItemsColumn(event), + AllAddonsIncludedColumn(event), + AllBundlesIncludedColumn(event), + ] + + for recv, resp in voucher_import_columns.send(sender=event): + default += resp + + return default diff --git a/src/pretix/base/models/vouchers.py b/src/pretix/base/models/vouchers.py index d57c7318ba..5d963f1350 100644 --- a/src/pretix/base/models/vouchers.py +++ b/src/pretix/base/models/vouchers.py @@ -517,9 +517,6 @@ class Voucher(LoggedModel): if item and seat.product != item: raise ValidationError(_('You need to choose the product "{prod}" for this seat.').format(prod=seat.product)) - if not seat.is_available(ignore_voucher_id=pk): - raise ValidationError(_('The seat "{id}" is already sold or currently blocked.').format(id=seat.seat_guid)) - return seat def save(self, *args, **kwargs): diff --git a/src/pretix/base/services/orderimport.py b/src/pretix/base/services/modelimport.py similarity index 63% rename from src/pretix/base/services/orderimport.py rename to src/pretix/base/services/modelimport.py index 0e1c6fe54e..8c4d2ef929 100644 --- a/src/pretix/base/services/orderimport.py +++ b/src/pretix/base/services/modelimport.py @@ -19,9 +19,8 @@ # You should have received a copy of the GNU Affero General Public License along with this program. If not, see # . # -import csv -import io from decimal import Decimal +from typing import List from django.conf import settings as django_settings from django.core.exceptions import ValidationError @@ -29,13 +28,15 @@ from django.db import transaction from django.utils.timezone import now from django.utils.translation import gettext as _ -from pretix.base.i18n import LazyLocaleException, language +from pretix.base.i18n import language +from pretix.base.modelimport import DataImportError, ImportColumn, parse_csv +from pretix.base.modelimport_orders import get_order_import_columns +from pretix.base.modelimport_vouchers import get_voucher_import_columns from pretix.base.models import ( CachedFile, Event, InvoiceAddress, Order, OrderPayment, OrderPosition, - User, + User, Voucher, ) from pretix.base.models.orders import Transaction -from pretix.base.orderimport import get_all_columns from pretix.base.services.invoices import generate_invoice, invoice_qualified from pretix.base.services.locking import lock_objects from pretix.base.services.tasks import ProfiledEventTask @@ -43,47 +44,36 @@ from pretix.base.signals import order_paid, order_placed from pretix.celery_app import app -class DataImportError(LazyLocaleException): - def __init__(self, *args): - msg = args[0] - msgargs = args[1] if len(args) > 1 else None - self.args = args - if msgargs: - msg = _(msg) % msgargs - else: - msg = _(msg) - super().__init__(msg) - - -def parse_csv(file, length=None, mode="strict", charset=None): - file.seek(0) - data = file.read(length) - if not charset: - try: - import chardet - charset = chardet.detect(data)['encoding'] - except ImportError: - charset = file.charset - data = data.decode(charset or "utf-8", mode) - # If the file was modified on a Mac, it only contains \r as line breaks - if '\r' in data and '\n' not in data: - data = data.replace('\r', '\n') - +def _validate(cf: CachedFile, charset: str, cols: List[ImportColumn], settings: dict): try: - dialect = csv.Sniffer().sniff(data.split("\n")[0], delimiters=";,.#:") - except csv.Error: - return None - - if dialect is None: - return None - - reader = csv.DictReader(io.StringIO(data), dialect=dialect) - return reader - - -def setif(record, obj, attr, setting): - if setting.startswith('csv:'): - setattr(obj, attr, record[setting[4:]] or '') + parsed = parse_csv(cf.file, charset=charset) + except UnicodeDecodeError as e: + raise DataImportError( + _( + 'Error decoding special characters in your file: {message}').format( + message=str(e) + ) + ) + data = [] + for i, record in enumerate(parsed): + if not any(record.values()): + continue + values = {} + for c in cols: + val = c.resolve(settings, record) + if isinstance(val, str): + val = val.strip() + try: + values[c.identifier] = c.clean(val, values) + except ValidationError as e: + raise DataImportError( + _( + 'Error while importing value "{value}" for column "{column}" in line "{line}": {message}').format( + value=val if val is not None else '', column=c.verbose_name, line=i + 1, message=e.message + ) + ) + data.append(values) + return data @app.task(base=ProfiledEventTask, throws=(DataImportError,)) @@ -91,45 +81,17 @@ def import_orders(event: Event, fileid: str, settings: dict, locale: str, user, cf = CachedFile.objects.get(id=fileid) user = User.objects.get(pk=user) with language(locale, event.settings.region): - cols = get_all_columns(event) - try: - parsed = parse_csv(cf.file, charset=charset) - except UnicodeDecodeError as e: - raise DataImportError( - _( - 'Error decoding special characters in your file: {message}').format( - message=str(e) - ) - ) - orders = [] - order = None - data = [] - - # Run validation - for i, record in enumerate(parsed): - if not any(record.values()): - continue - values = {} - for c in cols: - val = c.resolve(settings, record) - if isinstance(val, str): - val = val.strip() - try: - values[c.identifier] = c.clean(val, values) - except ValidationError as e: - raise DataImportError( - _( - 'Error while importing value "{value}" for column "{column}" in line "{line}": {message}').format( - value=val if val is not None else '', column=c.verbose_name, line=i + 1, message=e.message - ) - ) - data.append(values) + cols = get_order_import_columns(event) + data = _validate(cf, charset, cols, settings) if settings['orders'] == 'one' and len(data) > django_settings.PRETIX_MAX_ORDER_SIZE: raise DataImportError( _('Orders cannot have more than %(max)s positions.') % {'max': django_settings.PRETIX_MAX_ORDER_SIZE} ) + orders = [] + order = None + # Prepare model objects. Yes, this might consume lots of RAM, but allows us to make the actual SQL transaction # shorter. We'll see what works better in reality… lock_seats = [] @@ -149,16 +111,16 @@ def import_orders(event: Event, fileid: str, settings: dict, locale: str, user, position = OrderPosition(positionid=len(order._positions) + 1) position.attendee_name_parts = {'_scheme': event.settings.name_scheme} position.meta_info = {} - if position.seat is not None: - lock_seats.append(position.seat) order._positions.append(position) position.assign_pseudonymization_id() for c in cols: c.assign(record.get(c.identifier), order, position, order._address) - except ImportError as e: - raise ImportError( + if position.seat is not None: + lock_seats.append(position.seat) + except (ValidationError, ImportError) as e: + raise DataImportError( _('Invalid data in row {row}: {message}').format(row=i, message=str(e)) ) @@ -169,7 +131,7 @@ def import_orders(event: Event, fileid: str, settings: dict, locale: str, user, lock_objects(lock_seats, shared_lock_objects=[event]) for s in lock_seats: if not s.is_available(): - raise ImportError(_('The seat you selected has already been taken. Please select a different seat.')) + raise DataImportError(_('The seat you selected has already been taken. Please select a different seat.')) save_transactions = [] for o in orders: @@ -232,3 +194,62 @@ def import_orders(event: Event, fileid: str, settings: dict, locale: str, user, raise ValidationError(_('We were not able to process your request completely as the server was too busy. ' 'Please try again.')) cf.delete() + + +@app.task(base=ProfiledEventTask, throws=(DataImportError,)) +def import_vouchers(event: Event, fileid: str, settings: dict, locale: str, user, charset=None) -> None: + cf = CachedFile.objects.get(id=fileid) + user = User.objects.get(pk=user) + with language(locale, event.settings.region): + cols = get_voucher_import_columns(event) + data = _validate(cf, charset, cols, settings) + + # Prepare model objects. Yes, this might consume lots of RAM, but allows us to make the actual SQL transaction + # shorter. We'll see what works better in reality… + vouchers = [] + lock_seats = [] + for i, record in enumerate(data): + try: + voucher = Voucher(event=event) + vouchers.append(voucher) + + Voucher.clean_item_properties( + record, + event, + record.get('quota'), + record.get('item'), + record.get('variation'), + block_quota=record.get('block_quota') + ) + Voucher.clean_subevent(record, event) + Voucher.clean_max_usages(record, 0) + + for c in cols: + c.assign(record.get(c.identifier), voucher) + + if voucher.seat is not None: + lock_seats.append(voucher.seat) + except (ValidationError, ImportError) as e: + raise DataImportError( + _('Invalid data in row {row}: {message}').format(row=i, message=str(e)) + ) + + with transaction.atomic(): + # We don't support quotas here, so we only need to lock if seats are in use + if lock_seats: + lock_objects(lock_seats, shared_lock_objects=[event]) + for s in lock_seats: + if not s.is_available(): + raise DataImportError( + _('The seat you selected has already been taken. Please select a different seat.')) + + for v in vouchers: + v.save() + v.log_action( + 'pretix.voucher.added', + user=user, + data={'source': 'import'} + ) + for c in cols: + c.save(v) + cf.delete() diff --git a/src/pretix/base/signals.py b/src/pretix/base/signals.py index 2a09b8e480..dc8467d2c4 100644 --- a/src/pretix/base/signals.py +++ b/src/pretix/base/signals.py @@ -785,6 +785,15 @@ to define additional columns that can be read during import. You are expected to As with all event-plugin signals, the ``sender`` keyword argument will contain the event. """ +voucher_import_columns = EventPluginSignal() +""" +This signal is sent out if the user performs an import of vouchers from an external source. You can use this +to define additional columns that can be read during import. You are expected to return a list of instances of +``ImportColumn`` subclasses. + +As with all event-plugin signals, the ``sender`` keyword argument will contain the event. +""" + validate_event_settings = EventPluginSignal() """ Arguments: ``settings_dict`` diff --git a/src/pretix/control/forms/orderimport.py b/src/pretix/control/forms/modelimport.py similarity index 69% rename from src/pretix/control/forms/orderimport.py rename to src/pretix/control/forms/modelimport.py index c00335e6ec..565c48f7e4 100644 --- a/src/pretix/control/forms/orderimport.py +++ b/src/pretix/control/forms/modelimport.py @@ -22,10 +22,54 @@ from django import forms from django.utils.translation import gettext_lazy as _ -from pretix.base.services.orderimport import get_all_columns +from pretix.base.modelimport_orders import get_order_import_columns +from pretix.base.modelimport_vouchers import get_voucher_import_columns class ProcessForm(forms.Form): + + def __init__(self, *args, **kwargs): + headers = kwargs.pop('headers') + initital = kwargs.pop('initial', {}) or {} + kwargs['initial'] = initital + columns = self.get_columns() + column_keys = {c.identifier for c in columns} + + if not initital or all(k not in column_keys for k in initital.keys()): + for c in columns: + initital.setdefault(c.identifier, c.initial) + for h in headers: + if h == c.identifier or h == str(c.verbose_name): + initital[c.identifier] = 'csv:{}'.format(h) + break + + super().__init__(*args, **kwargs) + + header_choices = [ + ('csv:{}'.format(h), _('CSV column: "{name}"').format(name=h)) for h in headers + ] + + for c in columns: + choices = [] + if c.default_value: + choices.append((c.default_value, c.default_label)) + choices += header_choices + for k, v in c.static_choices(): + choices.append(('static:{}'.format(k), v)) + + self.fields[c.identifier] = forms.ChoiceField( + label=str(c.verbose_name), + choices=choices, + widget=forms.Select( + attrs={'data-static': 'true'} + ) + ) + + def get_columns(self): + raise NotImplementedError() # noqa + + +class OrdersProcessForm(ProcessForm): orders = forms.ChoiceField( label=_('Import mode'), choices=( @@ -46,29 +90,21 @@ class ProcessForm(forms.Form): ) def __init__(self, *args, **kwargs): - headers = kwargs.pop('headers') - initital = kwargs.pop('initial', {}) self.event = kwargs.pop('event') + initital = kwargs.pop('initial', {}) initital['testmode'] = self.event.testmode kwargs['initial'] = initital super().__init__(*args, **kwargs) - header_choices = [ - ('csv:{}'.format(h), _('CSV column: "{name}"').format(name=h)) for h in headers - ] + def get_columns(self): + return get_order_import_columns(self.event) - for c in get_all_columns(self.event): - choices = [] - if c.default_value: - choices.append((c.default_value, c.default_label)) - choices += header_choices - for k, v in c.static_choices(): - choices.append(('static:{}'.format(k), v)) - self.fields[c.identifier] = forms.ChoiceField( - label=str(c.verbose_name), - choices=choices, - widget=forms.Select( - attrs={'data-static': 'true'} - ) - ) +class VouchersProcessForm(ProcessForm): + + def __init__(self, *args, **kwargs): + self.event = kwargs.pop('event') + super().__init__(*args, **kwargs) + + def get_columns(self): + return get_voucher_import_columns(self.event) diff --git a/src/pretix/control/templates/pretixcontrol/vouchers/import_process.html b/src/pretix/control/templates/pretixcontrol/vouchers/import_process.html new file mode 100644 index 0000000000..bf2bf22d54 --- /dev/null +++ b/src/pretix/control/templates/pretixcontrol/vouchers/import_process.html @@ -0,0 +1,61 @@ +{% extends "pretixcontrol/event/base.html" %} +{% load i18n %} +{% load static %} +{% load getitem %} +{% load bootstrap3 %} +{% block title %}{% trans "Import vouchers" %}{% endblock %} +{% block content %} +

{% trans "Import vouchers" %}

+
+ {% csrf_token %} +
+
+

{% trans "Data preview" %}

+
+
+ + + + {% for fn in parsed.fieldnames %} + + {% endfor %} + + + + {% for r in sample_rows %} + + {% for fn in parsed.fieldnames %} + + {% endfor %} + + {% endfor %} + + + + +
{{ fn }}
{{ r|getitem:fn }}
+ … +
+
+
+
+
+

{% trans "Import settings" %}

+
+
+ {% bootstrap_form_errors form %} + {% bootstrap_form form layout="horizontal" %} +
+ {% blocktrans trimmed %} + The import will be performed regardless of your quotas, so it will be possible to overbook your event using this option. + {% endblocktrans %} +
+
+
+
+ +
+
+{% endblock %} diff --git a/src/pretix/control/templates/pretixcontrol/vouchers/import_start.html b/src/pretix/control/templates/pretixcontrol/vouchers/import_start.html new file mode 100644 index 0000000000..01f586f872 --- /dev/null +++ b/src/pretix/control/templates/pretixcontrol/vouchers/import_start.html @@ -0,0 +1,40 @@ +{% extends "pretixcontrol/event/base.html" %} +{% load i18n %} +{% load static %} +{% block title %}{% trans "Import vouchers" %}{% endblock %} +{% block content %} +

{% trans "Import vouchers" %}

+ +
+
+

{% trans "Upload a new file" %}

+
+
+
+ {% csrf_token %} +

+ {% blocktrans trimmed %} + The uploaded file should be a CSV file with a header row. You will be able to assign the + meanings of the different columns in the next step. + {% endblocktrans %} +

+
+ +
+
+ + +
+
+ +
+
+
+{% endblock %} diff --git a/src/pretix/control/templates/pretixcontrol/vouchers/index.html b/src/pretix/control/templates/pretixcontrol/vouchers/index.html index 028fd1672e..0620162146 100644 --- a/src/pretix/control/templates/pretixcontrol/vouchers/index.html +++ b/src/pretix/control/templates/pretixcontrol/vouchers/index.html @@ -77,6 +77,8 @@ class="btn btn-primary btn-lg"> {% trans "Create a new voucher" %} {% trans "Create multiple new vouchers" %} + {% trans "Import vouchers" %} {% endif %} {% else %} @@ -87,6 +89,9 @@ {% trans "Create multiple new vouchers" %} + + {% trans "Import vouchers" %} {% endif %} diff --git a/src/pretix/control/urls.py b/src/pretix/control/urls.py index 9960c31949..20fe27845f 100644 --- a/src/pretix/control/urls.py +++ b/src/pretix/control/urls.py @@ -38,7 +38,7 @@ from django.views.generic.base import RedirectView from pretix.control.views import ( auth, checkin, dashboards, discounts, event, geo, global_settings, item, - main, oauth, orderimport, orders, organizer, pdf, search, shredder, + main, modelimport, oauth, orders, organizer, pdf, search, shredder, subevents, typeahead, user, users, vouchers, waitinglist, ) @@ -349,6 +349,8 @@ urlpatterns = [ re_path(r'^vouchers/bulk_add$', vouchers.VoucherBulkCreate.as_view(), name='event.vouchers.bulk'), re_path(r'^vouchers/bulk_add/mail_preview$', vouchers.VoucherBulkMailPreview.as_view(), name='event.vouchers.bulk.mail_preview'), re_path(r'^vouchers/bulk_action$', vouchers.VoucherBulkAction.as_view(), name='event.vouchers.bulkaction'), + re_path(r'^vouchers/import/$', modelimport.VoucherImportView.as_view(), name='event.vouchers.import'), + re_path(r'^vouchers/import/(?P[^/]+)/$', modelimport.VoucherProcessView.as_view(), name='event.vouchers.import.process'), re_path(r'^orders/(?P[0-9A-Z]+)/transition$', orders.OrderTransition.as_view(), name='event.order.transition'), re_path(r'^orders/(?P[0-9A-Z]+)/resend$', orders.OrderResendLink.as_view(), @@ -415,8 +417,8 @@ urlpatterns = [ re_path(r'^invoice/(?P[^/]+)$', orders.InvoiceDownload.as_view(), name='event.invoice.download'), re_path(r'^orders/overview/$', orders.OverView.as_view(), name='event.orders.overview'), - re_path(r'^orders/import/$', orderimport.ImportView.as_view(), name='event.orders.import'), - re_path(r'^orders/import/(?P[^/]+)/$', orderimport.ProcessView.as_view(), name='event.orders.import.process'), + re_path(r'^orders/import/$', modelimport.OrderImportView.as_view(), name='event.orders.import'), + re_path(r'^orders/import/(?P[^/]+)/$', modelimport.OrderProcessView.as_view(), name='event.orders.import.process'), re_path(r'^orders/export/$', orders.ExportView.as_view(), name='event.orders.export'), re_path(r'^orders/export/(?P[^/]+)/run$', orders.RunScheduledExportView.as_view(), name='event.orders.export.scheduled.run'), re_path(r'^orders/export/(?P[^/]+)/delete$', orders.DeleteScheduledExportView.as_view(), name='event.orders.export.scheduled.delete'), diff --git a/src/pretix/control/views/orderimport.py b/src/pretix/control/views/modelimport.py similarity index 68% rename from src/pretix/control/views/orderimport.py rename to src/pretix/control/views/modelimport.py index 1912886468..0f82798f23 100644 --- a/src/pretix/control/views/orderimport.py +++ b/src/pretix/control/views/modelimport.py @@ -46,9 +46,13 @@ from django.utils.translation import gettext_lazy as _ from django.views.generic import FormView, TemplateView from pretix.base.models import CachedFile -from pretix.base.services.orderimport import import_orders, parse_csv +from pretix.base.services.modelimport import ( + import_orders, import_vouchers, parse_csv, +) from pretix.base.views.tasks import AsyncAction -from pretix.control.forms.orderimport import ProcessForm +from pretix.control.forms.modelimport import ( + OrdersProcessForm, VouchersProcessForm, +) from pretix.control.permissions import EventPermissionRequiredMixin from pretix.helpers.http import redirect_to_url @@ -64,28 +68,16 @@ ENCODINGS = ( ) -class ImportView(EventPermissionRequiredMixin, TemplateView): - template_name = 'pretixcontrol/orders/import_start.html' - permission = 'can_change_orders' - +class BaseImportView(TemplateView): def post(self, request, *args, **kwargs): if 'file' not in request.FILES: - return redirect_to_url(reverse('control:event.orders.import', kwargs={ - 'event': request.event.slug, - 'organizer': request.organizer.slug, - })) + return redirect_to_url(request.path) if not request.FILES['file'].name.lower().endswith('.csv'): messages.error(request, _('Please only upload CSV files.')) - return redirect_to_url(reverse('control:event.orders.import', kwargs={ - 'event': request.event.slug, - 'organizer': request.organizer.slug, - })) + return redirect_to_url(request.path) if request.FILES['file'].size > settings.FILE_UPLOAD_MAX_SIZE_OTHER: messages.error(request, _('Please do not upload files larger than 10 MB.')) - return redirect_to_url(reverse('control:event.orders.import', kwargs={ - 'event': request.event.slug, - 'organizer': request.organizer.slug, - })) + return redirect_to_url(request.path) cf = CachedFile.objects.create( expires=now() + timedelta(days=1), @@ -100,41 +92,47 @@ class ImportView(EventPermissionRequiredMixin, TemplateView): else: charset = "auto" - return redirect(reverse('control:event.orders.import.process', kwargs={ - 'event': request.event.slug, - 'organizer': request.organizer.slug, - 'file': cf.id - }) + "?charset=" + charset) + return redirect(self.get_process_url(request, cf, charset)) def get_context_data(self, **kwargs): return super().get_context_data(encodings=ENCODINGS) + def get_process_url(self, request, cf, charset): + raise NotImplementedError() # noqa -class ProcessView(EventPermissionRequiredMixin, AsyncAction, FormView): - permission = 'can_change_orders' - template_name = 'pretixcontrol/orders/import_process.html' - form_class = ProcessForm - task = import_orders + +class BaseProcessView(AsyncAction, FormView): known_errortypes = ['DataImportError'] + @property + def settings_key(self): + raise NotImplementedError() # noqa + + @property + def settings_holder(self): + raise NotImplementedError() # noqa + def get_form_kwargs(self): k = super().get_form_kwargs() k.update({ - 'event': self.request.event, - 'initial': self.request.event.settings.order_import_settings, + 'initial': self.settings_holder.settings.get(self.settings_key, as_type=dict), 'headers': self.parsed.fieldnames }) return k def form_valid(self, form): - self.request.event.settings.order_import_settings = form.cleaned_data + self.settings_holder.settings.set(self.settings_key, form.cleaned_data) if self.request.GET.get("charset") in ENCODINGS: charset = self.request.GET.get("charset") else: charset = None return self.do( - self.request.event.pk, self.file.id, form.cleaned_data, self.request.LANGUAGE_CODE, - self.request.user.pk, charset + self.settings_holder.pk, + self.file.id, + form.cleaned_data, + self.request.LANGUAGE_CODE, + self.request.user.pk, + charset, ) @cached_property @@ -176,28 +174,21 @@ class ProcessView(EventPermissionRequiredMixin, AsyncAction, FormView): return _('The import was successful.') def get_success_url(self, value): - return reverse('control:event.orders', kwargs={ - 'event': self.request.event.slug, - 'organizer': self.request.organizer.slug, - }) + raise NotImplementedError() # noqa + + def get_form_url(self): + raise NotImplementedError() # noqa def dispatch(self, request, *args, **kwargs): if 'async_id' in request.GET and settings.HAS_CELERY: return self.get_result(request) if not self.parsed or not self.parsed_list: messages.error(request, _('We\'ve been unable to parse the uploaded file as a CSV file.')) - return redirect(reverse('control:event.orders.import', kwargs={ - 'event': request.event.slug, - 'organizer': request.organizer.slug, - })) + return redirect(self.get_form_url()) return super().dispatch(request, *args, **kwargs) def get_error_url(self): - return reverse('control:event.orders.import.process', kwargs={ - 'event': self.request.event.slug, - 'organizer': self.request.organizer.slug, - 'file': self.file.id - }) + return self.request.path def get_context_data(self, **kwargs): ctx = super().get_context_data(**kwargs) @@ -205,3 +196,89 @@ class ProcessView(EventPermissionRequiredMixin, AsyncAction, FormView): ctx['parsed'] = self.parsed ctx['sample_rows'] = self.parsed_list[:3] return ctx + + +class OrderImportView(EventPermissionRequiredMixin, BaseImportView): + template_name = 'pretixcontrol/orders/import_start.html' + permission = 'can_change_orders' + + def get_process_url(self, request, cf, charset): + return reverse('control:event.orders.import.process', kwargs={ + 'event': request.event.slug, + 'organizer': request.organizer.slug, + 'file': cf.id + }) + "?charset=" + charset + + +class OrderProcessView(EventPermissionRequiredMixin, BaseProcessView): + permission = 'can_change_orders' + template_name = 'pretixcontrol/orders/import_process.html' + form_class = OrdersProcessForm + task = import_orders + settings_key = 'order_import_settings' + + @property + def settings_holder(self): + return self.request.event + + def get_form_kwargs(self): + k = super().get_form_kwargs() + k.update({ + 'event': self.request.event, + }) + return k + + def get_form_url(self): + return reverse('control:event.orders.import', kwargs={ + 'event': self.request.event.slug, + 'organizer': self.request.organizer.slug, + }) + + def get_success_url(self, value): + return reverse('control:event.orders', kwargs={ + 'event': self.request.event.slug, + 'organizer': self.request.organizer.slug, + }) + + +class VoucherImportView(EventPermissionRequiredMixin, BaseImportView): + template_name = 'pretixcontrol/vouchers/import_start.html' + permission = 'can_change_vouchers' + + def get_process_url(self, request, cf, charset): + return reverse('control:event.vouchers.import.process', kwargs={ + 'event': request.event.slug, + 'organizer': request.organizer.slug, + 'file': cf.id + }) + "?charset=" + charset + + +class VoucherProcessView(EventPermissionRequiredMixin, BaseProcessView): + permission = 'can_change_vouchers' + template_name = 'pretixcontrol/vouchers/import_process.html' + form_class = VouchersProcessForm + task = import_vouchers + settings_key = 'voucher_import_settings' + + @property + def settings_holder(self): + return self.request.event + + def get_form_kwargs(self): + k = super().get_form_kwargs() + k.update({ + 'event': self.request.event, + }) + return k + + def get_form_url(self): + return reverse('control:event.vouchers.import', kwargs={ + 'event': self.request.event.slug, + 'organizer': self.request.organizer.slug, + }) + + def get_success_url(self, value): + return reverse('control:event.vouchers', kwargs={ + 'event': self.request.event.slug, + 'organizer': self.request.organizer.slug, + }) diff --git a/src/tests/base/test_orderimport.py b/src/tests/base/test_modelimport_orders.py similarity index 99% rename from src/tests/base/test_orderimport.py rename to src/tests/base/test_modelimport_orders.py index b844a7b942..ce287a1d47 100644 --- a/src/tests/base/test_orderimport.py +++ b/src/tests/base/test_modelimport_orders.py @@ -35,7 +35,7 @@ from pretix.base.models import ( CachedFile, Event, Item, Order, OrderPayment, OrderPosition, Organizer, Question, QuestionAnswer, User, ) -from pretix.base.services.orderimport import DataImportError, import_orders +from pretix.base.services.modelimport import DataImportError, import_orders @pytest.fixture diff --git a/src/tests/base/test_modelimport_vouchers.py b/src/tests/base/test_modelimport_vouchers.py new file mode 100644 index 0000000000..d6419ca484 --- /dev/null +++ b/src/tests/base/test_modelimport_vouchers.py @@ -0,0 +1,182 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +import csv +from decimal import Decimal +from io import StringIO + +import pytest +from django.core.files.base import ContentFile +from django.utils.timezone import now +from django_scopes import scopes_disabled + +from pretix.base.models import CachedFile, Event, Item, Organizer, User +from pretix.base.services.modelimport import DataImportError, import_vouchers + + +@pytest.fixture +def event(): + o = Organizer.objects.create(name='Dummy', slug='dummy') + event = Event.objects.create( + organizer=o, name='Dummy', slug='dummy', + date_from=now(), plugins='pretix.plugins.banktransfer,pretix.plugins.paypal' + ) + return event + + +@pytest.fixture +def item(event): + return Item.objects.create(event=event, name="Ticket", default_price=23) + + +@pytest.fixture +def user(): + return User.objects.create_user('test@localhost', 'test') + + +def inputfile_factory(multiplier=1): + d = [ + { + 'A': 'ABCDE123', + 'B': 'Ticket', + 'C': 'True', + 'D': '2021-06-28 11:00:00', + 'E': '2', + 'F': '1', + }, + { + 'A': 'GHIJK432', + 'B': 'Ticket', + 'C': 'False', + 'D': '2021-05-28 11:00:00', + 'E': '2', + 'F': '1', + }, + ] + if multiplier > 1: + d = d * multiplier + f = StringIO() + w = csv.DictWriter(f, ['A', 'B', 'C', 'D', 'E', 'F'], dialect=csv.excel) + w.writeheader() + w.writerows(d) + f.seek(0) + c = CachedFile.objects.create(type="text/csv", filename="input.csv") + c.file.save("input.csv", ContentFile(f.read())) + return c + + +DEFAULT_SETTINGS = { + 'code': 'csv:A', + 'max_usages': 'static:1', + 'min_usages': 'static:1', + 'budget': 'empty', + 'valid_until': 'csv:D', + 'block_quota': 'static:false', + 'allow_ignore_quota': 'static:false', + 'price_mode': 'static:none', + 'value': 'empty', + 'item': 'csv:B', + 'variation': 'empty', + 'quota': 'empty', + 'seat': 'empty', + 'tag': 'empty', + 'comment': 'empty', + 'show_hidden_items': 'static:true', + 'all_addons_included': 'csv:C', + 'all_bundles_included': 'static:false', +} + + +@pytest.mark.django_db +@scopes_disabled() +def test_import_simple(event, item, user): + settings = dict(DEFAULT_SETTINGS) + import_vouchers.apply( + args=(event.pk, inputfile_factory().id, settings, 'en', user.pk) + ).get() + assert event.vouchers.count() == 2 + v = event.vouchers.get(code="ABCDE123") + assert v.item == item + assert v.all_addons_included + assert not v.all_bundles_included + assert v.valid_until.year == 2021 + + +@pytest.mark.django_db +@scopes_disabled() +def test_import_code_unique(event, item, user): + settings = dict(DEFAULT_SETTINGS) + import_vouchers.apply( + args=(event.pk, inputfile_factory().id, settings, 'en', user.pk) + ) + assert event.vouchers.count() == 2 + + with pytest.raises(DataImportError) as excinfo: + import_vouchers.apply( + args=(event.pk, inputfile_factory().id, settings, 'en', user.pk) + ).get() + assert ('Error while importing value "ABCDE123" for column "Voucher code" in line "1": ' + 'A voucher with this code already exists.') in str(excinfo.value) + + +@pytest.mark.django_db +@scopes_disabled() +def test_integer_invalid(event, item, user): + settings = dict(DEFAULT_SETTINGS) + settings['min_usages'] = 'csv:A' + with pytest.raises(DataImportError) as excinfo: + import_vouchers.apply( + args=(event.pk, inputfile_factory().id, settings, 'en', user.pk) + ).get() + assert 'column "Minimum usages" in line "1": Enter a valid integer.' in str(excinfo.value) + + +@pytest.mark.django_db +@scopes_disabled() +def test_model_validation(event, item, user): + settings = dict(DEFAULT_SETTINGS) + settings['min_usages'] = 'csv:E' + with pytest.raises(DataImportError) as excinfo: + import_vouchers.apply( + args=(event.pk, inputfile_factory().id, settings, 'en', user.pk) + ).get() + assert 'The maximum number of usages may not be lower than the minimum number of usages.' in str(excinfo.value) + + +@pytest.mark.django_db +@scopes_disabled() +def test_price_mode_validation(event, item, user): + settings = dict(DEFAULT_SETTINGS) + settings['value'] = 'csv:F' + with pytest.raises(DataImportError) as excinfo: + import_vouchers.apply( + args=(event.pk, inputfile_factory().id, settings, 'en', user.pk) + ).get() + assert 'It is pointless to set a value without a price mode.' in str(excinfo.value) + + settings['price_mode'] = 'static:percent' + import_vouchers.apply( + args=(event.pk, inputfile_factory().id, settings, 'en', user.pk) + ).get() + assert event.vouchers.count() == 2 + v = event.vouchers.get(code="ABCDE123") + assert v.price_mode == "percent" + assert v.value == Decimal("1.00") diff --git a/src/tests/control/test_orderimport.py b/src/tests/control/test_modelimport.py similarity index 78% rename from src/tests/control/test_orderimport.py rename to src/tests/control/test_modelimport.py index 74662d8a76..d24c0b219a 100644 --- a/src/tests/control/test_orderimport.py +++ b/src/tests/control/test_modelimport.py @@ -35,14 +35,15 @@ def env(): date_from=now(), plugins='pretix.plugins.banktransfer,pretix.plugins.paypal' ) user = User.objects.create_user('dummy@dummy.dummy', 'dummy') - t = Team.objects.create(organizer=event.organizer, can_view_orders=True, can_change_orders=True) + t = Team.objects.create(organizer=event.organizer, can_view_orders=True, can_change_orders=True, + can_change_vouchers=True) t.members.add(user) t.limit_events.add(event) return event, user @pytest.mark.django_db -def test_import_csv_file(client, env): +def test_orders_import_csv_file(client, env): client.login(email='dummy@dummy.dummy', password='dummy') r = client.get('/control/event/dummy/dummy/orders/import/') assert r.status_code == 200 @@ -65,3 +66,22 @@ Anke,Müller,anke@example.net assert b"Dieter" in r.content assert b"daniel@example.org" in r.content assert b"Anke" not in r.content + + +@pytest.mark.django_db +def test_vouchers_import_csv_file(client, env): + client.login(email='dummy@dummy.dummy', password='dummy') + r = client.get('/control/event/dummy/dummy/vouchers/import/') + assert r.status_code == 200 + + file = SimpleUploadedFile('file.csv', """Code,Product +ABC123,Ticket + +""".encode("utf-8"), content_type="text/csv") + + r = client.post('/control/event/dummy/dummy/vouchers/import/', { + 'file': file + }, follow=True) + doc = BeautifulSoup(r.content, "lxml") + assert doc.select("select[name=code]") + assert b"ABC123" in r.content