Generalize import process from orders to more models (#4002)

* Generalize import process from orders to more models

* Add voucher import

* Model import: Guess assignments of based on column headers

* Fix lock_seats being pointless

* Update docs

* Update doc/development/api/import.rst

Co-authored-by: Richard Schreiber <schreiber@rami.io>

* Update src/pretix/base/modelimport_vouchers.py

Co-authored-by: Richard Schreiber <schreiber@rami.io>

---------

Co-authored-by: Richard Schreiber <schreiber@rami.io>
This commit is contained in:
Raphael Michel
2024-04-03 10:15:30 +02:00
committed by GitHub
parent 4afb7a4976
commit 990e9da21d
17 changed files with 1298 additions and 362 deletions

View File

@@ -46,7 +46,7 @@ class PretixBaseConfig(AppConfig):
from . import invoice # NOQA
from . import notifications # NOQA
from . import email # NOQA
from .services import auth, checkin, currencies, export, mail, tickets, cart, orderimport, orders, invoices, cleanup, update_check, quotas, notifications, vouchers # NOQA
from .services import auth, checkin, currencies, export, mail, tickets, cart, modelimport, orders, invoices, cleanup, update_check, quotas, notifications, vouchers # NOQA
from .models import _transactions # NOQA
from django.conf import settings

View File

@@ -0,0 +1,287 @@
#
# This file is part of pretix (Community Edition).
#
# Copyright (C) 2014-2020 Raphael Michel and contributors
# Copyright (C) 2020-2021 rami.io GmbH and contributors
#
# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
# Public License as published by the Free Software Foundation in version 3 of the License.
#
# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
# this file, see <https://pretix.eu/about/en/license>.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
# <https://www.gnu.org/licenses/>.
#
import csv
import datetime
import io
import re
from decimal import Decimal, DecimalException
from django.core.exceptions import ValidationError
from django.core.validators import validate_integer
from django.utils import formats
from django.utils.functional import cached_property
from django.utils.translation import gettext as _, gettext_lazy, pgettext
from pretix.base.i18n import LazyLocaleException
from pretix.base.models import SubEvent
class DataImportError(LazyLocaleException):
def __init__(self, *args):
msg = args[0]
msgargs = args[1] if len(args) > 1 else None
self.args = args
if msgargs:
msg = _(msg) % msgargs
else:
msg = _(msg)
super().__init__(msg)
def parse_csv(file, length=None, mode="strict", charset=None):
file.seek(0)
data = file.read(length)
if not charset:
try:
import chardet
charset = chardet.detect(data)['encoding']
except ImportError:
charset = file.charset
data = data.decode(charset or "utf-8", mode)
# If the file was modified on a Mac, it only contains \r as line breaks
if '\r' in data and '\n' not in data:
data = data.replace('\r', '\n')
try:
dialect = csv.Sniffer().sniff(data.split("\n")[0], delimiters=";,.#:")
except csv.Error:
return None
if dialect is None:
return None
reader = csv.DictReader(io.StringIO(data), dialect=dialect)
return reader
class ImportColumn:
@property
def identifier(self):
"""
Unique, internal name of the column.
"""
raise NotImplementedError
@property
def verbose_name(self):
"""
Human-readable description of the column
"""
raise NotImplementedError
@property
def initial(self):
"""
Initial value for the form component
"""
return None
@property
def default_value(self):
"""
Internal default value for the assignment of this column. Defaults to ``empty``. Return ``None`` to disable this
option.
"""
return 'empty'
@property
def default_label(self):
"""
Human-readable description of the default assignment of this column, defaults to "Keep empty".
"""
return gettext_lazy('Keep empty')
def __init__(self, event):
self.event = event
def static_choices(self):
"""
This will be called when rendering the form component and allows you to return a list of values that can be
selected by the user statically during import.
:return: list of 2-tuples of strings
"""
return []
def resolve(self, settings, record):
"""
This method will be called to get the raw value for this field, usually by either using a static value or
inspecting the CSV file for the assigned header. You usually do not need to implement this on your own,
the default should be fine.
"""
k = settings.get(self.identifier, self.default_value)
if k == self.default_value:
return None
elif k.startswith('csv:'):
return record.get(k[4:], None) or None
elif k.startswith('static:'):
return k[7:]
raise ValidationError(_('Invalid setting for column "{header}".').format(header=self.verbose_name))
def clean(self, value, previous_values):
"""
Allows you to validate the raw input value for your column. Raise ``ValidationError`` if the value is invalid.
You do not need to include the column or row name or value in the error message as it will automatically be
included.
:param value: Contains the raw value of your column as returned by ``resolve``. This can usually be ``None``,
e.g. if the column is empty or does not exist in this row.
:param previous_values: Dictionary containing the validated values of all columns that have already been validated.
"""
return value
def assign(self, value, obj, **kwargs):
"""
This will be called to perform the actual import. You are supposed to set attributes on the ``obj`` or other
related objects that get passed in based on the input ``value``. This is called *before* the actual database
transaction, so the input objects do not yet have a primary key. If you want to create related objects, you
need to place them into some sort of internal queue and persist them when ``save`` is called.
"""
pass
def save(self, obj):
"""
This will be called to perform the actual import. This is called inside the actual database transaction and the
input object ``obj`` has already been saved to the database.
"""
pass
@property
def timezone(self):
return self.event.timezone
def i18n_flat(l):
if isinstance(l.data, dict):
return l.data.values()
return [l.data]
class BooleanColumnMixin:
default_value = None
initial = "static:false"
def static_choices(self):
return (
("false", _("No")),
("true", _("Yes")),
)
def clean(self, value, previous_values):
if not value:
return False
if value.lower() in ("true", "1", "yes", _("Yes").lower()):
return True
elif value.lower() in ("false", "0", "no", _("No").lower()):
return False
else:
raise ValidationError(_("Could not parse {value} as a yes/no value.").format(value=value))
class DatetimeColumnMixin:
def clean(self, value, previous_values):
if not value:
return
input_formats = formats.get_format('DATETIME_INPUT_FORMATS', use_l10n=True)
for format in input_formats:
try:
d = datetime.datetime.strptime(value, format)
d = d.replace(tzinfo=self.timezone)
return d
except (ValueError, TypeError):
pass
else:
raise ValidationError(_("Could not parse {value} as a date and time.").format(value=value))
class DecimalColumnMixin:
def clean(self, value, previous_values):
if value not in (None, ''):
value = formats.sanitize_separators(re.sub(r'[^0-9.,-]', '', value))
try:
value = Decimal(value)
except (DecimalException, TypeError):
raise ValidationError(_('You entered an invalid number.'))
return value
class IntegerColumnMixin:
def clean(self, value, previous_values):
if value is not None:
validate_integer(value)
return int(value)
class SubeventColumnMixin:
def __init__(self, *args, **kwargs):
self._subevent_cache = {}
super().__init__(*args, **kwargs)
@cached_property
def subevents(self):
return list(self.event.subevents.filter(active=True).order_by('date_from'))
def static_choices(self):
return [
(str(p.pk), str(p)) for p in self.subevents
]
def clean(self, value, previous_values):
if value in self._subevent_cache:
return self._subevent_cache[value]
input_formats = formats.get_format('DATETIME_INPUT_FORMATS', use_l10n=True)
for format in input_formats:
try:
d = datetime.datetime.strptime(value, format)
d = d.replace(tzinfo=self.event.timezone)
try:
se = self.event.subevents.get(
active=True,
date_from__gt=d - datetime.timedelta(seconds=1),
date_from__lt=d + datetime.timedelta(seconds=1),
)
self._subevent_cache[value] = se
return se
except SubEvent.DoesNotExist:
raise ValidationError(pgettext("subevent", "No matching date was found."))
except SubEvent.MultipleObjectsReturned:
raise ValidationError(pgettext("subevent", "Multiple matching dates were found."))
except (ValueError, TypeError):
continue
matches = [
p for p in self.subevents
if str(p.pk) == value or any(
(v and v == value) for v in i18n_flat(p.name)) or p.date_from.isoformat() == value
]
if len(matches) == 0:
raise ValidationError(pgettext("subevent", "No matching date was found."))
if len(matches) > 1:
raise ValidationError(pgettext("subevent", "Multiple matching dates were found."))
self._subevent_cache[value] = matches[0]
return matches[0]

View File

@@ -19,17 +19,13 @@
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
# <https://www.gnu.org/licenses/>.
#
import datetime
import re
from collections import defaultdict
from decimal import Decimal, DecimalException
import pycountry
from django.conf import settings
from django.core.exceptions import ValidationError
from django.core.validators import EmailValidator
from django.db.models import Q
from django.utils import formats
from django.utils.functional import cached_property
from django.utils.translation import (
gettext as _, gettext_lazy, pgettext, pgettext_lazy,
@@ -42,9 +38,13 @@ from phonenumbers import SUPPORTED_REGIONS
from pretix.base.channels import get_all_sales_channels
from pretix.base.forms.questions import guess_country
from pretix.base.modelimport import (
DatetimeColumnMixin, DecimalColumnMixin, ImportColumn, SubeventColumnMixin,
i18n_flat,
)
from pretix.base.models import (
Customer, ItemVariation, OrderPosition, Question, QuestionAnswer,
QuestionOption, Seat, SubEvent,
QuestionOption, Seat,
)
from pretix.base.services.pricing import get_price
from pretix.base.settings import (
@@ -53,99 +53,6 @@ from pretix.base.settings import (
from pretix.base.signals import order_import_columns
class ImportColumn:
@property
def identifier(self):
"""
Unique, internal name of the column.
"""
raise NotImplementedError
@property
def verbose_name(self):
"""
Human-readable description of the column
"""
raise NotImplementedError
@property
def initial(self):
"""
Initial value for the form component
"""
return None
@property
def default_value(self):
"""
Internal default value for the assignment of this column. Defaults to ``empty``. Return ``None`` to disable this
option.
"""
return 'empty'
@property
def default_label(self):
"""
Human-readable description of the default assignment of this column, defaults to "Keep empty".
"""
return gettext_lazy('Keep empty')
def __init__(self, event):
self.event = event
def static_choices(self):
"""
This will be called when rendering the form component and allows you to return a list of values that can be
selected by the user statically during import.
:return: list of 2-tuples of strings
"""
return []
def resolve(self, settings, record):
"""
This method will be called to get the raw value for this field, usually by either using a static value or
inspecting the CSV file for the assigned header. You usually do not need to implement this on your own,
the default should be fine.
"""
k = settings.get(self.identifier, self.default_value)
if k == self.default_value:
return None
elif k.startswith('csv:'):
return record.get(k[4:], None) or None
elif k.startswith('static:'):
return k[7:]
raise ValidationError(_('Invalid setting for column "{header}".').format(header=self.verbose_name))
def clean(self, value, previous_values):
"""
Allows you to validate the raw input value for your column. Raise ``ValidationError`` if the value is invalid.
You do not need to include the column or row name or value in the error message as it will automatically be
included.
:param value: Contains the raw value of your column as returned by ``resolve``. This can usually be ``None``,
e.g. if the column is empty or does not exist in this row.
:param previous_values: Dictionary containing the validated values of all columns that have already been validated.
"""
return value
def assign(self, value, order, position, invoice_address, **kwargs):
"""
This will be called to perform the actual import. You are supposed to set attributes on the ``order``, ``position``,
or ``invoice_address`` objects based on the input ``value``. This is called *before* the actual database
transaction, so these three objects do not yet have a primary key. If you want to create related objects, you
need to place them into some sort of internal queue and persist them when ``save`` is called.
"""
pass
def save(self, order):
"""
This will be called to perform the actual import. This is called inside the actual database transaction and the
input object ``order`` has already been saved to the database.
"""
pass
class EmailColumn(ImportColumn):
identifier = 'email'
verbose_name = gettext_lazy('E-mail address')
@@ -182,74 +89,20 @@ class PhoneColumn(ImportColumn):
order.phone = value
class SubeventColumn(ImportColumn):
class SubeventColumn(SubeventColumnMixin, ImportColumn):
identifier = 'subevent'
verbose_name = pgettext_lazy('subevents', 'Date')
default_value = None
def __init__(self, *args, **kwargs):
self._subevent_cache = {}
super().__init__(*args, **kwargs)
@cached_property
def subevents(self):
return list(self.event.subevents.filter(active=True).order_by('date_from'))
def static_choices(self):
return [
(str(p.pk), str(p)) for p in self.subevents
]
def clean(self, value, previous_values):
if not value:
raise ValidationError(pgettext("subevent", "You need to select a date."))
if value in self._subevent_cache:
return self._subevent_cache[value]
input_formats = formats.get_format('DATETIME_INPUT_FORMATS', use_l10n=True)
for format in input_formats:
try:
d = datetime.datetime.strptime(value, format)
d = d.replace(tzinfo=self.event.timezone)
try:
se = self.event.subevents.get(
active=True,
date_from__gt=d - datetime.timedelta(seconds=1),
date_from__lt=d + datetime.timedelta(seconds=1),
)
self._subevent_cache[value] = se
return se
except SubEvent.DoesNotExist:
raise ValidationError(pgettext("subevent", "No matching date was found."))
except SubEvent.MultipleObjectsReturned:
raise ValidationError(pgettext("subevent", "Multiple matching dates were found."))
except (ValueError, TypeError):
continue
matches = [
p for p in self.subevents
if str(p.pk) == value or any(
(v and v == value) for v in i18n_flat(p.name)) or p.date_from.isoformat() == value
]
if len(matches) == 0:
raise ValidationError(pgettext("subevent", "No matching date was found."))
if len(matches) > 1:
raise ValidationError(pgettext("subevent", "Multiple matching dates were found."))
self._subevent_cache[value] = matches[0]
return matches[0]
return super().clean(value, previous_values)
def assign(self, value, order, position, invoice_address, **kwargs):
position.subevent = value
def i18n_flat(l):
if isinstance(l.data, dict):
return l.data.values()
return [l.data]
class ItemColumn(ImportColumn):
identifier = 'item'
verbose_name = gettext_lazy('Product')
@@ -572,20 +425,11 @@ class AttendeeState(ImportColumn):
position.state = value or ''
class Price(ImportColumn):
class Price(DecimalColumnMixin, ImportColumn):
identifier = 'price'
verbose_name = gettext_lazy('Price')
default_label = gettext_lazy('Calculate from product')
def clean(self, value, previous_values):
if value not in (None, ''):
value = formats.sanitize_separators(re.sub(r'[^0-9.,-]', '', value))
try:
value = Decimal(value)
except (DecimalException, TypeError):
raise ValidationError(_('You entered an invalid number.'))
return value
def assign(self, value, order, position, invoice_address, **kwargs):
if value is None:
p = get_price(position.item, position.variation, position.voucher, subevent=position.subevent,
@@ -649,48 +493,18 @@ class Locale(ImportColumn):
order.locale = value
class ValidFrom(ImportColumn):
class ValidFrom(DatetimeColumnMixin, ImportColumn):
identifier = 'valid_from'
verbose_name = gettext_lazy('Valid from')
def clean(self, value, previous_values):
if not value:
return
input_formats = formats.get_format('DATETIME_INPUT_FORMATS', use_l10n=True)
for format in input_formats:
try:
d = datetime.datetime.strptime(value, format)
d = d.replace(tzinfo=self.event.timezone)
return d
except (ValueError, TypeError):
pass
else:
raise ValidationError(_("Could not parse {value} as a date and time.").format(value=value))
def assign(self, value, order, position, invoice_address, **kwargs):
position.valid_from = value
class ValidUntil(ImportColumn):
class ValidUntil(DatetimeColumnMixin, ImportColumn):
identifier = 'valid_until'
verbose_name = gettext_lazy('Valid until')
def clean(self, value, previous_values):
if not value:
return
input_formats = formats.get_format('DATETIME_INPUT_FORMATS', use_l10n=True)
for format in input_formats:
try:
d = datetime.datetime.strptime(value, format)
d = d.replace(tzinfo=self.event.timezone)
return d
except (ValueError, TypeError):
pass
else:
raise ValidationError(_("Could not parse {value} as a date and time.").format(value=value))
def assign(self, value, order, position, invoice_address, **kwargs):
position.valid_until = value
@@ -849,7 +663,7 @@ class CustomerColumn(ImportColumn):
order.customer = value
def get_all_columns(event):
def get_order_import_columns(event):
default = []
if event.has_subevents:
default.append(SubeventColumn(event))

View File

@@ -0,0 +1,378 @@
#
# This file is part of pretix (Community Edition).
#
# Copyright (C) 2014-2020 Raphael Michel and contributors
# Copyright (C) 2020-2021 rami.io GmbH and contributors
#
# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
# Public License as published by the Free Software Foundation in version 3 of the License.
#
# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
# this file, see <https://pretix.eu/about/en/license>.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
# <https://www.gnu.org/licenses/>.
#
from decimal import Decimal
from django.core.exceptions import ValidationError
from django.core.validators import MinLengthValidator
from django.utils.functional import cached_property
from django.utils.translation import gettext as _, gettext_lazy, pgettext_lazy
from pretix.base.modelimport import (
BooleanColumnMixin, DatetimeColumnMixin, DecimalColumnMixin, ImportColumn,
IntegerColumnMixin, i18n_flat,
)
from pretix.base.models import ItemVariation, Quota, Seat, Voucher
from pretix.base.signals import voucher_import_columns
class CodeColumn(ImportColumn):
identifier = 'code'
verbose_name = gettext_lazy('Voucher code')
default_value = None
def __init__(self, *args):
self._cached = set()
super().__init__(*args)
def clean(self, value, previous_values):
if value:
MinLengthValidator(5)(value)
if value and (value in self._cached or Voucher.objects.filter(event=self.event, code=value).exists()):
raise ValidationError(_('A voucher with this code already exists.'))
self._cached.add(value)
return value
def assign(self, value, obj: Voucher, **kwargs):
obj.code = value
class SubeventColumn(ImportColumn):
identifier = 'subevent'
verbose_name = pgettext_lazy('subevents', 'Date')
def assign(self, value, obj: Voucher, **kwargs):
obj.subevent = value
class MaxUsagesColumn(IntegerColumnMixin, ImportColumn):
identifier = 'max_usages'
verbose_name = gettext_lazy('Maximum usages')
initial = "static:1"
def static_choices(self):
return [
("1", "1")
]
def assign(self, value, obj: Voucher, **kwargs):
obj.max_usages = value if value is not None else 1
class MinUsagesColumn(IntegerColumnMixin, ImportColumn):
identifier = 'min_usages'
verbose_name = gettext_lazy('Minimum usages')
initial = "static:1"
def static_choices(self):
return [
("1", "1")
]
def assign(self, value, obj: Voucher, **kwargs):
obj.min_usages = value if value is not None else 1
class BudgetColumn(DecimalColumnMixin, ImportColumn):
identifier = 'budget'
verbose_name = gettext_lazy('Maximum discount budget')
def assign(self, value, obj: Voucher, **kwargs):
obj.budget = value
class ValidUntilColumn(DatetimeColumnMixin, ImportColumn):
identifier = 'valid_until'
verbose_name = gettext_lazy('Valid until')
def assign(self, value, obj: Voucher, **kwargs):
obj.valid_until = value
class BlockQuotaColumn(BooleanColumnMixin, ImportColumn):
identifier = 'block_quota'
verbose_name = gettext_lazy('Reserve ticket from quota')
def assign(self, value, obj: Voucher, **kwargs):
obj.block_quota = value
class AllowIgnoreQuotaColumn(BooleanColumnMixin, ImportColumn):
identifier = 'allow_ignore_quota'
verbose_name = gettext_lazy('Allow to bypass quota')
def assign(self, value, obj: Voucher, **kwargs):
obj.allow_ignore_quota = value
class PriceModeColumn(ImportColumn):
identifier = 'price_mode'
verbose_name = gettext_lazy('Price mode')
default_value = None
initial = 'static:none'
def static_choices(self):
return Voucher.PRICE_MODES
def clean(self, value, previous_values):
d = dict(Voucher.PRICE_MODES)
reverse = {v: k for k, v in Voucher.PRICE_MODES}
if value in d:
return value
elif value in reverse:
return reverse[value]
else:
raise ValidationError(_("Could not parse {value} as a price mode, use one of {options}.").format(
value=value, options=', '.join(d.keys())
))
def assign(self, value, voucher: Voucher, **kwargs):
voucher.price_mode = value
class ValueColumn(DecimalColumnMixin, ImportColumn):
identifier = 'value'
verbose_name = gettext_lazy('Voucher value')
def clean(self, value, previous_values):
value = super().clean(value, previous_values)
if value and previous_values.get("price_mode") == "none":
raise ValidationError(_("It is pointless to set a value without a price mode."))
return value
def assign(self, value, obj: Voucher, **kwargs):
obj.value = value or Decimal("0.00")
class ItemColumn(ImportColumn):
identifier = 'item'
verbose_name = gettext_lazy('Product')
@cached_property
def items(self):
return list(self.event.items.filter(active=True))
def static_choices(self):
return [
(str(p.pk), str(p)) for p in self.items
]
def clean(self, value, previous_values):
if not value:
return
matches = [
p for p in self.items
if str(p.pk) == value or (p.internal_name and p.internal_name == value) or any(
(v and v == value) for v in i18n_flat(p.name))
]
if len(matches) == 0:
raise ValidationError(_("No matching product was found."))
if len(matches) > 1:
raise ValidationError(_("Multiple matching products were found."))
return matches[0]
def assign(self, value, voucher, **kwargs):
voucher.item = value
class VariationColumn(ImportColumn):
identifier = 'variation'
verbose_name = gettext_lazy('Product variation')
@cached_property
def items(self):
return list(ItemVariation.objects.filter(
active=True, item__active=True, item__event=self.event
).select_related('item'))
def static_choices(self):
return [
(str(p.pk), '{} {}'.format(p.item, p.value)) for p in self.items
]
def clean(self, value, previous_values):
if value:
matches = [
p for p in self.items
if (str(p.pk) == value or any((v and v == value) for v in i18n_flat(p.value))) and p.item_id == previous_values['item'].pk
]
if len(matches) == 0:
raise ValidationError(_("No matching variation was found."))
if len(matches) > 1:
raise ValidationError(_("Multiple matching variations were found."))
return matches[0]
return value
def assign(self, value, voucher: Voucher, **kwargs):
voucher.variation = value
class QuotaColumn(ImportColumn):
identifier = 'quota'
verbose_name = gettext_lazy('Quota')
@cached_property
def quotas(self):
return list(Quota.objects.filter(
event=self.event
))
def static_choices(self):
return [
(str(q.pk), q.name) for q in self.quotas
]
def clean(self, value, previous_values):
if value:
if previous_values.get('item'):
raise ValidationError(_("You cannot specify a quota if you specified a product."))
matches = [
q for q in self.quotas
if str(q.pk) == value or any((v and v == value) for v in i18n_flat(q.name))
]
if len(matches) == 0:
raise ValidationError(_("No matching variation was found."))
if len(matches) > 1:
raise ValidationError(_("Multiple matching variations were found."))
return matches[0]
return value
def assign(self, value, voucher: Voucher, **kwargs):
voucher.quota = value
class SeatColumn(ImportColumn):
identifier = 'seat'
verbose_name = gettext_lazy('Seat ID')
def __init__(self, *args):
self._cached = set()
super().__init__(*args)
def clean(self, value, previous_values):
if value:
if self.event.has_subevents:
if not previous_values.get('subevent'):
raise ValidationError(_('You need to choose a date if you select a seat.'))
try:
value = Seat.objects.get(
event=self.event,
seat_guid=value,
subevent=previous_values.get('subevent')
)
except Seat.MultipleObjectsReturned:
raise ValidationError(_('Multiple matching seats were found.'))
except Seat.DoesNotExist:
raise ValidationError(_('No matching seat was found.'))
if not value.is_available() or value in self._cached:
raise ValidationError(
_('The seat you selected has already been taken. Please select a different seat.'))
if previous_values.get("quota"):
raise ValidationError(_('You need to choose a specific product if you select a seat.'))
if previous_values.get('max_usages', 1) > 1 or previous_values.get('min_usages', 1) > 1:
raise ValidationError(_('Seat-specific vouchers can only be used once.'))
if previous_values.get("item") and value.product != previous_values.get("item"):
raise ValidationError(
_('You need to choose the product "{prod}" for this seat.').format(prod=value.product)
)
self._cached.add(value)
return value
def assign(self, value, voucher: Voucher, **kwargs):
voucher.seat = value
class TagColumn(ImportColumn):
identifier = 'tag'
verbose_name = gettext_lazy('Tag')
def assign(self, value, voucher: Voucher, **kwargs):
voucher.tag = value or ''
class CommentColumn(ImportColumn):
identifier = 'comment'
verbose_name = gettext_lazy('Comment')
def assign(self, value, voucher: Voucher, **kwargs):
voucher.comment = value or ''
class ShowHiddenItemsColumn(BooleanColumnMixin, ImportColumn):
identifier = 'show_hidden_items'
verbose_name = gettext_lazy('Shows hidden products that match this voucher')
initial = "static:true"
def assign(self, value, obj: Voucher, **kwargs):
obj.show_hidden_items = value
class AllAddonsIncludedColumn(BooleanColumnMixin, ImportColumn):
identifier = 'all_addons_included'
verbose_name = gettext_lazy('Offer all add-on products for free when redeeming this voucher')
def assign(self, value, obj: Voucher, **kwargs):
obj.all_addons_included = value
class AllBundlesIncludedColumn(BooleanColumnMixin, ImportColumn):
identifier = 'all_bundles_included'
verbose_name = gettext_lazy('Include all bundled products without a designated price when redeeming this voucher')
def assign(self, value, obj: Voucher, **kwargs):
obj.all_bundles_included = value
def get_voucher_import_columns(event):
default = []
if event.has_subevents:
default.append(SubeventColumn(event))
default += [
CodeColumn(event),
MaxUsagesColumn(event),
MinUsagesColumn(event),
BudgetColumn(event),
ValidUntilColumn(event),
BlockQuotaColumn(event),
AllowIgnoreQuotaColumn(event),
PriceModeColumn(event),
ValueColumn(event),
ItemColumn(event),
VariationColumn(event),
QuotaColumn(event),
SeatColumn(event),
TagColumn(event),
CommentColumn(event),
ShowHiddenItemsColumn(event),
AllAddonsIncludedColumn(event),
AllBundlesIncludedColumn(event),
]
for recv, resp in voucher_import_columns.send(sender=event):
default += resp
return default

View File

@@ -517,9 +517,6 @@ class Voucher(LoggedModel):
if item and seat.product != item:
raise ValidationError(_('You need to choose the product "{prod}" for this seat.').format(prod=seat.product))
if not seat.is_available(ignore_voucher_id=pk):
raise ValidationError(_('The seat "{id}" is already sold or currently blocked.').format(id=seat.seat_guid))
return seat
def save(self, *args, **kwargs):

View File

@@ -19,9 +19,8 @@
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
# <https://www.gnu.org/licenses/>.
#
import csv
import io
from decimal import Decimal
from typing import List
from django.conf import settings as django_settings
from django.core.exceptions import ValidationError
@@ -29,13 +28,15 @@ from django.db import transaction
from django.utils.timezone import now
from django.utils.translation import gettext as _
from pretix.base.i18n import LazyLocaleException, language
from pretix.base.i18n import language
from pretix.base.modelimport import DataImportError, ImportColumn, parse_csv
from pretix.base.modelimport_orders import get_order_import_columns
from pretix.base.modelimport_vouchers import get_voucher_import_columns
from pretix.base.models import (
CachedFile, Event, InvoiceAddress, Order, OrderPayment, OrderPosition,
User,
User, Voucher,
)
from pretix.base.models.orders import Transaction
from pretix.base.orderimport import get_all_columns
from pretix.base.services.invoices import generate_invoice, invoice_qualified
from pretix.base.services.locking import lock_objects
from pretix.base.services.tasks import ProfiledEventTask
@@ -43,47 +44,36 @@ from pretix.base.signals import order_paid, order_placed
from pretix.celery_app import app
class DataImportError(LazyLocaleException):
def __init__(self, *args):
msg = args[0]
msgargs = args[1] if len(args) > 1 else None
self.args = args
if msgargs:
msg = _(msg) % msgargs
else:
msg = _(msg)
super().__init__(msg)
def parse_csv(file, length=None, mode="strict", charset=None):
file.seek(0)
data = file.read(length)
if not charset:
try:
import chardet
charset = chardet.detect(data)['encoding']
except ImportError:
charset = file.charset
data = data.decode(charset or "utf-8", mode)
# If the file was modified on a Mac, it only contains \r as line breaks
if '\r' in data and '\n' not in data:
data = data.replace('\r', '\n')
def _validate(cf: CachedFile, charset: str, cols: List[ImportColumn], settings: dict):
try:
dialect = csv.Sniffer().sniff(data.split("\n")[0], delimiters=";,.#:")
except csv.Error:
return None
if dialect is None:
return None
reader = csv.DictReader(io.StringIO(data), dialect=dialect)
return reader
def setif(record, obj, attr, setting):
if setting.startswith('csv:'):
setattr(obj, attr, record[setting[4:]] or '')
parsed = parse_csv(cf.file, charset=charset)
except UnicodeDecodeError as e:
raise DataImportError(
_(
'Error decoding special characters in your file: {message}').format(
message=str(e)
)
)
data = []
for i, record in enumerate(parsed):
if not any(record.values()):
continue
values = {}
for c in cols:
val = c.resolve(settings, record)
if isinstance(val, str):
val = val.strip()
try:
values[c.identifier] = c.clean(val, values)
except ValidationError as e:
raise DataImportError(
_(
'Error while importing value "{value}" for column "{column}" in line "{line}": {message}').format(
value=val if val is not None else '', column=c.verbose_name, line=i + 1, message=e.message
)
)
data.append(values)
return data
@app.task(base=ProfiledEventTask, throws=(DataImportError,))
@@ -91,45 +81,17 @@ def import_orders(event: Event, fileid: str, settings: dict, locale: str, user,
cf = CachedFile.objects.get(id=fileid)
user = User.objects.get(pk=user)
with language(locale, event.settings.region):
cols = get_all_columns(event)
try:
parsed = parse_csv(cf.file, charset=charset)
except UnicodeDecodeError as e:
raise DataImportError(
_(
'Error decoding special characters in your file: {message}').format(
message=str(e)
)
)
orders = []
order = None
data = []
# Run validation
for i, record in enumerate(parsed):
if not any(record.values()):
continue
values = {}
for c in cols:
val = c.resolve(settings, record)
if isinstance(val, str):
val = val.strip()
try:
values[c.identifier] = c.clean(val, values)
except ValidationError as e:
raise DataImportError(
_(
'Error while importing value "{value}" for column "{column}" in line "{line}": {message}').format(
value=val if val is not None else '', column=c.verbose_name, line=i + 1, message=e.message
)
)
data.append(values)
cols = get_order_import_columns(event)
data = _validate(cf, charset, cols, settings)
if settings['orders'] == 'one' and len(data) > django_settings.PRETIX_MAX_ORDER_SIZE:
raise DataImportError(
_('Orders cannot have more than %(max)s positions.') % {'max': django_settings.PRETIX_MAX_ORDER_SIZE}
)
orders = []
order = None
# Prepare model objects. Yes, this might consume lots of RAM, but allows us to make the actual SQL transaction
# shorter. We'll see what works better in reality…
lock_seats = []
@@ -149,16 +111,16 @@ def import_orders(event: Event, fileid: str, settings: dict, locale: str, user,
position = OrderPosition(positionid=len(order._positions) + 1)
position.attendee_name_parts = {'_scheme': event.settings.name_scheme}
position.meta_info = {}
if position.seat is not None:
lock_seats.append(position.seat)
order._positions.append(position)
position.assign_pseudonymization_id()
for c in cols:
c.assign(record.get(c.identifier), order, position, order._address)
except ImportError as e:
raise ImportError(
if position.seat is not None:
lock_seats.append(position.seat)
except (ValidationError, ImportError) as e:
raise DataImportError(
_('Invalid data in row {row}: {message}').format(row=i, message=str(e))
)
@@ -169,7 +131,7 @@ def import_orders(event: Event, fileid: str, settings: dict, locale: str, user,
lock_objects(lock_seats, shared_lock_objects=[event])
for s in lock_seats:
if not s.is_available():
raise ImportError(_('The seat you selected has already been taken. Please select a different seat.'))
raise DataImportError(_('The seat you selected has already been taken. Please select a different seat.'))
save_transactions = []
for o in orders:
@@ -232,3 +194,62 @@ def import_orders(event: Event, fileid: str, settings: dict, locale: str, user,
raise ValidationError(_('We were not able to process your request completely as the server was too busy. '
'Please try again.'))
cf.delete()
@app.task(base=ProfiledEventTask, throws=(DataImportError,))
def import_vouchers(event: Event, fileid: str, settings: dict, locale: str, user, charset=None) -> None:
cf = CachedFile.objects.get(id=fileid)
user = User.objects.get(pk=user)
with language(locale, event.settings.region):
cols = get_voucher_import_columns(event)
data = _validate(cf, charset, cols, settings)
# Prepare model objects. Yes, this might consume lots of RAM, but allows us to make the actual SQL transaction
# shorter. We'll see what works better in reality…
vouchers = []
lock_seats = []
for i, record in enumerate(data):
try:
voucher = Voucher(event=event)
vouchers.append(voucher)
Voucher.clean_item_properties(
record,
event,
record.get('quota'),
record.get('item'),
record.get('variation'),
block_quota=record.get('block_quota')
)
Voucher.clean_subevent(record, event)
Voucher.clean_max_usages(record, 0)
for c in cols:
c.assign(record.get(c.identifier), voucher)
if voucher.seat is not None:
lock_seats.append(voucher.seat)
except (ValidationError, ImportError) as e:
raise DataImportError(
_('Invalid data in row {row}: {message}').format(row=i, message=str(e))
)
with transaction.atomic():
# We don't support quotas here, so we only need to lock if seats are in use
if lock_seats:
lock_objects(lock_seats, shared_lock_objects=[event])
for s in lock_seats:
if not s.is_available():
raise DataImportError(
_('The seat you selected has already been taken. Please select a different seat.'))
for v in vouchers:
v.save()
v.log_action(
'pretix.voucher.added',
user=user,
data={'source': 'import'}
)
for c in cols:
c.save(v)
cf.delete()

View File

@@ -785,6 +785,15 @@ to define additional columns that can be read during import. You are expected to
As with all event-plugin signals, the ``sender`` keyword argument will contain the event.
"""
voucher_import_columns = EventPluginSignal()
"""
This signal is sent out if the user performs an import of vouchers from an external source. You can use this
to define additional columns that can be read during import. You are expected to return a list of instances of
``ImportColumn`` subclasses.
As with all event-plugin signals, the ``sender`` keyword argument will contain the event.
"""
validate_event_settings = EventPluginSignal()
"""
Arguments: ``settings_dict``