forked from CGM_Public/pretix_original
Generalize import process from orders to more models (#4002)
* Generalize import process from orders to more models * Add voucher import * Model import: Guess assignments of based on column headers * Fix lock_seats being pointless * Update docs * Update doc/development/api/import.rst Co-authored-by: Richard Schreiber <schreiber@rami.io> * Update src/pretix/base/modelimport_vouchers.py Co-authored-by: Richard Schreiber <schreiber@rami.io> --------- Co-authored-by: Richard Schreiber <schreiber@rami.io>
This commit is contained in:
Raphael Michel
@@ -19,9 +19,8 @@
|
||||
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
|
||||
# <https://www.gnu.org/licenses/>.
|
||||
#
|
||||
import csv
|
||||
import io
|
||||
from decimal import Decimal
|
||||
from typing import List
|
||||
|
||||
from django.conf import settings as django_settings
|
||||
from django.core.exceptions import ValidationError
|
||||
@@ -29,13 +28,15 @@ from django.db import transaction
|
||||
from django.utils.timezone import now
|
||||
from django.utils.translation import gettext as _
|
||||
|
||||
from pretix.base.i18n import LazyLocaleException, language
|
||||
from pretix.base.i18n import language
|
||||
from pretix.base.modelimport import DataImportError, ImportColumn, parse_csv
|
||||
from pretix.base.modelimport_orders import get_order_import_columns
|
||||
from pretix.base.modelimport_vouchers import get_voucher_import_columns
|
||||
from pretix.base.models import (
|
||||
CachedFile, Event, InvoiceAddress, Order, OrderPayment, OrderPosition,
|
||||
User,
|
||||
User, Voucher,
|
||||
)
|
||||
from pretix.base.models.orders import Transaction
|
||||
from pretix.base.orderimport import get_all_columns
|
||||
from pretix.base.services.invoices import generate_invoice, invoice_qualified
|
||||
from pretix.base.services.locking import lock_objects
|
||||
from pretix.base.services.tasks import ProfiledEventTask
|
||||
@@ -43,47 +44,36 @@ from pretix.base.signals import order_paid, order_placed
|
||||
from pretix.celery_app import app
|
||||
|
||||
|
||||
class DataImportError(LazyLocaleException):
|
||||
def __init__(self, *args):
|
||||
msg = args[0]
|
||||
msgargs = args[1] if len(args) > 1 else None
|
||||
self.args = args
|
||||
if msgargs:
|
||||
msg = _(msg) % msgargs
|
||||
else:
|
||||
msg = _(msg)
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
def parse_csv(file, length=None, mode="strict", charset=None):
|
||||
file.seek(0)
|
||||
data = file.read(length)
|
||||
if not charset:
|
||||
try:
|
||||
import chardet
|
||||
charset = chardet.detect(data)['encoding']
|
||||
except ImportError:
|
||||
charset = file.charset
|
||||
data = data.decode(charset or "utf-8", mode)
|
||||
# If the file was modified on a Mac, it only contains \r as line breaks
|
||||
if '\r' in data and '\n' not in data:
|
||||
data = data.replace('\r', '\n')
|
||||
|
||||
def _validate(cf: CachedFile, charset: str, cols: List[ImportColumn], settings: dict):
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(data.split("\n")[0], delimiters=";,.#:")
|
||||
except csv.Error:
|
||||
return None
|
||||
|
||||
if dialect is None:
|
||||
return None
|
||||
|
||||
reader = csv.DictReader(io.StringIO(data), dialect=dialect)
|
||||
return reader
|
||||
|
||||
|
||||
def setif(record, obj, attr, setting):
|
||||
if setting.startswith('csv:'):
|
||||
setattr(obj, attr, record[setting[4:]] or '')
|
||||
parsed = parse_csv(cf.file, charset=charset)
|
||||
except UnicodeDecodeError as e:
|
||||
raise DataImportError(
|
||||
_(
|
||||
'Error decoding special characters in your file: {message}').format(
|
||||
message=str(e)
|
||||
)
|
||||
)
|
||||
data = []
|
||||
for i, record in enumerate(parsed):
|
||||
if not any(record.values()):
|
||||
continue
|
||||
values = {}
|
||||
for c in cols:
|
||||
val = c.resolve(settings, record)
|
||||
if isinstance(val, str):
|
||||
val = val.strip()
|
||||
try:
|
||||
values[c.identifier] = c.clean(val, values)
|
||||
except ValidationError as e:
|
||||
raise DataImportError(
|
||||
_(
|
||||
'Error while importing value "{value}" for column "{column}" in line "{line}": {message}').format(
|
||||
value=val if val is not None else '', column=c.verbose_name, line=i + 1, message=e.message
|
||||
)
|
||||
)
|
||||
data.append(values)
|
||||
return data
|
||||
|
||||
|
||||
@app.task(base=ProfiledEventTask, throws=(DataImportError,))
|
||||
@@ -91,45 +81,17 @@ def import_orders(event: Event, fileid: str, settings: dict, locale: str, user,
|
||||
cf = CachedFile.objects.get(id=fileid)
|
||||
user = User.objects.get(pk=user)
|
||||
with language(locale, event.settings.region):
|
||||
cols = get_all_columns(event)
|
||||
try:
|
||||
parsed = parse_csv(cf.file, charset=charset)
|
||||
except UnicodeDecodeError as e:
|
||||
raise DataImportError(
|
||||
_(
|
||||
'Error decoding special characters in your file: {message}').format(
|
||||
message=str(e)
|
||||
)
|
||||
)
|
||||
orders = []
|
||||
order = None
|
||||
data = []
|
||||
|
||||
# Run validation
|
||||
for i, record in enumerate(parsed):
|
||||
if not any(record.values()):
|
||||
continue
|
||||
values = {}
|
||||
for c in cols:
|
||||
val = c.resolve(settings, record)
|
||||
if isinstance(val, str):
|
||||
val = val.strip()
|
||||
try:
|
||||
values[c.identifier] = c.clean(val, values)
|
||||
except ValidationError as e:
|
||||
raise DataImportError(
|
||||
_(
|
||||
'Error while importing value "{value}" for column "{column}" in line "{line}": {message}').format(
|
||||
value=val if val is not None else '', column=c.verbose_name, line=i + 1, message=e.message
|
||||
)
|
||||
)
|
||||
data.append(values)
|
||||
cols = get_order_import_columns(event)
|
||||
data = _validate(cf, charset, cols, settings)
|
||||
|
||||
if settings['orders'] == 'one' and len(data) > django_settings.PRETIX_MAX_ORDER_SIZE:
|
||||
raise DataImportError(
|
||||
_('Orders cannot have more than %(max)s positions.') % {'max': django_settings.PRETIX_MAX_ORDER_SIZE}
|
||||
)
|
||||
|
||||
orders = []
|
||||
order = None
|
||||
|
||||
# Prepare model objects. Yes, this might consume lots of RAM, but allows us to make the actual SQL transaction
|
||||
# shorter. We'll see what works better in reality…
|
||||
lock_seats = []
|
||||
@@ -149,16 +111,16 @@ def import_orders(event: Event, fileid: str, settings: dict, locale: str, user,
|
||||
position = OrderPosition(positionid=len(order._positions) + 1)
|
||||
position.attendee_name_parts = {'_scheme': event.settings.name_scheme}
|
||||
position.meta_info = {}
|
||||
if position.seat is not None:
|
||||
lock_seats.append(position.seat)
|
||||
order._positions.append(position)
|
||||
position.assign_pseudonymization_id()
|
||||
|
||||
for c in cols:
|
||||
c.assign(record.get(c.identifier), order, position, order._address)
|
||||
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
if position.seat is not None:
|
||||
lock_seats.append(position.seat)
|
||||
except (ValidationError, ImportError) as e:
|
||||
raise DataImportError(
|
||||
_('Invalid data in row {row}: {message}').format(row=i, message=str(e))
|
||||
)
|
||||
|
||||
@@ -169,7 +131,7 @@ def import_orders(event: Event, fileid: str, settings: dict, locale: str, user,
|
||||
lock_objects(lock_seats, shared_lock_objects=[event])
|
||||
for s in lock_seats:
|
||||
if not s.is_available():
|
||||
raise ImportError(_('The seat you selected has already been taken. Please select a different seat.'))
|
||||
raise DataImportError(_('The seat you selected has already been taken. Please select a different seat.'))
|
||||
|
||||
save_transactions = []
|
||||
for o in orders:
|
||||
@@ -232,3 +194,62 @@ def import_orders(event: Event, fileid: str, settings: dict, locale: str, user,
|
||||
raise ValidationError(_('We were not able to process your request completely as the server was too busy. '
|
||||
'Please try again.'))
|
||||
cf.delete()
|
||||
|
||||
|
||||
@app.task(base=ProfiledEventTask, throws=(DataImportError,))
|
||||
def import_vouchers(event: Event, fileid: str, settings: dict, locale: str, user, charset=None) -> None:
|
||||
cf = CachedFile.objects.get(id=fileid)
|
||||
user = User.objects.get(pk=user)
|
||||
with language(locale, event.settings.region):
|
||||
cols = get_voucher_import_columns(event)
|
||||
data = _validate(cf, charset, cols, settings)
|
||||
|
||||
# Prepare model objects. Yes, this might consume lots of RAM, but allows us to make the actual SQL transaction
|
||||
# shorter. We'll see what works better in reality…
|
||||
vouchers = []
|
||||
lock_seats = []
|
||||
for i, record in enumerate(data):
|
||||
try:
|
||||
voucher = Voucher(event=event)
|
||||
vouchers.append(voucher)
|
||||
|
||||
Voucher.clean_item_properties(
|
||||
record,
|
||||
event,
|
||||
record.get('quota'),
|
||||
record.get('item'),
|
||||
record.get('variation'),
|
||||
block_quota=record.get('block_quota')
|
||||
)
|
||||
Voucher.clean_subevent(record, event)
|
||||
Voucher.clean_max_usages(record, 0)
|
||||
|
||||
for c in cols:
|
||||
c.assign(record.get(c.identifier), voucher)
|
||||
|
||||
if voucher.seat is not None:
|
||||
lock_seats.append(voucher.seat)
|
||||
except (ValidationError, ImportError) as e:
|
||||
raise DataImportError(
|
||||
_('Invalid data in row {row}: {message}').format(row=i, message=str(e))
|
||||
)
|
||||
|
||||
with transaction.atomic():
|
||||
# We don't support quotas here, so we only need to lock if seats are in use
|
||||
if lock_seats:
|
||||
lock_objects(lock_seats, shared_lock_objects=[event])
|
||||
for s in lock_seats:
|
||||
if not s.is_available():
|
||||
raise DataImportError(
|
||||
_('The seat you selected has already been taken. Please select a different seat.'))
|
||||
|
||||
for v in vouchers:
|
||||
v.save()
|
||||
v.log_action(
|
||||
'pretix.voucher.added',
|
||||
user=user,
|
||||
data={'source': 'import'}
|
||||
)
|
||||
for c in cols:
|
||||
c.save(v)
|
||||
cf.delete()
|
||||
Reference in New Issue
Block a user