diff --git a/src/pretix/base/datasync/forms.py b/src/pretix/base/datasync/__init__.py similarity index 100% rename from src/pretix/base/datasync/forms.py rename to src/pretix/base/datasync/__init__.py diff --git a/src/pretix/base/datasync/datasync.py b/src/pretix/base/datasync/datasync.py index d3cd724105..05b9d641cd 100644 --- a/src/pretix/base/datasync/datasync.py +++ b/src/pretix/base/datasync/datasync.py @@ -1,5 +1,7 @@ import logging +from collections import namedtuple from datetime import datetime, timedelta +from functools import cached_property from itertools import groupby import sentry_sdk @@ -8,41 +10,65 @@ from django.db.models import Q from django.dispatch import receiver from django_scopes import scopes_disabled, scope +from pretix.base.datasync.sourcefields import get_data_fields, ORDER, EVENT, EVENT_OR_SUBEVENT, ORDER_POSITION from pretix.base.models import Order, Event from django.utils.translation import gettext_lazy as _ from pretix.base.services.tasks import TransactionAwareTask from pretix.base.signals import periodic_task, EventPluginRegistry from pretix.celery_app import app +import json logger = logging.getLogger(__name__) +MODE_OVERWRITE = "overwrite" +MODE_SET_IF_NEW = "if_new" +MODE_SET_IF_EMPTY = "if_empty" +MODE_APPEND_LIST = "append" + class OrderSyncQueue(models.Model): class Meta: - unique_together = (("order", "sync_target"),) + unique_together = (("order", "sync_provider"),) order = models.ForeignKey( - Order, on_delete=models.CASCADE, related_name="order_sync_jobs" + Order, on_delete=models.CASCADE, related_name="queued_sync_jobs" ) - sync_target = models.CharField(blank=False, null=False, max_length=128) + sync_provider = models.CharField(blank=False, null=False, max_length=128) triggered_by = models.CharField(blank=False, null=False, max_length=128) triggered = models.DateTimeField(blank=False, null=False, auto_now_add=True) failed_attempts = models.PositiveIntegerField(default=0) not_before = models.DateTimeField(blank=True, null=True) + @cached_property + def _provider_class_info(self): + return sync_targets.get(identifier=self.sync_provider) + + @property + def provider_class(self): + return self._provider_class_info[0] + + @property + def is_provider_active(self): + return self._provider_class_info[1] + + @property + def max_retry_attempts(self): + return self.provider_class.max_attempts + @receiver(periodic_task, dispatch_uid="data_sync_periodic") def on_periodic_task(sender, **kwargs): sync_all.apply_async() -sync_targets = EventPluginRegistry({"name": lambda o: o.__name__}) +sync_targets = EventPluginRegistry({"identifier": lambda o: o.identifier}) def sync_event_to_target(event, target_cls, queued_orders): with scope(organizer=event.organizer): with target_cls(event=event) as p: + # TODO: should I somehow lock the queued orders or events, to avoid syncing them twice at the same time? p.sync_queued_orders(queued_orders) @@ -50,10 +76,15 @@ def sync_event_to_target(event, target_cls, queued_orders): @app.task() def sync_all(): with scopes_disabled(): - queue = OrderSyncQueue.objects.filter(Q(not_before__isnull=True) | Q(not_before__lt=datetime.now()))[:1000] + queue = ( + OrderSyncQueue.objects + .select_related("order") + .prefetch_related("order__event") + .filter(Q(not_before__isnull=True) | Q(not_before__lt=datetime.now()))[:1000] + ) grouped = groupby(sorted(queue, key=lambda q: (q.sync_target, q.order.event)), lambda q: (q.sync_target, q.order.event)) for (target, event), queued_orders in grouped: - target_cls = sync_targets.get(name=target) + target_cls = sync_targets.get(identifier=target) sync_event_to_target(event, target_cls, queued_orders) @@ -63,8 +94,13 @@ class SyncConfigError(Exception): self.full_message = full_message -class SyncProvider: +StaticMapping = namedtuple('StaticMapping', ('pk', 'pretix_model', 'external_object_type', 'pretix_pk', 'external_pk', 'property_mapping')) + + +class OutboundSyncProvider: + #identifier = None max_attempts = 5 + syncer_class = None def __init__(self, event): self.event = event @@ -77,7 +113,23 @@ class SyncProvider: self.do_after_event() self.do_finally() - def sync_order(self, order): + @classmethod + @property + def display_name(cls): + return str(cls.identifier) + + @classmethod + def enqueue_order(cls, order, triggered_by, not_before=None): + OrderSyncQueue.objects.create( + order=order, + sync_provider=cls.identifier, + triggered_by=triggered_by, + not_before=not_before) + + def do_after_event(self): + pass + + def do_finally(self): pass def next_retry_date(self, sq): @@ -93,7 +145,7 @@ class SyncProvider: exc_info=True, ) sq.order.log_action( - "pretix.order_sync_failed", + "pretix.event.order.data_sync.failed", { "error": e.messages, "full_message": e.full_message, @@ -101,17 +153,19 @@ class SyncProvider: ) sq.delete() except Exception as e: - sentry_sdk.capture_exception(e) + # TODO: different handling per Exception, or even per HTTP response code? + # otherwise, SyncProviders should always throw SyncConfigError in non-recoverable situations sq.failed_attempts += 1 sq.not_before = self.next_retry_date(sq) logger.exception( f"Could not sync order {sq.order.code} to {self.__name__} (transient error, attempt #{sq.failed_attempts})" ) if sq.failed_attempts >= self.max_attempts: + sentry_sdk.capture_exception(e) sq.order.log_action( - "pretix.order_sync_failed", + "pretix.event.order.data_sync.failed", { - "error": [_("Marking as failed after {} retries").format(sq.failed_attempts)], + "error": [_("Maximum number of retries exceeded.")], "full_message": str(e), }, ) @@ -121,3 +175,90 @@ class SyncProvider: else: sq.delete() + def order_valid_for_sync(self, order): + return True + + @property + def mappings(self): + raise NotImplemented + + @cached_property + def data_fields(self): + return { + key: (from_model, label, ptype, enum_opts, getter) + for (from_model, key, label, ptype, enum_opts, getter) in get_data_fields(self.event) + } + + def get_field_value(self, inputs, mapping_entry): + key = mapping_entry["pretix_field"] + required_input, label, ptype, enum_opts, getter = self.data_fields.get(key) + input = inputs[required_input] + val = getter(input) + if isinstance(val, list): + if enum_opts and mapping_entry.get("value_map"): + map = json.loads(mapping_entry["value_map"]) + try: + val = [map[el] for el in val] + except KeyError: + raise SyncConfigError([f'Please update value mapping for field "{key}" - option "{val}" not assigned']) + + val = ",".join(val) + return val + + def get_properties(self, inputs: dict, property_mapping: str): + property_mapping = json.loads(property_mapping) + return [ + (m["external_field"], self.get_field_value(inputs, m), m["overwrite"]) + for m in property_mapping + ] + + def sync_object( + self, + inputs: dict, + mapping, + mapped_objects: dict, + ): + logger.debug("Syncing object %r, %r, %r", inputs, mapping, mapped_objects) + properties = self.get_properties(inputs, mapping.property_mapping) + logger.debug("Properties: %r", properties) + + pk_value = self.get_field_value(inputs, {"pretix_field": mapping.pretix_pk}) + if not pk_value: + return None + + return self.sync_object_with_properties(inputs, mapping, mapped_objects, pk_value, properties) + + def sync_order(self, order): + if not self.order_valid_for_sync(order): + logger.debug("Skipping order (not valid for sync)", order) + return + + logger.debug("Syncing order", order) + positions = list( + order.all_positions.filter(item__admission=True) + .prefetch_related("answers", "answers__question") + .select_related( + "voucher", + ) + ) + order_inputs = {ORDER: order, EVENT: self.event} + mapped_objects = {} + for mapping in self.mappings: + if mapping.pretix_model == 'Order': + mapped_objects[mapping.pk] = [ + self.sync_object(order_inputs, mapping, mapped_objects) + ] + elif mapping.pretix_model == 'OrderPosition': + mapped_objects[mapping.pk] = [ + self.sync_object({ + **order_inputs, EVENT_OR_SUBEVENT: op.subevent or self.event, ORDER_POSITION: op + }, mapping, mapped_objects) + for op in positions + ] + else: + raise SyncConfigError("Invalid pretix model '{}'".format(mapping.pretix_model)) + order.log_action( + "pretix.event.order.data_sync.success", {"objects": mapped_objects} + ) + + diff --git a/src/pretix/base/datasync/mapping.py b/src/pretix/base/datasync/mapping.py deleted file mode 100644 index 2091bcef19..0000000000 --- a/src/pretix/base/datasync/mapping.py +++ /dev/null @@ -1,50 +0,0 @@ -import json -from django import forms -from django.forms import ModelForm -from django.utils.translation import gettext_lazy as _ -from pretix.base.forms import SettingsForm - - -MODE_OVERWRITE = "overwrite" -MODE_SET_IF_NEW = "if_new" -MODE_SET_IF_EMPTY = "if_empty" -MODE_APPEND_LIST = "append" - - -class PropertyMappingForm(forms.Form): - pretix_field = forms.CharField() - external_field = forms.ChoiceField( - widget=forms.Select( - attrs={ - "data-model-select2": "json_script", - "data-select2-src": "#contact-props", - } - ) - ) - value_map = forms.CharField(required=False) - overwrite = forms.ChoiceField( - choices=[ - (MODE_OVERWRITE, _("Overwrite")), - (MODE_SET_IF_NEW, _("Fill if new contact")), - (MODE_SET_IF_EMPTY, _("Fill if empty")), - (MODE_APPEND_LIST, _("Add to list")), - ] - ) - - def __init__(self, pretix_fields, *args, **kwargs): - super().__init__(*args, **kwargs) - self.fields["pretix_field"] = forms.ChoiceField( - label=_("pretix Field"), - choices=pretix_fields_choices(pretix_fields), - required=False, - ) - self.fields["external_field"].choices = [ - (self["external_field"].value(), self["external_field"].value()) - ] - - -def pretix_fields_choices(pretix_fields): - return [ - (key, label + " [" + ptype.value + "]") - for (required_input, key, label, ptype, enum_opts, getter) in pretix_fields - ] diff --git a/src/pretix/base/datasync/sourcefields.py b/src/pretix/base/datasync/sourcefields.py index c02b600674..6c621f9400 100644 --- a/src/pretix/base/datasync/sourcefields.py +++ b/src/pretix/base/datasync/sourcefields.py @@ -74,7 +74,7 @@ AVAILABLE_MODELS = { } -def get_data_fields(event): +def get_data_fields(event, for_model=None): """ Returns tuple of (required_input, key, label, type, enum_opts, getter) @@ -376,6 +376,14 @@ def get_data_fields(event): None, get_payment_date, ), + ( + ORDER, + "order_locale", + _("Order locale country code"), + Question.TYPE_COUNTRYCODE, + None, + lambda order: order.locale.split("_")[0], + ), ] + [ ( @@ -400,7 +408,15 @@ def get_data_fields(event): for q in event.questions.all().prefetch_related("options") ] ) - return src_fields + if for_model: + available_inputs = AVAILABLE_MODELS[for_model] + return [ + (required_input, key, label, qtype, enum_opts, getter) + for required_input, key, label, qtype, enum_opts, getter in src_fields + if required_input in available_inputs + ] + else: + return src_fields def translate_property_mappings(property_mapping, checkin_list_map): @@ -418,3 +434,18 @@ def get_enum_opts(q): return [(opt.identifier, opt.answer) for opt in q.options.all()] else: return None + +QUESTION_TYPE_IDENTIFIERS = { + Question.TYPE_NUMBER: "NUMBER", + Question.TYPE_STRING: "STRING", + Question.TYPE_TEXT: "TEXT", + Question.TYPE_BOOLEAN: "BOOLEAN", + Question.TYPE_CHOICE: "CHOICE", + Question.TYPE_CHOICE_MULTIPLE: "CHOICE_MULTIPLE", + Question.TYPE_FILE: "FILE", + Question.TYPE_DATE: "DATE", + Question.TYPE_TIME: "TIME", + Question.TYPE_DATETIME: "DATETIME", + Question.TYPE_COUNTRYCODE: "COUNTRYCODE", + Question.TYPE_PHONENUMBER: "PHONENUMBER", +} diff --git a/src/pretix/base/signals.py b/src/pretix/base/signals.py index 7b996cffec..e20f93af7e 100644 --- a/src/pretix/base/signals.py +++ b/src/pretix/base/signals.py @@ -333,6 +333,23 @@ class EventPluginRegistry(Registry): def __init__(self, keys): super().__init__({"plugin": lambda o: get_defining_app(o), **keys}) + def filter(self, active_in=None, **kwargs): + result = super().filter(**kwargs) + if active_in is not None: + result = ( + (entry, meta) + for entry, meta in result + if is_app_active(active_in, meta['plugin']) + ) + return result + + def get(self, active_in=None, **kwargs): + item, meta = super().get(**kwargs) + if meta and active_in is not None: + if not is_app_active(active_in, meta['plugin']): + return None, None + return item, meta + event_live_issues = EventPluginSignal() """ diff --git a/src/pretix/control/apps.py b/src/pretix/control/apps.py index 1b2148618f..2d522c7462 100644 --- a/src/pretix/control/apps.py +++ b/src/pretix/control/apps.py @@ -42,3 +42,4 @@ class PretixControlConfig(AppConfig): def ready(self): from .views import dashboards # noqa from . import logdisplay # noqa + from . import datasync # noqa diff --git a/src/pretix/control/datasync.py b/src/pretix/control/datasync.py new file mode 100644 index 0000000000..537a81e9fe --- /dev/null +++ b/src/pretix/control/datasync.py @@ -0,0 +1,60 @@ +from django.contrib import messages +from django.dispatch import receiver +from django.http import HttpResponseNotAllowed +from django.shortcuts import redirect +from django.template.loader import get_template + +from pretix.base.datasync.datasync import sync_targets +from pretix.base.models import Event, Order +from pretix.control.signals import order_info +from pretix.control.views.orders import OrderView +from django.utils.translation import gettext_lazy as _ + +@receiver(order_info, dispatch_uid="datasync_control_order_info") +def on_control_order_info(sender: Event, request, order: Order, **kwargs): + providers = [provider for provider, meta in sync_targets.filter(active_in=sender)] + if not providers: return "" + + queued = order.queued_sync_jobs.all() + queued_provider_ids = {p.sync_provider for p in queued} + non_pending = [(provider.identifier, provider.display_name) for provider in providers if provider.identifier not in queued_provider_ids] + + #sync_logs = order.all_logentries().filter(action_type__in=( + # "pretix.event.order.data_sync.success", + # "pretix.event.order.data_sync.failed" + #)) + + template = get_template("pretixcontrol/datasync/control_order_info.html") + ctx = { + "order": order, + "request": request, + "event": sender, + "non_pending_providers": non_pending, + "queued_sync_jobs": queued, + } + return template.render(ctx, request=request) + + +class ControlSyncJob(OrderView): + permission = 'can_change_orders' + + def post(self, request, provider, *args, **kwargs): + prov, meta = sync_targets.get(active_in=self.request.event, identifier=provider) + + if self.request.POST.get("queue_sync") == "true": + prov.enqueue_order(self.order, 'user') + messages.success(self.request, _('The sync job has been enqueued and will run in the next minutes.')) + elif self.request.POST.get("cancel_job"): + job = self.order.queued_sync_jobs.get(pk=self.request.POST.get("cancel_job")) + job.delete() + messages.success(self.request, _('The sync job has been canceled.')) + elif self.request.POST.get("run_job_now"): + job = self.order.queued_sync_jobs.get(pk=self.request.POST.get("run_job_now")) + job.not_before = None + job.save() + messages.success(self.request, _('The sync job has been set to run as soon as possible.')) + + return redirect(self.get_order_url()) + + def get(self, *args, **kwargs): + return HttpResponseNotAllowed(['POST']) diff --git a/src/pretix/control/forms/mapping.py b/src/pretix/control/forms/mapping.py new file mode 100644 index 0000000000..adbba528e8 --- /dev/null +++ b/src/pretix/control/forms/mapping.py @@ -0,0 +1,72 @@ +from django import forms +from django.forms import formset_factory +from django.utils.translation import gettext_lazy as _ + +from pretix.base.datasync.sourcefields import QUESTION_TYPE_IDENTIFIERS +from pretix.base.datasync.datasync import MODE_SET_IF_NEW, MODE_SET_IF_EMPTY, MODE_OVERWRITE, MODE_APPEND_LIST + +class PropertyMappingForm(forms.Form): + pretix_field = forms.CharField() + external_field = forms.CharField() + value_map = forms.CharField(required=False) + overwrite = forms.ChoiceField( + choices=[ + (MODE_OVERWRITE, _("Overwrite")), + (MODE_SET_IF_NEW, _("Fill if new contact")), + (MODE_SET_IF_EMPTY, _("Fill if empty")), + (MODE_APPEND_LIST, _("Add to list")), + ] + ) + + def __init__(self, pretix_fields, external_fields_id, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["pretix_field"] = forms.ChoiceField( + label=_("pretix Field"), + choices=pretix_fields_choices(pretix_fields), + required=False, + ) + if external_fields_id: + self.fields["external_field"] = forms.ChoiceField( + widget=forms.Select( + attrs={ + "data-model-select2": "json_script", + "data-select2-src": "#" + external_fields_id, + }, + ), + ) + self.fields["external_field"].choices = [ + (self["external_field"].value(), self["external_field"].value()), + ] + print(self.fields) + + +class PropertyMappingFormSet(formset_factory( + PropertyMappingForm, + can_order=True, + can_delete=True, + extra=0, +)): + template_name = "pretixcontrol/datasync/property_mapping_formset.html" + + def __init__(self, pretix_fields, external_fields, prefix, *args, **kwargs): + super().__init__( + form_kwargs={ + "pretix_fields": pretix_fields, + "external_fields_id": prefix + "external-fields" if external_fields else None, + }, + prefix=prefix, + *args, **kwargs) + self.external_fields = external_fields + + def get_context(self): + ctx = super().get_context() + ctx["external_fields"] = self.external_fields + ctx["external_fields_id"] = self.prefix + "external-fields" + return ctx + + +def pretix_fields_choices(pretix_fields): + return [ + (key, label + " [" + QUESTION_TYPE_IDENTIFIERS[ptype] + "]") + for (required_input, key, label, ptype, enum_opts, getter) in pretix_fields + ] diff --git a/src/pretix/control/logdisplay.py b/src/pretix/control/logdisplay.py index 11f1321abe..2acadbbc7d 100644 --- a/src/pretix/control/logdisplay.py +++ b/src/pretix/control/logdisplay.py @@ -420,6 +420,23 @@ class OrderPrintLogEntryType(OrderLogEntryType): type=dict(PrintLog.PRINT_TYPES)[data["type"]], ) +@log_entry_types.new_from_dict({ + "pretix.event.order.data_sync.success": _("Ticket data successfully transferred to {provider}."), +}) +class OrderDataSyncLogentrytype(OrderLogEntryType): + pass + + +@log_entry_types.new_from_dict({ + "pretix.event.order.data_sync.failed": _("Error while transferring ticket data to {provider}:"), +}) +class OrderDataSyncErrorLogentrytype(OrderLogEntryType): + def display(self, logentry, data): + errmes = data["error"] + if not isinstance(errmes, list): + errmes = [errmes] + return mark_safe(escape(self.plain) + "".join("

" + escape(msg) + "

" for msg in errmes)) + @receiver(signal=logentry_display, dispatch_uid="pretixcontrol_logentry_display") def pretixcontrol_logentry_display(sender: Event, logentry: LogEntry, **kwargs): diff --git a/src/pretix/control/templates/pretixcontrol/datasync/control_order_info.html b/src/pretix/control/templates/pretixcontrol/datasync/control_order_info.html new file mode 100644 index 0000000000..8abb2376b2 --- /dev/null +++ b/src/pretix/control/templates/pretixcontrol/datasync/control_order_info.html @@ -0,0 +1,64 @@ +{% load i18n %} +{% load eventurl %} +{% load bootstrap3 %} +{% load escapejson %} +
+
+

+ {% trans "Data Sync" %} +

+
+
+ {{ test.hello }} + + + {% for pending in queued_sync_jobs %} + + + + + + {% endfor %} + {% for identifier, display_name in non_pending_providers %} + + + + + + {% endfor %} + + +
{{ pending.sync_provider }} + {% if pending.failed_attempts %} + + {% blocktrans trimmed with num=pending.failed_attempts max=pending.max_retry_attempts %} + Error, retry {{ num }} of {{ max }} + {% endblocktrans %}{% if pending.not_before %}, + {% blocktrans trimmed with datetime=pending.not_before %} + waiting until {{ datetime }} + {% endblocktrans %} + {% endif %} + {% elif pending.not_before %} + {% blocktrans trimmed with datetime=pending.not_before %} + Waiting until {{ datetime }} + {% endblocktrans %} + {% else %} + {% trans "Pending" %} + {% endif %} + +
+ {% csrf_token %} + {% if pending.not_before %} + + {% endif %} + +
+
{{ display_name }}- +
+ {% csrf_token %} + + +
+
+
+
diff --git a/src/pretix/control/templates/pretixcontrol/datasync/property_mapping_formset.html b/src/pretix/control/templates/pretixcontrol/datasync/property_mapping_formset.html new file mode 100644 index 0000000000..d431ceba9f --- /dev/null +++ b/src/pretix/control/templates/pretixcontrol/datasync/property_mapping_formset.html @@ -0,0 +1,77 @@ +{% load i18n %} +{% load bootstrap3 %} +{% load escapejson %} +{% load formset_tags %} +
+ {{ formset.management_form }} + {% bootstrap_formset_errors formset %} +
+ {% for f in formset %} + {% bootstrap_form_errors f %} +
+
+ {{ f.id }} + {% bootstrap_field f.DELETE form_group_class="" layout="inline" %} + {% bootstrap_field f.ORDER form_group_class="" layout="inline" %} +
+
+ {% bootstrap_field f.pretix_field layout='inline' form_group_class="" %} +
+
+ {% bootstrap_field f.external_field layout='inline' form_group_class="" %} +
+
+ {% bootstrap_field f.overwrite layout='inline' form_group_class="" %} +
+ {{ f.value_map.as_hidden }} +
+ + + + + + +
+
+ {% endfor %} +
+ +

+ +

+
+{% if external_fields %} + {{ external_fields|json_script:external_fields_id }} +{% endif %} diff --git a/src/pretix/control/urls.py b/src/pretix/control/urls.py index cd6d39a3a3..5ab4a8a41c 100644 --- a/src/pretix/control/urls.py +++ b/src/pretix/control/urls.py @@ -36,6 +36,7 @@ from django.urls import include, re_path from django.views.generic.base import RedirectView +from pretix.control.datasync import ControlSyncJob from pretix.control.views import ( auth, checkin, dashboards, discounts, event, geo, global_settings, item, main, modelimport, oauth, orders, organizer, pdf, search, shredder, @@ -428,6 +429,8 @@ urlpatterns = [ re_path(r'^orders/(?P[0-9A-Z]+)/cancellationrequests/(?P\d+)/delete$', orders.OrderCancellationRequestDelete.as_view(), name='event.order.cancellationrequests.delete'), + re_path(r'^orders/(?P[0-9A-Z]+)/sync_job/(?P[^/]+)/$', ControlSyncJob.as_view(), + name='event.order.sync_job'), re_path(r'^orders/(?P[0-9A-Z]+)/transactions/$', orders.OrderTransactions.as_view(), name='event.order.transactions'), re_path(r'^orders/(?P[0-9A-Z]+)/$', orders.OrderDetail.as_view(), name='event.order'), re_path(r'^invoice/(?P[^/]+)$', orders.InvoiceDownload.as_view(),