diff --git a/src/pretix/base/datasync/datasync.py b/src/pretix/base/datasync/datasync.py new file mode 100644 index 0000000000..d3cd724105 --- /dev/null +++ b/src/pretix/base/datasync/datasync.py @@ -0,0 +1,123 @@ +import logging +from datetime import datetime, timedelta +from itertools import groupby + +import sentry_sdk +from django.db import models +from django.db.models import Q +from django.dispatch import receiver +from django_scopes import scopes_disabled, scope + +from pretix.base.models import Order, Event +from django.utils.translation import gettext_lazy as _ + +from pretix.base.services.tasks import TransactionAwareTask +from pretix.base.signals import periodic_task, EventPluginRegistry +from pretix.celery_app import app + +logger = logging.getLogger(__name__) + + +class OrderSyncQueue(models.Model): + class Meta: + unique_together = (("order", "sync_target"),) + + order = models.ForeignKey( + Order, on_delete=models.CASCADE, related_name="order_sync_jobs" + ) + sync_target = models.CharField(blank=False, null=False, max_length=128) + triggered_by = models.CharField(blank=False, null=False, max_length=128) + triggered = models.DateTimeField(blank=False, null=False, auto_now_add=True) + failed_attempts = models.PositiveIntegerField(default=0) + not_before = models.DateTimeField(blank=True, null=True) + + +@receiver(periodic_task, dispatch_uid="data_sync_periodic") +def on_periodic_task(sender, **kwargs): + sync_all.apply_async() + + +sync_targets = EventPluginRegistry({"name": lambda o: o.__name__}) + + +def sync_event_to_target(event, target_cls, queued_orders): + with scope(organizer=event.organizer): + with target_cls(event=event) as p: + p.sync_queued_orders(queued_orders) + + + +@app.task() +def sync_all(): + with scopes_disabled(): + queue = OrderSyncQueue.objects.filter(Q(not_before__isnull=True) | Q(not_before__lt=datetime.now()))[:1000] + grouped = groupby(sorted(queue, key=lambda q: (q.sync_target, q.order.event)), lambda q: (q.sync_target, q.order.event)) + for (target, event), queued_orders in grouped: + target_cls = sync_targets.get(name=target) + sync_event_to_target(event, target_cls, queued_orders) + + +class SyncConfigError(Exception): + def __init__(self, messages, full_message=None): + self.messages = messages + self.full_message = full_message + + +class SyncProvider: + max_attempts = 5 + + def __init__(self, event): + self.event = event + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not exc_type: + self.do_after_event() + self.do_finally() + + def sync_order(self, order): + pass + + def next_retry_date(self, sq): + return datetime.now() + timedelta(days=1) + + def sync_queued_orders(self, queued_orders): + for sq in queued_orders: + try: + self.sync_order(sq.order) + except SyncConfigError as e: + logger.warning( + f"Could not sync order {sq.order.code} to {self.__name__} (config error)", + exc_info=True, + ) + sq.order.log_action( + "pretix.order_sync_failed", + { + "error": e.messages, + "full_message": e.full_message, + }, + ) + sq.delete() + except Exception as e: + sentry_sdk.capture_exception(e) + sq.failed_attempts += 1 + sq.not_before = self.next_retry_date(sq) + logger.exception( + f"Could not sync order {sq.order.code} to {self.__name__} (transient error, attempt #{sq.failed_attempts})" + ) + if sq.failed_attempts >= self.max_attempts: + sq.order.log_action( + "pretix.order_sync_failed", + { + "error": [_("Marking as failed after {} retries").format(sq.failed_attempts)], + "full_message": str(e), + }, + ) + sq.delete() + else: + sq.save() + else: + sq.delete() + diff --git a/src/pretix/base/datasync/forms.py b/src/pretix/base/datasync/forms.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/pretix/base/datasync/mapping.py b/src/pretix/base/datasync/mapping.py new file mode 100644 index 0000000000..2091bcef19 --- /dev/null +++ b/src/pretix/base/datasync/mapping.py @@ -0,0 +1,50 @@ +import json +from django import forms +from django.forms import ModelForm +from django.utils.translation import gettext_lazy as _ +from pretix.base.forms import SettingsForm + + +MODE_OVERWRITE = "overwrite" +MODE_SET_IF_NEW = "if_new" +MODE_SET_IF_EMPTY = "if_empty" +MODE_APPEND_LIST = "append" + + +class PropertyMappingForm(forms.Form): + pretix_field = forms.CharField() + external_field = forms.ChoiceField( + widget=forms.Select( + attrs={ + "data-model-select2": "json_script", + "data-select2-src": "#contact-props", + } + ) + ) + value_map = forms.CharField(required=False) + overwrite = forms.ChoiceField( + choices=[ + (MODE_OVERWRITE, _("Overwrite")), + (MODE_SET_IF_NEW, _("Fill if new contact")), + (MODE_SET_IF_EMPTY, _("Fill if empty")), + (MODE_APPEND_LIST, _("Add to list")), + ] + ) + + def __init__(self, pretix_fields, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["pretix_field"] = forms.ChoiceField( + label=_("pretix Field"), + choices=pretix_fields_choices(pretix_fields), + required=False, + ) + self.fields["external_field"].choices = [ + (self["external_field"].value(), self["external_field"].value()) + ] + + +def pretix_fields_choices(pretix_fields): + return [ + (key, label + " [" + ptype.value + "]") + for (required_input, key, label, ptype, enum_opts, getter) in pretix_fields + ] diff --git a/src/pretix/base/datasync/sourcefields.py b/src/pretix/base/datasync/sourcefields.py new file mode 100644 index 0000000000..c02b600674 --- /dev/null +++ b/src/pretix/base/datasync/sourcefields.py @@ -0,0 +1,420 @@ +import json +from django.db.models import Max +from django.utils.translation import gettext_lazy as _ +from functools import partial +from pretix.base.models import Checkin, Order, Question +from pretix.base.settings import PERSON_NAME_SCHEMES + + +def get_answer(op, question_identifier=None): + a = None + if op.addon_to: + if "answers" in getattr(op.addon_to, "_prefetched_objects_cache", {}): + try: + a = [ + a + for a in op.addon_to.answers.all() + if a.question.identifier == question_identifier + ][0] + except IndexError: + pass + else: + a = op.addon_to.answers.filter( + question__identifier=question_identifier + ).first() + + if "answers" in getattr(op, "_prefetched_objects_cache", {}): + try: + a = [ + a + for a in op.answers.all() + if a.question.identifier == question_identifier + ][0] + except IndexError: + pass + else: + a = op.answers.filter(question__identifier=question_identifier).first() + + if not a: + return "" + else: + if a.question.type in (Question.TYPE_CHOICE, Question.TYPE_CHOICE_MULTIPLE): + return [str(o.identifier) for o in a.options.all()] + if a.question.type == Question.TYPE_BOOLEAN: + return a.answer == "True" + return a.answer + + +def get_payment_date(order): + if order.status == Order.STATUS_PENDING: + return None + + return isoformat_or_none(order.payments.aggregate(m=Max("payment_date"))["m"]) + + +def isoformat_or_none(dt): + return dt and dt.isoformat() + + +def first_checkin_on_list(list_pk, position): + checkin = position.checkins.filter( + list__pk=list_pk, type=Checkin.TYPE_ENTRY + ).first() + if checkin: + return isoformat_or_none(checkin.datetime) + + +ORDER_POSITION = 'position' +ORDER = 'order' +EVENT = 'event' +EVENT_OR_SUBEVENT = 'event_or_subevent' +AVAILABLE_MODELS = { + 'OrderPosition': (ORDER_POSITION, ORDER, EVENT_OR_SUBEVENT, EVENT), + 'Order': (ORDER, EVENT), +} + + +def get_data_fields(event): + """ + Returns tuple of (required_input, key, label, type, enum_opts, getter) + + type is one of the hubspot data types as specified in + https://developers.hubspot.com/docs/api/crm/properties#property-type-and-fieldtype-values + """ + name_scheme = PERSON_NAME_SCHEMES[event.settings.name_scheme] + name_headers = [] + if name_scheme and len(name_scheme["fields"]) > 1: + for k, label, w in name_scheme["fields"]: + name_headers.append(label) + + src_fields = ( + [ + ( + ORDER_POSITION, + "attendee_name", + _("Attendee name"), + Question.TYPE_STRING, + None, + lambda position: position.attendee_name + or (position.addon_to.attendee_name if position.addon_to else None), + ), + ] + + [ + ( + ORDER_POSITION, + "attendee_name_" + k, + _("Attendee") + ": " + label, + Question.TYPE_STRING, + None, + partial( + lambda k, position: ( + position.attendee_name_parts + or (position.addon_to.attendee_name_parts if position.addon_to else {}) + or {} + ).get(k, ""), + k, + ), + ) + for k, label, w in name_scheme["fields"] + ] + + [ + ( + ORDER_POSITION, + "attendee_email", + _("Attendee email"), + Question.TYPE_STRING, + None, + lambda position: position.attendee_email + or (position.addon_to.attendee_email if position.addon_to else None), + ), + ( + ORDER_POSITION, + "attendee_or_order_email", + _("Attendee or order email"), + Question.TYPE_STRING, + None, + lambda position: position.attendee_email + or (position.addon_to.attendee_email if position.addon_to else None) + or position.order.email, + ), + ( + ORDER_POSITION, + "attendee_company", + _("Attendee company"), + Question.TYPE_STRING, + None, + lambda position: position.company or (position.addon_to.company if position.addon_to else None), + ), + ( + ORDER_POSITION, + "attendee_street", + _("Attendee address street"), + Question.TYPE_STRING, + None, + lambda position: position.street or (position.addon_to.street if position.addon_to else None), + ), + ( + ORDER_POSITION, + "attendee_zipcode", + _("Attendee address ZIP code"), + Question.TYPE_STRING, + None, + lambda position: position.zipcode or (position.addon_to.zipcode if position.addon_to else None), + ), + ( + ORDER_POSITION, + "attendee_city", + _("Attendee address city"), + Question.TYPE_STRING, + None, + lambda position: position.city or (position.addon_to.city if position.addon_to else None), + ), + ( + ORDER_POSITION, + "attendee_country", + _("Attendee address country"), + Question.TYPE_COUNTRYCODE, + None, + lambda position: str( + position.country or (position.addon_to.attendee_name if position.addon_to else "") + ), + ), + ( + ORDER, + "invoice_address_company", + _("Invoice address company"), + Question.TYPE_STRING, + None, + lambda order: order.invoice_address.company, + ), + ( + ORDER, + "invoice_address_name", + _("Invoice address name"), + Question.TYPE_STRING, + None, + lambda order: order.invoice_address.name, + ), + ] + + [ + ( + ORDER, + "invoice_address_name_" + k, + _("Invoice address") + ": " + label, + Question.TYPE_STRING, + None, + partial( + lambda k, order: (order.invoice_address.name_parts or {}).get( + k, "" + ), + k, + ), + ) + for k, label, w in name_scheme["fields"] + ] + + [ + ( + ORDER, + "invoice_address_street", + _("Invoice address street"), + Question.TYPE_STRING, + None, + lambda order: order.invoice_address.street, + ), + ( + ORDER, + "invoice_address_zipcode", + _("Invoice address ZIP code"), + Question.TYPE_STRING, + None, + lambda order: order.invoice_address.zipcode, + ), + ( + ORDER, + "invoice_address_city", + _("Invoice address city"), + Question.TYPE_STRING, + None, + lambda order: order.invoice_address.city, + ), + ( + ORDER, + "invoice_address_country", + _("Invoice address country"), + Question.TYPE_COUNTRYCODE, + None, + lambda order: str(order.invoice_address.country), + ), + ( + ORDER, + "email", + _("Order email"), + Question.TYPE_STRING, + None, + lambda order: order.email, + ), + ( + ORDER, + "order_code", + _("Order code"), + Question.TYPE_STRING, + None, + lambda order: order.code, + ), + ( + ORDER, + "event_order_code", + _("Event and order code"), + Question.TYPE_STRING, + None, + lambda order: order.full_code, + ), + ( + ORDER, + "order_total", + _("Order total"), + Question.TYPE_NUMBER, + None, + lambda order: str(order.total), + ), + ( + ORDER_POSITION, + "product", + _("Product and variation name"), + Question.TYPE_STRING, + None, + lambda position: str( + str(position.item.internal_name or position.item.name) + + ((" – " + str(position.variation.value)) if position.variation else "") + ), + ), + ( + ORDER_POSITION, + "product_id", + _("Product ID"), + Question.TYPE_NUMBER, + None, + lambda position: position.item.pk, + ), + ( + EVENT, + "event_slug", + _("Event short form"), + Question.TYPE_STRING, + None, + lambda event: str(event.slug), + ), + ( + EVENT, + "event_name", + _("Event name"), + Question.TYPE_STRING, + None, + lambda event: str(event.name), + ), + ( + EVENT_OR_SUBEVENT, + "event_date_from", + _("Event start date"), + Question.TYPE_DATETIME, + None, + lambda event_or_subevent: isoformat_or_none(event_or_subevent.date_from), + ), + ( + EVENT_OR_SUBEVENT, + "event_date_to", + _("Event end date"), + Question.TYPE_DATETIME, + None, + lambda event_or_subevent: isoformat_or_none(event_or_subevent.date_to), + ), + ( + ORDER_POSITION, + "voucher_code", + _("Voucher code"), + Question.TYPE_STRING, + None, + lambda position: position.voucher.code if position.voucher_id else "", + ), + ( + ORDER_POSITION, + "ticket_id", + _("Ticket ID"), + Question.TYPE_STRING, + None, + lambda position: position.code, + ), + ( + ORDER_POSITION, + "ticket_price", + _("Ticket price"), + Question.TYPE_NUMBER, + None, + lambda position: str(position.price), + ), + ( + ORDER, + "order_status", + _("Order status"), + Question.TYPE_CHOICE, + Order.STATUS_CHOICE, + lambda order: [str(order.status)], + ), + ( + ORDER, + "order_date", + _("Order date and time"), + Question.TYPE_DATETIME, + None, + lambda order: order.datetime.isoformat(), + ), + ( + ORDER, + "payment_date", + _("Payment date and time"), + Question.TYPE_DATETIME, + None, + get_payment_date, + ), + ] + + [ + ( + ORDER_POSITION, + "checkin_date_" + str(cl.pk), + _("Check-in datetime on list {}").format(cl.name), + Question.TYPE_DATETIME, + None, + partial(first_checkin_on_list, cl.pk), + ) + for cl in event.checkin_lists.all() + ] + + [ + ( + ORDER_POSITION, + "question_" + q.identifier, + _("Question: {name}").format(name=str(q.question)), + q.type, + get_enum_opts(q), + partial(lambda qq, position: get_answer(position, qq.identifier), q), + ) + for q in event.questions.all().prefetch_related("options") + ] + ) + return src_fields + + +def translate_property_mappings(property_mapping, checkin_list_map): + mappings = json.loads(property_mapping) + + for mapping in mappings: + if mapping["pretix_field"].startswith("checkin_date_"): + old_id = int(mapping["pretix_field"][len("checkin_date_") :]) + mapping["pretix_field"] = "checkin_date_%d" % checkin_list_map[old_id].pk + return json.dumps(mappings) + + +def get_enum_opts(q): + if q.type in (Question.TYPE_CHOICE, Question.TYPE_CHOICE_MULTIPLE): + return [(opt.identifier, opt.answer) for opt in q.options.all()] + else: + return None diff --git a/src/pretix/control/views/event.py b/src/pretix/control/views/event.py index a5d6fc408e..c428355f4e 100644 --- a/src/pretix/control/views/event.py +++ b/src/pretix/control/views/event.py @@ -1677,3 +1677,6 @@ class EventQRCode(EventPermissionRequiredMixin, View): r = HttpResponse(byte_io.read(), content_type='image/' + filetype) r['Content-Disposition'] = f'inline; filename="qrcode-{request.event.slug}.{filetype}"' return r + + +#class DataSyncSettings