diff --git a/doc/development/api/datasync.rst b/doc/development/api/datasync.rst new file mode 100644 index 000000000..e4f74d36c --- /dev/null +++ b/doc/development/api/datasync.rst @@ -0,0 +1,200 @@ +.. highlight:: python + :linenothreshold: 5 + +Data sync providers +=================== + +.. warning:: This feature is considered **experimental**. It might change at any time without prior notice. + +pretix provides connectivity to many external services through plugins. A common requirement +is unidirectionally sending (order, customer, ticket, ...) data into external systems. +The transfer is usually triggered by signals provided by pretix core (e.g. :data:`order_placed`), +but performed asynchronously. + +Such plugins should use the :class:`OutboundSyncProvider` API to utilize the queueing, retry and mapping mechanisms as well as the user interface for configuration and monitoring. + +An :class:`OutboundSyncProvider` for registering event participants in a mailing list could start +like this, for example: + +.. code-block:: python + + from pretix.base.datasync.datasync import OutboundSyncProvider + + class MyListSyncProvider(OutboundSyncProvider): + identifier = "my_list" + display_name = "My Mailing List Service" + # ...c + + +The plugin must register listeners in `signals.py` for all signals that should to trigger a sync and +within it has to call :meth:`MyListSyncProvider.enqueue_order` to enqueue the order for synchronization: + +.. code-block:: python + + @receiver(order_placed, dispatch_uid="mylist_order_placed") + def on_order_placed(sender, order, **kwargs): + MyListSyncProvider.enqueue_order(order, "order_placed") + + +Furthermore, most of these plugins need to translate data from some pretix objects (e.g. orders) +into an external system's data structures. Sometimes, there is only one reasonable way or the +plugin author makes an opinionated decision what information from which objects should be +transferred into which data structures in the external system. + +Otherwise, you can use a :class:`PropertyMappingFormSet` to let the user set up a mapping from pretix model fields +to external data fields. You could store the mapping information either in the event settings, or in a separate +data model. Your implementation of :attr:`OutboundSyncProvider.mappings` +needs to provide a list of mappings, which can be e.g. static objects or model instances, as long as they +have at least the properties defined in +:class:`pretix.base.datasync.datasync.StaticMapping`. + +.. code-block:: python + + # class MyListSyncProvider, contd. + def mappings(self): + return [ + StaticMapping( + id=1, pretix_model='Order', external_object_type='Contact', + pretix_id_field='email', external_id_field='email', + property_mappings=self.event.settings.mylist_order_mapping, + )) + ] + + +Currently, we support `orders` and `order positions` as data sources, with the data fields defined in +:func:`pretix.base.datasync.sourcefields.get_data_fields`. + +To perform the actual sync, implement :func:`sync_object_with_properties` and optionally +:func:`finalize_sync_order`. The former is called for each object to be created according to the ``mappings``. +For each order that was enqueued using :func:`enqueue_order`: + +- each Mapping with ``pretix_model == "Order"`` results in one call to :func:`sync_object_with_properties`, +- each Mapping with ``pretix_model == "OrderPosition"`` results in one call to + :func:`sync_object_with_properties` per order position, +- :func:`finalize_sync_order` is called one time after all calls to :func:`sync_object_with_properties`. + + +Implementation examples +----------------------- + +For example implementations, see the test cases in :mod:`tests.base.test_datasync`. + +In :class:`SimpleOrderSync`, a basic data transfer of order data only is +shown. Therein, a ``sync_object_with_properties`` method is defined as follows: + +.. code-block:: python + + from pretix.base.datasync.utils import assign_properties + + def sync_object_with_properties( + self, external_id_field, id_value, properties: list, inputs: dict, + mapping, mapped_objects: dict, **kwargs, + ): + # First, we query the external service if our object-to-sync already exists there. + # This is necessary to make sure our method is idempotent, i.e. handles already synced + # data gracefully. + pre_existing_object = self.fake_api_client.retrieve_object( + mapping.external_object_type, + external_id_field, + id_value + ) + + # We use the helper function ``assign_properties`` to update a pre-existing object. + update_values = assign_properties( + new_values=properties, + old_values=pre_existing_object or {}, + is_new=pre_existing_object is None, + list_sep=";", + ) + + # Then we can send our new data to the external service. The specifics of course depends + # on your API, e.g. you may need to use different endpoints for creating or updating an + # object, or pass the identifier separately instead of in the same dictionary as the + # other properties. + result = self.fake_api_client.create_or_update_object(mapping.external_object_type, { + **update_values, + external_id_field: id_value, + "_id": pre_existing_object and pre_existing_object.get("_id"), + }) + + # Finally, return a dictionary containing at least `object_type`, `external_id_field`, + # `id_value`, `external_link_href`, and `external_link_display_name` keys. + # Further keys may be provided for your internal use. This dictionary is provided + # in following calls in the ``mapped_objects`` dict, to allow creating associations + # to this object. + return { + "object_type": mapping.external_object_type, + "external_id_field": external_id_field, + "id_value": id_value, + "external_link_href": f"https://example.org/external-system/{mapping.external_object_type}/{id_value}/", + "external_link_display_name": f"Contact #{id_value} - Jane Doe", + "my_result": result, + } + +.. note:: The result dictionaries of earlier invocations of :func:`sync_object_with_properties` are + only provided in subsequent calls of the same sync run, such that a mapping can + refer to e.g. the external id of an object created by a preceding mapping. + However, the result dictionaries are currently not provided across runs. This will + likely change in a future revision of this API, to allow easier integration of external + systems that do not allow retrieving/updating data by a pretix-provided key. + +``mapped_objects`` is a dictionary of lists of dictionaries. The keys to the dictionary are +the mapping identifiers (``mapping.id``), the lists contain the result dictionaries returned +by :func:`sync_object_with_properties`. + + +In :class:`OrderAndTicketAssociationSync`, an example is given where orders, order positions, +and the association between them are transferred. + + +The OutboundSyncProvider base class +----------------------------------- + +.. autoclass:: pretix.base.datasync.datasync.OutboundSyncProvider + :members: + + +Property mapping format +----------------------- + +To allow the user to configure property mappings, you can use the PropertyMappingFormSet, +which will generate the required ``property_mappings`` value automatically. If you need +to specify the property mappings programmatically, you can refer to the description below +on their format. + +.. autoclass:: pretix.control.forms.mapping.PropertyMappingFormSet + :members: to_property_mappings_json + +A simple JSON-serialized ``property_mappings`` list for mapping some order information can look like this: + +.. code-block:: json + + [ + { + "pretix_field": "email", + "external_field": "orderemail", + "value_map": "", + "overwrite": "overwrite", + }, + { + "pretix_field": "order_status", + "external_field": "status", + "value_map": "{\"n\": \"pending\", \"p\": \"paid\", \"e\": \"expired\", \"c\": \"canceled\", \"r\": \"refunded\"}", + "overwrite": "overwrite", + }, + { + "pretix_field": "order_total", + "external_field": "total", + "value_map": "", + "overwrite": "overwrite", + } + ] + + +Translating mappings on Event copy +---------------------------------- + +Property mappings can contain references to event-specific primary keys. Therefore, plugins must register to the +event_copy_data signal and call translate_property_mappings on all property mappings they store. + +.. autofunction:: pretix.base.datasync.utils.translate_property_mappings diff --git a/doc/development/api/index.rst b/doc/development/api/index.rst index eeb7ea445..e891df35d 100644 --- a/doc/development/api/index.rst +++ b/doc/development/api/index.rst @@ -18,5 +18,6 @@ Contents: customview cookieconsent auth + datasync general quality diff --git a/src/pretix/base/apps.py b/src/pretix/base/apps.py index dd5d08f70..c32df6113 100644 --- a/src/pretix/base/apps.py +++ b/src/pretix/base/apps.py @@ -46,7 +46,7 @@ class PretixBaseConfig(AppConfig): from . import invoice # NOQA from . import notifications # NOQA from . import email # NOQA - from .services import auth, checkin, currencies, export, mail, tickets, cart, modelimport, orders, invoices, cleanup, update_check, quotas, notifications, vouchers # NOQA + from .services import auth, checkin, currencies, datasync, export, mail, tickets, cart, modelimport, orders, invoices, cleanup, update_check, quotas, notifications, vouchers # NOQA from .models import _transactions # NOQA from django.conf import settings diff --git a/src/pretix/base/datasync/__init__.py b/src/pretix/base/datasync/__init__.py new file mode 100644 index 000000000..9fd5bdc50 --- /dev/null +++ b/src/pretix/base/datasync/__init__.py @@ -0,0 +1,21 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# diff --git a/src/pretix/base/datasync/datasync.py b/src/pretix/base/datasync/datasync.py new file mode 100644 index 000000000..921e8d14d --- /dev/null +++ b/src/pretix/base/datasync/datasync.py @@ -0,0 +1,437 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# + +import json +import logging +from collections import namedtuple +from datetime import timedelta +from functools import cached_property +from typing import Optional, Protocol + +import sentry_sdk +from django.db import DatabaseError, transaction +from django.utils.timezone import now +from django.utils.translation import gettext_lazy as _ + +from pretix.base.datasync.sourcefields import ( + EVENT, EVENT_OR_SUBEVENT, ORDER, ORDER_POSITION, get_data_fields, +) +from pretix.base.i18n import language +from pretix.base.logentrytype_registry import make_link +from pretix.base.models.datasync import OrderSyncQueue, OrderSyncResult +from pretix.base.signals import EventPluginRegistry +from pretix.helpers import OF_SELF + +logger = logging.getLogger(__name__) + + +datasync_providers = EventPluginRegistry({"identifier": lambda o: o.identifier}) + + +class BaseSyncError(Exception): + def __init__(self, messages, full_message=None): + self.messages = messages + self.full_message = full_message + + +class UnrecoverableSyncError(BaseSyncError): + """ + A SyncProvider encountered a permanent problem, where a retry will not be successful. + """ + failure_mode = "permanent" + + +class SyncConfigError(UnrecoverableSyncError): + """ + A SyncProvider is misconfigured in a way where a retry without configuration change will + not be successful. + """ + failure_mode = "config" + + +class RecoverableSyncError(BaseSyncError): + """ + A SyncProvider has encountered a temporary problem, and the sync should be retried + at a later time. + """ + pass + + +class ObjectMapping(Protocol): + id: int + pretix_model: str + external_object_type: str + pretix_id_field: str + external_id_field: str + property_mappings: str + + +StaticMapping = namedtuple('StaticMapping', ('id', 'pretix_model', 'external_object_type', 'pretix_id_field', 'external_id_field', 'property_mappings')) + + +class OutboundSyncProvider: + max_attempts = 5 + + def __init__(self, event): + self.event = event + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + @classmethod + @property + def display_name(cls): + return str(cls.identifier) + + @classmethod + def enqueue_order(cls, order, triggered_by, not_before=None): + """ + Adds an order to the sync queue. May only be called on derived classes which define an ``identifier`` attribute. + + Should be called in the appropriate signal receivers, e.g.:: + + @receiver(order_placed, dispatch_uid="mysync_order_placed") + def on_order_placed(sender, order, **kwargs): + MySyncProvider.enqueue_order(order, "order_placed") + + :param order: the Order that should be synced + :param triggered_by: the reason why the order should be synced, e.g. name of the signal + (currently only used internally for logging) + """ + if not hasattr(cls, 'identifier'): + raise TypeError('Call this method on a derived class that defines an "identifier" attribute.') + OrderSyncQueue.objects.update_or_create( + order=order, + sync_provider=cls.identifier, + in_flight=False, + defaults={ + "event": order.event, + "triggered_by": triggered_by, + "not_before": not_before or now(), + "need_manual_retry": None, + }, + ) + + @classmethod + def get_external_link_info(cls, event, external_link_href, external_link_display_name): + return { + "href": external_link_href, + "val": external_link_display_name, + } + + @classmethod + def get_external_link_html(cls, event, external_link_href, external_link_display_name): + info = cls.get_external_link_info(event, external_link_href, external_link_display_name) + return make_link(info, '{val}') + + def next_retry_date(self, sq): + """ + Optionally override to configure a different retry backoff behavior + """ + return now() + timedelta(hours=1) + + def should_sync_order(self, order): + """ + Optionally override this method to exclude certain orders from sync by returning ``False`` + """ + return True + + @property + def mappings(self): + """ + Implementations must override this property to provide the data mappings as a list of objects. + + They can return instances of the ``StaticMapping`` `namedtuple` defined above, or create their own + class (e.g. a Django model). + + :return: The returned objects must have at least the following properties: + + - `id`: Unique identifier for this mapping. If the mappings are Django models, the database primary key + should be used. This may be referenced in other mappings, to establish relations between objects. + - `pretix_model`: Which pretix model to use as data source in this mapping. Possible values are + the keys of ``sourcefields.AVAILABLE_MODELS`` + - `external_object_type`: Destination object type in the target system. opaque string of maximum 128 characters. + - `pretix_id_field`: Which pretix data field should be used to identify the mapped object. Any ``DataFieldInfo.key`` + returned by ``sourcefields.get_data_fields()`` for the combination of ``Event`` and ``pretix_model``. + - `external_id_field`: Destination identifier field in the target system. + - `property_mappings`: Mapping configuration as generated by ``PropertyMappingFormSet.to_property_mappings_json()``. + """ + raise NotImplementedError + + def sync_queued_orders(self, queued_orders): + """ + This method should catch all Exceptions and handle them appropriately. It should never throw + an Exception, as that may block the entire queue. + """ + for queue_item in queued_orders: + with transaction.atomic(): + try: + sq = ( + OrderSyncQueue.objects + .select_for_update(of=OF_SELF, nowait=True) + .select_related("order") + .get(pk=queue_item.pk) + ) + if sq.in_flight: + continue + sq.in_flight = True + sq.in_flight_since = now() + sq.save() + except DatabaseError: + # Either select_for_update failed to lock the row, or we couldn't set in_flight + # as this order is already in flight (UNIQUE violation). In either case, we ignore + # this order for now. + continue + + try: + mapped_objects = self.sync_order(sq.order) + if not all(all(not res or res.sync_info.get("action", "") == "nothing_to_do" for res in res_list) for res_list in mapped_objects.values()): + sq.order.log_action("pretix.event.order.data_sync.success", { + "provider": self.identifier, + "objects": { + mapping_id: [osr and osr.to_result_dict() for osr in results] + for mapping_id, results in mapped_objects.items() + }, + }) + sq.delete() + except UnrecoverableSyncError as e: + sq.set_sync_error(e.failure_mode, e.messages, e.full_message) + except RecoverableSyncError as e: + sq.failed_attempts += 1 + sq.not_before = self.next_retry_date(sq) + # model changes saved by set_sync_error / clear_in_flight calls below + if sq.failed_attempts >= self.max_attempts: + logger.exception('Failed to sync order (max attempts exceeded)') + sentry_sdk.capture_exception(e) + sq.set_sync_error("exceeded", e.messages, e.full_message) + else: + logger.info( + f"Could not sync order {sq.order.code} to {type(self).__name__} " + f"(transient error, attempt #{sq.failed_attempts}, next {sq.not_before})", + exc_info=True, + ) + sq.clear_in_flight() + except Exception as e: + logger.exception('Failed to sync order (unhandled exception)') + sentry_sdk.capture_exception(e) + sq.set_sync_error("internal", [], str(e)) + + @cached_property + def data_fields(self): + return { + f.key: f + for f in get_data_fields(self.event) + } + + def get_field_value(self, inputs, mapping_entry): + key = mapping_entry["pretix_field"] + try: + field = self.data_fields[key] + except KeyError: + with language(self.event.settings.locale): + raise SyncConfigError([_( + 'Field "{field_name}" is not valid for {available_inputs}. Please check your {provider_name} settings.' + ).format(key=key, available_inputs="/".join(inputs.keys()), provider_name=self.display_name)]) + input = inputs[field.required_input] + val = field.getter(input) + if isinstance(val, list): + if field.enum_opts and mapping_entry.get("value_map"): + map = json.loads(mapping_entry["value_map"]) + try: + val = [map[el] for el in val] + except KeyError: + with language(self.event.settings.locale): + raise SyncConfigError([_( + 'Please update value mapping for field "{field_name}" - option "{val}" not assigned' + ).format(field_name=key, val=val)]) + + val = ",".join(val) + return val + + def get_properties(self, inputs: dict, property_mappings: str): + property_mappings = json.loads(property_mappings) + return [ + (m["external_field"], self.get_field_value(inputs, m), m["overwrite"]) + for m in property_mappings + ] + + def sync_object_with_properties( + self, + external_id_field: str, + id_value, + properties: list, + inputs: dict, + mapping: ObjectMapping, + mapped_objects: dict, + **kwargs, + ) -> Optional[dict]: + """ + This method is called for each object that needs to be created/updated in the external system -- which these are is + determined by the implementation of the `mapping` property. + + :param external_id_field: Identifier field in the external system as provided in ``mapping.external_identifier`` + :param id_value: Identifier contents as retrieved from the property specified by ``mapping.pretix_identifier`` of the model + specified by ``mapping.pretix_model`` + :param properties: All properties defined in ``mapping.property_mappings``, as list of three-tuples + ``(external_field, value, overwrite)`` + :param inputs: All pretix model instances from which data can be retrieved for this mapping. + Dictionary mapping from sourcefields.ORDER_POSITION, .ORDER, .EVENT, .EVENT_OR_SUBEVENT to the + relevant Django model. + Most providers don't need to use this parameter directly, as `properties` and `id_value` + already contain the values as evaluated from the available inputs. + :param mapping: The mapping object as returned by ``self.mappings`` + :param mapped_objects: Information about objects that were synced in the same sync run, by mapping definitions + *before* the current one in order of ``self.mappings``. + Type is a dictionary ``{mapping.id: [list of OrderSyncResult objects]}`` + Useful to create associations between objects in the target system. + + Example code to create return value:: + + return { + # optional: + "action": "nothing_to_do", # to inform that no action was taken, because the data was already up-to-date. + # other values for action (e.g. create, update) currently have no special + # meaning, but are visible for debugging purposes to admins. + + # optional: + "external_link_href": "https://external-system.example.com/backend/link/to/contact/123/", + "external_link_display_name": "Contact #123 - Jane Doe", + "...optionally further values you need in mapped_objects for association": 123456789, + } + + The return value needs to be a JSON serializable dict, or None. + + Return None only in case you decide this object should not be synced at all in this mapping. Do not return None in + case the object is already up-to-date in the target system (return "action": "nothing_to_do" instead). + + This method needs to be idempotent, i.e. calling it multiple times with the same input values should create + only a single object in the target system. + + Subsequent calls with the same mapping and id_value should update the existing object, instead of creating a new one. + In a SQL database, you might use an `INSERT OR UPDATE` or `UPSERT` statement; many REST APIs provide an equivalent API call. + """ + raise NotImplementedError() + + def sync_object( + self, + inputs: dict, + mapping, + mapped_objects: dict, + ): + logger.debug("Syncing object %r, %r, %r", inputs, mapping, mapped_objects) + properties = self.get_properties(inputs, mapping.property_mappings) + logger.debug("Properties: %r", properties) + + id_value = self.get_field_value(inputs, {"pretix_field": mapping.pretix_id_field}) + if not id_value: + return None + + info = self.sync_object_with_properties( + external_id_field=mapping.external_id_field, + id_value=id_value, + properties=properties, + inputs=inputs, + mapping=mapping, + mapped_objects=mapped_objects, + ) + if not info: + return None + external_link_href = info.pop('external_link_href', None) + external_link_display_name = info.pop('external_link_display_name', None) + obj, created = OrderSyncResult.objects.update_or_create( + order=inputs.get(ORDER), order_position=inputs.get(ORDER_POSITION), sync_provider=self.identifier, + mapping_id=mapping.id, + defaults=dict( + external_object_type=mapping.external_object_type, + external_id_field=mapping.external_id_field, + id_value=id_value, + external_link_href=external_link_href, + external_link_display_name=external_link_display_name, + sync_info=info, + transmitted=now(), + ) + ) + return obj + + def sync_order(self, order): + if not self.should_sync_order(order): + logger.debug("Skipping order %r", order) + return + + logger.debug("Syncing order %r", order) + positions = list( + order.all_positions + .prefetch_related("answers", "answers__question") + .select_related( + "voucher", + ) + ) + order_inputs = {ORDER: order, EVENT: self.event} + mapped_objects = {} + for mapping in self.mappings: + if mapping.pretix_model == 'Order': + mapped_objects[mapping.id] = [ + self.sync_object(order_inputs, mapping, mapped_objects) + ] + elif mapping.pretix_model == 'OrderPosition': + mapped_objects[mapping.id] = [ + self.sync_object({ + **order_inputs, EVENT_OR_SUBEVENT: op.subevent or self.event, ORDER_POSITION: op + }, mapping, mapped_objects) + for op in positions + ] + else: + raise SyncConfigError("Invalid pretix model '{}'".format(mapping.pretix_model)) + self.finalize_sync_order(order) + return mapped_objects + + def filter_mapped_objects(self, mapped_objects, inputs): + """ + For order positions, only + """ + if ORDER_POSITION in inputs: + return { + mapping_id: [ + osr for osr in results + if osr and (osr.order_position_id is None or osr.order_position_id == inputs[ORDER_POSITION].id) + ] + for mapping_id, results in mapped_objects.items() + } + else: + return mapped_objects + + def finalize_sync_order(self, order): + """ + Called after ``sync_object`` has been called successfully for all objects of a specific order. Can + be used for saving bulk information per order. + """ + pass + + def close(self): + """ + Called after all orders of an event have been synced. Can be used for clean-up tasks (e.g. closing + a session). + """ + pass diff --git a/src/pretix/base/datasync/sourcefields.py b/src/pretix/base/datasync/sourcefields.py new file mode 100644 index 000000000..5754f83fc --- /dev/null +++ b/src/pretix/base/datasync/sourcefields.py @@ -0,0 +1,534 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# + +from collections import namedtuple +from functools import partial + +from django.db.models import Max, Q +from django.utils.translation import gettext_lazy as _ + +from pretix.base.models import Checkin, InvoiceAddress, Order, Question +from pretix.base.settings import PERSON_NAME_SCHEMES + + +def get_answer(op, question_identifier=None): + a = None + if op.addon_to: + if "answers" in getattr(op.addon_to, "_prefetched_objects_cache", {}): + try: + a = [ + a + for a in op.addon_to.answers.all() + if a.question.identifier == question_identifier + ][0] + except IndexError: + pass + else: + a = op.addon_to.answers.filter( + question__identifier=question_identifier + ).first() + + if "answers" in getattr(op, "_prefetched_objects_cache", {}): + try: + a = [ + a + for a in op.answers.all() + if a.question.identifier == question_identifier + ][0] + except IndexError: + pass + else: + a = op.answers.filter(question__identifier=question_identifier).first() + + if not a: + return "" + else: + if a.question.type in (Question.TYPE_CHOICE, Question.TYPE_CHOICE_MULTIPLE): + return [str(o.identifier) for o in a.options.all()] + if a.question.type == Question.TYPE_BOOLEAN: + return a.answer == "True" + return a.answer + + +def get_payment_date(order): + if order.status == Order.STATUS_PENDING: + return None + + return isoformat_or_none(order.payments.aggregate(m=Max("payment_date"))["m"]) + + +def isoformat_or_none(dt): + return dt and dt.isoformat() + + +def first_checkin_on_list(list_pk, position): + checkin = position.checkins.filter( + list__pk=list_pk, type=Checkin.TYPE_ENTRY + ).first() + if checkin: + return isoformat_or_none(checkin.datetime) + + +def split_name_on_last_space(name, part): + name_parts = name.rsplit(" ", 1) + return name_parts[part] if len(name_parts) > part else "" + + +ORDER_POSITION = 'position' +ORDER = 'order' +EVENT = 'event' +EVENT_OR_SUBEVENT = 'event_or_subevent' +AVAILABLE_MODELS = { + 'OrderPosition': (ORDER_POSITION, ORDER, EVENT_OR_SUBEVENT, EVENT), + 'Order': (ORDER, EVENT), +} + + +DataFieldInfo = namedtuple( + 'DataFieldInfo', + field_names=('required_input', 'key', 'label', 'type', 'enum_opts', 'getter', 'deprecated'), + defaults=[False] +) + + +def get_invoice_address_or_empty(order): + try: + return order.invoice_address + except InvoiceAddress.DoesNotExist: + return InvoiceAddress() + + +def get_data_fields(event, for_model=None): + """ + Returns tuple of (required_input, key, label, type, enum_opts, getter) + + Type is one of the Question types as defined in Question.TYPE_CHOICES. + + The data type of the return value of `getter` depends on `type`: + - TYPE_CHOICE_MULTIPLE: list of strings + - TYPE_CHOICE: list, containing zero or one strings + - TYPE_BOOLEAN: boolean + - all other (including TYPE_NUMBER): string + """ + name_scheme = PERSON_NAME_SCHEMES[event.settings.name_scheme] + name_headers = [] + if name_scheme and len(name_scheme["fields"]) > 1: + for k, label, w in name_scheme["fields"]: + name_headers.append(label) + + src_fields = ( + [ + DataFieldInfo( + ORDER_POSITION, + "attendee_name", + _("Attendee name"), + Question.TYPE_STRING, + None, + lambda position: position.attendee_name + or (position.addon_to.attendee_name if position.addon_to else None), + ), + ] + + [ + DataFieldInfo( + ORDER_POSITION, + "attendee_name_" + k, + _("Attendee") + ": " + label, + Question.TYPE_STRING, + None, + partial( + lambda k, position: ( + position.attendee_name_parts + or (position.addon_to.attendee_name_parts if position.addon_to else {}) + or {} + ).get(k, ""), + k, + ), + deprecated=len(name_scheme["fields"]) == 1, + ) + for k, label, w in name_scheme["fields"] + ] + + [ + DataFieldInfo( + ORDER_POSITION, + "attendee_email", + _("Attendee email"), + Question.TYPE_STRING, + None, + lambda position: position.attendee_email + or (position.addon_to.attendee_email if position.addon_to else None), + ), + DataFieldInfo( + ORDER_POSITION, + "attendee_or_order_email", + _("Attendee or order email"), + Question.TYPE_STRING, + None, + lambda position: position.attendee_email + or (position.addon_to.attendee_email if position.addon_to else None) + or position.order.email, + ), + DataFieldInfo( + ORDER_POSITION, + "attendee_company", + _("Attendee company"), + Question.TYPE_STRING, + None, + lambda position: position.company or (position.addon_to.company if position.addon_to else None), + ), + DataFieldInfo( + ORDER_POSITION, + "attendee_street", + _("Attendee address street"), + Question.TYPE_STRING, + None, + lambda position: position.street or (position.addon_to.street if position.addon_to else None), + ), + DataFieldInfo( + ORDER_POSITION, + "attendee_zipcode", + _("Attendee address ZIP code"), + Question.TYPE_STRING, + None, + lambda position: position.zipcode or (position.addon_to.zipcode if position.addon_to else None), + ), + DataFieldInfo( + ORDER_POSITION, + "attendee_city", + _("Attendee address city"), + Question.TYPE_STRING, + None, + lambda position: position.city or (position.addon_to.city if position.addon_to else None), + ), + DataFieldInfo( + ORDER_POSITION, + "attendee_country", + _("Attendee address country"), + Question.TYPE_COUNTRYCODE, + None, + lambda position: str( + position.country or (position.addon_to.attendee_name if position.addon_to else "") + ), + ), + DataFieldInfo( + ORDER, + "invoice_address_company", + _("Invoice address company"), + Question.TYPE_STRING, + None, + lambda order: get_invoice_address_or_empty(order).company, + ), + DataFieldInfo( + ORDER, + "invoice_address_name", + _("Invoice address name"), + Question.TYPE_STRING, + None, + lambda order: get_invoice_address_or_empty(order).name, + ), + ] + + [ + DataFieldInfo( + ORDER, + "invoice_address_name_" + k, + _("Invoice address") + ": " + label, + Question.TYPE_STRING, + None, + partial( + lambda k, order: (get_invoice_address_or_empty(order).name_parts or {}).get( + k, "" + ), + k, + ), + deprecated=len(name_scheme["fields"]) == 1, + ) + for k, label, w in name_scheme["fields"] + ] + + [ + DataFieldInfo( + ORDER, + "invoice_address_street", + _("Invoice address street"), + Question.TYPE_STRING, + None, + lambda order: get_invoice_address_or_empty(order).street, + ), + DataFieldInfo( + ORDER, + "invoice_address_zipcode", + _("Invoice address ZIP code"), + Question.TYPE_STRING, + None, + lambda order: get_invoice_address_or_empty(order).zipcode, + ), + DataFieldInfo( + ORDER, + "invoice_address_city", + _("Invoice address city"), + Question.TYPE_STRING, + None, + lambda order: get_invoice_address_or_empty(order).city, + ), + DataFieldInfo( + ORDER, + "invoice_address_country", + _("Invoice address country"), + Question.TYPE_COUNTRYCODE, + None, + lambda order: str(get_invoice_address_or_empty(order).country), + ), + DataFieldInfo( + ORDER, + "email", + _("Order email"), + Question.TYPE_STRING, + None, + lambda order: order.email, + ), + DataFieldInfo( + ORDER, + "order_code", + _("Order code"), + Question.TYPE_STRING, + None, + lambda order: order.code, + ), + DataFieldInfo( + ORDER, + "event_order_code", + _("Event and order code"), + Question.TYPE_STRING, + None, + lambda order: order.full_code, + ), + DataFieldInfo( + ORDER, + "order_total", + _("Order total"), + Question.TYPE_NUMBER, + None, + lambda order: str(order.total), + ), + DataFieldInfo( + ORDER_POSITION, + "product", + _("Product and variation name"), + Question.TYPE_STRING, + None, + lambda position: str( + str(position.item.internal_name or position.item.name) + + ((" – " + str(position.variation.value)) if position.variation else "") + ), + ), + DataFieldInfo( + ORDER_POSITION, + "product_id", + _("Product ID"), + Question.TYPE_NUMBER, + None, + lambda position: str(position.item.pk), + ), + DataFieldInfo( + EVENT, + "event_slug", + _("Event short form"), + Question.TYPE_STRING, + None, + lambda event: str(event.slug), + ), + DataFieldInfo( + EVENT, + "event_name", + _("Event name"), + Question.TYPE_STRING, + None, + lambda event: str(event.name), + ), + DataFieldInfo( + EVENT_OR_SUBEVENT, + "event_date_from", + _("Event start date"), + Question.TYPE_DATETIME, + None, + lambda event_or_subevent: isoformat_or_none(event_or_subevent.date_from), + ), + DataFieldInfo( + EVENT_OR_SUBEVENT, + "event_date_to", + _("Event end date"), + Question.TYPE_DATETIME, + None, + lambda event_or_subevent: isoformat_or_none(event_or_subevent.date_to), + ), + DataFieldInfo( + ORDER_POSITION, + "voucher_code", + _("Voucher code"), + Question.TYPE_STRING, + None, + lambda position: position.voucher.code if position.voucher_id else "", + ), + DataFieldInfo( + ORDER_POSITION, + "ticket_id", + _("Order code and position number"), + Question.TYPE_STRING, + None, + lambda position: position.code, + ), + DataFieldInfo( + ORDER_POSITION, + "ticket_price", + _("Ticket price"), + Question.TYPE_NUMBER, + None, + lambda position: str(position.price), + ), + DataFieldInfo( + ORDER, + "order_status", + _("Order status"), + Question.TYPE_CHOICE, + Order.STATUS_CHOICE, + lambda order: [order.status], + ), + DataFieldInfo( + ORDER_POSITION, + "ticket_status", + _("Ticket status"), + Question.TYPE_CHOICE, + Order.STATUS_CHOICE, + lambda position: [Order.STATUS_CANCELED if position.canceled else position.order.status], + ), + DataFieldInfo( + ORDER, + "order_date", + _("Order date and time"), + Question.TYPE_DATETIME, + None, + lambda order: order.datetime.isoformat(), + ), + DataFieldInfo( + ORDER, + "payment_date", + _("Payment date and time"), + Question.TYPE_DATETIME, + None, + get_payment_date, + ), + DataFieldInfo( + ORDER, + "order_locale", + _("Order language code"), + Question.TYPE_CHOICE, + [(lc, lc) for lc in event.settings.locales], + lambda order: [order.locale], + ), + DataFieldInfo( + ORDER_POSITION, + "position_id", + _("Order position ID"), + Question.TYPE_NUMBER, + None, + lambda op: str(op.pk), + ), + ] + + [ + DataFieldInfo( + ORDER_POSITION, + "checkin_date_" + str(cl.pk), + _("Check-in datetime on list {}").format(cl.name), + Question.TYPE_DATETIME, + None, + partial(first_checkin_on_list, cl.pk), + ) + for cl in event.checkin_lists.all() + ] + + [ + DataFieldInfo( + ORDER_POSITION, + "question_" + q.identifier, + _("Question: {name}").format(name=str(q.question)), + q.type, + get_enum_opts(q), + partial(lambda qq, position: get_answer(position, qq.identifier), q), + ) + for q in event.questions.filter(~Q(type=Question.TYPE_FILE)).prefetch_related("options") + ] + ) + if not any(field_name == "given_name" for field_name, label, weight in name_scheme["fields"]): + src_fields += [ + DataFieldInfo( + ORDER_POSITION, + "attendee_name_given_name", + _("Attendee") + ": " + _("Given name") + " (⚠️ auto-generated, not recommended)", + Question.TYPE_STRING, + None, + lambda position: split_name_on_last_space(position.attendee_name, part=0), + deprecated=True, + ), + DataFieldInfo( + ORDER, + "invoice_address_name_given_name", + _("Invoice address") + ": " + _("Given name") + " (⚠️ auto-generated, not recommended)", + Question.TYPE_STRING, + None, + lambda order: split_name_on_last_space(get_invoice_address_or_empty(order).name, part=0), + deprecated=True, + ), + ] + + if not any(field_name == "family_name" for field_name, label, weight in name_scheme["fields"]): + src_fields += [ + DataFieldInfo( + ORDER_POSITION, + "attendee_name_family_name", + _("Attendee") + ": " + _("Family name") + " (⚠️ auto-generated, not recommended)", + Question.TYPE_STRING, + None, + lambda position: split_name_on_last_space(position.attendee_name, part=1), + deprecated=True, + ), + DataFieldInfo( + ORDER, + "invoice_address_name_family_name", + _("Invoice address") + ": " + _("Family name") + " (⚠️ auto-generated, not recommended)", + Question.TYPE_STRING, + None, + lambda order: split_name_on_last_space(get_invoice_address_or_empty(order).name, part=1), + deprecated=True, + ), + ] + + if for_model: + available_inputs = AVAILABLE_MODELS[for_model] + return [ + f for f in src_fields if f.required_input in available_inputs + ] + else: + return src_fields + + +def get_enum_opts(q): + if q.type in (Question.TYPE_CHOICE, Question.TYPE_CHOICE_MULTIPLE): + return [(opt.identifier, opt.answer) for opt in q.options.all()] + else: + return None diff --git a/src/pretix/base/datasync/utils.py b/src/pretix/base/datasync/utils.py new file mode 100644 index 000000000..da1f22d0d --- /dev/null +++ b/src/pretix/base/datasync/utils.py @@ -0,0 +1,122 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +import json +from typing import List, Tuple + +from pretix.base.datasync.datasync import SyncConfigError +from pretix.base.models.datasync import ( + MODE_APPEND_LIST, MODE_OVERWRITE, MODE_SET_IF_EMPTY, MODE_SET_IF_NEW, +) + + +def assign_properties( + new_values: List[Tuple[str, str, str]], old_values: dict, is_new, list_sep +): + """ + Generates a dictionary mapping property keys to new values, handling conditional overwrites and list updates + according to an update mode specified per property. + + Supported update modes are: + - `MODE_OVERWRITE`: Replaces the existing value with the new value. + - `MODE_SET_IF_NEW`: Only sets the property if `is_new` is True. + - `MODE_SET_IF_EMPTY`: Only sets the property if the field is empty or missing in old_values. + - `MODE_APPEND_LIST`: Appends the new value to the list from old_values (or the empty list if missing), + using `list_sep` as a separator. + + :param new_values: List of tuples, where each tuple contains (field_name, new_value, update_mode). + :param old_values: Dictionary, current property values in the external system. + :param is_new: Boolean, whether the object will be newly created in the external system. + :param list_sep: If string, used as a separator for MODE_APPEND_LIST. If None, native lists are used. + :raises SyncConfigError: If an invalid update mode is specified. + :returns: A dictionary containing the properties that need to be updated in the external system. + """ + + out = {} + + for field_name, new_value, update_mode in new_values: + if update_mode == MODE_OVERWRITE: + out[field_name] = new_value + continue + elif update_mode == MODE_SET_IF_NEW and not is_new: + continue + if not new_value: + continue + + current_value = old_values.get(field_name, out.get(field_name, "")) + if update_mode in (MODE_SET_IF_EMPTY, MODE_SET_IF_NEW): + if not current_value: + out[field_name] = new_value + elif update_mode == MODE_APPEND_LIST: + _add_to_list(out, field_name, current_value, new_value, list_sep) + else: + raise SyncConfigError(["Invalid update mode " + update_mode]) + return out + + +def _add_to_list(out, field_name, current_value, new_item, list_sep): + new_item = str(new_item) + if list_sep is not None: + new_item = new_item.replace(list_sep, "") + current_value = current_value.split(list_sep) if current_value else [] + elif not isinstance(current_value, (list, tuple)): + current_value = [str(current_value)] + if new_item not in current_value: + new_list = current_value + [new_item] + if list_sep is not None: + new_list = list_sep.join(new_list) + out[field_name] = new_list + + +def translate_property_mappings(property_mappings, checkin_list_map): + """ + To properly handle copied events, users of data fields as provided by get_data_fields need to register to the + event_copy_data signal and translate all stored references to those fields using this method. + + For example, if you store your mappings in a custom Django model with a ForeignKey to Event: + + .. code-block:: python + + @receiver(signal=event_copy_data, dispatch_uid="my_sync_event_copy_data") + def event_copy_data_receiver(sender, other, checkin_list_map, **kwargs): + object_mappings = other.my_object_mappings.all() + object_mapping_map = {} + for om in object_mappings: + om = copy.copy(om) + object_mapping_map[om.pk] = om + om.pk = None + om.event = sender + om.property_mappings = translate_property_mappings(om.property_mappings, checkin_list_map) + om.save() + + """ + mappings = json.loads(property_mappings) + + for mapping in mappings: + if mapping["pretix_field"].startswith("checkin_date_"): + old_id = int(mapping["pretix_field"][len("checkin_date_"):]) + if old_id not in checkin_list_map: + # old_id might not be in checkin_list_map, because copying of an event series only copies check-in + # lists covering the whole series, not individual dates. + mapping["pretix_field"] = "_invalid_" + mapping["pretix_field"] + else: + mapping["pretix_field"] = "checkin_date_%d" % checkin_list_map[old_id].pk + return json.dumps(mappings) diff --git a/src/pretix/base/migrations/0284_ordersyncresult_ordersyncqueue.py b/src/pretix/base/migrations/0284_ordersyncresult_ordersyncqueue.py new file mode 100644 index 000000000..0968715f5 --- /dev/null +++ b/src/pretix/base/migrations/0284_ordersyncresult_ordersyncqueue.py @@ -0,0 +1,54 @@ +# Generated by Django 4.2.21 on 2025-06-27 13:32 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('pretixbase', '0283_taxrule_default_taxrule_backfill'), + ] + + operations = [ + migrations.CreateModel( + name='OrderSyncResult', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('sync_provider', models.CharField(max_length=128)), + ('mapping_id', models.IntegerField()), + ('external_object_type', models.CharField(max_length=128)), + ('external_id_field', models.CharField(max_length=128)), + ('id_value', models.CharField(max_length=128)), + ('external_link_href', models.CharField(max_length=255, null=True)), + ('external_link_display_name', models.CharField(max_length=255, null=True)), + ('transmitted', models.DateTimeField(auto_now_add=True)), + ('sync_info', models.JSONField()), + ('order', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sync_results', to='pretixbase.order')), + ('order_position', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='sync_results', to='pretixbase.orderposition')), + ], + options={ + 'indexes': [models.Index(fields=['order', 'sync_provider'], name='pretixbase__order_i_3e3c84_idx')], + }, + ), + migrations.CreateModel( + name='OrderSyncQueue', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('sync_provider', models.CharField(max_length=128)), + ('triggered_by', models.CharField(max_length=128)), + ('triggered', models.DateTimeField(auto_now_add=True)), + ('failed_attempts', models.PositiveIntegerField(default=0)), + ('not_before', models.DateTimeField(db_index=True)), + ('need_manual_retry', models.CharField(null=True, max_length=20)), + ('in_flight', models.BooleanField(default=False)), + ('in_flight_since', models.DateTimeField(blank=True, null=True)), + ('event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='queued_sync_jobs', to='pretixbase.event')), + ('order', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='queued_sync_jobs', to='pretixbase.order')), + ], + options={ + 'ordering': ('triggered',), + 'unique_together': {('order', 'sync_provider', 'in_flight')}, + }, + ), + ] diff --git a/src/pretix/base/models/datasync.py b/src/pretix/base/models/datasync.py new file mode 100644 index 000000000..065832403 --- /dev/null +++ b/src/pretix/base/models/datasync.py @@ -0,0 +1,149 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# + +import logging +from functools import cached_property + +from django.db import IntegrityError, models +from django.utils.translation import gettext as _ + +from pretix.base.models import Event, Order, OrderPosition + +logger = logging.getLogger(__name__) + + +MODE_OVERWRITE = "overwrite" +MODE_SET_IF_NEW = "if_new" +MODE_SET_IF_EMPTY = "if_empty" +MODE_APPEND_LIST = "append" + + +class OrderSyncQueue(models.Model): + order = models.ForeignKey( + Order, on_delete=models.CASCADE, related_name="queued_sync_jobs" + ) + event = models.ForeignKey( + Event, on_delete=models.CASCADE, related_name="queued_sync_jobs" + ) + sync_provider = models.CharField(blank=False, null=False, max_length=128) + triggered_by = models.CharField(blank=False, null=False, max_length=128) + triggered = models.DateTimeField(blank=False, null=False, auto_now_add=True) + failed_attempts = models.PositiveIntegerField(default=0) + not_before = models.DateTimeField(blank=False, null=False, db_index=True) + need_manual_retry = models.CharField(blank=True, null=True, max_length=20, choices=[ + ('exceeded', _('Temporary error, auto-retry limit exceeded')), + ('permanent', _('Provider reported a permanent error')), + ('config', _('Misconfiguration, please check provider settings')), + ('internal', _('System error, needs manual intervention')), + ('timeout', _('System error, needs manual intervention')), + ]) + in_flight = models.BooleanField(default=False) + in_flight_since = models.DateTimeField(blank=True, null=True) + + class Meta: + unique_together = (("order", "sync_provider", "in_flight"),) + ordering = ("triggered",) + + @cached_property + def _provider_class_info(self): + from pretix.base.datasync.datasync import datasync_providers + return datasync_providers.get(identifier=self.sync_provider) + + @property + def provider_class(self): + return self._provider_class_info[0] + + @property + def provider_display_name(self): + return self.provider_class.display_name + + @property + def is_provider_active(self): + return self._provider_class_info[1] + + @property + def max_retry_attempts(self): + return self.provider_class.max_attempts + + def set_sync_error(self, failure_mode, messages, full_message): + logger.exception( + f"Could not sync order {self.order.code} to {type(self).__name__} ({failure_mode})" + ) + self.order.log_action(f"pretix.event.order.data_sync.failed.{failure_mode}", { + "provider": self.sync_provider, + "error": messages, + "full_message": full_message, + }) + self.need_manual_retry = failure_mode + self.clear_in_flight() + + def clear_in_flight(self): + self.in_flight = False + self.in_flight_since = None + try: + self.save() + except IntegrityError: + # if setting in_flight=False fails due to UNIQUE constraint, just delete the current instance + self.delete() + + +class OrderSyncResult(models.Model): + order = models.ForeignKey( + Order, on_delete=models.CASCADE, related_name="sync_results" + ) + sync_provider = models.CharField(blank=False, null=False, max_length=128) + order_position = models.ForeignKey( + OrderPosition, on_delete=models.CASCADE, related_name="sync_results", blank=True, null=True, + ) + mapping_id = models.IntegerField(blank=False, null=False) + external_object_type = models.CharField(blank=False, null=False, max_length=128) + external_id_field = models.CharField(blank=False, null=False, max_length=128) + id_value = models.CharField(blank=False, null=False, max_length=128) + external_link_href = models.CharField(blank=True, null=True, max_length=255) + external_link_display_name = models.CharField(blank=True, null=True, max_length=255) + transmitted = models.DateTimeField(blank=False, null=False, auto_now_add=True) + sync_info = models.JSONField() + + class Meta: + indexes = [ + models.Index(fields=("order", "sync_provider")), + ] + + def external_link_html(self): + if not self.external_link_display_name: + return None + + from pretix.base.datasync.datasync import datasync_providers + prov, meta = datasync_providers.get(identifier=self.sync_provider) + if prov: + return prov.get_external_link_html(self.order.event, self.external_link_href, self.external_link_display_name) + + def to_result_dict(self): + return { + "position": self.order_position_id, + "object_type": self.external_object_type, + "external_id_field": self.external_id_field, + "id_value": self.id_value, + "external_link_href": self.external_link_href, + "external_link_display_name": self.external_link_display_name, + **self.sync_info, + } diff --git a/src/pretix/base/services/datasync.py b/src/pretix/base/services/datasync.py new file mode 100644 index 000000000..a6a7aaf33 --- /dev/null +++ b/src/pretix/base/services/datasync.py @@ -0,0 +1,83 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# + +import logging +from datetime import timedelta +from itertools import groupby + +from django.db.models import F, Window +from django.db.models.functions import RowNumber +from django.dispatch import receiver +from django.utils.timezone import now +from django_scopes import scope, scopes_disabled + +from pretix.base.datasync.datasync import datasync_providers +from pretix.base.models.datasync import OrderSyncQueue +from pretix.base.signals import periodic_task +from pretix.celery_app import app + +logger = logging.getLogger(__name__) + + +@receiver(periodic_task, dispatch_uid="data_sync_periodic_sync_all") +def periodic_sync_all(sender, **kwargs): + sync_all.apply_async() + + +@receiver(periodic_task, dispatch_uid="data_sync_periodic_reset_in_flight") +def periodic_reset_in_flight(sender, **kwargs): + for sq in OrderSyncQueue.objects.filter( + in_flight=True, + in_flight_since__lt=now() - timedelta(minutes=20), + ): + sq.set_sync_error('timeout', [], 'Timeout') + + +@app.task() +def sync_all(): + with scopes_disabled(): + queue = ( + OrderSyncQueue.objects + .filter( + in_flight=False, + not_before__lt=now(), + need_manual_retry__isnull=True, + ) + .order_by(Window( + expression=RowNumber(), + partition_by=[F("event_id")], + order_by="not_before", + )) + .prefetch_related("event") + [:1000] + ) + grouped = groupby(sorted(queue, key=lambda q: (q.sync_provider, q.event.pk)), lambda q: (q.sync_provider, q.event)) + for (target, event), queued_orders in grouped: + target_cls, meta = datasync_providers.get(identifier=target, active_in=event) + + if not target_cls: + # sync plugin not found (plugin deactivated or uninstalled) -> drop outstanding jobs + OrderSyncQueue.objects.filter(pk__in=[sq.pk for sq in queued_orders]).delete() + + with scope(organizer=event.organizer): + with target_cls(event=event) as p: + p.sync_queued_orders(queued_orders) diff --git a/src/pretix/base/signals.py b/src/pretix/base/signals.py index 7b996cffe..252d4ac18 100644 --- a/src/pretix/base/signals.py +++ b/src/pretix/base/signals.py @@ -257,8 +257,14 @@ class Registry: When a new entry is registered, all accessor functions are called with the new entry as parameter. Their return value is stored as the metadata value for that key. """ - self.registered_entries = dict() self.keys = keys + self.clear() + + def clear(self): + """ + Removes all entries from the registry. + """ + self.registered_entries = dict() self.by_key = {key: {} for key in self.keys.keys()} def register(self, *objs): @@ -333,6 +339,23 @@ class EventPluginRegistry(Registry): def __init__(self, keys): super().__init__({"plugin": lambda o: get_defining_app(o), **keys}) + def filter(self, active_in=None, **kwargs): + result = super().filter(**kwargs) + if active_in is not None: + result = ( + (entry, meta) + for entry, meta in result + if is_app_active(active_in, meta['plugin']) + ) + return result + + def get(self, active_in=None, **kwargs): + item, meta = super().get(**kwargs) + if meta and active_in is not None: + if not is_app_active(active_in, meta['plugin']): + return None, None + return item, meta + event_live_issues = EventPluginSignal() """ diff --git a/src/pretix/control/apps.py b/src/pretix/control/apps.py index 1b2148618..05c626b6a 100644 --- a/src/pretix/control/apps.py +++ b/src/pretix/control/apps.py @@ -42,3 +42,4 @@ class PretixControlConfig(AppConfig): def ready(self): from .views import dashboards # noqa from . import logdisplay # noqa + from .views import datasync # noqa diff --git a/src/pretix/control/forms/mapping.py b/src/pretix/control/forms/mapping.py new file mode 100644 index 000000000..2ff14ac27 --- /dev/null +++ b/src/pretix/control/forms/mapping.py @@ -0,0 +1,127 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +import json + +from django import forms +from django.forms import formset_factory +from django.utils.translation import gettext_lazy as _ + +from pretix.base.models import Question +from pretix.base.models.datasync import ( + MODE_APPEND_LIST, MODE_OVERWRITE, MODE_SET_IF_EMPTY, MODE_SET_IF_NEW, +) + + +class PropertyMappingForm(forms.Form): + pretix_field = forms.CharField() + external_field = forms.CharField() + value_map = forms.CharField(required=False) + overwrite = forms.ChoiceField( + choices=[ + (MODE_OVERWRITE, _("Overwrite")), + (MODE_SET_IF_NEW, _("Fill if new")), + (MODE_SET_IF_EMPTY, _("Fill if empty")), + (MODE_APPEND_LIST, _("Add to list")), + ] + ) + + def __init__(self, pretix_fields, external_fields_id, available_modes, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["pretix_field"] = forms.ChoiceField( + label=_("pretix field"), + choices=pretix_fields_choices(pretix_fields, kwargs.get("initial", {}).get("pretix_field")), + required=False, + ) + if external_fields_id: + self.fields["external_field"] = forms.ChoiceField( + widget=forms.Select( + attrs={ + "data-model-select2": "json_script", + "data-select2-src": "#" + external_fields_id, + }, + ), + ) + self.fields["external_field"].choices = [ + (self["external_field"].value(), self["external_field"].value()), + ] + self.fields["overwrite"].choices = [ + (key, label) for (key, label) in self.fields["overwrite"].choices if key in available_modes + ] + + +class PropertyMappingFormSet(formset_factory( + PropertyMappingForm, + can_order=True, + can_delete=True, + extra=0, +)): + template_name = "pretixcontrol/datasync/property_mappings_formset.html" + + def __init__(self, pretix_fields, external_fields, available_modes, prefix, *args, initial_json=None, **kwargs): + if initial_json: + kwargs["initial"] = json.loads(initial_json) + super().__init__( + form_kwargs={ + "pretix_fields": pretix_fields, + "external_fields_id": prefix + "external-fields" if external_fields else None, + "available_modes": available_modes, + }, + prefix=prefix, + *args, **kwargs) + self.external_fields = external_fields + + def get_context(self): + ctx = super().get_context() + ctx["external_fields"] = self.external_fields + ctx["external_fields_id"] = self.prefix + "external-fields" + return ctx + + def to_property_mappings_json(self): + """ + Returns a property mapping configuration as a JSON-serialized list of dictionaries. + + Each entry specifies how to transfer data from one pretix field to one field in the external system: + + - `pretix_field`: Name of a pretix data source field as declared in `pretix.base.datasync.sourcefields.get_data_fields`. + - `external_field`: Name of the target field in the external system. Implementation-defined by the sync provider. + - `value_map`: Dictionary mapping pretix value to external value. Only used for enumeration-type fields. + - `overwrite`: Mode of operation if the object already exists in the target system. + + - `MODE_OVERWRITE` (`"overwrite"`) to always overwrite existing value. + - `MODE_SET_IF_NEW` (`"if_new"`) to only set the value if object does not exist in target system yet. + - `MODE_SET_IF_EMPTY` (`"if_empty"`) to only set the value if object does not exist in target system, + or the field is currently empty in target system. + - `MODE_APPEND_LIST` (`"append"`) if the field is an array or a multi-select: add the value to the list. + """ + mappings = [f.cleaned_data for f in self.ordered_forms] + return json.dumps(mappings) + + +QUESTION_TYPE_LABELS = dict(Question.TYPE_CHOICES) + + +def pretix_fields_choices(pretix_fields, initial_choice): + return [ + (f.key, f.label + " [" + QUESTION_TYPE_LABELS[f.type] + "]") + for f in pretix_fields + if not f.deprecated or f.key == initial_choice + ] diff --git a/src/pretix/control/logdisplay.py b/src/pretix/control/logdisplay.py index 11f1321ab..29fa97478 100644 --- a/src/pretix/control/logdisplay.py +++ b/src/pretix/control/logdisplay.py @@ -43,9 +43,11 @@ from django.dispatch import receiver from django.urls import reverse from django.utils.formats import date_format from django.utils.html import escape, format_html +from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _, pgettext_lazy from i18nfield.strings import LazyI18nString +from pretix.base.datasync.datasync import datasync_providers from pretix.base.logentrytypes import ( DiscountLogEntryType, EventLogEntryType, ItemCategoryLogEntryType, ItemLogEntryType, LogEntryType, OrderLogEntryType, QuestionLogEntryType, @@ -421,6 +423,51 @@ class OrderPrintLogEntryType(OrderLogEntryType): ) +class OrderDataSyncLogEntryType(OrderLogEntryType): + def display(self, logentry, data): + try: + from pretix.base.datasync.datasync import datasync_providers + provider_class, meta = datasync_providers.get(identifier=data['provider']) + data['provider_display_name'] = provider_class.display_name + except (KeyError, AttributeError): + data['provider_display_name'] = data.get('provider') + return super().display(logentry, data) + + +@log_entry_types.new_from_dict({ + "pretix.event.order.data_sync.success": _("Data successfully transferred to {provider_display_name}."), +}) +class OrderDataSyncSuccessLogEntryType(OrderDataSyncLogEntryType): + def display(self, logentry, data): + links = [] + if data.get('provider') and data.get('objects'): + prov, meta = datasync_providers.get(identifier=data['provider']) + if prov: + for objs in data['objects'].values(): + links.append(", ".join( + prov.get_external_link_html(logentry.event, obj['external_link_href'], obj['external_link_display_name']) + for obj in objs + if obj and 'external_link_href' in obj and 'external_link_display_name' in obj + )) + + return mark_safe(escape(super().display(logentry, data)) + "".join("

" + link + "

" for link in links)) + + +@log_entry_types.new_from_dict({ + "pretix.event.order.data_sync.failed.config": _("Transferring data to {provider_display_name} failed due to invalid configuration:"), + "pretix.event.order.data_sync.failed.exceeded": _("Maximum number of retries exceeded while transferring data to {provider_display_name}:"), + "pretix.event.order.data_sync.failed.permanent": _("Error while transferring data to {provider_display_name}:"), + "pretix.event.order.data_sync.failed.internal": _("Internal error while transferring data to {provider_display_name}."), + "pretix.event.order.data_sync.failed.timeout": _("Internal error while transferring data to {provider_display_name}."), +}) +class OrderDataSyncErrorLogEntryType(OrderDataSyncLogEntryType): + def display(self, logentry, data): + errmes = data["error"] + if not isinstance(errmes, list): + errmes = [errmes] + return mark_safe(escape(super().display(logentry, data)) + "".join("

" + escape(msg) + "

" for msg in errmes)) + + @receiver(signal=logentry_display, dispatch_uid="pretixcontrol_logentry_display") def pretixcontrol_logentry_display(sender: Event, logentry: LogEntry, **kwargs): diff --git a/src/pretix/control/navigation.py b/src/pretix/control/navigation.py index bfd5aef7c..cf8e43a79 100644 --- a/src/pretix/control/navigation.py +++ b/src/pretix/control/navigation.py @@ -451,6 +451,11 @@ def get_global_navigation(request): 'url': reverse('control:global.sysreport'), 'active': (url.url_name == 'global.sysreport'), }, + { + 'label': _('Data sync problems'), + 'url': reverse('control:global.datasync.failedjobs'), + 'active': (url.url_name == 'global.datasync.failedjobs'), + }, ] }) @@ -655,6 +660,18 @@ def get_organizer_navigation(request): 'icon': 'download', }) + if 'can_change_organizer_settings' in request.orgapermset: + merge_in(nav, [{ + 'parent': reverse('control:organizer.export', kwargs={ + 'organizer': request.organizer.slug, + }), + 'label': _('Data sync problems'), + 'url': reverse('control:organizer.datasync.failedjobs', kwargs={ + 'organizer': request.organizer.slug, + }), + 'active': (url.url_name == 'organizer.datasync.failedjobs'), + }]) + merge_in(nav, sorted( sum((list(a[1]) for a in nav_organizer.send(request.organizer, request=request, organizer=request.organizer)), []), diff --git a/src/pretix/control/templates/pretixcontrol/datasync/control_order_info.html b/src/pretix/control/templates/pretixcontrol/datasync/control_order_info.html new file mode 100644 index 000000000..156143733 --- /dev/null +++ b/src/pretix/control/templates/pretixcontrol/datasync/control_order_info.html @@ -0,0 +1,74 @@ +{% load i18n %} +{% load eventurl %} +{% load bootstrap3 %} +{% load escapejson %} +
+
+

+ {% trans "Data transfer to external systems" %} +

+
+
    + {% for identifier, display_name, pending, objects in providers %} +
  • +
    + {% csrf_token %} + {% if pending %} + {% if pending.not_before > now or pending.need_manual_retry %} + + {% endif %} + + {% else %} + + + {% endif %} +
    +

    {{ display_name }}

    + {% if pending %} +

    + {% if pending.need_manual_retry %} + + {% trans "Error" %}: {{ pending.get_need_manual_retry_display }} + {% elif pending.failed_attempts %} + + {% blocktrans trimmed with num=pending.failed_attempts max=pending.max_retry_attempts %} + Error. Retry {{ num }} of {{ max }}. + {% endblocktrans %} + {% if pending.not_before %} + {% blocktrans trimmed with datetime=pending.not_before|date:"SHORT_DATETIME_FORMAT" %} + Waiting until {{ datetime }} + {% endblocktrans %} + {% endif %} + {% elif pending.not_before > now %} + {% blocktrans trimmed with datetime=pending.not_before|date:"SHORT_DATETIME_FORMAT" %} + Waiting until {{ datetime }} + {% endblocktrans %} + {% else %} + {% trans "Pending" %} + {% endif %} + ({% blocktrans trimmed with datetime=pending.triggered|date:"SHORT_DATETIME_FORMAT" %}triggered at {{ datetime }} + {% endblocktrans %}) + +

    + {% endif %} + +
      + {% for obj in objects %} +
    • + {% if obj.external_link_html %} + {{ obj.external_link_html }} + {% else %} + {{ obj.external_object_type }} + {% trans "identified by" %} {{ obj.external_id_field }} + {{ obj.id_value }} + {% endif %} +   +
      • + {% empty %} +
    • {% trans "No data transmitted." %}
      • + {% endfor %} +
    +
    • + {% endfor %} +
+
diff --git a/src/pretix/control/templates/pretixcontrol/datasync/failed_jobs.html b/src/pretix/control/templates/pretixcontrol/datasync/failed_jobs.html new file mode 100644 index 000000000..263f87223 --- /dev/null +++ b/src/pretix/control/templates/pretixcontrol/datasync/failed_jobs.html @@ -0,0 +1,86 @@ +{% extends "pretixcontrol/event/base.html" %} +{% load i18n %} + +{% block content %} +

{% trans "Sync problems" %}

+

+ {% blocktrans trimmed %} + On this page, we provide a list of orders where data synchronisation to an external system has failed. + You can start another attempt to sync them manually. + {% endblocktrans %} +

+
+ {% csrf_token %} + + + + + + + + + {% if staff_session %} + + {% endif %} + + + + {% for item in queue_items %} + + + + + + + {% if staff_session %} + + {% endif %} + + {% empty %} + + + {% if staff_session %} + + {% endif %} + + {% endfor %} + + {% if queue_items %} + + + + {% if staff_session %} + + {% endif %} + + + {% endif %} +
+ {% if queue_items %} + + {% endif %} + {% trans "Order" %}{% trans "Sync provider" %}{% trans "Date" %}{% trans "Failure mode" %}in_flightretry
+ {% if staff_session %}{{ item.order.event.organizer.slug }} -{% endif %} + + {{ item.order.full_code }} + + {{ item.provider_display_name }} + {{ item.triggered }} + {% if staff_session %}({{ item.triggered_by }}){% endif %} + + {% if item.need_manual_retry %} + {{ item.get_need_manual_retry_display }} + {% else %} + {% blocktrans trimmed with datetime=item.not_before|date:"SHORT_DATETIME_FORMAT" %} + Temporary error, will retry after {{ datetime }} + {% endblocktrans %} + {% endif %} + {% if staff_session %}({{ item.need_manual_retry }}){% endif %} + {{ item.in_flight }} ({{ item.in_flight_since }}){{ item.failed_attempts }} / {{ item.max_retry_attempts }} ({{ item.not_before }})
{% trans "No problems." %}
+ + +
+
+ {% include "pretixcontrol/pagination.html" %} +{% endblock %} diff --git a/src/pretix/control/templates/pretixcontrol/datasync/property_mappings_formset.html b/src/pretix/control/templates/pretixcontrol/datasync/property_mappings_formset.html new file mode 100644 index 000000000..cba86405c --- /dev/null +++ b/src/pretix/control/templates/pretixcontrol/datasync/property_mappings_formset.html @@ -0,0 +1,81 @@ +{% load i18n %} +{% load bootstrap3 %} +{% load escapejson %} +{% load formset_tags %} +
+ {{ formset.management_form }} + {% bootstrap_formset_errors formset %} +
+ {% for f in formset %} + {% bootstrap_form_errors f %} +
+
+ {{ f.id }} + {% bootstrap_field f.DELETE form_group_class="" layout="inline" %} + {% bootstrap_field f.ORDER form_group_class="" layout="inline" %} +
+
+ {% bootstrap_field f.pretix_field layout='inline' form_group_class="" %} +
+
+ {% bootstrap_field f.external_field layout='inline' form_group_class="" %} +
+
+ {% bootstrap_field f.overwrite layout='inline' form_group_class="" %} +
+ {{ f.value_map.as_hidden }} +
+ + + + + + +
+
+ {% endfor %} +
+ +

+ +

+
+{% if external_fields %} + {{ external_fields|json_script:external_fields_id }} +{% endif %} diff --git a/src/pretix/control/templates/pretixcontrol/event/index.html b/src/pretix/control/templates/pretixcontrol/event/index.html index d89bb08ef..78aeb4f9a 100644 --- a/src/pretix/control/templates/pretixcontrol/event/index.html +++ b/src/pretix/control/templates/pretixcontrol/event/index.html @@ -79,6 +79,15 @@ class="btn btn-primary">{% trans "Show affected orders" %} {% endif %} + {% if has_sync_problems %} +
+ {% blocktrans trimmed %} + Orders in this event could not be synced to an external system as configured. + {% endblocktrans %} + {% trans "Show sync problems" %} +
+ {% endif %} {% eventsignal request.event "pretix.control.signals.event_dashboard_top" request=request %} {% if request.event.has_subevents %} diff --git a/src/pretix/control/urls.py b/src/pretix/control/urls.py index cd6d39a3a..f14286706 100644 --- a/src/pretix/control/urls.py +++ b/src/pretix/control/urls.py @@ -37,9 +37,9 @@ from django.urls import include, re_path from django.views.generic.base import RedirectView from pretix.control.views import ( - auth, checkin, dashboards, discounts, event, geo, global_settings, item, - main, modelimport, oauth, orders, organizer, pdf, search, shredder, - subevents, typeahead, user, users, vouchers, waitinglist, + auth, checkin, dashboards, datasync, discounts, event, geo, + global_settings, item, main, modelimport, oauth, orders, organizer, pdf, + search, shredder, subevents, typeahead, user, users, vouchers, waitinglist, ) urlpatterns = [ @@ -58,6 +58,7 @@ urlpatterns = [ re_path(r'^global/license/$', global_settings.LicenseCheckView.as_view(), name='global.license'), re_path(r'^global/sysreport/$', global_settings.SysReportView.as_view(), name='global.sysreport'), re_path(r'^global/message/$', global_settings.MessageView.as_view(), name='global.message'), + re_path(r'^global/datasync/failedjobs/$', datasync.GlobalFailedSyncJobsView.as_view(), name='global.datasync.failedjobs'), re_path(r'^logdetail/$', global_settings.LogDetailView.as_view(), name='global.logdetail'), re_path(r'^logdetail/payment/$', global_settings.PaymentDetailView.as_view(), name='global.paymentdetail'), re_path(r'^logdetail/refund/$', global_settings.RefundDetailView.as_view(), name='global.refunddetail'), @@ -248,6 +249,7 @@ urlpatterns = [ re_path(r'^organizer/(?P[^/]+)/export/(?P[^/]+)/delete$', organizer.DeleteScheduledExportView.as_view(), name='organizer.export.scheduled.delete'), re_path(r'^organizer/(?P[^/]+)/ticket_select2$', typeahead.ticket_select2, name='organizer.ticket_select2'), + re_path(r'^organizer/(?P[^/]+)/datasync/failedjobs/$', datasync.OrganizerFailedSyncJobsView.as_view(), name='organizer.datasync.failedjobs'), re_path(r'^nav/typeahead/$', typeahead.nav_context_list, name='nav.typeahead'), re_path(r'^events/$', main.EventList.as_view(), name='events'), re_path(r'^events/add$', main.EventWizard.as_view(), name='events.add'), @@ -428,6 +430,8 @@ urlpatterns = [ re_path(r'^orders/(?P[0-9A-Z]+)/cancellationrequests/(?P\d+)/delete$', orders.OrderCancellationRequestDelete.as_view(), name='event.order.cancellationrequests.delete'), + re_path(r'^orders/(?P[0-9A-Z]+)/sync_job/(?P[^/]+)/$', datasync.ControlSyncJob.as_view(), + name='event.order.sync_job'), re_path(r'^orders/(?P[0-9A-Z]+)/transactions/$', orders.OrderTransactions.as_view(), name='event.order.transactions'), re_path(r'^orders/(?P[0-9A-Z]+)/$', orders.OrderDetail.as_view(), name='event.order'), re_path(r'^invoice/(?P[^/]+)$', orders.InvoiceDownload.as_view(), @@ -474,6 +478,7 @@ urlpatterns = [ name='event.orders.checkinlists.edit'), re_path(r'^checkinlists/(?P\d+)/delete$', checkin.CheckinListDelete.as_view(), name='event.orders.checkinlists.delete'), + re_path(r'^datasync/failedjobs/$', datasync.EventFailedSyncJobsView.as_view(), name='event.datasync.failedjobs'), ])), re_path(r'^event/(?P[^/]+)/$', RedirectView.as_view(pattern_name='control:organizer'), name='event.organizerredirect'), ] diff --git a/src/pretix/control/views/dashboards.py b/src/pretix/control/views/dashboards.py index 8b5b47e2a..35932ceaf 100644 --- a/src/pretix/control/views/dashboards.py +++ b/src/pretix/control/views/dashboards.py @@ -383,6 +383,10 @@ def event_index(request, organizer, event): ctx['has_cancellation_requests'] = can_view_orders and CancellationRequest.objects.filter( order__event=request.event ).exists() + ctx['has_sync_problems'] = can_change_event_settings and request.event.queued_sync_jobs.filter( + Q(need_manual_retry__isnull=False) + | Q(failed_attempts__gt=0) + ).exists() ctx['timeline'] = [ { diff --git a/src/pretix/control/views/datasync.py b/src/pretix/control/views/datasync.py new file mode 100644 index 000000000..02a109842 --- /dev/null +++ b/src/pretix/control/views/datasync.py @@ -0,0 +1,150 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# + +from itertools import groupby + +from django.contrib import messages +from django.db.models import Q +from django.dispatch import receiver +from django.http import HttpResponseNotAllowed +from django.shortcuts import redirect +from django.template.loader import get_template +from django.utils.timezone import now +from django.utils.translation import gettext_lazy as _ +from django.views.generic import ListView + +from pretix.base.datasync.datasync import datasync_providers +from pretix.base.models import Event, Order +from pretix.base.models.datasync import OrderSyncQueue +from pretix.control.permissions import ( + AdministratorPermissionRequiredMixin, EventPermissionRequiredMixin, + OrganizerPermissionRequiredMixin, +) +from pretix.control.signals import order_info +from pretix.control.views.orders import OrderView + + +@receiver(order_info, dispatch_uid="datasync_control_order_info") +def on_control_order_info(sender: Event, request, order: Order, **kwargs): + providers = [provider for provider, meta in datasync_providers.filter(active_in=sender)] + if not providers: + return "" + + queued = {p.sync_provider: p for p in order.queued_sync_jobs.all()} + objects = { + provider: list(objects) + for (provider, objects) + in groupby(order.sync_results.order_by('sync_provider').all(), key=lambda o: o.sync_provider) + } + providers = [(provider.identifier, provider.display_name, queued.get(provider.identifier), objects.get(provider.identifier)) for provider in providers] + + template = get_template("pretixcontrol/datasync/control_order_info.html") + ctx = { + "order": order, + "request": request, + "event": sender, + "providers": providers, + "now": now(), + } + return template.render(ctx, request=request) + + +class ControlSyncJob(OrderView): + permission = 'can_change_orders' + + def post(self, request, provider, *args, **kwargs): + prov, meta = datasync_providers.get(active_in=self.request.event, identifier=provider) + + if self.request.POST.get("queue_sync") == "true": + prov.enqueue_order(self.order, 'user') + messages.success(self.request, _('The sync job has been enqueued and will run in the next minutes.')) + elif self.request.POST.get("cancel_job"): + job = self.order.queued_sync_jobs.get(pk=self.request.POST.get("cancel_job")) + if job.in_flight: + messages.warning(self.request, _('The sync job is already in progress.')) + else: + job.delete() + messages.success(self.request, _('The sync job has been canceled.')) + elif self.request.POST.get("run_job_now"): + job = self.order.queued_sync_jobs.get(pk=self.request.POST.get("run_job_now")) + job.not_before = now() + job.need_manual_retry = None + job.save() + messages.success(self.request, _('The sync job has been set to run as soon as possible.')) + + return redirect(self.get_order_url()) + + def get(self, *args, **kwargs): + return HttpResponseNotAllowed(['POST']) + + +class FailedSyncJobsView(ListView): + template_name = 'pretixcontrol/datasync/failed_jobs.html' + model = OrderSyncQueue + context_object_name = 'queue_items' + paginate_by = 100 + ordering = ('triggered',) + + def get_queryset(self): + return super().get_queryset().filter( + Q(need_manual_retry__isnull=False) + | Q(failed_attempts__gt=0) + ).select_related( + 'order' + ) + + def post(self, request, *args, **kwargs): + items = self.get_queryset().filter(pk__in=request.POST.getlist('idlist')) + + if self.request.POST.get("action") == "retry": + for item in items: + item.not_before = now() + item.need_manual_retry = None + item.save() + messages.success(self.request, _('The selected jobs have been set to run as soon as possible.')) + elif self.request.POST.get("action") == "cancel": + items.delete() + messages.success(self.request, _('The selected jobs have been canceled.')) + + return redirect(request.get_full_path()) + + +class GlobalFailedSyncJobsView(AdministratorPermissionRequiredMixin, FailedSyncJobsView): + pass + + +class OrganizerFailedSyncJobsView(OrganizerPermissionRequiredMixin, FailedSyncJobsView): + permission = "can_change_organizer_settings" + + def get_queryset(self): + return super().get_queryset().filter( + event__organizer=self.request.organizer + ) + + +class EventFailedSyncJobsView(EventPermissionRequiredMixin, FailedSyncJobsView): + permission = "can_change_event_settings" + + def get_queryset(self): + return super().get_queryset().filter( + event=self.request.event + ) diff --git a/src/tests/base/test_datasync.py b/src/tests/base/test_datasync.py new file mode 100644 index 000000000..8e55a181d --- /dev/null +++ b/src/tests/base/test_datasync.py @@ -0,0 +1,611 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +import json +from collections import defaultdict, namedtuple +from datetime import timedelta +from decimal import Decimal + +import pytest +from django.utils.timezone import now +from django_scopes import scope + +from pretix.base.datasync.datasync import ( + OutboundSyncProvider, StaticMapping, datasync_providers, +) +from pretix.base.datasync.utils import assign_properties +from pretix.base.models import ( + Event, InvoiceAddress, Item, Order, Organizer, Question, +) +from pretix.base.models.datasync import ( + MODE_APPEND_LIST, MODE_OVERWRITE, MODE_SET_IF_EMPTY, MODE_SET_IF_NEW, +) +from pretix.base.services.datasync import sync_all + + +@pytest.fixture(scope='function') +def event(): + o = Organizer.objects.create(name='Dummy', slug='dummy') + event = Event.objects.create( + organizer=o, name='Dummy', slug='dummy', + date_from=now(), + plugins='pretix.plugins.banktransfer,testplugin' + ) + event.settings.name_scheme = 'given_family' + with scope(organizer=o): + ticket = Item.objects.create(event=event, name='Early-bird ticket', + default_price=Decimal('23.00'), admission=True) + question = ticket.questions.create(question="Whats's your favourite colour?", type=Question.TYPE_STRING, + event=event, required=False, identifier="FAV_COLOR") + question2 = ticket.questions.create(question="Food preference", type=Question.TYPE_CHOICE, + event=event, required=False, identifier="FOOD_PREF") + option1 = question2.options.create(identifier="F1", answer="vegetarian") + option2 = question2.options.create(identifier="F2", answer="vegan") + + o1 = Order.objects.create( + code='1AAA', event=event, email='anonymous@example.org', + status=Order.STATUS_PENDING, locale='en', + datetime=now(), expires=now() + timedelta(days=10), + total=46, + sales_channel=event.organizer.sales_channels.get(identifier="web"), + ) + op1 = o1.positions.create( + item=ticket, variation=None, + price=Decimal("23.00"), attendee_name_parts={'_scheme': 'given_family', 'given_name': "Alice", 'family_name': "Anonymous"}, positionid=1 + ) + op1.answers.create(question=question, answer="#3b1c4a") + op1.answers.create(question=question2, answer="vegan").options.set([option2]) + op2 = o1.positions.create( + item=ticket, variation=None, + price=Decimal("23.00"), attendee_name_parts={'_scheme': 'given_family', 'given_name': "Charlie", 'family_name': "de l'Exemple"}, positionid=2 + ) + op2.answers.create(question=question, answer="Red") + op2.answers.create(question=question2, answer="vegetarian").options.set([option1]) + + o2 = Order.objects.create( + code='2EEE', event=event, email='ephemeral@example.com', + status=Order.STATUS_PENDING, locale='en', + datetime=now(), expires=now() + timedelta(days=10), + total=23, + sales_channel=event.organizer.sales_channels.get(identifier="web"), + ) + o2.positions.create( + item=ticket, variation=None, + price=Decimal("23.00"), attendee_name_parts={'_scheme': 'given_family', 'given_name': "Eve", 'family_name': "Ephemeral"}, positionid=1 + ) + yield event + + +def expected_order_sync_result(): + return { + 'ticketorders': [ + { + '_id': 0, + 'ordernumber': 'DUMMY-1AAA', + 'orderemail': 'anonymous@example.org', + 'status': 'pending', + 'total': '46.00', + 'payment_date': None, + }, + { + '_id': 1, + 'ordernumber': 'DUMMY-2EEE', + 'orderemail': 'ephemeral@example.com', + 'status': 'pending', + 'total': '23.00', + 'payment_date': None, + }, + ], + } + + +def expected_sync_result_with_associations(): + return { + 'tickets': [ + { + '_id': 0, + 'ticketnumber': '1AAA-1', + 'amount': '23.00', + 'firstname': 'Alice', + 'lastname': 'Anonymous', + 'status': 'pending', + 'fav_color': '#3b1c4a', + 'food': 'VEGAN', + 'links': [], + }, + { + '_id': 1, + 'ticketnumber': '1AAA-2', + 'amount': '23.00', + 'firstname': 'Charlie', + 'lastname': "de l'Exemple", + 'status': 'pending', + 'fav_color': 'Red', + 'food': 'VEGETARIAN', + 'links': [], + }, + { + '_id': 2, + 'ticketnumber': '2EEE-1', + 'amount': '23.00', + 'firstname': 'Eve', + 'lastname': 'Ephemeral', + 'status': 'pending', + 'fav_color': '', + 'food': '', + 'links': [], + }, + ], + 'ticketorders': [ + { + '_id': 0, + 'ordernumber': 'DUMMY-1AAA', + 'orderemail': 'anonymous@example.org', + 'firstname': '', + 'lastname': '', + 'status': 'pending', + 'links': ['link:tickets:0', 'link:tickets:1'], + }, + { + '_id': 1, + 'ordernumber': 'DUMMY-2EEE', + 'orderemail': 'ephemeral@example.com', + 'firstname': '', + 'lastname': '', + 'status': 'pending', + 'links': ['link:tickets:2'], + }, + ], + } + + +def _register_with_fake_plugin_name(registry, obj, plugin_name): + registry.clear() + + class App: + name = plugin_name + registry.register(obj) + registry.registered_entries[obj]['plugin'] = App + + +class FakeSyncAPI: + def __init__(self): + self.fake_database = defaultdict(list) + + def retrieve_object(self, table, search_by_attribute, search_for_value): + t = self.fake_database[table] + for idx, record in enumerate(t): + if record.get(search_by_attribute) == search_for_value: + return {**record, "_id": idx} + return None + + def create_or_update_object(self, table, record): + t = self.fake_database[table] + if record.get("_id") is not None: + t[record["_id"]].update(record) + else: + record["_id"] = len(t) + t.append(record) + return record + + +class SimpleOrderSync(OutboundSyncProvider): + identifier = "example1" + fake_api_client = None + + @property + def mappings(self): + return [ + StaticMapping( + id=1, + pretix_model='Order', external_object_type='ticketorders', + pretix_id_field='event_order_code', external_id_field='ordernumber', + property_mappings=json.dumps([ + { + "pretix_field": "email", + "external_field": "orderemail", + "value_map": "", + "overwrite": MODE_OVERWRITE, + }, + { + "pretix_field": "order_status", + "external_field": "status", + "value_map": json.dumps({ + Order.STATUS_PENDING: "pending", + Order.STATUS_PAID: "paid", + Order.STATUS_EXPIRED: "expired", + Order.STATUS_CANCELED: "canceled", + Order.STATUS_REFUNDED: "refunded", + }), + "overwrite": MODE_OVERWRITE, + }, + { + "pretix_field": "order_total", + "external_field": "total", + "value_map": "", + "overwrite": MODE_OVERWRITE, + }, + { + "pretix_field": "payment_date", + "external_field": "payment_date", + "value_map": "", + "overwrite": MODE_OVERWRITE, + }, + ]) + ) + ] + + def sync_object_with_properties( + self, + external_id_field, + id_value, + properties: list, + inputs: dict, + mapping, + mapped_objects: dict, + **kwargs, + ): + pre_existing_object = self.fake_api_client.retrieve_object(mapping.external_object_type, external_id_field, id_value) + update_values = assign_properties(properties, pre_existing_object or {}, is_new=pre_existing_object is None, list_sep=";") + result = self.fake_api_client.create_or_update_object(mapping.external_object_type, { + **update_values, + external_id_field: id_value, + "_id": pre_existing_object and pre_existing_object.get("_id"), + }) + + return { + "object_type": mapping.external_object_type, + "external_id_field": external_id_field, + "id_value": id_value, + "external_link_href": f"https://external-system.example.com/backend/link/to/{mapping.external_object_type}/123/", + "external_link_display_name": "Contact #123 - Jane Doe", + "my_result": result, + } + + +@pytest.mark.django_db +def test_simple_order_sync(event): + _register_with_fake_plugin_name(datasync_providers, SimpleOrderSync, 'testplugin') + + for order in event.orders.order_by("code").all(): + SimpleOrderSync.enqueue_order(order, 'testcase') + + SimpleOrderSync.fake_api_client = FakeSyncAPI() + + sync_all() + + expected = expected_order_sync_result() + assert SimpleOrderSync.fake_api_client.fake_database == expected + + order_1a = event.orders.get(code='1AAA') + paydate = now() + order_1a.payments.create(payment_date=paydate, amount=order_1a.total) + order_1a.status = Order.STATUS_PAID + order_1a.save() + + for order in event.orders.order_by("code").all(): + SimpleOrderSync.enqueue_order(order, 'testcase') + + sync_all() + + expected['ticketorders'][0]['status'] = 'paid' + expected['ticketorders'][0]['payment_date'] = paydate.isoformat() + assert SimpleOrderSync.fake_api_client.fake_database == expected + + +@pytest.mark.django_db +def test_enqueue_order_twice(event): + _register_with_fake_plugin_name(datasync_providers, SimpleOrderSync, 'testplugin') + + for order in event.orders.order_by("code").all(): + SimpleOrderSync.enqueue_order(order, 'testcase_1st') + + for order in event.orders.order_by("code").all(): + SimpleOrderSync.enqueue_order(order, 'testcase_2nd') + + +StaticMappingWithAssociations = namedtuple('StaticMappingWithAssociations', ( + 'id', 'pretix_model', 'external_object_type', 'pretix_id_field', 'external_id_field', 'property_mappings', 'association_mappings' +)) +AssociationMapping = namedtuple('AssociationMapping', ( + 'via_mapping_id' +)) + + +class OrderAndTicketAssociationSync(OutboundSyncProvider): + identifier = "example2" + fake_api_client = None + + @property + def mappings(self): + return [ + StaticMappingWithAssociations( + id=1, + pretix_model='OrderPosition', external_object_type='tickets', + pretix_id_field='ticket_id', external_id_field='ticketnumber', + property_mappings=json.dumps([ + { + "pretix_field": "ticket_price", + "external_field": "amount", + "value_map": "", + "overwrite": MODE_OVERWRITE, + }, + { + "pretix_field": "attendee_name_given_name", + "external_field": "firstname", + "value_map": "", + "overwrite": MODE_OVERWRITE, + }, + { + "pretix_field": "attendee_name_family_name", + "external_field": "lastname", + "value_map": "", + "overwrite": MODE_OVERWRITE, + }, + { + "pretix_field": "order_status", + "external_field": "status", + "value_map": json.dumps({ + Order.STATUS_PENDING: "pending", + Order.STATUS_PAID: "paid", + Order.STATUS_EXPIRED: "expired", + Order.STATUS_CANCELED: "canceled", + Order.STATUS_REFUNDED: "refunded", + }), + "overwrite": MODE_OVERWRITE, + }, + { + "pretix_field": "question_FAV_COLOR", + "external_field": "fav_color", + "value_map": "", + "overwrite": MODE_OVERWRITE, + }, + { + "pretix_field": "question_FOOD_PREF", + "external_field": "food", + "value_map": json.dumps({ + "F1": "VEGETARIAN", + "F2": "VEGAN", + }), + "overwrite": MODE_OVERWRITE, + }, + ]), + association_mappings=[], + ), + StaticMappingWithAssociations( + id=2, + pretix_model='Order', external_object_type='ticketorders', + pretix_id_field='event_order_code', external_id_field='ordernumber', + property_mappings=json.dumps([ + { + "pretix_field": "email", + "external_field": "orderemail", + "value_map": "", + "overwrite": MODE_OVERWRITE, + }, + { + "pretix_field": "invoice_address_name_given_name", + "external_field": "firstname", + "value_map": "", + "overwrite": MODE_OVERWRITE, + }, + { + "pretix_field": "invoice_address_name_family_name", + "external_field": "lastname", + "value_map": "", + "overwrite": MODE_OVERWRITE, + }, + { + "pretix_field": "order_status", + "external_field": "status", + "value_map": json.dumps({ + Order.STATUS_PENDING: "pending", + Order.STATUS_PAID: "paid", + Order.STATUS_EXPIRED: "expired", + Order.STATUS_CANCELED: "canceled", + Order.STATUS_REFUNDED: "refunded", + }), + "overwrite": MODE_OVERWRITE, + }, + ]), + association_mappings=[ + AssociationMapping(via_mapping_id=1) + ], + ), + ] + + def sync_object_with_properties( + self, + external_id_field, + id_value, + properties: list, + inputs: dict, + mapping, + mapped_objects: dict, + **kwargs, + ): + pre_existing_object = self.fake_api_client.retrieve_object(mapping.external_object_type, external_id_field, id_value) + update_values = assign_properties(properties, pre_existing_object or {}, is_new=pre_existing_object is None, list_sep=";") + result = self.fake_api_client.create_or_update_object(mapping.external_object_type, { + **update_values, + external_id_field: id_value, + "_id": pre_existing_object and pre_existing_object.get("_id"), + "links": [ + f"link:{obj.external_object_type}:{obj.sync_info['my_result']['_id']}" + for am in mapping.association_mappings + for obj in mapped_objects[am.via_mapping_id] + ] + }) + + return { + "object_type": mapping.external_object_type, + "external_id_field": external_id_field, + "id_value": id_value, + "external_link_href": f"https://external-system.example.com/backend/link/to/{mapping.external_object_type}/123/", + "external_link_display_name": "Contact #123 - Jane Doe", + "my_result": result, + } + + +@pytest.mark.django_db +def test_association_sync(event): + _register_with_fake_plugin_name(datasync_providers, OrderAndTicketAssociationSync, 'testplugin') + + for order in event.orders.order_by("code").all(): + OrderAndTicketAssociationSync.enqueue_order(order, 'testcase') + + OrderAndTicketAssociationSync.fake_api_client = FakeSyncAPI() + + sync_all() + + expected = expected_sync_result_with_associations() + assert OrderAndTicketAssociationSync.fake_api_client.fake_database == expected + + order_1a = event.orders.get(code='1AAA') + order_1a.status = Order.STATUS_PAID + order_1a.save() + + for order in event.orders.order_by("code").all(): + OrderAndTicketAssociationSync.enqueue_order(order, 'testcase') + + sync_all() + + expected['tickets'][0]['status'] = 'paid' + expected['tickets'][1]['status'] = 'paid' + expected['ticketorders'][0]['status'] = 'paid' + assert OrderAndTicketAssociationSync.fake_api_client.fake_database == expected + + +@pytest.mark.django_db +def test_legacy_name_splitting(event): + _register_with_fake_plugin_name(datasync_providers, OrderAndTicketAssociationSync, 'testplugin') + + for order in event.orders.order_by("code").all(): + OrderAndTicketAssociationSync.enqueue_order(order, 'testcase') + InvoiceAddress.objects.create(order=order, name_parts={'_scheme': 'full', 'full_name': 'A B C D'}) + order.refresh_from_db() + print(order.invoice_address.name_parts) + print(order.invoice_address.name) + + event.settings.name_scheme = 'full' + + OrderAndTicketAssociationSync.fake_api_client = FakeSyncAPI() + + sync_all() + + expected = expected_sync_result_with_associations() + expected['tickets'][1]['firstname'] = "Charlie de" # yes, this splits incorrectly, hence it's legacy + expected['tickets'][1]['lastname'] = "l'Exemple" + expected['ticketorders'][1]['firstname'] = "A B C" + expected['ticketorders'][1]['lastname'] = "D" + assert OrderAndTicketAssociationSync.fake_api_client.fake_database == expected + + +def test_assign_properties(): + assert assign_properties( + [("name", "Alice", MODE_OVERWRITE)], {"name": "A"}, is_new=False, list_sep=";" + ) == {"name": "Alice"} + assert ( + assign_properties([("name", "Alice", MODE_SET_IF_NEW)], {}, is_new=False, list_sep=";") == {} + ) + assert assign_properties([("name", "Alice", MODE_SET_IF_NEW)], {}, is_new=True, list_sep=";") == { + "name": "Alice" + } + assert assign_properties( + [ + ("name", "Alice", MODE_SET_IF_NEW), + ("name", "A", MODE_SET_IF_NEW), + ], + {}, + is_new=True, + list_sep=";", + ) == {"name": "Alice"} + assert ( + assign_properties( + [ + ("name", "Alice", MODE_SET_IF_NEW), + ("name", "A", MODE_SET_IF_NEW), + ], + {"name": "Bob"}, + is_new=False, + list_sep=";", + ) + == {} + ) + assert ( + assign_properties( + [ + ("name", "Alice", MODE_SET_IF_NEW), + ("name", "A", MODE_SET_IF_NEW), + ], + {}, + is_new=False, + list_sep=";", + ) + == {} + ) + assert assign_properties( + [ + ("name", "Alice", MODE_SET_IF_EMPTY), + ("name", "A", MODE_SET_IF_EMPTY), + ], + {}, + is_new=True, + list_sep=";", + ) == {"name": "Alice"} + assert ( + assign_properties( + [ + ("name", "Alice", MODE_SET_IF_EMPTY), + ("name", "A", MODE_SET_IF_EMPTY), + ], + {"name": "Bob"}, + is_new=False, + list_sep=";", + ) + == {} + ) + assert assign_properties( + [("name", "Alice", MODE_SET_IF_EMPTY)], {}, is_new=False, list_sep=";" + ) == {"name": "Alice"} + + assert assign_properties( + [("name", "Alice", MODE_SET_IF_EMPTY)], {}, is_new=False, list_sep=";" + ) == {"name": "Alice"} + + assert assign_properties( + [("colors", "red", MODE_APPEND_LIST)], {}, is_new=False, list_sep=";" + ) == {"colors": "red"} + assert assign_properties( + [("colors", "red", MODE_APPEND_LIST)], {"colors": "red"}, is_new=False, list_sep=";" + ) == {} + assert assign_properties( + [("colors", "red", MODE_APPEND_LIST)], {"colors": "blue"}, is_new=False, list_sep=";" + ) == {"colors": "blue;red"} + assert assign_properties( + [("colors", "red", MODE_APPEND_LIST)], {"colors": "green;blue"}, is_new=False, list_sep=";" + ) == {"colors": "green;blue;red"} + assert assign_properties( + [("colors", "red", MODE_APPEND_LIST)], {"colors": ["green", "blue"]}, is_new=False, list_sep=None + ) == {"colors": ["green", "blue", "red"]} + assert assign_properties( + [("colors", "green", MODE_APPEND_LIST)], {"colors": ["green", "blue"]}, is_new=False, list_sep=None + ) == {}