refactor: move some utils into core

src/pretix/base/datasync/__init__.py

@@ -0,0 +1,21 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020 Raphael Michel and contributors
+# Copyright (C) 2020-2021 rami.io GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#

src/pretix/base/datasync/datasync.py

@@ -150,7 +150,6 @@ StaticMapping = namedtuple('StaticMapping', ('pk', 'pretix_model', 'external_obj
 
 class OutboundSyncProvider:
     max_attempts = 5
-    syncer_class = None
 
     def __init__(self, event):
         self.event = event
@@ -197,9 +196,39 @@ class OutboundSyncProvider:
         info = cls.get_external_link_info(event, external_link_href, external_link_display_name)
         return make_link(info, '{val}')
 
+    """
+    Optionally override to configure a different retry backoff behavior
+    """
     def next_retry_date(self, sq):
         return datetime.now() + timedelta(hours=1)
 
+    """
+    Optionally override this method to exclude certain orders from sync by returning False
+    """
+    def order_valid_for_sync(self, order):
+        return True
+
+    """
+    Implementations must override this property to provide the data mappings as a list of objects.
+
+    They can return instances of the StaticMapping namedtuple defined above, or create their own
+    class (e.g. a Django model).
+
+    The returned objects must have at least the following properties:
+    - pk:                   unique identifier
+    - pretix_model:         which pretix model to use as data source in this mapping. possible values are
+                            the keys of sourcefields.AVAILABLE_MODELS
+    - external_object_type: destination object type in the target system. opaque string of maximum 128 characters.
+    - pretix_pk:            which pretix data field should be used to identify the mapped object. any
+                            DataFieldInfo.key returned by sourcefields.get_data_fields for the combination of
+                            Event and pretix_model
+    - external_pk:          destination identifier field in the target system
+    - property_mapping:     mapping configuration as generated by PropertyMappingFormSet.to_property_mapping_json()
+    """
+    @property
+    def mappings(self):
+        raise NotImplementedError
+
     def sync_queued_orders(self, queued_orders):
         for sq in queued_orders:
             try:
@@ -240,13 +269,6 @@ class OutboundSyncProvider:
                 })
                 sq.delete()
 
-    def order_valid_for_sync(self, order):
-        return True
-
-    @property
-    def mappings(self):
-        raise NotImplementedError
-
     @cached_property
     def data_fields(self):
         return {
@@ -284,7 +306,25 @@ class OutboundSyncProvider:
     This method is called for each object that needs to be created/updated in the external system -- which these are is
     determined by the implementation of the `mapping` property.
 
-    TODO: describe the parameters
+    :param pk_field: Identifier field in the target system as provided in mapping.external_pk
+    :param pk_value: Identifier contents as retrieved from the property specified by mapping.pretix_pk of the model
+                     specified by mapping.pretix_model
+    :param properties: All properties defined in mapping.property_mapping, as list of three-tuples
+                       (external_field, value, overwrite)
+    :param inputs: All pretix model instances from which data can be retrieved for this mapping
+    :param mapping: The mapping object as returned by self.mappings
+    :param mapped_objects: Information about objects that were synced in the same sync run, by mapping definitions
+                           *before* the current one in order of self.mappings.
+                           Type is a dictionary {mapping.pk: [list of return values of this method]}
+                           Useful to create associations between objects in the target system.
+    :return: {
+                "object_type": mapping.external_object_type,
+                "pk_field": pk_field,
+                "pk_value": pk_value,
+                "external_link_href": "https://external-system.example.com/backend/link/to/contact/123/",
+                "external_link_display_name": "Contact #123 - Jane Doe",
+                "...optionally further values you need in mapped_objects for association": 123456789,
+             }
 
     This method needs to be idempotent, i.e. calling it multiple times with the same input values should create
     only a single object in the target system.

src/pretix/base/datasync/utils.py

@@ -0,0 +1,45 @@
+from typing import List, Tuple
+
+from pretix.base.datasync.datasync import (
+    MODE_APPEND_LIST, MODE_OVERWRITE, MODE_SET_IF_EMPTY, MODE_SET_IF_NEW,
+    SyncConfigError,
+)
+
+
+def assign_properties(
+    new_values: List[Tuple[str, str, str]], old_values: dict, is_new=True, list_sep=";",
+):
+    out = {}
+
+    for k, v, mode in new_values:
+        if mode == MODE_OVERWRITE:
+            out[k] = v
+            continue
+        elif mode == MODE_SET_IF_NEW and not is_new:
+            continue
+        if not v:
+            continue
+
+        current_value = old_values.get(k, out.get(k, ""))
+        if mode in (MODE_SET_IF_EMPTY, MODE_SET_IF_NEW):
+            if not current_value:
+                out[k] = v
+        elif mode == MODE_APPEND_LIST:
+            _add_to_list(out, k, current_value, v, list_sep)
+        else:
+            raise SyncConfigError(["Invalid update mode " + mode])
+    return out
+
+
+def _add_to_list(out, key, current_value, new_item, list_sep):
+    new_item = str(new_item)
+    if list_sep is not None:
+        new_item = new_item.replace(list_sep, "")
+        current_value = current_value.split(list_sep) if current_value else []
+    else:
+        current_value = list(current_value)
+    if new_item not in current_value:
+        new_list = current_value + [new_item]
+        if list_sep is not None:
+            new_list = list_sep.join(new_list)
+        out[key] = new_list

src/pretix/control/forms/mapping.py

@@ -19,6 +19,7 @@
 # You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
 # <https://www.gnu.org/licenses/>.
 #
+import json
 
 from django import forms
 from django.forms import formset_factory
@@ -76,7 +77,9 @@ class PropertyMappingFormSet(formset_factory(
 )):
     template_name = "pretixcontrol/datasync/property_mapping_formset.html"
 
-    def __init__(self, pretix_fields, external_fields, available_modes, prefix, *args, **kwargs):
+    def __init__(self, pretix_fields, external_fields, available_modes, prefix, *args, initial_json=None, **kwargs):
+        if initial_json:
+            kwargs["initial"] = json.loads(initial_json)
         super().__init__(
             form_kwargs={
                 "pretix_fields": pretix_fields,
@@ -93,6 +96,10 @@ class PropertyMappingFormSet(formset_factory(
         ctx["external_fields_id"] = self.prefix + "external-fields"
         return ctx
 
+    def to_property_mapping_json(self):
+        mappings = [f.cleaned_data for f in self.ordered_forms]
+        return json.dumps(mappings)
+
 
 def pretix_fields_choices(pretix_fields, initial_choice):
     return [

src/tests/base/test_datasync.py

@@ -0,0 +1,110 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020 Raphael Michel and contributors
+# Copyright (C) 2020-2021 rami.io GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+from pretix.base.datasync.datasync import MODE_OVERWRITE, MODE_SET_IF_EMPTY, MODE_SET_IF_NEW, MODE_APPEND_LIST
+from pretix.base.datasync.utils import assign_properties
+
+
+def test_assign_properties():
+    assert assign_properties(
+        [("name", "Alice", MODE_OVERWRITE)], {"name": "A"}, is_new=False
+    ) == {"name": "Alice"}
+    assert (
+        assign_properties([("name", "Alice", MODE_SET_IF_NEW)], {}, is_new=False) == {}
+    )
+    assert assign_properties([("name", "Alice", MODE_SET_IF_NEW)], {}, is_new=True) == {
+        "name": "Alice"
+    }
+    assert assign_properties(
+        [
+            ("name", "Alice", MODE_SET_IF_NEW),
+            ("name", "A", MODE_SET_IF_NEW),
+        ],
+        {},
+        is_new=True,
+    ) == {"name": "Alice"}
+    assert (
+        assign_properties(
+            [
+                ("name", "Alice", MODE_SET_IF_NEW),
+                ("name", "A", MODE_SET_IF_NEW),
+            ],
+            {"name": "Bob"},
+            is_new=False,
+        )
+        == {}
+    )
+    assert (
+        assign_properties(
+            [
+                ("name", "Alice", MODE_SET_IF_NEW),
+                ("name", "A", MODE_SET_IF_NEW),
+            ],
+            {},
+            is_new=False,
+        )
+        == {}
+    )
+    assert assign_properties(
+        [
+            ("name", "Alice", MODE_SET_IF_EMPTY),
+            ("name", "A", MODE_SET_IF_EMPTY),
+        ],
+        {},
+        is_new=True,
+    ) == {"name": "Alice"}
+    assert (
+        assign_properties(
+            [
+                ("name", "Alice", MODE_SET_IF_EMPTY),
+                ("name", "A", MODE_SET_IF_EMPTY),
+            ],
+            {"name": "Bob"},
+            is_new=False,
+        )
+        == {}
+    )
+    assert assign_properties(
+        [("name", "Alice", MODE_SET_IF_EMPTY)], {}, is_new=False
+    ) == {"name": "Alice"}
+
+    assert assign_properties(
+        [("name", "Alice", MODE_SET_IF_EMPTY)], {}, is_new=False
+    ) == {"name": "Alice"}
+
+    assert assign_properties(
+        [("colors", "red", MODE_APPEND_LIST)], {}, is_new=False
+    ) == {"colors": "red"}
+    assert assign_properties(
+        [("colors", "red", MODE_APPEND_LIST)], {"colors": "red"}, is_new=False
+    ) == {"colors": "red"}
+    assert assign_properties(
+        [("colors", "red", MODE_APPEND_LIST)], {"colors": "blue"}, is_new=False
+    ) == {"colors": "blue;red"}
+    assert assign_properties(
+        [("colors", "red", MODE_APPEND_LIST)], {"colors": "green;blue"}, is_new=False
+    ) == {"colors": "green;blue;red"}
+    assert assign_properties(
+        [("colors", "red", MODE_APPEND_LIST)], {"colors": ["green","blue"]}, is_new=False, list_sep=None
+    ) == {"colors": ["green", "blue", "red"]}
+    assert assign_properties(
+        [("colors", "green", MODE_APPEND_LIST)], {"colors": ["green","blue"]}, is_new=False, list_sep=None
+    ) == {"colors": ["green", "blue"]}