From a0c69bb480c40854cebdc9e15b82d6bf3895d6cc Mon Sep 17 00:00:00 2001 From: Mira Weller Date: Thu, 3 Apr 2025 12:28:28 +0200 Subject: [PATCH] refactor: move some utils into core --- src/pretix/base/datasync/__init__.py | 21 +++++ src/pretix/base/datasync/datasync.py | 58 +++++++++++--- src/pretix/base/datasync/utils.py | 45 +++++++++++ src/pretix/control/forms/mapping.py | 9 ++- src/tests/base/test_datasync.py | 110 +++++++++++++++++++++++++++ 5 files changed, 233 insertions(+), 10 deletions(-) create mode 100644 src/pretix/base/datasync/utils.py create mode 100644 src/tests/base/test_datasync.py diff --git a/src/pretix/base/datasync/__init__.py b/src/pretix/base/datasync/__init__.py index e69de29bb2..9fd5bdc500 100644 --- a/src/pretix/base/datasync/__init__.py +++ b/src/pretix/base/datasync/__init__.py @@ -0,0 +1,21 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# diff --git a/src/pretix/base/datasync/datasync.py b/src/pretix/base/datasync/datasync.py index 46481f0166..8a21eb3a68 100644 --- a/src/pretix/base/datasync/datasync.py +++ b/src/pretix/base/datasync/datasync.py @@ -150,7 +150,6 @@ StaticMapping = namedtuple('StaticMapping', ('pk', 'pretix_model', 'external_obj class OutboundSyncProvider: max_attempts = 5 - syncer_class = None def __init__(self, event): self.event = event @@ -197,9 +196,39 @@ class OutboundSyncProvider: info = cls.get_external_link_info(event, external_link_href, external_link_display_name) return make_link(info, '{val}') + """ + Optionally override to configure a different retry backoff behavior + """ def next_retry_date(self, sq): return datetime.now() + timedelta(hours=1) + """ + Optionally override this method to exclude certain orders from sync by returning False + """ + def order_valid_for_sync(self, order): + return True + + """ + Implementations must override this property to provide the data mappings as a list of objects. + + They can return instances of the StaticMapping namedtuple defined above, or create their own + class (e.g. a Django model). + + The returned objects must have at least the following properties: + - pk: unique identifier + - pretix_model: which pretix model to use as data source in this mapping. possible values are + the keys of sourcefields.AVAILABLE_MODELS + - external_object_type: destination object type in the target system. opaque string of maximum 128 characters. + - pretix_pk: which pretix data field should be used to identify the mapped object. any + DataFieldInfo.key returned by sourcefields.get_data_fields for the combination of + Event and pretix_model + - external_pk: destination identifier field in the target system + - property_mapping: mapping configuration as generated by PropertyMappingFormSet.to_property_mapping_json() + """ + @property + def mappings(self): + raise NotImplementedError + def sync_queued_orders(self, queued_orders): for sq in queued_orders: try: @@ -240,13 +269,6 @@ class OutboundSyncProvider: }) sq.delete() - def order_valid_for_sync(self, order): - return True - - @property - def mappings(self): - raise NotImplementedError - @cached_property def data_fields(self): return { @@ -284,7 +306,25 @@ class OutboundSyncProvider: This method is called for each object that needs to be created/updated in the external system -- which these are is determined by the implementation of the `mapping` property. - TODO: describe the parameters + :param pk_field: Identifier field in the target system as provided in mapping.external_pk + :param pk_value: Identifier contents as retrieved from the property specified by mapping.pretix_pk of the model + specified by mapping.pretix_model + :param properties: All properties defined in mapping.property_mapping, as list of three-tuples + (external_field, value, overwrite) + :param inputs: All pretix model instances from which data can be retrieved for this mapping + :param mapping: The mapping object as returned by self.mappings + :param mapped_objects: Information about objects that were synced in the same sync run, by mapping definitions + *before* the current one in order of self.mappings. + Type is a dictionary {mapping.pk: [list of return values of this method]} + Useful to create associations between objects in the target system. + :return: { + "object_type": mapping.external_object_type, + "pk_field": pk_field, + "pk_value": pk_value, + "external_link_href": "https://external-system.example.com/backend/link/to/contact/123/", + "external_link_display_name": "Contact #123 - Jane Doe", + "...optionally further values you need in mapped_objects for association": 123456789, + } This method needs to be idempotent, i.e. calling it multiple times with the same input values should create only a single object in the target system. diff --git a/src/pretix/base/datasync/utils.py b/src/pretix/base/datasync/utils.py new file mode 100644 index 0000000000..2ce6af4a72 --- /dev/null +++ b/src/pretix/base/datasync/utils.py @@ -0,0 +1,45 @@ +from typing import List, Tuple + +from pretix.base.datasync.datasync import ( + MODE_APPEND_LIST, MODE_OVERWRITE, MODE_SET_IF_EMPTY, MODE_SET_IF_NEW, + SyncConfigError, +) + + +def assign_properties( + new_values: List[Tuple[str, str, str]], old_values: dict, is_new=True, list_sep=";", +): + out = {} + + for k, v, mode in new_values: + if mode == MODE_OVERWRITE: + out[k] = v + continue + elif mode == MODE_SET_IF_NEW and not is_new: + continue + if not v: + continue + + current_value = old_values.get(k, out.get(k, "")) + if mode in (MODE_SET_IF_EMPTY, MODE_SET_IF_NEW): + if not current_value: + out[k] = v + elif mode == MODE_APPEND_LIST: + _add_to_list(out, k, current_value, v, list_sep) + else: + raise SyncConfigError(["Invalid update mode " + mode]) + return out + + +def _add_to_list(out, key, current_value, new_item, list_sep): + new_item = str(new_item) + if list_sep is not None: + new_item = new_item.replace(list_sep, "") + current_value = current_value.split(list_sep) if current_value else [] + else: + current_value = list(current_value) + if new_item not in current_value: + new_list = current_value + [new_item] + if list_sep is not None: + new_list = list_sep.join(new_list) + out[key] = new_list diff --git a/src/pretix/control/forms/mapping.py b/src/pretix/control/forms/mapping.py index 3dd5a1a7c8..dccb071756 100644 --- a/src/pretix/control/forms/mapping.py +++ b/src/pretix/control/forms/mapping.py @@ -19,6 +19,7 @@ # You should have received a copy of the GNU Affero General Public License along with this program. If not, see # . # +import json from django import forms from django.forms import formset_factory @@ -76,7 +77,9 @@ class PropertyMappingFormSet(formset_factory( )): template_name = "pretixcontrol/datasync/property_mapping_formset.html" - def __init__(self, pretix_fields, external_fields, available_modes, prefix, *args, **kwargs): + def __init__(self, pretix_fields, external_fields, available_modes, prefix, *args, initial_json=None, **kwargs): + if initial_json: + kwargs["initial"] = json.loads(initial_json) super().__init__( form_kwargs={ "pretix_fields": pretix_fields, @@ -93,6 +96,10 @@ class PropertyMappingFormSet(formset_factory( ctx["external_fields_id"] = self.prefix + "external-fields" return ctx + def to_property_mapping_json(self): + mappings = [f.cleaned_data for f in self.ordered_forms] + return json.dumps(mappings) + def pretix_fields_choices(pretix_fields, initial_choice): return [ diff --git a/src/tests/base/test_datasync.py b/src/tests/base/test_datasync.py new file mode 100644 index 0000000000..7766c93f56 --- /dev/null +++ b/src/tests/base/test_datasync.py @@ -0,0 +1,110 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +from pretix.base.datasync.datasync import MODE_OVERWRITE, MODE_SET_IF_EMPTY, MODE_SET_IF_NEW, MODE_APPEND_LIST +from pretix.base.datasync.utils import assign_properties + + +def test_assign_properties(): + assert assign_properties( + [("name", "Alice", MODE_OVERWRITE)], {"name": "A"}, is_new=False + ) == {"name": "Alice"} + assert ( + assign_properties([("name", "Alice", MODE_SET_IF_NEW)], {}, is_new=False) == {} + ) + assert assign_properties([("name", "Alice", MODE_SET_IF_NEW)], {}, is_new=True) == { + "name": "Alice" + } + assert assign_properties( + [ + ("name", "Alice", MODE_SET_IF_NEW), + ("name", "A", MODE_SET_IF_NEW), + ], + {}, + is_new=True, + ) == {"name": "Alice"} + assert ( + assign_properties( + [ + ("name", "Alice", MODE_SET_IF_NEW), + ("name", "A", MODE_SET_IF_NEW), + ], + {"name": "Bob"}, + is_new=False, + ) + == {} + ) + assert ( + assign_properties( + [ + ("name", "Alice", MODE_SET_IF_NEW), + ("name", "A", MODE_SET_IF_NEW), + ], + {}, + is_new=False, + ) + == {} + ) + assert assign_properties( + [ + ("name", "Alice", MODE_SET_IF_EMPTY), + ("name", "A", MODE_SET_IF_EMPTY), + ], + {}, + is_new=True, + ) == {"name": "Alice"} + assert ( + assign_properties( + [ + ("name", "Alice", MODE_SET_IF_EMPTY), + ("name", "A", MODE_SET_IF_EMPTY), + ], + {"name": "Bob"}, + is_new=False, + ) + == {} + ) + assert assign_properties( + [("name", "Alice", MODE_SET_IF_EMPTY)], {}, is_new=False + ) == {"name": "Alice"} + + assert assign_properties( + [("name", "Alice", MODE_SET_IF_EMPTY)], {}, is_new=False + ) == {"name": "Alice"} + + assert assign_properties( + [("colors", "red", MODE_APPEND_LIST)], {}, is_new=False + ) == {"colors": "red"} + assert assign_properties( + [("colors", "red", MODE_APPEND_LIST)], {"colors": "red"}, is_new=False + ) == {"colors": "red"} + assert assign_properties( + [("colors", "red", MODE_APPEND_LIST)], {"colors": "blue"}, is_new=False + ) == {"colors": "blue;red"} + assert assign_properties( + [("colors", "red", MODE_APPEND_LIST)], {"colors": "green;blue"}, is_new=False + ) == {"colors": "green;blue;red"} + assert assign_properties( + [("colors", "red", MODE_APPEND_LIST)], {"colors": ["green","blue"]}, is_new=False, list_sep=None + ) == {"colors": ["green", "blue", "red"]} + assert assign_properties( + [("colors", "green", MODE_APPEND_LIST)], {"colors": ["green","blue"]}, is_new=False, list_sep=None + ) == {"colors": ["green", "blue"]}