Add more sourcefields for datasync (#5378)

* add email domain field

* add order and ticket URL fields

* add "is admission product" field

* fix types

* Display sourcefields grouped into categories (#5379)

src/pretix/base/datasync/sourcefields.py

@@ -28,6 +28,7 @@ from django.utils.translation import gettext_lazy as _
 
 from pretix.base.models import Checkin, InvoiceAddress, Order, Question
 from pretix.base.settings import PERSON_NAME_SCHEMES
+from pretix.multidomain.urlreverse import build_absolute_uri
 
 
 def get_answer(op, question_identifier=None):
@@ -102,6 +103,14 @@ def normalize_email(email):
         return None
 
 
+def get_email_domain(email):
+    if email:
+        local, host = email.split("@")
+        return host
+    else:
+        return None
+
+
 ORDER_POSITION = 'position'
 ORDER = 'order'
 EVENT = 'event'
@@ -111,10 +120,23 @@ AVAILABLE_MODELS = {
     'Order': (ORDER, EVENT),
 }
 
+DataFieldCategory = namedtuple(
+    'DataFieldCategory',
+    field_names=('sort_index', 'label',),
+)
+
+CAT_ORDER_POSITION = DataFieldCategory(10, _('Order position details'))
+CAT_ATTENDEE = DataFieldCategory(11, _('Attendee details'))
+CAT_QUESTIONS = DataFieldCategory(12, _('Questions'))
+CAT_PRODUCT = DataFieldCategory(20, _('Product details'))
+CAT_ORDER = DataFieldCategory(21, _('Order details'))
+CAT_INVOICE_ADDRESS = DataFieldCategory(22, _('Invoice address'))
+CAT_EVENT = DataFieldCategory(30, _('Event information'))
+CAT_EVENT_OR_SUBEVENT = DataFieldCategory(31, _('Event or subevent information'))
 
 DataFieldInfo = namedtuple(
     'DataFieldInfo',
-    field_names=('required_input', 'key', 'label', 'type', 'enum_opts', 'getter', 'deprecated'),
+    field_names=('required_input', 'category', 'key', 'label', 'type', 'enum_opts', 'getter', 'deprecated'),
     defaults=[False]
 )
 
@@ -148,6 +170,7 @@ def get_data_fields(event, for_model=None):
         [
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ATTENDEE,
                 "attendee_name",
                 _("Attendee name"),
                 Question.TYPE_STRING,
@@ -159,6 +182,7 @@ def get_data_fields(event, for_model=None):
         + [
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ATTENDEE,
                 "attendee_name_" + k,
                 _("Attendee") + ": " + label,
                 Question.TYPE_STRING,
@@ -178,6 +202,7 @@ def get_data_fields(event, for_model=None):
         + [
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ATTENDEE,
                 "attendee_email",
                 _("Attendee email"),
                 Question.TYPE_STRING,
@@ -189,6 +214,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ATTENDEE,
                 "attendee_or_order_email",
                 _("Attendee or order email"),
                 Question.TYPE_STRING,
@@ -201,6 +227,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ATTENDEE,
                 "attendee_company",
                 _("Attendee company"),
                 Question.TYPE_STRING,
@@ -209,6 +236,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ATTENDEE,
                 "attendee_street",
                 _("Attendee address street"),
                 Question.TYPE_STRING,
@@ -217,6 +245,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ATTENDEE,
                 "attendee_zipcode",
                 _("Attendee address ZIP code"),
                 Question.TYPE_STRING,
@@ -225,6 +254,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ATTENDEE,
                 "attendee_city",
                 _("Attendee address city"),
                 Question.TYPE_STRING,
@@ -233,6 +263,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ATTENDEE,
                 "attendee_country",
                 _("Attendee address country"),
                 Question.TYPE_COUNTRYCODE,
@@ -243,6 +274,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_INVOICE_ADDRESS,
                 "invoice_address_company",
                 _("Invoice address company"),
                 Question.TYPE_STRING,
@@ -251,6 +283,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_INVOICE_ADDRESS,
                 "invoice_address_name",
                 _("Invoice address name"),
                 Question.TYPE_STRING,
@@ -261,6 +294,7 @@ def get_data_fields(event, for_model=None):
         + [
             DataFieldInfo(
                 ORDER,
+                CAT_INVOICE_ADDRESS,
                 "invoice_address_name_" + k,
                 _("Invoice address") + ": " + label,
                 Question.TYPE_STRING,
@@ -278,6 +312,7 @@ def get_data_fields(event, for_model=None):
         + [
             DataFieldInfo(
                 ORDER,
+                CAT_INVOICE_ADDRESS,
                 "invoice_address_street",
                 _("Invoice address street"),
                 Question.TYPE_STRING,
@@ -286,6 +321,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_INVOICE_ADDRESS,
                 "invoice_address_zipcode",
                 _("Invoice address ZIP code"),
                 Question.TYPE_STRING,
@@ -294,6 +330,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_INVOICE_ADDRESS,
                 "invoice_address_city",
                 _("Invoice address city"),
                 Question.TYPE_STRING,
@@ -302,6 +339,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_INVOICE_ADDRESS,
                 "invoice_address_country",
                 _("Invoice address country"),
                 Question.TYPE_COUNTRYCODE,
@@ -310,6 +348,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_ORDER,
                 "email",
                 _("Order email"),
                 Question.TYPE_STRING,
@@ -318,6 +357,16 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_ORDER,
+                "email_domain",
+                _("Order email domain"),
+                Question.TYPE_STRING,
+                None,
+                lambda order: get_email_domain(normalize_email(order.email)),
+            ),
+            DataFieldInfo(
+                ORDER,
+                CAT_ORDER,
                 "order_code",
                 _("Order code"),
                 Question.TYPE_STRING,
@@ -326,6 +375,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_ORDER,
                 "event_order_code",
                 _("Event and order code"),
                 Question.TYPE_STRING,
@@ -334,6 +384,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_ORDER,
                 "order_total",
                 _("Order total"),
                 Question.TYPE_NUMBER,
@@ -342,6 +393,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_PRODUCT,
                 "product",
                 _("Product and variation name"),
                 Question.TYPE_STRING,
@@ -353,14 +405,25 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_PRODUCT,
                 "product_id",
                 _("Product ID"),
                 Question.TYPE_NUMBER,
                 None,
                 lambda position: str(position.item.pk),
             ),
+            DataFieldInfo(
+                ORDER_POSITION,
+                CAT_PRODUCT,
+                "product_is_admission",
+                _("Product is admission product"),
+                Question.TYPE_BOOLEAN,
+                None,
+                lambda position: bool(position.item.admission),
+            ),
             DataFieldInfo(
                 EVENT,
+                CAT_EVENT,
                 "event_slug",
                 _("Event short form"),
                 Question.TYPE_STRING,
@@ -369,6 +432,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 EVENT,
+                CAT_EVENT,
                 "event_name",
                 _("Event name"),
                 Question.TYPE_STRING,
@@ -377,6 +441,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 EVENT_OR_SUBEVENT,
+                CAT_EVENT_OR_SUBEVENT,
                 "event_date_from",
                 _("Event start date"),
                 Question.TYPE_DATETIME,
@@ -385,6 +450,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 EVENT_OR_SUBEVENT,
+                CAT_EVENT_OR_SUBEVENT,
                 "event_date_to",
                 _("Event end date"),
                 Question.TYPE_DATETIME,
@@ -393,6 +459,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ORDER_POSITION,
                 "voucher_code",
                 _("Voucher code"),
                 Question.TYPE_STRING,
@@ -401,6 +468,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ORDER_POSITION,
                 "ticket_id",
                 _("Order code and position number"),
                 Question.TYPE_STRING,
@@ -409,6 +477,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ORDER_POSITION,
                 "ticket_price",
                 _("Ticket price"),
                 Question.TYPE_NUMBER,
@@ -417,6 +486,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_ORDER,
                 "order_status",
                 _("Order status"),
                 Question.TYPE_CHOICE,
@@ -425,6 +495,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ORDER_POSITION,
                 "ticket_status",
                 _("Ticket status"),
                 Question.TYPE_CHOICE,
@@ -433,6 +504,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_ORDER,
                 "order_date",
                 _("Order date and time"),
                 Question.TYPE_DATETIME,
@@ -441,6 +513,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_ORDER,
                 "payment_date",
                 _("Payment date and time"),
                 Question.TYPE_DATETIME,
@@ -449,6 +522,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_ORDER,
                 "order_locale",
                 _("Order language code"),
                 Question.TYPE_CHOICE,
@@ -457,16 +531,49 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ORDER_POSITION,
                 "position_id",
                 _("Order position ID"),
                 Question.TYPE_NUMBER,
                 None,
                 lambda op: str(op.pk),
             ),
+            DataFieldInfo(
+                ORDER,
+                CAT_ORDER,
+                "presale_order_url",
+                _("Order URL"),
+                Question.TYPE_STRING,
+                None,
+                lambda order: build_absolute_uri(
+                    event,
+                    'presale:event.order', kwargs={
+                        'order': order.code,
+                        'secret': order.secret,
+                    }
+                ),
+            ),
+            DataFieldInfo(
+                ORDER_POSITION,
+                CAT_ORDER_POSITION,
+                "presale_ticket_url",
+                _("Ticket URL"),
+                Question.TYPE_STRING,
+                None,
+                lambda op: build_absolute_uri(
+                    event,
+                    'presale:event.order.position', kwargs={
+                        'order': op.order.code,
+                        'secret': op.web_secret,
+                        'position': op.positionid
+                    }
+                ),
+            ),
         ]
         + [
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ORDER_POSITION,
                 "checkin_date_" + str(cl.pk),
                 _("Check-in datetime on list {}").format(cl.name),
                 Question.TYPE_DATETIME,
@@ -478,6 +585,7 @@ def get_data_fields(event, for_model=None):
         + [
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_QUESTIONS,
                 "question_" + q.identifier,
                 _("Question: {name}").format(name=str(q.question)),
                 q.type,
@@ -491,6 +599,7 @@ def get_data_fields(event, for_model=None):
         src_fields += [
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ATTENDEE,
                 "attendee_name_given_name",
                 _("Attendee") + ": " + _("Given name") + " (⚠️ auto-generated, not recommended)",
                 Question.TYPE_STRING,
@@ -500,6 +609,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_INVOICE_ADDRESS,
                 "invoice_address_name_given_name",
                 _("Invoice address") + ": " + _("Given name") + " (⚠️ auto-generated, not recommended)",
                 Question.TYPE_STRING,
@@ -513,6 +623,7 @@ def get_data_fields(event, for_model=None):
         src_fields += [
             DataFieldInfo(
                 ORDER_POSITION,
+                CAT_ATTENDEE,
                 "attendee_name_family_name",
                 _("Attendee") + ": " + _("Family name") + " (⚠️ auto-generated, not recommended)",
                 Question.TYPE_STRING,
@@ -522,6 +633,7 @@ def get_data_fields(event, for_model=None):
             ),
             DataFieldInfo(
                 ORDER,
+                CAT_INVOICE_ADDRESS,
                 "invoice_address_name_family_name",
                 _("Invoice address") + ": " + _("Family name") + " (⚠️ auto-generated, not recommended)",
                 Question.TYPE_STRING,

src/pretix/control/forms/mapping.py

@@ -19,6 +19,8 @@
 # You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
 # <https://www.gnu.org/licenses/>.
 #
+from itertools import groupby
+
 from django import forms
 from django.forms import formset_factory
 from django.utils.translation import gettext_lazy as _
@@ -116,8 +118,13 @@ QUESTION_TYPE_LABELS = dict(Question.TYPE_CHOICES)
 
 
 def pretix_fields_choices(pretix_fields, initial_choice):
+    pretix_fields = sorted(pretix_fields, key=lambda f: f.category)
+    grouped_fields = groupby(pretix_fields, lambda f: f.category)
     return [
-        (f.key, f.label + " [" + QUESTION_TYPE_LABELS[f.type] + "]")
+        (f"{cat}", [
-        for f in pretix_fields
+            (f.key, f.label + " [" + QUESTION_TYPE_LABELS[f.type] + "]")
-        if not f.deprecated or f.key == initial_choice
+            for f in fields
+            if not f.deprecated or f.key == initial_choice
+        ])
+        for ((idx, cat), fields) in grouped_fields
     ]