API: Generalize concept of including/excluding/expanding fields

doc/api/fundamentals.rst

@@ -203,9 +203,35 @@ Query parameters
 Most list endpoints allow a filtering of the results using query parameters. In this case, booleans should be passed
 as the string values ``true`` and ``false``.
 
+Ordering
+--------
+
 If the ``ordering`` parameter is documented for a resource, you can use it to sort the result set by one of the allowed
 fields. Prepend a ``-`` to the field name to reverse the sort order.
 
+Filtering and expanding fields
+------------------------------
+
+On many endpoints, you can modify what fields are being returned:
+
+- Using the ``include`` query parameter, you can chose which fields will be returned as part of the response.
+  For example, if you pass ``include=code&include=email`` to the list of orders, you will receive a list of only
+  order codes and email addresses.
+
+- Using the ``exclude`` query parameter, you can chose which fields will not be returned as part of the response.
+  For example, if you pass ``exclude=payments&exclude=refunds`` to the list of orders, you will receive a list
+  without the payment and refund objects.
+
+- Using the ``expand`` query parameter, you can chose which fields will be expanded into full objects. For example,
+  if you pass ``expand=voucher`` to the list of order positions, the response will contain a full voucher object
+  instead of just the ID. If you do not have permission to view vouchers, a 403 status code is returned.
+  For performance reasons, this option is only available for a limited number of fields that are noted as
+  "expandable" in the documentation of the respective object.
+
+In all of these, you can use dotted notation to address fields of sub-objects, such as ``positions.checkins.gate``.
+
+These options are not available everywhere as we are slowly rolling them out throughout the codebase. Please check
+the individual endpoint documentation for availability.
 
 Idempotency
 -----------

doc/api/resources/organizers.rst

@@ -61,6 +61,8 @@ Endpoints
    :query page: The page number in case of a multi-page result set, default is 1
    :query string ordering: Manually set the ordering of results. Valid fields to be used are ``slug`` and
                            ``name``. Default: ``slug``.
+   :query string include: Limit the output to the given field. Can be passed multiple times.
+   :query string exclude: Exclude a field from the output. Can be passed multiple times.
    :statuscode 200: no error
    :statuscode 401: Authentication failure
 

src/pretix/api/serializers/__init__.py

@@ -23,7 +23,7 @@ import json
 
 from django.db.models import prefetch_related_objects
 from rest_framework import serializers
-from rest_framework.exceptions import ValidationError
+from rest_framework.exceptions import PermissionDenied, ValidationError
 
 
 class AsymmetricField(serializers.Field):
@@ -135,3 +135,117 @@ class SalesChannelMigrationMixin:
         else:
             value["sales_channels"] = value["limit_sales_channels"]
         return value
+
+
+class ConfigurableSerializerMixin:
+    expand_fields = {}
+
+    def get_exclude_requests(self):
+        if hasattr(self, "initial_data"):
+            # Do not support include requests when the serializer is used for writing
+            # TODO: think about this
+            return set()
+        if 'exclude' in self.context:
+            return self.context['exclude']
+        elif 'request' in self.context:
+            return self.context['request'].query_params.getlist('exclude')
+        raise TypeError("Could not discover list of fields to exclude")
+
+    def get_include_requests(self):
+        if hasattr(self, "initial_data"):
+            # Do not support include requests when the serializer is used for writing
+            # TODO: think about this
+            return set()
+        if 'include' in self.context:
+            return self.context['include']
+        elif 'request' in self.context:
+            return self.context['request'].query_params.getlist('include')
+        raise TypeError("Could not discover list of fields to include")
+
+    def get_expand_requests(self):
+        if hasattr(self, "initial_data"):
+            # Do not support expand requests when the serializer is used for writing
+            # TODO: think about this
+            return set()
+        if 'expand' in self.context:
+            return self.context['expand']
+        elif 'request' in self.context:
+            return self.context['request'].query_params.getlist('expand')
+        raise TypeError("Could not discover list of fields to expand")
+
+    def _exclude_field(self, serializer, path):
+        if path[0] not in serializer.fields:
+            return  # field does not exist, nothing to do
+
+        if len(path) == 1:
+            del serializer.fields[path[0]]
+        elif len(path) >= 2 and hasattr(serializer.fields[path[0]], "child"):
+            self._exclude_field(serializer.fields[path[0]].child, path[1:])
+        elif len(path) >= 2 and isinstance(serializer.fields[path[0]], serializers.Serializer):
+            self._exclude_field(serializer.fields[path[0]], path[1:])
+
+    def _filter_fields_to_included(self, serializer, includes):
+        any_field_remaining = False
+        for fname, field in list(serializer.fields.items()):
+            if fname in includes:
+                any_field_remaining = True
+                continue
+            elif hasattr(field, 'child'):  # Nested list serializers
+                child_includes = {i.removeprefix(f'{fname}.') for i in includes if i.startswith(f'{fname}.')}
+                if child_includes and self._filter_fields_to_included(field.child, child_includes):
+                    any_field_remaining = True
+                    continue
+                serializer.fields.pop(fname)
+            elif isinstance(field, serializers.Serializer):  # Nested serializers
+                child_includes = {i.removeprefix(f'{fname}.') for i in includes if i.startswith(f'{fname}.')}
+                if child_includes and self._filter_fields_to_included(field, child_includes):
+                    any_field_remaining = True
+                    continue
+                serializer.fields.pop(fname)
+            else:
+                serializer.fields.pop(fname)
+        return any_field_remaining
+
+    def _expand_field(self, serializer, path, original_field):
+        if path[0] not in serializer.fields or not self.is_field_expandable(original_field):
+            return False  # field does not exist, nothing to do
+
+        if len(path) == 1:
+            serializer.fields[path[0]] = self.get_expand_serializer(original_field)
+            return True
+        elif len(path) >= 2 and hasattr(serializer.fields[path[0]], "child"):
+            return self._expand_field(serializer.fields[path[0]].child, path[1:], original_field)
+        elif len(path) >= 2 and isinstance(serializer.fields[path[0]], serializers.Serializer):
+            return self._expand_field(serializer.fields[path[0]], path[1:], original_field)
+
+    def is_field_expandable(self, field):
+        return field in self.expand_fields
+
+    def get_expand_serializer(self, field):
+        from pretix.base.models import Device, TeamAPIToken
+
+        ef = self.expand_fields[field]
+        if "permission" in ef:
+            request = self.context["request"]
+            perm_holder = request.auth if isinstance(request.auth, (Device, TeamAPIToken)) else request.user
+            if not perm_holder.has_event_permission(request.organizer, request.event, ef["permission"], request=request):
+                raise PermissionDenied(f"No permission to expand field {field}")
+
+        return ef["serializer"](
+            read_only=True,
+            context=self.context,
+        )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        expanded = False
+        for expand in sorted(list(self.get_expand_requests())):
+            expanded = expanded or self._expand_field(self, expand.split('.'), expand)
+
+        includes = set(self.get_include_requests())
+        if includes:
+            self._filter_fields_to_included(self, includes)
+
+        for exclude_field in self.get_exclude_requests():
+            self._exclude_field(self, exclude_field.split('.'))

src/pretix/api/serializers/checkin.py

@@ -23,15 +23,19 @@ from django.utils.translation import gettext as _
 from rest_framework import serializers
 from rest_framework.exceptions import ValidationError
 
+from pretix.api.serializers import ConfigurableSerializerMixin
 from pretix.api.serializers.event import SubEventSerializer
 from pretix.api.serializers.i18n import I18nAwareModelSerializer
 from pretix.base.media import MEDIA_TYPES
 from pretix.base.models import Checkin, CheckinList
 
 
-class CheckinListSerializer(I18nAwareModelSerializer):
+class CheckinListSerializer(ConfigurableSerializerMixin, I18nAwareModelSerializer):
     checkin_count = serializers.IntegerField(read_only=True)
     position_count = serializers.IntegerField(read_only=True)
+    expand_fields = {
+        "subevent": SubEventSerializer,
+    }
 
     class Meta:
         model = CheckinList
@@ -42,17 +46,6 @@ class CheckinListSerializer(I18nAwareModelSerializer):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        if 'subevent' in self.context['request'].query_params.getlist('expand'):
-            self.fields['subevent'] = SubEventSerializer(read_only=True)
-
-        for exclude_field in self.context['request'].query_params.getlist('exclude'):
-            p = exclude_field.split('.')
-            if p[0] in self.fields:
-                if len(p) == 1:
-                    del self.fields[p[0]]
-                elif len(p) == 2:
-                    self.fields[p[0]].child.fields.pop(p[1])
-
     def validate(self, data):
         data = super().validate(data)
         event = self.context['event']

src/pretix/api/serializers/order.py

@@ -40,12 +40,15 @@ from rest_framework.exceptions import ValidationError
 from rest_framework.relations import SlugRelatedField
 from rest_framework.reverse import reverse
 
-from pretix.api.serializers import CompatibleJSONField
+from pretix.api.serializers import (
+    CompatibleJSONField, ConfigurableSerializerMixin,
+)
 from pretix.api.serializers.event import SubEventSerializer
 from pretix.api.serializers.i18n import I18nAwareModelSerializer
 from pretix.api.serializers.item import (
     InlineItemVariationSerializer, ItemSerializer, QuestionSerializer,
 )
+from pretix.api.serializers.voucher import VoucherSerializer
 from pretix.api.signals import order_api_details, orderposition_api_details
 from pretix.base.decimal import round_decimal
 from pretix.base.i18n import language
@@ -175,7 +178,7 @@ class AnswerSerializer(I18nAwareModelSerializer):
 
     def to_representation(self, instance):
         r = super().to_representation(instance)
-        if r['answer'].startswith('file://') and instance.orderposition:
+        if r.get('answer') and r.get('answer').startswith('file://') and instance.orderposition:
             r['answer'] = reverse('api-v1:orderposition-answer', kwargs={
                 'organizer': instance.orderposition.order.event.organizer.slug,
                 'event': instance.orderposition.order.event.slug,
@@ -757,7 +760,7 @@ class OrderPluginDataField(serializers.Field):
         return d
 
 
-class OrderSerializer(I18nAwareModelSerializer):
+class OrderSerializer(ConfigurableSerializerMixin, I18nAwareModelSerializer):
     event = SlugRelatedField(slug_field='slug', read_only=True)
     invoice_address = InvoiceAddressSerializer(allow_null=True)
     positions = OrderPositionSerializer(many=True, read_only=True)
@@ -775,6 +778,12 @@ class OrderSerializer(I18nAwareModelSerializer):
         required=False,
     )
     plugin_data = OrderPluginDataField(source='*', allow_null=True, read_only=True)
+    expand_fields = {
+        "positions.voucher": {
+            "serializer": VoucherSerializer,
+            "permission": "can_view_vouchers",
+        },
+    }
 
     class Meta:
         model = Order
@@ -793,47 +802,14 @@ class OrderSerializer(I18nAwareModelSerializer):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        if "organizer" in self.context:
+        if "sales_channel" in self.fields:
-            self.fields["sales_channel"].queryset = self.context["organizer"].sales_channels.all()
+            if "organizer" in self.context:
-        else:
+                self.fields["sales_channel"].queryset = self.context["organizer"].sales_channels.all()
-            self.fields["sales_channel"].queryset = self.context["event"].organizer.sales_channels.all()
+            else:
-        if not self.context['pdf_data']:
+                self.fields["sales_channel"].queryset = self.context["event"].organizer.sales_channels.all()
+        if not self.context['pdf_data'] and "positions" in self.fields:
             self.fields['positions'].child.fields.pop('pdf_data', None)
 
-        includes = set(self.context['include'])
-        if includes:
-            for fname, field in list(self.fields.items()):
-                if fname in includes:
-                    continue
-                elif hasattr(field, 'child'):  # Nested list serializers
-                    found_any = False
-                    for childfname, childfield in list(field.child.fields.items()):
-                        if f'{fname}.{childfname}' not in includes:
-                            field.child.fields.pop(childfname)
-                        else:
-                            found_any = True
-                    if not found_any:
-                        self.fields.pop(fname)
-                elif isinstance(field, serializers.Serializer):  # Nested serializers
-                    found_any = False
-                    for childfname, childfield in list(field.fields.items()):
-                        if f'{fname}.{childfname}' not in includes:
-                            field.fields.pop(childfname)
-                        else:
-                            found_any = True
-                    if not found_any:
-                        self.fields.pop(fname)
-                else:
-                    self.fields.pop(fname)
-
-        for exclude_field in self.context['exclude']:
-            p = exclude_field.split('.')
-            if p[0] in self.fields:
-                if len(p) == 1:
-                    del self.fields[p[0]]
-                elif len(p) == 2:
-                    self.fields[p[0]].child.fields.pop(p[1])
-
     def validate_locale(self, l):
         if l not in set(k for k in self.instance.event.settings.locales):
             raise ValidationError('"{}" is not a supported locale for this event.'.format(l))

src/pretix/api/serializers/organizer.py

@@ -31,7 +31,7 @@ from rest_framework import serializers
 from rest_framework.exceptions import ValidationError
 
 from pretix.api.auth.devicesecurity import get_all_security_profiles
-from pretix.api.serializers import AsymmetricField
+from pretix.api.serializers import AsymmetricField, ConfigurableSerializer
 from pretix.api.serializers.i18n import I18nAwareModelSerializer
 from pretix.api.serializers.order import CompatibleJSONField
 from pretix.api.serializers.settings import SettingsSerializer
@@ -51,7 +51,7 @@ from pretix.multidomain.urlreverse import build_absolute_uri
 logger = logging.getLogger(__name__)
 
 
-class OrganizerSerializer(I18nAwareModelSerializer):
+class OrganizerSerializer(ConfigurableSerializer, I18nAwareModelSerializer):
     public_url = serializers.SerializerMethodField('get_organizer_url', read_only=True)
 
     def get_organizer_url(self, organizer):

src/tests/api/test_orders.py

@@ -31,6 +31,7 @@ from django.utils.timezone import now
 from django_countries.fields import Country
 from django_scopes import scopes_disabled
 from stripe import error
+from tests.api.utils import _test_configurable_serializer
 from tests.plugins.stripe.test_checkout import apple_domain_create
 from tests.plugins.stripe.test_provider import MockedCharge
 
@@ -400,13 +401,18 @@ def test_order_list_filter_subevent_date(token_client, device, organizer, event,
 
 
 @pytest.mark.django_db
-def test_order_list(token_client, organizer, event, order, item, taxrule, question, device):
+def test_order_list(token_client, organizer, event, order, item, team, taxrule, question, device):
     res = dict(TEST_ORDER_RES)
     with scopes_disabled():
+        voucher = event.vouchers.create(code="FOO")
+        opos = order.positions.first()
+        opos.voucher = voucher
+        opos.save()
         res["positions"][0]["id"] = order.positions.first().pk
         res["fees"][0]["id"] = order.fees.first().pk
         res["positions"][0]["print_logs"][0]["id"] = order.positions.first().print_logs.first().pk
         res["positions"][0]["print_logs"][0]["device_id"] = device.device_id
+        res["positions"][0]["voucher"] = voucher.pk
     res["positions"][0]["item"] = item.pk
     res["positions"][0]["answers"][0]["question"] = question.pk
     res["last_modified"] = order.last_modified.isoformat().replace('+00:00', 'Z')
@@ -514,6 +520,22 @@ def test_order_list(token_client, organizer, event, order, item, taxrule, questi
     assert resp.status_code == 200
     assert len(resp.data['results'][0]['fees']) == 2
 
+    _test_configurable_serializer(
+        token_client,
+        "/api/v1/organizers/{}/events/{}/orders/".format(organizer.slug, event.slug),
+        [
+            "status", "invoice_address.company", "fees.value", "payments.state",
+            "positions.print_logs.type", "positions.answers.answer"
+        ],
+        expands=["positions.voucher"],
+    )
+
+    team.can_view_vouchers = False
+    team.save()
+    resp = token_client.get('/api/v1/organizers/{}/events/{}/orders/?expand=positions.voucher'.format(organizer.slug, event.slug))
+    assert resp.status_code == 200
+    assert resp.data == "No permission to expand field positions.voucher"
+
 
 @pytest.mark.django_db
 def test_order_detail(token_client, organizer, event, order, item, taxrule, question):

src/tests/api/utils.py

@@ -0,0 +1,88 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020 Raphael Michel and contributors
+# Copyright (C) 2020-2021 rami.io GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+import urllib.parse
+
+
+def _add_params(url, params):
+    url_parts = list(urllib.parse.urlparse(url))
+    query = urllib.parse.parse_qs(url_parts[4])
+    query = [*query, *params]
+    url_parts[4] = urllib.parse.urlencode(query)
+    return urllib.parse.urlunparse(url_parts)
+
+
+def _find_field_names(d: dict, path):
+    names = set()
+    for k, v in d.items():
+        if isinstance(v, dict):
+            names |= _find_field_names(v, path=(*path, k))
+        elif isinstance(v, list) and len(v) > 0:
+            names |= _find_field_names(v[0], path=(*path, k))
+        else:
+            names.add(".".join([*path, k]))
+    return names
+
+
+def _test_configurable_serializer(client, url, field_name_samples, expands):
+    # Test include
+    resp = client.get(_add_params(url, [("include", f) for f in field_name_samples]))
+    if "results" in resp.data:
+        o = resp.data["results"][0]
+    else:
+        o = resp.data
+
+    found_field_names = _find_field_names(o, tuple())
+    # Assert no unexpected fields
+    for f in found_field_names:
+        depth = f.count(".")
+        assert f in field_name_samples or any(f.rsplit(".", c)[0] in field_name_samples for c in range(depth))
+    # Assert all fields are there
+    for f in field_name_samples:
+        assert f in found_field_names
+
+    # Test exclude
+    resp = client.get(_add_params(url, [("exclude", f) for f in field_name_samples]))
+    if "results" in resp.data:
+        o = resp.data["results"][0]
+    else:
+        o = resp.data
+    found_field_names = _find_field_names(o, [])
+    # Assert all fields are not there
+    for f in found_field_names:
+        assert f not in field_name_samples
+
+    # Test expand
+    if expands:
+        resp = client.get(_add_params(url, [("expand", f) for f in expands]))
+        if "results" in resp.data:
+            o = resp.data["results"][0]
+        else:
+            o = resp.data
+        for e in expands:
+            path = e.split(".")
+            obj = o
+            while len(path) > 1:
+                obj = o[path[0]]
+                if isinstance(obj, list):
+                    obj = obj[0]
+                path = path[1:]
+            assert isinstance(obj[path[0]], dict)