Fix #1780 -- Trigger exports through API (#1839)

src/pretix/api/serializers/exporters.py

@@ -0,0 +1,110 @@
+from django import forms
+from rest_framework import serializers
+
+
+class FormFieldWrapperField(serializers.Field):
+    def __init__(self, *args, **kwargs):
+        self.form_field = kwargs.pop('form_field')
+        super().__init__(*args, **kwargs)
+
+    def to_representation(self, value):
+        return self.form_field.widget.format_value(value)
+
+    def to_internal_value(self, data):
+        d = self.form_field.widget.value_from_datadict({'name': data}, {}, 'name')
+        d = self.form_field.clean(d)
+        return d
+
+
+simple_mappings = (
+    (forms.DateField, serializers.DateField, tuple()),
+    (forms.TimeField, serializers.TimeField, tuple()),
+    (forms.SplitDateTimeField, serializers.DateTimeField, tuple()),
+    (forms.DateTimeField, serializers.DateTimeField, tuple()),
+    (forms.DecimalField, serializers.DecimalField, ('max_digits', 'decimal_places', 'min_value', 'max_value')),
+    (forms.FloatField, serializers.FloatField, tuple()),
+    (forms.IntegerField, serializers.IntegerField, tuple()),
+    (forms.EmailField, serializers.EmailField, tuple()),
+    (forms.UUIDField, serializers.UUIDField, tuple()),
+    (forms.URLField, serializers.URLField, tuple()),
+    (forms.NullBooleanField, serializers.NullBooleanField, tuple()),
+    (forms.BooleanField, serializers.BooleanField, tuple()),
+)
+
+
+class SerializerDescriptionField(serializers.Field):
+    def to_representation(self, value):
+        fields = []
+        for k, v in value.fields.items():
+            d = {
+                'name': k,
+                'required': v.required,
+            }
+            if isinstance(v, serializers.ChoiceField):
+                d['choices'] = list(v.choices.keys())
+            fields.append(d)
+
+        return fields
+
+
+class ExporterSerializer(serializers.Serializer):
+    identifier = serializers.CharField()
+    verbose_name = serializers.CharField()
+    input_parameters = SerializerDescriptionField(source='_serializer')
+
+
+class JobRunSerializer(serializers.Serializer):
+    def __init__(self, *args, **kwargs):
+        ex = kwargs.pop('exporter')
+        events = kwargs.pop('events', None)
+        super().__init__(*args, **kwargs)
+        if events is not None:
+            self.fields["events"] = serializers.SlugRelatedField(
+                queryset=events,
+                required=True,
+                allow_empty=False,
+                slug_field='slug',
+                many=True
+            )
+        for k, v in ex.export_form_fields.items():
+            for m_from, m_to, m_kwargs in simple_mappings:
+                if isinstance(v, m_from):
+                    self.fields[k] = m_to(
+                        required=v.required,
+                        allow_null=not v.required,
+                        validators=v.validators,
+                        **{kwarg: getattr(v, kwargs, None) for kwarg in m_kwargs}
+                    )
+                    break
+
+            if isinstance(v, forms.ModelMultipleChoiceField):
+                self.fields[k] = serializers.PrimaryKeyRelatedField(
+                    queryset=v.queryset,
+                    required=v.required,
+                    allow_empty=not v.required,
+                    validators=v.validators,
+                    many=True
+                )
+            elif isinstance(v, forms.ModelChoiceField):
+                self.fields[k] = serializers.PrimaryKeyRelatedField(
+                    queryset=v.queryset,
+                    required=v.required,
+                    allow_null=not v.required,
+                    validators=v.validators,
+                )
+            elif isinstance(v, forms.MultipleChoiceField):
+                self.fields[k] = serializers.MultipleChoiceField(
+                    choices=v.choices,
+                    required=v.required,
+                    allow_empty=not v.required,
+                    validators=v.validators,
+                )
+            elif isinstance(v, forms.ChoiceField):
+                self.fields[k] = serializers.ChoiceField(
+                    choices=v.choices,
+                    required=v.required,
+                    allow_null=not v.required,
+                    validators=v.validators,
+                )
+            else:
+                self.fields[k] = FormFieldWrapperField(form_field=v, required=v.required, allow_null=not v.required)

src/pretix/api/urls.py

@@ -7,8 +7,8 @@ from rest_framework import routers
 from pretix.api.views import cart
 
 from .views import (
-    checkin, device, event, item, oauth, order, organizer, user, version,
-    voucher, waitinglist, webhooks,
+    checkin, device, event, exporters, item, oauth, order, organizer, user,
+    version, voucher, waitinglist, webhooks,
 )
 
 router = routers.DefaultRouter()
@@ -22,6 +22,7 @@ orga_router.register(r'seatingplans', organizer.SeatingPlanViewSet)
 orga_router.register(r'giftcards', organizer.GiftCardViewSet)
 orga_router.register(r'teams', organizer.TeamViewSet)
 orga_router.register(r'devices', organizer.DeviceViewSet)
+orga_router.register(r'exporters', exporters.OrganizerExportersViewSet, basename='exporters')
 
 team_router = routers.DefaultRouter()
 team_router.register(r'members', organizer.TeamMemberViewSet)
@@ -44,6 +45,7 @@ event_router.register(r'taxrules', event.TaxRuleViewSet)
 event_router.register(r'waitinglistentries', waitinglist.WaitingListViewSet)
 event_router.register(r'checkinlists', checkin.CheckinListViewSet)
 event_router.register(r'cartpositions', cart.CartPositionViewSet)
+event_router.register(r'exporters', exporters.EventExportersViewSet, basename='exporters')
 
 checkinlist_router = routers.DefaultRouter()
 checkinlist_router.register(r'positions', checkin.CheckinListPositionViewSet, basename='checkinlistpos')

src/pretix/api/views/exporters.py

@@ -0,0 +1,154 @@
+from datetime import timedelta
+
+from celery.result import AsyncResult
+from django.conf import settings
+from django.http import Http404
+from django.shortcuts import get_object_or_404
+from django.utils.functional import cached_property
+from django.utils.timezone import now
+from rest_framework import status, viewsets
+from rest_framework.decorators import action
+from rest_framework.response import Response
+from rest_framework.reverse import reverse
+
+from pretix.api.serializers.exporters import (
+    ExporterSerializer, JobRunSerializer,
+)
+from pretix.base.models import CachedFile, Device, TeamAPIToken
+from pretix.base.services.export import export, multiexport
+from pretix.base.signals import (
+    register_data_exporters, register_multievent_data_exporters,
+)
+from pretix.helpers.http import ChunkBasedFileResponse
+
+
+class ExportersMixin:
+    def list(self, request, *args, **kwargs):
+        res = ExporterSerializer(self.exporters, many=True)
+        return Response({
+            "count": len(self.exporters),
+            "next": None,
+            "previous": None,
+            "results": res.data
+        })
+
+    def get_object(self):
+        instances = [e for e in self.exporters if e.identifier == self.kwargs.get('pk')]
+        if not instances:
+            raise Http404()
+        return instances[0]
+
+    def retrieve(self, request, *args, **kwargs):
+        instance = self.get_object()
+        serializer = ExporterSerializer(instance)
+        return Response(serializer.data)
+
+    @action(detail=True, methods=['GET'], url_name='download', url_path='download/(?P<asyncid>[^/]+)/(?P<cfid>[^/]+)')
+    def download(self, *args, **kwargs):
+        cf = get_object_or_404(CachedFile, id=kwargs['cfid'])
+        if cf.file:
+            resp = ChunkBasedFileResponse(cf.file.file, content_type=cf.type)
+            resp['Content-Disposition'] = 'attachment; filename="{}"'.format(cf.filename)
+            return resp
+        elif not settings.HAS_CELERY:
+            return Response(
+                {'status': 'failed', 'message': 'Unknown file ID or export failed'},
+                status=status.HTTP_410_GONE
+            )
+
+        res = AsyncResult(kwargs['asyncid'])
+        if res.failed():
+            if isinstance(res.info, dict) and res.info['exc_type'] == 'ExportError':
+                msg = res.info['exc_message']
+            else:
+                msg = 'Internal error'
+            return Response(
+                {'status': 'failed', 'message': msg},
+                status=status.HTTP_410_GONE
+            )
+
+        return Response(
+            {
+                'status': 'running' if res.state in ('PROGRESS', 'STARTED', 'SUCCESS') else 'waiting',
+                'percentage': res.result.get('value', None) if res.result else None,
+            },
+            status=status.HTTP_409_CONFLICT
+        )
+
+    @action(detail=True, methods=['POST'])
+    def run(self, *args, **kwargs):
+        instance = self.get_object()
+        serializer = JobRunSerializer(exporter=instance, data=self.request.data, **self.get_serializer_kwargs())
+        serializer.is_valid(raise_exception=True)
+
+        cf = CachedFile()
+        cf.date = now()
+        cf.expires = now() + timedelta(days=3)
+        cf.save()
+        d = serializer.data
+        for k, v in d.items():
+            if isinstance(v, set):
+                d[k] = list(v)
+        async_result = self.do_export(cf, instance, d)
+
+        url_kwargs = {
+            'asyncid': str(async_result.id),
+            'cfid': str(cf.id),
+        }
+        url_kwargs.update(self.kwargs)
+        return Response({
+            'download': reverse('api-v1:exporters-download', kwargs=url_kwargs, request=self.request)
+        }, status=status.HTTP_202_ACCEPTED)
+
+
+class EventExportersViewSet(ExportersMixin, viewsets.ViewSet):
+    permission = 'can_view_orders'
+
+    def get_serializer_kwargs(self):
+        return {}
+
+    @cached_property
+    def exporters(self):
+        exporters = []
+        responses = register_data_exporters.send(self.request.event)
+        for ex in sorted([response(self.request.event) for r, response in responses], key=lambda ex: str(ex.verbose_name)):
+            ex._serializer = JobRunSerializer(exporter=ex)
+            exporters.append(ex)
+        return exporters
+
+    def do_export(self, cf, instance, data):
+        return export.apply_async(args=(self.request.event.id, str(cf.id), instance.identifier, data))
+
+
+class OrganizerExportersViewSet(ExportersMixin, viewsets.ViewSet):
+    permission = None
+
+    @cached_property
+    def exporters(self):
+        exporters = []
+        events = (self.request.auth or self.request.user).get_events_with_permission('can_view_orders', request=self.request).filter(
+            organizer=self.request.organizer
+        )
+        responses = register_multievent_data_exporters.send(self.request.organizer)
+        for ex in sorted([response(events) for r, response in responses if response], key=lambda ex: str(ex.verbose_name)):
+            ex._serializer = JobRunSerializer(exporter=ex, events=events)
+            exporters.append(ex)
+        return exporters
+
+    def get_serializer_kwargs(self):
+        return {
+            'events': self.request.auth.get_events_with_permission('can_view_orders', request=self.request).filter(
+                organizer=self.request.organizer
+            )
+        }
+
+    def do_export(self, cf, instance, data):
+        return multiexport.apply_async(kwargs={
+            'organizer': self.request.organizer.id,
+            'user': self.request.user.id if self.request.user.is_authenticated else None,
+            'token': self.request.auth.pk if isinstance(self.request.auth, TeamAPIToken) else None,
+            'device': self.request.auth.pk if isinstance(self.request.auth, Device) else None,
+            'fileid': str(cf.id),
+            'provider': instance.identifier,
+            'form_data': data
+        })

src/pretix/base/exporters/mail.py

@@ -41,7 +41,7 @@ class MailExporter(BaseExporter):
                      initial=[Order.STATUS_PENDING, Order.STATUS_PAID],
                      choices=Order.STATUS_CHOICE,
                      widget=forms.CheckboxSelectMultiple,
-                     required=False
+                     required=True
                  )),
             ]
         )

src/pretix/base/models/devices.py

@@ -222,3 +222,15 @@ class Device(LoggedModel):
             return self.organizer.events.all()
         else:
             return self.limit_events.all()
+
+    def get_events_with_permission(self, permission, request=None):
+        """
+        Returns a queryset of events the device has a specific permissions to.
+
+        :param request: Ignored, for compatibility with User model
+        :return: Iterable of Events
+        """
+        if permission in self.permission_set():
+            return self.get_events_with_any_permission()
+        else:
+            return self.organizer.events.none()

src/pretix/base/models/organizer.py

@@ -357,3 +357,15 @@ class TeamAPIToken(models.Model):
             return self.team.organizer.events.all()
         else:
             return self.team.limit_events.all()
+
+    def get_events_with_permission(self, permission, request=None):
+        """
+        Returns a queryset of events the token has a specific permissions to.
+
+        :param request: Ignored, for compatibility with User model
+        :return: Iterable of Events
+        """
+        if getattr(self.team, permission, False):
+            return self.get_events_with_any_permission()
+        else:
+            return self.team.organizer.events.none()

src/pretix/base/services/export.py

@@ -1,12 +1,13 @@
 from typing import Any, Dict
 
+from django.conf import settings
 from django.core.files.base import ContentFile
 from django.utils.timezone import override
 from django.utils.translation import gettext
 
 from pretix.base.i18n import LazyLocaleException, language
 from pretix.base.models import (
-    CachedFile, Event, Organizer, User, cachedfile_name,
+    CachedFile, Device, Event, Organizer, TeamAPIToken, User, cachedfile_name,
 )
 from pretix.base.services.tasks import (
     ProfiledEventTask, ProfiledOrganizerUserTask,
@@ -48,7 +49,13 @@ def export(self, event: Event, fileid: str, provider: str, form_data: Dict[str,
 
 
 @app.task(base=ProfiledOrganizerUserTask, throws=(ExportError,), bind=True)
-def multiexport(self, organizer: Organizer, user: User, fileid: str, provider: str, form_data: Dict[str, Any]) -> None:
+def multiexport(self, organizer: Organizer, user: User, device: int, token: int, fileid: str, provider: str, form_data: Dict[str, Any]) -> None:
+    if device:
+        device = Device.objects.get(pk=device)
+    if token:
+        device = TeamAPIToken.objects.get(pk=token)
+    allowed_events = (device or token or user).get_events_with_permission('can_view_orders')
+
     def set_progress(val):
         if not self.request.called_directly:
             self.update_state(
@@ -57,10 +64,22 @@ def multiexport(self, organizer: Organizer, user: User, fileid: str, provider: s
             )
 
     file = CachedFile.objects.get(id=fileid)
-    with language(user.locale), override(user.timezone):
-        allowed_events = user.get_events_with_permission('can_view_orders')
-
-        events = allowed_events.filter(pk__in=form_data.get('events'))
+    if user:
+        locale = user.locale
+        timezone = user.timezone
+    else:
+        e = allowed_events.first()
+        if e:
+            locale = e.settings.locale
+            timezone = e.settings.timezone
+        else:
+            locale = settings.LANGUAGE_CODE
+            timezone = settings.TIME_ZONE
+    with language(locale), override(timezone):
+        if isinstance(form_data['events'][0], str):
+            events = allowed_events.filter(slug__in=form_data.get('events'), organizer=organizer)
+        else:
+            events = allowed_events.filter(pk__in=form_data.get('events'))
         responses = register_multievent_data_exporters.send(organizer)
 
         for receiver, response in responses:

src/pretix/base/services/tasks.py

@@ -96,8 +96,9 @@ class OrganizerUserTask(app.Task):
         kwargs['organizer'] = organizer
 
         user_id = kwargs['user']
-        user = User.objects.get(pk=user_id)
-        kwargs['user'] = user
+        if user_id is not None:
+            user = User.objects.get(pk=user_id)
+            kwargs['user'] = user
 
         with scope(organizer=organizer):
             ret = super().__call__(*args, **kwargs)