Fix #1780 -- Trigger exports through API (#1839)

src/pretix/api/serializers/exporters.py

@@ -0,0 +1,110 @@
+from django import forms
+from rest_framework import serializers
+
+
+class FormFieldWrapperField(serializers.Field):
+    def __init__(self, *args, **kwargs):
+        self.form_field = kwargs.pop('form_field')
+        super().__init__(*args, **kwargs)
+
+    def to_representation(self, value):
+        return self.form_field.widget.format_value(value)
+
+    def to_internal_value(self, data):
+        d = self.form_field.widget.value_from_datadict({'name': data}, {}, 'name')
+        d = self.form_field.clean(d)
+        return d
+
+
+simple_mappings = (
+    (forms.DateField, serializers.DateField, tuple()),
+    (forms.TimeField, serializers.TimeField, tuple()),
+    (forms.SplitDateTimeField, serializers.DateTimeField, tuple()),
+    (forms.DateTimeField, serializers.DateTimeField, tuple()),
+    (forms.DecimalField, serializers.DecimalField, ('max_digits', 'decimal_places', 'min_value', 'max_value')),
+    (forms.FloatField, serializers.FloatField, tuple()),
+    (forms.IntegerField, serializers.IntegerField, tuple()),
+    (forms.EmailField, serializers.EmailField, tuple()),
+    (forms.UUIDField, serializers.UUIDField, tuple()),
+    (forms.URLField, serializers.URLField, tuple()),
+    (forms.NullBooleanField, serializers.NullBooleanField, tuple()),
+    (forms.BooleanField, serializers.BooleanField, tuple()),
+)
+
+
+class SerializerDescriptionField(serializers.Field):
+    def to_representation(self, value):
+        fields = []
+        for k, v in value.fields.items():
+            d = {
+                'name': k,
+                'required': v.required,
+            }
+            if isinstance(v, serializers.ChoiceField):
+                d['choices'] = list(v.choices.keys())
+            fields.append(d)
+
+        return fields
+
+
+class ExporterSerializer(serializers.Serializer):
+    identifier = serializers.CharField()
+    verbose_name = serializers.CharField()
+    input_parameters = SerializerDescriptionField(source='_serializer')
+
+
+class JobRunSerializer(serializers.Serializer):
+    def __init__(self, *args, **kwargs):
+        ex = kwargs.pop('exporter')
+        events = kwargs.pop('events', None)
+        super().__init__(*args, **kwargs)
+        if events is not None:
+            self.fields["events"] = serializers.SlugRelatedField(
+                queryset=events,
+                required=True,
+                allow_empty=False,
+                slug_field='slug',
+                many=True
+            )
+        for k, v in ex.export_form_fields.items():
+            for m_from, m_to, m_kwargs in simple_mappings:
+                if isinstance(v, m_from):
+                    self.fields[k] = m_to(
+                        required=v.required,
+                        allow_null=not v.required,
+                        validators=v.validators,
+                        **{kwarg: getattr(v, kwargs, None) for kwarg in m_kwargs}
+                    )
+                    break
+
+            if isinstance(v, forms.ModelMultipleChoiceField):
+                self.fields[k] = serializers.PrimaryKeyRelatedField(
+                    queryset=v.queryset,
+                    required=v.required,
+                    allow_empty=not v.required,
+                    validators=v.validators,
+                    many=True
+                )
+            elif isinstance(v, forms.ModelChoiceField):
+                self.fields[k] = serializers.PrimaryKeyRelatedField(
+                    queryset=v.queryset,
+                    required=v.required,
+                    allow_null=not v.required,
+                    validators=v.validators,
+                )
+            elif isinstance(v, forms.MultipleChoiceField):
+                self.fields[k] = serializers.MultipleChoiceField(
+                    choices=v.choices,
+                    required=v.required,
+                    allow_empty=not v.required,
+                    validators=v.validators,
+                )
+            elif isinstance(v, forms.ChoiceField):
+                self.fields[k] = serializers.ChoiceField(
+                    choices=v.choices,
+                    required=v.required,
+                    allow_null=not v.required,
+                    validators=v.validators,
+                )
+            else:
+                self.fields[k] = FormFieldWrapperField(form_field=v, required=v.required, allow_null=not v.required)

src/pretix/api/urls.py

@@ -7,8 +7,8 @@ from rest_framework import routers
 from pretix.api.views import cart
 
 from .views import (
-    checkin, device, event, item, oauth, order, organizer, user, version,
-    voucher, waitinglist, webhooks,
+    checkin, device, event, exporters, item, oauth, order, organizer, user,
+    version, voucher, waitinglist, webhooks,
 )
 
 router = routers.DefaultRouter()
@@ -22,6 +22,7 @@ orga_router.register(r'seatingplans', organizer.SeatingPlanViewSet)
 orga_router.register(r'giftcards', organizer.GiftCardViewSet)
 orga_router.register(r'teams', organizer.TeamViewSet)
 orga_router.register(r'devices', organizer.DeviceViewSet)
+orga_router.register(r'exporters', exporters.OrganizerExportersViewSet, basename='exporters')
 
 team_router = routers.DefaultRouter()
 team_router.register(r'members', organizer.TeamMemberViewSet)
@@ -44,6 +45,7 @@ event_router.register(r'taxrules', event.TaxRuleViewSet)
 event_router.register(r'waitinglistentries', waitinglist.WaitingListViewSet)
 event_router.register(r'checkinlists', checkin.CheckinListViewSet)
 event_router.register(r'cartpositions', cart.CartPositionViewSet)
+event_router.register(r'exporters', exporters.EventExportersViewSet, basename='exporters')
 
 checkinlist_router = routers.DefaultRouter()
 checkinlist_router.register(r'positions', checkin.CheckinListPositionViewSet, basename='checkinlistpos')

src/pretix/api/views/exporters.py

@@ -0,0 +1,154 @@
+from datetime import timedelta
+
+from celery.result import AsyncResult
+from django.conf import settings
+from django.http import Http404
+from django.shortcuts import get_object_or_404
+from django.utils.functional import cached_property
+from django.utils.timezone import now
+from rest_framework import status, viewsets
+from rest_framework.decorators import action
+from rest_framework.response import Response
+from rest_framework.reverse import reverse
+
+from pretix.api.serializers.exporters import (
+    ExporterSerializer, JobRunSerializer,
+)
+from pretix.base.models import CachedFile, Device, TeamAPIToken
+from pretix.base.services.export import export, multiexport
+from pretix.base.signals import (
+    register_data_exporters, register_multievent_data_exporters,
+)
+from pretix.helpers.http import ChunkBasedFileResponse
+
+
+class ExportersMixin:
+    def list(self, request, *args, **kwargs):
+        res = ExporterSerializer(self.exporters, many=True)
+        return Response({
+            "count": len(self.exporters),
+            "next": None,
+            "previous": None,
+            "results": res.data
+        })
+
+    def get_object(self):
+        instances = [e for e in self.exporters if e.identifier == self.kwargs.get('pk')]
+        if not instances:
+            raise Http404()
+        return instances[0]
+
+    def retrieve(self, request, *args, **kwargs):
+        instance = self.get_object()
+        serializer = ExporterSerializer(instance)
+        return Response(serializer.data)
+
+    @action(detail=True, methods=['GET'], url_name='download', url_path='download/(?P<asyncid>[^/]+)/(?P<cfid>[^/]+)')
+    def download(self, *args, **kwargs):
+        cf = get_object_or_404(CachedFile, id=kwargs['cfid'])
+        if cf.file:
+            resp = ChunkBasedFileResponse(cf.file.file, content_type=cf.type)
+            resp['Content-Disposition'] = 'attachment; filename="{}"'.format(cf.filename)
+            return resp
+        elif not settings.HAS_CELERY:
+            return Response(
+                {'status': 'failed', 'message': 'Unknown file ID or export failed'},
+                status=status.HTTP_410_GONE
+            )
+
+        res = AsyncResult(kwargs['asyncid'])
+        if res.failed():
+            if isinstance(res.info, dict) and res.info['exc_type'] == 'ExportError':
+                msg = res.info['exc_message']
+            else:
+                msg = 'Internal error'
+            return Response(
+                {'status': 'failed', 'message': msg},
+                status=status.HTTP_410_GONE
+            )
+
+        return Response(
+            {
+                'status': 'running' if res.state in ('PROGRESS', 'STARTED', 'SUCCESS') else 'waiting',
+                'percentage': res.result.get('value', None) if res.result else None,
+            },
+            status=status.HTTP_409_CONFLICT
+        )
+
+    @action(detail=True, methods=['POST'])
+    def run(self, *args, **kwargs):
+        instance = self.get_object()
+        serializer = JobRunSerializer(exporter=instance, data=self.request.data, **self.get_serializer_kwargs())
+        serializer.is_valid(raise_exception=True)
+
+        cf = CachedFile()
+        cf.date = now()
+        cf.expires = now() + timedelta(days=3)
+        cf.save()
+        d = serializer.data
+        for k, v in d.items():
+            if isinstance(v, set):
+                d[k] = list(v)
+        async_result = self.do_export(cf, instance, d)
+
+        url_kwargs = {
+            'asyncid': str(async_result.id),
+            'cfid': str(cf.id),
+        }
+        url_kwargs.update(self.kwargs)
+        return Response({
+            'download': reverse('api-v1:exporters-download', kwargs=url_kwargs, request=self.request)
+        }, status=status.HTTP_202_ACCEPTED)
+
+
+class EventExportersViewSet(ExportersMixin, viewsets.ViewSet):
+    permission = 'can_view_orders'
+
+    def get_serializer_kwargs(self):
+        return {}
+
+    @cached_property
+    def exporters(self):
+        exporters = []
+        responses = register_data_exporters.send(self.request.event)
+        for ex in sorted([response(self.request.event) for r, response in responses], key=lambda ex: str(ex.verbose_name)):
+            ex._serializer = JobRunSerializer(exporter=ex)
+            exporters.append(ex)
+        return exporters
+
+    def do_export(self, cf, instance, data):
+        return export.apply_async(args=(self.request.event.id, str(cf.id), instance.identifier, data))
+
+
+class OrganizerExportersViewSet(ExportersMixin, viewsets.ViewSet):
+    permission = None
+
+    @cached_property
+    def exporters(self):
+        exporters = []
+        events = (self.request.auth or self.request.user).get_events_with_permission('can_view_orders', request=self.request).filter(
+            organizer=self.request.organizer
+        )
+        responses = register_multievent_data_exporters.send(self.request.organizer)
+        for ex in sorted([response(events) for r, response in responses if response], key=lambda ex: str(ex.verbose_name)):
+            ex._serializer = JobRunSerializer(exporter=ex, events=events)
+            exporters.append(ex)
+        return exporters
+
+    def get_serializer_kwargs(self):
+        return {
+            'events': self.request.auth.get_events_with_permission('can_view_orders', request=self.request).filter(
+                organizer=self.request.organizer
+            )
+        }
+
+    def do_export(self, cf, instance, data):
+        return multiexport.apply_async(kwargs={
+            'organizer': self.request.organizer.id,
+            'user': self.request.user.id if self.request.user.is_authenticated else None,
+            'token': self.request.auth.pk if isinstance(self.request.auth, TeamAPIToken) else None,
+            'device': self.request.auth.pk if isinstance(self.request.auth, Device) else None,
+            'fileid': str(cf.id),
+            'provider': instance.identifier,
+            'form_data': data
+        })