Fix #526 -- Add a webhook system (#1073)

- [x] Data model
- [x] UI
- [x] Fire hooks
- [x] Unit tests
- [x] Display logs
- [x] API to modify hooks
- [x] Documentation
- [x] More hooks!

src/pretix/api/__init__.py

@@ -5,5 +5,8 @@ class PretixApiConfig(AppConfig):
     name = 'pretix.api'
     label = 'pretixapi'
 
+    def ready(self):
+        from . import signals, webhooks  # noqa
+
 
 default_app_config = 'pretix.api.PretixApiConfig'

src/pretix/api/migrations/0003_webhook_webhookcall_webhookeventlistener.py

@@ -0,0 +1,79 @@
+# Generated by Django 2.1.1 on 2018-11-07 10:46
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pretixbase', '0102_auto_20181017_0024'),
+        ('pretixapi', '0002_auto_20180604_1120'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='WebHook',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('enabled', models.BooleanField(default=True, verbose_name='Enable webhook')),
+                ('target_url', models.URLField(verbose_name='Target URL')),
+                ('all_events', models.BooleanField(default=False, verbose_name='All events (including newly created ones)')),
+                ('limit_events', models.ManyToManyField(blank=True, to='pretixbase.Event', verbose_name='Limit to events')),
+                ('organizer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pretixbase.Organizer')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='WebHookCall',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('datetime', models.DateTimeField(auto_now_add=True)),
+                ('target_url', models.URLField()),
+                ('is_retry', models.BooleanField(default=False)),
+                ('execution_time', models.FloatField(null=True)),
+                ('return_code', models.PositiveIntegerField(default=0)),
+                ('payload', models.TextField()),
+                ('response_body', models.TextField()),
+                ('webhook', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pretixapi.WebHook')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='WebHookEventListener',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('action_type', models.CharField(max_length=255)),
+                ('webhook', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pretixapi.WebHook')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='webhookcall',
+            name='success',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='webhook',
+            name='all_events',
+            field=models.BooleanField(default=True, verbose_name='All events (including newly created ones)'),
+        ),
+        migrations.AlterField(
+            model_name='webhook',
+            name='organizer',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='webhooks', to='pretixbase.Organizer'),
+        ),
+        migrations.AlterField(
+            model_name='webhookcall',
+            name='webhook',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='calls', to='pretixapi.WebHook'),
+        ),
+        migrations.AlterField(
+            model_name='webhookeventlistener',
+            name='webhook',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='listeners', to='pretixapi.WebHook'),
+        ),
+        migrations.AddField(
+            model_name='webhookcall',
+            name='action_type',
+            field=models.CharField(default='', max_length=255),
+            preserve_default=False,
+        ),
+    ]

src/pretix/api/models.py

@@ -68,3 +68,41 @@ class OAuthRefreshToken(AbstractRefreshToken):
         OAuthAccessToken, on_delete=models.SET_NULL, blank=True, null=True,
         related_name="refresh_token"
     )
+
+
+class WebHook(models.Model):
+    organizer = models.ForeignKey('pretixbase.Organizer', on_delete=models.CASCADE, related_name='webhooks')
+    enabled = models.BooleanField(default=True, verbose_name=_("Enable webhook"))
+    target_url = models.URLField(verbose_name=_("Target URL"))
+    all_events = models.BooleanField(default=True, verbose_name=_("All events (including newly created ones)"))
+    limit_events = models.ManyToManyField('pretixbase.Event', verbose_name=_("Limit to events"), blank=True)
+
+    @property
+    def action_types(self):
+        return [
+            l.action_type for l in self.listeners.all()
+        ]
+
+
+class WebHookEventListener(models.Model):
+    webhook = models.ForeignKey('WebHook', on_delete=models.CASCADE, related_name='listeners')
+    action_type = models.CharField(max_length=255)
+
+    class Meta:
+        ordering = ("action_type",)
+
+
+class WebHookCall(models.Model):
+    webhook = models.ForeignKey('WebHook', on_delete=models.CASCADE, related_name='calls')
+    datetime = models.DateTimeField(auto_now_add=True)
+    target_url = models.URLField()
+    action_type = models.CharField(max_length=255)
+    is_retry = models.BooleanField(default=False)
+    execution_time = models.FloatField(null=True)
+    return_code = models.PositiveIntegerField(default=0)
+    success = models.BooleanField(default=False)
+    payload = models.TextField()
+    response_body = models.TextField()
+
+    class Meta:
+        ordering = ("-datetime",)

src/pretix/api/serializers/webhooks.py

@@ -0,0 +1,71 @@
+from django.core.exceptions import ValidationError
+from rest_framework import serializers
+
+from pretix.api.models import WebHook
+from pretix.api.serializers.i18n import I18nAwareModelSerializer
+from pretix.api.webhooks import get_all_webhook_events
+from pretix.base.models import Event
+
+
+class EventRelatedField(serializers.SlugRelatedField):
+    def get_queryset(self):
+        return self.context['organizer'].events.all()
+
+
+class ActionTypesField(serializers.Field):
+    def to_representation(self, instance: WebHook):
+        return instance.action_types
+
+    def to_internal_value(self, data):
+        types = get_all_webhook_events()
+        for d in data:
+            if d not in types:
+                raise ValidationError('Invalid action type "%s".' % d)
+        return {'action_types': data}
+
+
+class WebHookSerializer(I18nAwareModelSerializer):
+    limit_events = EventRelatedField(
+        slug_field='slug',
+        queryset=Event.objects.none(),
+        many=True
+    )
+    action_types = ActionTypesField(source='*')
+
+    class Meta:
+        model = WebHook
+        fields = ('id', 'enabled', 'target_url', 'all_events', 'limit_events', 'action_types')
+
+    def validate(self, data):
+        data = super().validate(data)
+
+        full_data = self.to_internal_value(self.to_representation(self.instance)) if self.instance else {}
+        full_data.update(data)
+
+        for event in full_data.get('limit_events'):
+            if self.context['organizer'] != event.organizer:
+                raise ValidationError('One or more events do not belong to this organizer.')
+
+        if full_data.get('limit_events') and full_data.get('all_events'):
+            raise ValidationError('You can set either limit_events or all_events.')
+
+        return data
+
+    def create(self, validated_data):
+        action_types = validated_data.pop('action_types')
+        inst = super().create(validated_data)
+        for l in action_types:
+            inst.listeners.create(action_type=l)
+        return inst
+
+    def update(self, instance, validated_data):
+        action_types = validated_data.pop('action_types', None)
+        instance = super().update(instance, validated_data)
+        if action_types is not None:
+            current_listeners = set(instance.listeners.values_list('action_type', flat=True))
+            new_listeners = set(action_types)
+            for l in current_listeners - new_listeners:
+                instance.listeners.filter(action_type=l).delete()
+            for l in new_listeners - current_listeners:
+                instance.listeners.create(action_type=l)
+        return instance

src/pretix/api/signals.py

@@ -0,0 +1,21 @@
+from datetime import timedelta
+
+from django.dispatch import Signal, receiver
+from django.utils.timezone import now
+
+from pretix.api.models import WebHookCall
+from pretix.base.signals import periodic_task
+
+register_webhook_events = Signal(
+    providing_args=[]
+)
+"""
+This signal is sent out to get all known webhook events. Receivers should return an
+instance of a subclass of pretix.api.webhooks.WebhookEvent or a list of such
+instances.
+"""
+
+
+@receiver(periodic_task)
+def cleanup_webhook_logs(sender, **kwargs):
+    WebHookCall.objects.filter(datetime__lte=now() - timedelta(days=30)).delete()

src/pretix/api/urls.py

@@ -8,7 +8,7 @@ from pretix.api.views import cart
 
 from .views import (
     checkin, device, event, item, oauth, order, organizer, voucher,
-    waitinglist,
+    waitinglist, webhooks,
 )
 
 router = routers.DefaultRouter()
@@ -17,6 +17,7 @@ router.register(r'organizers', organizer.OrganizerViewSet)
 orga_router = routers.DefaultRouter()
 orga_router.register(r'events', event.EventViewSet)
 orga_router.register(r'subevents', event.SubEventViewSet)
+orga_router.register(r'webhooks', webhooks.WebHookViewSet)
 
 event_router = routers.DefaultRouter()
 event_router.register(r'subevents', event.SubEventViewSet)

src/pretix/api/views/checkin.py

@@ -244,7 +244,9 @@ class CheckinListPositionViewSet(viewsets.ReadOnlyModelViewSet):
                 ignore_unpaid=ignore_unpaid,
                 nonce=nonce,
                 datetime=dt,
-                questions_supported=self.request.data.get('questions_supported', True)
+                questions_supported=self.request.data.get('questions_supported', True),
+                user=self.request.user,
+                auth=self.request.auth,
             )
         except RequiredQuestionsError as e:
             return Response({

src/pretix/api/views/webhooks.py

@@ -0,0 +1,49 @@
+from rest_framework import viewsets
+
+from pretix.api.models import WebHook
+from pretix.api.serializers.webhooks import WebHookSerializer
+from pretix.helpers.dicts import merge_dicts
+
+
+class WebHookViewSet(viewsets.ModelViewSet):
+    serializer_class = WebHookSerializer
+    queryset = WebHook.objects.none()
+    permission = 'can_change_organizer_settings'
+    write_permission = 'can_change_organizer_settings'
+
+    def get_queryset(self):
+        return self.request.organizer.webhooks.prefetch_related('listeners')
+
+    def get_serializer_context(self):
+        ctx = super().get_serializer_context()
+        ctx['organizer'] = self.request.organizer
+        return ctx
+
+    def perform_create(self, serializer):
+        inst = serializer.save(organizer=self.request.organizer)
+        self.request.organizer.log_action(
+            'pretix.webhook.created',
+            user=self.request.user,
+            auth=self.request.auth,
+            data=merge_dicts(self.request.data, {'id': inst.pk})
+        )
+
+    def perform_update(self, serializer):
+        inst = serializer.save(organizer=self.request.organizer)
+        self.request.organizer.log_action(
+            'pretix.webhook.changed',
+            user=self.request.user,
+            auth=self.request.auth,
+            data=merge_dicts(self.request.data, {'id': serializer.instance.pk})
+        )
+        return inst
+
+    def perform_destroy(self, instance):
+        self.request.organizer.log_action(
+            'pretix.webhook.changed',
+            user=self.request.user,
+            auth=self.request.auth,
+            data={'id': instance.pk, 'enabled': False}
+        )
+        instance.enabled = False
+        instance.save(update_fields=['enabled'])

src/pretix/api/webhooks.py

@@ -0,0 +1,252 @@
+import json
+import logging
+import time
+from collections import OrderedDict
+
+import requests
+from celery.exceptions import MaxRetriesExceededError
+from django.db.models import Exists, OuterRef, Q
+from django.dispatch import receiver
+from django.utils.translation import ugettext_lazy as _
+from requests import RequestException
+
+from pretix.api.models import WebHook, WebHookCall, WebHookEventListener
+from pretix.api.signals import register_webhook_events
+from pretix.base.models import LogEntry
+from pretix.base.services.tasks import ProfiledTask, TransactionAwareTask
+from pretix.celery_app import app
+
+logger = logging.getLogger(__name__)
+_ALL_EVENTS = None
+
+
+class WebhookEvent:
+    def __init__(self):
+        pass
+
+    def __repr__(self):
+        return '<WebhookEvent: {}>'.format(self.action_type)
+
+    @property
+    def action_type(self) -> str:
+        """
+        The action_type string that this notification handles, for example
+        ``"pretix.event.order.paid"``. Only one notification type should be registered
+        per action type.
+        """
+        raise NotImplementedError()  # NOQA
+
+    @property
+    def verbose_name(self) -> str:
+        """
+        A human-readable name of this notification type.
+        """
+        raise NotImplementedError()  # NOQA
+
+    def build_payload(self, logentry: LogEntry) -> dict:
+        """
+        This is the main function that you should override. It is supposed to turn a log entry
+        object into a dictionary that can be used as the webhook payload.
+        """
+        raise NotImplementedError()  # NOQA
+
+
+def get_all_webhook_events():
+    global _ALL_EVENTS
+
+    if _ALL_EVENTS:
+        return _ALL_EVENTS
+
+    types = OrderedDict()
+    for recv, ret in register_webhook_events.send(None):
+        if isinstance(ret, (list, tuple)):
+            for r in ret:
+                types[r.action_type] = r
+        else:
+            types[ret.action_type] = ret
+    _ALL_EVENTS = types
+    return types
+
+
+class ParametrizedOrderWebhookEvent(WebhookEvent):
+    def __init__(self, action_type, verbose_name):
+        self._action_type = action_type
+        self._verbose_name = verbose_name
+        super().__init__()
+
+    @property
+    def action_type(self):
+        return self._action_type
+
+    @property
+    def verbose_name(self):
+        return self._verbose_name
+
+    def build_payload(self, logentry: LogEntry):
+        order = logentry.content_object
+
+        return {
+            'notification_id': logentry.pk,
+            'organizer': order.event.organizer.slug,
+            'event': order.event.slug,
+            'code': order.code,
+            'action': logentry.action_type,
+        }
+
+
+class ParametrizedOrderPositionWebhookEvent(ParametrizedOrderWebhookEvent):
+
+    def build_payload(self, logentry: LogEntry):
+        d = super().build_payload(logentry)
+        d['orderposition_id'] = logentry.parsed_data.get('position')
+        d['orderposition_positionid'] = logentry.parsed_data.get('positionid')
+        d['checkin_list'] = logentry.parsed_data.get('list')
+        d['first_checkin'] = logentry.parsed_data.get('first_checkin')
+
+
+@receiver(register_webhook_events, dispatch_uid="base_register_default_webhook_events")
+def register_default_webhook_events(sender, **kwargs):
+    return (
+        ParametrizedOrderWebhookEvent(
+            'pretix.event.order.placed',
+            _('New order placed'),
+        ),
+        ParametrizedOrderWebhookEvent(
+            'pretix.event.order.paid',
+            _('Order marked as paid'),
+        ),
+        ParametrizedOrderWebhookEvent(
+            'pretix.event.order.canceled',
+            _('Order canceled'),
+        ),
+        ParametrizedOrderWebhookEvent(
+            'pretix.event.order.expired',
+            _('Order expired'),
+        ),
+        ParametrizedOrderWebhookEvent(
+            'pretix.event.order.modified',
+            _('Order information changed'),
+        ),
+        ParametrizedOrderWebhookEvent(
+            'pretix.event.order.contact.changed',
+            _('Order contact address changed'),
+        ),
+        ParametrizedOrderWebhookEvent(
+            'pretix.event.order.changed.*',
+            _('Order changed'),
+        ),
+        ParametrizedOrderWebhookEvent(
+            'pretix.event.order.refund.created.externally',
+            _('External refund of payment'),
+        ),
+        ParametrizedOrderWebhookEvent(
+            'pretix.event.order.refunded',
+            _('Order refunded'),
+        ),
+        ParametrizedOrderWebhookEvent(
+            'pretix.event.order.approved',
+            _('Order approved'),
+        ),
+        ParametrizedOrderWebhookEvent(
+            'pretix.event.order.denied',
+            _('Order denied'),
+        ),
+        ParametrizedOrderPositionWebhookEvent(
+            'pretix.event.checkin',
+            _('Ticket checked in'),
+        ),
+        ParametrizedOrderPositionWebhookEvent(
+            'pretix.event.checkin.reverted',
+            _('Ticket check-in reverted'),
+        ),
+    )
+
+
+@app.task(base=TransactionAwareTask)
+def notify_webhooks(logentry_id: int):
+    logentry = LogEntry.all.get(id=logentry_id)
+
+    if not logentry.organizer:
+        return  # We need to know the organizer
+
+    types = get_all_webhook_events()
+    notification_type = None
+    typepath = logentry.action_type
+    while not notification_type and '.' in typepath:
+        notification_type = types.get(typepath + ('.*' if typepath != logentry.action_type else ''))
+        typepath = typepath.rsplit('.', 1)[0]
+
+    if not notification_type:
+        return  # Ignore, no webhooks for this event type
+
+    # All webhooks that registered for this notification
+    event_listener = WebHookEventListener.objects.filter(
+        webhook=OuterRef('pk'),
+        action_type=notification_type.action_type
+    )
+
+    webhooks = WebHook.objects.annotate(has_el=Exists(event_listener)).filter(
+        organizer=logentry.organizer,
+        has_el=True,
+        enabled=True
+    )
+    if logentry.event_id:
+        webhooks = webhooks.filter(
+            Q(all_events=True) | Q(limit_events__pk=logentry.event_id)
+        )
+
+    for wh in webhooks:
+        send_webhook.apply_async(args=(logentry_id, notification_type.action_type, wh.pk))
+
+
+@app.task(base=ProfiledTask, bind=True, max_retries=9)
+def send_webhook(self, logentry_id: int, action_type: str, webhook_id: int):
+    # 9 retries with 2**(2*x) timing is roughly 72 hours
+    logentry = LogEntry.all.get(id=logentry_id)
+    webhook = WebHook.objects.get(id=webhook_id)
+
+    types = get_all_webhook_events()
+    event_type = types.get(action_type)
+    if not event_type or not webhook.enabled:
+        return  # Ignore, e.g. plugin not installed
+
+    payload = event_type.build_payload(logentry)
+    t = time.time()
+
+    try:
+        try:
+            resp = requests.post(
+                webhook.target_url,
+                json=payload,
+                allow_redirects=False
+            )
+            WebHookCall.objects.create(
+                webhook=webhook,
+                action_type=logentry.action_type,
+                target_url=webhook.target_url,
+                is_retry=self.request.retries > 0,
+                execution_time=time.time() - t,
+                return_code=resp.status_code,
+                payload=json.dumps(payload),
+                response_body=resp.text[:1024 * 1024],
+                success=200 <= resp.status_code <= 299
+            )
+            if resp.status_code == 410:
+                webhook.enabled = False
+                webhook.save()
+            elif resp.status_code > 299:
+                raise self.retry(countdown=2 ** (self.request.retries * 2))
+        except RequestException as e:
+            WebHookCall.objects.create(
+                webhook=webhook,
+                action_type=logentry.action_type,
+                target_url=webhook.target_url,
+                is_retry=self.request.retries > 0,
+                execution_time=time.time() - t,
+                return_code=0,
+                payload=json.dumps(payload),
+                response_body=str(e)[:1024 * 1024]
+            )
+            raise self.retry(countdown=2 ** (self.request.retries * 2))
+    except MaxRetriesExceededError:
+        pass