Fix #526 -- Add a webhook system (#1073)

- [x] Data model
- [x] UI
- [x] Fire hooks
- [x] Unit tests
- [x] Display logs
- [x] API to modify hooks
- [x] Documentation
- [x] More hooks!

src/tests/api/conftest.py

@@ -68,6 +68,7 @@ def team(organizer):
         can_change_vouchers=True,
         can_view_vouchers=True,
         can_change_orders=True,
+        can_change_organizer_settings=True
     )
 
 

src/tests/api/test_permissions.py

@@ -124,6 +124,15 @@ event_permission_sub_urls = [
     ('delete', 'can_change_orders', 'cartpositions/1/', 404),
 ]
 
+org_permission_sub_urls = [
+    ('get', 'can_change_organizer_settings', 'webhooks/', 200),
+    ('post', 'can_change_organizer_settings', 'webhooks/', 400),
+    ('get', 'can_change_organizer_settings', 'webhooks/1/', 404),
+    ('put', 'can_change_organizer_settings', 'webhooks/1/', 404),
+    ('patch', 'can_change_organizer_settings', 'webhooks/1/', 404),
+    ('delete', 'can_change_organizer_settings', 'webhooks/1/', 404),
+]
+
 
 event_permission_root_urls = [
     ('post', 'can_create_events', 400),
@@ -400,3 +409,32 @@ def test_device_subresource_permission_check(device_client, device, organizer, e
             assert resp.status_code == 403
         else:
             assert resp.status_code in (404, 403)
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize("urlset", org_permission_sub_urls)
+def test_token_org_subresources_permission_allowed(token_client, team, organizer, event, urlset):
+    team.all_events = True
+    if urlset[1]:
+        setattr(team, urlset[1], True)
+    team.save()
+    resp = getattr(token_client, urlset[0])('/api/v1/organizers/{}/{}'.format(
+        organizer.slug, urlset[2]))
+    assert resp.status_code == urlset[3]
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize("urlset", org_permission_sub_urls)
+def test_token_org_subresources_permission_not_allowed(token_client, team, organizer, event, urlset):
+    if urlset[1] is None:
+        team.all_events = False
+    else:
+        team.all_events = True
+        setattr(team, urlset[1], False)
+    team.save()
+    resp = getattr(token_client, urlset[0])('/api/v1/organizers/{}/{}'.format(
+        organizer.slug, urlset[2]))
+    if urlset[3] == 404:
+        assert resp.status_code == 403
+    else:
+        assert resp.status_code in (404, 403)

src/tests/api/test_webhooks.py

@@ -0,0 +1,169 @@
+import copy
+
+import pytest
+
+from pretix.api.models import WebHook
+
+
+@pytest.fixture
+def webhook(organizer, event):
+    wh = organizer.webhooks.create(
+        enabled=True,
+        target_url='https://google.com',
+        all_events=False
+    )
+    wh.limit_events.add(event)
+    wh.listeners.create(action_type='pretix.event.order.placed')
+    wh.listeners.create(action_type='pretix.event.order.paid')
+    return wh
+
+
+TEST_WEBHOOK_RES = {
+    "id": 1,
+    "enabled": True,
+    "target_url": "https://google.com",
+    "all_events": False,
+    "limit_events": ['dummy'],
+    "action_types": ['pretix.event.order.paid', 'pretix.event.order.placed'],
+}
+
+
+@pytest.mark.django_db
+def test_hook_list(token_client, organizer, event, webhook):
+    res = dict(TEST_WEBHOOK_RES)
+    res["id"] = webhook.pk
+
+    resp = token_client.get('/api/v1/organizers/{}/webhooks/'.format(organizer.slug))
+    assert resp.status_code == 200
+    assert [res] == resp.data['results']
+
+
+@pytest.mark.django_db
+def test_hook_detail(token_client, organizer, event, webhook):
+    res = dict(TEST_WEBHOOK_RES)
+    res["id"] = webhook.pk
+    resp = token_client.get('/api/v1/organizers/{}/webhooks/{}/'.format(organizer.slug, webhook.pk))
+    assert resp.status_code == 200
+    assert res == resp.data
+
+
+TEST_WEBHOOK_CREATE_PAYLOAD = {
+    "enabled": True,
+    "target_url": "https://google.com",
+    "all_events": False,
+    "limit_events": ['dummy'],
+    "action_types": ['pretix.event.order.placed', 'pretix.event.order.paid'],
+}
+
+
+@pytest.mark.django_db
+def test_hook_create(token_client, organizer, event):
+    resp = token_client.post(
+        '/api/v1/organizers/{}/webhooks/'.format(organizer.slug),
+        TEST_WEBHOOK_CREATE_PAYLOAD,
+        format='json'
+    )
+    assert resp.status_code == 201
+    cl = WebHook.objects.get(pk=resp.data['id'])
+    assert cl.target_url == "https://google.com"
+    assert cl.limit_events.count() == 1
+    assert set(cl.listeners.values_list('action_type', flat=True)) == {'pretix.event.order.placed',
+                                                                       'pretix.event.order.paid'}
+    assert not cl.all_events
+
+
+@pytest.mark.django_db
+def test_hook_create_either_all_or_limit(token_client, organizer, event):
+    res = copy.copy(TEST_WEBHOOK_CREATE_PAYLOAD)
+    res['all_events'] = True
+    resp = token_client.post(
+        '/api/v1/organizers/{}/webhooks/'.format(organizer.slug),
+        res,
+        format='json'
+    )
+    assert resp.status_code == 400
+    assert resp.data == {'non_field_errors': ['You can set either limit_events or all_events.']}
+
+
+@pytest.mark.django_db
+def test_hook_create_invalid_url(token_client, organizer, event):
+    res = copy.copy(TEST_WEBHOOK_CREATE_PAYLOAD)
+    res['target_url'] = 'foo.bar'
+    resp = token_client.post(
+        '/api/v1/organizers/{}/webhooks/'.format(organizer.slug),
+        res,
+        format='json'
+    )
+    assert resp.status_code == 400
+    assert resp.data == {'target_url': ['Enter a valid URL.']}
+
+
+@pytest.mark.django_db
+def test_hook_create_invalid_event(token_client, organizer, event):
+    res = copy.copy(TEST_WEBHOOK_CREATE_PAYLOAD)
+    res['limit_events'] = ['foo']
+    resp = token_client.post(
+        '/api/v1/organizers/{}/webhooks/'.format(organizer.slug),
+        res,
+        format='json'
+    )
+    assert resp.status_code == 400
+    assert resp.data == {'limit_events': ['Object with slug=foo does not exist.']}
+
+
+@pytest.mark.django_db
+def test_hook_create_invalid_action_types(token_client, organizer, event):
+    res = copy.copy(TEST_WEBHOOK_CREATE_PAYLOAD)
+    res['action_types'] = ['foo']
+    resp = token_client.post(
+        '/api/v1/organizers/{}/webhooks/'.format(organizer.slug),
+        res,
+        format='json'
+    )
+    assert resp.status_code == 400
+    assert resp.data == {'action_types': ['Invalid action type "foo".']}
+
+
+@pytest.mark.django_db
+def test_hook_patch_url(token_client, organizer, event, webhook):
+    resp = token_client.patch(
+        '/api/v1/organizers/{}/webhooks/{}/'.format(organizer.slug, webhook.pk),
+        {
+            'target_url': 'https://pretix.eu'
+        },
+        format='json'
+    )
+    assert resp.status_code == 200
+    webhook.refresh_from_db()
+    assert webhook.target_url == "https://pretix.eu"
+    assert webhook.limit_events.count() == 1
+    assert set(webhook.listeners.values_list('action_type', flat=True)) == {'pretix.event.order.placed',
+                                                                            'pretix.event.order.paid'}
+    assert webhook.enabled
+
+
+@pytest.mark.django_db
+def test_hook_patch_types(token_client, organizer, event, webhook):
+    resp = token_client.patch(
+        '/api/v1/organizers/{}/webhooks/{}/'.format(organizer.slug, webhook.pk),
+        {
+            'action_types': ['pretix.event.order.placed', 'pretix.event.order.canceled']
+        },
+        format='json'
+    )
+    assert resp.status_code == 200
+    webhook.refresh_from_db()
+    assert webhook.limit_events.count() == 1
+    assert set(webhook.listeners.values_list('action_type', flat=True)) == {'pretix.event.order.placed',
+                                                                            'pretix.event.order.canceled'}
+    assert webhook.enabled
+
+
+@pytest.mark.django_db
+def test_hook_delete(token_client, organizer, event, webhook):
+    resp = token_client.delete(
+        '/api/v1/organizers/{}/webhooks/{}/'.format(organizer.slug, webhook.pk),
+    )
+    assert resp.status_code == 204
+    webhook.refresh_from_db()
+    assert not webhook.enabled

src/tests/base/test_notifications.py

@@ -78,6 +78,17 @@ def test_notification_trigger_global(event, order, user, monkeypatch_on_commit):
     assert len(djmail.outbox) == 1
 
 
+@pytest.mark.django_db
+def test_notification_trigger_global_wildcard(event, order, user, monkeypatch_on_commit):
+    djmail.outbox = []
+    user.notification_settings.create(
+        method='mail', event=None, action_type='pretix.event.order.changed.*', enabled=True
+    )
+    with transaction.atomic():
+        order.log_action('pretix.event.order.changed.item', {})
+    assert len(djmail.outbox) == 1
+
+
 @pytest.mark.django_db
 def test_notification_enabled_global_ignored_specific(event, order, user, monkeypatch_on_commit):
     djmail.outbox = []

src/tests/base/test_webhooks.py

@@ -0,0 +1,199 @@
+import json
+from datetime import timedelta
+from decimal import Decimal
+
+import pytest
+import responses
+from django.db import transaction
+from django.utils.timezone import now
+
+from pretix.base.models import Event, Item, Order, OrderPosition, Organizer
+
+
+@pytest.fixture
+def organizer():
+    return Organizer.objects.create(name='Dummy', slug='dummy')
+
+
+@pytest.fixture
+def event(organizer):
+    event = Event.objects.create(
+        organizer=organizer, name='Dummy', slug='dummy',
+        date_from=now()
+    )
+    return event
+
+
+@pytest.fixture
+def webhook(organizer, event):
+    wh = organizer.webhooks.create(
+        enabled=True,
+        target_url='https://google.com',
+        all_events=False
+    )
+    wh.limit_events.add(event)
+    wh.listeners.create(action_type='pretix.event.order.placed')
+    wh.listeners.create(action_type='pretix.event.order.paid')
+    return wh
+
+
+@pytest.fixture
+def order(event):
+    o = Order.objects.create(
+        code='FOO', event=event, email='dummy@dummy.test',
+        status=Order.STATUS_PENDING, locale='en',
+        datetime=now(), expires=now() + timedelta(days=10),
+        total=Decimal('46.00'),
+    )
+    tr19 = event.tax_rules.create(rate=Decimal('19.00'))
+    ticket = Item.objects.create(event=event, name='Early-bird ticket', tax_rule=tr19,
+                                 default_price=Decimal('23.00'), admission=True)
+    OrderPosition.objects.create(
+        order=o, item=ticket, variation=None,
+        price=Decimal("23.00"), attendee_name_parts={'full_name': "Peter"}, positionid=1
+    )
+    return o
+
+
+def force_str(v):
+    return v.decode() if isinstance(v, bytes) else str(v)
+
+
+@pytest.fixture
+def monkeypatch_on_commit(monkeypatch):
+    monkeypatch.setattr("django.db.transaction.on_commit", lambda t: t())
+
+
+@pytest.mark.django_db
+@responses.activate
+def test_webhook_trigger_event_specific(event, order, webhook, monkeypatch_on_commit):
+    responses.add_callback(
+        responses.POST, 'https://google.com',
+        callback=lambda r: (200, {}, 'ok'),
+        content_type='application/json',
+    )
+
+    with transaction.atomic():
+        le = order.log_action('pretix.event.order.paid', {})
+    assert len(responses.calls) == 1
+    assert json.loads(force_str(responses.calls[0].request.body)) == {
+        "notification_id": le.pk,
+        "organizer": "dummy",
+        "event": "dummy",
+        "code": "FOO",
+        "action": "pretix.event.order.paid"
+    }
+    first = webhook.calls.last()
+    assert first.webhook == webhook
+    assert first.target_url == 'https://google.com'
+    assert first.action_type == 'pretix.event.order.paid'
+    assert not first.is_retry
+    assert first.return_code == 200
+    assert first.success
+
+
+@pytest.mark.django_db
+@responses.activate
+def test_webhook_trigger_global(event, order, webhook, monkeypatch_on_commit):
+    webhook.limit_events.clear()
+    webhook.all_events = True
+    webhook.save()
+    responses.add(responses.POST, 'https://google.com', status=200)
+    with transaction.atomic():
+        le = order.log_action('pretix.event.order.paid', {})
+    assert len(responses.calls) == 1
+    assert json.loads(force_str(responses.calls[0].request.body)) == {
+        "notification_id": le.pk,
+        "organizer": "dummy",
+        "event": "dummy",
+        "code": "FOO",
+        "action": "pretix.event.order.paid"
+    }
+
+
+@pytest.mark.django_db
+@responses.activate
+def test_webhook_trigger_global_wildcard(event, order, webhook, monkeypatch_on_commit):
+    webhook.listeners.create(action_type="pretix.event.order.changed.*")
+    webhook.limit_events.clear()
+    webhook.all_events = True
+    webhook.save()
+    responses.add(responses.POST, 'https://google.com', status=200)
+    with transaction.atomic():
+        le = order.log_action('pretix.event.order.changed.item', {})
+    assert len(responses.calls) == 1
+    assert json.loads(force_str(responses.calls[0].request.body)) == {
+        "notification_id": le.pk,
+        "organizer": "dummy",
+        "event": "dummy",
+        "code": "FOO",
+        "action": "pretix.event.order.changed.item"
+    }
+
+
+@pytest.mark.django_db
+@responses.activate
+def test_webhook_ignore_wrong_action_type(event, order, webhook, monkeypatch_on_commit):
+    responses.add(responses.POST, 'https://google.com', status=200)
+    with transaction.atomic():
+        order.log_action('pretix.event.order.changed.item', {})
+    assert len(responses.calls) == 0
+
+
+@pytest.mark.django_db
+@responses.activate
+def test_webhook_ignore_disabled(event, order, webhook, monkeypatch_on_commit):
+    webhook.enabled = False
+    webhook.save()
+    responses.add(responses.POST, 'https://google.com', status=200)
+    with transaction.atomic():
+        order.log_action('pretix.event.order.changed.item', {})
+    assert len(responses.calls) == 0
+
+
+@pytest.mark.django_db
+@responses.activate
+def test_webhook_ignore_wrong_event(event, order, webhook, monkeypatch_on_commit):
+    webhook.limit_events.clear()
+    responses.add(responses.POST, 'https://google.com', status=200)
+    with transaction.atomic():
+        order.log_action('pretix.event.order.changed.item', {})
+    assert len(responses.calls) == 0
+
+
+@pytest.mark.django_db
+@pytest.mark.xfail(reason="retries can't be tested with celery_always_eager")
+@responses.activate
+def test_webhook_retry(event, order, webhook, monkeypatch_on_commit):
+    responses.add(responses.POST, 'https://google.com', status=500)
+    responses.add(responses.POST, 'https://google.com', status=200)
+    with transaction.atomic():
+        order.log_action('pretix.event.order.paid', {})
+    assert len(responses.calls) == 2
+    second = webhook.objects.first()
+    first = webhook.objects.last()
+
+    assert first.webhook == webhook
+    assert first.target_url == 'https://google.com'
+    assert first.action_type == 'pretix.event.order.paid'
+    assert not first.is_retry
+    assert first.return_code == 500
+    assert not first.success
+
+    assert second.webhook == webhook
+    assert second.target_url == 'https://google.com'
+    assert second.action_type == 'pretix.event.order.paid'
+    assert first.is_retry
+    assert first.return_code == 200
+    assert first.success
+
+
+@pytest.mark.django_db
+@responses.activate
+def test_webhook_disable_gone(event, order, webhook, monkeypatch_on_commit):
+    responses.add(responses.POST, 'https://google.com', status=410)
+    with transaction.atomic():
+        order.log_action('pretix.event.order.paid', {})
+    assert len(responses.calls) == 1
+    webhook.refresh_from_db()
+    assert not webhook.enabled

src/tests/control/test_checkins.py

@@ -262,7 +262,7 @@ def test_manual_checkins(client, checkin_list_env):
     })
     assert checkin_list_env[5][3].checkins.exists()
     assert LogEntry.objects.filter(
-        action_type='pretix.control.views.checkin', object_id=checkin_list_env[5][3].order.pk
+        action_type='pretix.event.checkin', object_id=checkin_list_env[5][3].order.pk
     ).exists()
 
 
@@ -279,10 +279,10 @@ def test_manual_checkins_revert(client, checkin_list_env):
     })
     assert not checkin_list_env[5][3].checkins.exists()
     assert LogEntry.objects.filter(
-        action_type='pretix.control.views.checkin', object_id=checkin_list_env[5][3].order.pk
+        action_type='pretix.event.checkin', object_id=checkin_list_env[5][3].order.pk
     ).exists()
     assert LogEntry.objects.filter(
-        action_type='pretix.control.views.checkin.reverted', object_id=checkin_list_env[5][3].order.pk
+        action_type='pretix.event.checkin.reverted', object_id=checkin_list_env[5][3].order.pk
     ).exists()
 
 

src/tests/control/test_permissions.py

@@ -137,6 +137,10 @@ organizer_urls = [
     'organizer/abc/device/1/edit',
     'organizer/abc/device/1/connect',
     'organizer/abc/device/1/revoke',
+    'organizer/abc/webhooks',
+    'organizer/abc/webhook/add',
+    'organizer/abc/webhook/1/edit',
+    'organizer/abc/webhook/1/logs',
 ]
 
 
@@ -378,6 +382,15 @@ organizer_permission_urls = [
     ("can_change_teams", "organizer/dummy/team/1/delete", 200),
     ("can_change_organizer_settings", "organizer/dummy/edit", 200),
     ("can_change_organizer_settings", "organizer/dummy/settings/display", 200),
+    ("can_change_organizer_settings", "organizer/dummy/devices", 200),
+    ("can_change_organizer_settings", "organizer/dummy/device/add", 200),
+    ("can_change_organizer_settings", "organizer/dummy/device/1/edit", 404),
+    ("can_change_organizer_settings", "organizer/dummy/device/1/connect", 404),
+    ("can_change_organizer_settings", "organizer/dummy/device/1/revoke", 404),
+    ("can_change_organizer_settings", "organizer/dummy/webhooks", 200),
+    ("can_change_organizer_settings", "organizer/dummy/webhook/add", 200),
+    ("can_change_organizer_settings", "organizer/dummy/webhook/1/edit", 404),
+    ("can_change_organizer_settings", "organizer/dummy/webhook/1/logs", 404),
 ]
 
 

src/tests/control/test_views.py

@@ -100,6 +100,7 @@ def logged_in_client(client, event):
     ('/control/organizer/{orga}/edit', 200),
     ('/control/organizer/{orga}/teams', 200),
     ('/control/organizer/{orga}/devices', 200),
+    ('/control/organizer/{orga}/webhooks', 200),
 
     ('/control/events/', 200),
     ('/control/events/add', 200),

src/tests/control/test_webhooks.py

@@ -0,0 +1,101 @@
+import pytest
+from django.utils.timezone import now
+
+from pretix.api.models import WebHook
+from pretix.base.models import Event, Organizer, Team, User
+
+
+@pytest.fixture
+def organizer():
+    return Organizer.objects.create(name='Dummy', slug='dummy')
+
+
+@pytest.fixture
+def event(organizer):
+    event = Event.objects.create(
+        organizer=organizer, name='Dummy', slug='dummy',
+        date_from=now()
+    )
+    return event
+
+
+@pytest.fixture
+def webhook(organizer, event):
+    wh = organizer.webhooks.create(
+        enabled=True,
+        target_url='https://google.com',
+        all_events=False
+    )
+    wh.limit_events.add(event)
+    wh.listeners.create(action_type='pretix.event.order.placed')
+    wh.listeners.create(action_type='pretix.event.order.paid')
+    return wh
+
+
+@pytest.fixture
+def admin_user(admin_team):
+    u = User.objects.create_user('dummy@dummy.dummy', 'dummy')
+    admin_team.members.add(u)
+    return u
+
+
+@pytest.fixture
+def admin_team(organizer):
+    return Team.objects.create(organizer=organizer, can_change_organizer_settings=True, name='Admin team')
+
+
+@pytest.mark.django_db
+def test_list_of_webhooks(event, admin_user, client, webhook):
+    client.login(email='dummy@dummy.dummy', password='dummy')
+    resp = client.get('/control/organizer/dummy/webhooks')
+    assert 'https://google.com' in resp.rendered_content
+
+
+@pytest.mark.django_db
+def test_create_webhook(event, admin_user, admin_team, client):
+    client.login(email='dummy@dummy.dummy', password='dummy')
+    client.post('/control/organizer/dummy/webhook/add', {
+        'target_url': 'https://google.com',
+        'enabled': 'on',
+        'events': 'pretix.event.order.paid',
+        'limit_events': str(event.pk),
+    }, follow=True)
+    w = WebHook.objects.last()
+    assert w.target_url == "https://google.com"
+    assert w.limit_events.count() == 1
+    assert list(w.listeners.values_list('action_type', flat=True)) == ['pretix.event.order.paid']
+    assert not w.all_events
+
+
+@pytest.mark.django_db
+def test_update_webhook(event, admin_user, admin_team, webhook, client):
+    client.login(email='dummy@dummy.dummy', password='dummy')
+    client.post('/control/organizer/dummy/webhook/{}/edit'.format(webhook.pk), {
+        'target_url': 'https://google.com',
+        'enabled': 'on',
+        'events': ['pretix.event.order.paid', 'pretix.event.order.canceled'],
+        'limit_events': str(event.pk),
+    }, follow=True)
+    webhook.refresh_from_db()
+    assert webhook.target_url == "https://google.com"
+    assert webhook.limit_events.count() == 1
+    assert list(webhook.listeners.values_list('action_type', flat=True)) == ['pretix.event.order.canceled',
+                                                                             'pretix.event.order.paid']
+    assert not webhook.all_events
+
+
+@pytest.mark.django_db
+def test_webhook_logs(event, admin_user, admin_team, webhook, client):
+    client.login(email='dummy@dummy.dummy', password='dummy')
+    webhook.calls.create(
+        webhook=webhook,
+        action_type='pretix.event.order.paid',
+        target_url=webhook.target_url,
+        is_retry=False,
+        execution_time=2,
+        return_code=0,
+        payload='foo',
+        response_body='bar'
+    )
+    resp = client.get('/control/organizer/dummy/webhook/{}/logs'.format(webhook.pk))
+    assert 'pretix.event.order.paid' in resp.rendered_content