Banktransfer: Big import refactoring

src/pretix/base/views/async.py

@@ -0,0 +1,134 @@
+import logging
+
+from django.conf import settings
+from django.contrib import messages
+from django.http import JsonResponse
+from django.shortcuts import redirect, render
+from django.utils.translation import ugettext as _
+
+logger = logging.getLogger('pretix.base.async')
+
+
+class AsyncAction:
+    task = None
+    success_url = None
+    error_url = None
+
+    def do(self, *args):
+        if settings.HAS_CELERY:
+            from pretix.celery import app
+
+            if hasattr(self.task, 'task') and isinstance(self.task.task, app.Task):
+                return self._do_celery(args)
+            else:
+                raise TypeError('Method has no task attached')
+        else:
+            return self._do_sync(args)
+
+    def get_success_url(self, value):
+        return self.success_url
+
+    def get_error_url(self):
+        return self.error_url
+
+    def get_check_url(self, task_id, ajax):
+        return self.request.path + '?async_id=%s' % task_id + ('&ajax=1' if ajax else '')
+
+    def get(self, request, *args, **kwargs):
+        if 'async_id' in request.GET and settings.HAS_CELERY:
+            return self.get_result(request)
+        return self.http_method_not_allowed(request)
+
+    def _return_celery_result(self, res, timeout=.5):
+        import celery.exceptions
+
+        if not res.ready():
+            try:
+                res.get(timeout=timeout)
+            except celery.exceptions.TimeoutError:
+                pass
+        ready = res.ready()
+        data = {
+            'async_id': res.id,
+            'ready': ready
+        }
+        if ready:
+            if res.successful() and not isinstance(res.info, Exception):
+                smes = self.get_success_message(res.info)
+                if smes:
+                    messages.success(self.request, smes)
+                # TODO: Do not store message if the ajax client stats that it will not redirect
+                # but handle the mssage itself
+                data.update({
+                    'redirect': self.get_success_url(res.info),
+                    'message': str(self.get_success_message(res.info))
+                })
+            else:
+                messages.error(self.request, self.get_error_message(res.info))
+                # TODO: Do not store message if the ajax client stats that it will not redirect
+                # but handle the mssage itself
+                data.update({
+                    'redirect': self.get_error_url(),
+                    'message': str(self.get_error_message(res.info))
+                })
+        return data
+
+    def get_result(self, request):
+        from celery.result import AsyncResult
+
+        res = AsyncResult(request.GET.get('async_id'))
+        if 'ajax' in self.request.GET:
+            return JsonResponse(self._return_celery_result(res, timeout=0.25))
+        else:
+            if res.ready():
+                if res.successful():
+                    return self.success(res.info)
+                else:
+                    return self.error(res.info)
+            return render(request, 'pretixpresale/waiting.html')
+
+    def _do_celery(self, args):
+        res = self.task.task.apply_async(args=args)
+        if 'ajax' in self.request.GET or 'ajax' in self.request.POST:
+            data = self._return_celery_result(res)
+            data['check_url'] = self.get_check_url(res.id, True)
+            return JsonResponse(data)
+        else:
+            return redirect(self.get_check_url(res.id, False))
+
+    def _do_sync(self, args):
+        try:
+            rs = getattr(self.__class__, 'task')(*args)
+            return self.success(rs)
+        except Exception as e:
+            logger.exception('Error while executing task synchronously')
+            return self.error(e)
+
+    def success(self, value):
+        smes = self.get_success_message(value)
+        if smes:
+            messages.success(self.request, smes)
+        if "ajax" in self.request.POST or "ajax" in self.request.GET:
+            return JsonResponse({
+                'ready': True,
+                'redirect': self.get_success_url(value),
+                'message': str(self.get_success_message(value))
+            })
+        return redirect(self.get_success_url(value))
+
+    def error(self, exception):
+        messages.error(self.request, self.get_error_message(exception))
+        if "ajax" in self.request.POST or "ajax" in self.request.GET:
+            return JsonResponse({
+                'ready': True,
+                'redirect': self.get_error_url(),
+                'message': str(self.get_error_message(exception))
+            })
+        return redirect(self.get_error_url())
+
+    def get_error_message(self, exception):
+        logger.error('Unexpected exception: %r' % exception)
+        return _('An unexpected error has occured.')
+
+    def get_success_message(self, value):
+        return _('The task has been completed.')

src/pretix/control/templates/pretixcontrol/base.html

@@ -13,6 +13,7 @@
 		{% compress js %}
             <script type="text/javascript" src="{% static "jquery/js/jquery-2.1.1.min.js" %}"></script>
             <script type="text/javascript" src="{% static "js/jquery.formset.js" %}"></script>
+            <script type="text/javascript" src="{% static "typeahead/typeahead.bundle.js" %}"></script>
             <script type="text/javascript" src="{% static "bootstrap/js/bootstrap.js" %}"></script>
             <script type="text/javascript" src="{% static "pretixcontrol/js/metisMenu.min.js" %}"></script>
             <script type="text/javascript" src="{% static "pretixcontrol/js/sb-admin-2.js" %}"></script>
@@ -132,5 +133,14 @@
                 </div>
             </div>
         </div>
+        <div id="ajaxerr">
+        </div>
+        <div id="loadingmodal">
+            <i class="fa fa-cog big-rotating-icon"></i>
+            <h1>{% trans "We are processing your request…" %}</h1>
+            <p>
+                {% trans "If this takes longer than a few minutes, please contact us." %}
+            </p>
+        </div>
 	</body>
 </html>

src/pretix/control/views/orders.py

@@ -42,8 +42,8 @@ from pretix.multidomain.urlreverse import build_absolute_uri
 class OrderList(EventPermissionRequiredMixin, ListView):
     model = Order
     context_object_name = 'orders'
-    template_name = 'pretixcontrol/orders/index.html'
     paginate_by = 30
+    template_name = 'pretixcontrol/orders/index.html'
     permission = 'can_view_orders'
 
     def get_queryset(self):

src/pretix/plugins/banktransfer/__init__.py

@@ -20,6 +20,7 @@ class BankTransferApp(AppConfig):
 
     def ready(self):
         from . import signals  # NOQA
+        from . import tasks  # NOQA
 
     @cached_property
     def compatibility_warnings(self):
@@ -28,10 +29,6 @@ class BankTransferApp(AppConfig):
             import chardet  # NOQA
         except ImportError:
             errs.append(_("Install the python package 'chardet' for better CSV import capabilities."))
-        try:
-            import defusedxml  # NOQA
-        except ImportError:
-            errs.append(_("Please install the python package 'defusedxml' for security reasons."))
         return errs
 
 

src/pretix/plugins/banktransfer/migrations/0001_initial.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2016-09-07 09:35
+from __future__ import unicode_literals
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('pretixbase', '0036_auto_20160902_0755'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='BankImportJob',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('state', models.CharField(choices=[('running', 'pending'), ('running', 'running'), ('error', 'error'), ('completed', 'completed')], default='running', max_length=32)),
+                ('event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pretixbase.Event')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='BankTransaction',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('state', models.CharField(choices=[('imported', 'imported, unchecked'), ('nomatch', 'no match'), ('invalid', 'not valid'), ('error', 'error'), ('valid', 'valid'), ('already', 'valid, already paid'), ('discarded', 'manually discarded')], default='imported', max_length=32)),
+                ('message', models.TextField()),
+                ('checksum', models.CharField(db_index=True, max_length=190)),
+                ('payer', models.TextField(blank=True)),
+                ('reference', models.TextField(blank=True)),
+                ('amount', models.DecimalField(decimal_places=2, max_digits=10)),
+                ('date', models.DateField()),
+                ('event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pretixbase.Event')),
+                ('import_job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='banktransfer.BankImportJob')),
+                ('order', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='pretixbase.Order')),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name='banktransaction',
+            unique_together=set([('event', 'checksum')]),
+        ),
+    ]

src/pretix/plugins/banktransfer/models.py

@@ -0,0 +1,68 @@
+import hashlib
+import re
+
+from django.db import models
+
+
+class BankImportJob(models.Model):
+    STATE_PENDING = 'pending'
+    STATE_RUNNING = 'running'
+    STATE_ERROR = 'error'
+    STATE_COMPLETED = 'completed'
+    STATES = (
+        (STATE_PENDING, 'pending'),
+        (STATE_RUNNING, 'running'),
+        (STATE_ERROR, 'error'),
+        (STATE_COMPLETED, 'completed'),
+    )
+
+    event = models.ForeignKey('pretixbase.Event')
+    created = models.DateTimeField(auto_now_add=True)
+    state = models.CharField(max_length=32, choices=STATES, default=STATE_PENDING)
+
+
+class BankTransaction(models.Model):
+    STATE_UNCHECKED = 'imported'
+    STATE_NOMATCH = 'nomatch'
+    STATE_INVALID = 'invalid'
+    STATE_ERROR = 'error'
+    STATE_VALID = 'valid'
+    STATE_DISCARDED = 'discarded'
+    STATE_DUPLICATE = 'already'
+
+    STATES = (
+        (STATE_UNCHECKED, 'imported, unchecked'),
+        (STATE_NOMATCH, 'no match'),
+        (STATE_INVALID, 'not valid'),
+        (STATE_ERROR, 'error'),
+        (STATE_VALID, 'valid'),
+        (STATE_DUPLICATE, 'valid, already paid'),
+        (STATE_DISCARDED, 'manually discarded'),
+    )
+
+    event = models.ForeignKey('pretixbase.Event')
+    import_job = models.ForeignKey('BankImportJob', related_name='transactions')
+    state = models.CharField(max_length=32, choices=STATES, default=STATE_UNCHECKED)
+    message = models.TextField()
+    checksum = models.CharField(max_length=190, db_index=True)
+    payer = models.TextField(blank=True)
+    reference = models.TextField(blank=True)
+    amount = models.DecimalField(max_digits=10, decimal_places=2)
+    date = models.DateField()
+    order = models.ForeignKey('pretixbase.Order', null=True, blank=True)
+
+    def calculate_checksum(self):
+        clean = re.compile('[^a-zA-Z0-9.-]')
+        hasher = hashlib.sha1()
+        hasher.update(clean.sub('', self.payer.lower()).encode('utf-8'))
+        hasher.update(clean.sub('', self.reference.lower()).encode('utf-8'))
+        hasher.update(clean.sub('', str(self.amount).lower()).encode('utf-8'))
+        hasher.update(clean.sub('', str(self.date).lower()).encode('utf-8'))
+        return str(hasher.hexdigest())
+
+    def shred_private_data(self):
+        self.payer = ""
+        self.reference = ""
+
+    class Meta:
+        unique_together = ('event', 'checksum')

src/pretix/plugins/banktransfer/signals.py

@@ -1,9 +1,11 @@
 from django.core.urlresolvers import resolve, reverse
 from django.dispatch import receiver
+from django.template import Context
+from django.template.loader import get_template
 from django.utils.translation import ugettext_lazy as _
 
 from pretix.base.signals import register_payment_providers
-from pretix.control.signals import nav_event
+from pretix.control.signals import html_head, nav_event
 
 from .payment import BankTransfer
 
@@ -29,3 +31,14 @@ def control_nav_import(sender, request=None, **kwargs):
             'icon': 'upload',
         }
     ]
+
+
+@receiver(html_head, dispatch_uid="banktransfer_html_head")
+def html_head_presale(sender, request=None, **kwargs):
+    url = resolve(request.path_info)
+    if url.namespace == 'plugins:banktransfer':
+        template = get_template('pretixplugins/banktransfer/control_head.html')
+        ctx = Context({})
+        return template.render(ctx)
+    else:
+        return ""

src/pretix/plugins/banktransfer/static/pretixplugins/banktransfer/ui.css

@@ -0,0 +1,10 @@
+table.transaction-list td.actions {
+    max-width: 100px;
+}
+table.transaction-list td.actions .input-group {
+    display: inline-block;
+    width: 110px;
+}
+table.transaction-list td.actions .input-group button {
+    line-height: 1.43;
+}

src/pretix/plugins/banktransfer/static/pretixplugins/banktransfer/ui.js

@@ -0,0 +1,132 @@
+/*global $, gettext*/
+var bankimport_transactionlist = {
+
+    _btn_click: function (e) {
+        console.log(e.delegateTarget);
+        var trans_id = parseInt($(e.delegateTarget).attr("name").split("_")[1]);
+        var value = $(e.delegateTarget).val();
+        if (value === "discard") {
+            bankimport_transactionlist.discard(trans_id);
+        } else if (value === "accept") {
+            bankimport_transactionlist.accept(trans_id);
+        } else if (value === "retry") {
+            bankimport_transactionlist.retry(trans_id);
+        } else if (value === "assign") {
+            bankimport_transactionlist.assign(trans_id);
+        }
+        return false;
+    },
+
+    _action: function (id, action, success) {
+        $("tr[data-id=" + id + "] button").prop("disabled", true);
+        var data = {
+            "csrfmiddlewaretoken": $("[name=csrfmiddlewaretoken]").val()
+        };
+        data["action_" + id] = action;
+        $.ajax({
+            "method": "POST",
+            "url": $(".transaction-list").attr("data-url"),
+            "data": data,
+            "dataType": "json",
+            "success": function (data) {
+                if (data.status == "ok") {
+                    $("tr[data-id=" + id + "]").removeClass("has-error");
+                    success();
+                } else {
+                    $("tr[data-id=" + id + "] button").prop("disabled", false);
+                    $("tr[data-id=" + id + "] .help-block").remove();
+                    $("tr[data-id=" + id + "]").addClass("has-error");
+                    $("<p>").addClass("help-block").text(data.message).appendTo($("tr[data-id=" + id + "] td.actions"));
+                }
+            }
+        });
+    },
+
+    discard: function (id) {
+        bankimport_transactionlist._action(id, "discard", function () {
+            $("tr[data-id=" + id + "] td").remove();
+        });
+    },
+
+    retry: function (id) {
+        bankimport_transactionlist._action(id, "retry", function () {
+            $("tr[data-id=" + id + "] td.actions").html('').text(gettext("Marked as paid"));
+        });
+    },
+
+    accept: function (id) {
+        bankimport_transactionlist._action(id, "accept", function () {
+            $("tr[data-id=" + id + "] td.actions").html('').text(gettext("Marked as paid"));
+        });
+    },
+
+    assign: function (id) {
+        bankimport_transactionlist._action(id, "assign:" + $("tr[data-id=" + id + "] input.form-control:not(.tt-hint)").val(), function () {
+            $("tr[data-id=" + id + "] td.actions").html('').text(gettext("Marked as paid"));
+        });
+    },
+
+    typeahead_source: function () {
+        return new Bloodhound({
+            datumTokenizer: Bloodhound.tokenizers.obj.whitespace('value'),
+            queryTokenizer: Bloodhound.tokenizers.whitespace,
+            remote: {
+                url: $(".transaction-list").attr("data-url"),
+                prepare: function (query, settings) {
+                    settings.url = settings.url + '?query=' + encodeURIComponent(query);
+                    return settings;
+                },
+                transform: function (object) {
+                    var results = object.results;
+                    var suggs = [];
+                    var reslen = results.length;
+                    for (var i = 0; i < reslen; i++) {
+                        suggs.push(results[i]);
+                    }
+                    return suggs;
+                }
+            }
+        });
+    },
+
+    init: function () {
+        if ($(".transaction-list").length) {
+            $(".transaction-list button").click(bankimport_transactionlist._btn_click);
+            $(".transaction-list .form-control").typeahead(null, {
+                minLength: 2,
+                name: 'order-dataset',
+                source: bankimport_transactionlist.typeahead_source(),
+                display: function (obj) {
+                    return obj.code;
+                },
+                templates: {
+                    suggestion: function (obj) {
+                        return '<div>' + obj.code + ' (' + obj.total + ', ' + obj.status + ')</div>';
+                    }
+                }
+            }).keypress(function (e) {
+                if (e.keyCode === 13) {
+                    $(this).parent().parent().find("button[value=assign]").click();
+                }
+            });
+        }
+
+        if ($("[data-job-waiting]").length) {
+            window.setInterval(bankimport_transactionlist.check_state, 750);
+        }
+    },
+
+    check_state: function () {
+        $.getJSON(location.pathname + '?ajax=1', function (data) {
+            if (data.state == 'running' || data.state == 'pending') {
+                window.setInterval(check_state, 750);
+            } else {
+                location.reload();
+            }
+        });
+    }
+};
+
+$(function () {
+    bankimport_transactionlist.init();
+});

src/pretix/plugins/banktransfer/tasks.py

@@ -0,0 +1,130 @@
+import json
+import logging
+import re
+from decimal import Decimal
+
+import dateutil.parser
+from django.conf import settings
+from django.db import transaction
+from django.utils.translation import ugettext_noop
+
+from pretix.base.i18n import language
+from pretix.base.models import Event, Order, Quota
+from pretix.base.services.mail import SendMailException
+from pretix.base.services.orders import mark_order_paid
+
+from .models import BankImportJob, BankTransaction
+
+logger = logging.getLogger(__name__)
+
+
+def _handle_transaction(event: Event, trans: BankTransaction, code: str):
+    try:
+        trans.order = event.orders.get(code=code)
+    except Order.DoesNotExist:
+        normalized_code = Order.normalize_code(code)
+        try:
+            trans.order = event.orders.get(code=normalized_code)
+        except Order.DoesNotExist:
+            trans.state = BankTransaction.STATE_NOMATCH
+            trans.save()
+            return
+
+    if trans.order.status == Order.STATUS_PAID:
+        trans.state = BankTransaction.STATE_DUPLICATE
+    elif trans.order.status == Order.STATUS_REFUNDED:
+        trans.state = BankTransaction.STATE_ERROR
+        trans.message = ugettext_noop('The order has already been refunded.')
+    elif trans.order.status == Order.STATUS_CANCELLED:
+        trans.state = BankTransaction.STATE_ERROR
+        trans.message = ugettext_noop('The order has already been cancelled.')
+    elif trans.amount != trans.order.total:
+        trans.state = BankTransaction.STATE_INVALID
+        trans.message = ugettext_noop('The transaction amount is incorrect.')
+    else:
+        try:
+            mark_order_paid(trans.order, provider='banktransfer', info=json.dumps({
+                'reference': trans.reference,
+                'date': trans.date.isoformat(),
+                'payer': trans.payer,
+                'trans_id': trans.pk
+            }))
+        except Quota.QuotaExceededException as e:
+            trans.state = BankTransaction.STATE_ERROR
+            trans.message = str(e)
+        except SendMailException:
+            trans.state = BankTransaction.STATE_ERROR
+            trans.message = ugettext_noop('Problem sending email.')
+        else:
+            trans.state = BankTransaction.STATE_VALID
+    trans.save()
+
+
+def _get_unknown_transactions(event: Event, job: BankImportJob, data: list):
+    amount_pattern = re.compile("[^0-9.-]")
+    known_checksums = set(t['checksum'] for t in BankTransaction.objects.filter(event=event).values('checksum'))
+
+    transactions = []
+    for row in data:
+        amount = amount_pattern.sub("", row['amount'].replace(",", "."))
+        try:
+            amount = Decimal(amount)
+        except:
+            logger.exception('Could not parse amount of transaction: {}'.format(amount))
+            amount = Decimal("0.00")
+
+        date = dateutil.parser.parse(row['date'])
+        trans = BankTransaction(event=event, import_job=job,
+                                payer=row['payer'],
+                                reference=row['reference'],
+                                amount=amount,
+                                date=date)
+        trans.checksum = trans.calculate_checksum()
+        if trans.checksum not in known_checksums:
+            trans.state = BankTransaction.STATE_UNCHECKED
+            trans.save()
+            transactions.append(trans)
+            known_checksums.add(trans.checksum)
+
+    return transactions
+
+
+def process_banktransfers(event: int, job: int, data: list) -> None:
+    with language("en"):  # We'll translate error messages at display time
+        event = Event.objects.get(pk=event)
+        job = BankImportJob.objects.get(pk=job)
+        job.state = BankImportJob.STATE_RUNNING
+        job.save()
+
+        try:
+            transactions = _get_unknown_transactions(event, job, data)
+
+            code_len = settings.ENTROPY['order_code']
+            pattern = re.compile(event.slug.upper() + "[ -_]*([A-Z0-9]{%s})" % code_len)
+
+            for trans in transactions:
+                match = pattern.search(trans.reference.upper())
+
+                if match:
+                    code = match.group(1)
+                    with transaction.atomic():
+                        _handle_transaction(event, trans, code)
+                else:
+                    trans.state = BankTransaction.STATE_NOMATCH
+                    trans.save()
+        except Exception as e:
+            job.state = BankImportJob.STATE_ERROR
+            job.save()
+            raise e
+        else:
+            job.state = BankImportJob.STATE_COMPLETED
+            job.save()
+
+
+if settings.HAS_CELERY:
+    from pretix.celery import app
+
+    process_task = app.task(process_banktransfers)
+
+    def process_banktransfers(*args, **kwargs):
+        process_task.apply_async(args=args, kwargs=kwargs)

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/control_head.html

@@ -0,0 +1,6 @@
+{% load staticfiles %}
+{% load compress %}
+
+{% compress css %}
+    <link type="text/css" rel="stylesheet" href="{% static "pretixplugins/banktransfer/ui.css" %}">
+{% endcompress %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/hbci_log.html

@@ -1,9 +0,0 @@
-{% extends "pretixplugins/banktransfer/import_base.html" %}
-{% load i18n %}
-{% load bootstrap3 %}
-{% block inner %}
-    <div class="alert alert-danger">
-        {% trans "An error occured during the HBCI transaction." %}
-    </div>
-    <pre>{{ log }}</pre>
-{% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_base.html

@@ -1,8 +1,10 @@
 {% extends "pretixcontrol/event/base.html" %}
 {% load i18n %}
+{% load static %}
 {% block title %}{% trans "Import bank data" %}{% endblock %}
 {% block content %}
     <h1>{% trans "Import bank data" %}</h1>
     {% block inner %}
     {% endblock %}
+    <script type="application/javascript" src="{% static "pretixplugins/banktransfer/ui.js" %}"></script>
 {% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_confirm.html

@@ -1,54 +0,0 @@
-{% extends "pretixplugins/banktransfer/import_base.html" %}
-{% load i18n %}
-{% block inner %}
-    <p>{% blocktrans trimmed %}
-        We detected the following payments. Please review them and click the 'Confirm' button below.
-    {% endblocktrans %}</p>
-    <form method="post" action="">
-        {% csrf_token %}
-        <table class="table table-condensed">
-            <thead>
-                <tr>
-                    <th></th>
-                    <th>{% trans "Date" %}</th>
-                    <th>{% trans "Reference" %}</th>
-                    <th>{% trans "Amount" %}</th>
-                    <th>{% trans "Payer" %}</th>
-                    <th>{% trans "Order" %}</th>
-                    <th>{% trans "Status" %}</th>
-                </tr>
-            </thead>
-            <tbody>
-                {% for row in rows %}
-                    <tr class="{{ row.class }}">
-                        <th>{% if row.ok %}
-                            <input type="checkbox" name="mark_paid" value="{{ row.order.code }}"
-                                checked="checked" />
-                            <input type="hidden" name="date_{{ row.order.code }}" value="{{ row.date }}" />
-                            <input type="hidden" name="payer_{{ row.order.code }}" value="{{ row.payer }}" />
-                            <input type="hidden" name="reference_{{ row.order.code }}" value="{{ row.reference }}" />
-                        {% endif %}</th>
-                        <td>{{ row.date }}</td>
-                        <td>{{ row.reference }}</td>
-                        <td>{{ row.amount }}</td>
-                        <td>{{ row.payer }}</td>
-                        <td>{% if row.order %}
-                            <a href="{% url "control:event.order" event=request.event.slug organizer=request.event.organizer.slug code=row.order.code %}"
-                                    target="_blank">
-                                {{ row.order.code }}
-                            </a>
-                            {% endif %}
-                        </td>
-                        <td>{{ row.message }}</td>
-                    </tr>
-                {% endfor %}
-            </tbody>
-            <input type="hidden" name="confirm" value="true" />
-
-        </table>
-        <button class="btn btn-primary btn-lg pull-right helper-space-below" type="submit">
-            <span class="icon icon-upload"></span> {% trans "Confirm" %}
-        </button>
-        <div class="clearfix"></div>
-    </form>
-{% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_form.html

@@ -16,7 +16,7 @@
             <form action="" method="post" enctype="multipart/form-data" class="form-inline">
                 {% csrf_token %}
                 <div class="form-group">
-                    <label for="file">{% trans "Import file" %}: </label> <input id="file" type="file" name="file" />
+                    <label for="file">{% trans "Import file" %}: </label> <input id="file" type="file" name="file"/>
                 </div>
                 <div class="clearfix"></div>
                 <button class="btn btn-primary pull-right" type="submit">
@@ -25,38 +25,32 @@
             </form>
         </div>
     </div>
-    <!--
-    <div class="panel panel-default">
-        <div class="panel-heading">
-            <h3 class="panel-title">{% trans "HBCI import" %}</h3>
-        </div>
-        <div class="panel-body">
-            {% if hbci_available %}
-            <form action="" method="post" class="form-horizontal">
-                {% csrf_token %}
-                {% bootstrap_form hbci_form layout='horizontal' %}
-                <div class="clearfix"></div>
+    {% if transactions_unhandled|length > 0 %}
+        <div class="panel panel-default">
+            <div class="panel-heading">
+                <h3 class="panel-title">{% trans "Unresolved transactions" %}</h3>
+            </div>
+            <div class="panel-body">
                 <p>
-                    {% trans "Please note that this step might take a few minutes." %}
+                    <form class="form-inline helper-display-inline" action="" method="get">
+                        <input type="text" name="search" class="form-control" placeholder="{% trans "Search" %}" value="{{ request.GET.search }}">
+                    </form>
+                    <button class="btn btn-primary" type="submit">{% trans "Filter" %}</button>
+                    {% if not request.GET.search %}
+                        <form action="" method="post" class="helper-display-inline pull-right">
+                            {% csrf_token %}
+                            <button class="btn btn-danger" type="submit" name="discard" value="all">
+                                <span class="fa fa-trash"></span>
+                                {% trans "Discard all" %}
+                            </button>
+                        </form>
+                    {% endif %}
                 </p>
-                <div class="alert alert-warning">
-                    {% blocktrans trimmed %}
-                        Your HBCI PIN code is only stored on the server for a very short time and
-                        immediately deleted after the communication with the bank is over. However,
-                        we <strong>strongly advise</strong> you to ask your bank for HBCI credentials
-                        that have <strong>read-only</strong> access to your bank account.
-                    {% endblocktrans %}
-                </div>
-                <button class="btn btn-primary pull-right" type="submit">
-                    <span class="icon icon-upload"></span> {% trans "Import" %}
-                </button>
-            </form>
-            {% else %}
-                <div class="alert alert-error">
-                    {% trans "HBCI is only available with aqbanking-cli and aqhbci-tool4 installed." %}
-                </div>
-            {% endif %}
+
+                {% include "pretixcontrol/pagination.html" %}
+                {% include "pretixplugins/banktransfer/transaction_list.html" with list=transactions_unhandled %}
+                {% include "pretixcontrol/pagination.html" %}
+            </div>
         </div>
-    </div>
-    -->
+    {% endif %}
 {% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/job_detail.html

@@ -0,0 +1,47 @@
+{% extends "pretixplugins/banktransfer/import_base.html" %}
+{% load i18n %}
+{% block inner %}
+    <h2>{% trans "Import result" %}</h2>
+    {% if job.state == "running" or job.state == "pending" %}
+        <div class="empty-collection" data-job-waiting>
+            <p>
+                <span class="fa big-grey-icon fa-cog fa big-rotating-icon"></span>
+            </p>
+            <p>
+                {% trans "The result of your import is in progress. Please be patient while we process the data..." %}
+            </p>
+        </div>
+    {% else %}
+        {% if job.state == "error" %}
+            <div class="alert alert-danger">
+                {% trans "An internal error occured during processing your data." %}
+            </div>
+        {% elif transactions_ignored == 0 and transactions_invalid == 0 and transactions_valid == 0 %}
+            <div class="alert alert-info">
+                {% trans "Your import did not contain any transactions that you did not import before." %}
+            </div>
+        {% endif %}
+
+        <table class="table table-condensed">
+            <tr class="success">
+                <td>{{ transactions_valid }}</td>
+                <td>{% trans "Orders marked as paid" %}</td>
+            </tr>
+            <tr class="warning">
+                <td>{{ transactions_invalid }}</td>
+                <td>{% trans "Invalid payments" %}</td>
+            </tr>
+            <tr class="info">
+                <td>{{ transactions_ignored }}</td>
+                <td>{% trans "Ignored payments" %}</td>
+            </tr>
+        </table>
+        {% if transactions_ignored or transactions_invalid %}
+            <a href="{% url "plugins:banktransfer:import" event=request.event.slug organizer=request.organizer.slug %}"
+                class="btn btn-primary">
+                {% trans "Review invalid and ignored payments" %} &raquo;
+            </a>
+        {% endif %}
+
+    {% endif %}
+{% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/transaction_list.html

@@ -0,0 +1,86 @@
+{% load i18n %}
+{% load staticfiles %}
+<div class="table-responsive">
+    {% csrf_token %}
+    <table class="table table-condensed transaction-list" data-url="{% url "plugins:banktransfer:import.action" event=request.event.slug organizer=request.event.organizer.slug %}">
+        <thead>
+        <tr>
+            <th></th>
+            <th>{% trans "Date" %}</th>
+            <th>{% trans "Payer and reference" %}</th>
+            <th>{% trans "Amount" %}</th>
+            <th>{% trans "Result" %}</th>
+            <th>{% trans "Order" %}</th>
+        </tr>
+        </thead>
+        <tbody>
+        {% for trans in list %}
+            <tr data-id="{{ trans.id }}">
+                <td class="actions">
+                    {% if trans.order and trans.state == 'invalid' %}
+                        <div class="btn-group" role="group">
+                            <button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="accept"
+                                data-toggle="tooltip" title="{% trans "Accept anyway" %}" data-placement="right">
+                                <span class="fa fa-check"></span>
+                            </button>
+                            <button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="discard"
+                                data-toggle="tooltip" title="{% trans "Discard" %}">
+                                <span class="fa fa-trash"></span>
+                            </button>
+                        </div>
+                    {% elif trans.state == 'nomatch' %}
+                        <input type="text" class="form-control" placeholder="{% trans "Order code" %}">
+                        <div class="btn-group" role="group">
+                            <button class="btn btn-default" type="button" name="action_{{ trans.id }}"
+                                    value="assign" data-toggle="tooltip" title="{% trans "Assign to order" %}" data-placement="right">
+                                <span class="fa fa-check"></span>
+                            </button>
+                            <button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="discard"
+                                data-toggle="tooltip" title="{% trans "Discard" %}">
+                                <span class="fa fa-trash"></span>
+                            </button>
+                        </div>
+                    {% elif trans.state == 'error' %}
+                        <div class="btn-group" role="group">
+                            <button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="retry"
+                                data-toggle="tooltip" title="{% trans "Retry" %}" data-placement="right">
+                                <span class="fa fa-refresh"></span>
+                            </button>
+                            <button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="discard"
+                                data-toggle="tooltip" title="{% trans "Discard" %}">
+                                <span class="fa fa-trash"></span>
+                            </button>
+                        </div>
+                    {% endif %}
+                </td>
+                <td>{{ trans.date|date:"SHORT_DATE_FORMAT" }}</td>
+                <td>{{ trans.payer }}<br/>{{ trans.reference }}</td>
+                <td>{{ trans.amount|floatformat:2 }}</td>
+                <td>
+                    {% if trans.message %}
+                        {% trans trans.message %}
+                    {% elif trans.state == 'nomatch' %}
+                        {% trans "No order code detected" %}
+                    {% elif trans.state == 'invalid' %}
+                        {% trans "Invalid for this order" %}
+                    {% elif trans.state == 'error' %}
+                        {% trans "Error while processing" %}
+                    {% elif trans.state == 'valid' %}
+                        {% trans "The order is already marked as paid" %}
+                    {% elif trans.state == 'already' %}
+                        {% trans "Order already paid" %}
+                    {% endif %}
+                </td>
+                <td>
+                    {% if trans.order %}
+                        <a href="{% url "control:event.order" event=request.event.slug organizer=request.event.organizer.slug code=trans.order.code %}"
+                            data-toggle="tooltip" title="{{ trans.order.total|floatformat:2 }} {{ request.event.currency }}">
+                            {{ trans.order.code }}
+                        </a>
+                    {% endif %}
+                </td>
+            </tr>
+        {% endfor %}
+        </tbody>
+    </table>
+</div>

src/pretix/plugins/banktransfer/urls.py

@@ -5,4 +5,8 @@ from . import views
 urlpatterns = [
     url(r'^control/event/(?P<organizer>[^/]+)/(?P<event>[^/]+)/banktransfer/import/', views.ImportView.as_view(),
         name='import'),
+    url(r'^control/event/(?P<organizer>[^/]+)/(?P<event>[^/]+)/banktransfer/job/(?P<job>\d+)/',
+        views.JobDetailView.as_view(), name='import.job'),
+    url(r'^control/event/(?P<organizer>[^/]+)/(?P<event>[^/]+)/banktransfer/action/',
+        views.ActionView.as_view(), name='import.action'),
 ]

src/pretix/plugins/banktransfer/views.py

@@ -1,18 +1,16 @@
 import csv
 import json
 import logging
-import re
-import shutil
-from decimal import Decimal
+from locale import format as lformat
 
-from django import forms
-from django.conf import settings
 from django.contrib import messages
-from django.shortcuts import redirect, render
+from django.core.urlresolvers import reverse
+from django.db.models import Q
+from django.http import JsonResponse
+from django.shortcuts import get_object_or_404, redirect, render
 from django.utils.functional import cached_property
-from django.utils.timezone import now
-from django.utils.translation import ugettext_lazy as _
-from django.views.generic import TemplateView
+from django.utils.translation import ugettext as _
+from django.views.generic import DetailView, ListView, TemplateView, View
 
 from pretix.base.models import Order, Quota
 from pretix.base.services.mail import SendMailException
@@ -20,15 +18,204 @@ from pretix.base.services.orders import mark_order_paid
 from pretix.base.settings import SettingsSandbox
 from pretix.control.permissions import EventPermissionRequiredMixin
 from pretix.plugins.banktransfer import csvimport, mt940import
+from pretix.plugins.banktransfer.models import BankImportJob, BankTransaction
+from pretix.plugins.banktransfer.tasks import process_banktransfers
 
 logger = logging.getLogger('pretix.plugins.banktransfer')
 
 
-class ImportView(EventPermissionRequiredMixin, TemplateView):
-    template_name = 'pretixplugins/banktransfer/import_form.html'
+class ActionView(EventPermissionRequiredMixin, View):
     permission = 'can_change_orders'
 
+    def _discard(self, trans):
+        trans.state = BankTransaction.STATE_DISCARDED
+        trans.shred_private_data()
+        trans.save()
+        return JsonResponse({
+            'status': 'ok'
+        })
+
+    def _retry(self, trans):
+        if trans.amount != trans.order.total:
+            return JsonResponse({
+                'status': 'error',
+                'message': _('The transaction amount is incorrect.')
+            })
+        return self._accept_ignore_amount(trans)
+
+    def _accept_ignore_amount(self, trans):
+        if trans.order.status == Order.STATUS_PAID:
+            return JsonResponse({
+                'status': 'error',
+                'message': _('The order is already marked as paid.')
+            })
+        elif trans.order.status == Order.STATUS_REFUNDED:
+            return JsonResponse({
+                'status': 'error',
+                'message': _('The order has already been refunded.')
+            })
+        elif trans.order.status == Order.STATUS_CANCELLED:
+            return JsonResponse({
+                'status': 'error',
+                'message': _('The order has already been cancelled.')
+            })
+
+        try:
+            mark_order_paid(trans.order, provider='banktransfer', info=json.dumps({
+                'reference': trans.reference,
+                'date': trans.date.isoformat(),
+                'payer': trans.payer,
+                'trans_id': trans.pk
+            }))
+            trans.state = BankTransaction.STATE_VALID
+            trans.save()
+        except Quota.QuotaExceededException as e:
+            return JsonResponse({
+                'status': 'error',
+                'message': str(e)
+            })
+        except SendMailException:
+            return JsonResponse({
+                'status': 'error',
+                'message': _('Problem sending email.')
+            })
+        else:
+            trans.state = BankTransaction.STATE_VALID
+            trans.save()
+            return JsonResponse({
+                'status': 'ok'
+            })
+
+    def _assign(self, trans, code):
+        try:
+            trans.order = self.request.event.orders.get(code=code)
+        except Order.DoesNotExist:
+            return JsonResponse({
+                'status': 'error',
+                'message': _('Unknown order code.')
+            })
+        else:
+            return self._retry(trans)
+
+    def post(self, request, *args, **kwargs):
+        for k, v in request.POST.items():
+            if not k.startswith('action_'):
+                continue
+            trans = get_object_or_404(BankTransaction, id=k.split('_')[1], event=self.request.event)
+
+            if v == 'discard' and trans.state in (BankTransaction.STATE_INVALID, BankTransaction.STATE_ERROR,
+                                                  BankTransaction.STATE_NOMATCH):
+                return self._discard(trans)
+
+            elif v == 'accept' and trans.state == BankTransaction.STATE_INVALID:
+                # Accept anyway even with wrong amount
+                return self._accept_ignore_amount(trans)
+
+            elif v.startswith('assign:') and trans.state == BankTransaction.STATE_NOMATCH:
+                return self._assign(trans, v[7:])
+
+            elif v == 'retry' and trans.state == BankTransaction.STATE_ERROR:
+                return self._retry(trans)
+
+            return JsonResponse({
+                'status': 'error',
+                'message': 'Unknown action'
+            })
+
+    def get(self, request, *args, **kwargs):
+        query = request.GET.get('query', '')
+        if len(query) < 2:
+            return JsonResponse({'results': []})
+
+        qs = self.request.event.orders.filter(Q(code__icontains=query) | Q(code__icontains=Order.normalize_code(query)))
+        return JsonResponse({
+            'results': [
+                {
+                    'code': o.code,
+                    'status': o.get_status_display(),
+                    'total': lformat("%.2f", o.total) + ' ' + self.request.event.currency
+                } for o in qs
+            ]
+        })
+
+
+class JobDetailView(EventPermissionRequiredMixin, DetailView):
+    template_name = 'pretixplugins/banktransfer/job_detail.html'
+    permission = 'can_change_orders'
+    context_objectname = 'job'
+
+    def redirect_form(self):
+        return redirect(reverse('plugins:banktransfer:import', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+        }))
+
+    def redirect_back(self):
+        return redirect(reverse('plugins:banktransfer:import.job', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+            'job': self.kwargs['job']
+        }))
+
+    def get_object(self, queryset=None):
+        return get_object_or_404(BankImportJob, id=self.kwargs['job'], event=self.request.event)
+
+    def get(self, request, *args, **kwargs):
+        if 'ajax' in request.GET:
+            self.object = self.get_object()
+            return JsonResponse({
+                'state': self.object.state
+            })
+
+        return super().get(request, *args, **kwargs)
+
+    def get_context_data(self, **kwargs):
+        ctx = super().get_context_data()
+
+        qs = self.object.transactions.select_related('order')
+
+        ctx['transactions_valid'] = qs.filter(state=BankTransaction.STATE_VALID).count()
+        ctx['transactions_invalid'] = qs.filter(state__in=[
+            BankTransaction.STATE_INVALID, BankTransaction.STATE_ERROR
+        ]).count()
+        ctx['transactions_ignored'] = qs.filter(state__in=[
+            BankTransaction.STATE_DUPLICATE, BankTransaction.STATE_NOMATCH
+        ]).count()
+        ctx['job'] = self.object
+
+        return ctx
+
+
+class ImportView(EventPermissionRequiredMixin, ListView):
+    template_name = 'pretixplugins/banktransfer/import_form.html'
+    permission = 'can_change_orders'
+    context_object_name = 'transactions_unhandled'
+    paginate_by = 30
+
+    def get_queryset(self):
+        qs = BankTransaction.objects.filter(
+            event=self.request.event
+        ).select_related('order').filter(state__in=[
+            BankTransaction.STATE_INVALID, BankTransaction.STATE_ERROR,
+            BankTransaction.STATE_DUPLICATE, BankTransaction.STATE_NOMATCH
+        ])
+        if 'search' in self.request.GET:
+            q = self.request.GET.get('search')
+            qs = qs.filter(
+                Q(payer__icontains=q) | Q(reference__icontains=q)
+            )
+
+        return qs
+
+    def discard_all(self):
+        self.get_queryset().update(payer='', reference='', state=BankTransaction.STATE_DISCARDED)
+        messages.success(self.request, _('All unresolved transactions have been discarded.'))
+
     def post(self, *args, **kwargs):
+        if self.request.POST.get('discard', '') == 'all':
+            self.discard_all()
+            return self.redirect_back()
+
         if ('file' in self.request.FILES and 'csv' in self.request.FILES.get('file').name.lower()) \
                 or 'amount' in self.request.POST:
             # Process CSV
@@ -37,35 +224,6 @@ class ImportView(EventPermissionRequiredMixin, TemplateView):
         if 'file' in self.request.FILES and 'txt' in self.request.FILES.get('file').name.lower():
             return self.process_mt940()
 
-        if 'confirm' in self.request.POST:
-            orders = Order.objects.filter(event=self.request.event,
-                                          code__in=self.request.POST.getlist('mark_paid'))
-            some_failed = False
-            mail_failures = False
-            for order in orders:
-                try:
-                    mark_order_paid(order, provider='banktransfer', info=json.dumps({
-                        'reference': self.request.POST.get('reference_%s' % order.code),
-                        'date': self.request.POST.get('date_%s' % order.code),
-                        'payer': self.request.POST.get('payer_%s' % order.code),
-                        'import': now().isoformat(),
-                    }))
-                except Quota.QuotaExceededException:
-                    some_failed = True
-                except SendMailException:
-                    mail_failures = True
-
-            if some_failed:
-                messages.warning(self.request, _('Not all of the selected orders could be marked as '
-                                                 'paid as some of them have expired and the selected '
-                                                 'items are sold out.'))
-            else:
-                messages.success(self.request, _('The selected orders have been marked as paid.'))
-                # TODO: Display a list of them!
-            if mail_failures:
-                messages.warning(self.request, _('Some confirmation mails could not be sent.'))
-            return self.redirect_back()
-
         messages.error(self.request, _('We were unable to detect the file type of this import. Please '
                                        'contact support for help.'))
         return self.redirect_back()
@@ -76,7 +234,7 @@ class ImportView(EventPermissionRequiredMixin, TemplateView):
 
     def process_mt940(self):
         try:
-            return self.confirm_view(mt940import.parse(self.request.FILES.get('file')))
+            return self.start_processing(mt940import.parse(self.request.FILES.get('file')))
         except:
             logger.exception('Failed to import MT940 file')
             messages.error(self.request, _('We were unable to process your input.'))
@@ -102,7 +260,7 @@ class ImportView(EventPermissionRequiredMixin, TemplateView):
             except csvimport.HintMismatchError:  # TODO: narrow down
                 logger.exception('Import using stored hint failed')
             else:
-                return self.confirm_view(parsed)
+                return self.start_processing(parsed)
 
         return self.assign_view(data)
 
@@ -113,7 +271,7 @@ class ImportView(EventPermissionRequiredMixin, TemplateView):
                 [
                     self.request.POST.get('col[%d][%d]' % (i, j))
                     for j in range(int(self.request.POST.get('cols')))
-                ]
+                    ]
             )
         if 'reference' not in self.request.POST:
             messages.error(self.request, _('You need to select the column containing the payment reference.'))
@@ -131,7 +289,7 @@ class ImportView(EventPermissionRequiredMixin, TemplateView):
             pass
         else:
             parsed = csvimport.parse(data, hint)
-            return self.confirm_view(parsed)
+            return self.start_processing(parsed)
 
     def process_csv(self):
         if 'file' in self.request.FILES:
@@ -140,81 +298,22 @@ class ImportView(EventPermissionRequiredMixin, TemplateView):
             return self.process_csv_hint()
         return super().get(self.request)
 
-    def confirm_view(self, parsed):
-        parsed = self.annotate_data(parsed)
-        return render(self.request, 'pretixplugins/banktransfer/import_confirm.html', {
-            'rows': parsed
-        })
-
     def assign_view(self, parsed):
         return render(self.request, 'pretixplugins/banktransfer/import_assign.html', {
             'rows': parsed
         })
 
     def redirect_back(self):
-        return redirect('plugins:banktransfer:import',
-                        event=self.request.event.slug,
-                        organizer=self.request.event.organizer.slug)
+        return redirect(reverse('plugins:banktransfer:import', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+        }))
 
-    def annotate_data(self, data):
-        code_len = settings.ENTROPY['order_code']
-        pattern = re.compile(self.request.event.slug.upper() + "[ -_]*([A-Z0-9]{%s})" % code_len)
-        amount_pattern = re.compile("[^0-9.-]")
-        order_codes = []
-        for row in data:
-            row['ok'] = False
-            try:
-                amount = Decimal(amount_pattern.sub("", row['amount'].replace(",", ".")))
-                row['amount'] = str(amount)
-            except:
-                logger.exception('Could not parse amount of transaction')
-                amount = 0
-            match = pattern.search(row['reference'].upper())
-            if not match:
-                row['class'] = 'warning' if amount > 0 else ''
-                row['message'] = _('No order code detected')
-                continue
-
-            code = match.group(1)
-            row['code'] = code
-            order_codes.append(code)
-            normalized_code = Order.normalize_code(code)
-            if normalized_code != code:
-                order_codes.append(normalized_code)
-
-        orders = {}
-        # Perform query in bulks because of SQLite's default of SQLITE_MAX_VARIABLE_NUMBER = 999
-        for i in range(0, len(order_codes), 500):
-            orders.update({
-                o.code: o for o in self.request.event.orders.filter(code__in=order_codes[i:500])
-            })
-
-        for row in data:
-            if 'code' not in row:
-                continue
-            normalized_code = Order.normalize_code(row['code'])
-            if row['code'] in orders or normalized_code in orders:
-                order = orders[row['code']] if row['code'] in orders else orders[normalized_code]
-                row['order'] = order
-                if order.status == Order.STATUS_PENDING:
-                    amount = Decimal(row['amount'])
-                    if amount != order.total:
-                        row['class'] = 'danger'
-                        row['message'] = _('Found wrong amount. Expected: %s' % str(order.total))
-                    else:
-                        row['class'] = 'success'
-                        row['message'] = _('Valid payment')
-                        row['ok'] = True
-                elif order.status == Order.STATUS_CANCELLED:
-                    row['class'] = 'danger'
-                    row['message'] = _('Order has been cancelled')
-                elif order.status == Order.STATUS_PAID:
-                    row['class'] = ''
-                    row['message'] = _('Order already has been paid')
-                elif order.status == Order.STATUS_REFUNDED:
-                    row['class'] = 'warning'
-                    row['message'] = _('Order has been refunded')
-            else:
-                row['class'] = 'danger'
-                row['message'] = _('Unknown order code detected')
-        return data
+    def start_processing(self, parsed):
+        job = BankImportJob.objects.create(event=self.request.event)
+        process_banktransfers(event=self.request.event.pk, job=job.pk, data=parsed)
+        return redirect(reverse('plugins:banktransfer:import.job', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+            'job': job.pk
+        }))

src/pretix/presale/views/async.py

@@ -1,134 +1,3 @@
-import logging
+from pretix.base.views.async import *  # noqa
 
-from django.conf import settings
-from django.contrib import messages
-from django.http import JsonResponse
-from django.shortcuts import redirect, render
-from django.utils.translation import ugettext as _
-
-logger = logging.getLogger('pretix.presale.async')
-
-
-class AsyncAction:
-    task = None
-    success_url = None
-    error_url = None
-
-    def do(self, *args):
-        if settings.HAS_CELERY:
-            from pretix.celery import app
-
-            if hasattr(self.task, 'task') and isinstance(self.task.task, app.Task):
-                return self._do_celery(args)
-            else:
-                raise TypeError('Method has no task attached')
-        else:
-            return self._do_sync(args)
-
-    def get_success_url(self, value):
-        return self.success_url
-
-    def get_error_url(self):
-        return self.error_url
-
-    def get_check_url(self, task_id, ajax):
-        return self.request.path + '?async_id=%s' % task_id + ('&ajax=1' if ajax else '')
-
-    def get(self, request, *args, **kwargs):
-        if 'async_id' in request.GET and settings.HAS_CELERY:
-            return self.get_result(request)
-        return self.http_method_not_allowed(request)
-
-    def _return_celery_result(self, res, timeout=.5):
-        import celery.exceptions
-
-        if not res.ready():
-            try:
-                res.get(timeout=timeout)
-            except celery.exceptions.TimeoutError:
-                pass
-        ready = res.ready()
-        data = {
-            'async_id': res.id,
-            'ready': ready
-        }
-        if ready:
-            if res.successful() and not isinstance(res.info, Exception):
-                smes = self.get_success_message(res.info)
-                if smes:
-                    messages.success(self.request, smes)
-                # TODO: Do not store message if the ajax client stats that it will not redirect
-                # but handle the mssage itself
-                data.update({
-                    'redirect': self.get_success_url(res.info),
-                    'message': str(self.get_success_message(res.info))
-                })
-            else:
-                messages.error(self.request, self.get_error_message(res.info))
-                # TODO: Do not store message if the ajax client stats that it will not redirect
-                # but handle the mssage itself
-                data.update({
-                    'redirect': self.get_error_url(),
-                    'message': str(self.get_error_message(res.info))
-                })
-        return data
-
-    def get_result(self, request):
-        from celery.result import AsyncResult
-
-        res = AsyncResult(request.GET.get('async_id'))
-        if 'ajax' in self.request.GET:
-            return JsonResponse(self._return_celery_result(res, timeout=0.25))
-        else:
-            if res.ready():
-                if res.successful():
-                    return self.success(res.info)
-                else:
-                    return self.error(res.info)
-            return render(request, 'pretixpresale/waiting.html')
-
-    def _do_celery(self, args):
-        res = self.task.task.apply_async(args=args)
-        if 'ajax' in self.request.GET or 'ajax' in self.request.POST:
-            data = self._return_celery_result(res)
-            data['check_url'] = self.get_check_url(res.id, True)
-            return JsonResponse(data)
-        else:
-            return redirect(self.get_check_url(res.id, False))
-
-    def _do_sync(self, args):
-        try:
-            rs = getattr(self.__class__, 'task')(*args)
-            return self.success(rs)
-        except Exception as e:
-            logger.exception('Error while executing task synchronously')
-            return self.error(e)
-
-    def success(self, value):
-        smes = self.get_success_message(value)
-        if smes:
-            messages.success(self.request, smes)
-        if "ajax" in self.request.POST or "ajax" in self.request.GET:
-            return JsonResponse({
-                'ready': True,
-                'redirect': self.get_success_url(value),
-                'message': str(self.get_success_message(value))
-            })
-        return redirect(self.get_success_url(value))
-
-    def error(self, exception):
-        messages.error(self.request, self.get_error_message(exception))
-        if "ajax" in self.request.POST or "ajax" in self.request.GET:
-            return JsonResponse({
-                'ready': True,
-                'redirect': self.get_error_url(),
-                'message': str(self.get_error_message(exception))
-            })
-        return redirect(self.get_error_url())
-
-    def get_error_message(self, exception):
-        logger.error('Unexpected exception: %r' % exception)
-        return _('An unexpected error has occured.')
-
-    def get_success_message(self, value):
-        return _('The task has been completed.')
+# file kept for backwards compatibility

src/requirements/banktransfer.txt

@@ -1,4 +1,3 @@
 chardet>=2.3,<3
-defusedxml
 mt-940==3.2
 

src/setup.py

@@ -61,7 +61,7 @@ setup(
         'easy-thumbnails>=2.2,<3'
         'PyPDF2', 'BeautifulSoup4', 'html5lib',
         'slimit', 'lxml', 'static3==0.6.1', 'dj-static', 'chardet',
-        'csscompressor', 'defusedxml', 'mt-940', 'django-markup', 'markdown'
+        'csscompressor', 'mt-940', 'django-markup', 'markdown'
     ],
     extras_require={
         'dev': ['django-debug-toolbar>=1.3.0,<2.0'],

src/static/pretixcontrol/js/ui/main.js

@@ -1,4 +1,4 @@
-/*global $*/
+/*global $,gettext*/
 
 function question_page_toggle_view() {
     var show = $("#id_type").val() == "C" || $("#id_type").val() == "M";
@@ -8,6 +8,42 @@ function question_page_toggle_view() {
     $(".alert-required-boolean").toggle(show);
 }
 
+var waitingDialog = {
+    show: function (message) {
+        "use strict";
+        $("#loadingmodal").find("h1").html(message);
+        $("body").addClass("loading");
+    },
+    hide: function () {
+        "use strict";
+        $("body").removeClass("loading");
+    }
+};
+
+var ajaxErrDialog = {
+    show: function (c) {
+        "use strict";
+        $("#ajaxerr").html(c);
+        $("#ajaxerr .links").html("<a class='btn btn-default ajaxerr-close'>"
+                                  + gettext("Close message") + "</a>");
+        $("body").addClass("ajaxerr");
+    },
+    hide: function () {
+        "use strict";
+        $("body").removeClass("ajaxerr");
+    }
+};
+
+$(document).ajaxError(function (event, jqXHR, settings, thrownError) {
+    waitingDialog.hide();
+    var c = $(jqXHR.responseText).filter('.container');
+    if (c.length > 0) {
+        ajaxErrDialog.show(c.first().html());
+    } else {
+        alert(gettext('Unknown error.'));
+    }
+});
+
 $(function () {
     "use strict";
 
@@ -45,6 +81,8 @@ $(function () {
     });
 
     $('.collapsible').collapse();
+    
+    $('[data-toggle="tooltip"]').tooltip()
 
     // Question editor
     if ($("#answer-options").length) {
@@ -63,4 +101,6 @@ $(function () {
             $("#id_codes").text(data.codes.join("\n"));
         });
     });
+
+    $("#ajaxerr").on("click", ".ajaxerr-close", ajaxErrDialog.hide);
 });

src/static/pretixcontrol/scss/main.scss

@@ -2,6 +2,7 @@
 @import "../../bootstrap/scss/_bootstrap.scss";
 $fa-font-path: static("fontawesome/fonts");
 @import "../../fontawesome/scss/font-awesome.scss";
+@import "../../typeahead/typeahead.css";
 @import "../css/metisMenu.min.css";
 @import "_sb-admin-2.scss";
 @import "_forms.scss";
@@ -129,3 +130,49 @@ h1 .btn-sm {
 .form-order-change .form-group {
     margin: 0;
 }
+
+#loadingmodal, #ajaxerr {
+    position: fixed;
+    top: 0;
+    left: 0;
+    width: 100vw;
+    height: 100vh;
+    background: rgba(255, 255, 255, .7);
+    opacity: 0;
+    text-align: center;
+    z-index: 900000;
+    visibility: hidden;
+
+    .big-icon {
+        margin-top: 50px;
+        font-size: 200px;
+        color: $brand-primary;
+    }
+
+    .big-rotating-icon {
+        -webkit-animation: fa-spin 8s infinite linear;
+        animation: fa-spin 8s infinite linear;
+        margin-top: 50px;
+        font-size: 200px;
+        color: $brand-primary;
+    }
+}
+#ajaxerr {
+    background: rgba(236, 236, 236, .9);
+}
+
+.loading #loadingmodal, .ajaxerr #ajaxerr {
+    opacity: 1;
+    visibility: visible;
+    transition: opacity .5s ease-in-out;
+    -moz-transition: opacity .5s ease-in-out;
+    -webkit-transition: opacity .5s ease-in-out;
+}
+
+.big-rotating-icon {
+    margin-top: 50px;
+    -webkit-animation: fa-spin 8s infinite linear;
+    animation: fa-spin 8s infinite linear;
+    font-size: 200px;
+    color: $brand-primary;
+}

src/static/pretixpresale/scss/main.scss

@@ -135,4 +135,4 @@ body.loading .container {
     h2 {
         margin-top: 0;
     }
-}
+}

src/static/typeahead/typeahead.css

@@ -0,0 +1,93 @@
+span.twitter-typeahead .tt-menu,
+span.twitter-typeahead .tt-dropdown-menu {
+  position: absolute;
+  top: 100%;
+  left: 0;
+  z-index: 1000;
+  display: none;
+  float: left;
+  min-width: 160px;
+  padding: 5px 0;
+  margin: 2px 0 0;
+  list-style: none;
+  font-size: 14px;
+  text-align: left;
+  background-color: #ffffff;
+  border: 1px solid #cccccc;
+  border: 1px solid rgba(0, 0, 0, 0.15);
+  border-radius: 4px;
+  -webkit-box-shadow: 0 6px 12px rgba(0, 0, 0, 0.175);
+  box-shadow: 0 6px 12px rgba(0, 0, 0, 0.175);
+  background-clip: padding-box;
+}
+span.twitter-typeahead .tt-suggestion {
+  display: block;
+  padding: 3px 20px;
+  clear: both;
+  font-weight: normal;
+  line-height: 1.42857143;
+  color: #333333;
+  white-space: nowrap;
+}
+span.twitter-typeahead .tt-suggestion.tt-cursor,
+span.twitter-typeahead .tt-suggestion:hover,
+span.twitter-typeahead .tt-suggestion:focus {
+  color: #ffffff;
+  text-decoration: none;
+  outline: 0;
+  background-color: #337ab7;
+}
+.input-group.input-group-lg span.twitter-typeahead .form-control {
+  height: 46px;
+  padding: 10px 16px;
+  font-size: 18px;
+  line-height: 1.3333333;
+  border-radius: 6px;
+}
+.input-group.input-group-sm span.twitter-typeahead .form-control {
+  height: 30px;
+  padding: 5px 10px;
+  font-size: 12px;
+  line-height: 1.5;
+  border-radius: 3px;
+}
+span.twitter-typeahead {
+  width: 100%;
+}
+.input-group span.twitter-typeahead {
+  display: block !important;
+  height: 34px;
+}
+.input-group span.twitter-typeahead .tt-menu,
+.input-group span.twitter-typeahead .tt-dropdown-menu {
+  top: 32px !important;
+}
+.input-group span.twitter-typeahead:not(:first-child):not(:last-child) .form-control {
+  border-radius: 0;
+}
+.input-group span.twitter-typeahead:first-child .form-control {
+  border-top-left-radius: 4px;
+  border-bottom-left-radius: 4px;
+  border-top-right-radius: 0;
+  border-bottom-right-radius: 0;
+}
+.input-group span.twitter-typeahead:last-child .form-control {
+  border-top-left-radius: 0;
+  border-bottom-left-radius: 0;
+  border-top-right-radius: 4px;
+  border-bottom-right-radius: 4px;
+}
+.input-group.input-group-sm span.twitter-typeahead {
+  height: 30px;
+}
+.input-group.input-group-sm span.twitter-typeahead .tt-menu,
+.input-group.input-group-sm span.twitter-typeahead .tt-dropdown-menu {
+  top: 30px !important;
+}
+.input-group.input-group-lg span.twitter-typeahead {
+  height: 46px;
+}
+.input-group.input-group-lg span.twitter-typeahead .tt-menu,
+.input-group.input-group-lg span.twitter-typeahead .tt-dropdown-menu {
+  top: 46px !important;
+}

src/tests/plugins/banktransfer/test_import.py

@@ -8,6 +8,8 @@ from django.utils.timezone import now
 from pretix.base.models import (
     Event, EventPermission, Item, Order, OrderPosition, Organizer, Quota, User,
 )
+from pretix.plugins.banktransfer.models import BankImportJob
+from pretix.plugins.banktransfer.tasks import process_banktransfers
 
 
 @pytest.fixture
@@ -20,7 +22,7 @@ def env():
     user = User.objects.create_user('dummy@dummy.dummy', 'dummy')
     EventPermission.objects.create(user=user, event=event)
     o1 = Order.objects.create(
-        code='1234S', event=event,
+        code='1Z3AS', event=event,
         status=Order.STATUS_PENDING,
         datetime=now(), expires=now() + timedelta(days=10),
         total=23, payment_provider='banktransfer'
@@ -77,47 +79,57 @@ Buchungstag;Valuta;Buchungstext;Auftraggeber / Empfänger;Verwendungszweck;Betra
     for inp in doc.select("input[type=hidden]"):
         data[inp.attrs['name']] = inp.attrs['value']
     r = client.post('/control/event/dummy/dummy/banktransfer/import/', data)
-    doc = BeautifulSoup(r.content)
-    assert r.status_code == 200
-    assert len(doc.select("form table tbody tr")) == 6
-    trs = doc.select("form table tbody tr")
-    assert trs[0].select("td")[0].text == "09.04.2015"
-    assert trs[0].select("td")[1].text == "Bestellung 2015ABCDE"
-    assert trs[0].select("td")[2].text == "23.00"
-    assert trs[0].select("td")[3].text == "Karl Kunde"
-    assert trs[0].select("td")[5].text == "No order code detected"
-    assert trs[1].select("td")[0].text == "09.04.2015"
-    assert trs[1].select("td")[1].text == "Bestellung DUMMYFGHIJ"
-    assert trs[1].select("td")[2].text == "42.00"
-    assert trs[1].select("td")[3].text == "Karla Kundin"
-    assert trs[1].select("td")[5].text == "Unknown order code detected"
-    assert trs[2].select("td")[0].text == "09.04.2015"
-    assert trs[2].select("td")[1].text == "Bestellung DUMMY1234S"
-    assert trs[2].select("td")[2].text == "42.00"
-    assert trs[2].select("td")[3].text == "Karla Kundin"
-    assert trs[2].select("td")[5].text == "Found wrong amount. Expected: 23.00"
-    assert trs[3].select("td")[0].text == "09.04.2015"
-    assert trs[3].select("td")[1].text == "Bestellung DUMMY1234S"
-    assert trs[3].select("td")[2].text == "23.00"
-    assert trs[3].select("td")[3].text == "Karla Kundin"
-    assert trs[3].select("td")[5].text == "Valid payment"
-    assert trs[4].select("td")[0].text == "09.04.2015"
-    assert trs[4].select("td")[1].text == "Bestellung DUMMY6789Z"
-    assert trs[4].select("td")[2].text == "23.00"
-    assert trs[4].select("td")[3].text == "Karla Kundin"
-    assert trs[4].select("td")[5].text == "Order has been cancelled"
-    assert trs[5].select("td")[0].text == "09.04.2015"
-    # Test "autocorrection"
-    assert trs[5].select("td")[1].text == "Bestellung DUMMY65892"
-    assert trs[5].select("td")[2].text == "23.00"
-    assert trs[5].select("td")[3].text == "Karla Kundin"
-    assert "GS89Z" in trs[5].select("td")[4].text
-    assert trs[5].select("td")[5].text == "Order has been cancelled"
+    assert '/job/' in r['Location']
 
-    data = {}
-    for inp in doc.select("form input"):
-        data[inp.attrs['name']] = inp.attrs['value']
-    client.post('/control/event/dummy/dummy/banktransfer/import/', data)
 
-    assert Order.objects.get(id=env[2].id).status == Order.STATUS_PAID
-    assert Order.objects.get(id=env[3].id).status == Order.STATUS_CANCELLED
+@pytest.fixture
+def job(env):
+    return BankImportJob.objects.create(event=env[0]).pk
+
+
+@pytest.mark.django_db
+def test_mark_paid(env, job):
+    process_banktransfers(env[0].pk, job, [{
+        'payer': 'Karla Kundin',
+        'reference': 'Bestellung DUMMY1234S',
+        'date': '2016-01-26',
+        'amount': '23.00'
+    }])
+    env[2].refresh_from_db()
+    assert env[2].status == Order.STATUS_PAID
+
+
+@pytest.mark.django_db
+def test_check_amount(env, job):
+    process_banktransfers(env[0].pk, job, [{
+        'payer': 'Karla Kundin',
+        'reference': 'Bestellung DUMMY1Z3AS',
+        'date': '2016-01-26',
+        'amount': '23.50'
+    }])
+    env[2].refresh_from_db()
+    assert env[2].status == Order.STATUS_PENDING
+
+
+@pytest.mark.django_db
+def test_ignore_cancelled(env, job):
+    process_banktransfers(env[0].pk, job, [{
+        'payer': 'Karla Kundin',
+        'reference': 'Bestellung DUMMY6789Z',
+        'date': '2016-01-26',
+        'amount': '23.00'
+    }])
+    env[3].refresh_from_db()
+    assert env[3].status == Order.STATUS_CANCELLED
+
+
+@pytest.mark.django_db
+def test_autocorrection(env, job):
+    process_banktransfers(env[0].pk, job, [{
+        'payer': 'Karla Kundin',
+        'reference': 'Bestellung DUMMY12345',
+        'amount': '23.00',
+        'date': '2016-01-26',
+    }])
+    env[2].refresh_from_db()
+    assert env[2].status == Order.STATUS_PAID