Banktransfer: Big import refactoring

src/pretix/plugins/banktransfer/__init__.py

@@ -20,6 +20,7 @@ class BankTransferApp(AppConfig):
 
     def ready(self):
         from . import signals  # NOQA
+        from . import tasks  # NOQA
 
     @cached_property
     def compatibility_warnings(self):
@@ -28,10 +29,6 @@ class BankTransferApp(AppConfig):
             import chardet  # NOQA
         except ImportError:
             errs.append(_("Install the python package 'chardet' for better CSV import capabilities."))
-        try:
-            import defusedxml  # NOQA
-        except ImportError:
-            errs.append(_("Please install the python package 'defusedxml' for security reasons."))
         return errs
 
 

src/pretix/plugins/banktransfer/migrations/0001_initial.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2016-09-07 09:35
+from __future__ import unicode_literals
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('pretixbase', '0036_auto_20160902_0755'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='BankImportJob',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('state', models.CharField(choices=[('running', 'pending'), ('running', 'running'), ('error', 'error'), ('completed', 'completed')], default='running', max_length=32)),
+                ('event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pretixbase.Event')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='BankTransaction',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('state', models.CharField(choices=[('imported', 'imported, unchecked'), ('nomatch', 'no match'), ('invalid', 'not valid'), ('error', 'error'), ('valid', 'valid'), ('already', 'valid, already paid'), ('discarded', 'manually discarded')], default='imported', max_length=32)),
+                ('message', models.TextField()),
+                ('checksum', models.CharField(db_index=True, max_length=190)),
+                ('payer', models.TextField(blank=True)),
+                ('reference', models.TextField(blank=True)),
+                ('amount', models.DecimalField(decimal_places=2, max_digits=10)),
+                ('date', models.DateField()),
+                ('event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pretixbase.Event')),
+                ('import_job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='banktransfer.BankImportJob')),
+                ('order', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='pretixbase.Order')),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name='banktransaction',
+            unique_together=set([('event', 'checksum')]),
+        ),
+    ]

src/pretix/plugins/banktransfer/models.py

@@ -0,0 +1,68 @@
+import hashlib
+import re
+
+from django.db import models
+
+
+class BankImportJob(models.Model):
+    STATE_PENDING = 'pending'
+    STATE_RUNNING = 'running'
+    STATE_ERROR = 'error'
+    STATE_COMPLETED = 'completed'
+    STATES = (
+        (STATE_PENDING, 'pending'),
+        (STATE_RUNNING, 'running'),
+        (STATE_ERROR, 'error'),
+        (STATE_COMPLETED, 'completed'),
+    )
+
+    event = models.ForeignKey('pretixbase.Event')
+    created = models.DateTimeField(auto_now_add=True)
+    state = models.CharField(max_length=32, choices=STATES, default=STATE_PENDING)
+
+
+class BankTransaction(models.Model):
+    STATE_UNCHECKED = 'imported'
+    STATE_NOMATCH = 'nomatch'
+    STATE_INVALID = 'invalid'
+    STATE_ERROR = 'error'
+    STATE_VALID = 'valid'
+    STATE_DISCARDED = 'discarded'
+    STATE_DUPLICATE = 'already'
+
+    STATES = (
+        (STATE_UNCHECKED, 'imported, unchecked'),
+        (STATE_NOMATCH, 'no match'),
+        (STATE_INVALID, 'not valid'),
+        (STATE_ERROR, 'error'),
+        (STATE_VALID, 'valid'),
+        (STATE_DUPLICATE, 'valid, already paid'),
+        (STATE_DISCARDED, 'manually discarded'),
+    )
+
+    event = models.ForeignKey('pretixbase.Event')
+    import_job = models.ForeignKey('BankImportJob', related_name='transactions')
+    state = models.CharField(max_length=32, choices=STATES, default=STATE_UNCHECKED)
+    message = models.TextField()
+    checksum = models.CharField(max_length=190, db_index=True)
+    payer = models.TextField(blank=True)
+    reference = models.TextField(blank=True)
+    amount = models.DecimalField(max_digits=10, decimal_places=2)
+    date = models.DateField()
+    order = models.ForeignKey('pretixbase.Order', null=True, blank=True)
+
+    def calculate_checksum(self):
+        clean = re.compile('[^a-zA-Z0-9.-]')
+        hasher = hashlib.sha1()
+        hasher.update(clean.sub('', self.payer.lower()).encode('utf-8'))
+        hasher.update(clean.sub('', self.reference.lower()).encode('utf-8'))
+        hasher.update(clean.sub('', str(self.amount).lower()).encode('utf-8'))
+        hasher.update(clean.sub('', str(self.date).lower()).encode('utf-8'))
+        return str(hasher.hexdigest())
+
+    def shred_private_data(self):
+        self.payer = ""
+        self.reference = ""
+
+    class Meta:
+        unique_together = ('event', 'checksum')

src/pretix/plugins/banktransfer/signals.py

@@ -1,9 +1,11 @@
 from django.core.urlresolvers import resolve, reverse
 from django.dispatch import receiver
+from django.template import Context
+from django.template.loader import get_template
 from django.utils.translation import ugettext_lazy as _
 
 from pretix.base.signals import register_payment_providers
-from pretix.control.signals import nav_event
+from pretix.control.signals import html_head, nav_event
 
 from .payment import BankTransfer
 
@@ -29,3 +31,14 @@ def control_nav_import(sender, request=None, **kwargs):
             'icon': 'upload',
         }
     ]
+
+
+@receiver(html_head, dispatch_uid="banktransfer_html_head")
+def html_head_presale(sender, request=None, **kwargs):
+    url = resolve(request.path_info)
+    if url.namespace == 'plugins:banktransfer':
+        template = get_template('pretixplugins/banktransfer/control_head.html')
+        ctx = Context({})
+        return template.render(ctx)
+    else:
+        return ""

src/pretix/plugins/banktransfer/static/pretixplugins/banktransfer/ui.css

@@ -0,0 +1,10 @@
+table.transaction-list td.actions {
+    max-width: 100px;
+}
+table.transaction-list td.actions .input-group {
+    display: inline-block;
+    width: 110px;
+}
+table.transaction-list td.actions .input-group button {
+    line-height: 1.43;
+}

src/pretix/plugins/banktransfer/static/pretixplugins/banktransfer/ui.js

@@ -0,0 +1,132 @@
+/*global $, gettext*/
+var bankimport_transactionlist = {
+
+    _btn_click: function (e) {
+        console.log(e.delegateTarget);
+        var trans_id = parseInt($(e.delegateTarget).attr("name").split("_")[1]);
+        var value = $(e.delegateTarget).val();
+        if (value === "discard") {
+            bankimport_transactionlist.discard(trans_id);
+        } else if (value === "accept") {
+            bankimport_transactionlist.accept(trans_id);
+        } else if (value === "retry") {
+            bankimport_transactionlist.retry(trans_id);
+        } else if (value === "assign") {
+            bankimport_transactionlist.assign(trans_id);
+        }
+        return false;
+    },
+
+    _action: function (id, action, success) {
+        $("tr[data-id=" + id + "] button").prop("disabled", true);
+        var data = {
+            "csrfmiddlewaretoken": $("[name=csrfmiddlewaretoken]").val()
+        };
+        data["action_" + id] = action;
+        $.ajax({
+            "method": "POST",
+            "url": $(".transaction-list").attr("data-url"),
+            "data": data,
+            "dataType": "json",
+            "success": function (data) {
+                if (data.status == "ok") {
+                    $("tr[data-id=" + id + "]").removeClass("has-error");
+                    success();
+                } else {
+                    $("tr[data-id=" + id + "] button").prop("disabled", false);
+                    $("tr[data-id=" + id + "] .help-block").remove();
+                    $("tr[data-id=" + id + "]").addClass("has-error");
+                    $("<p>").addClass("help-block").text(data.message).appendTo($("tr[data-id=" + id + "] td.actions"));
+                }
+            }
+        });
+    },
+
+    discard: function (id) {
+        bankimport_transactionlist._action(id, "discard", function () {
+            $("tr[data-id=" + id + "] td").remove();
+        });
+    },
+
+    retry: function (id) {
+        bankimport_transactionlist._action(id, "retry", function () {
+            $("tr[data-id=" + id + "] td.actions").html('').text(gettext("Marked as paid"));
+        });
+    },
+
+    accept: function (id) {
+        bankimport_transactionlist._action(id, "accept", function () {
+            $("tr[data-id=" + id + "] td.actions").html('').text(gettext("Marked as paid"));
+        });
+    },
+
+    assign: function (id) {
+        bankimport_transactionlist._action(id, "assign:" + $("tr[data-id=" + id + "] input.form-control:not(.tt-hint)").val(), function () {
+            $("tr[data-id=" + id + "] td.actions").html('').text(gettext("Marked as paid"));
+        });
+    },
+
+    typeahead_source: function () {
+        return new Bloodhound({
+            datumTokenizer: Bloodhound.tokenizers.obj.whitespace('value'),
+            queryTokenizer: Bloodhound.tokenizers.whitespace,
+            remote: {
+                url: $(".transaction-list").attr("data-url"),
+                prepare: function (query, settings) {
+                    settings.url = settings.url + '?query=' + encodeURIComponent(query);
+                    return settings;
+                },
+                transform: function (object) {
+                    var results = object.results;
+                    var suggs = [];
+                    var reslen = results.length;
+                    for (var i = 0; i < reslen; i++) {
+                        suggs.push(results[i]);
+                    }
+                    return suggs;
+                }
+            }
+        });
+    },
+
+    init: function () {
+        if ($(".transaction-list").length) {
+            $(".transaction-list button").click(bankimport_transactionlist._btn_click);
+            $(".transaction-list .form-control").typeahead(null, {
+                minLength: 2,
+                name: 'order-dataset',
+                source: bankimport_transactionlist.typeahead_source(),
+                display: function (obj) {
+                    return obj.code;
+                },
+                templates: {
+                    suggestion: function (obj) {
+                        return '<div>' + obj.code + ' (' + obj.total + ', ' + obj.status + ')</div>';
+                    }
+                }
+            }).keypress(function (e) {
+                if (e.keyCode === 13) {
+                    $(this).parent().parent().find("button[value=assign]").click();
+                }
+            });
+        }
+
+        if ($("[data-job-waiting]").length) {
+            window.setInterval(bankimport_transactionlist.check_state, 750);
+        }
+    },
+
+    check_state: function () {
+        $.getJSON(location.pathname + '?ajax=1', function (data) {
+            if (data.state == 'running' || data.state == 'pending') {
+                window.setInterval(check_state, 750);
+            } else {
+                location.reload();
+            }
+        });
+    }
+};
+
+$(function () {
+    bankimport_transactionlist.init();
+});

src/pretix/plugins/banktransfer/tasks.py

@@ -0,0 +1,130 @@
+import json
+import logging
+import re
+from decimal import Decimal
+
+import dateutil.parser
+from django.conf import settings
+from django.db import transaction
+from django.utils.translation import ugettext_noop
+
+from pretix.base.i18n import language
+from pretix.base.models import Event, Order, Quota
+from pretix.base.services.mail import SendMailException
+from pretix.base.services.orders import mark_order_paid
+
+from .models import BankImportJob, BankTransaction
+
+logger = logging.getLogger(__name__)
+
+
+def _handle_transaction(event: Event, trans: BankTransaction, code: str):
+    try:
+        trans.order = event.orders.get(code=code)
+    except Order.DoesNotExist:
+        normalized_code = Order.normalize_code(code)
+        try:
+            trans.order = event.orders.get(code=normalized_code)
+        except Order.DoesNotExist:
+            trans.state = BankTransaction.STATE_NOMATCH
+            trans.save()
+            return
+
+    if trans.order.status == Order.STATUS_PAID:
+        trans.state = BankTransaction.STATE_DUPLICATE
+    elif trans.order.status == Order.STATUS_REFUNDED:
+        trans.state = BankTransaction.STATE_ERROR
+        trans.message = ugettext_noop('The order has already been refunded.')
+    elif trans.order.status == Order.STATUS_CANCELLED:
+        trans.state = BankTransaction.STATE_ERROR
+        trans.message = ugettext_noop('The order has already been cancelled.')
+    elif trans.amount != trans.order.total:
+        trans.state = BankTransaction.STATE_INVALID
+        trans.message = ugettext_noop('The transaction amount is incorrect.')
+    else:
+        try:
+            mark_order_paid(trans.order, provider='banktransfer', info=json.dumps({
+                'reference': trans.reference,
+                'date': trans.date.isoformat(),
+                'payer': trans.payer,
+                'trans_id': trans.pk
+            }))
+        except Quota.QuotaExceededException as e:
+            trans.state = BankTransaction.STATE_ERROR
+            trans.message = str(e)
+        except SendMailException:
+            trans.state = BankTransaction.STATE_ERROR
+            trans.message = ugettext_noop('Problem sending email.')
+        else:
+            trans.state = BankTransaction.STATE_VALID
+    trans.save()
+
+
+def _get_unknown_transactions(event: Event, job: BankImportJob, data: list):
+    amount_pattern = re.compile("[^0-9.-]")
+    known_checksums = set(t['checksum'] for t in BankTransaction.objects.filter(event=event).values('checksum'))
+
+    transactions = []
+    for row in data:
+        amount = amount_pattern.sub("", row['amount'].replace(",", "."))
+        try:
+            amount = Decimal(amount)
+        except:
+            logger.exception('Could not parse amount of transaction: {}'.format(amount))
+            amount = Decimal("0.00")
+
+        date = dateutil.parser.parse(row['date'])
+        trans = BankTransaction(event=event, import_job=job,
+                                payer=row['payer'],
+                                reference=row['reference'],
+                                amount=amount,
+                                date=date)
+        trans.checksum = trans.calculate_checksum()
+        if trans.checksum not in known_checksums:
+            trans.state = BankTransaction.STATE_UNCHECKED
+            trans.save()
+            transactions.append(trans)
+            known_checksums.add(trans.checksum)
+
+    return transactions
+
+
+def process_banktransfers(event: int, job: int, data: list) -> None:
+    with language("en"):  # We'll translate error messages at display time
+        event = Event.objects.get(pk=event)
+        job = BankImportJob.objects.get(pk=job)
+        job.state = BankImportJob.STATE_RUNNING
+        job.save()
+
+        try:
+            transactions = _get_unknown_transactions(event, job, data)
+
+            code_len = settings.ENTROPY['order_code']
+            pattern = re.compile(event.slug.upper() + "[ -_]*([A-Z0-9]{%s})" % code_len)
+
+            for trans in transactions:
+                match = pattern.search(trans.reference.upper())
+
+                if match:
+                    code = match.group(1)
+                    with transaction.atomic():
+                        _handle_transaction(event, trans, code)
+                else:
+                    trans.state = BankTransaction.STATE_NOMATCH
+                    trans.save()
+        except Exception as e:
+            job.state = BankImportJob.STATE_ERROR
+            job.save()
+            raise e
+        else:
+            job.state = BankImportJob.STATE_COMPLETED
+            job.save()
+
+
+if settings.HAS_CELERY:
+    from pretix.celery import app
+
+    process_task = app.task(process_banktransfers)
+
+    def process_banktransfers(*args, **kwargs):
+        process_task.apply_async(args=args, kwargs=kwargs)

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/control_head.html

@@ -0,0 +1,6 @@
+{% load staticfiles %}
+{% load compress %}
+
+{% compress css %}
+    <link type="text/css" rel="stylesheet" href="{% static "pretixplugins/banktransfer/ui.css" %}">
+{% endcompress %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/hbci_log.html

@@ -1,9 +0,0 @@
-{% extends "pretixplugins/banktransfer/import_base.html" %}
-{% load i18n %}
-{% load bootstrap3 %}
-{% block inner %}
-    <div class="alert alert-danger">
-        {% trans "An error occured during the HBCI transaction." %}
-    </div>
-    <pre>{{ log }}</pre>
-{% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_base.html

@@ -1,8 +1,10 @@
 {% extends "pretixcontrol/event/base.html" %}
 {% load i18n %}
+{% load static %}
 {% block title %}{% trans "Import bank data" %}{% endblock %}
 {% block content %}
     <h1>{% trans "Import bank data" %}</h1>
     {% block inner %}
     {% endblock %}
+    <script type="application/javascript" src="{% static "pretixplugins/banktransfer/ui.js" %}"></script>
 {% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_confirm.html

@@ -1,54 +0,0 @@
-{% extends "pretixplugins/banktransfer/import_base.html" %}
-{% load i18n %}
-{% block inner %}
-    <p>{% blocktrans trimmed %}
-        We detected the following payments. Please review them and click the 'Confirm' button below.
-    {% endblocktrans %}</p>
-    <form method="post" action="">
-        {% csrf_token %}
-        <table class="table table-condensed">
-            <thead>
-                <tr>
-                    <th></th>
-                    <th>{% trans "Date" %}</th>
-                    <th>{% trans "Reference" %}</th>
-                    <th>{% trans "Amount" %}</th>
-                    <th>{% trans "Payer" %}</th>
-                    <th>{% trans "Order" %}</th>
-                    <th>{% trans "Status" %}</th>
-                </tr>
-            </thead>
-            <tbody>
-                {% for row in rows %}
-                    <tr class="{{ row.class }}">
-                        <th>{% if row.ok %}
-                            <input type="checkbox" name="mark_paid" value="{{ row.order.code }}"
-                                checked="checked" />
-                            <input type="hidden" name="date_{{ row.order.code }}" value="{{ row.date }}" />
-                            <input type="hidden" name="payer_{{ row.order.code }}" value="{{ row.payer }}" />
-                            <input type="hidden" name="reference_{{ row.order.code }}" value="{{ row.reference }}" />
-                        {% endif %}</th>
-                        <td>{{ row.date }}</td>
-                        <td>{{ row.reference }}</td>
-                        <td>{{ row.amount }}</td>
-                        <td>{{ row.payer }}</td>
-                        <td>{% if row.order %}
-                            <a href="{% url "control:event.order" event=request.event.slug organizer=request.event.organizer.slug code=row.order.code %}"
-                                    target="_blank">
-                                {{ row.order.code }}
-                            </a>
-                            {% endif %}
-                        </td>
-                        <td>{{ row.message }}</td>
-                    </tr>
-                {% endfor %}
-            </tbody>
-            <input type="hidden" name="confirm" value="true" />
-
-        </table>
-        <button class="btn btn-primary btn-lg pull-right helper-space-below" type="submit">
-            <span class="icon icon-upload"></span> {% trans "Confirm" %}
-        </button>
-        <div class="clearfix"></div>
-    </form>
-{% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_form.html

@@ -16,7 +16,7 @@
             <form action="" method="post" enctype="multipart/form-data" class="form-inline">
                 {% csrf_token %}
                 <div class="form-group">
-                    <label for="file">{% trans "Import file" %}: </label> <input id="file" type="file" name="file" />
+                    <label for="file">{% trans "Import file" %}: </label> <input id="file" type="file" name="file"/>
                 </div>
                 <div class="clearfix"></div>
                 <button class="btn btn-primary pull-right" type="submit">
@@ -25,38 +25,32 @@
             </form>
         </div>
     </div>
-    <!--
-    <div class="panel panel-default">
-        <div class="panel-heading">
-            <h3 class="panel-title">{% trans "HBCI import" %}</h3>
-        </div>
-        <div class="panel-body">
-            {% if hbci_available %}
-            <form action="" method="post" class="form-horizontal">
-                {% csrf_token %}
-                {% bootstrap_form hbci_form layout='horizontal' %}
-                <div class="clearfix"></div>
+    {% if transactions_unhandled|length > 0 %}
+        <div class="panel panel-default">
+            <div class="panel-heading">
+                <h3 class="panel-title">{% trans "Unresolved transactions" %}</h3>
+            </div>
+            <div class="panel-body">
                 <p>
-                    {% trans "Please note that this step might take a few minutes." %}
+                    <form class="form-inline helper-display-inline" action="" method="get">
+                        <input type="text" name="search" class="form-control" placeholder="{% trans "Search" %}" value="{{ request.GET.search }}">
+                    </form>
+                    <button class="btn btn-primary" type="submit">{% trans "Filter" %}</button>
+                    {% if not request.GET.search %}
+                        <form action="" method="post" class="helper-display-inline pull-right">
+                            {% csrf_token %}
+                            <button class="btn btn-danger" type="submit" name="discard" value="all">
+                                <span class="fa fa-trash"></span>
+                                {% trans "Discard all" %}
+                            </button>
+                        </form>
+                    {% endif %}
                 </p>
-                <div class="alert alert-warning">
-                    {% blocktrans trimmed %}
-                        Your HBCI PIN code is only stored on the server for a very short time and
-                        immediately deleted after the communication with the bank is over. However,
-                        we <strong>strongly advise</strong> you to ask your bank for HBCI credentials
-                        that have <strong>read-only</strong> access to your bank account.
-                    {% endblocktrans %}
-                </div>
-                <button class="btn btn-primary pull-right" type="submit">
-                    <span class="icon icon-upload"></span> {% trans "Import" %}
-                </button>
-            </form>
-            {% else %}
-                <div class="alert alert-error">
-                    {% trans "HBCI is only available with aqbanking-cli and aqhbci-tool4 installed." %}
-                </div>
-            {% endif %}
+
+                {% include "pretixcontrol/pagination.html" %}
+                {% include "pretixplugins/banktransfer/transaction_list.html" with list=transactions_unhandled %}
+                {% include "pretixcontrol/pagination.html" %}
+            </div>
         </div>
-    </div>
-    -->
+    {% endif %}
 {% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/job_detail.html

@@ -0,0 +1,47 @@
+{% extends "pretixplugins/banktransfer/import_base.html" %}
+{% load i18n %}
+{% block inner %}
+    <h2>{% trans "Import result" %}</h2>
+    {% if job.state == "running" or job.state == "pending" %}
+        <div class="empty-collection" data-job-waiting>
+            <p>
+                <span class="fa big-grey-icon fa-cog fa big-rotating-icon"></span>
+            </p>
+            <p>
+                {% trans "The result of your import is in progress. Please be patient while we process the data..." %}
+            </p>
+        </div>
+    {% else %}
+        {% if job.state == "error" %}
+            <div class="alert alert-danger">
+                {% trans "An internal error occured during processing your data." %}
+            </div>
+        {% elif transactions_ignored == 0 and transactions_invalid == 0 and transactions_valid == 0 %}
+            <div class="alert alert-info">
+                {% trans "Your import did not contain any transactions that you did not import before." %}
+            </div>
+        {% endif %}
+
+        <table class="table table-condensed">
+            <tr class="success">
+                <td>{{ transactions_valid }}</td>
+                <td>{% trans "Orders marked as paid" %}</td>
+            </tr>
+            <tr class="warning">
+                <td>{{ transactions_invalid }}</td>
+                <td>{% trans "Invalid payments" %}</td>
+            </tr>
+            <tr class="info">
+                <td>{{ transactions_ignored }}</td>
+                <td>{% trans "Ignored payments" %}</td>
+            </tr>
+        </table>
+        {% if transactions_ignored or transactions_invalid %}
+            <a href="{% url "plugins:banktransfer:import" event=request.event.slug organizer=request.organizer.slug %}"
+                class="btn btn-primary">
+                {% trans "Review invalid and ignored payments" %} &raquo;
+            </a>
+        {% endif %}
+
+    {% endif %}
+{% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/transaction_list.html

@@ -0,0 +1,86 @@
+{% load i18n %}
+{% load staticfiles %}
+<div class="table-responsive">
+    {% csrf_token %}
+    <table class="table table-condensed transaction-list" data-url="{% url "plugins:banktransfer:import.action" event=request.event.slug organizer=request.event.organizer.slug %}">
+        <thead>
+        <tr>
+            <th></th>
+            <th>{% trans "Date" %}</th>
+            <th>{% trans "Payer and reference" %}</th>
+            <th>{% trans "Amount" %}</th>
+            <th>{% trans "Result" %}</th>
+            <th>{% trans "Order" %}</th>
+        </tr>
+        </thead>
+        <tbody>
+        {% for trans in list %}
+            <tr data-id="{{ trans.id }}">
+                <td class="actions">
+                    {% if trans.order and trans.state == 'invalid' %}
+                        <div class="btn-group" role="group">
+                            <button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="accept"
+                                data-toggle="tooltip" title="{% trans "Accept anyway" %}" data-placement="right">
+                                <span class="fa fa-check"></span>
+                            </button>
+                            <button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="discard"
+                                data-toggle="tooltip" title="{% trans "Discard" %}">
+                                <span class="fa fa-trash"></span>
+                            </button>
+                        </div>
+                    {% elif trans.state == 'nomatch' %}
+                        <input type="text" class="form-control" placeholder="{% trans "Order code" %}">
+                        <div class="btn-group" role="group">
+                            <button class="btn btn-default" type="button" name="action_{{ trans.id }}"
+                                    value="assign" data-toggle="tooltip" title="{% trans "Assign to order" %}" data-placement="right">
+                                <span class="fa fa-check"></span>
+                            </button>
+                            <button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="discard"
+                                data-toggle="tooltip" title="{% trans "Discard" %}">
+                                <span class="fa fa-trash"></span>
+                            </button>
+                        </div>
+                    {% elif trans.state == 'error' %}
+                        <div class="btn-group" role="group">
+                            <button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="retry"
+                                data-toggle="tooltip" title="{% trans "Retry" %}" data-placement="right">
+                                <span class="fa fa-refresh"></span>
+                            </button>
+                            <button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="discard"
+                                data-toggle="tooltip" title="{% trans "Discard" %}">
+                                <span class="fa fa-trash"></span>
+                            </button>
+                        </div>
+                    {% endif %}
+                </td>
+                <td>{{ trans.date|date:"SHORT_DATE_FORMAT" }}</td>
+                <td>{{ trans.payer }}<br/>{{ trans.reference }}</td>
+                <td>{{ trans.amount|floatformat:2 }}</td>
+                <td>
+                    {% if trans.message %}
+                        {% trans trans.message %}
+                    {% elif trans.state == 'nomatch' %}
+                        {% trans "No order code detected" %}
+                    {% elif trans.state == 'invalid' %}
+                        {% trans "Invalid for this order" %}
+                    {% elif trans.state == 'error' %}
+                        {% trans "Error while processing" %}
+                    {% elif trans.state == 'valid' %}
+                        {% trans "The order is already marked as paid" %}
+                    {% elif trans.state == 'already' %}
+                        {% trans "Order already paid" %}
+                    {% endif %}
+                </td>
+                <td>
+                    {% if trans.order %}
+                        <a href="{% url "control:event.order" event=request.event.slug organizer=request.event.organizer.slug code=trans.order.code %}"
+                            data-toggle="tooltip" title="{{ trans.order.total|floatformat:2 }} {{ request.event.currency }}">
+                            {{ trans.order.code }}
+                        </a>
+                    {% endif %}
+                </td>
+            </tr>
+        {% endfor %}
+        </tbody>
+    </table>
+</div>

src/pretix/plugins/banktransfer/urls.py

@@ -5,4 +5,8 @@ from . import views
 urlpatterns = [
     url(r'^control/event/(?P<organizer>[^/]+)/(?P<event>[^/]+)/banktransfer/import/', views.ImportView.as_view(),
         name='import'),
+    url(r'^control/event/(?P<organizer>[^/]+)/(?P<event>[^/]+)/banktransfer/job/(?P<job>\d+)/',
+        views.JobDetailView.as_view(), name='import.job'),
+    url(r'^control/event/(?P<organizer>[^/]+)/(?P<event>[^/]+)/banktransfer/action/',
+        views.ActionView.as_view(), name='import.action'),
 ]

src/pretix/plugins/banktransfer/views.py

@@ -1,18 +1,16 @@
 import csv
 import json
 import logging
-import re
-import shutil
-from decimal import Decimal
+from locale import format as lformat
 
-from django import forms
-from django.conf import settings
 from django.contrib import messages
-from django.shortcuts import redirect, render
+from django.core.urlresolvers import reverse
+from django.db.models import Q
+from django.http import JsonResponse
+from django.shortcuts import get_object_or_404, redirect, render
 from django.utils.functional import cached_property
-from django.utils.timezone import now
-from django.utils.translation import ugettext_lazy as _
-from django.views.generic import TemplateView
+from django.utils.translation import ugettext as _
+from django.views.generic import DetailView, ListView, TemplateView, View
 
 from pretix.base.models import Order, Quota
 from pretix.base.services.mail import SendMailException
@@ -20,15 +18,204 @@ from pretix.base.services.orders import mark_order_paid
 from pretix.base.settings import SettingsSandbox
 from pretix.control.permissions import EventPermissionRequiredMixin
 from pretix.plugins.banktransfer import csvimport, mt940import
+from pretix.plugins.banktransfer.models import BankImportJob, BankTransaction
+from pretix.plugins.banktransfer.tasks import process_banktransfers
 
 logger = logging.getLogger('pretix.plugins.banktransfer')
 
 
-class ImportView(EventPermissionRequiredMixin, TemplateView):
-    template_name = 'pretixplugins/banktransfer/import_form.html'
+class ActionView(EventPermissionRequiredMixin, View):
     permission = 'can_change_orders'
 
+    def _discard(self, trans):
+        trans.state = BankTransaction.STATE_DISCARDED
+        trans.shred_private_data()
+        trans.save()
+        return JsonResponse({
+            'status': 'ok'
+        })
+
+    def _retry(self, trans):
+        if trans.amount != trans.order.total:
+            return JsonResponse({
+                'status': 'error',
+                'message': _('The transaction amount is incorrect.')
+            })
+        return self._accept_ignore_amount(trans)
+
+    def _accept_ignore_amount(self, trans):
+        if trans.order.status == Order.STATUS_PAID:
+            return JsonResponse({
+                'status': 'error',
+                'message': _('The order is already marked as paid.')
+            })
+        elif trans.order.status == Order.STATUS_REFUNDED:
+            return JsonResponse({
+                'status': 'error',
+                'message': _('The order has already been refunded.')
+            })
+        elif trans.order.status == Order.STATUS_CANCELLED:
+            return JsonResponse({
+                'status': 'error',
+                'message': _('The order has already been cancelled.')
+            })
+
+        try:
+            mark_order_paid(trans.order, provider='banktransfer', info=json.dumps({
+                'reference': trans.reference,
+                'date': trans.date.isoformat(),
+                'payer': trans.payer,
+                'trans_id': trans.pk
+            }))
+            trans.state = BankTransaction.STATE_VALID
+            trans.save()
+        except Quota.QuotaExceededException as e:
+            return JsonResponse({
+                'status': 'error',
+                'message': str(e)
+            })
+        except SendMailException:
+            return JsonResponse({
+                'status': 'error',
+                'message': _('Problem sending email.')
+            })
+        else:
+            trans.state = BankTransaction.STATE_VALID
+            trans.save()
+            return JsonResponse({
+                'status': 'ok'
+            })
+
+    def _assign(self, trans, code):
+        try:
+            trans.order = self.request.event.orders.get(code=code)
+        except Order.DoesNotExist:
+            return JsonResponse({
+                'status': 'error',
+                'message': _('Unknown order code.')
+            })
+        else:
+            return self._retry(trans)
+
+    def post(self, request, *args, **kwargs):
+        for k, v in request.POST.items():
+            if not k.startswith('action_'):
+                continue
+            trans = get_object_or_404(BankTransaction, id=k.split('_')[1], event=self.request.event)
+
+            if v == 'discard' and trans.state in (BankTransaction.STATE_INVALID, BankTransaction.STATE_ERROR,
+                                                  BankTransaction.STATE_NOMATCH):
+                return self._discard(trans)
+
+            elif v == 'accept' and trans.state == BankTransaction.STATE_INVALID:
+                # Accept anyway even with wrong amount
+                return self._accept_ignore_amount(trans)
+
+            elif v.startswith('assign:') and trans.state == BankTransaction.STATE_NOMATCH:
+                return self._assign(trans, v[7:])
+
+            elif v == 'retry' and trans.state == BankTransaction.STATE_ERROR:
+                return self._retry(trans)
+
+            return JsonResponse({
+                'status': 'error',
+                'message': 'Unknown action'
+            })
+
+    def get(self, request, *args, **kwargs):
+        query = request.GET.get('query', '')
+        if len(query) < 2:
+            return JsonResponse({'results': []})
+
+        qs = self.request.event.orders.filter(Q(code__icontains=query) | Q(code__icontains=Order.normalize_code(query)))
+        return JsonResponse({
+            'results': [
+                {
+                    'code': o.code,
+                    'status': o.get_status_display(),
+                    'total': lformat("%.2f", o.total) + ' ' + self.request.event.currency
+                } for o in qs
+            ]
+        })
+
+
+class JobDetailView(EventPermissionRequiredMixin, DetailView):
+    template_name = 'pretixplugins/banktransfer/job_detail.html'
+    permission = 'can_change_orders'
+    context_objectname = 'job'
+
+    def redirect_form(self):
+        return redirect(reverse('plugins:banktransfer:import', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+        }))
+
+    def redirect_back(self):
+        return redirect(reverse('plugins:banktransfer:import.job', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+            'job': self.kwargs['job']
+        }))
+
+    def get_object(self, queryset=None):
+        return get_object_or_404(BankImportJob, id=self.kwargs['job'], event=self.request.event)
+
+    def get(self, request, *args, **kwargs):
+        if 'ajax' in request.GET:
+            self.object = self.get_object()
+            return JsonResponse({
+                'state': self.object.state
+            })
+
+        return super().get(request, *args, **kwargs)
+
+    def get_context_data(self, **kwargs):
+        ctx = super().get_context_data()
+
+        qs = self.object.transactions.select_related('order')
+
+        ctx['transactions_valid'] = qs.filter(state=BankTransaction.STATE_VALID).count()
+        ctx['transactions_invalid'] = qs.filter(state__in=[
+            BankTransaction.STATE_INVALID, BankTransaction.STATE_ERROR
+        ]).count()
+        ctx['transactions_ignored'] = qs.filter(state__in=[
+            BankTransaction.STATE_DUPLICATE, BankTransaction.STATE_NOMATCH
+        ]).count()
+        ctx['job'] = self.object
+
+        return ctx
+
+
+class ImportView(EventPermissionRequiredMixin, ListView):
+    template_name = 'pretixplugins/banktransfer/import_form.html'
+    permission = 'can_change_orders'
+    context_object_name = 'transactions_unhandled'
+    paginate_by = 30
+
+    def get_queryset(self):
+        qs = BankTransaction.objects.filter(
+            event=self.request.event
+        ).select_related('order').filter(state__in=[
+            BankTransaction.STATE_INVALID, BankTransaction.STATE_ERROR,
+            BankTransaction.STATE_DUPLICATE, BankTransaction.STATE_NOMATCH
+        ])
+        if 'search' in self.request.GET:
+            q = self.request.GET.get('search')
+            qs = qs.filter(
+                Q(payer__icontains=q) | Q(reference__icontains=q)
+            )
+
+        return qs
+
+    def discard_all(self):
+        self.get_queryset().update(payer='', reference='', state=BankTransaction.STATE_DISCARDED)
+        messages.success(self.request, _('All unresolved transactions have been discarded.'))
+
     def post(self, *args, **kwargs):
+        if self.request.POST.get('discard', '') == 'all':
+            self.discard_all()
+            return self.redirect_back()
+
         if ('file' in self.request.FILES and 'csv' in self.request.FILES.get('file').name.lower()) \
                 or 'amount' in self.request.POST:
             # Process CSV
@@ -37,35 +224,6 @@ class ImportView(EventPermissionRequiredMixin, TemplateView):
         if 'file' in self.request.FILES and 'txt' in self.request.FILES.get('file').name.lower():
             return self.process_mt940()
 
-        if 'confirm' in self.request.POST:
-            orders = Order.objects.filter(event=self.request.event,
-                                          code__in=self.request.POST.getlist('mark_paid'))
-            some_failed = False
-            mail_failures = False
-            for order in orders:
-                try:
-                    mark_order_paid(order, provider='banktransfer', info=json.dumps({
-                        'reference': self.request.POST.get('reference_%s' % order.code),
-                        'date': self.request.POST.get('date_%s' % order.code),
-                        'payer': self.request.POST.get('payer_%s' % order.code),
-                        'import': now().isoformat(),
-                    }))
-                except Quota.QuotaExceededException:
-                    some_failed = True
-                except SendMailException:
-                    mail_failures = True
-
-            if some_failed:
-                messages.warning(self.request, _('Not all of the selected orders could be marked as '
-                                                 'paid as some of them have expired and the selected '
-                                                 'items are sold out.'))
-            else:
-                messages.success(self.request, _('The selected orders have been marked as paid.'))
-                # TODO: Display a list of them!
-            if mail_failures:
-                messages.warning(self.request, _('Some confirmation mails could not be sent.'))
-            return self.redirect_back()
-
         messages.error(self.request, _('We were unable to detect the file type of this import. Please '
                                        'contact support for help.'))
         return self.redirect_back()
@@ -76,7 +234,7 @@ class ImportView(EventPermissionRequiredMixin, TemplateView):
 
     def process_mt940(self):
         try:
-            return self.confirm_view(mt940import.parse(self.request.FILES.get('file')))
+            return self.start_processing(mt940import.parse(self.request.FILES.get('file')))
         except:
             logger.exception('Failed to import MT940 file')
             messages.error(self.request, _('We were unable to process your input.'))
@@ -102,7 +260,7 @@ class ImportView(EventPermissionRequiredMixin, TemplateView):
             except csvimport.HintMismatchError:  # TODO: narrow down
                 logger.exception('Import using stored hint failed')
             else:
-                return self.confirm_view(parsed)
+                return self.start_processing(parsed)
 
         return self.assign_view(data)
 
@@ -113,7 +271,7 @@ class ImportView(EventPermissionRequiredMixin, TemplateView):
                 [
                     self.request.POST.get('col[%d][%d]' % (i, j))
                     for j in range(int(self.request.POST.get('cols')))
-                ]
+                    ]
             )
         if 'reference' not in self.request.POST:
             messages.error(self.request, _('You need to select the column containing the payment reference.'))
@@ -131,7 +289,7 @@ class ImportView(EventPermissionRequiredMixin, TemplateView):
             pass
         else:
             parsed = csvimport.parse(data, hint)
-            return self.confirm_view(parsed)
+            return self.start_processing(parsed)
 
     def process_csv(self):
         if 'file' in self.request.FILES:
@@ -140,81 +298,22 @@ class ImportView(EventPermissionRequiredMixin, TemplateView):
             return self.process_csv_hint()
         return super().get(self.request)
 
-    def confirm_view(self, parsed):
-        parsed = self.annotate_data(parsed)
-        return render(self.request, 'pretixplugins/banktransfer/import_confirm.html', {
-            'rows': parsed
-        })
-
     def assign_view(self, parsed):
         return render(self.request, 'pretixplugins/banktransfer/import_assign.html', {
             'rows': parsed
         })
 
     def redirect_back(self):
-        return redirect('plugins:banktransfer:import',
-                        event=self.request.event.slug,
-                        organizer=self.request.event.organizer.slug)
+        return redirect(reverse('plugins:banktransfer:import', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+        }))
 
-    def annotate_data(self, data):
-        code_len = settings.ENTROPY['order_code']
-        pattern = re.compile(self.request.event.slug.upper() + "[ -_]*([A-Z0-9]{%s})" % code_len)
-        amount_pattern = re.compile("[^0-9.-]")
-        order_codes = []
-        for row in data:
-            row['ok'] = False
-            try:
-                amount = Decimal(amount_pattern.sub("", row['amount'].replace(",", ".")))
-                row['amount'] = str(amount)
-            except:
-                logger.exception('Could not parse amount of transaction')
-                amount = 0
-            match = pattern.search(row['reference'].upper())
-            if not match:
-                row['class'] = 'warning' if amount > 0 else ''
-                row['message'] = _('No order code detected')
-                continue
-
-            code = match.group(1)
-            row['code'] = code
-            order_codes.append(code)
-            normalized_code = Order.normalize_code(code)
-            if normalized_code != code:
-                order_codes.append(normalized_code)
-
-        orders = {}
-        # Perform query in bulks because of SQLite's default of SQLITE_MAX_VARIABLE_NUMBER = 999
-        for i in range(0, len(order_codes), 500):
-            orders.update({
-                o.code: o for o in self.request.event.orders.filter(code__in=order_codes[i:500])
-            })
-
-        for row in data:
-            if 'code' not in row:
-                continue
-            normalized_code = Order.normalize_code(row['code'])
-            if row['code'] in orders or normalized_code in orders:
-                order = orders[row['code']] if row['code'] in orders else orders[normalized_code]
-                row['order'] = order
-                if order.status == Order.STATUS_PENDING:
-                    amount = Decimal(row['amount'])
-                    if amount != order.total:
-                        row['class'] = 'danger'
-                        row['message'] = _('Found wrong amount. Expected: %s' % str(order.total))
-                    else:
-                        row['class'] = 'success'
-                        row['message'] = _('Valid payment')
-                        row['ok'] = True
-                elif order.status == Order.STATUS_CANCELLED:
-                    row['class'] = 'danger'
-                    row['message'] = _('Order has been cancelled')
-                elif order.status == Order.STATUS_PAID:
-                    row['class'] = ''
-                    row['message'] = _('Order already has been paid')
-                elif order.status == Order.STATUS_REFUNDED:
-                    row['class'] = 'warning'
-                    row['message'] = _('Order has been refunded')
-            else:
-                row['class'] = 'danger'
-                row['message'] = _('Unknown order code detected')
-        return data
+    def start_processing(self, parsed):
+        job = BankImportJob.objects.create(event=self.request.event)
+        process_banktransfers(event=self.request.event.pk, job=job.pk, data=parsed)
+        return redirect(reverse('plugins:banktransfer:import.job', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+            'job': job.pk
+        }))