Allow import bank data from CSV files

src/pretix/control/context.py

@@ -8,7 +8,7 @@ def contextprocessor(request):
     Adds data to all template contexts
     """
     url = resolve(request.path_info)
-    if url.namespace != 'control':
+    if not request.path.startswith('/control'):
         return {}
     ctx = {
         'url_name': url.url_name,

src/pretix/control/middleware.py

@@ -24,9 +24,8 @@ class PermissionMiddleware:
 
     def process_request(self, request):
         url = resolve(request.path_info)
-        url_namespace = url.namespace
         url_name = url.url_name
-        if url_namespace != 'control' or url_name in self.EXCEPTIONS:
+        if not request.path.startswith('/control') or url_name in self.EXCEPTIONS:
             return
         if not request.user.is_authenticated():
             # Taken from django/contrib/auth/decorators.py
@@ -47,7 +46,7 @@ class PermissionMiddleware:
 
         request.user.events_cache = request.user.events.current.order_by(
             "organizer", "date_from").prefetch_related("organizer")
-        if 'event.' in url_name and 'event' in url.kwargs:
+        if 'event' in url.kwargs and 'organizer' in url.kwargs:
             try:
                 request.event = Event.objects.current.filter(
                     slug=url.kwargs['event'],

src/pretix/control/static/pretixcontrol/less/main.less

@@ -53,4 +53,15 @@ nav.navbar {
 }
 .container-fluid > .alert:first-child {
     margin-top: 20px;
+}
+
+.flipped-scroll-wrapper {
+    overflow-y: auto;
+}
+.flipped-scroll-wrapper, .flipped-scroll-inner {
+    /* This nasty hack puts the scroll bar at the top, so the user really
+       notices that there is one */
+    transform: rotateX(180deg);
+    -ms-transform: rotateX(180deg); /* IE 9 */
+    -webkit-transform: rotateX(180deg); /* Safari and Chrome */
 }

src/pretix/plugins/banktransfer/csvimport.py

@@ -0,0 +1,85 @@
+import csv
+import io
+
+
+class HintMismatchError(Exception):
+    pass
+
+
+def parse(data, hint):
+    result = []
+    if 'cols' not in hint:
+        raise HintMismatchError('Invalid hint')
+    if len(data[0]) != hint['cols']:
+        raise HintMismatchError('Wrong column count')
+    for row in data:
+        resrow = {}
+        if None in row or len(row) == 0:
+            # Wrong column count
+            continue
+        if hint.get('payer') is not None:
+            resrow['payer'] = "\n".join([row[int(i)].strip() for i in hint.get('payer')])
+        if hint.get('reference') is not None:
+            resrow['reference'] = "\n".join([row[int(i)].strip() for i in hint.get('reference')])
+        if hint.get('amount') is not None:
+            resrow['amount'] = row[int(hint.get('amount'))].strip()
+        if hint.get('date') is not None:
+            resrow['date'] = row[int(hint.get('date'))].strip()
+        if len(resrow['amount']) == 0 or 'amount' not in resrow \
+                or resrow['amount'][0] not in list("1234567890," "+- ") \
+                or len(resrow['reference']) == 0:
+            # This is probably a headline or something other special.
+            continue
+        result.append(resrow)
+    return result
+
+
+def get_rows_from_file(file):
+    data = file.read()
+    try:
+        import chardet
+        charset = chardet.detect(data)['encoding']
+    except ImportError:
+        charset = file.charset
+    data = data.decode(charset or 'utf-8')
+    # Sniffing line by line is necessary as some banks like to include
+    # one-column garbage at the beginning of the file which breaks the sniffer.
+    # See also: http://bugs.python.org/issue2078
+    last_e = None
+    dialect = None
+    for line in data.split("\n"):
+        line = line.strip()
+        if len(line) == 0:
+            continue
+        try:
+            dialect = csv.Sniffer().sniff(line, delimiters=";,.#:")
+        except Exception as e:
+            last_e = e
+        else:
+            last_e = None
+            break
+    if dialect is None:
+        raise last_e
+    reader = csv.reader(io.StringIO(data), dialect)
+    rows = []
+    for row in reader:
+        if rows and len(row) > len(rows[0]):
+            # Some banks put metadata above the real data, things like
+            # a headline, the bank's name, the user's name, etc.
+            # In many cases, we can identify this because these rows
+            # have less columns than the rows containing the real data.
+            # Therefore, if the number of columns suddenly grows, we start
+            # over with parsing.
+            rows = []
+        rows.append(row)
+    return rows
+
+
+def new_hint(data):
+    return {
+        'payer': data.getlist('payer') if 'payer' in data else None,
+        'reference': data.getlist('reference') if 'date' in data else None,
+        'date': int(data.get('date')) if 'date' in data else None,
+        'amount': int(data.get('amount')) if 'amount' in data else None,
+        'cols': int(data.get('cols')) if 'cols' in data else None
+    }

src/pretix/plugins/banktransfer/signals.py

@@ -1,10 +1,29 @@
+from django.core.urlresolvers import reverse, resolve
 from django.dispatch import receiver
+from django.utils.translation import ugettext_lazy as _
 
 from pretix.base.signals import register_payment_providers
 
 from .payment import BankTransfer
+from pretix.control.signals import nav_event
 
 
 @receiver(register_payment_providers)
 def register_payment_provider(sender, **kwargs):
     return BankTransfer
+
+
+@receiver(nav_event)
+def html_head_presale(sender, request=None, **kwargs):
+    url = resolve(request.path_info)
+    return [
+        {
+            'label': _('Import bank data'),
+            'url': reverse('plugins:banktransfer.import', kwargs={
+                'event': request.event.slug,
+                'organizer': request.event.organizer.slug,
+            }),
+            'active': (url.namespace == 'plugins' and url.url_name == 'banktransfer.import'),
+            'icon': 'upload',
+        }
+    ]

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_assign.html

@@ -0,0 +1,68 @@
+{% extends "pretixplugins/banktransfer/import_base.html" %}
+{% load i18n %}
+{% block inner %}
+    <p>{% blocktrans trimmed %}
+        We've been unable to automatically determine how the columns in your file are aligned. Please help us
+        by selecting which column contain what kind of data.
+    {% endblocktrans %}</p>
+    <form method="post" action="">
+        {% csrf_token %}
+        <button class="btn btn-primary pull-right" type="submit" style="margin-bottom: 10px">
+            <span class="icon icon-upload"></span> {% trans "Continue" %}
+        </button>
+        <div class="flipped-scroll-wrapper" style="clear: both;">
+            <table class="table table-condensed flipped-scroll-inner">
+                <thead>
+                    <tr>
+                        <th>{% trans "Date" %}</th>
+                        {% for col in rows.0 %}
+                            <th>
+                                <input type="radio" name="date" value="{{ forloop.counter0 }}" />
+                            </th>
+                        {% endfor %}
+                    </tr>
+                    <tr>
+                        <th>{% trans "Amount" %}</th>
+                        {% for col in rows.0 %}
+                            <th>
+                                <input type="radio" name="amount" value="{{ forloop.counter0 }}" required="required" />
+                            </th>
+                        {% endfor %}
+                    </tr>
+                    <tr>
+                        <th>{% trans "Reference" %}</th>
+                        {% for col in rows.0 %}
+                            <th>
+                                <input type="checkbox" name="reference" value="{{ forloop.counter0 }}" />
+                            </th>
+                        {% endfor %}
+                    </tr>
+                    <tr>
+                        <th>{% trans "Payer" %}</th>
+                        {% for col in rows.0 %}
+                            <th>
+                                <input type="checkbox" name="payer" value="{{ forloop.counter0 }}" />
+                            </th>
+                        {% endfor %}
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for row in rows %}
+                        {% with forloop.counter0 as rowid %}
+                            <tr>
+                                <td></td>
+                                {% for col in row %}
+                                    <td>{{ col }}<input type="hidden" name="col[{{ rowid }}][{{ forloop.counter0 }}]"
+                                            value="{{ col }}" /></td>
+                                {% endfor %}
+                            </tr>
+                        {% endwith %}
+                    {% endfor %}
+                </tbody>
+                <input type="hidden" name="cols" value="{{ rows.0|length }}" />
+                <input type="hidden" name="rows" value="{{ rows|length }}" />
+
+            </table>
+        </div>
+    </form>
+{% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_base.html

@@ -0,0 +1,8 @@
+{% extends "pretixcontrol/event/base.html" %}
+{% load i18n %}
+{% block title %}{% trans "Import bank data" %}{% endblock %}
+{% block content %}
+    <h1>{% trans "Import bank data" %}</h1>
+    {% block inner %}
+    {% endblock %}
+{% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_confirm.html

@@ -0,0 +1,55 @@
+{% extends "pretixplugins/banktransfer/import_base.html" %}
+{% load i18n %}
+{% block inner %}
+    <p>{% blocktrans trimmed %}
+        We detected the following payments. Please review them and click the 'Confirm' button below.
+    {% endblocktrans %}</p>
+    <form method="post" action="">
+        {% csrf_token %}
+        <table class="table table-condensed">
+            <thead>
+                <tr>
+                    <th></th>
+                    <th>{% trans "Date" %}</th>
+                    <th>{% trans "Reference" %}</th>
+                    <th>{% trans "Amount" %}</th>
+                    <th>{% trans "Payer" %}</th>
+                    <th>{% trans "Order" %}</th>
+                    <th>{% trans "Status" %}</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for row in rows %}
+                    <tr class="{{ row.class }}">
+                        <th>{% if row.ok %}
+                            <input type="checkbox" name="mark_paid" value="{{ row.order.code }}"
+                                checked="checked" />
+                            <input type="hidden" name="date_{{ row.date }}" value="{{ row.date }}" />
+                            <input type="hidden" name="payer_{{ row.payer }}" value="{{ row.payer }}" />
+                            <input type="hidden" name="reference_{{ row.reference }}" value="{{ row.reference }}" />
+                        {% endif %}</th>
+                        <td>{{ row.date }}</td>
+                        <td>{{ row.reference }}</td>
+                        <td>{{ row.amount }}</td>
+                        <td>{{ row.payer }}</td>
+                        <td>{% if row.order %}
+                            <a href="{% url "control:event.order" event=request.event.slug organizer=request.event.organizer.slug code=row.order.code %}"
+                                    target="_blank">
+                                {{ row.order.code }}
+                            </a>
+                            {% endif %}
+                        </td>
+                        <td>{{ row.message }}</td>
+                    </tr>
+                {% endfor %}
+            </tbody>
+            <input type="hidden" name="cols" value="{{ rows.0|length }}" />
+            <input type="hidden" name="rows" value="{{ rows|length }}" />
+
+        </table>
+        <button class="btn btn-primary btn-lg pull-right" type="submit" style="margin-bottom: 10px">
+            <span class="icon icon-upload"></span> {% trans "Confirm" %}
+        </button>
+        <div class="clearfix"></div>
+    </form>
+{% endblock %}

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_form.html

@@ -0,0 +1,25 @@
+{% extends "pretixplugins/banktransfer/import_base.html" %}
+{% load i18n %}
+{% block inner %}
+    <p>{% blocktrans trimmed %}
+        This page allows you to upload bank statement files to process incoming payments.
+    {% endblocktrans %}</p>
+    <p>{% blocktrans trimmed %}
+        Currently, only <code>.csv</code> files are supported.
+    {% endblocktrans %}</p>
+    <div class="panel panel-default">
+        <div class="panel-heading">
+            <h3 class="panel-title">{% trans "Upload a new file" %}</h3>
+        </div>
+        <div class="panel-body">
+            <form action="" method="post" enctype="multipart/form-data" class="form-inline">
+                {% csrf_token %}
+                <div class="form-group">
+                    <input type="file" name="file" />
+                </div>
+                <button class="btn btn-primary pull-right" type="submit">
+                    <span class="icon icon-upload"></span> {% trans "Start upload" %}
+                </button>
+            </form></div>
+    </div>
+{% endblock %}

src/pretix/plugins/banktransfer/urls.py

@@ -0,0 +1,9 @@
+from django.conf.urls import url
+
+from .views import *
+
+
+urlpatterns = [
+    url(r'^control/event/(?P<organizer>[^/]+)/(?P<event>[^/]+)/banktransfer/import/', ImportView.as_view(),
+        name='banktransfer.import'),
+]

src/pretix/plugins/banktransfer/views.py

@@ -0,0 +1,148 @@
+import csv
+from decimal import Decimal
+import json
+import logging
+import re
+from django.contrib import messages
+from django.core.urlresolvers import reverse
+from django.shortcuts import redirect, render
+from django.utils.timezone import now
+from django.views.generic import TemplateView
+from pretix.base.models import Order
+from pretix.control.permissions import EventPermissionRequiredMixin
+from pretix.plugins.banktransfer import csvimport
+from django.utils.translation import ugettext_lazy as _
+
+
+logger = logging.getLogger('pretix.plugins.banktransfer')
+
+
+class ImportView(EventPermissionRequiredMixin, TemplateView):
+    template_name = 'pretixplugins/banktransfer/import_form.html'
+    permission = 'can_change_orders'
+
+    def post(self, *args, **kwargs):
+        if 'mark_paid' in self.request.POST:
+            orders = Order.objects.filter(event=self.request.event,
+                                          code__in=self.request.POST.getlist('mark_paid'))
+            for order in orders:
+                order.mark_paid(provider='banktransfer', info=json.dumps({
+                    'reference': self.request.POST.get('reference_%s' % order.code),
+                    'date': self.request.POST.get('date_%s' % order.code),
+                    'payer': self.request.POST.get('payer_%s' % order.code),
+                    'import': now().isoformat(),
+                }))
+
+            messages.success(self.request, _('The selected orders have been marked as paid.'))
+            return self.redirect_back()
+        return self.process_csv()
+
+    def process_csv(self):
+        if 'file' in self.request.FILES:
+            # if file is csv file
+            try:
+                data = csvimport.get_rows_from_file(self.request.FILES['file'])
+            except csv.Error as e:  # TODO: narrow down
+                logger.error('Import failed: ' + str(e))
+                messages.error(self.request, _('I\'m sorry, but we were unable to import this CSV file. Please '
+                                               'contact support for help.'))
+                return self.redirect_back()
+
+            if len(data) == 0:
+                messages.error(self.request, _('I\'m sorry, but we detected this file as empty. Please '
+                                               'contact support for help.'))
+
+            if self.request.event.settings.get('banktransfer_csvhint') is not None:
+                hint = self.request.event.settings.get('banktransfer_csvhint', as_type=dict)
+                try:
+                    parsed = csvimport.parse(data, hint)
+                except csvimport.HintMismatchError as e:  # TODO: narrow down
+                    logger.error('Import using stored hint failed: ' + str(e))
+                else:
+                    return self.confirm_view(parsed)
+
+            return self.assign_view(data)
+
+        elif 'amount' in self.request.POST:  # CSV hint given
+            data = []
+            for i in range(int(self.request.POST.get('rows'))):
+                data.append([
+                    self.request.POST.get('col[%d][%d]' % (i, j))
+                    for j in range(int(self.request.POST.get('cols')))
+                ])
+            if 'reference' not in self.request.POST:
+                messages.error(self.request, _('You need to select the column containing the payment reference.'))
+                return self.assign_view(data)
+            try:
+                hint = csvimport.new_hint(self.request.POST)
+            except Exception as e:
+                logger.error('Parsing hint failed: ' + str(e))
+                messages.error(self.request, _('We were unable to process your input.'))
+                return self.assign_view(data)
+            try:
+                self.request.event.settings.set('banktransfer_csvhint', hint)
+            except Exception as e:  # TODO: narrow down
+                logger.error('Import using stored hint failed: ' + str(e))
+                pass
+            else:
+                parsed = csvimport.parse(data, hint)
+                return self.confirm_view(parsed)
+        return super().get(self.request)
+
+    def confirm_view(self, parsed):
+        parsed = self.annotate_data(parsed)
+        return render(self.request, 'pretixplugins/banktransfer/import_confirm.html', {
+            'rows': parsed
+        })
+
+    def assign_view(self, parsed):
+        return render(self.request, 'pretixplugins/banktransfer/import_assign.html', {
+            'rows': parsed
+        })
+
+    def redirect_back(self):
+        return redirect(reverse('plugins:banktransfer.import', kwargs={
+            'event': self.request.event.slug,
+            'organizer': self.request.event.organizer.slug,
+        }))
+
+    def annotate_data(self, data):
+        pattern = re.compile(self.request.event.slug.upper() + "([A-Z0-9]{5})")
+        amount_pattern = re.compile("[^0-9.-]")
+        for row in data:
+            row['ok'] = False
+            match = pattern.search(row['reference'].upper())
+            if not match:
+                row['class'] = ''
+                row['message'] = _('No order code detected')
+                continue
+
+            code = match.group(1)
+            try:
+                order = Order.objects.current.get(event=self.request.event,
+                                                  code=code)
+            except Order.DoesNotExist:
+                row['class'] = 'error'
+                row['message'] = _('Unknown order code detected')
+            else:
+                row['order'] = order
+                if order.status == Order.STATUS_PENDING:
+                    amount = Decimal(amount_pattern.sub("", row['amount'].replace(",", ".")))
+                    if amount != order.total:
+                        row['class'] = 'error'
+                        row['message'] = _('Found wrong amount. Expected: %s' % str(order.total))
+                    else:
+                        row['class'] = 'success'
+                        row['message'] = _('Valid payment')
+                        row['ok'] = True
+                elif order.status == Order.STATUS_CANCELLED:
+                    row['class'] = 'error'
+                    row['message'] = _('Order has been cancelled')
+                elif order.status == Order.STATUS_PAID:
+                    # TODO: Do a plausibility check to tell duplicate payments from overlapping import files
+                    row['class'] = ''
+                    row['message'] = _('Order already has been paid')
+                elif order.status == Order.STATUS_REFUNDED:
+                    row['class'] = 'warning'
+                    row['message'] = _('Order has been refunded')
+        return data