From b55b02f4b81db7d72ebf286c53334f2b56a6b23d Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Thu, 20 Aug 2015 21:53:10 +0200 Subject: [PATCH 1/3] Banktransfer: Added experimental HBCI support --- deployment/docker/standalone/Dockerfile | 1 + src/pretix/plugins/banktransfer/__init__.py | 4 + src/pretix/plugins/banktransfer/hbci.py | 112 ++++++++++++++++++ .../pretixplugins/banktransfer/hbci_log.html | 9 ++ .../banktransfer/import_form.html | 41 +++++-- src/pretix/plugins/banktransfer/views.py | 72 ++++++++++- src/requirements/banktransfer.txt | 1 + 7 files changed, 227 insertions(+), 13 deletions(-) create mode 100644 src/pretix/plugins/banktransfer/hbci.py create mode 100644 src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/hbci_log.html diff --git a/deployment/docker/standalone/Dockerfile b/deployment/docker/standalone/Dockerfile index 6c2001a5a5..94bfb7b546 100644 --- a/deployment/docker/standalone/Dockerfile +++ b/deployment/docker/standalone/Dockerfile @@ -4,6 +4,7 @@ RUN apt-get update && apt-get install -y python3 git python3-pip \ libxml2-dev libxslt1-dev python-dev python-virtualenv locales libffi-dev \ build-essential python3-dev zlib1g-dev libssl-dev npm gettext git \ libpq-dev libmysqlclient-dev libmemcached-dev libjpeg-dev \ + aqbanking-tools \ --no-install-recommends WORKDIR / diff --git a/src/pretix/plugins/banktransfer/__init__.py b/src/pretix/plugins/banktransfer/__init__.py index b513649877..9a52cfcf35 100644 --- a/src/pretix/plugins/banktransfer/__init__.py +++ b/src/pretix/plugins/banktransfer/__init__.py @@ -27,6 +27,10 @@ class BankTransferApp(AppConfig): import chardet # NOQA except ImportError: errs.append(_("Install the python package 'chardet' for better CSV import capabilities.")) + try: + import defusedxml # NOQA + except ImportError: + errs.append(_("Please install the python package 'defusedxml' for security reasons.")) return errs diff --git a/src/pretix/plugins/banktransfer/hbci.py b/src/pretix/plugins/banktransfer/hbci.py new file mode 100644 index 0000000000..8748179d08 --- /dev/null +++ b/src/pretix/plugins/banktransfer/hbci.py @@ -0,0 +1,112 @@ +import subprocess +import tempfile +import time +from decimal import Decimal + + +def hbci_transactions(event, conf): + try: + from defusedxml import ElementTree + except: + from xml.etree import ElementTree + + log = [] + data = [] + accname = event.identity + '_' + str(int(time.time())) + try: + try: + subprocess.call([ + 'aqhbci-tool4', 'deluser', '-a', '--all', + '-b', conf['hbci_blz'], + '-u', conf['hbci_userid'] + ]) + except subprocess.CalledProcessError: + pass + aqhbci_params = [ + 'aqhbci-tool4', 'adduser', + '-N', accname, + '-b', conf['hbci_blz'], + '-s', conf['hbci_server'], + '-t', conf['hbci_tokentype'], + '-u', conf['hbci_userid'] + ] + if conf['hbci_customerid']: + aqhbci_params += ['-c', conf['hbci_customerid']] + if conf['hbci_tokenname']: + aqhbci_params += ['-n', conf['hbci_tokenname']] + if conf['hbci_version']: + aqhbci_params += ['--hbciversion=' + str(conf['hbci_version'])] + aqhbci_add = subprocess.check_output(aqhbci_params) + log.append("$ " + " ".join(aqhbci_params)) + log.append(aqhbci_add.decode("utf-8")) + with tempfile.NamedTemporaryFile() as f, tempfile.NamedTemporaryFile() as g: + f.write(('PIN_%s_%s = "%s"\n' % ( + conf['hbci_blz'], + conf['hbci_userid'], + conf['pin'], + )).encode("utf-8")) + f.flush() + aqhbci_params = [ + 'aqhbci-tool4', + '-P', f.name, + '-n', '-A', + 'getsysid' + ] + aqhbci_test = subprocess.check_output(aqhbci_params) + log.append("$ " + " ".join(aqhbci_params)) + log.append(aqhbci_test.decode("utf-8")) + aqbanking_params = [ + 'aqbanking-cli', + '-P', f.name, '-A', '-n', + 'request', + '--transactions', + '-c', g.name + ] + aqbanking_trans = subprocess.check_output(aqbanking_params) + log.append("$ " + " ".join(aqbanking_params)) + log.append(aqbanking_trans.decode("utf-8")) + aqbanking_params = [ + 'aqbanking-cli', + 'listtrans', + '-c', g.name, + '--exporter=xmldb', + ] + aqbanking_conv = subprocess.check_output(aqbanking_params) + log.append("$ " + " ".join(aqbanking_params)) + + root = ElementTree.fromstring(aqbanking_conv) + trans_list = root.find('accountInfoList').find('accountInfo').find('transactionList') + for trans in trans_list.findall('transaction'): + payer = [] + for child in trans: + if child.tag.startswith('remote'): + payer.append(child.find('value').text) + date = '%s-%02d-%02d' % ( + trans.find('date').find('date').find('year').find('value').text, + int(trans.find('date').find('date').find('month').find('value').text), + int(trans.find('date').find('date').find('day').find('value').text) + ) + value = trans.find('value').find('value').find('value').text + if "/" in value: + parts = value.split("/") + num = int(parts[0]) + denom = int(parts[1]) + value = Decimal(num) / Decimal(denom) + value = str(value.quantize(Decimal('.01'))) + data.append({ + 'payer': "\n".join(payer), + 'reference': trans.find('purpose').find('value').text, + 'amount': value, + 'date': date + }) + except subprocess.CalledProcessError as e: + log.append("Command %s failed with %d and output:" % (e.cmd, e.returncode)) + log.append(e.output.decode("utf-8")) + except Exception as e: + log.append(str(e)) + finally: + subprocess.call([ + 'aqhbci-tool4', 'deluser', '-a', '-N', accname + ]) + log = "\n".join(log) + return data, log diff --git a/src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/hbci_log.html b/src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/hbci_log.html new file mode 100644 index 0000000000..a4311e7c7e --- /dev/null +++ b/src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/hbci_log.html @@ -0,0 +1,9 @@ +{% extends "pretixplugins/banktransfer/import_base.html" %} +{% load i18n %} +{% load bootstrap3 %} +{% block inner %} +
+ {% trans "An error occured during the HBCI transaction." %} +
+ 
{{ log }}
+{% endblock %} diff --git a/src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_form.html b/src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_form.html index 8d461b20cb..44cad1e0a9 100644 --- a/src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_form.html +++ b/src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_form.html @@ -1,25 +1,50 @@ {% extends "pretixplugins/banktransfer/import_base.html" %} {% load i18n %} +{% load bootstrap3 %} {% block inner %} -

{% blocktrans trimmed %} - This page allows you to upload bank statement files to process incoming payments. - {% endblocktrans %}

-

{% blocktrans trimmed %} - Currently, this feature supports .csv files and files in the MT940 format. - {% endblocktrans %}

{% trans "Upload a new file" %}

+

{% blocktrans trimmed %} + This page allows you to upload bank statement files to process incoming payments. + {% endblocktrans %}

+

{% blocktrans trimmed %} + Currently, this feature supports .csv files and files in the MT940 format. + {% endblocktrans %}

{% csrf_token %}
- +
+
-
+ +
+ +
+
+

{% trans "HBCI import" %}

+
+
+ {% if hbci_available %} +
+ {% csrf_token %} + {% bootstrap_form hbci_form layout='horizontal' %} +
+ {% trans "Please note that this step might take a few minutes." %} + +
+ {% else %} +
+ {% trans "HBCI is only available with Python 3.3 or newer and with aqbanking-cli and aqhbci-tool4 installed." %} +
+ {% endif %} +
{% endblock %} diff --git a/src/pretix/plugins/banktransfer/views.py b/src/pretix/plugins/banktransfer/views.py index bbbff55748..97165d324b 100644 --- a/src/pretix/plugins/banktransfer/views.py +++ b/src/pretix/plugins/banktransfer/views.py @@ -2,27 +2,48 @@ import csv import json import logging import re +import shutil +import sys from decimal import Decimal +from django import forms from django.contrib import messages from django.shortcuts import redirect, render +from django.utils.decorators import method_decorator from django.utils.timezone import now from django.utils.translation import ugettext_lazy as _ +from django.views.decorators.debug import sensitive_post_parameters from django.views.generic import TemplateView +from pip.utils import cached_property from pretix.base.models import Order, Quota from pretix.base.services.orders import mark_order_paid +from pretix.base.settings import SettingsSandbox from pretix.control.permissions import EventPermissionRequiredMixin -from pretix.plugins.banktransfer import csvimport, mt940import +from pretix.plugins.banktransfer import csvimport, hbci, mt940import logger = logging.getLogger('pretix.plugins.banktransfer') +class HbciForm(forms.Form): + hbci_blz = forms.CharField(label=_("Bank code")) + hbci_userid = forms.CharField(label=_("User ID")) + hbci_customerid = forms.CharField(label=_("Customer ID"), required=False) + hbci_tokentype = forms.CharField(label=_("Token type"), initial='pintan') + hbci_tokenname = forms.CharField(label=_("Token name"), required=False) + hbci_server = forms.URLField(label=_("Server URL")) + hbci_version = forms.IntegerField(label=_("HBCI version"), required=False, initial=220) + pin = forms.CharField(label=_("PIN"), widget=forms.PasswordInput) + + class ImportView(EventPermissionRequiredMixin, TemplateView): template_name = 'pretixplugins/banktransfer/import_form.html' permission = 'can_change_orders' def post(self, *args, **kwargs): + if 'hbci_server' in self.request.POST: + return self.process_hbci() + if ('file' in self.request.FILES and 'csv' in self.request.FILES.get('file').name.lower()) \ or 'amount' in self.request.POST: # Process CSV @@ -59,9 +80,44 @@ class ImportView(EventPermissionRequiredMixin, TemplateView): 'contact support for help.')) return self.redirect_back() + @cached_property + def settings(self): + return SettingsSandbox('payment', 'banktransfer', self.request.event) + + def process_hbci(self): + form = HbciForm(data=self.request.POST if self.request.method == "POST" else None, + initial=self.settings) + if form.is_valid(): + for key, value in form.cleaned_data.items(): + if key.startswith('hbci_'): + self.settings.set(key, value) + data, log = hbci.hbci_transactions(self.request.event, form.cleaned_data) + if data: + return self.confirm_view(data) + return render(self.request, 'pretixplugins/banktransfer/hbci_log.html', { + 'log': log + }) + else: + return self.get(*self.args, **self.kwargs) + def process_mt940(self): return self.confirm_view(mt940import.parse(self.request.FILES.get('file'))) + @cached_property + def hbci_form(self): + return HbciForm(data=self.request.POST if self.request.method == "POST" else None, + initial=self.settings) + + def get_context_data(self, **kwargs): + ctx = super().get_context_data(**kwargs) + if sys.version_info[:2] >= (3, 3): + ctx['hbci_available'] = shutil.which('aqbanking-cli') and shutil.which('aqhbci-tool4') + if ctx['hbci_available']: + ctx['hbci_form'] = self.hbci_form + else: + ctx['hbci_available'] = False + return ctx + def process_csv_file(self): try: data = csvimport.get_rows_from_file(self.request.FILES['file']) @@ -89,10 +145,12 @@ class ImportView(EventPermissionRequiredMixin, TemplateView): def process_csv_hint(self): data = [] for i in range(int(self.request.POST.get('rows'))): - data.append([ - self.request.POST.get('col[%d][%d]' % (i, j)) - for j in range(int(self.request.POST.get('cols'))) - ]) + data.append( + [ + self.request.POST.get('col[%d][%d]' % (i, j)) + for j in range(int(self.request.POST.get('cols'))) + ] + ) if 'reference' not in self.request.POST: messages.error(self.request, _('You need to select the column containing the payment reference.')) return self.assign_view(data) @@ -174,3 +232,7 @@ class ImportView(EventPermissionRequiredMixin, TemplateView): row['class'] = 'warning' row['message'] = _('Order has been refunded') return data + + @method_decorator(sensitive_post_parameters) + def dispatch(self, *args, **kwargs): + return super().dispatch(*args, **kwargs) diff --git a/src/requirements/banktransfer.txt b/src/requirements/banktransfer.txt index 9bbf199d32..658120c0d3 100644 --- a/src/requirements/banktransfer.txt +++ b/src/requirements/banktransfer.txt @@ -1,2 +1,3 @@ chardet>=2.3,<3 +defusedxml From 4f28c01692aaa1f3128f1caa804da337bc9df0fa Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Thu, 20 Aug 2015 22:26:32 +0200 Subject: [PATCH 2/3] Fixed #94 -- Improved UI for bankdata import --- src/pretix/plugins/banktransfer/views.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/pretix/plugins/banktransfer/views.py b/src/pretix/plugins/banktransfer/views.py index 3642ca29a0..ac446cb23a 100644 --- a/src/pretix/plugins/banktransfer/views.py +++ b/src/pretix/plugins/banktransfer/views.py @@ -199,7 +199,7 @@ class ImportView(EventPermissionRequiredMixin, TemplateView): row['ok'] = False match = pattern.search(row['reference'].upper()) if not match: - row['class'] = '' + row['class'] = 'warning' row['message'] = _('No order code detected') continue @@ -208,24 +208,23 @@ class ImportView(EventPermissionRequiredMixin, TemplateView): order = Order.objects.current.get(event=self.request.event, code=code) except Order.DoesNotExist: - row['class'] = 'error' + row['class'] = 'danger' row['message'] = _('Unknown order code detected') else: row['order'] = order if order.status == Order.STATUS_PENDING: amount = Decimal(amount_pattern.sub("", row['amount'].replace(",", "."))) if amount != order.total: - row['class'] = 'error' + row['class'] = 'danger' row['message'] = _('Found wrong amount. Expected: %s' % str(order.total)) else: row['class'] = 'success' row['message'] = _('Valid payment') row['ok'] = True elif order.status == Order.STATUS_CANCELLED: - row['class'] = 'error' + row['class'] = 'danger' row['message'] = _('Order has been cancelled') elif order.status == Order.STATUS_PAID: - # TODO: Do a plausibility check to tell duplicate payments from overlapping import files row['class'] = '' row['message'] = _('Order already has been paid') elif order.status == Order.STATUS_REFUNDED: From aed73b0c61450d640d93fa813945c3dd7c4be2ea Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Thu, 20 Aug 2015 22:37:38 +0200 Subject: [PATCH 3/3] Changed plugin URL config to create better error messages, fixed an import error --- src/pretix/plugins/banktransfer/views.py | 4 +--- src/pretix/urls.py | 6 ++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/pretix/plugins/banktransfer/views.py b/src/pretix/plugins/banktransfer/views.py index ac446cb23a..6f6390c3ec 100644 --- a/src/pretix/plugins/banktransfer/views.py +++ b/src/pretix/plugins/banktransfer/views.py @@ -9,12 +9,10 @@ from decimal import Decimal from django import forms from django.contrib import messages from django.shortcuts import redirect, render -from django.utils.decorators import method_decorator +from django.utils.functional import cached_property from django.utils.timezone import now from django.utils.translation import ugettext_lazy as _ -from django.views.decorators.debug import sensitive_post_parameters from django.views.generic import TemplateView -from pip.utils import cached_property from pretix.base.models import Order, Quota from pretix.base.services.orders import mark_order_paid diff --git a/src/pretix/urls.py b/src/pretix/urls.py index 896e78d1cf..ebda5b5a63 100644 --- a/src/pretix/urls.py +++ b/src/pretix/urls.py @@ -1,4 +1,4 @@ -import importlib +import importlib.util from django.apps import apps from django.conf import settings @@ -22,13 +22,11 @@ if settings.DEBUG: pluginpatterns = [] for app in apps.get_app_configs(): if hasattr(app, 'PretixPluginMeta'): - try: + if importlib.util.find_spec(app.name + '.urls'): urlmod = importlib.import_module(app.name + '.urls') pluginpatterns.append( url(r'', include(urlmod, namespace=app.label)) ) - except ImportError: - pass urlpatterns.append( url(r'', include(pluginpatterns, namespace='plugins')) )