Improvements for bank transfer importing (#1762)

Co-authored-by: Raphael Michel <michel@rami.io>
Co-authored-by: Raphael Michel <mail@raphaelmichel.de>
This commit is contained in:
Felix Rindt
2020-10-22 11:00:36 +02:00
committed by GitHub
parent 9e4dc344a4
commit a62c7939ae
26 changed files with 1204 additions and 139 deletions

View File

@@ -717,7 +717,7 @@ class BasePaymentProvider:
The default implementation returns an empty string.
:param order: The order object
:param refund: The refund object
"""
return ''

View File

@@ -2,6 +2,8 @@ import csv
import io
import re
from django.utils.text import Truncator
class HintMismatchError(Exception):
pass
@@ -28,6 +30,11 @@ def parse(data, hint):
resrow['amount'] = re.sub('[^0-9,+.-]', '', resrow['amount'])
if hint.get('date') is not None:
resrow['date'] = row[int(hint.get('date'))].strip()
if hint.get('iban') is not None:
resrow['iban'] = Truncator(row[int(hint.get('iban'))].strip()).chars(200)
if hint.get('bic') is not None:
resrow['bic'] = Truncator(row[int(hint.get('bic'))].strip()).chars(200)
if len(resrow['amount']) == 0 or 'amount' not in resrow or resrow.get('date') == '':
# This is probably a headline or something other special.
continue
@@ -88,5 +95,7 @@ def new_hint(data):
'reference': data.getlist('reference') if 'reference' in data else None,
'date': int(data.get('date')) if 'date' in data else None,
'amount': int(data.get('amount')) if 'amount' in data else None,
'cols': int(data.get('cols')) if 'cols' in data else None
'cols': int(data.get('cols')) if 'cols' in data else None,
'iban': int(data.get('iban')) if 'iban' in data else None,
'bic': int(data.get('bic')) if 'bic' in data else None,
}

View File

@@ -0,0 +1,30 @@
# Generated by Django 3.0.9 on 2020-09-01 14:19
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('banktransfer', '0005_auto_20181023_2209'),
]
operations = [
migrations.AddField(
model_name='banktransaction',
name='bic',
field=models.CharField(default='', max_length=250),
preserve_default=False,
),
migrations.AddField(
model_name='banktransaction',
name='date_parsed',
field=models.DateField(null=True),
),
migrations.AddField(
model_name='banktransaction',
name='iban',
field=models.CharField(default='', max_length=250),
preserve_default=False,
),
]

View File

@@ -0,0 +1,27 @@
# Generated by Django 3.0.9 on 2020-09-09 15:43
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('pretixbase', '0162_remove_seat_name'),
('banktransfer', '0006_auto_20200901_1419'),
]
operations = [
migrations.CreateModel(
name='RefundExport',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False)),
('datetime', models.DateTimeField(auto_now_add=True)),
('testmode', models.BooleanField(default=False)),
('rows', models.TextField(default='[]')),
('event', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='banktransfer_refund_exports', to='pretixbase.Event')),
('organizer', models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='banktransfer_refund_exports', to='pretixbase.Organizer')),
('downloaded', models.BooleanField(default=False)),
],
),
]

View File

@@ -1,7 +1,10 @@
import hashlib
import json
import re
from decimal import Decimal
from django.db import models
from django.utils.functional import cached_property
class BankImportJob(models.Model):
@@ -61,6 +64,9 @@ class BankTransaction(models.Model):
reference = models.TextField(blank=True)
amount = models.DecimalField(max_digits=10, decimal_places=2)
date = models.CharField(max_length=50)
date_parsed = models.DateField(null=True)
iban = models.CharField(max_length=250, blank=True)
bic = models.CharField(max_length=250, blank=True)
order = models.ForeignKey('pretixbase.Order', null=True, blank=True, on_delete=models.CASCADE)
comment = models.TextField(blank=True)
@@ -80,3 +86,37 @@ class BankTransaction(models.Model):
class Meta:
unique_together = ('event', 'organizer', 'checksum')
ordering = ('date', 'id')
class RefundExport(models.Model):
event = models.ForeignKey('pretixbase.Event', related_name='banktransfer_refund_exports', on_delete=models.CASCADE, null=True, blank=True)
organizer = models.ForeignKey('pretixbase.Organizer', related_name='banktransfer_refund_exports', on_delete=models.PROTECT, null=True, blank=True)
datetime = models.DateTimeField(auto_now_add=True)
testmode = models.BooleanField(default=False)
rows = models.TextField(default="[]")
downloaded = models.BooleanField(default=False)
@cached_property
def entity_slug(self):
if self.organizer:
return self.organizer.slug
else:
return self.event.slug
@cached_property
def currency(self):
if self.event:
return self.event.currency
return self.organizer.events.first().currency
@property
def rows_data(self):
return json.loads(self.rows)
@property
def sum(self):
return sum(Decimal(row["amount"]) for row in self.rows_data)
@property
def cnt(self):
return len(self.rows_data)

View File

@@ -158,8 +158,9 @@ def parse(file):
result.append({
'amount': str(round_decimal(t.data['amount'].amount)),
'reference': reference + (' EREF: {}'.format(eref) if eref else ''),
'payer': (payer.get('name', '') + ' - ' + payer.get('iban', '')).strip(),
'date': t.data['date'].isoformat()
'payer': payer['name'].strip(),
'date': t.data['date'].isoformat(),
**{k: payer[k].strip() for k in ("iban", "bic") if payer.get(k)}
})
else:
result.append({

View File

@@ -11,8 +11,9 @@ from i18nfield.fields import I18nFormField, I18nTextarea
from i18nfield.forms import I18nTextInput
from i18nfield.strings import LazyI18nString
from localflavor.generic.forms import BICFormField, IBANFormField
from localflavor.generic.validators import BICValidator, IBANValidator
from pretix.base.models import OrderPayment
from pretix.base.models import OrderPayment, OrderRefund
from pretix.base.payment import BasePaymentProvider
@@ -144,9 +145,7 @@ class BankTransfer(BasePaymentProvider):
def settings_form_clean(self, cleaned_data):
if cleaned_data.get('payment_banktransfer_bank_details_type') == 'sepa':
for f in (
'bank_details_sepa_name', 'bank_details_sepa_bank', 'bank_details_sepa_bic',
'bank_details_sepa_iban'):
for f in ('bank_details_sepa_name', 'bank_details_sepa_bank', 'bank_details_sepa_bic', 'bank_details_sepa_iban'):
if not cleaned_data.get('payment_banktransfer_%s' % f):
raise ValidationError(
{'payment_banktransfer_%s' % f: _('Please fill out your bank account details.')})
@@ -213,10 +212,17 @@ class BankTransfer(BasePaymentProvider):
return template.render(ctx)
def payment_control_render(self, request: HttpRequest, payment: OrderPayment) -> str:
warning = None
if not self.payment_refund_supported(payment):
warning = _("Invalid IBAN/BIC")
return self._render_control_info(request, payment.order, payment.info_data, warning=warning)
def _render_control_info(self, request, order, info_data, **extra_context):
template = get_template('pretixplugins/banktransfer/control.html')
ctx = {'request': request, 'event': self.event,
'code': self._code(payment.order),
'payment_info': payment.info_data, 'order': payment.order}
'code': self._code(order),
'payment_info': info_data, 'order': order,
**extra_context}
return template.render(ctx)
def _code(self, order):
@@ -234,3 +240,39 @@ class BankTransfer(BasePaymentProvider):
d['_shredded'] = True
obj.info = json.dumps(d)
obj.save(update_fields=['info'])
@staticmethod
def norm(s):
return s.strip().upper().replace(" ", "")
def payment_refund_supported(self, payment: OrderPayment) -> bool:
if not all(payment.info_data.get(key) for key in ("payer", "iban", "bic")):
return False
try:
IBANValidator()(self.norm(payment.info_data['iban']))
BICValidator()(self.norm(payment.info_data['bic']))
except ValidationError:
return False
else:
return True
def payment_partial_refund_supported(self, payment: OrderPayment) -> bool:
return self.payment_refund_supported(payment)
def execute_refund(self, refund: OrderRefund):
"""
We just keep a created refund object. It will be marked as done using the control view
for bank transfer refunds.
"""
if refund.payment is None:
raise ValueError(_("Can only create a bank transfer refund from an existing payment."))
refund.info_data = {
'payer': refund.payment.info_data['payer'],
'iban': self.norm(refund.payment.info_data['iban']),
'bic': self.norm(refund.payment.info_data['bic']),
}
refund.save(update_fields=["info"])
def refund_control_render(self, request: HttpRequest, refund: OrderRefund) -> str:
return self._render_control_info(request, refund.order, refund.info_data)

View File

@@ -0,0 +1,68 @@
import codecs
import datetime
import io
from decimal import Decimal
from defusedcsv import csv
from django.templatetags.l10n import localize
from django.utils.translation import gettext_lazy as _
from pretix.plugins.banktransfer.models import RefundExport
def _get_filename(refund_export):
return 'bank_transfer_refunds-{}_{}-{}'.format(refund_export.entity_slug, refund_export.datetime.strftime("%Y-%m-%d"), refund_export.id)
def get_refund_export_csv(refund_export: RefundExport):
byte_data = io.BytesIO()
StreamWriter = codecs.getwriter('utf-8')
output = StreamWriter(byte_data)
writer = csv.writer(output)
writer.writerow([_("Payer"), "IBAN", "BIC", _("Amount"), _("Currency"), _("Code")])
for row in refund_export.rows_data:
writer.writerow([
row['payer'],
row['iban'],
row['bic'],
localize(Decimal(row['amount'])),
refund_export.currency,
row['id'],
])
filename = _get_filename(refund_export) + ".csv"
byte_data.seek(0)
return filename, 'text/csv', byte_data
from sepaxml import SepaTransfer
def build_sepa_xml(refund_export: RefundExport, account_holder, iban, bic):
if refund_export.currency != "EUR":
raise ValueError("Cannot create SEPA export for currency other than EUR.")
config = {
"name": account_holder,
"IBAN": iban,
"BIC": bic,
"batch": True,
"currency": refund_export.currency,
}
sepa = SepaTransfer(config, clean=True)
for row in refund_export.rows_data:
payment = {
"name": row['payer'],
"IBAN": row["iban"],
"BIC": row["bic"],
"amount": int(Decimal(row['amount']) * 100), # in euro-cents
"execution_date": datetime.date.today(),
"description": f"{_('Refund')} {refund_export.entity_slug} {row['id']}",
}
sepa.add_payment(payment)
data = sepa.export(validate=True)
filename = _get_filename(refund_export) + ".xml"
return filename, 'application/xml', io.BytesIO(data)

View File

@@ -21,14 +21,31 @@ def control_nav_import(sender, request=None, **kwargs):
return []
return [
{
'label': _('Import bank data'),
'label': _("Bank transfer"),
'url': reverse('plugins:banktransfer:import', kwargs={
'event': request.event.slug,
'organizer': request.event.organizer.slug,
}),
'active': (url.namespace == 'plugins:banktransfer' and url.url_name == 'import'),
'icon': 'upload',
}
'icon': 'university',
'children': [
{
'label': _('Import bank data'),
'url': reverse('plugins:banktransfer:import', kwargs={
'event': request.event.slug,
'organizer': request.event.organizer.slug,
}),
'active': (url.namespace == 'plugins:banktransfer' and url.url_name == 'import'),
},
{
'label': _('Export refunds'),
'url': reverse('plugins:banktransfer:refunds.list', kwargs={
'event': request.event.slug,
'organizer': request.event.organizer.slug,
}),
'active': (url.namespace == 'plugins:banktransfer' and url.url_name.startswith("refunds")),
},
]
},
]
@@ -41,12 +58,28 @@ def control_nav_orga_import(sender, request=None, **kwargs):
return []
return [
{
'label': _('Import bank data'),
'label': _("Bank transfer"),
'url': reverse('plugins:banktransfer:import', kwargs={
'organizer': request.organizer.slug,
}),
'active': (url.namespace == 'plugins:banktransfer' and url.url_name == 'import'),
'icon': 'upload',
'icon': 'university',
'children': [
{
'label': _('Import bank data'),
'url': reverse('plugins:banktransfer:import', kwargs={
'organizer': request.organizer.slug,
}),
'active': (url.namespace == 'plugins:banktransfer' and url.url_name == 'import'),
'icon': 'upload',
},
{
'label': _('Export refunds'),
'url': reverse('plugins:banktransfer:refunds.list', kwargs={
'organizer': request.organizer.slug,
}),
'active': (url.namespace == 'plugins:banktransfer' and url.url_name.startswith("refunds")),
},
]
}
]

View File

@@ -104,6 +104,7 @@ var bankimport_transactionlist = {
var text = $box.find("textarea").val();
$box.find("input, textarea, button").prop("disabled", true);
bankimport_transactionlist._action(id, "comment:" + text, function () {
$("tr[data-id=" + id + "] button").prop("disabled", false);
});
});
$btn2.click(function () {

View File

@@ -2,6 +2,7 @@ import logging
import re
from decimal import Decimal
import dateutil.parser
from celery.exceptions import MaxRetriesExceededError
from django.conf import settings
from django.db import transaction
@@ -41,9 +42,9 @@ def notify_incomplete_payment(o: Order):
def cancel_old_payments(order):
for p in order.payments.filter(
state__in=(OrderPayment.PAYMENT_STATE_PENDING,
OrderPayment.PAYMENT_STATE_CREATED),
provider='banktransfer',
state__in=(OrderPayment.PAYMENT_STATE_PENDING,
OrderPayment.PAYMENT_STATE_CREATED),
provider='banktransfer',
):
try:
with transaction.atomic():
@@ -64,8 +65,8 @@ def cancel_old_payments(order):
@transaction.atomic
def _handle_transaction(trans: BankTransaction, code: str, event: Event=None, organizer: Organizer=None,
slug: str=None):
def _handle_transaction(trans: BankTransaction, code: str, event: Event = None, organizer: Organizer = None,
slug: str = None):
if event:
try:
trans.order = event.orders.get(code=code)
@@ -117,8 +118,10 @@ def _handle_transaction(trans: BankTransaction, code: str, event: Event=None, or
p.info_data = {
'reference': trans.reference,
'date': trans.date,
'date': trans.date_parsed.isoformat() if trans.date_parsed else trans.date,
'payer': trans.payer,
'iban': trans.iban,
'bic': trans.bic,
'trans_id': trans.pk
}
@@ -150,7 +153,18 @@ def _handle_transaction(trans: BankTransaction, code: str, event: Event=None, or
trans.save()
def _get_unknown_transactions(job: BankImportJob, data: list, event: Event=None, organizer: Organizer=None):
def parse_date(date_str):
try:
return dateutil.parser.parse(
date_str,
dayfirst="." in date_str,
).date()
except (ValueError, OverflowError):
pass
return None
def _get_unknown_transactions(job: BankImportJob, data: list, event: Event = None, organizer: Organizer = None):
amount_pattern = re.compile("[^0-9.-]")
known_checksums = set(t['checksum'] for t in BankTransaction.objects.filter(
Q(event=event) if event else Q(organizer=organizer)
@@ -176,8 +190,11 @@ def _get_unknown_transactions(job: BankImportJob, data: list, event: Event=None,
trans = BankTransaction(event=event, organizer=organizer, import_job=job,
payer=row.get('payer', ''),
reference=row['reference'],
amount=amount,
date=row['date'])
amount=amount, date=row['date'],
iban=row.get('iban', ''), bic=row.get('bic', ''))
trans.date_parsed = parse_date(trans.date)
trans.checksum = trans.calculate_checksum()
if trans.checksum not in known_checksums:
trans.state = BankTransaction.STATE_UNCHECKED

View File

@@ -4,10 +4,24 @@
<dl class="dl-horizontal">
<dt>{% trans "Payer" %}</dt>
<dd>{{ payment_info.payer }}</dd>
<dt>{% trans "Payment date" %}</dt>
<dd>{{ payment_info.date }}</dd>
<dt>{% trans "Reference" %}</dt>
<dd>{{ payment_info.reference }}</dd>
{% if payment_info.iban %}
<dt>{% trans "Account" %}
<dd>
{{ payment_info.iban }} {{ payment_info.bic }}
{% if warning %}
<span class="fa fa-exclamation-triangle text-danger" data-toggle="tooltip" title=""
data-original-title="{{ warning }}"></span>
{% endif %}
</dd>
{% endif %}
{% if payment_info.date %}
<dt>{% trans "Payment date" %}</dt>
<dd>{{ payment_info.date }}</dd>
{% endif %}
{% if payment_info.reference %}
<dt>{% trans "Reference" %}</dt>
<dd>{{ payment_info.reference }}</dd>
{% endif %}
</dl>
{% else %}
<dl class="dl-horizontal">

View File

@@ -13,62 +13,94 @@
<div class="flipped-scroll-wrapper clearfix">
<table class="table table-condensed flipped-scroll-inner">
<thead>
<tr>
<th>{% trans "Date" %}</th>
{% for col in rows.0 %}
<th>
<input type="radio" name="date" value="{{ forloop.counter0 }}" />
</th>
{% endfor %}
</tr>
<tr>
<th>{% trans "Amount" %}</th>
{% for col in rows.0 %}
<th>
<input type="radio" name="amount" value="{{ forloop.counter0 }}" required="required" />
</th>
{% endfor %}
</tr>
<tr>
<th>{% trans "Reference" %}</th>
{% for col in rows.0 %}
<th>
<input type="checkbox" name="reference" value="{{ forloop.counter0 }}" />
</th>
{% endfor %}
</tr>
<tr>
<th>{% trans "Payer" %}</th>
{% for col in rows.0 %}
<th>
<input type="checkbox" name="payer" value="{{ forloop.counter0 }}" />
</th>
{% endfor %}
</tr>
<tr>
<th>{% trans "Date" %}</th>
{% for col in rows.0 %}
<th>
<input type="radio" name="date" value="{{ forloop.counter0 }}"/>
</th>
{% endfor %}
</tr>
<tr>
<th>{% trans "Amount" %}</th>
{% for col in rows.0 %}
<th>
<input type="radio" name="amount" value="{{ forloop.counter0 }}" required="required"/>
</th>
{% endfor %}
</tr>
<tr>
<th>{% trans "Reference" %}</th>
{% for col in rows.0 %}
<th>
<input type="checkbox" name="reference" value="{{ forloop.counter0 }}"/>
</th>
{% endfor %}
</tr>
<tr>
<th>{% trans "Payer" %}</th>
{% for col in rows.0 %}
<th>
<input type="checkbox" name="payer" value="{{ forloop.counter0 }}"/>
</th>
{% endfor %}
</tr>
<tr>
<th>
{% trans "IBAN" %}
<label for="id_iban_clear">
<span class="btn btn-default btn-sm fa fa-close"></span>
</label>
<span class="sr-only">
<input id="id_iban_clear" type="radio" name="iban">
</span>
</th>
{% for col in rows.0 %}
<th>
<input type="radio" name="iban" value="{{ forloop.counter0 }}"/>
</th>
{% endfor %}
</tr>
<tr>
<th>
{% trans "BIC" %}
<label for="id_bic_clear">
<span class="btn btn-default btn-sm fa fa-close"></span>
</label>
<span class="sr-only">
<input id="id_bic_clear" type="radio" name="bic">
</span>
</th>
{% for col in rows.0 %}
<th>
<input type="radio" name="bic" value="{{ forloop.counter0 }}"/>
</th>
{% endfor %}
</tr>
</thead>
<tbody>
{% for row in rows|slice:":100" %}
{% with forloop.counter0 as rowid %}
<tr>
<td></td>
{% for col in row %}
<td>{{ col }}</td>
{% endfor %}
</tr>
{% endwith %}
{% endfor %}
{% if rows|length > 100 %}
{% for row in rows|slice:":100" %}
{% with forloop.counter0 as rowid %}
<tr>
<td colspan="{{ rows.0|length|add:1 }}" class="text-center">
<em>{% trans "More data was uploaded but is not shown here. It will still be processed" %}</em>
</td>
<td></td>
{% for col in row %}
<td>{{ col }}</td>
{% endfor %}
</tr>
{% endif %}
{% endwith %}
{% endfor %}
{% if rows|length > 100 %}
<tr>
<td colspan="{{ rows.0|length|add:1 }}" class="text-center">
<em>{% trans "More data was uploaded but is not shown here. It will still be processed" %}</em>
</td>
</tr>
{% endif %}
</tbody>
</table>
</div>
<input type="hidden" name="cols" value="{{ rows.0|length }}" />
<input type="hidden" name="rows" value="{{ rows|length }}" />
<input type="hidden" name="cols" value="{{ rows.0|length }}"/>
<input type="hidden" name="rows" value="{{ rows|length }}"/>
<textarea class="helper-display-none" name="data">
{{ json|escape }}
</textarea>

View File

@@ -23,7 +23,8 @@
Currently, this feature supports <code>.csv</code> files and files in the MT940 format.
{% endblocktrans %}</p>
{% if job_running %}
<div class="alert alert-info" data-job-waiting data-job-waiting-url="{% url "plugins:banktransfer:import.job" event=request.event.slug organizer=request.event.organizer.slug job=job_running.pk %}?ajax=1">
<div class="alert alert-info" data-job-waiting
data-job-waiting-url="{% url "plugins:banktransfer:import.job" event=request.event.slug organizer=request.event.organizer.slug job=job_running.pk %}?ajax=1">
<span class="fa fa-cog fa-spin"></span>
{% trans "An import is currently being processed, please try again in a few minutes." %}
</div>
@@ -31,7 +32,8 @@
<form action="" method="post" enctype="multipart/form-data" class="form-inline">
{% csrf_token %}
<div class="form-group">
<label for="file">{% trans "Import file" %}: </label> <input id="file" type="file" name="file"/>
<label for="file">{% trans "Import file" %}: </label> <input id="file" type="file"
name="file"/>
</div>
<div class="clearfix"></div>
<button class="btn btn-primary pull-right flip" type="submit">
@@ -42,7 +44,7 @@
</div>
</div>
{% endif %}
{% if transactions_unhandled|length > 0 or request.GET.search %}
{% if transactions_unhandled|length > 0 or filter_form.is_valid %}
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">{% trans "Unresolved transactions" %}</h3>
@@ -55,25 +57,42 @@
unmatched transactions imported directly for this event.
{% endblocktrans %}
<a href="{% url "plugins:banktransfer:import" organizer=request.organizer.slug %}"
class="btn btn-default btn-xs">{% trans "Go to organizer-level import" %}</a>
class="btn btn-default btn-xs">{% trans "Go to organizer-level import" %}</a>
</p>
{% endif %}
<p>
<form class="form-inline helper-display-inline" action="" method="get">
<input type="text" name="search" class="form-control" placeholder="{% trans "Search" %}"
value="{{ request.GET.search }}">
<button class="btn btn-primary" type="submit">{% trans "Filter" %}</button>
</form>
{% if not request.GET.search %}
<form action="" method="post" class="helper-display-inline pull-right flip">
{% csrf_token %}
<button class="btn btn-danger" type="submit" name="discard" value="all">
<span class="fa fa-trash"></span>
{% trans "Discard all" %}
</button>
<div class="row">
<form class="form-inline helper-display-inline" action="" method="get">
<div class="col-md-3">
{% trans "Amount from" %}
{{ filter_form.amount_min }}
{% trans "up to" %}
{{ filter_form.amount_max }}
</div>
<div class="col-md-3">
{% trans "Date from" %}
{{ filter_form.date_min }}
{% trans "up to" %}
{{ filter_form.date_max }}
</div>
<div class="col-md-4">
{{ filter_form.search_text }}
<button class="btn btn-primary" type="submit">{% trans "Filter" %}</button>
<a class="btn btn-default" href="{{ request.path }}">{% trans "Clear" %}</a>
</div>
</form>
{% endif %}
</p>
<div class="col-md-2">
{% if not filter_form.is_valid %}
<form action="" method="post" class="helper-display-inline pull-right flip">
{% csrf_token %}
<button class="btn btn-danger" type="submit" name="discard" value="all">
<span class="fa fa-trash"></span>
{% trans "Discard all" %}
</button>
</form>
{% endif %}
</div>
</div>
{% if transactions_unhandled|length > 0 %}
{% include "pretixcontrol/pagination.html" %}

View File

@@ -0,0 +1,113 @@
{% extends basetpl %}
{% load i18n %}
{% load money %}
{% load static %}
{% block title %}{% trans "Export bank transfer refunds" %}{% endblock %}
{% block content %}
<h1>{% trans "Export bank transfer refunds" %}</h1>
<p>
{% blocktrans trimmed %}
<strong>{{ num_new }}</strong> Bank transfer refunds have been placed and are not yet part of an export.
{% endblocktrans %}
</p>
{% if request.event.testmode %}
<div class="alert alert-warning">
{% trans "In test mode, your exports will only contain test mode orders." %}
</div>
{% elif request.event %}
<div class="alert alert-info">
{% trans "If you want, you can now also create these exports for multiple events combined." %}
<strong>
<a href="{% url "plugins:banktransfer:refunds.list" organizer=request.organizer.slug %}">
{% trans "Go to organizer-level exports" %}
</a>
</strong>
</div>
{% endif %}
{% if num_new > 0 %}
<form action="" method="post">
{% csrf_token %}
<button class="btn btn-primary">
{% trans "Create new export file" %}
</button>
<div class="form-group">
<div class="checkbox">
<label for="id_unite_transactions">
<input type="checkbox" name="unite_transactions" id="id_unite_transactions" checked>
{% trans "Aggregate transactions to the same bank account" %}
</label>
</div>
</div>
<p>
{% blocktrans %}
Beware that refunds will be marked as done once an export is created.
Make sure to download the export and execute the refunds.
{% endblocktrans %}
</p>
</form>
{% endif %}
<h2>{% trans "Exported files" %}</h2>
<div class="table-responsive">
<table class="table table-hover">
<thead>
<tr>
<th>{% trans "Export date" %}</th>
<th>{% trans "Number of orders" %}</th>
<th>{% trans "Total amount" %}</th>
<th></th>
</tr>
</thead>
<tbody>
{% for export in exports %}
<tr>
<td>
{{ export.datetime|date:"SHORT_DATETIME_FORMAT" }}
{% if export.testmode %}
<span class="label label-warning">{% trans "TEST MODE" %}</span>
{% endif %}
</td>
<td>{{ export.cnt }}</td>
<td>
{{ export.sum|default_if_none:0|money:export.currency }}
</td>
<td class="text-right">
{% if not export.downloaded %}
<span class="label label-warning">{% trans "not downloaded" %}</span>
{% endif %}
{% if export.event %}
<a class="btn btn-primary"
href="{% url "plugins:banktransfer:refunds.download" organizer=request.organizer.slug event=export.event.slug id=export.id %}">
<span class="fa fa-download"></span> {% trans "Download CSV" %}
</a>
{% if export.currency == "EUR" %}
<a class="btn btn-secondary"
href="{% url "plugins:banktransfer:refunds.sepa" organizer=request.organizer.slug event=export.event.slug id=export.id %}">
{% trans "SEPA XML" %}
</a>
{% endif %}
{% else %}
<a class="btn btn-primary"
href="{% url "plugins:banktransfer:refunds.download" organizer=export.organizer.slug id=export.id %}">
<span class="fa fa-download"></span> {% trans "Download CSV" %}
</a>
{% if export.currency == "EUR" %}
<a class="btn btn-secondary"
href="{% url "plugins:banktransfer:refunds.sepa" organizer=export.organizer.slug id=export.id %}">
{% trans "SEPA XML" %}
</a>
{% endif %}
{% endif %}
</td>
</tr>
{% empty %}
<tr>
<td colspan="3">
{% trans "No exports have been created yet." %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% endblock %}

View File

@@ -0,0 +1,36 @@
{% extends basetpl %}
{% load bootstrap3 %}
{% load i18n %}
{% load rich_text %}
{% load money %}
{% load static %}
{% block title %}{% trans "Export bank transfer refunds" %}{% endblock %}
{% block content %}
<h1>{% trans "Export SEPA xml" %}</h1>
<p>
{% blocktrans with cnt=export.cnt sum=export.sum|money:export.currency date=export.datetime|date %}
You are trying to download a refund export from {{ date }} with {{ cnt }} order{{ cnt|pluralize }} and a
total of {{ sum }}.
{% endblocktrans %}
</p>
<p>
{% blocktrans %}
Please state from which bank account the refunds should be transferred from.
{% endblocktrans %}
</p>
<form method="post" class="form-horizontal">
{% csrf_token %}
<div class="row">
<div class="col-lg-8 col-md-12">
{% bootstrap_form form layout="control" %}
</div>
</div>
<button class="btn btn-primary"><span class="fa fa-download"></span> {% trans "Download" %}</button>
</form>
{% endblock %}

View File

@@ -22,11 +22,11 @@
{% if trans.order and trans.state == 'invalid' %}
<div class="btn-group" role="group">
<button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="accept"
data-toggle="tooltip" title="{% trans "Accept anyway" %}" data-placement="right">
data-toggle="tooltip" title="{% trans "Accept anyway" %}" data-placement="right">
<span class="fa fa-check"></span>
</button>
<button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="discard"
data-toggle="tooltip" title="{% trans "Discard" %}">
data-toggle="tooltip" title="{% trans "Discard" %}">
<span class="fa fa-trash"></span>
</button>
</div>
@@ -34,22 +34,23 @@
<input type="text" class="form-control" placeholder="{% trans "Order code" %}">
<div class="btn-group" role="group">
<button class="btn btn-default" type="button" name="action_{{ trans.id }}"
value="assign" data-toggle="tooltip" title="{% trans "Assign to order" %}" data-placement="right">
value="assign" data-toggle="tooltip" title="{% trans "Assign to order" %}"
data-placement="right">
<span class="fa fa-check"></span>
</button>
<button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="discard"
data-toggle="tooltip" title="{% trans "Discard" %}">
data-toggle="tooltip" title="{% trans "Discard" %}">
<span class="fa fa-trash"></span>
</button>
</div>
{% elif trans.state == 'error' %}
<div class="btn-group" role="group">
<button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="retry"
data-toggle="tooltip" title="{% trans "Retry" %}" data-placement="right">
data-toggle="tooltip" title="{% trans "Retry" %}" data-placement="right">
<span class="fa fa-refresh"></span>
</button>
<button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="discard"
data-toggle="tooltip" title="{% trans "Discard" %}">
data-toggle="tooltip" title="{% trans "Discard" %}">
<span class="fa fa-trash"></span>
</button>
</div>
@@ -57,23 +58,44 @@
<input type="text" class="form-control" placeholder="{% trans "Order code" %}">
<div class="btn-group" role="group">
<button class="btn btn-default" type="button" name="action_{{ trans.id }}"
value="assign" data-toggle="tooltip" title="{% trans "Assign to order" %}" data-placement="right">
value="assign" data-toggle="tooltip" title="{% trans "Assign to order" %}"
data-placement="right">
<span class="fa fa-check"></span>
</button>
<button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="retry"
data-toggle="tooltip" title="{% trans "Retry" %}" data-placement="right">
data-toggle="tooltip" title="{% trans "Retry" %}" data-placement="right">
<span class="fa fa-refresh"></span>
</button>
<button type="button" class="btn btn-default" name="action_{{ trans.id }}" value="discard"
data-toggle="tooltip" title="{% trans "Discard" %}">
data-toggle="tooltip" title="{% trans "Discard" %}">
<span class="fa fa-trash"></span>
</button>
</div>
{% endif %}
</td>
<td>{{ trans.date }}</td>
<td>
{{ trans.payer }}<br/>
{% if trans.date_parsed is not None %}
{{ trans.date_parsed|date:"SHORT_DATE_FORMAT" }}
{% else %}
{{ trans.date }}
{% endif %}
</td>
<td>
<div class="clearfix">
{% if trans.payer %}
{{ trans.payer }}
<br>
{% endif %}
{% if trans.iban or trans.bic %}
{% if trans.iban %}
<em>{{ trans.iban }}</em>
{% endif %}
{% if trans.bic %}
<em>{{ trans.bic }}</em>
{% endif %}
<br>
{% endif %}
</div>
{{ trans.reference }}
<div class="comment-box" data-plain="{{ trans.comment }}">
<strong>{% trans "Comment:" %}</strong>
@@ -102,7 +124,7 @@
<td>
{% if trans.order %}
<a href="{% url "control:event.order" event=trans.order.event.slug organizer=request.organizer.slug code=trans.order.code %}"
data-toggle="tooltip" title="{{ trans.order.total|money:trans.order.event.currency }}">
data-toggle="tooltip" title="{{ trans.order.total|money:trans.order.event.currency }}">
{% if not request.event %}
{{ trans.order.event.slug|upper }}-{{ trans.order.code }}
{% else %}

View File

@@ -13,6 +13,14 @@ urlpatterns = [
views.OrganizerJobDetailView.as_view(), name='import.job'),
url(r'^control/organizer/(?P<organizer>[^/]+)/banktransfer/action/',
views.OrganizerActionView.as_view(), name='import.action'),
url(r'^control/organizer/(?P<organizer>[^/]+)/banktransfer/refunds/',
views.OrganizerRefundExportListView.as_view(), name='refunds.list'),
url(r'^control/organizer/(?P<organizer>[^/]+)/banktransfer/export/(?P<id>\d+)/$',
views.OrganizerDownloadRefundExportView.as_view(),
name='refunds.download'),
url(r'^control/organizer/(?P<organizer>[^/]+)/banktransfer/sepa-export/(?P<id>\d+)/$',
views.OrganizerSepaXMLExportView.as_view(),
name='refunds.sepa'),
url(r'^control/event/(?P<organizer>[^/]+)/(?P<event>[^/]+)/banktransfer/import/',
views.EventImportView.as_view(),
@@ -21,6 +29,15 @@ urlpatterns = [
views.EventJobDetailView.as_view(), name='import.job'),
url(r'^control/event/(?P<organizer>[^/]+)/(?P<event>[^/]+)/banktransfer/action/',
views.EventActionView.as_view(), name='import.action'),
url(r'^control/event/(?P<organizer>[^/]+)/(?P<event>[^/]+)/banktransfer/refunds/',
views.EventRefundExportListView.as_view(),
name='refunds.list'),
url(r'^control/event/(?P<organizer>[^/]+)/(?P<event>[^/]+)/banktransfer/export/(?P<id>\d+)/$',
views.EventDownloadRefundExportView.as_view(),
name='refunds.download'),
url(r'^control/event/(?P<organizer>[^/]+)/(?P<event>[^/]+)/banktransfer/sepa-export/(?P<id>\d+)/$',
views.EventSepaXMLExportView.as_view(),
name='refunds.sepa'),
]
orga_router.register('bankimportjobs', BankImportJobViewSet)

View File

@@ -1,21 +1,28 @@
import csv
import itertools
import json
import logging
from datetime import timedelta
from decimal import Decimal
from typing import Set
from django import forms
from django.contrib import messages
from django.db.models import Count, Q
from django.db import transaction
from django.db.models import Count, Q, QuerySet
from django.db.models.functions import Concat
from django.http import JsonResponse
from django.http import FileResponse, JsonResponse
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django.utils.functional import cached_property
from django.utils.timezone import now
from django.utils.translation import gettext as _
from django.views.generic import DetailView, ListView, View
from django.views.generic import DetailView, FormView, ListView, View
from django.views.generic.detail import SingleObjectMixin
from localflavor.generic.forms import BICFormField, IBANFormField
from pretix.base.models import Order, OrderPayment, OrderRefund, Quota
from pretix.base.forms.widgets import DatePickerWidget
from pretix.base.models import Event, Order, OrderPayment, OrderRefund, Quota
from pretix.base.services.mail import SendMailException
from pretix.base.settings import SettingsSandbox
from pretix.base.templatetags.money import money_filter
@@ -23,8 +30,15 @@ from pretix.control.permissions import (
EventPermissionRequiredMixin, OrganizerPermissionRequiredMixin,
)
from pretix.control.views.organizer import OrganizerDetailViewMixin
from pretix.helpers.json import CustomJSONEncoder
from pretix.plugins.banktransfer import csvimport, mt940import
from pretix.plugins.banktransfer.models import BankImportJob, BankTransaction
from pretix.plugins.banktransfer.models import (
BankImportJob, BankTransaction, RefundExport,
)
from pretix.plugins.banktransfer.payment import BankTransfer
from pretix.plugins.banktransfer.refund_export import (
build_sepa_xml, get_refund_export_csv,
)
from pretix.plugins.banktransfer.tasks import process_banktransfers
logger = logging.getLogger('pretix.plugins.banktransfer')
@@ -70,6 +84,8 @@ class ActionView(View):
'reference': trans.reference,
'date': trans.date,
'payer': trans.payer,
'iban': trans.iban,
'bic': trans.bic,
'trans_id': trans.pk
})
)
@@ -96,6 +112,8 @@ class ActionView(View):
'reference': trans.reference,
'date': trans.date,
'payer': trans.payer,
'iban': trans.iban,
'bic': trans.bic,
'trans_id': trans.pk
}
try:
@@ -274,6 +292,33 @@ class JobDetailView(DetailView):
return ctx
class BankTransactionFilterForm(forms.Form):
search_text = forms.CharField(required=False, widget=forms.TextInput(attrs={'class': "form-control", "placeholder": _("Search text")}))
amount_min = forms.DecimalField(required=False, localize=True, widget=forms.TextInput(attrs={'class': "form-control", "placeholder": _("min"), "size": 8}))
amount_max = forms.DecimalField(required=False, localize=True, widget=forms.TextInput(attrs={'class': "form-control", "placeholder": _("max"), "size": 8}))
date_min = forms.DateField(required=False, widget=DatePickerWidget(attrs={"size": 8}))
date_max = forms.DateField(required=False, widget=DatePickerWidget(attrs={"size": 8}))
def is_valid(self):
return super().is_valid() and any(value for value in self.cleaned_data.values())
def filter(self, qs):
if not self.is_valid():
raise ValueError(_("Filter form is not valid."))
if self.cleaned_data.get('search_text'):
q = self.cleaned_data['search_text']
qs = qs.filter(Q(payer__icontains=q) | Q(reference__icontains=q) | Q(comment__icontains=q) | Q(iban__icontains=q) | Q(bic__icontains=q))
if self.cleaned_data.get('amount_min') is not None:
qs = qs.filter(amount__gte=self.cleaned_data['amount_min'])
if self.cleaned_data.get("amount_max") is not None:
qs = qs.filter(amount__lte=self.cleaned_data['amount_max'])
if self.cleaned_data.get('date_min') is not None:
qs = qs.filter(ate_parsed__gte=self.cleaned_data['date_min'])
if self.cleaned_data.get('date_max') is not None:
qs = qs.filter(date_parsed__lte=self.cleaned_data['date_max'])
return qs
class ImportView(ListView):
template_name = 'pretixplugins/banktransfer/import_form.html'
permission = 'can_change_orders'
@@ -293,15 +338,12 @@ class ImportView(ListView):
BankTransaction.STATE_INVALID, BankTransaction.STATE_ERROR,
BankTransaction.STATE_DUPLICATE, BankTransaction.STATE_NOMATCH
])
if 'search' in self.request.GET:
q = self.request.GET.get('search')
qs = qs.filter(
Q(payer__icontains=q) | Q(reference__icontains=q) | Q(comment__icontains=q)
).order_by(
'-import_job__created'
)
return qs
filter_form = BankTransactionFilterForm(self.request.GET or None)
if filter_form.is_valid():
qs = filter_form.filter(qs)
return qs.order_by('-import_job__created')
def discard_all(self):
self.get_queryset().update(payer='', reference='', state=BankTransaction.STATE_DISCARDED)
@@ -312,8 +354,7 @@ class ImportView(ListView):
self.discard_all()
return self.redirect_back()
elif ('file' in self.request.FILES and '.csv' in self.request.FILES.get('file').name.lower()) \
or 'amount' in self.request.POST:
elif ('file' in self.request.FILES and '.csv' in self.request.FILES.get('file').name.lower()) or 'amount' in self.request.POST:
# Process CSV
return self.process_csv()
@@ -465,6 +506,7 @@ class ImportView(ListView):
ctx = super().get_context_data()
ctx['job_running'] = self.job_running
ctx['no_more_payments'] = False
ctx['filter_form'] = BankTransactionFilterForm(self.request.GET or None)
if 'event' in self.kwargs:
ctx['basetpl'] = 'pretixplugins/banktransfer/import_base.html'
@@ -543,3 +585,232 @@ class OrganizerActionView(OrganizerBanktransferView, OrganizerPermissionRequired
organizer=self.request.organizer, can_change_orders=True, can_view_orders=True
).values_list('limit_events__id', flat=True)
)
def _row_key_func(row):
return row['iban'], row['bic']
def _unite_transaction_rows(transaction_rows):
united_transactions_rows = []
transaction_rows = sorted(transaction_rows, key=_row_key_func)
for (iban, bic), group in itertools.groupby(transaction_rows, _row_key_func):
rows = list(group)
united_transactions_rows.append({
"iban": iban,
"bic": bic,
"id": ", ".join(sorted(set(r['id'] for r in rows))),
"payer": ", ".join(sorted(set(r['payer'] for r in rows))),
"amount": sum(r['amount'] for r in rows),
})
return united_transactions_rows
class RefundExportListView(ListView):
template_name = 'pretixplugins/banktransfer/refund_export.html'
model = RefundExport
context_object_name = 'exports'
def get_success_url(self):
raise NotImplementedError
def get_unexported(self) -> QuerySet:
raise NotImplementedError()
def get_context_data(self, **kwargs):
ctx = super().get_context_data()
ctx['num_new'] = self.get_unexported().count()
ctx['basetpl'] = "pretixcontrol/event/base.html"
if not hasattr(self.request, 'event'):
ctx['basetpl'] = "pretixcontrol/organizers/base.html"
return ctx
@transaction.atomic()
def post(self, request, *args, **kwargs):
unite_transactions = request.POST.get("unite_transactions", False)
valid_refunds: Set[OrderRefund] = set()
for refund in self.get_unexported().select_related('order', 'order__event'):
if not refund.info_data:
# Should not happen
messages.warning(request,
_("We could not find bank account information for the refund {refund_id}. It was marked as failed.")
.format(refund_id=refund.full_id))
refund.state = OrderRefund.REFUND_STATE_FAILED
refund.save()
continue
else:
valid_refunds.add(refund)
if valid_refunds:
transaction_rows = []
for refund in valid_refunds:
data = refund.info_data
transaction_rows.append({
"amount": refund.amount,
"id": refund.full_id,
**{key: data[key] for key in ("payer", "iban", "bic")}
})
refund.done(user=self.request.user)
if unite_transactions:
transaction_rows = _unite_transaction_rows(transaction_rows)
rows_data = json.dumps(transaction_rows, cls=CustomJSONEncoder)
if hasattr(request, 'event'):
RefundExport.objects.create(event=self.request.event, testmode=self.request.event.testmode, rows=rows_data)
else:
RefundExport.objects.create(organizer=self.request.organizer, testmode=False, rows=rows_data)
else:
messages.warning(request, _('No valid orders have been found.'))
return redirect(self.get_success_url())
class EventRefundExportListView(EventPermissionRequiredMixin, RefundExportListView):
permission = 'can_change_orders'
def get_success_url(self):
return reverse('plugins:banktransfer:refunds.list', kwargs={
'event': self.request.event.slug,
'organizer': self.request.organizer.slug,
})
def get_queryset(self):
return RefundExport.objects.filter(
event=self.request.event
).order_by('-datetime')
def get_unexported(self):
return OrderRefund.objects.filter(
order__event=self.request.event,
provider='banktransfer',
state=OrderRefund.REFUND_STATE_CREATED,
order__testmode=self.request.event.testmode,
)
class OrganizerRefundExportListView(OrganizerPermissionRequiredMixin, RefundExportListView):
permission = 'can_change_orders'
def dispatch(self, request, *args, **kwargs):
if len(request.organizer.events.order_by('currency').values_list('currency', flat=True).distinct()) > 1:
messages.error(request, _('Please perform per-event refund exports as this organizer has events with '
'multiple currencies.'))
return redirect('control:organizer', organizer=request.organizer.slug)
return super().dispatch(request, *args, **kwargs)
def get_success_url(self):
return reverse('plugins:banktransfer:refunds.list', kwargs={
'organizer': self.request.organizer.slug,
})
def get_queryset(self):
return RefundExport.objects.filter(
Q(organizer=self.request.organizer) | Q(event__organizer=self.request.organizer)
).order_by('-datetime')
def get_unexported(self):
return OrderRefund.objects.filter(
order__event__organizer=self.request.organizer,
provider='banktransfer',
state=OrderRefund.REFUND_STATE_CREATED,
order__testmode=False,
)
class DownloadRefundExportView(DetailView):
model = RefundExport
def get(self, request, *args, **kwargs):
self.object: RefundExport = self.get_object()
self.object.downloaded = True
self.object.save(update_fields=["downloaded"])
filename, content_type, data = get_refund_export_csv(self.object)
return FileResponse(data, as_attachment=True, filename=filename, content_type=content_type)
class EventDownloadRefundExportView(EventPermissionRequiredMixin, DownloadRefundExportView):
permission = 'can_change_orders'
def get_object(self, *args, **kwargs):
return get_object_or_404(
RefundExport,
event=self.request.event,
pk=self.kwargs.get('id')
)
class OrganizerDownloadRefundExportView(OrganizerPermissionRequiredMixin, OrganizerDetailViewMixin, DownloadRefundExportView):
permission = 'can_change_orders'
def get_object(self, *args, **kwargs):
return get_object_or_404(
RefundExport,
organizer=self.request.organizer,
pk=self.kwargs.get('id')
)
class SepaXMLExportForm(forms.Form):
account_holder = forms.CharField(label=_("Account holder"))
iban = IBANFormField(label="IBAN")
bic = BICFormField(label="BIC")
def set_initial_from_event(self, event: Event):
banktransfer = event.get_payment_providers(cached=True)[BankTransfer.identifier]
self.initial["account_holder"] = banktransfer.settings.get("bank_details_sepa_name")
self.initial["iban"] = banktransfer.settings.get("bank_details_sepa_iban")
self.initial["bic"] = banktransfer.settings.get("bank_details_sepa_bic")
class SepaXMLExportView(SingleObjectMixin, FormView):
form_class = SepaXMLExportForm
model = RefundExport
template_name = 'pretixplugins/banktransfer/sepa_export.html'
context_object_name = "export"
def setup(self, request, *args, **kwargs):
super().setup(request, *args, **kwargs)
self.object: RefundExport = self.get_object()
def form_valid(self, form):
self.object.downloaded = True
self.object.save(update_fields=["downloaded"])
filename, content_type, data = build_sepa_xml(self.object, **form.cleaned_data)
return FileResponse(data, as_attachment=True, filename=filename, content_type=content_type)
def get_context_data(self, **kwargs):
ctx = super().get_context_data()
ctx['basetpl'] = "pretixcontrol/event/base.html"
if not hasattr(self.request, 'event'):
ctx['basetpl'] = "pretixcontrol/organizers/base.html"
return ctx
class EventSepaXMLExportView(EventPermissionRequiredMixin, SepaXMLExportView):
permission = 'can_change_orders'
def get_object(self, *args, **kwargs):
return get_object_or_404(
RefundExport,
event=self.request.event,
pk=self.kwargs.get('id')
)
def get_form(self, form_class=None):
form = super().get_form(form_class)
form.set_initial_from_event(self.object.event)
return form
class OrganizerSepaXMLExportView(OrganizerPermissionRequiredMixin, OrganizerDetailViewMixin, SepaXMLExportView):
permission = 'can_change_orders'
def get_object(self, *args, **kwargs):
return get_object_or_404(
RefundExport,
organizer=self.request.organizer,
pk=self.kwargs.get('id')
)