Bank transfer: Enable organizer-level features with multiple currencies (#3177)

Co-authored-by: Martin Gross <gross@rami.io>
This commit is contained in:
Raphael Michel
2023-03-27 09:31:41 +02:00
committed by GitHub
parent 27f0ed69d7
commit 2774eb442d
13 changed files with 330 additions and 41 deletions

View File

@@ -26,7 +26,7 @@ from django.db.models import Q
from django.utils.timezone import now
from django_filters.rest_framework import DjangoFilterBackend, FilterSet
from rest_framework import serializers, status, viewsets
from rest_framework.exceptions import PermissionDenied
from rest_framework.exceptions import PermissionDenied, ValidationError
from rest_framework.mixins import CreateModelMixin
from rest_framework.response import Response
@@ -46,7 +46,7 @@ class BankTransactionSerializer(serializers.ModelSerializer):
class Meta:
model = BankTransaction
fields = ('state', 'message', 'checksum', 'payer', 'reference', 'amount', 'date', 'order',
'comment', 'iban', 'bic')
'comment', 'iban', 'bic', 'currency')
class BankImportJobSerializer(serializers.ModelSerializer):
@@ -57,7 +57,7 @@ class BankImportJobSerializer(serializers.ModelSerializer):
class Meta:
model = BankImportJob
fields = ('id', 'event', 'created', 'state', 'transactions')
fields = ('id', 'event', 'created', 'state', 'transactions', 'currency')
def __init__(self, *args, **kwargs):
self.organizer = kwargs.pop('organizer')
@@ -65,6 +65,18 @@ class BankImportJobSerializer(serializers.ModelSerializer):
self.fields['event'].queryset = self.organizer.events.all()
super().__init__(*args, **kwargs)
def validate(self, attrs):
if not attrs.get("event"):
if "currency" not in attrs:
currencies = list(
self.organizer.events.order_by('currency').values_list('currency', flat=True).distinct()
)
if len(currencies) != 1:
raise ValidationError({"currency": ["Currency is ambiguous, please set explicitly."]})
else:
attrs["currency"] = currencies[0]
return attrs
def create(self, validated_data):
trans_data = validated_data.pop('transactions')
job = BankImportJob.objects.create(organizer=self.organizer, **validated_data)

View File

@@ -0,0 +1,81 @@
# Generated by Django 3.2.18 on 2023-03-22 13:10
from django.db import migrations, models
from django.db.models import Q
from django_scopes import scopes_disabled
from pretix.base.models import Event
@scopes_disabled()
def set_currency(apps, schema_editor):
BankTransaction = apps.get_model('banktransfer', 'BankTransaction')
for row in BankTransaction.objects.order_by('organizer', 'event').values('organizer', 'event',
'order__event').distinct():
if row['event'] or row['order__event']:
currency = Event.objects.get(pk=row['event'] or row['order__event']).currency
BankTransaction.objects.filter(
Q(event_id=row['event'] or row['order__event']) | Q(
order__event_id=row['event'] or row['order__event']),
organizer_id=row['organizer'],
).update(
currency=currency
)
else:
currencies = list(
Event.objects.filter(organizer_id=row['organizer']).order_by('currency').values_list('currency',
flat=True).distinct())
if len(currencies) == 1:
BankTransaction.objects.filter(
organizer_id=row['organizer'],
).update(
currency=currencies[0]
)
RefundExport = apps.get_model('banktransfer', 'RefundExport')
for row in RefundExport.objects.order_by('organizer', 'event').values('organizer', 'event').distinct():
if row['event']:
currency = Event.objects.get(pk=row['event']).currency
RefundExport.objects.filter(
event_id=row['event'],
organizer_id=row['organizer'],
).update(
currency=currency
)
else:
currencies = list(
Event.objects.filter(organizer_id=row['organizer']).order_by('currency').values_list('currency', flat=True).distinct()
)
if len(currencies) == 1:
RefundExport.objects.filter(
organizer_id=row['organizer'],
).update(
currency=currencies[0]
)
class Migration(migrations.Migration):
dependencies = [
('banktransfer', '0008_alter_banktransaction_amount'),
]
operations = [
migrations.AddField(
model_name='banktransaction',
name='currency',
field=models.CharField(max_length=10, null=True),
),
migrations.AddField(
model_name='bankimportjob',
name='currency',
field=models.CharField(max_length=10, null=True),
),
migrations.AddField(
model_name='refundexport',
name='currency',
field=models.CharField(max_length=10, null=True),
),
migrations.RunPython(set_currency, migrations.RunPython.noop),
]

View File

@@ -42,6 +42,7 @@ class BankImportJob(models.Model):
event = models.ForeignKey('pretixbase.Event', null=True, on_delete=models.CASCADE)
organizer = models.ForeignKey('pretixbase.Organizer', null=True, on_delete=models.CASCADE)
currency = models.CharField(max_length=10, null=True)
created = models.DateTimeField(auto_now_add=True)
state = models.CharField(max_length=32, choices=STATES, default=STATE_PENDING)
@@ -78,6 +79,7 @@ class BankTransaction(models.Model):
event = models.ForeignKey('pretixbase.Event', null=True, on_delete=models.CASCADE)
organizer = models.ForeignKey('pretixbase.Organizer', null=True, on_delete=models.CASCADE)
import_job = models.ForeignKey('BankImportJob', related_name='transactions', on_delete=models.CASCADE)
currency = models.CharField(max_length=10, null=True)
state = models.CharField(max_length=32, choices=STATES, default=STATE_UNCHECKED)
message = models.TextField()
checksum = models.CharField(max_length=190, db_index=True)
@@ -112,6 +114,7 @@ class BankTransaction(models.Model):
class RefundExport(models.Model):
event = models.ForeignKey('pretixbase.Event', related_name='banktransfer_refund_exports', on_delete=models.CASCADE, null=True, blank=True)
organizer = models.ForeignKey('pretixbase.Organizer', related_name='banktransfer_refund_exports', on_delete=models.PROTECT, null=True, blank=True)
currency = models.CharField(max_length=10, null=True)
datetime = models.DateTimeField(auto_now_add=True)
testmode = models.BooleanField(default=False)
rows = models.TextField(default="[]")
@@ -124,12 +127,6 @@ class RefundExport(models.Model):
else:
return self.event.slug
@cached_property
def currency(self):
if self.event:
return self.event.currency
return self.organizer.events.first().currency
@property
def rows_data(self):
return json.loads(self.rows)

View File

@@ -171,6 +171,12 @@ def _handle_transaction(trans: BankTransaction, matches: tuple, event: Event = N
trans.save()
return
if trans.currency is not None and trans.currency != o.event.currency:
trans.state = BankTransaction.STATE_ERROR
trans.message = gettext_noop('Currencies do not match.')
trans.save()
return
if len(orders) > 1:
# Multi-match! Can we split this automatically?
order_pending_sum = sum(o.pending_sum for o in orders)
@@ -280,7 +286,8 @@ def _get_unknown_transactions(job: BankImportJob, data: list, event: Event = Non
payer=row.get('payer', ''),
reference=row.get('reference', ''),
amount=amount, date=row.get('date', ''),
iban=row.get('iban', ''), bic=row.get('bic', ''))
iban=row.get('iban', ''), bic=row.get('bic', ''),
currency=event.currency if event else job.currency)
trans.date_parsed = parse_date(trans.date)

View File

@@ -99,6 +99,9 @@
</tbody>
</table>
</div>
{% if "currency" in request.POST %}
<input type="hidden" name="currency" value="{{ request.POST.currency }}"/>
{% endif %}
<input type="hidden" name="cols" value="{{ rows.0|length }}"/>
<input type="hidden" name="rows" value="{{ rows|length }}"/>
<textarea class="helper-display-none" name="data">

View File

@@ -31,9 +31,20 @@
{% else %}
<form action="" method="post" enctype="multipart/form-data" class="form-inline">
{% csrf_token %}
{% if currencies|length > 1 %}
<div class="form-group helper-display-block">
<label for="currency">{% trans "Currency" %}: </label>
<select required name="currency" class="form-control">
<option></option>
{% for c in currencies %}
<option value="{{c}}">{{c}}</option>
{% endfor %}
</select>
</div>
{% endif %}
<div class="form-group">
<label for="file">{% trans "Import file" %}: </label> <input id="file" type="file"
name="file"/>
<label for="file">{% trans "Import file" %}: </label>
<input id="file" type="file" name="file" required />
</div>
<div class="clearfix"></div>
<button class="btn btn-primary pull-right flip" type="submit">

View File

@@ -69,7 +69,11 @@
</td>
<td>{{ export.cnt }}</td>
<td>
{% if export.currency %}
{{ export.sum|default_if_none:0|money:export.currency }}
{% else %}
{{ export.sum|default_if_none:0|floatformat:2 }}
{% endif %}
</td>
<td class="text-right">
{% if not export.downloaded %}

View File

@@ -84,7 +84,13 @@
</a>
</div>
</td>
<td>{{ trans.amount|floatformat:2 }}</td>
<td>
{% if trans.currency %}
{{ trans.amount|money:trans.currency }}
{% else %}
{{ trans.amount|floatformat:2 }}
{% endif %}
</td>
<td>
{% if trans.message %}
{% trans trans.message %}

View File

@@ -37,6 +37,7 @@ import csv
import itertools
import json
import logging
from collections import defaultdict
from datetime import timedelta
from decimal import Decimal
from typing import Set
@@ -94,6 +95,11 @@ class ActionView(View):
return self._accept_ignore_amount(trans)
def _accept_ignore_amount(self, trans):
if trans.currency and trans.order and trans.currency != trans.order.event.currency:
return JsonResponse({
'status': 'error',
'message': _('Currencies do not match.')
})
if trans.amount < Decimal('0.00'):
ref = trans.order.refunds.filter(
amount=trans.amount * -1,
@@ -176,6 +182,11 @@ class ActionView(View):
trans.order = self.order_qs().get(code=code.rsplit('-', 1)[1], event__slug__iexact=code.rsplit('-', 1)[0])
else:
trans.order = self.order_qs().get(code=code.rsplit('-', 1)[-1])
if trans.currency and trans.order and trans.currency != trans.order.event.currency:
return JsonResponse({
'status': 'error',
'message': _('Currencies do not match.')
})
except Order.DoesNotExist:
return JsonResponse({
'status': 'error',
@@ -422,6 +433,11 @@ class ImportView(ListView):
messages.error(self.request, _('We were unable to process your input.'))
return self.redirect_back()
def _hint_settings_name(self, currency):
if len(self.currencies) > 1:
return f'banktransfer_csvhint_{currency}'
return 'banktransfer_csvhint'
def process_csv_file(self):
o = getattr(self.request, 'event', self.request.organizer)
try:
@@ -436,8 +452,8 @@ class ImportView(ListView):
messages.error(self.request, _('I\'m sorry, but we detected this file as empty. Please '
'contact support for help.'))
if o.settings.get('banktransfer_csvhint') is not None:
hint = o.settings.get('banktransfer_csvhint', as_type=dict)
if o.settings.get(self._hint_settings_name(self.request.POST.get('currency'))) is not None:
hint = o.settings.get(self._hint_settings_name(self.request.POST.get('currency')), as_type=dict)
try:
parsed, good = csvimport.parse(data, hint)
@@ -467,7 +483,7 @@ class ImportView(ListView):
return self.assign_view(data)
o = getattr(self.request, 'event', self.request.organizer)
try:
o.settings.set('banktransfer_csvhint', hint)
o.settings.set(self._hint_settings_name(self.request.POST.get('currency')), hint)
except Exception as e: # TODO: narrow down
logger.error('Import using stored hint failed: ' + str(e))
pass
@@ -517,6 +533,15 @@ class ImportView(ListView):
kwargs['event'] = self.kwargs['event']
return redirect(reverse('plugins:banktransfer:import', kwargs=kwargs))
@cached_property
def currencies(self):
if hasattr(self.request, 'event'):
return [self.request.event.currency]
else:
return list(
self.request.organizer.events.order_by('currency').values_list('currency', flat=True).distinct()
)
def start_processing(self, parsed):
if self.job_running:
messages.error(self.request,
@@ -525,7 +550,15 @@ class ImportView(ListView):
if 'event' in self.kwargs:
job = BankImportJob.objects.create(event=self.request.event, organizer=self.request.organizer)
else:
job = BankImportJob.objects.create(organizer=self.request.organizer)
if len(self.currencies) != 1:
currency = self.request.POST.get("currency")
if not currency or currency not in self.currencies:
messages.error(self.request,
_('No currency has been selected.'))
return self.redirect_back()
job = BankImportJob.objects.create(organizer=self.request.organizer, currency=currency)
else:
job = BankImportJob.objects.create(organizer=self.request.organizer, currency=self.currencies[0])
process_banktransfers.apply_async(kwargs={
'job': job.pk,
'data': parsed
@@ -544,6 +577,9 @@ class ImportView(ListView):
ctx['no_more_payments'] = False
ctx['filter_form'] = BankTransactionFilterForm(self.request.GET or None)
if not hasattr(self.request, 'event'):
ctx['currencies'] = self.currencies
if 'event' in self.kwargs:
ctx['basetpl'] = 'pretixplugins/banktransfer/import_base.html'
if not self.request.event.has_subevents and self.request.event.settings.get('payment_term_last'):
@@ -577,10 +613,6 @@ class ImportView(ListView):
class OrganizerBanktransferView:
def dispatch(self, request, *args, **kwargs):
if len(request.organizer.events.order_by('currency').values_list('currency', flat=True).distinct()) > 1:
messages.error(request, _('Please perform per-event bank imports as this organizer has events with '
'multiple currencies.'))
return redirect('control:organizer', organizer=request.organizer.slug)
return super().dispatch(request, *args, **kwargs)
@@ -679,11 +711,11 @@ class RefundExportListView(ListView):
valid_refunds.add(refund)
if valid_refunds:
transaction_rows = []
transaction_rows = defaultdict(list)
for refund in valid_refunds:
data = refund.info_data
transaction_rows.append({
transaction_rows[refund.order.event.currency].append({
"amount": refund.amount,
"id": refund.full_id,
"comment": refund.comment,
@@ -692,13 +724,15 @@ class RefundExportListView(ListView):
refund.done(user=self.request.user)
if unite_transactions:
transaction_rows = _unite_transaction_rows(transaction_rows)
for currency, rows in transaction_rows.items():
transaction_rows[currency] = _unite_transaction_rows(rows)
rows_data = json.dumps(transaction_rows, cls=CustomJSONEncoder)
if hasattr(request, 'event'):
RefundExport.objects.create(event=self.request.event, testmode=self.request.event.testmode, rows=rows_data)
else:
RefundExport.objects.create(organizer=self.request.organizer, testmode=False, rows=rows_data)
for currency, rows in transaction_rows.items():
rows_data = json.dumps(rows, cls=CustomJSONEncoder)
if hasattr(request, 'event'):
RefundExport.objects.create(event=self.request.event, testmode=self.request.event.testmode, rows=rows_data, currency=currency)
else:
RefundExport.objects.create(organizer=self.request.organizer, testmode=False, rows=rows_data, currency=currency)
else:
messages.warning(request, _('No valid orders have been found.'))
@@ -732,13 +766,6 @@ class EventRefundExportListView(EventPermissionRequiredMixin, RefundExportListVi
class OrganizerRefundExportListView(OrganizerPermissionRequiredMixin, RefundExportListView):
permission = 'can_change_orders'
def dispatch(self, request, *args, **kwargs):
if len(request.organizer.events.order_by('currency').values_list('currency', flat=True).distinct()) > 1:
messages.error(request, _('Please perform per-event refund exports as this organizer has events with '
'multiple currencies.'))
return redirect('control:organizer', organizer=request.organizer.slug)
return super().dispatch(request, *args, **kwargs)
def get_success_url(self):
return reverse('plugins:banktransfer:refunds.list', kwargs={
'organizer': self.request.organizer.slug,

View File

@@ -96,6 +96,21 @@ def test_assign_order(env, client):
assert env[2].status == Order.STATUS_PAID
@pytest.mark.django_db
def test_assign_order_invalid_currency(env, client):
job = BankImportJob.objects.create(event=env[0])
trans = BankTransaction.objects.create(event=env[0], import_job=job, payer='Foo',
state=BankTransaction.STATE_NOMATCH,
amount=23, date='unknown', currency='HUF')
client.login(email='dummy@dummy.dummy', password='dummy')
r = json.loads(client.post('/control/event/{}/{}/banktransfer/action/'.format(env[0].organizer.slug, env[0].slug), {
'action_{}'.format(trans.pk): 'assign:FOO'
}).content.decode('utf-8'))
assert r['status'] == 'error'
trans.refresh_from_db()
assert trans.state == BankTransaction.STATE_NOMATCH
@pytest.mark.django_db
def test_assign_order_unknown(env, client):
job = BankImportJob.objects.create(event=env[0])

View File

@@ -79,11 +79,13 @@ RES_JOB = {
'amount': '0.00',
'date': 'unknown',
'state': 'error',
'currency': 'EUR',
'order': None
}
],
'created': '2017-06-27T09:13:35.785251Z',
'state': 'pending'
'state': 'pending',
'currency': 'EUR'
}
@@ -93,10 +95,10 @@ def test_api_list(env, client):
with mock.patch('django.utils.timezone.now') as mock_now:
mock_now.return_value = testtime
job = BankImportJob.objects.create(event=env[0], organizer=env[0].organizer)
job = BankImportJob.objects.create(event=env[0], organizer=env[0].organizer, currency='EUR')
BankTransaction.objects.create(event=env[0], import_job=job, payer='Foo',
state=BankTransaction.STATE_ERROR,
amount=0, date='unknown')
amount=0, date='unknown', currency='EUR')
res = copy.copy(RES_JOB)
res['id'] = job.pk
res['created'] = testtime.isoformat().replace('+00:00', 'Z')
@@ -113,10 +115,10 @@ def test_api_detail(env, client):
with mock.patch('django.utils.timezone.now') as mock_now:
mock_now.return_value = testtime
job = BankImportJob.objects.create(event=env[0], organizer=env[0].organizer)
job = BankImportJob.objects.create(event=env[0], organizer=env[0].organizer, currency='EUR')
BankTransaction.objects.create(event=env[0], import_job=job, payer='Foo',
state=BankTransaction.STATE_ERROR,
amount=0, date='unknown')
amount=0, date='unknown', currency='EUR')
res = copy.copy(RES_JOB)
res['id'] = job.pk
res['created'] = testtime.isoformat().replace('+00:00', 'Z')
@@ -151,6 +153,7 @@ def test_api_create(env, client):
assert rdata['state'] == 'completed'
assert len(rdata['transactions']) == 1
assert rdata['transactions'][0]['checksum']
assert rdata['transactions'][0]['currency'] == 'EUR'
env[2].refresh_from_db()
assert env[2].status == Order.STATUS_PAID
@@ -179,7 +182,79 @@ def test_api_create_with_iban_bic(env, client):
assert rdata['state'] == 'completed'
assert len(rdata['transactions']) == 1
assert rdata['transactions'][0]['checksum']
assert rdata['transactions'][0]['currency'] == 'EUR'
env[2].refresh_from_db()
assert env[2].status == Order.STATUS_PAID
with scopes_disabled():
assert env[2].payments.first().info_data['iban'] == 'NL79RABO5373380466'
@pytest.mark.django_db(transaction=True)
def test_api_create_org_auto_currency(env, client):
client.login(email='dummy@dummy.dummy', password='dummy')
r = client.post(
'/api/v1/organizers/{}/bankimportjobs/'.format(env[0].organizer.slug), json.dumps({
'transactions': [
{
'payer': 'Foo',
'reference': 'DUMMY-1Z3AS',
'amount': '23.00',
'date': 'yesterday' # test bogus date format
}
]
}), content_type="application/json"
)
assert r.status_code == 201
rdata = json.loads(r.content.decode('utf-8'))
# This is only because we don't run celery in tests, otherwise it wouldn't be completed yet.
assert rdata['state'] == 'completed'
assert len(rdata['transactions']) == 1
assert rdata['transactions'][0]['checksum']
assert rdata['transactions'][0]['currency'] == 'EUR'
env[2].refresh_from_db()
assert env[2].status == Order.STATUS_PAID
@pytest.mark.django_db(transaction=True)
def test_api_create_org_unclear_currency(env, client):
Event.objects.create(
organizer=env[0].organizer, name='Dummy', slug='dummy2', currency='HUF',
date_from=now(), plugins='pretix.plugins.banktransfer'
)
client.login(email='dummy@dummy.dummy', password='dummy')
r = client.post(
'/api/v1/organizers/{}/bankimportjobs/'.format(env[0].organizer.slug), json.dumps({
'transactions': [
{
'payer': 'Foo',
'reference': 'DUMMY-1Z3AS',
'amount': '23.00',
'date': 'yesterday' # test bogus date format
}
]
}), content_type="application/json"
)
assert r.status_code == 400
r = client.post(
'/api/v1/organizers/{}/bankimportjobs/'.format(env[0].organizer.slug), json.dumps({
'currency': 'EUR',
'transactions': [
{
'payer': 'Foo',
'reference': 'DUMMY-1Z3AS',
'amount': '23.00',
'date': 'yesterday' # test bogus date format
}
]
}), content_type="application/json"
)
assert r.status_code == 201
rdata = json.loads(r.content.decode('utf-8'))
# This is only because we don't run celery in tests, otherwise it wouldn't be completed yet.
assert rdata['state'] == 'completed'
assert len(rdata['transactions']) == 1
assert rdata['transactions'][0]['checksum']
assert rdata['transactions'][0]['currency'] == 'EUR'
env[2].refresh_from_db()
assert env[2].status == Order.STATUS_PAID

View File

@@ -338,6 +338,19 @@ def test_mark_paid_organizer(env, orga_job):
assert env[2].status == Order.STATUS_PAID
@pytest.mark.django_db
def test_incorrect_currency(env, orga_job):
BankImportJob.objects.filter(pk=orga_job).update(currency='HUF')
process_banktransfers(orga_job, [{
'payer': 'Karla Kundin',
'reference': 'Bestellung DUMMY-1234S',
'date': '2016-01-26',
'amount': '23.00'
}])
env[2].refresh_from_db()
assert env[2].status == Order.STATUS_PENDING
@pytest.mark.django_db
def test_mark_paid_double_reference(env, orga_job):
process_banktransfers(orga_job, [{

View File

@@ -64,6 +64,32 @@ def env():
return event, user, refund
@pytest.fixture
def refund_huf(env):
event = Event.objects.create(
organizer=env[0].organizer, name='Dummy', slug='dummy2', currency='HUF',
date_from=now(), plugins='pretix.plugins.banktransfer,pretix.plugins.paypal'
)
order = Order.objects.create(
code='1Z3AS', event=event, email='admin@localhost',
status=Order.STATUS_PAID,
datetime=now(), expires=now() + timedelta(days=10),
total=42
)
refund = OrderRefund.objects.create(
order=order,
amount=Decimal("42"),
provider='banktransfer',
state=OrderRefund.REFUND_STATE_CREATED,
info=json.dumps({
'payer': "Abc Def",
'iban': "DE27520521540534534466",
'bic': "HELADEF1MEG",
})
)
return refund
url_prefixes = [
"/control/event/dummy/dummy/",
"/control/organizer/dummy/"
@@ -106,6 +132,18 @@ def test_export_refunds(client, env, url_prefix):
assert "HELADEF" in r
@pytest.mark.django_db
def test_export_refunds_multi_currency(client, env, refund_huf):
client.login(email='dummy@dummy.dummy', password='dummy')
r = client.get('/control/organizer/dummy/banktransfer/refunds/')
assert r.status_code == 200
r = client.post('/control/organizer/dummy/banktransfer/refunds/', {"unite_transactions": True}, follow=True)
assert r.status_code == 200
assert RefundExport.objects.count() == 2
assert RefundExport.objects.get(currency="EUR").sum == Decimal("23.00")
assert RefundExport.objects.get(currency="HUF").sum == Decimal("42.00")
@pytest.mark.django_db
@pytest.mark.parametrize("url_prefix", url_prefixes)
def test_export_refunds_omit_invalid_bic(client, env, url_prefix):