From f98f25fb6b6feac2e13c2d79c797344a3f80b199 Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Fri, 25 Aug 2017 14:49:57 +0200 Subject: [PATCH] Improve MT940 import --- .../plugins/banktransfer/mt940import.py | 160 +++++++++++++++++- src/pretix/plugins/banktransfer/tasks.py | 2 +- src/tests/plugins/banktransfer/test_import.py | 24 +++ src/tests/plugins/banktransfer/test_mt940.py | 76 +++++---- 4 files changed, 225 insertions(+), 37 deletions(-) diff --git a/src/pretix/plugins/banktransfer/mt940import.py b/src/pretix/plugins/banktransfer/mt940import.py index af04e9313e..56b04130af 100644 --- a/src/pretix/plugins/banktransfer/mt940import.py +++ b/src/pretix/plugins/banktransfer/mt940import.py @@ -1,10 +1,136 @@ import io +import string import mt940 from pretix.base.decimal import round_decimal +""" +The parse_transaction_details and join_reference functions are +Copyright (c) 2017 Nicole Klünder + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + + +def parse_transaction_details(raw_data): + transaction_details = { + 'code': [raw_data[:3]], + } + + code_mapping = { + '00': 'description', + '10': 'primanota', + '30': 'blz', + '31': 'accountnumber', + '32': 'accountholder', + '33': 'accountholder', + '34': 'chargeback', + '35': 'recipient', + '36': 'recipient', + } + for i in range(20, 30): + code_mapping[str(i)] = 'reference' + for i in range(60, 64): + code_mapping[str(i)] = 'additional' + + delimiter = raw_data[3] + + lines = sorted((line[:2], line[2:].strip()) for line in raw_data.split(delimiter)[1:]) + for code, data in lines: + transaction_details.setdefault(code_mapping.get(code, code), []).append(data) + + transaction_details = {name: '\n'.join(elems) for name, elems in transaction_details.items()} + + if 'reference' in transaction_details: + fragments = {'': []} + current_code = '' + for line in transaction_details['reference'].split('\n'): + code = line.split('+', 1)[0] + if code in ('EREF', 'SVWZ'): + current_code = code + line = line[len(code) + 1:] + fragments.setdefault(current_code, []).append(line) + + fragments = {code: '\n'.join(elems) for code, elems in fragments.items()} + + if 'EREF' in fragments: + transaction_details['eref'] = fragments['EREF'].replace('\n', '') + + if 'SVWZ' in fragments: + transaction_details['reference'] = fragments['SVWZ'] + + return transaction_details + + +def join_reference(reference_list, payer): + # Join Reference into one line. + reference = '' + if reference_list and ''.join(reference_list): + reference += reference_list.pop(0) + for d in reference_list: + if not d: + continue + if not ( + (reference[-1] in string.ascii_lowercase and d[0] in string.ascii_lowercase) or + (reference[-1] in string.ascii_uppercase and d[0] in string.ascii_uppercase) or + (reference[-1] in string.digits + string.ascii_uppercase and d[0] in ('-', ':')) or + (reference[-1] == ' ' or d[0] == ' ') + ): + reference += ' ' + reference += d + reference = [s for s in reference.split(' ') if s] + + eref = '' + if len(reference) >= 2 and reference[-2] == 'ABWA:': + payer['abwa'] = reference[-1] + reference = reference[:-2] + elif len(reference) >= 3 and reference[-3] == 'ABWA:': + payer['abwa'] = ''.join(reference[-2:]) + reference = reference[:-3] + + if len(reference) >= 2 and reference[-2] == 'BIC:': + payer['bic'] = reference[-1] + reference = reference[:-2] + elif len(reference) >= 3 and reference[-3] == 'BIC:': + payer['bic'] = ''.join(reference[-2:]) + reference = reference[:-3] + + if len(reference) >= 2 and reference[-2] == 'IBAN:': + payer['iban'] = reference[-1] + reference = reference[:-2] + elif len(reference) >= 3 and reference[-3] == 'IBAN:': + payer['iban'] = ''.join(reference[-2:]) + reference = reference[:-3] + + reference = ' '.join(reference) + + if ' EREF: ' in reference: + reference = reference.split(' EREF: ') + eref = reference[-1] + reference = reference[:-1] + reference = ' EREF: '.join(reference) + + return reference, eref + + def parse(file): data = file.read() try: @@ -17,11 +143,31 @@ def parse(file): mt = mt940.parse(io.StringIO(data.strip())) result = [] for t in mt: - result.append({ - 'reference': "\n".join([ - t.data.get(f) for f in ('transaction_details', 'customer_reference', 'bank_reference', - 'extra_details', 'non_swift_text') if t.data.get(f, '')]), - 'amount': str(round_decimal(t.data['amount'].amount)), - 'date': t.data['date'].isoformat() - }) + td = t.data.get('transaction_details', '') + if len(td) >= 4 and td[3] == '?': + # SEPA content + transaction_details = parse_transaction_details(td.replace("\n", "")) + + payer = { + 'name': transaction_details.get('accountholder', ''), + 'iban': transaction_details.get('accountnumber', ''), + } + reference, eref = join_reference(transaction_details.get('reference', '').split('\n'), payer) + if not eref: + eref = transaction_details.get('eref', '') + + result.append({ + 'amount': str(round_decimal(t.data['amount'].amount)), + 'reference': reference + (' EREF: {}'.format(eref) if eref else ''), + 'payer': (payer.get('name', '') + ' - ' + payer.get('iban', '')).strip(), + 'date': t.data['date'].isoformat() + }) + else: + result.append({ + 'reference': "\n".join([ + t.data.get(f) for f in ('transaction_details', 'customer_reference', 'bank_reference', + 'extra_details', 'non_swift_text') if t.data.get(f, '')]), + 'amount': str(round_decimal(t.data['amount'].amount)), + 'date': t.data['date'].isoformat() + }) return result diff --git a/src/pretix/plugins/banktransfer/tasks.py b/src/pretix/plugins/banktransfer/tasks.py index 8fc8162165..ac2aaad4bf 100644 --- a/src/pretix/plugins/banktransfer/tasks.py +++ b/src/pretix/plugins/banktransfer/tasks.py @@ -142,7 +142,7 @@ def process_banktransfers(self, job: int, data: list) -> None: pattern = re.compile("(%s)[ \-_]*([A-Z0-9]{%s})" % ("|".join(prefixes), code_len)) for trans in transactions: - match = pattern.search(trans.reference.replace(" ", "").upper()) + match = pattern.search(trans.reference.replace(" ", "").replace("\n", "").upper()) if match: if job.event: diff --git a/src/tests/plugins/banktransfer/test_import.py b/src/tests/plugins/banktransfer/test_import.py index 46f96657ba..863d581456 100644 --- a/src/tests/plugins/banktransfer/test_import.py +++ b/src/tests/plugins/banktransfer/test_import.py @@ -155,6 +155,30 @@ def test_random_spaces(env, job): assert env[2].status == Order.STATUS_PAID +@pytest.mark.django_db +def test_random_newlines(env, job): + process_banktransfers(job, [{ + 'payer': 'Karla Kundin', + 'reference': 'Bestellung DUM\nMY123\n 45NEXTLINE', + 'amount': '23.00', + 'date': '2016-01-26', + }]) + env[2].refresh_from_db() + assert env[2].status == Order.STATUS_PAID + + +@pytest.mark.django_db +def test_end_comma(env, job): + process_banktransfers(job, [{ + 'payer': 'Karla Kundin', + 'reference': 'Bestellung DUMMY12345,NEXTLINE', + 'amount': '23.00', + 'date': '2016-01-26', + }]) + env[2].refresh_from_db() + assert env[2].status == Order.STATUS_PAID + + @pytest.mark.django_db def test_huge_amount(env, job): env[2].total = Decimal('23000.00') diff --git a/src/tests/plugins/banktransfer/test_mt940.py b/src/tests/plugins/banktransfer/test_mt940.py index 69e872aacc..e969b7f57a 100644 --- a/src/tests/plugins/banktransfer/test_mt940.py +++ b/src/tests/plugins/banktransfer/test_mt940.py @@ -122,6 +122,33 @@ MUELLER?34999 :NS:01bekannt 1812345 :62F:C02032495000,00 +""", + # From a customer (N26) + """ +:20:STARTUMS +:25:DE13495179316396679327 +:28C:0 +:60F:C170823EUR0, +:61:1708230823C12,NMSCNONREF +:86:000?32Peter Schneider?31DE13495179316396679327?30NOTABIC?20De +mocon-Abcde (Peter Schneider?21), Kategorie: Alles - E?22innahmen - V +eranstaltungen ?23Democon #1111 +:61:1708230823C12,NMSCNONREF +:86:000?32Peter Schneider?31DE13495179316396679327?30NOTABIC?20De +mocon-Abcde (Peter Schneider?21), Kategorie: Alles - E?22innahmen - V +eranstaltungen ?23Democon #1111 +:62F:C170823EUR24, +- +:20:STARTUMS +:25:DE13495179316396679327 +:28C:0 +:60F:C170824EUR24, +:61:1708240824C12,NMSCNONREF +:86:000?32Peter Schneider?31DE13495179316396679327?30NOTABIC?20De +mocon-Abcde (Peter Schneider?21), Kategorie: Alles- E?22innahmen - V +eranstaltungen ?23Democon #1111 +:62F:C170824EUR36, +- """ ] @@ -164,47 +191,30 @@ EXPECTED = [ [ {'amount': '-800.00', 'date': '2002-11-01', - 'reference': '008?00DAUERAUFTRAG?100599?20Miete Novem\n' - 'ber?3010020030?31234567\n' - '?32MUELLER?34339\n' - 'NONREF//55555'}, + 'payer': 'MUELLER - 234567', + 'reference': 'Miete November'}, {'amount': '3000.00', 'date': '2002-11-02', - 'reference': '051?00UEBERWEISUNG?100599?20Gehalt Oktob\n' - 'er\n' - '?21Firma Mustermann GmbH?3050060400?31084756\n' - '4700?32MUELLER?34339\n' - 'NONREF//55555'} + 'payer': 'MUELLER - 0847564700', + 'reference': 'Gehalt Oktober Firma Mustermann GmbH'}, ], [ {'amount': '-400.62', 'date': '2012-02-02', - 'reference': '077?00Überweisung beleglos?109310?20RECHNUNGSNR. ' - '1210815 ?21K\n' - 'UNDENNR. 01234 ?22DATUM ' - '01.02.2012?3020020020?2222222222?32MARTHA\n' - 'MUELLER?34999\n' - 'NONREF'}, + 'payer': 'MARTHAMUELLER -', + 'reference': 'RECHNUNGSNR. 1210815 KUNDENNR. 01234 22222222 DATUM 01.02.2012'}, {'amount': '-1210.00', 'date': '2012-02-03', - 'reference': '008?00Dauerauftrag?107000?20MIETE GOETHESTR. ' - '12?3030030030?31\n' - '3333333333?32ABC IMMOBILIEN GMBH?34997\n' - 'NONREF'}, + 'reference': 'MIETE GOETHESTR. 12', + 'payer': 'ABC IMMOBILIEN GMBH - 3333333333'}, {'amount': '30.00', 'date': '2012-02-03', - 'reference': '051?00Überweisungseingang?109265?20RECHNUNG ' - '20120188?21STEFAN\n' - ' SCHMIDT?23KUNDENR. ' - '4711,?3040040040?4444444444?32STEFAN SCHMIDT\n' - 'NONREF'}, + 'payer': 'STEFAN SCHMIDT -', + 'reference': 'RECHNUNG 20120188 STEFAN SCHMIDTKUNDENR. 4711,'}, {'amount': '89.97', 'date': '2012-02-03', - 'reference': '052?00Überweisungseingang?109265?20RECHNUNG ' - '20120165?21PETER\n' - ' PETERSEN?3050050050?315555555555?32PETER PETERSEN\n' - 'NONREF//00000000\n' - '0001'} + 'payer': 'PETER PETERSEN - 5555555555', + 'reference': 'RECHNUNG 20120165 PETER PETERSEN'} ], [ {'amount': '5000.00', 'date': '2002-03-17', 'reference': '68790452'}, @@ -216,6 +226,14 @@ EXPECTED = [ {'amount': '20000.00', 'date': '2002-03-22', 'reference': ''}, {'amount': '20000.00', 'date': '2002-03-22', 'reference': ''}, {'amount': '-50000.00', 'date': '2002-03-24', 'reference': ''} + ], + [ + {'amount': '12.00', 'date': '2017-08-23', 'payer': 'Peter Schneider - DE13495179316396679327', 'reference': + 'Democon-Abcde (Peter Schneider ), Kategorie: Alles - E innahmen - Veranstaltungen Democon #1111'}, + {'amount': '12.00', 'date': '2017-08-23', 'payer': 'Peter Schneider - DE13495179316396679327', 'reference': + 'Democon-Abcde (Peter Schneider ), Kategorie: Alles - E innahmen - Veranstaltungen Democon #1111'}, + {'amount': '12.00', 'date': '2017-08-24', 'payer': 'Peter Schneider - DE13495179316396679327', 'reference': + 'Democon-Abcde (Peter Schneider ), Kategorie: Alles- E innahmen - Veranstaltungen Democon #1111'}, ] ]