From 5d4b218aa6b9fc1df181fdf07339ab10e98ca409 Mon Sep 17 00:00:00 2001
From: Kian Cross <kian@kiancross.co.uk>
Date: Wed, 5 Feb 2025 15:56:28 +0000
Subject: [PATCH] Banktransfer: Handle trailing commas in headers for Lloyds
 Bank CSV files (#4782)

Lloyds Bank (UK) CSV files include a trailing comma in the header row
but not in the data rows, causing the `csvimport.parse` function to
skip the data rows. This occurs because the header length exceeds the
row length, making them unequal to `hint.cols`.

This commit adjusts the length check to allow a range of acceptable row
lengths, from the index of the last non-empty column in the header to
`hint.cols`. This ensures compatibility with headers containing one or
more trailing commas without affecting rows with correctly labelled columns.

The solution avoids breaking changes by leaving underlying data structures
untouched. Alternative approaches, such as dropping trailing commas before
parsing or removing empty elements after parsing, were avoided due to
potential risks. Specifically, trailing columns might contain data that
banks provide but fail to label in the header row.
---
 src/pretix/plugins/banktransfer/csvimport.py  | 15 ++++++++++-
 .../banktransfer/csvimport_data_gb_lloyds.csv |  3 +++
 .../plugins/banktransfer/test_csvparser.py    | 25 +++++++++++++++++++
 3 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 src/tests/plugins/banktransfer/csvimport_data_gb_lloyds.csv

diff --git a/src/pretix/plugins/banktransfer/csvimport.py b/src/pretix/plugins/banktransfer/csvimport.py
index 0cd6cfe2c..33f7f3459 100644
--- a/src/pretix/plugins/banktransfer/csvimport.py
+++ b/src/pretix/plugins/banktransfer/csvimport.py
@@ -30,6 +30,19 @@ class HintMismatchError(Exception):
     pass
 
 
+def check_row_length(data, hint, row):
+    valid_lengths = [hint['cols']]
+    header = data[0]
+
+    for i in range(len(header) - 1, 0, -1):
+        if header[i]:
+            break
+        else:
+            valid_lengths.append(hint['cols'] - (len(header) - i))
+
+    return None not in row and len(row) in valid_lengths
+
+
 def parse(data, hint):
     result = []
     if 'cols' not in hint:
@@ -39,7 +52,7 @@ def parse(data, hint):
     good_hint = False
     for row in data:
         resrow = {}
-        if None in row or len(row) != hint['cols']:
+        if not check_row_length(data, hint, row):
             # Wrong column count
             continue
         if hint.get('payer') is not None:
diff --git a/src/tests/plugins/banktransfer/csvimport_data_gb_lloyds.csv b/src/tests/plugins/banktransfer/csvimport_data_gb_lloyds.csv
new file mode 100644
index 000000000..bb0deb99f
--- /dev/null
+++ b/src/tests/plugins/banktransfer/csvimport_data_gb_lloyds.csv
@@ -0,0 +1,3 @@
+Transaction Date,Transaction Type,Sort Code,Account Number,Transaction Description,Debit Amount,Credit Amount,Balance,
+27/01/2025,FPI,'99-99-99,11111111,SMITH J ABCDE 111111111111111111 111111     10 25JAN25 13:34,,214,500
+25/01/2025,FPI,'99-99-99,11111111,JONES A FGHIJ 111111111111111112 111112     10 25JAN25 10:34,,213,286
diff --git a/src/tests/plugins/banktransfer/test_csvparser.py b/src/tests/plugins/banktransfer/test_csvparser.py
index bf960c887..c2aea00ac 100644
--- a/src/tests/plugins/banktransfer/test_csvparser.py
+++ b/src/tests/plugins/banktransfer/test_csvparser.py
@@ -166,3 +166,28 @@ class CsvImportTest(TestCase):
         ]
         filename = "csvimport_data_de_postbank.csv"
         self._test_from_sample_file(filename, expected, hint, expected_parsed)
+
+    def test_sample_file_lloyds(self):
+        expected = [
+            # Lloyds Bank includes a trailing comma at the end of the header row, making the header one column longer than the data rows.
+            ['Transaction Date', 'Transaction Type', 'Sort Code', 'Account Number', 'Transaction Description', 'Debit Amount',
+             'Credit Amount', 'Balance', ''],
+            ["27/01/2025", "FPI", "'99-99-99", "11111111", "SMITH J ABCDE 111111111111111111 111111     10 25JAN25 13:34",
+             "", "214", "500"],
+            ["25/01/2025", "FPI", "'99-99-99", "11111111", "JONES A FGHIJ 111111111111111112 111112     10 25JAN25 10:34",
+             "", "213", "286"],
+        ]
+        hint = {
+            'reference': [4],
+            'date': 0,
+            'amount': 6,
+            'cols': 9,
+        }
+        expected_parsed = [
+            {'reference': 'SMITH J ABCDE 111111111111111111 111111     10 25JAN25 13:34', 'amount': '214',
+             'date': '27/01/2025'},
+            {'reference': 'JONES A FGHIJ 111111111111111112 111112     10 25JAN25 10:34', 'amount': '213',
+             'date': '25/01/2025'},
+        ]
+        filename = "csvimport_data_gb_lloyds.csv"
+        self._test_from_sample_file(filename, expected, hint, expected_parsed)