Bank CSV import: Allow dealing with files of more than a few hundred lines during first import

src/pretix/plugins/banktransfer/templates/pretixplugins/banktransfer/import_assign.html

@@ -47,22 +47,30 @@
                     </tr>
                 </thead>
                 <tbody>
-                    {% for row in rows %}
+                    {% for row in rows|slice:":100" %}
                         {% with forloop.counter0 as rowid %}
                             <tr>
                                 <td></td>
                                 {% for col in row %}
-                                    <td>{{ col }}<input type="hidden" name="col[{{ rowid }}][{{ forloop.counter0 }}]"
-                                            value="{{ col }}" /></td>
+                                    <td>{{ col }}</td>
                                 {% endfor %}
                             </tr>
                         {% endwith %}
                     {% endfor %}
+                    {% if rows|length > 100 %}
+                        <tr>
+                            <td colspan="{{ rows.0|length|add:1 }}" class="text-center">
+                                <em>{% trans "More data was uploaded but is not shown here. It will still be processed" %}</em>
+                            </td>
+                        </tr>
+                    {% endif %}
                 </tbody>
-                <input type="hidden" name="cols" value="{{ rows.0|length }}" />
-                <input type="hidden" name="rows" value="{{ rows|length }}" />
-
             </table>
         </div>
+        <input type="hidden" name="cols" value="{{ rows.0|length }}" />
+        <input type="hidden" name="rows" value="{{ rows|length }}" />
+        <textarea class="helper-display-none" name="data">
+            {{ json|escape }}
+        </textarea>
     </form>
 {% endblock %}

src/pretix/plugins/banktransfer/views.py

@@ -347,14 +347,12 @@ class ImportView(ListView):
         return self.assign_view(data)
 
     def process_csv_hint(self):
-        data = []
-        for i in range(int(self.request.POST.get('rows'))):
-            data.append(
-                [
-                    self.request.POST.get('col[%d][%d]' % (i, j))
-                    for j in range(int(self.request.POST.get('cols')))
-                ]
-            )
+        try:
+            data = json.loads(self.request.POST.get('data').strip())
+        except ValueError:
+            messages.error(self.request, _('Invalid input data.'))
+            return self.get(self.request, *self.args, **self.kwargs)
+
         if 'reference' not in self.request.POST:
             messages.error(self.request, _('You need to select the column containing the payment reference.'))
             return self.assign_view(data)
@@ -382,7 +380,10 @@ class ImportView(ListView):
         return super().get(self.request)
 
     def assign_view(self, parsed):
-        ctx = {'rows': parsed}
+        ctx = {
+            'json': json.dumps(parsed),
+            'rows': parsed,
+        }
         if 'event' in self.kwargs:
             ctx['basetpl'] = 'pretixplugins/banktransfer/import_base.html'
         else:

src/pretix/static/pretixcontrol/scss/main.scss

@@ -124,6 +124,9 @@ h1 .btn-sm {
 .helper-display-inline {
     display: inline !important;
 }
+.helper-display-none {
+    display: none !important;
+}
 .helper-width-auto {
     width: auto;
 }

src/tests/plugins/banktransfer/test_import.py

@@ -81,6 +81,8 @@ Buchungstag;Valuta;Buchungstext;Auftraggeber / Empfänger;Verwendungszweck;Betra
     }
     for inp in doc.select("input[type=hidden]"):
         data[inp.attrs['name']] = inp.attrs['value']
+    for inp in doc.select("textarea"):
+        data[inp.attrs['name']] = inp.text
     r = client.post('/control/event/dummy/dummy/banktransfer/import/', data)
     assert '/job/' in r['Location']
 
@@ -233,3 +235,43 @@ def test_wrong_event_organizer(env, orga_job):
     }])
     env[2].refresh_from_db()
     assert env[2].status == Order.STATUS_PENDING
+
+
+@pytest.mark.django_db
+def test_import_very_long_csv_file(client, env):
+    client.login(email='dummy@dummy.dummy', password='dummy')
+    r = client.get('/control/event/dummy/dummy/banktransfer/import/')
+    assert r.status_code == 200
+
+    payload = """
+Buchungstag;Valuta;Buchungstext;Auftraggeber / Empfänger;Verwendungszweck;Betrag in EUR;
+09.04.2015;09.04.2015;SEPA-Überweisung;Karl Kunde;Bestellung 2015ABCDE;23,00;
+09.04.2015;09.04.2015;SEPA-Überweisung;Karla Kundin;Bestellung DUMMYFGHIJ;42,00;
+09.04.2015;09.04.2015;SEPA-Überweisung;Karla Kundin;Bestellung DUMMY1234S;42,00;
+09.04.2015;09.04.2015;SEPA-Überweisung;Karla Kundin;Bestellung DUMMY1234S;23,00;
+09.04.2015;09.04.2015;SEPA-Überweisung;Karla Kundin;Bestellung DUMMY6789Z;23,00;
+09.04.2015;09.04.2015;SEPA-Überweisung;Karla Kundin;Bestellung DUMMY6789Z;23,00;
+"""
+    payload += "09.04.2015;09.04.2015;SEPA-Überweisung;Karla Kundin;Bestellung DUMMY6789Z;23,00;\n" * 1000
+
+    file = SimpleUploadedFile('file.csv', payload.encode("utf-8"), content_type="text/csv")
+
+    r = client.post('/control/event/dummy/dummy/banktransfer/import/', {
+        'file': file
+    })
+    doc = BeautifulSoup(r.content, "lxml")
+    assert r.status_code == 200
+    assert len(doc.select("input[name=date]")) > 0
+    data = {
+        'payer': [3],
+        'reference': [4],
+        'date': 1,
+        'amount': 5,
+        'cols': 7
+    }
+    for inp in doc.select("input[type=hidden]"):
+        data[inp.attrs['name']] = inp.attrs['value']
+    for inp in doc.select("textarea"):
+        data[inp.attrs['name']] = inp.text
+    r = client.post('/control/event/dummy/dummy/banktransfer/import/', data)
+    assert '/job/' in r['Location']