Fix #571 -- Partial payments and refunds

src/pretix/base/exporters/invoices.py

@@ -5,9 +5,12 @@ from zipfile import ZipFile
 
 import dateutil.parser
 from django import forms
+from django.db.models import Exists, OuterRef, Q
 from django.dispatch import receiver
 from django.utils.translation import ugettext_lazy as _
 
+from pretix.base.models import OrderPayment
+
 from ..exporter import BaseExporter
 from ..services.invoices import invoice_pdf_task
 from ..signals import register_data_exporters
@@ -21,7 +24,14 @@ class InvoiceExporter(BaseExporter):
         qs = self.event.invoices.filter(shredded=False)
 
         if form_data.get('payment_provider'):
-            qs = qs.filter(order__payment_provider=form_data.get('payment_provider'))
+            qs = qs.annotate(
+                has_payment_with_provider=Exists(
+                    OrderPayment.objects.filter(
+                        Q(order=OuterRef('pk')) & Q(provider=form_data.get('payment_provider'))
+                    )
+                )
+            )
+            qs = qs.filter(has_payment_with_provider=1)
 
         if form_data.get('date_from'):
             date_value = form_data.get('date_from')
@@ -84,10 +94,10 @@ class InvoiceExporter(BaseExporter):
                          (k, v.verbose_name) for k, v in self.event.get_payment_providers().items()
                      ],
                      required=False,
-                     help_text=_('Only include invoices for orders that are currently set to this payment provider. '
-                                 'Note that this might include some invoices of other payment providers or misses '
-                                 'some invoices if the payment provider of an order has been changed and a new invoice '
-                                 'has been generated.')
+                     help_text=_('Only include invoices for orders that have at least one payment attempt '
+                                 'with this payment provider. '
+                                 'Note that this might include some invoices of orders which in the end have been '
+                                 'fully or partially paid with a different provider.')
                  )),
             ]
         )

src/pretix/base/exporters/orderlist.py

@@ -5,13 +5,13 @@ from decimal import Decimal
 import pytz
 from defusedcsv import csv
 from django import forms
-from django.db.models import Sum
+from django.db.models import DateTimeField, Max, OuterRef, Subquery, Sum
 from django.dispatch import receiver
 from django.utils.formats import localize
 from django.utils.translation import ugettext as _, ugettext_lazy
 
 from pretix.base.models import InvoiceAddress, Order, OrderPosition
-from pretix.base.models.orders import OrderFee
+from pretix.base.models.orders import OrderFee, OrderPayment, OrderRefund
 
 from ..exporter import BaseExporter
 from ..signals import register_data_exporters
@@ -55,7 +55,19 @@ class OrderListExporter(BaseExporter):
         tz = pytz.timezone(self.event.settings.timezone)
         writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC, delimiter=",")
 
-        qs = self.event.orders.all().select_related('invoice_address').prefetch_related('invoices')
+        p_date = OrderPayment.objects.filter(
+            order=OuterRef('pk'),
+            state__in=(OrderPayment.PAYMENT_STATE_CONFIRMED, OrderPayment.PAYMENT_STATE_REFUNDED),
+            payment_date__isnull=False
+        ).order_by().values('order').annotate(
+            m=Max('payment_date')
+        ).values(
+            'm'
+        )
+
+        qs = self.event.orders.annotate(
+            payment_date=Subquery(p_date, output_field=DateTimeField())
+        ).select_related('invoice_address').prefetch_related('invoices')
         if form_data['paid_only']:
             qs = qs.filter(status=Order.STATUS_PAID)
         tax_rates = self._get_all_tax_rates(qs)
@@ -63,7 +75,7 @@ class OrderListExporter(BaseExporter):
         headers = [
             _('Order code'), _('Order total'), _('Status'), _('Email'), _('Order date'),
             _('Company'), _('Name'), _('Address'), _('ZIP code'), _('City'), _('Country'), _('VAT ID'),
-            _('Payment date'), _('Payment type'), _('Fees'), _('Order locale')
+            _('Date of last payment'), _('Fees'), _('Order locale')
         ]
 
         for tr in tax_rates:
@@ -77,11 +89,6 @@ class OrderListExporter(BaseExporter):
 
         writer.writerow(headers)
 
-        provider_names = {
-            k: v.verbose_name
-            for k, v in self.event.get_payment_providers().items()
-        }
-
         full_fee_sum_cache = {
             o['order__id']: o['grosssum'] for o in
             OrderFee.objects.values('tax_rate', 'order__id').order_by().annotate(grosssum=Sum('value'))
@@ -114,7 +121,8 @@ class OrderListExporter(BaseExporter):
                     order.invoice_address.street,
                     order.invoice_address.zipcode,
                     order.invoice_address.city,
-                    order.invoice_address.country if order.invoice_address.country else order.invoice_address.country_old,
+                    order.invoice_address.country if order.invoice_address.country else
+                    order.invoice_address.country_old,
                     order.invoice_address.vat_id,
                 ]
             except InvoiceAddress.DoesNotExist:
@@ -122,14 +130,14 @@ class OrderListExporter(BaseExporter):
 
             row += [
                 order.payment_date.astimezone(tz).strftime('%Y-%m-%d') if order.payment_date else '',
-                provider_names.get(order.payment_provider, order.payment_provider),
                 localize(full_fee_sum_cache.get(order.id) or Decimal('0.00')),
                 order.locale,
             ]
 
             for tr in tax_rates:
                 taxrate_values = sum_cache.get((order.id, tr), {'grosssum': Decimal('0.00'), 'taxsum': Decimal('0.00')})
-                fee_taxrate_values = fee_sum_cache.get((order.id, tr), {'grosssum': Decimal('0.00'), 'taxsum': Decimal('0.00')})
+                fee_taxrate_values = fee_sum_cache.get((order.id, tr),
+                                                       {'grosssum': Decimal('0.00'), 'taxsum': Decimal('0.00')})
 
                 row += [
                     localize(taxrate_values['grosssum'] + fee_taxrate_values['grosssum']),
@@ -144,6 +152,77 @@ class OrderListExporter(BaseExporter):
         return '{}_orders.csv'.format(self.event.slug), 'text/csv', output.getvalue().encode("utf-8")
 
 
+class PaymentListExporter(BaseExporter):
+    identifier = 'paymentlistcsv'
+    verbose_name = ugettext_lazy('List of payments and refunds (CSV)')
+
+    @property
+    def export_form_fields(self):
+        return OrderedDict(
+            [
+                ('successful_only',
+                 forms.BooleanField(
+                     label=_('Only successful payments'),
+                     initial=True,
+                     required=False
+                 )),
+            ]
+        )
+
+    def render(self, form_data: dict):
+        output = io.StringIO()
+        tz = pytz.timezone(self.event.settings.timezone)
+        writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC, delimiter=",")
+
+        provider_names = {
+            k: v.verbose_name
+            for k, v in self.event.get_payment_providers().items()
+        }
+
+        payments = OrderPayment.objects.filter(
+            order__event=self.event,
+        ).order_by('created')
+        refunds = OrderRefund.objects.filter(
+            order__event=self.event
+        ).order_by('created')
+
+        if form_data['successful_only']:
+            payments = payments.filter(
+                state__in=(OrderPayment.PAYMENT_STATE_CONFIRMED, OrderPayment.PAYMENT_STATE_REFUNDED),
+            )
+            refunds = refunds.filter(
+                state=OrderRefund.REFUND_STATE_DONE,
+            )
+
+        objs = sorted(list(payments) + list(refunds), key=lambda o: o.created)
+
+        headers = [
+            _('Order'), _('Payment ID'), _('Creation date'), _('Completion date'), _('Status'),
+            _('Amount'), _('Payment method')
+        ]
+        writer.writerow(headers)
+
+        for obj in objs:
+            if isinstance(obj, OrderPayment) and obj.payment_date:
+                d2 = obj.payment_date.astimezone(tz).date().strftime('%Y-%m-%d')
+            elif isinstance(obj, OrderRefund) and obj.execution_date:
+                d2 = obj.execution_date.astimezone(tz).date().strftime('%Y-%m-%d')
+            else:
+                d2 = ''
+            row = [
+                obj.order.code,
+                obj.full_id,
+                obj.created.astimezone(tz).date().strftime('%Y-%m-%d'),
+                d2,
+                obj.get_state_display(),
+                localize(obj.amount * (-1 if isinstance(obj, OrderRefund) else 1)),
+                provider_names.get(obj.provider, obj.provider)
+            ]
+            writer.writerow(row)
+
+        return '{}_payments.csv'.format(self.event.slug), 'text/csv', output.getvalue().encode("utf-8")
+
+
 class QuotaListExporter(BaseExporter):
     identifier = 'quotalistcsv'
     verbose_name = ugettext_lazy('Quota availabilities (CSV)')
@@ -180,6 +259,11 @@ def register_orderlist_exporter(sender, **kwargs):
     return OrderListExporter
 
 
+@receiver(register_data_exporters, dispatch_uid="exporter_paymentlist")
+def register_paymentlist_exporter(sender, **kwargs):
+    return PaymentListExporter
+
+
 @receiver(register_data_exporters, dispatch_uid="exporter_quotalist")
 def register_quotalist_exporter(sender, **kwargs):
     return QuotaListExporter