Pluggable invoice transmission methods (#5020)

* Flexible invoice transmission

* UI work

* Add peppol and output

* API support

* Profile integration

* Simplify form for individuals

* Remove sent_to_customer usage

* more steps

* Revert "Bank transfer: Allow to send the invoice direclty to the accounting department (#2975)"

This reverts commit cea6c340be.

* minor fixes

* Fixes after rebase

* update stati

* Backend view

* Transmit and show status

* status, retransmission

* API retransmission

* More fields

* API docs

* Plugin docs

* Update migration

* Add missing license headers

* Remove dead code, fix current tests

* Run isort

* Update regex

* Rebase migration

* Fix migration

* Add tests, fix bugs

* Rebase migration

* Apply suggestion from @luelista

Co-authored-by: luelista <weller@rami.io>

* Apply suggestion from @luelista

Co-authored-by: luelista <weller@rami.io>

* Apply suggestion from @luelista

Co-authored-by: luelista <weller@rami.io>

* Apply suggestion from @luelista

Co-authored-by: luelista <weller@rami.io>

* Apply suggestion from @luelista

Co-authored-by: luelista <weller@rami.io>

* Make migration reversible

* Add TransmissionType.enforce_transmission

* Fix registries API usage after rebase

* Remove code I forgot to delete

* Update transmission status display depending on type

* Add testmode_supported

* Update src/pretix/static/pretixbase/js/addressform.js

Co-authored-by: luelista <weller@rami.io>

* Update src/pretix/static/pretixbase/js/addressform.js

Co-authored-by: luelista <weller@rami.io>

* Update src/pretix/static/pretixbase/js/addressform.js

Co-authored-by: luelista <weller@rami.io>

* New mechanism for non-required invoice forms

* Update src/pretix/base/invoicing/transmission.py

Co-authored-by: luelista <weller@rami.io>

* Declare testmode_supported for email

* Make transmission_email_other an implementation detail

* Fix failing tests and add new ones

* Update src/pretix/base/services/invoices.py

Co-authored-by: luelista <weller@rami.io>

* Add emails to email history

* Fix comma error

* More generic default email text

* Cleanup

* Remove "email invoices" button and refine logic

* Rebase migration

* Fix edge case

---------

Co-authored-by: luelista <weller@rami.io>

src/pretix/base/services/invoices.py

@@ -51,11 +51,16 @@ from django_scopes import scope, scopes_disabled
 from i18nfield.strings import LazyI18nString
 
 from pretix.base.i18n import language
+from pretix.base.invoicing.transmission import (
+    get_transmission_types, transmission_providers,
+)
 from pretix.base.models import (
     ExchangeRate, Invoice, InvoiceAddress, InvoiceLine, Order, OrderFee,
 )
 from pretix.base.models.tax import EU_CURRENCIES
-from pretix.base.services.tasks import TransactionAwareTask
+from pretix.base.services.tasks import (
+    TransactionAwareProfiledEventTask, TransactionAwareTask,
+)
 from pretix.base.signals import invoice_line_text, periodic_task
 from pretix.celery_app import app
 from pretix.helpers.database import OF_SELF, rolledback_transaction
@@ -71,12 +76,13 @@ def _location_oneliner(loc):
 @transaction.atomic
 def build_invoice(invoice: Invoice) -> Invoice:
     invoice.locale = invoice.event.settings.get('invoice_language', invoice.event.settings.locale)
+    invoice.transmission_status = Invoice.TRANSMISSION_STATUS_PENDING
     if invoice.locale == '__user__':
         invoice.locale = invoice.order.locale or invoice.event.settings.locale
 
     lp = invoice.order.payments.last()
 
-    with language(invoice.locale, invoice.event.settings.region):
+    with (language(invoice.locale, invoice.event.settings.region)):
         invoice.invoice_from = invoice.event.settings.get('invoice_address_from')
         invoice.invoice_from_name = invoice.event.settings.get('invoice_address_from_name')
         invoice.invoice_from_zipcode = invoice.event.settings.get('invoice_address_from_zipcode')
@@ -127,6 +133,7 @@ def build_invoice(invoice: Invoice) -> Invoice:
             invoice.internal_reference = ia.internal_reference
             invoice.custom_field = ia.custom_field
             invoice.invoice_to_company = ia.company
+            invoice.invoice_to_is_business = ia.is_business
             invoice.invoice_to_name = ia.name
             invoice.invoice_to_street = ia.street
             invoice.invoice_to_zipcode = ia.zipcode
@@ -134,6 +141,8 @@ def build_invoice(invoice: Invoice) -> Invoice:
             invoice.invoice_to_country = ia.country
             invoice.invoice_to_state = ia.state
             invoice.invoice_to_beneficiary = ia.beneficiary
+            invoice.invoice_to_transmission_info = ia.transmission_info or {}
+            invoice.transmission_type = ia.transmission_type
 
             if ia.vat_id:
                 invoice.invoice_to += "\n" + pgettext("invoice", "VAT-ID: %s") % ia.vat_id
@@ -356,7 +365,9 @@ def generate_cancellation(invoice: Invoice, trigger_pdf=True):
     cancellation.payment_provider_stamp = ''
     cancellation.file = None
     cancellation.sent_to_organizer = None
-    cancellation.sent_to_customer = None
+    cancellation.transmission_provider = None
+    cancellation.transmission_status = Invoice.TRANSMISSION_STATUS_PENDING
+    cancellation.transmission_date = None
     with language(invoice.locale, invoice.event.settings.region):
         cancellation.invoice_from = invoice.event.settings.get('invoice_address_from')
         cancellation.invoice_from_name = invoice.event.settings.get('invoice_address_from_name')
@@ -512,6 +523,36 @@ def build_preview_invoice_pdf(event):
         return event.invoice_renderer.generate(invoice)
 
 
+def order_invoice_transmission_separately(order):
+    try:
+        info = order.invoice_address.transmission_info or {}
+        return (
+            order.invoice_address.transmission_type != "email" or
+            (
+                info.get("transmission_email_address") and
+                order.email != info["transmission_email_address"]
+            )
+        )
+    except InvoiceAddress.DoesNotExist:
+        return False
+
+
+def invoice_transmission_separately(invoice):
+    if not invoice:
+        return False
+    try:
+        info = invoice.invoice_to_transmission_info or {}
+        return (
+            invoice.transmission_type != "email" or
+            (
+                info.get("transmission_email_address") and
+                invoice.order.email != info["transmission_email_address"]
+            )
+        )
+    except InvoiceAddress.DoesNotExist:
+        return False
+
+
 @receiver(signal=periodic_task)
 @scopes_disabled()
 def send_invoices_to_organizer(sender, **kwargs):
@@ -551,3 +592,124 @@ def send_invoices_to_organizer(sender, **kwargs):
                 else:
                     i.sent_to_organizer = False
                 i.save(update_fields=['sent_to_organizer'])
+
+
+@receiver(signal=periodic_task)
+@scopes_disabled()
+def retry_stuck_invoices(sender, **kwargs):
+    with transaction.atomic():
+        qs = Invoice.objects.filter(
+            transmission_status=Invoice.TRANSMISSION_STATUS_INFLIGHT,
+            transmission_date__lte=now() - timedelta(hours=24),
+        ).select_for_update(
+            of=OF_SELF, skip_locked=connection.features.has_select_for_update_skip_locked
+        )
+        batch_size = 5000
+        for invoice in qs[:batch_size]:
+            invoice.transmission_status = Invoice.TRANSMISSION_STATUS_PENDING
+            invoice.transmission_date = now()
+            invoice.save(update_fields=["transmission_status", "transmission_date"])
+            transmit_invoice.apply_async(args=(invoice.event_id, invoice.pk, True))
+
+
+@receiver(signal=periodic_task)
+@scopes_disabled()
+def send_pending_invoices(sender, **kwargs):
+    with transaction.atomic():
+        # Transmit all invoices that have not been transmitted by another process if the provider enforces
+        # transmission
+        types = [
+            tt.identifier for tt in get_transmission_types()
+            if tt.enforce_transmission
+        ]
+        qs = Invoice.objects.filter(
+            transmission_type__in=types,
+            transmission_status=Invoice.TRANSMISSION_STATUS_PENDING,
+            created__lte=now() - timedelta(minutes=15),
+        ).select_for_update(
+            of=OF_SELF, skip_locked=connection.features.has_select_for_update_skip_locked
+        )
+        batch_size = 5000
+        for invoice in qs[:batch_size]:
+            transmit_invoice.apply_async(args=(invoice.event_id, invoice.pk, False))
+
+
+@app.task(base=TransactionAwareProfiledEventTask)
+def transmit_invoice(sender, invoice_id, allow_retransmission=True, **kwargs):
+    with transaction.atomic(durable='tests.testdummy' not in settings.INSTALLED_APPS):
+        # We need durable=True for transactional correctness, but can't have it during tests
+        invoice = Invoice.objects.select_for_update(of=OF_SELF).get(pk=invoice_id)
+
+        if invoice.transmission_status == Invoice.TRANSMISSION_STATUS_INFLIGHT:
+            logger.info(f"Did not transmit invoice {invoice.pk} due to being in inflight state.")
+            return
+
+        if invoice.transmission_status != Invoice.TRANSMISSION_STATUS_PENDING and not allow_retransmission:
+            logger.info(f"Did not transmit invoice {invoice.pk} due to status being {invoice.transmission_status}.")
+            return
+
+        invoice.transmission_status = Invoice.TRANSMISSION_STATUS_INFLIGHT
+        invoice.transmission_date = now()
+        invoice.save(update_fields=["transmission_status", "transmission_date"])
+
+    providers = sorted([
+        provider
+        for provider, __ in transmission_providers.filter(type=invoice.transmission_type, active_in=sender)
+    ], key=lambda p: (-p.priority, p.identifier))
+
+    provider = None
+    for p in providers:
+        if p.is_available(sender, invoice.invoice_to_country, invoice.invoice_to_is_business):
+            provider = p
+            break
+
+    if not provider:
+        invoice.transmission_status = Invoice.TRANSMISSION_STATUS_FAILED
+        invoice.transmission_date = now()
+        invoice.save(update_fields=["transmission_status", "transmission_date"])
+        invoice.order.log_action(
+            "pretix.event.order.invoice.sending_failed",
+            data={
+                "full_invoice_no": invoice.full_invoice_no,
+                "transmission_provider": None,
+                "transmission_type": invoice.transmission_type,
+                "data": {
+                    "reason": "no_provider",
+                },
+            }
+        )
+        return
+
+    if invoice.order.testmode and not provider.testmode_supported:
+        invoice.transmission_status = Invoice.TRANSMISSION_STATUS_TESTMODE_IGNORED
+        invoice.transmission_date = now()
+        invoice.save(update_fields=["transmission_status", "transmission_date"])
+        invoice.order.log_action(
+            "pretix.event.order.invoice.testmode_ignored",
+            data={
+                "full_invoice_no": invoice.full_invoice_no,
+                "transmission_provider": None,
+                "transmission_type": invoice.transmission_type,
+            }
+        )
+        return
+
+    try:
+        provider.transmit(invoice)
+    except Exception as e:
+        logger.exception(f"Transmission of invoice {invoice.pk} failed with exception.")
+        invoice.transmission_status = Invoice.TRANSMISSION_STATUS_FAILED
+        invoice.transmission_date = now()
+        invoice.save(update_fields=["transmission_status", "transmission_date"])
+        invoice.order.log_action(
+            "pretix.event.order.invoice.sending_failed",
+            data={
+                "full_invoice_no": invoice.full_invoice_no,
+                "transmission_provider": None,
+                "transmission_type": invoice.transmission_type,
+                "data": {
+                    "reason": "exception",
+                    "exception": str(e),
+                },
+            }
+        )

src/pretix/base/services/mail.py

@@ -658,8 +658,55 @@ def mail_send_task(self, *args, to: List[str], subject: str, body: str, html: st
             raise SendMailException('Failed to send an email to {}.'.format(to))
         else:
             for i in invoices_sent:
-                i.sent_to_customer = now()
-                i.save(update_fields=['sent_to_customer'])
+                if i.transmission_type == "email":
+                    # Mark invoice as sent when it was sent to the requested address *either* at the time of invoice
+                    # creation *or* as of right now.
+                    expected_recipients = [
+                        (i.invoice_to_transmission_info or {}).get("transmission_email_address") or i.order.email,
+                    ]
+                    try:
+                        expected_recipients.append(order.invoice_address.transmission_info.get("transmission_email_address") or i.order.email)
+                    except InvoiceAddress.DoesNotExist:
+                        pass
+                    if not any(t in expected_recipients for t in to):
+                        continue
+                    if i.transmission_status != Invoice.TRANSMISSION_STATUS_COMPLETED:
+                        i.transmission_date = now()
+                        i.transmission_status = Invoice.TRANSMISSION_STATUS_COMPLETED
+                        i.transmission_provider = "email_pdf"
+                        i.transmission_info = {
+                            "sent": [
+                                {
+                                    "recipients": to,
+                                    "datetime": now().isoformat(),
+                                }
+                            ]
+                        }
+                        i.save(update_fields=[
+                            "transmission_date", "transmission_provider", "transmission_status",
+                            "transmission_info"
+                        ])
+                    elif i.transmission_provider == "email_pdf":
+                        i.transmission_info["sent"].append(
+                            {
+                                "recipients": to,
+                                "datetime": now().isoformat(),
+                            }
+                        )
+                        i.save(update_fields=[
+                            "transmission_info"
+                        ])
+                    i.order.log_action(
+                        "pretix.event.order.invoice.sent",
+                        data={
+                            "full_invoice_no": i.full_invoice_no,
+                            "transmission_provider": "email_pdf",
+                            "transmission_type": "email",
+                            "data": {
+                                "recipients": [to],
+                            },
+                        }
+                    )
 
 
 def mail_send(*args, **kwargs):

src/pretix/base/services/orders.py

@@ -84,6 +84,8 @@ from pretix.base.secrets import assign_ticket_secret
 from pretix.base.services import tickets
 from pretix.base.services.invoices import (
     generate_cancellation, generate_invoice, invoice_qualified,
+    invoice_transmission_separately, order_invoice_transmission_separately,
+    transmit_invoice,
 )
 from pretix.base.services.locking import (
     LOCK_TRUST_WINDOW, LockTimeoutException, lock_objects,
@@ -389,13 +391,19 @@ def approve_order(order, user=None, send_mail: bool=True, auth=None, force=False
     order_approved.send(order.event, order=order)
 
     invoice = order.invoices.last()  # Might be generated by plugin already
+
+    transmit_invoice_task = order_invoice_transmission_separately(order)
+    transmit_invoice_mail = not transmit_invoice_task and order.event.settings.invoice_email_attachment and order.email
+
     if order.event.settings.get('invoice_generate') == 'True' and invoice_qualified(order):
         if not invoice:
             invoice = generate_invoice(
                 order,
-                trigger_pdf=not order.event.settings.invoice_email_attachment or not order.email
+                # send_mail will trigger PDF generation later
+                trigger_pdf=not transmit_invoice_mail
             )
-            # send_mail will trigger PDF generation later
+            if transmit_invoice_task:
+                transmit_invoice.apply_async(args=(order.event_id, invoice.pk, False))
 
     if send_mail:
         with language(order.locale, order.event.settings.region):
@@ -423,7 +431,7 @@ def approve_order(order, user=None, send_mail: bool=True, auth=None, force=False
                         order.total == Decimal('0.00') or
                         order.valid_if_pending
                     ),
-                    invoices=[invoice] if invoice and order.event.settings.invoice_email_attachment else []
+                    invoices=[invoice] if invoice and transmit_invoice_mail else []
                 )
             except SendMailException:
                 logger.exception('Order approved email could not be sent')
@@ -619,6 +627,11 @@ def _cancel_order(order, user=None, send_mail: bool=True, api_token=None, device
 
         order.create_transactions()
 
+        transmit_invoices_task = [i for i in invoices if invoice_transmission_separately(i)]
+        transmit_invoices_mail = [i for i in invoices if i not in transmit_invoices_task and order.event.settings.invoice_email_attachment]
+        for i in transmit_invoices_task:
+            transmit_invoice.apply_async(args=(order.event_id, i.pk, False))
+
         if send_mail:
             with language(order.locale, order.event.settings.region):
                 email_template = order.event.settings.mail_text_order_canceled
@@ -628,7 +641,7 @@ def _cancel_order(order, user=None, send_mail: bool=True, api_token=None, device
                     order.send_mail(
                         email_subject, email_template, email_context,
                         'pretix.event.order.email.order_canceled', user,
-                        invoices=invoices if order.event.settings.invoice_email_attachment else []
+                        invoices=transmit_invoices_mail,
                     )
                 except SendMailException:
                     logger.exception('Order canceled email could not be sent')
@@ -1085,11 +1098,12 @@ def _create_order(event: Event, *, email: str, positions: List[CartPosition], no
 def _order_placed_email(event: Event, order: Order, email_template, subject_template,
                         log_entry: str, invoice, payments: List[OrderPayment], is_free=False):
     email_context = get_email_context(event=event, order=order, payments=payments)
+
     try:
         order.send_mail(
             subject_template, email_template, email_context,
             log_entry,
-            invoices=[invoice] if invoice and event.settings.invoice_email_attachment else [],
+            invoices=[invoice] if invoice else [],
             attach_tickets=True,
             attach_ical=event.settings.mail_attach_ical and (
                 not event.settings.mail_attach_ical_paid_only or
@@ -1278,6 +1292,9 @@ def _perform_order(event: Event, payment_requests: List[dict], position_ids: Lis
         not order.require_approval
     )
 
+    transmit_invoice_task = order_invoice_transmission_separately(order)
+    transmit_invoice_mail = not transmit_invoice_task and order.event.settings.invoice_email_attachment and order.email
+
     invoice = order.invoices.last()  # Might be generated by plugin already
     if not invoice and invoice_qualified(order):
         invoice_required = (
@@ -1291,9 +1308,11 @@ def _perform_order(event: Event, payment_requests: List[dict], position_ids: Lis
         if invoice_required:
             invoice = generate_invoice(
                 order,
-                trigger_pdf=not event.settings.invoice_email_attachment or not order.email
+                # send_mail will trigger PDF generation later
+                trigger_pdf=not transmit_invoice_mail
             )
-            # send_mail will trigger PDF generation later
+            if transmit_invoice_task:
+                transmit_invoice.apply_async(args=(event.pk, invoice.pk, False))
 
     if order.email:
         if order.require_approval:
@@ -1320,8 +1339,16 @@ def _perform_order(event: Event, payment_requests: List[dict], position_ids: Lis
             subject_attendees_template = event.settings.mail_subject_order_placed_attendee
 
         if sales_channel.identifier in event.settings.mail_sales_channel_placed_paid:
-            _order_placed_email(event, order, email_template, subject_template, log_entry, invoice, payment_objs,
-                                is_free=free_order_flow)
+            _order_placed_email(
+                event,
+                order,
+                email_template,
+                subject_template,
+                log_entry,
+                invoice if transmit_invoice_mail else None,
+                payment_objs,
+                is_free=free_order_flow
+            )
             if email_attendees:
                 for p in order.positions.all():
                     if p.addon_to_id is None and p.attendee_email and p.attendee_email != order.email:
@@ -2924,17 +2951,36 @@ class OrderChangeManager:
             if self.split_order:
                 self.split_order.create_transactions()
 
+        transmit_invoices_task = [i for i in self._invoices if invoice_transmission_separately(i)]
+        transmit_invoices_mail = [
+            i for i in self._invoices
+            if i not in transmit_invoices_task and self.event.settings.invoice_email_attachment and self.order.email
+        ]
+
+        if self.split_order:
+            split_invoices = list(self.split_order.invoices.all())
+            transmit_invoices_task += [
+                i for i in split_invoices if invoice_transmission_separately(i)
+            ]
+            split_transmit_invoices_mail = [
+                i for i in split_invoices
+                if i not in transmit_invoices_task and self.event.settings.invoice_email_attachment and self.order.email
+            ]
+
         if self.notify:
             notify_user_changed_order(
                 self.order, self.user, self.auth,
-                self._invoices if self.event.settings.invoice_email_attachment else []
+                transmit_invoices_mail,
             )
             if self.split_order:
                 notify_user_changed_order(
                     self.split_order, self.user, self.auth,
-                    list(self.split_order.invoices.all()) if self.event.settings.invoice_email_attachment else []
+                    split_transmit_invoices_mail,
                 )
 
+        for i in transmit_invoices_task:
+            transmit_invoice.apply_async(args=(self.event.pk, i.pk, False))
+
         order_changed.send(self.order.event, order=self.order)
 
     def _clear_tickets_cache(self):