Invoice generation: Add way for renderers to signal they are not ready

src/pretix/base/invoicing/email.py

@@ -33,7 +33,8 @@ from pretix.base.invoicing.transmission import (
     transmission_types,
 )
 from pretix.base.models import Invoice, InvoiceAddress
-from pretix.base.services.mail import mail
+from pretix.base.services.mail import mail, render_mail
+from pretix.helpers.format import format_map
 
 
 @transmission_types.new()
@@ -133,7 +134,9 @@ class EmailTransmissionProvider(TransmissionProvider):
             subject = invoice.order.event.settings.get('mail_subject_order_invoice', as_type=LazyI18nString)
 
             # Do not set to completed because that is done by the email sending task
-            outgoing_mail = mail(
+            subject = format_map(subject, context)
+            email_content = render_mail(template, context)
+            mail(
                 [recipient],
                 subject,
                 template,
@@ -148,10 +151,19 @@ class EmailTransmissionProvider(TransmissionProvider):
                 plain_text_only=True,
                 no_order_links=True,
             )
-            if outgoing_mail:
-                invoice.order.log_action(
-                    'pretix.event.order.email.invoice',
-                    user=None,
-                    auth=None,
-                    data=outgoing_mail.log_data()
-                )
+            invoice.order.log_action(
+                'pretix.event.order.email.invoice',
+                user=None,
+                auth=None,
+                data={
+                    'subject': subject,
+                    'message': email_content,
+                    'position': None,
+                    'recipient': recipient,
+                    'invoices': [invoice.pk],
+                    'attach_tickets': False,
+                    'attach_ical': False,
+                    'attach_other_files': [],
+                    'attach_cached_files': [],
+                }
+            )

src/pretix/base/invoicing/pdf.py

@@ -148,6 +148,10 @@ class NumberedCanvas(Canvas):
         self.restoreState()
 
 
+class InvoiceNotReadyException(Exception):
+    pass
+
+
 class BaseInvoiceRenderer:
     """
     This is the base class for all invoice renderers.

src/pretix/base/models/mail.py

@@ -220,20 +220,3 @@ class OutgoingMail(models.Model):
             error_log_action_type = 'pretix.email.error'
             log_target = None
         return log_target, error_log_action_type
-
-    def log_data(self):
-        return {
-            "subject": self.subject,
-            "message": self.body_plain,
-            "to": self.to,
-            "cc": self.cc,
-            "bcc": self.bcc,
-
-            "invoices": [i.pk for i in self.should_attach_invoices.all()],
-            "attach_tickets": self.should_attach_tickets,
-            "attach_ical": self.should_attach_ical,
-            "attach_other_files": self.should_attach_other_files,
-            "attach_cached_files": [cf.filename for cf in self.should_attach_cached_files.all()],
-
-            "position": self.orderposition.positionid if self.orderposition else None,
-        }

src/pretix/base/models/orders.py

@@ -87,6 +87,7 @@ from pretix.base.timemachine import time_machine_now
 
 from ...helpers import OF_SELF
 from ...helpers.countries import CachedCountries, FastCountryField
+from ...helpers.format import FormattedString, format_map
 from ...helpers.names import build_name
 from ...testutils.middleware import debugflags_var
 from ._transactions import (
@@ -1166,7 +1167,7 @@ class Order(LockModel, LoggedModel):
                          only be attached for this position and child positions, the link will only point to the
                          position and the attendee email will be used if available.
         """
-        from pretix.base.services.mail import mail
+        from pretix.base.services.mail import mail, render_mail
 
         if not self.email and not (position and position.attendee_email):
             return
@@ -1176,20 +1177,32 @@ class Order(LockModel, LoggedModel):
             if position and position.attendee_email:
                 recipient = position.attendee_email
 
-            outgoing_mail = mail(
+            email_content = render_mail(template, context)
+            if not isinstance(subject, FormattedString):
+                subject = format_map(subject, context)
+            mail(
                 recipient, subject, template, context,
                 self.event, self.locale, self, headers=headers, sender=sender,
                 invoices=invoices, attach_tickets=attach_tickets,
                 position=position, auto_email=auto_email, attach_ical=attach_ical,
                 attach_other_files=attach_other_files, attach_cached_files=attach_cached_files,
             )
-            if outgoing_mail:
-                self.log_action(
-                    log_entry_type,
-                    user=user,
-                    auth=auth,
-                    data=outgoing_mail.log_data(),
-                )
+            self.log_action(
+                log_entry_type,
+                user=user,
+                auth=auth,
+                data={
+                    'subject': subject,
+                    'message': email_content,
+                    'position': position.positionid if position else None,
+                    'recipient': recipient,
+                    'invoices': [i.pk for i in invoices] if invoices else [],
+                    'attach_tickets': attach_tickets,
+                    'attach_ical': attach_ical,
+                    'attach_other_files': attach_other_files,
+                    'attach_cached_files': [cf.filename for cf in attach_cached_files] if attach_cached_files else [],
+                }
+            )
 
     def resend_link(self, user=None, auth=None):
         with language(self.locale, self.event.settings.region):
@@ -2887,14 +2900,17 @@ class OrderPosition(AbstractPosition):
         :param attach_tickets: Attach tickets of this order, if they are existing and ready to download
         :param attach_ical: Attach relevant ICS files
         """
-        from pretix.base.services.mail import mail
+        from pretix.base.services.mail import mail, render_mail
 
         if not self.attendee_email:
             return
 
         with language(self.order.locale, self.order.event.settings.region):
             recipient = self.attendee_email
-            outgoing_mail = mail(
+            email_content = render_mail(template, context)
+            if not isinstance(subject, FormattedString):
+                subject = format_map(subject, context)
+            mail(
                 recipient, subject, template, context,
                 self.event, self.order.locale, order=self.order, headers=headers, sender=sender,
                 position=self,
@@ -2903,13 +2919,21 @@ class OrderPosition(AbstractPosition):
                 attach_ical=attach_ical,
                 attach_other_files=attach_other_files,
             )
-            if outgoing_mail:
-                self.order.log_action(
-                    log_entry_type,
-                    user=user,
-                    auth=auth,
-                    data=outgoing_mail.log_data(),
-                )
+            self.order.log_action(
+                log_entry_type,
+                user=user,
+                auth=auth,
+                data={
+                    'subject': subject,
+                    'message': email_content,
+                    'recipient': recipient,
+                    'invoices': [i.pk for i in invoices] if invoices else [],
+                    'attach_tickets': attach_tickets,
+                    'attach_ical': attach_ical,
+                    'attach_other_files': attach_other_files,
+                    'attach_cached_files': [],
+                }
+            )
 
     def resend_link(self, user=None, auth=None):
 

src/pretix/base/models/waitinglist.py

@@ -34,9 +34,10 @@ from phonenumber_field.modelfields import PhoneNumberField
 from pretix.base.email import get_email_context
 from pretix.base.i18n import language
 from pretix.base.models import User, Voucher
-from pretix.base.services.mail import mail
+from pretix.base.services.mail import mail, render_mail
 from pretix.helpers import OF_SELF
 
+from ...helpers.format import format_map
 from ...helpers.names import build_name
 from .base import LoggedModel
 from .event import Event, SubEvent
@@ -271,7 +272,9 @@ class WaitingListEntry(LoggedModel):
         with language(self.locale, self.event.settings.region):
             recipient = self.email
 
-            outgoing_mail = mail(
+            email_content = render_mail(template, context)
+            subject = format_map(subject, context)
+            mail(
                 recipient, subject, template, context,
                 self.event,
                 self.locale,
@@ -281,13 +284,18 @@ class WaitingListEntry(LoggedModel):
                 attach_other_files=attach_other_files,
                 attach_cached_files=attach_cached_files,
             )
-            if outgoing_mail:
-                self.log_action(
-                    log_entry_type,
-                    user=user,
-                    auth=auth,
-                    data=outgoing_mail.log_data(),
-                )
+            self.log_action(
+                log_entry_type,
+                user=user,
+                auth=auth,
+                data={
+                    'subject': subject,
+                    'message': email_content,
+                    'recipient': recipient,
+                    'attach_other_files': attach_other_files,
+                    'attach_cached_files': [cf.filename for cf in attach_cached_files] if attach_cached_files else [],
+                }
+            )
 
     @staticmethod
     def clean_itemvar(event, item, variation):

src/pretix/base/payment.py

@@ -1295,7 +1295,6 @@ class ManualPayment(BasePaymentProvider):
 
     def format_map(self, order, payment):
         return {
-            # Possible placeholder injection, we should make sure to never include user-controlled variables here
             'order': order.code,
             'amount': payment.amount,
             'currency': self.event.currency,

src/pretix/base/services/cancelevent.py

@@ -45,6 +45,7 @@ from pretix.base.services.tax import split_fee_for_taxes
 from pretix.base.templatetags.money import money_filter
 from pretix.celery_app import app
 from pretix.helpers import OF_SELF
+from pretix.helpers.format import format_map
 
 logger = logging.getLogger(__name__)
 
@@ -54,7 +55,7 @@ def _send_wle_mail(wle: WaitingListEntry, subject: LazyI18nString, message: Lazy
         email_context = get_email_context(event_or_subevent=subevent or wle.event, event=wle.event)
         mail(
             wle.email,
-            str(subject),
+            format_map(subject, email_context),
             message,
             email_context,
             wle.event,
@@ -72,8 +73,9 @@ def _send_mail(order: Order, subject: LazyI18nString, message: LazyI18nString, s
 
         email_context = get_email_context(event_or_subevent=subevent or order.event, refund_amount=refund_amount,
                                           order=order, position_or_address=ia, event=order.event)
+        real_subject = format_map(subject, email_context)
         order.send_mail(
-            subject, message, email_context,
+            real_subject, message, email_context,
             'pretix.event.order.email.event_canceled',
             user,
         )
@@ -83,13 +85,14 @@ def _send_mail(order: Order, subject: LazyI18nString, message: LazyI18nString, s
                 continue
 
             if p.addon_to_id is None and p.attendee_email and p.attendee_email != order.email:
+                real_subject = format_map(subject, email_context)
                 email_context = get_email_context(event_or_subevent=p.subevent or order.event,
                                                   event=order.event,
                                                   refund_amount=refund_amount,
                                                   position_or_address=p,
                                                   order=order, position=p)
                 order.send_mail(
-                    subject, message, email_context,
+                    real_subject, message, email_context,
                     'pretix.event.order.email.event_canceled',
                     position=p,
                     user=user

src/pretix/base/services/invoices.py

@@ -51,6 +51,7 @@ from django_scopes import scope, scopes_disabled
 from i18nfield.strings import LazyI18nString
 
 from pretix.base.i18n import language
+from pretix.base.invoicing.pdf import InvoiceNotReadyException
 from pretix.base.invoicing.transmission import (
     get_transmission_types, transmission_providers,
 )
@@ -504,7 +505,7 @@ def generate_invoice(order: Order, trigger_pdf=True):
     return invoice
 
 
-@app.task(base=TransactionAwareTask)
+@app.task(base=TransactionAwareTask, throws=(InvoiceNotReadyException,))
 def invoice_pdf_task(invoice: int):
     with scopes_disabled():
         i = Invoice.objects.get(pk=invoice)

src/pretix/base/services/mail.py

@@ -149,13 +149,13 @@ def prefix_subject(settings_holder, subject, highlight=False):
     return subject
 
 
-def mail(email: Union[str, Sequence[str]], subject: Union[str, FormattedString], template: Union[str, LazyI18nString],
+def mail(email: Union[str, Sequence[str]], subject: str, template: Union[str, LazyI18nString],
          context: Dict[str, Any] = None, event: Event = None, locale: str = None, order: Order = None,
          position: OrderPosition = None, *, headers: dict = None, sender: str = None, organizer: Organizer = None,
          customer: Customer = None, invoices: Sequence = None, attach_tickets=False, auto_email=True, user=None,
          attach_ical=False, attach_cached_files: Sequence = None, attach_other_files: list=None,
          plain_text_only=False, no_order_links=False, cc: Sequence[str]=None, bcc: Sequence[str]=None,
-         sensitive: bool=False) -> Optional[OutgoingMail]:
+         sensitive: bool=False):
     """
     Sends out an email to a user. The mail will be sent synchronously or asynchronously depending on the installation.
 
@@ -335,26 +335,14 @@ def mail(email: Union[str, Sequence[str]], subject: Union[str, FormattedString],
             should_attach_other_files=attach_other_files or [],
             sensitive=sensitive,
         )
-        m._prefetched_objects_cache = {}
         if invoices and not position:
             m.should_attach_invoices.add(*invoices)
-            # Hack: For logging, we'll later make a `should_attach_invoices.all()` call. We can prevent a useless
-            # DB query by filling the cache
-            m._prefetched_objects_cache[m.should_attach_invoices.prefetch_cache_name] = invoices
-        else:
-            m._prefetched_objects_cache[m.should_attach_invoices.prefetch_cache_name] = Invoice.objects.none()
         if attach_cached_files:
-            cf_list = []
             for cf in attach_cached_files:
                 if not isinstance(cf, CachedFile):
-                    cf = CachedFile.objects.get(pk=cf)
-                m.should_attach_cached_files.add(cf)
-                cf_list.append(cf)
-            # Hack: For logging, we'll later make a `should_attach_cached_files.all()` call. We can prevent a useless
-            # DB query by filling the cache
-            m._prefetched_objects_cache[m.should_attach_cached_files.prefetch_cache_name] = cf_list
-        else:
-            m._prefetched_objects_cache[m.should_attach_cached_files.prefetch_cache_name] = CachedFile.objects.none()
+                    m.should_attach_cached_files.add(CachedFile.objects.get(pk=cf))
+                else:
+                    m.should_attach_cached_files.add(cf)
 
         send_task = mail_send_task.si(
             outgoing_mail=m.id
@@ -376,8 +364,6 @@ def mail(email: Union[str, Sequence[str]], subject: Union[str, FormattedString],
                 lambda: chain(*task_chain).apply_async()
             )
 
-    return m
-
 
 class CustomEmail(EmailMultiAlternatives):
     def _create_mime_attachment(self, content, mimetype):

src/pretix/base/services/vouchers.py

@@ -39,7 +39,7 @@ def vouchers_send(event: Event, vouchers: list, subject: str, message: str, reci
         with language(event.settings.locale):
             email_context = get_email_context(event=event, name=r.get('name') or '',
                                               voucher_list=[v.code for v in voucher_list])
-            outgoing_mail = mail(
+            mail(
                 r['email'],
                 subject,
                 LazyI18nString(message),
@@ -60,8 +60,8 @@ def vouchers_send(event: Event, vouchers: list, subject: str, message: str, reci
                     data={
                         'recipient': r['email'],
                         'name': r.get('name'),
-                        'subject': outgoing_mail.subject,
-                        'message': outgoing_mail.body_plain,
+                        'subject': subject,
+                        'message': message,
                     },
                     save=False
                 ))

src/pretix/base/shredder.py

@@ -363,7 +363,7 @@ class EmailAddressShredder(BaseDataShredder):
                     le.save(update_fields=['data', 'shredded'])
             else:
                 shred_log_fields(le, banlist=[
-                    'recipient', 'message', 'subject', 'full_mail', 'old_email', 'new_email', 'bcc', 'cc',
+                    'recipient', 'message', 'subject', 'full_mail', 'old_email', 'new_email'
                 ])
 
 

src/pretix/control/templates/pretixcontrol/order/mail_history.html

@@ -24,9 +24,7 @@
                         {% if log.display %}
                             <br/><span class="fa fa-fw fa-comment-o"></span> {{ log.display }}
                         {% endif %}
-                        {% if log.parsed_data.to %}
-                            <br/><span class="fa fa-fw fa-envelope-o"></span> {{ log.parsed_data.to|join:", " }}
-                        {% elif log.parsed_data.recipient %} {# legacy #}
+                        {% if log.parsed_data.recipient %}
                             <br/><span class="fa fa-fw fa-envelope-o"></span> {{ log.parsed_data.recipient }}
                         {% endif %}
                     </p>

src/pretix/control/views/orders.py

@@ -2413,9 +2413,9 @@ class OrderSendMail(EventPermissionRequiredMixin, OrderViewMixin, FormView):
         with language(order.locale, self.request.event.settings.region):
             email_context = get_email_context(event=order.event, order=order)
         email_template = LazyI18nString(form.cleaned_data['message'])
+        email_subject = format_map(str(form.cleaned_data['subject']), email_context)
+        email_content = render_mail(email_template, email_context)
         if self.request.POST.get('action') == 'preview':
-            email_subject = format_map(str(form.cleaned_data['subject']), email_context)
-            email_content = render_mail(email_template, email_context)
             self.preview_output = {
                 'subject': mark_safe(_('Subject: {subject}').format(
                     subject=prefix_subject(order.event, escape(email_subject), highlight=True)
@@ -2477,9 +2477,9 @@ class OrderPositionSendMail(OrderSendMail):
         with language(position.order.locale, self.request.event.settings.region):
             email_context = get_email_context(event=position.order.event, order=position.order, position=position)
         email_template = LazyI18nString(form.cleaned_data['message'])
+        email_subject = format_map(str(form.cleaned_data['subject']), email_context)
+        email_content = render_mail(email_template, email_context)
         if self.request.POST.get('action') == 'preview':
-            email_subject = format_map(str(form.cleaned_data['subject']), email_context)
-            email_content = render_mail(email_template, email_context)
             self.preview_output = {
                 'subject': mark_safe(_('Subject: {subject}').format(
                     subject=prefix_subject(position.order.event, escape(email_subject), highlight=True))

src/pretix/plugins/sendmail/tasks.py

@@ -38,10 +38,13 @@ from i18nfield.strings import LazyI18nString
 
 from pretix.base.email import get_email_context
 from pretix.base.i18n import language
-from pretix.base.models import Checkin, Event, InvoiceAddress, Order, User
+from pretix.base.models import (
+    CachedFile, Checkin, Event, InvoiceAddress, Order, User,
+)
 from pretix.base.services.mail import mail
 from pretix.base.services.tasks import ProfiledEventTask
 from pretix.celery_app import app
+from pretix.helpers.format import format_map
 
 
 def _chunks(lst, n):
@@ -61,6 +64,7 @@ def send_mails_to_orders(event: Event, user: int, subject: dict, message: dict,
     user = User.objects.get(pk=user) if user else None
     subject = LazyI18nString(subject)
     message = LazyI18nString(message)
+    attachments_for_log = [cf.filename for cf in CachedFile.objects.filter(pk__in=attachments)] if attachments else []
 
     def _send_to_order(o):
         send_to_order = recipients in ('both', 'orders')
@@ -118,7 +122,7 @@ def send_mails_to_orders(event: Event, user: int, subject: dict, message: dict,
 
                 with language(o.locale, event.settings.region):
                     email_context = get_email_context(event=event, order=o, invoice_address=ia, position=p)
-                    outgoing_mail = mail(
+                    mail(
                         p.attendee_email,
                         subject,
                         message,
@@ -131,17 +135,25 @@ def send_mails_to_orders(event: Event, user: int, subject: dict, message: dict,
                         attach_ical=attach_ical,
                         attach_cached_files=attachments
                     )
-                    if outgoing_mail:
-                        o.log_action(
-                            'pretix.plugins.sendmail.order.email.sent.attendee',
-                            user=user,
-                            data=outgoing_mail.log_data(),
-                        )
+                    o.log_action(
+                        'pretix.plugins.sendmail.order.email.sent.attendee',
+                        user=user,
+                        data={
+                            'position': p.positionid,
+                            'subject': format_map(subject.localize(o.locale), email_context),
+                            'message': format_map(message.localize(o.locale), email_context),
+                            'recipient': p.attendee_email,
+                            'attach_tickets': attach_tickets,
+                            'attach_ical': attach_ical,
+                            'attach_other_files': [],
+                            'attach_cached_files': attachments_for_log,
+                        }
+                    )
 
         if send_to_order and o.email:
             with language(o.locale, event.settings.region):
                 email_context = get_email_context(event=event, order=o, invoice_address=ia)
-                outgoing_mail = mail(
+                mail(
                     o.email,
                     subject,
                     message,
@@ -153,12 +165,19 @@ def send_mails_to_orders(event: Event, user: int, subject: dict, message: dict,
                     attach_ical=attach_ical,
                     attach_cached_files=attachments,
                 )
-                if outgoing_mail:
-                    o.log_action(
-                        'pretix.plugins.sendmail.order.email.sent',
-                        user=user,
-                        data=outgoing_mail.log_data(),
-                    )
+                o.log_action(
+                    'pretix.plugins.sendmail.order.email.sent',
+                    user=user,
+                    data={
+                        'subject': format_map(subject.localize(o.locale), email_context),
+                        'message': format_map(message.localize(o.locale), email_context),
+                        'recipient': o.email,
+                        'attach_tickets': attach_tickets,
+                        'attach_ical': attach_ical,
+                        'attach_other_files': [],
+                        'attach_cached_files': attachments_for_log,
+                    }
+                )
 
     for chunk in _chunks(objects, 1000):
         orders = Order.objects.filter(pk__in=chunk, event=event)

src/pretix/presale/templates/pretixpresale/event/resend_link.html

@@ -1,14 +1,14 @@
 {% extends "pretixpresale/event/base.html" %}
 {% load i18n %}
 {% load bootstrap3 %}
-{% block title %}{% trans "Resend order links" %}{% endblock %}
+{% block title %}{% trans "Resend order link" %}{% endblock %}
 {% block custom_header %}
     {{ block.super }}
     <meta name="robots" content="noindex, nofollow">
 {% endblock %}
 {% block content %}
     <h2>
-        {% trans "Resend order links" %}
+        {% trans "Resend order link" %}
     </h2>
     <p>
         {% blocktrans trimmed %}

src/pretix/presale/views/order.py

@@ -1510,7 +1510,10 @@ class OrderChangeMixin:
                             'max_count': iao.max_count,
                             'iao': iao,
                             'items': [i for i in items if not i.require_voucher],
-                            'items_missing': {k: v for k, v in current_addon_products_missing.items() if v},
+                            'items_missing': {
+                                k: v for k, v in current_addon_products_missing.items()
+                                if v and k[0].category_id == iao.addon_category_id
+                            },
                         })
 
         return positions