Mail: Handle all rendering in mail.py, return values for log (#5895)

* Mail: Handle all rendering in mail.py, return values for log

* Apply suggestions from code review

src/pretix/base/invoicing/email.py

@@ -33,8 +33,7 @@ from pretix.base.invoicing.transmission import (
     transmission_types,
 )
 from pretix.base.models import Invoice, InvoiceAddress
-from pretix.base.services.mail import mail, render_mail
+from pretix.base.services.mail import mail
-from pretix.helpers.format import format_map
 
 
 @transmission_types.new()
@@ -134,9 +133,7 @@ class EmailTransmissionProvider(TransmissionProvider):
             subject = invoice.order.event.settings.get('mail_subject_order_invoice', as_type=LazyI18nString)
 
             # Do not set to completed because that is done by the email sending task
-            subject = format_map(subject, context)
+            outgoing_mail = mail(
-            email_content = render_mail(template, context)
-            mail(
                 [recipient],
                 subject,
                 template,
@@ -151,19 +148,10 @@ class EmailTransmissionProvider(TransmissionProvider):
                 plain_text_only=True,
                 no_order_links=True,
             )
-            invoice.order.log_action(
+            if outgoing_mail:
-                'pretix.event.order.email.invoice',
+                invoice.order.log_action(
-                user=None,
+                    'pretix.event.order.email.invoice',
-                auth=None,
+                    user=None,
-                data={
+                    auth=None,
-                    'subject': subject,
+                    data=outgoing_mail.log_data()
-                    'message': email_content,
+                )
-                    'position': None,
-                    'recipient': recipient,
-                    'invoices': [invoice.pk],
-                    'attach_tickets': False,
-                    'attach_ical': False,
-                    'attach_other_files': [],
-                    'attach_cached_files': [],
-                }
-            )

src/pretix/base/models/mail.py

@@ -220,3 +220,20 @@ class OutgoingMail(models.Model):
             error_log_action_type = 'pretix.email.error'
             log_target = None
         return log_target, error_log_action_type
+
+    def log_data(self):
+        return {
+            "subject": self.subject,
+            "message": self.body_plain,
+            "to": self.to,
+            "cc": self.cc,
+            "bcc": self.bcc,
+
+            "invoices": [i.pk for i in self.should_attach_invoices.all()],
+            "attach_tickets": self.should_attach_tickets,
+            "attach_ical": self.should_attach_ical,
+            "attach_other_files": self.should_attach_other_files,
+            "attach_cached_files": [cf.filename for cf in self.should_attach_cached_files.all()],
+
+            "position": self.orderposition.positionid if self.orderposition else None,
+        }

src/pretix/base/models/orders.py

@@ -87,7 +87,6 @@ from pretix.base.timemachine import time_machine_now
 
 from ...helpers import OF_SELF
 from ...helpers.countries import CachedCountries, FastCountryField
-from ...helpers.format import FormattedString, format_map
 from ...helpers.names import build_name
 from ...testutils.middleware import debugflags_var
 from ._transactions import (
@@ -1167,7 +1166,7 @@ class Order(LockModel, LoggedModel):
                          only be attached for this position and child positions, the link will only point to the
                          position and the attendee email will be used if available.
         """
-        from pretix.base.services.mail import mail, render_mail
+        from pretix.base.services.mail import mail
 
         if not self.email and not (position and position.attendee_email):
             return
@@ -1177,32 +1176,20 @@ class Order(LockModel, LoggedModel):
             if position and position.attendee_email:
                 recipient = position.attendee_email
 
-            email_content = render_mail(template, context)
+            outgoing_mail = mail(
-            if not isinstance(subject, FormattedString):
-                subject = format_map(subject, context)
-            mail(
                 recipient, subject, template, context,
                 self.event, self.locale, self, headers=headers, sender=sender,
                 invoices=invoices, attach_tickets=attach_tickets,
                 position=position, auto_email=auto_email, attach_ical=attach_ical,
                 attach_other_files=attach_other_files, attach_cached_files=attach_cached_files,
             )
-            self.log_action(
+            if outgoing_mail:
-                log_entry_type,
+                self.log_action(
-                user=user,
+                    log_entry_type,
-                auth=auth,
+                    user=user,
-                data={
+                    auth=auth,
-                    'subject': subject,
+                    data=outgoing_mail.log_data(),
-                    'message': email_content,
+                )
-                    'position': position.positionid if position else None,
-                    'recipient': recipient,
-                    'invoices': [i.pk for i in invoices] if invoices else [],
-                    'attach_tickets': attach_tickets,
-                    'attach_ical': attach_ical,
-                    'attach_other_files': attach_other_files,
-                    'attach_cached_files': [cf.filename for cf in attach_cached_files] if attach_cached_files else [],
-                }
-            )
 
     def resend_link(self, user=None, auth=None):
         with language(self.locale, self.event.settings.region):
@@ -2900,17 +2887,14 @@ class OrderPosition(AbstractPosition):
         :param attach_tickets: Attach tickets of this order, if they are existing and ready to download
         :param attach_ical: Attach relevant ICS files
         """
-        from pretix.base.services.mail import mail, render_mail
+        from pretix.base.services.mail import mail
 
         if not self.attendee_email:
             return
 
         with language(self.order.locale, self.order.event.settings.region):
             recipient = self.attendee_email
-            email_content = render_mail(template, context)
+            outgoing_mail = mail(
-            if not isinstance(subject, FormattedString):
-                subject = format_map(subject, context)
-            mail(
                 recipient, subject, template, context,
                 self.event, self.order.locale, order=self.order, headers=headers, sender=sender,
                 position=self,
@@ -2919,21 +2903,13 @@ class OrderPosition(AbstractPosition):
                 attach_ical=attach_ical,
                 attach_other_files=attach_other_files,
             )
-            self.order.log_action(
+            if outgoing_mail:
-                log_entry_type,
+                self.order.log_action(
-                user=user,
+                    log_entry_type,
-                auth=auth,
+                    user=user,
-                data={
+                    auth=auth,
-                    'subject': subject,
+                    data=outgoing_mail.log_data(),
-                    'message': email_content,
+                )
-                    'recipient': recipient,
-                    'invoices': [i.pk for i in invoices] if invoices else [],
-                    'attach_tickets': attach_tickets,
-                    'attach_ical': attach_ical,
-                    'attach_other_files': attach_other_files,
-                    'attach_cached_files': [],
-                }
-            )
 
     def resend_link(self, user=None, auth=None):
 

src/pretix/base/models/waitinglist.py

@@ -34,10 +34,9 @@ from phonenumber_field.modelfields import PhoneNumberField
 from pretix.base.email import get_email_context
 from pretix.base.i18n import language
 from pretix.base.models import User, Voucher
-from pretix.base.services.mail import mail, render_mail
+from pretix.base.services.mail import mail
 from pretix.helpers import OF_SELF
 
-from ...helpers.format import format_map
 from ...helpers.names import build_name
 from .base import LoggedModel
 from .event import Event, SubEvent
@@ -273,9 +272,7 @@ class WaitingListEntry(LoggedModel):
         with language(self.locale, self.event.settings.region):
             recipient = self.email
 
-            email_content = render_mail(template, context)
+            outgoing_mail = mail(
-            subject = format_map(subject, context)
-            mail(
                 recipient, subject, template, context,
                 self.event,
                 self.locale,
@@ -285,18 +282,13 @@ class WaitingListEntry(LoggedModel):
                 attach_other_files=attach_other_files,
                 attach_cached_files=attach_cached_files,
             )
-            self.log_action(
+            if outgoing_mail:
-                log_entry_type,
+                self.log_action(
-                user=user,
+                    log_entry_type,
-                auth=auth,
+                    user=user,
-                data={
+                    auth=auth,
-                    'subject': subject,
+                    data=outgoing_mail.log_data(),
-                    'message': email_content,
+                )
-                    'recipient': recipient,
-                    'attach_other_files': attach_other_files,
-                    'attach_cached_files': [cf.filename for cf in attach_cached_files] if attach_cached_files else [],
-                }
-            )
 
     @staticmethod
     def clean_itemvar(event, item, variation):

src/pretix/base/payment.py

@@ -1295,6 +1295,7 @@ class ManualPayment(BasePaymentProvider):
 
     def format_map(self, order, payment):
         return {
+            # Possible placeholder injection, we should make sure to never include user-controlled variables here
             'order': order.code,
             'amount': payment.amount,
             'currency': self.event.currency,

src/pretix/base/services/cancelevent.py

@@ -45,7 +45,6 @@ from pretix.base.services.tax import split_fee_for_taxes
 from pretix.base.templatetags.money import money_filter
 from pretix.celery_app import app
 from pretix.helpers import OF_SELF
-from pretix.helpers.format import format_map
 
 logger = logging.getLogger(__name__)
 
@@ -55,7 +54,7 @@ def _send_wle_mail(wle: WaitingListEntry, subject: LazyI18nString, message: Lazy
         email_context = get_email_context(event_or_subevent=subevent or wle.event, event=wle.event)
         mail(
             wle.email,
-            format_map(subject, email_context),
+            str(subject),
             message,
             email_context,
             wle.event,
@@ -73,9 +72,8 @@ def _send_mail(order: Order, subject: LazyI18nString, message: LazyI18nString, s
 
         email_context = get_email_context(event_or_subevent=subevent or order.event, refund_amount=refund_amount,
                                           order=order, position_or_address=ia, event=order.event)
-        real_subject = format_map(subject, email_context)
         order.send_mail(
-            real_subject, message, email_context,
+            subject, message, email_context,
             'pretix.event.order.email.event_canceled',
             user,
         )
@@ -85,14 +83,13 @@ def _send_mail(order: Order, subject: LazyI18nString, message: LazyI18nString, s
                 continue
 
             if p.addon_to_id is None and p.attendee_email and p.attendee_email != order.email:
-                real_subject = format_map(subject, email_context)
                 email_context = get_email_context(event_or_subevent=p.subevent or order.event,
                                                   event=order.event,
                                                   refund_amount=refund_amount,
                                                   position_or_address=p,
                                                   order=order, position=p)
                 order.send_mail(
-                    real_subject, message, email_context,
+                    subject, message, email_context,
                     'pretix.event.order.email.event_canceled',
                     position=p,
                     user=user

src/pretix/base/services/mail.py

@@ -149,13 +149,13 @@ def prefix_subject(settings_holder, subject, highlight=False):
     return subject
 
 
-def mail(email: Union[str, Sequence[str]], subject: str, template: Union[str, LazyI18nString],
+def mail(email: Union[str, Sequence[str]], subject: Union[str, FormattedString], template: Union[str, LazyI18nString],
          context: Dict[str, Any] = None, event: Event = None, locale: str = None, order: Order = None,
          position: OrderPosition = None, *, headers: dict = None, sender: str = None, organizer: Organizer = None,
          customer: Customer = None, invoices: Sequence = None, attach_tickets=False, auto_email=True, user=None,
          attach_ical=False, attach_cached_files: Sequence = None, attach_other_files: list=None,
          plain_text_only=False, no_order_links=False, cc: Sequence[str]=None, bcc: Sequence[str]=None,
-         sensitive: bool=False):
+         sensitive: bool=False) -> Optional[OutgoingMail]:
     """
     Sends out an email to a user. The mail will be sent synchronously or asynchronously depending on the installation.
 
@@ -335,14 +335,26 @@ def mail(email: Union[str, Sequence[str]], subject: str, template: Union[str, La
             should_attach_other_files=attach_other_files or [],
             sensitive=sensitive,
         )
+        m._prefetched_objects_cache = {}
         if invoices and not position:
             m.should_attach_invoices.add(*invoices)
+            # Hack: For logging, we'll later make a `should_attach_invoices.all()` call. We can prevent a useless
+            # DB query by filling the cache
+            m._prefetched_objects_cache[m.should_attach_invoices.prefetch_cache_name] = invoices
+        else:
+            m._prefetched_objects_cache[m.should_attach_invoices.prefetch_cache_name] = Invoice.objects.none()
         if attach_cached_files:
+            cf_list = []
             for cf in attach_cached_files:
                 if not isinstance(cf, CachedFile):
-                    m.should_attach_cached_files.add(CachedFile.objects.get(pk=cf))
+                    cf = CachedFile.objects.get(pk=cf)
-                else:
+                m.should_attach_cached_files.add(cf)
-                    m.should_attach_cached_files.add(cf)
+                cf_list.append(cf)
+            # Hack: For logging, we'll later make a `should_attach_cached_files.all()` call. We can prevent a useless
+            # DB query by filling the cache
+            m._prefetched_objects_cache[m.should_attach_cached_files.prefetch_cache_name] = cf_list
+        else:
+            m._prefetched_objects_cache[m.should_attach_cached_files.prefetch_cache_name] = CachedFile.objects.none()
 
         send_task = mail_send_task.si(
             outgoing_mail=m.id
@@ -364,6 +376,8 @@ def mail(email: Union[str, Sequence[str]], subject: str, template: Union[str, La
                 lambda: chain(*task_chain).apply_async()
             )
 
+    return m
+
 
 class CustomEmail(EmailMultiAlternatives):
     def _create_mime_attachment(self, content, mimetype):

src/pretix/base/services/vouchers.py

@@ -39,7 +39,7 @@ def vouchers_send(event: Event, vouchers: list, subject: str, message: str, reci
         with language(event.settings.locale):
             email_context = get_email_context(event=event, name=r.get('name') or '',
                                               voucher_list=[v.code for v in voucher_list])
-            mail(
+            outgoing_mail = mail(
                 r['email'],
                 subject,
                 LazyI18nString(message),
@@ -60,8 +60,8 @@ def vouchers_send(event: Event, vouchers: list, subject: str, message: str, reci
                     data={
                         'recipient': r['email'],
                         'name': r.get('name'),
-                        'subject': subject,
+                        'subject': outgoing_mail.subject,
-                        'message': message,
+                        'message': outgoing_mail.body_plain,
                     },
                     save=False
                 ))

src/pretix/base/shredder.py

@@ -363,7 +363,7 @@ class EmailAddressShredder(BaseDataShredder):
                     le.save(update_fields=['data', 'shredded'])
             else:
                 shred_log_fields(le, banlist=[
-                    'recipient', 'message', 'subject', 'full_mail', 'old_email', 'new_email'
+                    'recipient', 'message', 'subject', 'full_mail', 'old_email', 'new_email', 'bcc', 'cc',
                 ])
 
 

src/pretix/control/templates/pretixcontrol/order/mail_history.html

@@ -24,7 +24,9 @@
                         {% if log.display %}
                             <br/><span class="fa fa-fw fa-comment-o"></span> {{ log.display }}
                         {% endif %}
-                        {% if log.parsed_data.recipient %}
+                        {% if log.parsed_data.to %}
+                            <br/><span class="fa fa-fw fa-envelope-o"></span> {{ log.parsed_data.to|join:", " }}
+                        {% elif log.parsed_data.recipient %} {# legacy #}
                             <br/><span class="fa fa-fw fa-envelope-o"></span> {{ log.parsed_data.recipient }}
                         {% endif %}
                     </p>

src/pretix/control/views/orders.py

@@ -2421,9 +2421,9 @@ class OrderSendMail(EventPermissionRequiredMixin, OrderViewMixin, FormView):
         with language(order.locale, self.request.event.settings.region):
             email_context = get_email_context(event=order.event, order=order)
         email_template = LazyI18nString(form.cleaned_data['message'])
-        email_subject = format_map(str(form.cleaned_data['subject']), email_context)
-        email_content = render_mail(email_template, email_context)
         if self.request.POST.get('action') == 'preview':
+            email_subject = format_map(form.cleaned_data['subject'], email_context)
+            email_content = render_mail(email_template, email_context)
             self.preview_output = {
                 'subject': mark_safe(_('Subject: {subject}').format(
                     subject=prefix_subject(order.event, escape(email_subject), highlight=True)
@@ -2485,9 +2485,9 @@ class OrderPositionSendMail(OrderSendMail):
         with language(position.order.locale, self.request.event.settings.region):
             email_context = get_email_context(event=position.order.event, order=position.order, position=position)
         email_template = LazyI18nString(form.cleaned_data['message'])
-        email_subject = format_map(str(form.cleaned_data['subject']), email_context)
-        email_content = render_mail(email_template, email_context)
         if self.request.POST.get('action') == 'preview':
+            email_subject = format_map(str(form.cleaned_data['subject']), email_context)
+            email_content = render_mail(email_template, email_context)
             self.preview_output = {
                 'subject': mark_safe(_('Subject: {subject}').format(
                     subject=prefix_subject(position.order.event, escape(email_subject), highlight=True))

src/pretix/plugins/sendmail/tasks.py

@@ -38,13 +38,10 @@ from i18nfield.strings import LazyI18nString
 
 from pretix.base.email import get_email_context
 from pretix.base.i18n import language
-from pretix.base.models import (
+from pretix.base.models import Checkin, Event, InvoiceAddress, Order, User
-    CachedFile, Checkin, Event, InvoiceAddress, Order, User,
-)
 from pretix.base.services.mail import mail
 from pretix.base.services.tasks import ProfiledEventTask
 from pretix.celery_app import app
-from pretix.helpers.format import format_map
 
 
 def _chunks(lst, n):
@@ -64,7 +61,6 @@ def send_mails_to_orders(event: Event, user: int, subject: dict, message: dict,
     user = User.objects.get(pk=user) if user else None
     subject = LazyI18nString(subject)
     message = LazyI18nString(message)
-    attachments_for_log = [cf.filename for cf in CachedFile.objects.filter(pk__in=attachments)] if attachments else []
 
     def _send_to_order(o):
         send_to_order = recipients in ('both', 'orders')
@@ -122,7 +118,7 @@ def send_mails_to_orders(event: Event, user: int, subject: dict, message: dict,
 
                 with language(o.locale, event.settings.region):
                     email_context = get_email_context(event=event, order=o, invoice_address=ia, position=p)
-                    mail(
+                    outgoing_mail = mail(
                         p.attendee_email,
                         subject,
                         message,
@@ -135,25 +131,17 @@ def send_mails_to_orders(event: Event, user: int, subject: dict, message: dict,
                         attach_ical=attach_ical,
                         attach_cached_files=attachments
                     )
-                    o.log_action(
+                    if outgoing_mail:
-                        'pretix.plugins.sendmail.order.email.sent.attendee',
+                        o.log_action(
-                        user=user,
+                            'pretix.plugins.sendmail.order.email.sent.attendee',
-                        data={
+                            user=user,
-                            'position': p.positionid,
+                            data=outgoing_mail.log_data(),
-                            'subject': format_map(subject.localize(o.locale), email_context),
+                        )
-                            'message': format_map(message.localize(o.locale), email_context),
-                            'recipient': p.attendee_email,
-                            'attach_tickets': attach_tickets,
-                            'attach_ical': attach_ical,
-                            'attach_other_files': [],
-                            'attach_cached_files': attachments_for_log,
-                        }
-                    )
 
         if send_to_order and o.email:
             with language(o.locale, event.settings.region):
                 email_context = get_email_context(event=event, order=o, invoice_address=ia)
-                mail(
+                outgoing_mail = mail(
                     o.email,
                     subject,
                     message,
@@ -165,19 +153,12 @@ def send_mails_to_orders(event: Event, user: int, subject: dict, message: dict,
                     attach_ical=attach_ical,
                     attach_cached_files=attachments,
                 )
-                o.log_action(
+                if outgoing_mail:
-                    'pretix.plugins.sendmail.order.email.sent',
+                    o.log_action(
-                    user=user,
+                        'pretix.plugins.sendmail.order.email.sent',
-                    data={
+                        user=user,
-                        'subject': format_map(subject.localize(o.locale), email_context),
+                        data=outgoing_mail.log_data(),
-                        'message': format_map(message.localize(o.locale), email_context),
+                    )
-                        'recipient': o.email,
-                        'attach_tickets': attach_tickets,
-                        'attach_ical': attach_ical,
-                        'attach_other_files': [],
-                        'attach_cached_files': attachments_for_log,
-                    }
-                )
 
     for chunk in _chunks(objects, 1000):
         orders = Order.objects.filter(pk__in=chunk, event=event)