diff --git a/src/pretix/api/serializers/organizer.py b/src/pretix/api/serializers/organizer.py index ce3ed39b72..6d8f6743b9 100644 --- a/src/pretix/api/serializers/organizer.py +++ b/src/pretix/api/serializers/organizer.py @@ -49,7 +49,7 @@ from pretix.base.plugins import ( PLUGIN_LEVEL_EVENT, PLUGIN_LEVEL_EVENT_ORGANIZER_HYBRID, PLUGIN_LEVEL_ORGANIZER, ) -from pretix.base.services.mail import SendMailException, mail +from pretix.base.services.mail import mail from pretix.base.settings import validate_organizer_settings from pretix.helpers.urls import build_absolute_uri as build_global_uri from pretix.multidomain.urlreverse import build_absolute_uri @@ -363,24 +363,21 @@ class TeamInviteSerializer(serializers.ModelSerializer): ) def _send_invite(self, instance): - try: - mail( - instance.email, - _('pretix account invitation'), - 'pretixcontrol/email/invitation.txt', - { - 'user': self, - 'organizer': self.context['organizer'].name, - 'team': instance.team.name, - 'url': build_global_uri('control:auth.invite', kwargs={ - 'token': instance.token - }) - }, - event=None, - locale=get_language_without_region() # TODO: expose? - ) - except SendMailException: - pass # Already logged + mail( + instance.email, + _('pretix account invitation'), + 'pretixcontrol/email/invitation.txt', + { + 'user': self, + 'organizer': self.context['organizer'].name, + 'team': instance.team.name, + 'url': build_global_uri('control:auth.invite', kwargs={ + 'token': instance.token + }) + }, + event=None, + locale=get_language_without_region() # TODO: expose? + ) def create(self, validated_data): if 'email' in validated_data: diff --git a/src/pretix/api/views/order.py b/src/pretix/api/views/order.py index 2b5a7f82bd..8ec7828763 100644 --- a/src/pretix/api/views/order.py +++ b/src/pretix/api/views/order.py @@ -90,7 +90,6 @@ from pretix.base.services.invoices import ( generate_cancellation, generate_invoice, invoice_pdf, invoice_qualified, regenerate_invoice, transmit_invoice, ) -from pretix.base.services.mail import SendMailException from pretix.base.services.orders import ( OrderChangeManager, OrderError, _order_placed_email, _order_placed_email_attendee, approve_order, cancel_order, deny_order, @@ -439,8 +438,6 @@ class EventOrderViewSet(OrderViewSetMixin, viewsets.ModelViewSet): return Response({'detail': str(e)}, status=status.HTTP_400_BAD_REQUEST) except PaymentException as e: return Response({'detail': str(e)}, status=status.HTTP_400_BAD_REQUEST) - except SendMailException: - pass return self.retrieve(request, [], **kwargs) return Response( @@ -634,10 +631,7 @@ class EventOrderViewSet(OrderViewSetMixin, viewsets.ModelViewSet): order = self.get_object() if not order.email: return Response({'detail': 'There is no email address associated with this order.'}, status=status.HTTP_400_BAD_REQUEST) - try: - order.resend_link(user=self.request.user, auth=self.request.auth) - except SendMailException: - return Response({'detail': _('There was an error sending the mail. Please try again later.')}, status=status.HTTP_503_SERVICE_UNAVAILABLE) + order.resend_link(user=self.request.user, auth=self.request.auth) return Response( status=status.HTTP_204_NO_CONTENT @@ -1616,8 +1610,6 @@ class PaymentViewSet(CreateModelMixin, viewsets.ReadOnlyModelViewSet): ) except Quota.QuotaExceededException: pass - except SendMailException: - pass serializer = OrderPaymentSerializer(r, context=serializer.context) @@ -1655,8 +1647,6 @@ class PaymentViewSet(CreateModelMixin, viewsets.ReadOnlyModelViewSet): return Response({'detail': str(e)}, status=status.HTTP_400_BAD_REQUEST) except PaymentException as e: return Response({'detail': str(e)}, status=status.HTTP_400_BAD_REQUEST) - except SendMailException: - pass return self.retrieve(request, [], **kwargs) @action(detail=True, methods=['POST']) diff --git a/src/pretix/base/invoicing/email.py b/src/pretix/base/invoicing/email.py index d3064af778..155e7906a0 100644 --- a/src/pretix/base/invoicing/email.py +++ b/src/pretix/base/invoicing/email.py @@ -33,7 +33,7 @@ from pretix.base.invoicing.transmission import ( transmission_types, ) from pretix.base.models import Invoice, InvoiceAddress -from pretix.base.services.mail import SendMailException, mail, render_mail +from pretix.base.services.mail import mail, render_mail from pretix.helpers.format import format_map @@ -133,41 +133,37 @@ class EmailTransmissionProvider(TransmissionProvider): template = invoice.order.event.settings.get('mail_text_order_invoice', as_type=LazyI18nString) subject = invoice.order.event.settings.get('mail_subject_order_invoice', as_type=LazyI18nString) - try: - # Do not set to completed because that is done by the email sending task - subject = format_map(subject, context) - email_content = render_mail(template, context) - mail( - [recipient], - subject, - template, - context=context, - event=invoice.order.event, - locale=invoice.order.locale, - order=invoice.order, - invoices=[invoice], - attach_tickets=False, - auto_email=True, - attach_ical=False, - plain_text_only=True, - no_order_links=True, - ) - except SendMailException: - raise - else: - invoice.order.log_action( - 'pretix.event.order.email.invoice', - user=None, - auth=None, - data={ - 'subject': subject, - 'message': email_content, - 'position': None, - 'recipient': recipient, - 'invoices': [invoice.pk], - 'attach_tickets': False, - 'attach_ical': False, - 'attach_other_files': [], - 'attach_cached_files': [], - } - ) + # Do not set to completed because that is done by the email sending task + subject = format_map(subject, context) + email_content = render_mail(template, context) + mail( + [recipient], + subject, + template, + context=context, + event=invoice.order.event, + locale=invoice.order.locale, + order=invoice.order, + invoices=[invoice], + attach_tickets=False, + auto_email=True, + attach_ical=False, + plain_text_only=True, + no_order_links=True, + ) + invoice.order.log_action( + 'pretix.event.order.email.invoice', + user=None, + auth=None, + data={ + 'subject': subject, + 'message': email_content, + 'position': None, + 'recipient': recipient, + 'invoices': [invoice.pk], + 'attach_tickets': False, + 'attach_ical': False, + 'attach_other_files': [], + 'attach_cached_files': [], + } + ) diff --git a/src/pretix/base/migrations/0297_outgoingmail.py b/src/pretix/base/migrations/0297_outgoingmail.py new file mode 100644 index 0000000000..034d77ac4e --- /dev/null +++ b/src/pretix/base/migrations/0297_outgoingmail.py @@ -0,0 +1,120 @@ +# Generated by Django 4.2.26 on 2026-01-22 13:44 +import uuid + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + +import pretix.base.models.mail + + +class Migration(migrations.Migration): + + dependencies = [ + ("pretixbase", "0296_invoice_invoice_from_state"), + ] + + operations = [ + migrations.CreateModel( + name="OutgoingMail", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False + ), + ), + ("guid", models.UUIDField(db_index=True, default=uuid.uuid4)), + ("status", models.CharField(default="queued", max_length=200)), + ("created", models.DateTimeField(auto_now_add=True)), + ("sent", models.DateTimeField(blank=True, null=True)), + ("inflight_since", models.DateTimeField(blank=True, null=True)), + ("retry_after", models.DateTimeField(blank=True, null=True)), + ("error", models.TextField(null=True)), + ("error_detail", models.TextField(null=True)), + ("sensitive", models.BooleanField(default=False)), + ("subject", models.TextField()), + ("body_plain", models.TextField()), + ("body_html", models.TextField(null=True)), + ("sender", models.CharField(max_length=500)), + ("headers", models.JSONField(default=dict)), + ("to", models.JSONField(default=list)), + ("cc", models.JSONField(default=list)), + ("bcc", models.JSONField(default=list)), + ("recipient_count", models.IntegerField()), + ("should_attach_tickets", models.BooleanField(default=False)), + ("should_attach_ical", models.BooleanField(default=False)), + ("should_attach_other_files", models.JSONField(default=list)), + ("actual_attachments", models.JSONField(default=list)), + ( + "customer", + models.ForeignKey( + null=True, + on_delete=pretix.base.models.mail.CASCADE_IF_QUEUED, + related_name="outgoing_mails", + to="pretixbase.customer", + ), + ), + ( + "event", + models.ForeignKey( + null=True, + on_delete=pretix.base.models.mail.CASCADE_IF_QUEUED, + related_name="outgoing_mails", + to="pretixbase.event", + ), + ), + ( + "order", + models.ForeignKey( + null=True, + on_delete=pretix.base.models.mail.CASCADE_IF_QUEUED, + related_name="outgoing_mails", + to="pretixbase.order", + ), + ), + ( + "orderposition", + models.ForeignKey( + null=True, + on_delete=pretix.base.models.mail.CASCADE_IF_QUEUED, + related_name="outgoing_mails", + to="pretixbase.orderposition", + ), + ), + ( + "organizer", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="outgoing_mails", + to="pretixbase.organizer", + ), + ), + ( + "should_attach_cached_files", + models.ManyToManyField( + related_name="outgoing_mails", to="pretixbase.cachedfile" + ), + ), + ( + "should_attach_invoices", + models.ManyToManyField( + related_name="outgoing_mails", to="pretixbase.invoice" + ), + ), + ( + "user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="outgoing_mails", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "ordering": ("-created",), + }, + ), + ] diff --git a/src/pretix/base/models/__init__.py b/src/pretix/base/models/__init__.py index 372fc8287c..3657cfb4c2 100644 --- a/src/pretix/base/models/__init__.py +++ b/src/pretix/base/models/__init__.py @@ -41,6 +41,7 @@ from .items import ( itempicture_upload_to, ) from .log import LogEntry +from .mail import OutgoingMail from .media import ReusableMedium from .memberships import Membership, MembershipType from .notifications import NotificationSetting diff --git a/src/pretix/base/models/auth.py b/src/pretix/base/models/auth.py index 44ff3587d6..0d2ba97a27 100644 --- a/src/pretix/base/models/auth.py +++ b/src/pretix/base/models/auth.py @@ -334,27 +334,24 @@ class User(AbstractBaseUser, PermissionsMixin, LoggingMixin): return self.email def send_security_notice(self, messages, email=None): - from pretix.base.services.mail import SendMailException, mail + from pretix.base.services.mail import mail - try: - with language(self.locale): - msg = '- ' + '\n- '.join(str(m) for m in messages) + with language(self.locale): + msg = '- ' + '\n- '.join(str(m) for m in messages) - mail( - email or self.email, - _('Account information changed'), - 'pretixcontrol/email/security_notice.txt', - { - 'user': self, - 'messages': msg, - 'url': build_absolute_uri('control:user.settings') - }, - event=None, - user=self, - locale=self.locale - ) - except SendMailException: - pass # Already logged + mail( + email or self.email, + _('Account information changed'), + 'pretixcontrol/email/security_notice.txt', + { + 'user': self, + 'messages': msg, + 'url': build_absolute_uri('control:user.settings') + }, + event=None, + user=self, + locale=self.locale + ) def send_confirmation_code(self, session, reason, email=None, state=None): """ diff --git a/src/pretix/base/models/customers.py b/src/pretix/base/models/customers.py index d7d34db98d..51fbd7490b 100644 --- a/src/pretix/base/models/customers.py +++ b/src/pretix/base/models/customers.py @@ -293,6 +293,7 @@ class Customer(LoggedModel): locale=self.locale, customer=self, organizer=self.organizer, + sensitive=True, ) def usable_gift_cards(self, used_cards=[]): diff --git a/src/pretix/base/models/mail.py b/src/pretix/base/models/mail.py new file mode 100644 index 0000000000..2128c28592 --- /dev/null +++ b/src/pretix/base/models/mail.py @@ -0,0 +1,222 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-today pretix GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +import uuid + +from django.core.mail import get_connection +from django.db import models +from django.utils.translation import gettext_lazy as _ +from django_scopes import scope, scopes_disabled + + +def CASCADE_IF_QUEUED(collector, field, sub_objs, using): + # If the email is still queued and the thing it is related to vanishes, the email can vanish as well + cascade_objs = [ + o for o in sub_objs if o.status == OutgoingMail.STATUS_QUEUED + ] + if cascade_objs: + models.CASCADE(collector, field, cascade_objs, using) + + # In all other cases, set to NULL to keep the email on record + models.SET_NULL(collector, field, [o for o in sub_objs if o not in cascade_objs], using) + + +class OutgoingMail(models.Model): + STATUS_QUEUED = "queued" + STATUS_WITHHELD = "withheld" + STATUS_INFLIGHT = "inflight" + STATUS_AWAITING_RETRY = "awaiting_retry" + STATUS_FAILED = "failed" + STATUS_SENT = "sent" + STATUS_BOUNCED = "bounced" + STATUS_ABORTED = "aborted" + STATUS_CHOICES = ( + (STATUS_QUEUED, _("queued")), + (STATUS_INFLIGHT, _("being sent")), + (STATUS_AWAITING_RETRY, _("awaiting retry")), + (STATUS_WITHHELD, _("withheld")), # for plugin use + (STATUS_FAILED, _("failed")), + (STATUS_ABORTED, _("aborted")), + (STATUS_SENT, _("sent")), + (STATUS_BOUNCED, _("bounced")), # for plugin use + ) + STATUS_LIST_ABORTABLE = { + STATUS_QUEUED, + STATUS_WITHHELD, + STATUS_AWAITING_RETRY, + } + STATUS_LIST_RETRYABLE = { + STATUS_FAILED, + STATUS_WITHHELD, + } + + # The GUID is a globally unique ID for the email added to a header of the email for later tracing + # in bug reports etc. We could theoretically also use this as a basis for the Message-ID header, but + # we currently don't since we are unsure if some intermediary SMTP servers have opinions on setting + # their own Message-ID headers. + guid = models.UUIDField(db_index=True, default=uuid.uuid4) + + status = models.CharField(max_length=200, choices=STATUS_CHOICES, default=STATUS_QUEUED) + created = models.DateTimeField(auto_now_add=True) + + # sent will be the time the email was sent or the email failed + sent = models.DateTimeField(null=True, blank=True) + + inflight_since = models.DateTimeField(null=True, blank=True) + retry_after = models.DateTimeField(null=True, blank=True) + + error = models.TextField(null=True, blank=True) + error_detail = models.TextField(null=True, blank=True) + + # There is a conflict here between the different purposes of the model. As a system administrator, + # one wants *all* emails to be persisted as long as possible to debug issues. This means that if + # e.g. the event or order is deleted, we want SET_NULL behavior. However, in that case, the email + # would be an "orphan" forever and there's no way to remove the personal information. + # We try to find a middle-ground with the following behaviour: + # - The email is always deleted if the entire organizer or user is deleted + # - The email is always deleted if it has not yet been sent + # - The email is kept in all other cases + # This is only an acceptable trade-off since emails are stored for a short period only, and because + # orders and customers are never deleted during normal operation. If we ever make this a long-term + # storage / email archive, we'd need to find another way to make sure personal information is removed + # if personal information of orders etc is removed. + organizer = models.ForeignKey( + 'pretixbase.Organizer', + on_delete=models.CASCADE, + related_name='outgoing_mails', + null=True, blank=True, + ) + event = models.ForeignKey( + 'pretixbase.Event', + on_delete=CASCADE_IF_QUEUED, + related_name='outgoing_mails', + null=True, blank=True, + ) + order = models.ForeignKey( + 'pretixbase.Order', + on_delete=CASCADE_IF_QUEUED, + related_name='outgoing_mails', + null=True, blank=True, + ) + orderposition = models.ForeignKey( + 'pretixbase.OrderPosition', + on_delete=CASCADE_IF_QUEUED, + related_name='outgoing_mails', + null=True, blank=True, + ) + customer = models.ForeignKey( + 'pretixbase.Customer', + on_delete=CASCADE_IF_QUEUED, + related_name='outgoing_mails', + null=True, blank=True, + ) + user = models.ForeignKey( + 'pretixbase.User', + on_delete=models.CASCADE, + related_name='outgoing_mails', + null=True, blank=True, + ) + + sensitive = models.BooleanField(default=False) + subject = models.TextField() + body_plain = models.TextField() + body_html = models.TextField(null=True) + sender = models.CharField(max_length=500) + headers = models.JSONField(default=dict) + to = models.JSONField(default=list) + cc = models.JSONField(default=list) + bcc = models.JSONField(default=list) + recipient_count = models.IntegerField() + + # We don't store the actual invoices, tickets or calendar invites, so if the email is re-sent at a later time, a + # newer version of the files might be used. We accept that risk to save on storage and also because the new + # version might actually be more useful. + should_attach_invoices = models.ManyToManyField( + 'pretixbase.Invoice', + related_name='outgoing_mails' + ) + should_attach_tickets = models.BooleanField(default=False) + should_attach_ical = models.BooleanField(default=False) + + # clean_cached_files makes sure not to delete these as long as the email is in a retryable state + should_attach_cached_files = models.ManyToManyField( + 'pretixbase.CachedFile', + related_name='outgoing_mails', + ) + + # This is used to send files stored in settings. In most cases, these aren't short-lived and should still be there + # if the email is sent. Otherwise, they will be skipped. We accept that risk. + should_attach_other_files = models.JSONField(default=list) + + # [{name, type size}] of the attachments we actually setn + actual_attachments = models.JSONField(default=list) + + class Meta: + ordering = ('-created',) + + def get_mail_backend(self): + if self.event: + return self.event.get_mail_backend() + elif self.organizer: + return self.organizer.get_mail_backend() + else: + return get_connection(fail_silently=False) + + def scope_manager(self): + if self.organizer: + return scope(organizer=self.organizer) # noqa + else: + return scopes_disabled() # noqa + + @property + def is_failed(self): + return self.status in ( + OutgoingMail.STATUS_FAILED, + OutgoingMail.STATUS_AWAITING_RETRY, + OutgoingMail.STATUS_BOUNCED, + ) + + def save(self, *args, **kwargs): + if self.orderposition_id and not self.order_id: + self.order = self.orderposition.order + if self.order_id and not self.event_id: + self.event = self.order.event + if self.event_id and not self.organizer_id: + self.organizer = self.event.organizer + if self.customer_id and not self.organizer_id: + self.organizer = self.customer.organizer + self.recipient_count = len(self.to) + len(self.cc) + len(self.bcc) + super().save(*args, **kwargs) + + def log_parameters(self): + if self.order: + error_log_action_type = 'pretix.event.order.email.error' + log_target = self.order + elif self.customer: + error_log_action_type = 'pretix.customer.email.error' + log_target = self.customer + elif self.user: + error_log_action_type = 'pretix.user.email.error' + log_target = self.user + else: + error_log_action_type = 'pretix.email.error' + log_target = None + return log_target, error_log_action_type diff --git a/src/pretix/base/models/orders.py b/src/pretix/base/models/orders.py index c9c4da8719..6ae0e027af 100644 --- a/src/pretix/base/models/orders.py +++ b/src/pretix/base/models/orders.py @@ -1167,9 +1167,7 @@ class Order(LockModel, LoggedModel): only be attached for this position and child positions, the link will only point to the position and the attendee email will be used if available. """ - from pretix.base.services.mail import ( - SendMailException, mail, render_mail, - ) + from pretix.base.services.mail import mail, render_mail if not self.email and not (position and position.attendee_email): return @@ -1179,35 +1177,31 @@ class Order(LockModel, LoggedModel): if position and position.attendee_email: recipient = position.attendee_email - try: - email_content = render_mail(template, context) - subject = format_map(subject, context) - mail( - recipient, subject, template, context, - self.event, self.locale, self, headers=headers, sender=sender, - invoices=invoices, attach_tickets=attach_tickets, - position=position, auto_email=auto_email, attach_ical=attach_ical, - attach_other_files=attach_other_files, attach_cached_files=attach_cached_files, - ) - except SendMailException: - raise - else: - self.log_action( - log_entry_type, - user=user, - auth=auth, - data={ - 'subject': subject, - 'message': email_content, - 'position': position.positionid if position else None, - 'recipient': recipient, - 'invoices': [i.pk for i in invoices] if invoices else [], - 'attach_tickets': attach_tickets, - 'attach_ical': attach_ical, - 'attach_other_files': attach_other_files, - 'attach_cached_files': [cf.filename for cf in attach_cached_files] if attach_cached_files else [], - } - ) + email_content = render_mail(template, context) + subject = format_map(subject, context) + mail( + recipient, subject, template, context, + self.event, self.locale, self, headers=headers, sender=sender, + invoices=invoices, attach_tickets=attach_tickets, + position=position, auto_email=auto_email, attach_ical=attach_ical, + attach_other_files=attach_other_files, attach_cached_files=attach_cached_files, + ) + self.log_action( + log_entry_type, + user=user, + auth=auth, + data={ + 'subject': subject, + 'message': email_content, + 'position': position.positionid if position else None, + 'recipient': recipient, + 'invoices': [i.pk for i in invoices] if invoices else [], + 'attach_tickets': attach_tickets, + 'attach_ical': attach_ical, + 'attach_other_files': attach_other_files, + 'attach_cached_files': [cf.filename for cf in attach_cached_files] if attach_cached_files else [], + } + ) def resend_link(self, user=None, auth=None): with language(self.locale, self.event.settings.region): @@ -2024,40 +2018,30 @@ class OrderPayment(models.Model): transmit_invoice.apply_async(args=(self.order.event_id, invoice.pk, False)) def _send_paid_mail_attendee(self, position, user): - from pretix.base.services.mail import SendMailException - with language(self.order.locale, self.order.event.settings.region): email_template = self.order.event.settings.mail_text_order_paid_attendee email_subject = self.order.event.settings.mail_subject_order_paid_attendee email_context = get_email_context(event=self.order.event, order=self.order, position=position) - try: - position.send_mail( - email_subject, email_template, email_context, - 'pretix.event.order.email.order_paid', user, - invoices=[], - attach_tickets=True, - attach_ical=self.order.event.settings.mail_attach_ical - ) - except SendMailException: - logger.exception('Order paid email could not be sent') + position.send_mail( + email_subject, email_template, email_context, + 'pretix.event.order.email.order_paid', user, + invoices=[], + attach_tickets=True, + attach_ical=self.order.event.settings.mail_attach_ical + ) def _send_paid_mail(self, invoice, user, mail_text): - from pretix.base.services.mail import SendMailException - with language(self.order.locale, self.order.event.settings.region): email_template = self.order.event.settings.mail_text_order_paid email_subject = self.order.event.settings.mail_subject_order_paid email_context = get_email_context(event=self.order.event, order=self.order, payment_info=mail_text) - try: - self.order.send_mail( - email_subject, email_template, email_context, - 'pretix.event.order.email.order_paid', user, - invoices=[invoice] if invoice else [], - attach_tickets=True, - attach_ical=self.order.event.settings.mail_attach_ical - ) - except SendMailException: - logger.exception('Order paid email could not be sent') + self.order.send_mail( + email_subject, email_template, email_context, + 'pretix.event.order.email.order_paid', user, + invoices=[invoice] if invoice else [], + attach_tickets=True, + attach_ical=self.order.event.settings.mail_attach_ical + ) @property def refunded_amount(self): @@ -2915,45 +2899,39 @@ class OrderPosition(AbstractPosition): :param attach_tickets: Attach tickets of this order, if they are existing and ready to download :param attach_ical: Attach relevant ICS files """ - from pretix.base.services.mail import ( - SendMailException, mail, render_mail, - ) + from pretix.base.services.mail import mail, render_mail if not self.attendee_email: return with language(self.order.locale, self.order.event.settings.region): recipient = self.attendee_email - try: - email_content = render_mail(template, context) - subject = format_map(subject, context) - mail( - recipient, subject, template, context, - self.event, self.order.locale, order=self.order, headers=headers, sender=sender, - position=self, - invoices=invoices, - attach_tickets=attach_tickets, - attach_ical=attach_ical, - attach_other_files=attach_other_files, - ) - except SendMailException: - raise - else: - self.order.log_action( - log_entry_type, - user=user, - auth=auth, - data={ - 'subject': subject, - 'message': email_content, - 'recipient': recipient, - 'invoices': [i.pk for i in invoices] if invoices else [], - 'attach_tickets': attach_tickets, - 'attach_ical': attach_ical, - 'attach_other_files': attach_other_files, - 'attach_cached_files': [], - } - ) + email_content = render_mail(template, context) + subject = format_map(subject, context) + mail( + recipient, subject, template, context, + self.event, self.order.locale, order=self.order, headers=headers, sender=sender, + position=self, + invoices=invoices, + attach_tickets=attach_tickets, + attach_ical=attach_ical, + attach_other_files=attach_other_files, + ) + self.order.log_action( + log_entry_type, + user=user, + auth=auth, + data={ + 'subject': subject, + 'message': email_content, + 'recipient': recipient, + 'invoices': [i.pk for i in invoices] if invoices else [], + 'attach_tickets': attach_tickets, + 'attach_ical': attach_ical, + 'attach_other_files': attach_other_files, + 'attach_cached_files': [], + } + ) def resend_link(self, user=None, auth=None): diff --git a/src/pretix/base/models/waitinglist.py b/src/pretix/base/models/waitinglist.py index 68d46cac41..6ae14591c2 100644 --- a/src/pretix/base/models/waitinglist.py +++ b/src/pretix/base/models/waitinglist.py @@ -34,7 +34,7 @@ from phonenumber_field.modelfields import PhoneNumberField from pretix.base.email import get_email_context from pretix.base.i18n import language from pretix.base.models import User, Voucher -from pretix.base.services.mail import SendMailException, mail, render_mail +from pretix.base.services.mail import mail, render_mail from pretix.helpers import OF_SELF from ...helpers.format import format_map @@ -272,34 +272,30 @@ class WaitingListEntry(LoggedModel): with language(self.locale, self.event.settings.region): recipient = self.email - try: - email_content = render_mail(template, context) - subject = format_map(subject, context) - mail( - recipient, subject, template, context, - self.event, - self.locale, - headers=headers, - sender=sender, - auto_email=auto_email, - attach_other_files=attach_other_files, - attach_cached_files=attach_cached_files, - ) - except SendMailException: - raise - else: - self.log_action( - log_entry_type, - user=user, - auth=auth, - data={ - 'subject': subject, - 'message': email_content, - 'recipient': recipient, - 'attach_other_files': attach_other_files, - 'attach_cached_files': [cf.filename for cf in attach_cached_files] if attach_cached_files else [], - } - ) + email_content = render_mail(template, context) + subject = format_map(subject, context) + mail( + recipient, subject, template, context, + self.event, + self.locale, + headers=headers, + sender=sender, + auto_email=auto_email, + attach_other_files=attach_other_files, + attach_cached_files=attach_cached_files, + ) + self.log_action( + log_entry_type, + user=user, + auth=auth, + data={ + 'subject': subject, + 'message': email_content, + 'recipient': recipient, + 'attach_other_files': attach_other_files, + 'attach_cached_files': [cf.filename for cf in attach_cached_files] if attach_cached_files else [], + } + ) @staticmethod def clean_itemvar(event, item, variation): diff --git a/src/pretix/base/services/cancelevent.py b/src/pretix/base/services/cancelevent.py index 694ea35987..012f01b85d 100644 --- a/src/pretix/base/services/cancelevent.py +++ b/src/pretix/base/services/cancelevent.py @@ -36,7 +36,7 @@ from pretix.base.models import ( SubEvent, TaxRule, User, WaitingListEntry, ) from pretix.base.services.locking import LockTimeoutException -from pretix.base.services.mail import SendMailException, mail +from pretix.base.services.mail import mail from pretix.base.services.orders import ( OrderChangeManager, OrderError, _cancel_order, _try_auto_refund, ) @@ -53,17 +53,14 @@ logger = logging.getLogger(__name__) def _send_wle_mail(wle: WaitingListEntry, subject: LazyI18nString, message: LazyI18nString, subevent: SubEvent): with language(wle.locale, wle.event.settings.region): email_context = get_email_context(event_or_subevent=subevent or wle.event, event=wle.event) - try: - mail( - wle.email, - format_map(subject, email_context), - message, - email_context, - wle.event, - locale=wle.locale - ) - except SendMailException: - logger.exception('Waiting list canceled email could not be sent') + mail( + wle.email, + format_map(subject, email_context), + message, + email_context, + wle.event, + locale=wle.locale + ) def _send_mail(order: Order, subject: LazyI18nString, message: LazyI18nString, subevent: SubEvent, @@ -77,14 +74,11 @@ def _send_mail(order: Order, subject: LazyI18nString, message: LazyI18nString, s email_context = get_email_context(event_or_subevent=subevent or order.event, refund_amount=refund_amount, order=order, position_or_address=ia, event=order.event) real_subject = format_map(subject, email_context) - try: - order.send_mail( - real_subject, message, email_context, - 'pretix.event.order.email.event_canceled', - user, - ) - except SendMailException: - logger.exception('Order canceled email could not be sent') + order.send_mail( + real_subject, message, email_context, + 'pretix.event.order.email.event_canceled', + user, + ) for p in positions: if subevent and p.subevent_id != subevent.id: @@ -97,15 +91,12 @@ def _send_mail(order: Order, subject: LazyI18nString, message: LazyI18nString, s refund_amount=refund_amount, position_or_address=p, order=order, position=p) - try: - order.send_mail( - real_subject, message, email_context, - 'pretix.event.order.email.event_canceled', - position=p, - user=user - ) - except SendMailException: - logger.exception('Order canceled email could not be sent to attendee') + order.send_mail( + real_subject, message, email_context, + 'pretix.event.order.email.event_canceled', + position=p, + user=user + ) @app.task(base=ProfiledEventTask, bind=True, max_retries=5, default_retry_delay=1, throws=(OrderError,)) diff --git a/src/pretix/base/services/cleanup.py b/src/pretix/base/services/cleanup.py index 6667b2c3f5..41376b920f 100644 --- a/src/pretix/base/services/cleanup.py +++ b/src/pretix/base/services/cleanup.py @@ -23,11 +23,12 @@ from datetime import timedelta from django.conf import settings from django.core.management import call_command +from django.db.models import Exists, OuterRef from django.dispatch import receiver from django.utils.timezone import now from django_scopes import scopes_disabled -from pretix.base.models import CachedCombinedTicket, CachedTicket +from pretix.base.models import CachedCombinedTicket, CachedTicket, OutgoingMail from pretix.base.models.customers import CustomerSSOGrant from ..models import CachedFile, CartPosition, InvoiceAddress @@ -49,7 +50,18 @@ def clean_cart_positions(sender, **kwargs): @receiver(signal=periodic_task) @scopes_disabled() def clean_cached_files(sender, **kwargs): - for cf in CachedFile.objects.filter(expires__isnull=False, expires__lt=now()): + has_queued_email = Exists( + OutgoingMail.objects.filter( + should_attach_cached_files__pk=OuterRef("pk"), + status__in=( + OutgoingMail.STATUS_QUEUED, + OutgoingMail.STATUS_INFLIGHT, + OutgoingMail.STATUS_AWAITING_RETRY, + OutgoingMail.STATUS_FAILED, + ), + ) + ) + for cf in CachedFile.objects.filter(expires__isnull=False, expires__lt=now()).exclude(has_queued_email): cf.delete() diff --git a/src/pretix/base/services/mail.py b/src/pretix/base/services/mail.py index a21eec4194..9d0943f9de 100644 --- a/src/pretix/base/services/mail.py +++ b/src/pretix/base/services/mail.py @@ -39,10 +39,12 @@ import mimetypes import os import re import smtplib +import uuid import warnings +from datetime import timedelta from email.mime.image import MIMEImage from email.utils import formataddr -from typing import Any, Dict, List, Sequence, Union +from typing import Any, Dict, List, Optional, Sequence, Union from urllib.parse import urljoin, urlparse from zoneinfo import ZoneInfo @@ -51,16 +53,16 @@ from celery import chain from celery.exceptions import MaxRetriesExceededError from django.conf import settings from django.core.files.storage import default_storage -from django.core.mail import ( - EmailMultiAlternatives, SafeMIMEMultipart, get_connection, -) +from django.core.mail import EmailMultiAlternatives, SafeMIMEMultipart from django.core.mail.message import SafeMIMEText -from django.db import transaction +from django.db import connection, transaction +from django.db.models import Q +from django.dispatch import receiver from django.template.loader import get_template from django.utils.html import escape from django.utils.timezone import now, override from django.utils.translation import gettext as _, pgettext -from django_scopes import scope, scopes_disabled +from django_scopes import scopes_disabled from i18nfield.strings import LazyI18nString from text_unidecode import unidecode @@ -70,11 +72,15 @@ from pretix.base.models import ( CachedFile, Customer, Event, Invoice, InvoiceAddress, Order, OrderPosition, Organizer, User, ) +from pretix.base.models.mail import OutgoingMail from pretix.base.services.invoices import invoice_pdf_task from pretix.base.services.tasks import TransactionAwareTask from pretix.base.services.tickets import get_tickets_for_order -from pretix.base.signals import email_filter, global_email_filter +from pretix.base.signals import ( + email_filter, global_email_filter, periodic_task, +) from pretix.celery_app import app +from pretix.helpers import OF_SELF from pretix.helpers.format import SafeFormatter, format_map from pretix.helpers.hierarkey import clean_filename from pretix.multidomain.urlreverse import build_absolute_uri @@ -91,9 +97,18 @@ class TolerantDict(dict): class SendMailException(Exception): + """ + Deprecated, not thrown any more. + """ pass +class WithholdMailException(Exception): + def __init__(self, error, error_detail): + self.error = error + self.error_detail = error_detail + + def clean_sender_name(sender_name: str) -> str: # Even though we try to properly escape sender names, some characters seem to cause problems when the escaping # fails due to some forwardings, etc. @@ -137,7 +152,8 @@ def mail(email: Union[str, Sequence[str]], subject: str, template: Union[str, La position: OrderPosition = None, *, headers: dict = None, sender: str = None, organizer: Organizer = None, customer: Customer = None, invoices: Sequence = None, attach_tickets=False, auto_email=True, user=None, attach_ical=False, attach_cached_files: Sequence = None, attach_other_files: list=None, - plain_text_only=False, no_order_links=False, cc: Sequence[str]=None, bcc: Sequence[str]=None): + plain_text_only=False, no_order_links=False, cc: Sequence[str]=None, bcc: Sequence[str]=None, + sensitive: bool=False): """ Sends out an email to a user. The mail will be sent synchronously or asynchronously depending on the installation. @@ -193,6 +209,9 @@ def mail(email: Union[str, Sequence[str]], subject: str, template: Union[str, La only allowed to use together with ``plain_text_only`` since HTML renderers add their own links. + :param sensitive: If set to ``True``, the email content will not be shown as part of log entries, used e.g. for + password resets. Bcc will also not be used. + :raises MailOrderException: on obvious, immediate failures. Not raising an exception does not necessarily mean that the email has been sent, just that it has been queued by the email backend. """ @@ -202,162 +221,123 @@ def mail(email: Union[str, Sequence[str]], subject: str, template: Union[str, La if no_order_links and not plain_text_only: raise ValueError('If you set no_order_links, you also need to set plain_text_only.') + settings_holder = event or organizer + headers = headers or {} + guid = uuid.uuid4() if auto_email: headers['X-Auto-Response-Suppress'] = 'OOF, NRN, AutoReply, RN' headers['Auto-Submitted'] = 'auto-generated' headers.setdefault('X-Mailer', 'pretix') + headers.setdefault('X-PX-Correlation', str(guid)) + + bcc = list(bcc or []) + if settings_holder and settings_holder.settings.mail_bcc and not sensitive: + for bcc_mail in settings_holder.settings.mail_bcc.split(','): + bcc.append(bcc_mail.strip()) + + if (settings_holder + and settings_holder.settings.mail_from in (settings.DEFAULT_FROM_EMAIL, settings.MAIL_FROM_ORGANIZERS) + and settings_holder.settings.contact_mail and not headers.get('Reply-To')): + headers['Reply-To'] = settings_holder.settings.contact_mail + + if settings_holder: + timezone = settings_holder.timezone + elif user: + timezone = ZoneInfo(user.timezone) + else: + timezone = ZoneInfo(settings.TIME_ZONE) + + if event and attach_tickets and not event.settings.mail_attach_tickets: + attach_tickets = False with language(locale): if isinstance(context, dict) and order: - try: - context.update({ - 'invoice_name': order.invoice_address.name, - 'invoice_company': order.invoice_address.company - }) - except InvoiceAddress.DoesNotExist: - context.update({ - 'invoice_name': '', - 'invoice_company': '' - }) - renderer = ClassicMailRenderer(None, organizer) + _autoextend_context(context, order) + + # Build raw content body_plain = render_mail(template, context, placeholder_mode=SafeFormatter.MODE_RICH_TO_PLAIN) - subject = str(subject).format_map(TolerantDict(context)) - sender = ( - sender or - (event.settings.get('mail_from') if event else None) or - (organizer.settings.get('mail_from') if organizer else None) or - settings.MAIL_FROM - ) - if event: - sender_name = clean_sender_name(event.settings.mail_from_name or str(event.name)) - sender = formataddr((sender_name, sender)) - elif organizer: - sender_name = clean_sender_name(organizer.settings.mail_from_name or str(organizer.name)) - sender = formataddr((sender_name, sender)) - else: - sender = formataddr((clean_sender_name(settings.PRETIX_INSTANCE_NAME), sender)) - - subject = raw_subject = str(subject).replace('\n', ' ').replace('\r', '')[:900] - signature = "" - - bcc = list(bcc or []) - - settings_holder = event or organizer - - if event: - timezone = event.timezone - elif user: - timezone = ZoneInfo(user.timezone) - elif organizer: - timezone = organizer.timezone - else: - timezone = ZoneInfo(settings.TIME_ZONE) - if settings_holder: - if settings_holder.settings.mail_bcc: - for bcc_mail in settings_holder.settings.mail_bcc.split(','): - bcc.append(bcc_mail.strip()) - - if settings_holder.settings.mail_from in (settings.DEFAULT_FROM_EMAIL, settings.MAIL_FROM_ORGANIZERS) \ - and settings_holder.settings.contact_mail and not headers.get('Reply-To'): - headers['Reply-To'] = settings_holder.settings.contact_mail - - subject = prefix_subject(settings_holder, subject) - - body_plain += "\r\n\r\n-- \r\n" - signature = str(settings_holder.settings.get('mail_text_signature')) - if signature: - signature = signature.format(event=event.name if event else '') - body_plain += signature - body_plain += "\r\n\r\n-- \r\n" - - if event: - renderer = event.get_html_mail_renderer() - - if order and order.testmode: - subject = "[TESTMODE] " + subject - - if order and position and not no_order_links: - body_plain += _( - "You are receiving this email because someone placed an order for {event} for you." - ).format(event=event.name) - body_plain += "\r\n" - body_plain += _( - "You can view your order details at the following URL:\n{orderurl}." - ).replace("\n", "\r\n").format( - event=event.name, orderurl=build_absolute_uri( - order.event, 'presale:event.order.position', kwargs={ - 'order': order.code, - 'secret': position.web_secret, - 'position': position.positionid, - } - ) - ) - elif order and not no_order_links: - body_plain += _( - "You are receiving this email because you placed an order for {event}." - ).format(event=event.name) - body_plain += "\r\n" - body_plain += _( - "You can view your order details at the following URL:\n{orderurl}." - ).replace("\n", "\r\n").format( - event=event.name, orderurl=build_absolute_uri( - order.event, 'presale:event.order.open', kwargs={ - 'order': order.code, - 'secret': order.secret, - 'hash': order.email_confirm_secret() - } - ) - ) - body_plain += "\r\n" - - with override(timezone): - try: - content_plain = render_mail(template, context, placeholder_mode=None) - if plain_text_only: - body_html = None - elif 'context' in inspect.signature(renderer.render).parameters: - body_html = renderer.render(content_plain, signature, raw_subject, order, position, context) - elif 'position' in inspect.signature(renderer.render).parameters: - # Backwards compatibility - warnings.warn('Email renderer called without context argument because context argument is not ' - 'supported.', - DeprecationWarning) - body_html = renderer.render(content_plain, signature, raw_subject, order, position) - else: - # Backwards compatibility - warnings.warn('Email renderer called without position argument because position argument is not ' - 'supported.', - DeprecationWarning) - body_html = renderer.render(content_plain, signature, raw_subject, order) - except: - logger.exception('Could not render HTML body') - body_html = None + else: + signature = "" + # Build full plain-text body + body_plain = _wrap_plain_body(body_plain, signature, event, order, position, no_order_links) body_plain = format_map(body_plain, context, mode=SafeFormatter.MODE_RICH_TO_PLAIN) - send_task = mail_send_task.si( - to=[email] if isinstance(email, str) else list(email), - cc=cc, - bcc=bcc, + # Build subject + subject = str(subject).format_map(TolerantDict(context)) + subject = raw_subject = subject.replace('\n', ' ').replace('\r', '')[:900] + if settings_holder: + subject = prefix_subject(settings_holder, subject) + if (order and order.testmode) or (not order and event and event.testmode): + subject = "[TESTMODE] " + subject + + # Build sender + sender = _full_sender(sender, event, organizer) + + # Build HTML body + if plain_text_only: + body_html = None + else: + if event: + renderer = event.get_html_mail_renderer() + else: + renderer = ClassicMailRenderer(None, organizer) + + with override(timezone): + content_plain = render_mail(template, context, placeholder_mode=None) + try: + if 'context' in inspect.signature(renderer.render).parameters: + body_html = renderer.render(content_plain, signature, raw_subject, order, position, context) + elif 'position' in inspect.signature(renderer.render).parameters: + # Backwards compatibility + warnings.warn('Email renderer called without context argument because context argument is not ' + 'supported.', + DeprecationWarning) + body_html = renderer.render(content_plain, signature, raw_subject, order, position) + else: + # Backwards compatibility + warnings.warn('Email renderer called without position argument because position argument is not ' + 'supported.', + DeprecationWarning) + body_html = renderer.render(content_plain, signature, raw_subject, order) + except: + logger.exception('Could not render HTML body') + body_html = None + + m = OutgoingMail.objects.create( + organizer=organizer, + event=event, + order=order, + orderposition=position, + customer=customer, + user=user, + to=[email.lower()] if isinstance(email, str) else [e.lower() for e in email], + cc=[e.lower() for e in cc] if cc else [], + bcc=[e.lower() for e in bcc] if bcc else [], subject=subject, - body=body_plain, - html=body_html, + body_plain=body_plain, + body_html=body_html, sender=sender, - event=event.id if event else None, headers=headers, - invoices=[i.pk for i in invoices] if invoices and not position else [], - order=order.pk if order else None, - position=position.pk if position else None, - attach_tickets=attach_tickets, - attach_ical=attach_ical, - user=user.pk if user else None, - organizer=organizer.pk if organizer else None, - customer=customer.pk if customer else None, - attach_cached_files=[(cf.id if isinstance(cf, CachedFile) else cf) for cf in attach_cached_files] if attach_cached_files else [], - attach_other_files=attach_other_files, + should_attach_tickets=attach_tickets, + should_attach_ical=attach_ical, + should_attach_other_files=attach_other_files or [], + sensitive=sensitive, + ) + if invoices and not position: + m.should_attach_invoices.add(*invoices) + if attach_cached_files: + for cf in attach_cached_files: + if not isinstance(cf, CachedFile): + m.should_attach_cached_files.add(CachedFile.objects.get(pk=cf)) + else: + m.should_attach_cached_files.add(cf) + + send_task = mail_send_task.si( + outgoing_mail=m.id ) if invoices: @@ -392,194 +372,250 @@ class CustomEmail(EmailMultiAlternatives): @app.task(base=TransactionAwareTask, bind=True, acks_late=True) -def mail_send_task(self, *args, to: List[str], subject: str, body: str, html: str, sender: str, - event: int = None, position: int = None, headers: dict = None, cc: List[str] = None, bcc: List[str] = None, - invoices: List[int] = None, order: int = None, attach_tickets=False, user=None, - organizer=None, customer=None, attach_ical=False, attach_cached_files: List[int] = None, - attach_other_files: List[str] = None) -> bool: - email = CustomEmail(subject, body, sender, to=to, cc=cc, bcc=bcc, headers=headers) - if html is not None: +def mail_send_task(self, **kwargs) -> bool: + if "outgoing_mail" in kwargs: + outgoing_mail = kwargs.get("outgoing_mail") + elif "to" in kwargs: + # May only occur while upgrading from pretix versions before OutgoingMail when celery tasks are still in-queue + # during the upgrade. Can be removed after 2026.2.x is released, and then the signature can be changed to + # mail_send_task(self, *, outgoing_mail) + mail_send(**kwargs) + return + else: + raise ValueError("Unknown arguments") + + with transaction.atomic(): + try: + outgoing_mail = OutgoingMail.objects.select_for_update(of=OF_SELF).get(pk=outgoing_mail) + except OutgoingMail.DoesNotExist: + logger.info(f"Ignoring job for non existing email {outgoing_mail.guid}") + return False + if outgoing_mail.status == OutgoingMail.STATUS_INFLIGHT: + logger.info(f"Ignoring job for inflight email {outgoing_mail.guid}") + return False + elif outgoing_mail.status not in (OutgoingMail.STATUS_AWAITING_RETRY, OutgoingMail.STATUS_QUEUED): + logger.info(f"Ignoring job for email {outgoing_mail.guid} in final state {outgoing_mail.status}") + return False + outgoing_mail.status = OutgoingMail.STATUS_INFLIGHT + outgoing_mail.inflight_since = now() + outgoing_mail.save(update_fields=["status", "inflight_since"]) + + headers = dict(outgoing_mail.headers) + headers.setdefault('X-PX-Correlation', str(outgoing_mail.guid)) + email = CustomEmail( + subject=outgoing_mail.subject, + body=outgoing_mail.body_plain, + from_email=outgoing_mail.sender, + to=outgoing_mail.to, + cc=outgoing_mail.cc, + bcc=outgoing_mail.bcc, + headers=headers, + ) + + # Rewrite all tags from real URLs or data URLs to inline attachments referred to by content ID + if outgoing_mail.body_html is not None: html_message = SafeMIMEMultipart(_subtype='related', encoding=settings.DEFAULT_CHARSET) - html_with_cid, cid_images = replace_images_with_cid_paths(html) + html_with_cid, cid_images = replace_images_with_cid_paths(outgoing_mail.body_html) html_message.attach(SafeMIMEText(html_with_cid, 'html', settings.DEFAULT_CHARSET)) attach_cid_images(html_message, cid_images, verify_ssl=True) email.attach_alternative(html_message, "multipart/related") - log_target = None + log_target, error_log_action_type = outgoing_mail.log_parameters() + invoices_attached = [] - if user: - user = User.objects.get(pk=user) - error_log_action_type = 'pretix.user.email.error' - log_target = user + with outgoing_mail.scope_manager(): + # Attach tickets + if outgoing_mail.should_attach_tickets and outgoing_mail.order: + with language(outgoing_mail.order.locale, outgoing_mail.event.settings.region): + args = [] + attach_size = 0 + for name, ct in get_tickets_for_order(outgoing_mail.order, base_position=outgoing_mail.orderposition): + try: + content = ct.file.read() + args.append((name, content, ct.type)) + attach_size += len(content) + except Exception: + # This sometimes fails e.g. with FileNotFoundError. We haven't been able to figure out + # why (probably some race condition with ticket cache invalidation?), so retry later. + try: + self.retry(max_retries=5, countdown=60) + except MaxRetriesExceededError: + # Well then, something is really wrong, let's send it without attachment before we + # don't send at all + logger.exception(f'Could not attach tickets to email {outgoing_mail.guid}') + pass - if event: - with scopes_disabled(): - event = Event.objects.get(id=event) - organizer = event.organizer - backend = event.get_mail_backend() - cm = lambda: scope(organizer=event.organizer) # noqa - elif organizer: - with scopes_disabled(): - organizer = Organizer.objects.get(id=organizer) - backend = organizer.get_mail_backend() - cm = lambda: scope(organizer=organizer) # noqa - else: - backend = get_connection(fail_silently=False) - cm = lambda: scopes_disabled() # noqa - - with cm(): - if customer: - customer = Customer.objects.get(pk=customer) - if not user: - error_log_action_type = 'pretix.customer.email.error' - log_target = customer - - if event: - if order: - try: - order = event.orders.get(pk=order) - error_log_action_type = 'pretix.event.order.email.error' - log_target = order - except Order.DoesNotExist: - order = None + if attach_size * 1.37 < settings.FILE_UPLOAD_MAX_SIZE_EMAIL_ATTACHMENT - 1024 * 1024: + # Do not attach more than (limit - 1 MB) in tickets (1MB space for invoice, email itself, …), + # it will bounce way too often. + # 1 MB is the buffer for the rest of the email (text, invoice, calendar, pictures) + # 1.37 is the factor for base64 encoding https://en.wikipedia.org/wiki/Base64 + for a in args: + try: + email.attach(*a) + except: + pass else: - with language(order.locale, event.settings.region): - if not event.settings.mail_attach_tickets: - attach_tickets = False - if position: - try: - position = order.positions.get(pk=position) - except OrderPosition.DoesNotExist: - attach_tickets = False - if attach_tickets: - args = [] - attach_size = 0 - for name, ct in get_tickets_for_order(order, base_position=position): - try: - content = ct.file.read() - args.append((name, content, ct.type)) - attach_size += len(content) - except: - # This sometimes fails e.g. with FileNotFoundError. We haven't been able to figure out - # why (probably some race condition with ticket cache invalidation?), so retry later. - try: - self.retry(max_retries=5, countdown=60) - except MaxRetriesExceededError: - # Well then, something is really wrong, let's send it without attachment before we - # don't sent at all - logger.exception('Could not attach invoice to email') - pass + outgoing_mail.order.log_action( + 'pretix.event.order.email.attachments.skipped', + data={ + 'subject': 'Attachments skipped', + 'message': 'Attachment have not been send because {} bytes are likely too large to arrive.'.format(attach_size), + 'recipient': '', + 'invoices': [], + } + ) - if attach_size * 1.37 < settings.FILE_UPLOAD_MAX_SIZE_EMAIL_ATTACHMENT - 1024 * 1024: - # Do not attach more than (limit - 1 MB) in tickets (1MB space for invoice, email itself, …), - # it will bounce way to often. - # 1 MB is the buffer for the rest of the email (text, invoice, calendar, pictures) - # 1.37 is the factor for base64 encoding https://en.wikipedia.org/wiki/Base64 - for a in args: - try: - email.attach(*a) - except: - pass - else: - order.log_action( - 'pretix.event.order.email.attachments.skipped', - data={ - 'subject': 'Attachments skipped', - 'message': 'Attachment have not been send because {} bytes are likely too large to arrive.'.format(attach_size), - 'recipient': '', - 'invoices': [], - } - ) - if attach_ical: - fname = re.sub('[^a-zA-Z0-9 ]', '-', unidecode(pgettext('attachment_filename', 'Calendar invite'))) - for i, cal in enumerate(get_private_icals(event, [position] if position else order.positions.all())): - email.attach('{}{}.ics'.format(fname, f'-{i + 1}' if i > 0 else ''), cal.serialize(), 'text/calendar') - - email = email_filter.send_chained(event, 'message', message=email, order=order, user=user) + # Attach calendar files + if outgoing_mail.should_attach_ical and outgoing_mail.order: + fname = re.sub('[^a-zA-Z0-9 ]', '-', unidecode(pgettext('attachment_filename', 'Calendar invite'))) + icals = get_private_icals( + outgoing_mail.event, + [outgoing_mail.orderposition] if outgoing_mail.orderposition else outgoing_mail.order.positions.all() + ) + for i, cal in enumerate(icals): + name = '{}{}.ics'.format(fname, f'-{i + 1}' if i > 0 else '') + content = cal.serialize() + mimetype = 'text/calendar' + email.attach(name, content, mimetype) invoices_to_mark_transmitted = [] - if invoices: - invoices = Invoice.objects.filter(pk__in=invoices) - for inv in invoices: - if inv.file: - try: - # We try to give the invoice a more human-readable name, e.g. "Invoice_ABC-123.pdf" instead of - # just "ABC-123.pdf", but we only do so if our currently selected language allows to do this - # as ASCII text. For example, we would not want a "فاتورة_" prefix for our filename since this - # has shown to cause deliverability problems of the email and deliverability wins. - with language(order.locale if order else inv.locale, event.settings.region if event else None): - filename = pgettext('invoice', 'Invoice {num}').format(num=inv.number).replace(' ', '_') + '.pdf' - if not re.match("^[a-zA-Z0-9-_%./,&:# ]+$", filename): - filename = inv.number.replace(' ', '_') + '.pdf' - filename = re.sub("[^a-zA-Z0-9-_.]+", "_", filename) - with language(inv.order.locale): - email.attach( - filename, - inv.file.file.read(), - 'application/pdf' + for inv in outgoing_mail.should_attach_invoices.all(): + if inv.file: + try: + # We try to give the invoice a more human-readable name, e.g. "Invoice_ABC-123.pdf" instead of + # just "ABC-123.pdf", but we only do so if our currently selected language allows to do this + # as ASCII text. For example, we would not want a "فاتورة_" prefix for our filename since this + # has shown to cause deliverability problems of the email and deliverability wins. + with language(outgoing_mail.order.locale if outgoing_mail.order else inv.locale, outgoing_mail.event.settings.region): + filename = pgettext('invoice', 'Invoice {num}').format(num=inv.number).replace(' ', '_') + '.pdf' + if not re.match("^[a-zA-Z0-9-_%./,&:# ]+$", filename): + filename = inv.number.replace(' ', '_') + '.pdf' + filename = re.sub("[^a-zA-Z0-9-_.]+", "_", filename) + content = inv.file.file.read() + with language(inv.order.locale): + email.attach( + filename, + content, + 'application/pdf' + ) + invoices_attached.append(inv) + except Exception: + logger.exception(f'Could not attach invoice to email {outgoing_mail.guid}') + pass + else: + if inv.transmission_type == "email": + # Mark invoice as sent when it was sent to the requested address *either* at the time of invoice + # creation *or* as of right now. + expected_recipients = [ + (inv.invoice_to_transmission_info or {}).get("transmission_email_address") + or inv.order.email, + ] + try: + expected_recipients.append( + (inv.order.invoice_address.transmission_info or {}).get("transmission_email_address") + or inv.order.email ) + except InvoiceAddress.DoesNotExist: + pass + if any(t in expected_recipients for t in outgoing_mail.to): + invoices_to_mark_transmitted.append(inv) - if inv.transmission_type == "email": - # Mark invoice as sent when it was sent to the requested address *either* at the time of - # invoice creation *or* as of right now. - expected_recipients = [ - (inv.invoice_to_transmission_info or {}).get("transmission_email_address") - or inv.order.email, - ] - try: - expected_recipients.append( - (inv.order.invoice_address.transmission_info or {}).get("transmission_email_address") - or inv.order.email - ) - except InvoiceAddress.DoesNotExist: - pass - if any(t in expected_recipients for t in to): - invoices_to_mark_transmitted.append(inv) - except: - logger.exception('Could not attach invoice to email') - pass + for fname in outgoing_mail.should_attach_other_files: + ftype, _ = mimetypes.guess_type(fname) + data = default_storage.open(fname).read() + try: + email.attach( + clean_filename(os.path.basename(fname)), + data, + ftype + ) + except: + logger.exception(f'Could not attach file to email {outgoing_mail.guid}') + pass - if attach_other_files: - for fname in attach_other_files: - ftype, _ = mimetypes.guess_type(fname) - data = default_storage.open(fname).read() + for cf in outgoing_mail.should_attach_cached_files.all(): + if cf.file: try: email.attach( - clean_filename(os.path.basename(fname)), - data, - ftype + cf.filename, + cf.file.file.read(), + cf.type, ) except: - logger.exception('Could not attach file to email') + logger.exception(f'Could not attach file to email {outgoing_mail.guid}') pass - if attach_cached_files: - for cf in CachedFile.objects.filter(id__in=attach_cached_files): - if cf.file: - try: - email.attach( - cf.filename, - cf.file.file.read(), - cf.type, - ) - except: - logger.exception('Could not attach file to email') - pass - - email = global_email_filter.send_chained(event, 'message', message=email, user=user, order=order, - organizer=organizer, customer=customer) + outgoing_mail.actual_attachments = [ + { + "name": a[0], + "size": len(a[1]), + "type": a[2], + } for a in email.attachments + ] + try: + if outgoing_mail.event: + with outgoing_mail.scope_manager(): + email = email_filter.send_chained( + sender=outgoing_mail.event, + chain_kwarg_name='message', + message=email, + order=outgoing_mail.order, + user=outgoing_mail.user, + outgoing_mail=outgoing_mail, + ) + + email = global_email_filter.send_chained( + sender=outgoing_mail.event, + chain_kwarg_name='message', + message=email, + user=outgoing_mail.user, + order=outgoing_mail.order, + organizer=outgoing_mail.organizer, + customer=outgoing_mail.customer, + outgoing_mail=outgoing_mail, + ) + except WithholdMailException as e: + outgoing_mail.status = OutgoingMail.STATUS_WITHHELD + outgoing_mail.error = e.error + outgoing_mail.error_detail = e.error_detail + outgoing_mail.sent = now() + outgoing_mail.retry_after = None + outgoing_mail.actual_attachments = [ + { + "name": a[0], + "size": len(a[1]), + "type": a[2], + } for a in email.attachments + ] + outgoing_mail.save(update_fields=["status", "error", "error_detail", "sent", "retry_after", "actual_attachments"]) + logger.info(f"Email {outgoing_mail.guid} withheld") + return False + + # Seems duplicate, but needs to be in this order since plugins might change this + outgoing_mail.actual_attachments = [ + { + "name": a[0], + "size": len(a[1]), + "type": a[2], + } for a in email.attachments + ] + backend = outgoing_mail.get_mail_backend() try: backend.send_messages([email]) - except (smtplib.SMTPResponseException, smtplib.SMTPSenderRefused) as e: - if e.smtp_code in (101, 111, 421, 422, 431, 432, 442, 447, 452): - if e.smtp_code == 432 and settings.HAS_REDIS: - # This is likely Microsoft Exchange Online which has a pretty bad rate limit of max. 3 concurrent - # SMTP connections which is *easily* exceeded with many celery threads. Just retrying with exponential - # backoff won't be good enough if we have a lot of emails, instead we'll need to make sure our retry - # intervals scatter such that the email won't all be retried at the same time again and cause the - # same problem. - # See also https://docs.microsoft.com/en-us/exchange/troubleshoot/send-emails/smtp-submission-improvements + except Exception as e: + logger.exception(f'Error sending email {outgoing_mail.guid}') + retry_strategy = _retry_strategy(e) + err, err_detail = _format_error(e) + + outgoing_mail.error = err + outgoing_mail.error_detail = err_detail + outgoing_mail.sent = now() + + # Run retries + try: + if retry_strategy == "microsoft_concurrency" and settings.HAS_REDIS: from django_redis import get_redis_connection redis_key = "pretix_mail_retry_" + hashlib.sha1(f"{getattr(backend, 'username', '_')}@{getattr(backend, 'host', '_')}".encode()).hexdigest() @@ -589,124 +625,80 @@ def mail_send_task(self, *args, to: List[str], subject: str, body: str, html: st max_retries = 10 retry_after = min(30 + cnt * 10, 1800) - else: - # Most likely some other kind of temporary failure, retry again (but pretty soon) + + outgoing_mail.status = OutgoingMail.STATUS_AWAITING_RETRY + outgoing_mail.retry_after = now() + timedelta(seconds=retry_after) + outgoing_mail.save(update_fields=["status", "error", "error_detail", "sent", "retry_after", "actual_attachments"]) + self.retry(max_retries=max_retries, countdown=retry_after) # throws RetryException, ends function flow + elif retry_strategy in ("microsoft_concurrency", "quick"): max_retries = 5 retry_after = [10, 30, 60, 300, 900, 900][self.request.retries] + outgoing_mail.status = OutgoingMail.STATUS_AWAITING_RETRY + outgoing_mail.retry_after = now() + timedelta(seconds=retry_after) + outgoing_mail.save(update_fields=["status", "error", "error_detail", "sent", "retry_after", "actual_attachments"]) + self.retry(max_retries=max_retries, countdown=retry_after) # throws RetryException, ends function flow - try: - self.retry(max_retries=max_retries, countdown=retry_after) - except MaxRetriesExceededError: - if log_target: - log_target.log_action( - error_log_action_type, - data={ - 'subject': 'SMTP code {}, max retries exceeded'.format(e.smtp_code), - 'message': e.smtp_error.decode() if isinstance(e.smtp_error, bytes) else str(e.smtp_error), - 'recipient': '', - 'invoices': [], - } - ) - for i in invoices_to_mark_transmitted: - i.set_transmission_failed(provider="email_pdf", data={ - "reason": "exception", - "exception": "SMTP code {}, max retries exceeded".format(e.smtp_code), - }) - raise e + elif retry_strategy == "slow": + retry_after = [60, 300, 600, 1200, 1800, 1800][self.request.retries] + outgoing_mail.status = OutgoingMail.STATUS_AWAITING_RETRY + outgoing_mail.retry_after = now() + timedelta(seconds=retry_after) + outgoing_mail.save(update_fields=["status", "error", "error_detail", "sent", "retry_after", "actual_attachments"]) + self.retry(max_retries=5, countdown=retry_after) # throws RetryException, ends function flow - logger.exception('Error sending email') + except MaxRetriesExceededError: + for i in invoices_to_mark_transmitted: + i.set_transmission_failed(provider="email_pdf", data={ + "reason": "exception", + "exception": "{}, max retries exceeded".format(err), + "detail": err_detail, + }) + + if log_target: + log_target.log_action( + error_log_action_type, + data={ + 'subject': f'{err} (max retries exceeded)', + 'message': err_detail, + 'recipient': '', + 'invoices': [], + } + ) + + outgoing_mail.status = OutgoingMail.STATUS_FAILED + outgoing_mail.sent = now() + outgoing_mail.retry_after = None + outgoing_mail.save(update_fields=["status", "error", "error_detail", "sent", "retry_after", "actual_attachments"]) + return False + + # If we reach this, it's a non-retryable error + outgoing_mail.status = OutgoingMail.STATUS_FAILED + outgoing_mail.sent = now() + outgoing_mail.retry_after = None + outgoing_mail.save(update_fields=["status", "error", "error_detail", "sent", "retry_after", "actual_attachments"]) + for i in invoices_to_mark_transmitted: + i.set_transmission_failed(provider="email_pdf", data={ + "reason": "exception", + "exception": err, + "detail": err_detail, + }) if log_target: log_target.log_action( error_log_action_type, data={ - 'subject': 'SMTP code {}'.format(e.smtp_code), - 'message': e.smtp_error.decode() if isinstance(e.smtp_error, bytes) else str(e.smtp_error), + 'subject': err, + 'message': err_detail, 'recipient': '', 'invoices': [], } ) - for i in invoices_to_mark_transmitted: - i.set_transmission_failed(provider="email_pdf", data={ - "reason": "exception", - "exception": "SMTP code {}".format(e.smtp_code), - }) - - raise SendMailException('Failed to send an email to {}.'.format(to)) - except smtplib.SMTPRecipientsRefused as e: - smtp_codes = [a[0] for a in e.recipients.values()] - - if not any(c >= 500 for c in smtp_codes) or any(b'Message is too large' in a[1] for a in e.recipients.values()): - # This is not a permanent failure (mailbox full, service unavailable), retry later, but with large - # intervals. One would think that "Message is too lage" is a permanent failure, but apparently it is not. - # We have documented cases of emails to Microsoft returning the error occasionally and then later - # allowing the very same email. - try: - self.retry(max_retries=5, countdown=[60, 300, 600, 1200, 1800, 1800][self.request.retries]) - except MaxRetriesExceededError: - # ignore and go on with logging the error - pass - - logger.exception('Error sending email') - if log_target: - message = [] - for e, val in e.recipients.items(): - message.append(f'{e}: {val[0]} {val[1].decode()}') - - log_target.log_action( - error_log_action_type, - data={ - 'subject': 'SMTP error', - 'message': '\n'.join(message), - 'recipient': '', - 'invoices': [], - } - ) - for i in invoices_to_mark_transmitted: - i.set_transmission_failed(provider="email_pdf", data={ - "reason": "exception", - "exception": "SMTP error", - }) - - raise SendMailException('Failed to send an email to {}.'.format(to)) - except Exception as e: - if isinstance(e, OSError) and not isinstance(e, smtplib.SMTPNotSupportedError): - try: - self.retry(max_retries=5, countdown=[10, 30, 60, 300, 900, 900][self.request.retries]) - except MaxRetriesExceededError: - if log_target: - log_target.log_action( - error_log_action_type, - data={ - 'subject': 'Internal error', - 'message': f'Max retries exceeded after error "{str(e)}"', - 'recipient': '', - 'invoices': [], - } - ) - for i in invoices_to_mark_transmitted: - i.set_transmission_failed(provider="email_pdf", data={ - "reason": "exception", - "exception": "Internal error", - }) - raise e - if log_target: - log_target.log_action( - error_log_action_type, - data={ - 'subject': 'Internal error', - 'message': str(e), - 'recipient': '', - 'invoices': [], - } - ) - for i in invoices_to_mark_transmitted: - i.set_transmission_failed(provider="email_pdf", data={ - "reason": "exception", - "exception": "Internal error", - }) - logger.exception('Error sending email') - raise SendMailException('Failed to send an email to {}.'.format(to)) + return False else: + outgoing_mail.status = OutgoingMail.STATUS_SENT + outgoing_mail.error = None + outgoing_mail.error_detail = None + outgoing_mail.sent = now() + outgoing_mail.retry_after = None + outgoing_mail.save(update_fields=["status", "error", "error_detail", "sent", "actual_attachments", "retry_after"]) for i in invoices_to_mark_transmitted: if i.transmission_status != Invoice.TRANSMISSION_STATUS_COMPLETED: i.transmission_date = now() @@ -715,7 +707,7 @@ def mail_send_task(self, *args, to: List[str], subject: str, body: str, html: st i.transmission_info = { "sent": [ { - "recipients": to, + "recipients": outgoing_mail.to, "datetime": now().isoformat(), } ] @@ -727,7 +719,7 @@ def mail_send_task(self, *args, to: List[str], subject: str, body: str, html: st elif i.transmission_provider == "email_pdf": i.transmission_info["sent"].append( { - "recipients": to, + "recipients": outgoing_mail.to, "datetime": now().isoformat(), } ) @@ -741,17 +733,55 @@ def mail_send_task(self, *args, to: List[str], subject: str, body: str, html: st "transmission_provider": "email_pdf", "transmission_type": "email", "data": { - "recipients": [to], + "recipients": outgoing_mail.to, }, } ) + return True -def mail_send(*args, **kwargs): - mail_send_task.apply_async(args=args, kwargs=kwargs) +def mail_send(to: List[str], subject: str, body: str, html: Optional[str], sender: str, + event: int | Event = None, position: int | OrderPosition = None, headers: dict = None, + cc: List[str] = None, bcc: List[str] = None, invoices: List[int | Invoice] = None, order: int | Order = None, + attach_tickets=False, user: int | User=None, organizer: int | Organizer=None, customer: int | Customer=None, + attach_ical=False, attach_cached_files: List[int | CachedFile] = None, attach_other_files: List[str] = None): + """ + Low-level function to send mails, kept for backwards-compatibility. You should usually use mail() instead. + """ + m = OutgoingMail.objects.create( + organizer_id=organizer.pk if isinstance(organizer, Organizer) else organizer, + event_id=event.pk if isinstance(event, Event) else event, + order_id=order.pk if isinstance(order, Order) else order, + orderposition_id=position.pk if isinstance(position, OrderPosition) else position, + customer_id=customer.pk if isinstance(customer, Customer) else customer, + user_id=user.pk if isinstance(user, User) else user, + to=[to.lower()] if isinstance(to, str) else [e.lower() for e in to], + cc=[e.lower() for e in cc] if cc else [], + bcc=[e.lower() for e in bcc] if bcc else [], + subject=subject, + body_plain=body, + body_html=html, + sender=sender, + headers=headers, + should_attach_tickets=attach_tickets, + should_attach_ical=attach_ical, + should_attach_other_files=attach_other_files or [], + ) + if invoices and not position: + if isinstance(invoices[0], int): + invoices = Invoice.objects.filter(pk__in=invoices) + m.should_attach_invoices.add(*invoices) + if attach_cached_files: + for cf in attach_cached_files: + if not isinstance(cf, CachedFile): + m.should_attach_cached_files.add(CachedFile.objects.get(pk=cf)) + else: + m.should_attach_cached_files.add(cf) + + mail_send_task.apply_async(kwargs={"outgoing_mail": m.pk}) -def render_mail(template, context, placeholder_mode=SafeFormatter.MODE_RICH_TO_PLAIN): +def render_mail(template, context, placeholder_mode: Optional[int]=SafeFormatter.MODE_RICH_TO_PLAIN): if isinstance(template, LazyI18nString): body = str(template) if context and placeholder_mode: @@ -866,3 +896,203 @@ def normalize_image_url(url): else: url = urljoin(settings.MEDIA_URL, url) return url + + +def _autoextend_context(context, order): + try: + context.update({ + 'invoice_name': order.invoice_address.name, + 'invoice_company': order.invoice_address.company + }) + except InvoiceAddress.DoesNotExist: + context.update({ + 'invoice_name': '', + 'invoice_company': '' + }) + + +def _full_sender(sender_address, event, organizer): + sender_address = ( + sender_address or + (event.settings.get('mail_from') if event else None) or + (organizer.settings.get('mail_from') if organizer else None) or + settings.MAIL_FROM + ) + if event: + sender_name = event.settings.mail_from_name or str(event.name) + elif organizer: + sender_name = organizer.settings.mail_from_name or str(organizer.name) + else: + sender_name = settings.PRETIX_INSTANCE_NAME + + sender = formataddr((clean_sender_name(sender_name), sender_address)) + return sender + + +def _wrap_plain_body(content_plain, signature, event, order, position, no_order_links): + body_plain = content_plain + body_plain += "\r\n\r\n-- \r\n" + + if signature: + signature = signature.format(event=event.name if event else '') + body_plain += signature + body_plain += "\r\n\r\n-- \r\n" + + if event and order and position and not no_order_links: + body_plain += _( + "You are receiving this email because someone placed an order for {event} for you." + ).format(event=event.name) + body_plain += "\r\n" + body_plain += _( + "You can view your order details at the following URL:\n{orderurl}." + ).replace("\n", "\r\n").format( + event=event.name, orderurl=build_absolute_uri( + order.event, 'presale:event.order.position', kwargs={ + 'order': order.code, + 'secret': position.web_secret, + 'position': position.positionid, + } + ) + ) + elif event and order and not no_order_links: + body_plain += _( + "You are receiving this email because you placed an order for {event}." + ).format(event=event.name) + body_plain += "\r\n" + body_plain += _( + "You can view your order details at the following URL:\n{orderurl}." + ).replace("\n", "\r\n").format( + event=event.name, orderurl=build_absolute_uri( + order.event, 'presale:event.order.open', kwargs={ + 'order': order.code, + 'secret': order.secret, + 'hash': order.email_confirm_secret() + } + ) + ) + body_plain += "\r\n" + + return body_plain + + +def _retry_strategy(e: Exception): + if isinstance(e, (smtplib.SMTPResponseException, smtplib.SMTPSenderRefused)): + if e.smtp_code == 432: + # This is likely Microsoft Exchange Online which has a pretty bad rate limit of max. 3 concurrent + # SMTP connections which is *easily* exceeded with many celery threads. Just retrying with exponential + # backoff won't be good enough if we have a lot of emails, instead we'll need to make sure our retry + # intervals scatter such that the email won't all be retried at the same time again and cause the + # same problem. + # See also https://docs.microsoft.com/en-us/exchange/troubleshoot/send-emails/smtp-submission-improvements + return "microsoft_concurrency" + + if e.smtp_code in (101, 111, 421, 422, 431, 432, 442, 447, 452): + return "quick" + + elif isinstance(e, smtplib.SMTPRecipientsRefused): + smtp_codes = [a[0] for a in e.recipients.values()] + + if not any(c >= 500 for c in smtp_codes) or any(b'Message is too large' in a[1] for a in e.recipients.values()): + # This is not a permanent failure (mailbox full, service unavailable), retry later, but with large + # intervals. One would think that "Message is too lage" is a permanent failure, but apparently it is not. + # We have documented cases of emails to Microsoft returning the error occasionally and then later + # allowing the very same email. + return "slow" + + elif isinstance(e, OSError) and not isinstance(e, smtplib.SMTPNotSupportedError): + # Most likely some other kind of temporary failure, retry again (but pretty soon) + return "quick" + + +def _format_error(e: Exception): + if isinstance(e, (smtplib.SMTPResponseException, smtplib.SMTPSenderRefused)): + return 'SMTP code {}'.format(e.smtp_code), e.smtp_error.decode() if isinstance(e.smtp_error, bytes) else str(e.smtp_error) + elif isinstance(e, smtplib.SMTPRecipientsRefused): + message = [] + for e, val in e.recipients.items(): + message.append(f'{e}: {val[0]} {val[1].decode()}') + return 'SMTP recipients refudes', '\n'.join(message) + else: + return 'Internal error', str(e) + + +def _is_queue_long(queue_name="mail"): + """ + Checks an estimate if there is currently a long celery queue for emails. If so, + there's no reason to retry stuck emails, because they are stuck because of the + queue and we don't need to add more oil to the fire. + + This does not need to be perfect, as it is safe to run the same task twice, it just + wastes ressources. + """ + if not settings.HAS_CELERY: + return False + if not settings.CELERY_BROKER_URL.startswith("redis://"): + return False # check not supported + priority_steps = settings.CELERY_BROKER_TRANSPORT_OPTIONS.get("priority_steps", [0]) + sep = settings.CELERY_BROKER_TRANSPORT_OPTIONS.get("sep", ":") + client = app.broker_connection().channel().client + queue_length = 0 + for prio in priority_steps: + if prio: + qname = f"{queue_name}{sep}{prio}" + else: + qname = queue_name + queue_length += client.llen(qname) + + return queue_length > 100 + + +@receiver(signal=periodic_task) +@scopes_disabled() +def retry_stuck_inflight_mails(sender, **kwargs): + """ + Retry emails that are stuck in "inflight" state, e.g. their celery task just died. + """ + with transaction.atomic(): + for m in OutgoingMail.objects.filter( + status=OutgoingMail.STATUS_INFLIGHT, + inflight_since__lt=now() - timedelta(hours=1), + ).select_for_update(of=OF_SELF, skip_locked=connection.features.has_select_for_update_skip_locked): + m.status = OutgoingMail.STATUS_QUEUED + m.save() + mail_send_task.apply_async(kwargs={"outgoing_mail": m.pk}) + + +@receiver(signal=periodic_task) +@scopes_disabled() +def retry_stuck_queued_mails(sender, **kwargs): + """ + Retry emails that are stuck in "queued" state, e.g. their celery task never started. We do this only + when there is currently almost no queue, to avoid many tasks being scheduled for the same mail if that + mail is still waiting in the queue (even if that would be safe, all tasks except the first one would be a no-op, + but it would create many more useless tasks in a high-load situation). + """ + if _is_queue_long(): + logger.info("Do not retry stuck mails as the queue is long.") + return + + for m in OutgoingMail.objects.filter( + Q( + status=OutgoingMail.STATUS_QUEUED, + created__lt=now() - timedelta(hours=1), + ) | Q( + status=OutgoingMail.STATUS_AWAITING_RETRY, + retry_after__lt=now() - timedelta(hours=1), + ) + ): + mail_send_task.apply_async(kwargs={"outgoing_mail": m.pk}) + + +@receiver(signal=periodic_task) +@scopes_disabled() +def delete_old_emails(sender, **kwargs): + """ + OutgoingMail is currently not intended to be an archive, because it would be hard to do in a + privacy-first design, so we delete after some time. + """ + cutoff = now() - timedelta(seconds=settings.OUTGOING_MAIL_RETENTION) + OutgoingMail.objects.filter( + Q(sent__lt=cutoff) | + Q(sent__isnull=True, created__lt=cutoff) + ).delete() diff --git a/src/pretix/base/services/notifications.py b/src/pretix/base/services/notifications.py index 5a75dfafbe..0a70e7c32d 100644 --- a/src/pretix/base/services/notifications.py +++ b/src/pretix/base/services/notifications.py @@ -19,6 +19,8 @@ # You should have received a copy of the GNU Affero General Public License along with this program. If not, see # . # +import uuid + import css_inline from django.conf import settings from django.template.loader import get_template @@ -26,7 +28,9 @@ from django.utils.timezone import override from django_scopes import scope, scopes_disabled from pretix.base.i18n import language -from pretix.base.models import LogEntry, NotificationSetting, User +from pretix.base.models import ( + LogEntry, NotificationSetting, OutgoingMail, User, +) from pretix.base.notifications import Notification, get_all_notification_types from pretix.base.services.mail import mail_send_task from pretix.base.services.tasks import ProfiledTask, TransactionAwareTask @@ -153,16 +157,26 @@ def send_notification_mail(notification: Notification, user: User): tpl_plain = get_template('pretixbase/email/notification.txt') body_plain = tpl_plain.render(ctx) - mail_send_task.apply_async(kwargs={ - 'to': [user.email], - 'subject': '[{}] {}: {}'.format( + guid = uuid.uuid4() + m = OutgoingMail.objects.create( + guid=guid, + user=user, + to=[user.email], + subject='[{}] {}: {}'.format( settings.PRETIX_INSTANCE_NAME, notification.event.settings.mail_prefix or notification.event.slug.upper(), notification.title ), - 'body': body_plain, - 'html': body_html, - 'sender': settings.MAIL_FROM_NOTIFICATIONS, - 'headers': {}, - 'user': user.pk + body_plain=body_plain, + body_html=body_html, + sender=settings.MAIL_FROM_NOTIFICATIONS, + headers={ + 'X-Auto-Response-Suppress': 'OOF, NRN, AutoReply, RN', + 'Auto-Submitted': 'auto-generated', + 'X-Mailer': 'pretix', + 'X-PX-Correlation': str(guid), + }, + ) + mail_send_task.apply_async(kwargs={ + 'outgoing_mail': m.pk, }) diff --git a/src/pretix/base/services/orders.py b/src/pretix/base/services/orders.py index 5d1d611531..b49d656169 100644 --- a/src/pretix/base/services/orders.py +++ b/src/pretix/base/services/orders.py @@ -90,7 +90,6 @@ from pretix.base.services.invoices import ( from pretix.base.services.locking import ( LOCK_TRUST_WINDOW, LockTimeoutException, lock_objects, ) -from pretix.base.services.mail import SendMailException from pretix.base.services.memberships import ( create_membership, validate_memberships_in_order, ) @@ -438,33 +437,27 @@ def approve_order(order, user=None, send_mail: bool=True, auth=None, force=False email_attendee_subject = order.event.settings.mail_subject_order_approved_attendee email_context = get_email_context(event=order.event, order=order) - try: - order.send_mail( - email_subject, email_template, email_context, - 'pretix.event.order.email.order_approved', user, - attach_tickets=True, - attach_ical=order.event.settings.mail_attach_ical and ( - not order.event.settings.mail_attach_ical_paid_only or - order.total == Decimal('0.00') or - order.valid_if_pending - ), - invoices=[invoice] if invoice and transmit_invoice_mail else [] - ) - except SendMailException: - logger.exception('Order approved email could not be sent') + order.send_mail( + email_subject, email_template, email_context, + 'pretix.event.order.email.order_approved', user, + attach_tickets=True, + attach_ical=order.event.settings.mail_attach_ical and ( + not order.event.settings.mail_attach_ical_paid_only or + order.total == Decimal('0.00') or + order.valid_if_pending + ), + invoices=[invoice] if invoice and transmit_invoice_mail else [] + ) if email_attendees: for p in order.positions.all(): if p.addon_to_id is None and p.attendee_email and p.attendee_email != order.email: email_attendee_context = get_email_context(event=order.event, order=order, position=p) - try: - p.send_mail( - email_attendee_subject, email_attendee_template, email_attendee_context, - 'pretix.event.order.email.order_approved', user, - attach_tickets=True, - ) - except SendMailException: - logger.exception('Order approved email could not be sent to attendee') + p.send_mail( + email_attendee_subject, email_attendee_template, email_attendee_context, + 'pretix.event.order.email.order_approved', user, + attach_tickets=True, + ) return order.pk @@ -501,13 +494,10 @@ def deny_order(order, comment='', user=None, send_mail: bool=True, auth=None): email_template = order.event.settings.mail_text_order_denied email_subject = order.event.settings.mail_subject_order_denied email_context = get_email_context(event=order.event, order=order, comment=comment) - try: - order.send_mail( - email_subject, email_template, email_context, - 'pretix.event.order.email.order_denied', user - ) - except SendMailException: - logger.exception('Order denied email could not be sent') + order.send_mail( + email_subject, email_template, email_context, + 'pretix.event.order.email.order_denied', user + ) return order.pk @@ -660,14 +650,11 @@ def _cancel_order(order, user=None, send_mail: bool=True, api_token=None, device email_template = order.event.settings.mail_text_order_canceled email_subject = order.event.settings.mail_subject_order_canceled email_context = get_email_context(event=order.event, order=order, comment=comment or "") - try: - order.send_mail( - email_subject, email_template, email_context, - 'pretix.event.order.email.order_canceled', user, - invoices=transmit_invoices_mail, - ) - except SendMailException: - logger.exception('Order canceled email could not be sent') + order.send_mail( + email_subject, email_template, email_context, + 'pretix.event.order.email.order_canceled', user, + invoices=transmit_invoices_mail, + ) for p in order.payments.filter(state__in=(OrderPayment.PAYMENT_STATE_CREATED, OrderPayment.PAYMENT_STATE_PENDING)): try: @@ -1108,46 +1095,40 @@ def _order_placed_email(event: Event, order: Order, email_template, subject_temp log_entry: str, invoice, payments: List[OrderPayment], is_free=False): email_context = get_email_context(event=event, order=order, payments=payments) - try: - order.send_mail( - subject_template, email_template, email_context, - log_entry, - invoices=[invoice] if invoice else [], - attach_tickets=True, - attach_ical=event.settings.mail_attach_ical and ( - not event.settings.mail_attach_ical_paid_only or - is_free or - order.valid_if_pending - ), - attach_other_files=[a for a in [ - event.settings.get('mail_attachment_new_order', as_type=str, default='')[len('file://'):] - ] if a], - ) - except SendMailException: - logger.exception('Order received email could not be sent') + order.send_mail( + subject_template, email_template, email_context, + log_entry, + invoices=[invoice] if invoice else [], + attach_tickets=True, + attach_ical=event.settings.mail_attach_ical and ( + not event.settings.mail_attach_ical_paid_only or + is_free or + order.valid_if_pending + ), + attach_other_files=[a for a in [ + event.settings.get('mail_attachment_new_order', as_type=str, default='')[len('file://'):] + ] if a], + ) def _order_placed_email_attendee(event: Event, order: Order, position: OrderPosition, email_template, subject_template, log_entry: str, is_free=False): email_context = get_email_context(event=event, order=order, position=position) - try: - position.send_mail( - subject_template, email_template, email_context, - log_entry, - invoices=[], - attach_tickets=True, - attach_ical=event.settings.mail_attach_ical and ( - not event.settings.mail_attach_ical_paid_only or - is_free or - order.valid_if_pending - ), - attach_other_files=[a for a in [ - event.settings.get('mail_attachment_new_order', as_type=str, default='')[len('file://'):] - ] if a], - ) - except SendMailException: - logger.exception('Order received email could not be sent to attendee') + position.send_mail( + subject_template, email_template, email_context, + log_entry, + invoices=[], + attach_tickets=True, + attach_ical=event.settings.mail_attach_ical and ( + not event.settings.mail_attach_ical_paid_only or + is_free or + order.valid_if_pending + ), + attach_other_files=[a for a in [ + event.settings.get('mail_attachment_new_order', as_type=str, default='')[len('file://'):] + ] if a], + ) def _perform_order(event: Event, payment_requests: List[dict], position_ids: List[str], @@ -1476,13 +1457,10 @@ def send_expiry_warnings(sender, **kwargs): email_template = settings.mail_text_order_pending_warning email_subject = settings.mail_subject_order_pending_warning - try: - o.send_mail( - email_subject, email_template, email_context, - 'pretix.event.order.email.expire_warning_sent' - ) - except SendMailException: - logger.exception('Reminder email could not be sent') + o.send_mail( + email_subject, email_template, email_context, + 'pretix.event.order.email.expire_warning_sent' + ) @receiver(signal=periodic_task) @@ -1543,14 +1521,11 @@ def send_download_reminders(sender, **kwargs): email_template = event.settings.mail_text_download_reminder email_subject = event.settings.mail_subject_download_reminder email_context = get_email_context(event=event, order=o) - try: - o.send_mail( - email_subject, email_template, email_context, - 'pretix.event.order.email.download_reminder_sent', - attach_tickets=True - ) - except SendMailException: - logger.exception('Reminder email could not be sent') + o.send_mail( + email_subject, email_template, email_context, + 'pretix.event.order.email.download_reminder_sent', + attach_tickets=True + ) if event.settings.mail_send_download_reminder_attendee: for p in positions: @@ -1564,14 +1539,11 @@ def send_download_reminders(sender, **kwargs): email_template = event.settings.mail_text_download_reminder_attendee email_subject = event.settings.mail_subject_download_reminder_attendee email_context = get_email_context(event=event, order=o, position=p) - try: - o.send_mail( - email_subject, email_template, email_context, - 'pretix.event.order.email.download_reminder_sent', - attach_tickets=True, position=p - ) - except SendMailException: - logger.exception('Reminder email could not be sent to attendee') + o.send_mail( + email_subject, email_template, email_context, + 'pretix.event.order.email.download_reminder_sent', + attach_tickets=True, position=p + ) def notify_user_changed_order(order, user=None, auth=None, invoices=[]): @@ -1579,13 +1551,10 @@ def notify_user_changed_order(order, user=None, auth=None, invoices=[]): email_template = order.event.settings.mail_text_order_changed email_context = get_email_context(event=order.event, order=order) email_subject = order.event.settings.mail_subject_order_changed - try: - order.send_mail( - email_subject, email_template, email_context, - 'pretix.event.order.email.order_changed', user, auth=auth, invoices=invoices, attach_tickets=True, - ) - except SendMailException: - logger.exception('Order changed email could not be sent') + order.send_mail( + email_subject, email_template, email_context, + 'pretix.event.order.email.order_changed', user, auth=auth, invoices=invoices, attach_tickets=True, + ) class OrderChangeManager: diff --git a/src/pretix/base/services/shredder.py b/src/pretix/base/services/shredder.py index 84ef4eee00..6211ad4a85 100644 --- a/src/pretix/base/services/shredder.py +++ b/src/pretix/base/services/shredder.py @@ -48,7 +48,7 @@ from django.utils.translation import gettext_lazy as _ from pretix.base.i18n import language from pretix.base.models import CachedFile, Event, User, cachedfile_name -from pretix.base.services.mail import SendMailException, mail +from pretix.base.services.mail import mail from pretix.base.services.tasks import ProfiledEventTask from pretix.base.shredder import ShredError from pretix.celery_app import app @@ -171,21 +171,18 @@ def shred(self, event: Event, fileid: str, confirm_code: str, user: int=None, lo if user: with language(user.locale): - try: - mail( - user.email, - _('Data shredding completed'), - 'pretixbase/email/shred_completed.txt', - { - 'user': user, - 'organizer': event.organizer.name, - 'event': str(event.name), - 'start_time': date_format(parse(indexdata['time']).astimezone(event.timezone), 'SHORT_DATETIME_FORMAT'), - 'shredders': ', '.join([str(s.verbose_name) for s in shredders]) - }, - event=None, - user=user, - locale=user.locale, - ) - except SendMailException: - pass # Already logged + mail( + user.email, + _('Data shredding completed'), + 'pretixbase/email/shred_completed.txt', + { + 'user': user, + 'organizer': event.organizer.name, + 'event': str(event.name), + 'start_time': date_format(parse(indexdata['time']).astimezone(event.timezone), 'SHORT_DATETIME_FORMAT'), + 'shredders': ', '.join([str(s.verbose_name) for s in shredders]) + }, + event=None, + user=user, + locale=user.locale, + ) diff --git a/src/pretix/base/shredder.py b/src/pretix/base/shredder.py index a8110e9874..21b2803053 100644 --- a/src/pretix/base/shredder.py +++ b/src/pretix/base/shredder.py @@ -51,7 +51,7 @@ from pretix.api.serializers.waitinglist import WaitingListSerializer from pretix.base.i18n import LazyLocaleException from pretix.base.models import ( CachedCombinedTicket, CachedTicket, Event, InvoiceAddress, OrderPayment, - OrderPosition, OrderRefund, QuestionAnswer, + OrderPosition, OrderRefund, OutgoingMail, QuestionAnswer, ) from pretix.base.services.invoices import invoice_pdf_task from pretix.base.signals import register_data_shredders @@ -329,6 +329,10 @@ class EmailAddressShredder(BaseDataShredder): sleep_time=2, ) + slow_delete( + OutgoingMail.objects.filter(event=self.event) + ) + for o in _progress_helper(qs_orders, progress_callback, qs_op_cnt, total): changed = bool(o.email) or bool(o.customer) o.email = None diff --git a/src/pretix/base/signals.py b/src/pretix/base/signals.py index 3ab54bf0eb..c917f318bd 100644 --- a/src/pretix/base/signals.py +++ b/src/pretix/base/signals.py @@ -944,32 +944,40 @@ As with all event-plugin signals, the ``sender`` keyword argument will contain t email_filter = EventPluginSignal() """ -Arguments: ``message``, ``order``, ``user`` +Arguments: ``message``, ``order``, ``user``, ``outgoing_mail`` This signal allows you to implement a middleware-style filter on all outgoing emails. You are expected to return a (possibly modified) copy of the message object passed to you. As with all event-plugin signals, the ``sender`` keyword argument will contain the event. The ``message`` argument will contain an ``EmailMultiAlternatives`` object. +The ``outgoing_mail`` argument will contain the ``OutgoingMail`` model instance. Note that the ``message`` object +might have newer information if a previous plugin already modified the email. If the email is associated with a specific order, the ``order`` argument will be passed as well, otherwise it will be ``None``. If the email is associated with a specific user, e.g. a notification email, the ``user`` argument will be passed as well, otherwise it will be ``None``. + +You can raise ``WithholdMailException`` to prevent the email from being sent, e.g. when implementing rate limiting. """ global_email_filter = GlobalSignal() """ -Arguments: ``message``, ``order``, ``user``, ``customer``, ``organizer`` +Arguments: ``message``, ``order``, ``user``, ``customer``, ``organizer``, ``outgoing_mail`` This signal allows you to implement a middleware-style filter on all outgoing emails. You are expected to return a (possibly modified) copy of the message object passed to you. This signal is called on all events and even if there is no known event. ``sender`` is an event or None. The ``message`` argument will contain an ``EmailMultiAlternatives`` object. +The ``outgoing_mail`` argument will contain the ``OutgoingMail`` model instance. Note that the ``message`` object +might have newer information if a previous plugin already modified the email. If the email is associated with a specific order, the ``order`` argument will be passed as well, otherwise it will be ``None``. If the email is associated with a specific user, e.g. a notification email, the ``user`` argument will be passed as well, otherwise it will be ``None``. + +You can raise ``WithholdMailException`` to prevent the email from being sent, e.g. when implementing rate limiting. """ diff --git a/src/pretix/control/forms/filter.py b/src/pretix/control/forms/filter.py index 783c12e857..204820ac30 100644 --- a/src/pretix/control/forms/filter.py +++ b/src/pretix/control/forms/filter.py @@ -57,8 +57,9 @@ from pretix.base.forms.widgets import ( from pretix.base.models import ( Checkin, CheckinList, Device, Event, EventMetaProperty, EventMetaValue, Gate, Invoice, InvoiceAddress, Item, Order, OrderPayment, OrderPosition, - OrderRefund, Organizer, Question, QuestionAnswer, Quota, SalesChannel, - SubEvent, SubEventMetaValue, Team, TeamAPIToken, TeamInvite, Voucher, + OrderRefund, Organizer, OutgoingMail, Question, QuestionAnswer, Quota, + SalesChannel, SubEvent, SubEventMetaValue, Team, TeamAPIToken, TeamInvite, + Voucher, ) from pretix.base.signals import register_payment_providers from pretix.base.timeframes import ( @@ -2815,3 +2816,61 @@ class DeviceFilterForm(FilterForm): qs = qs.order_by('-device_id') return qs + + +class OutgoingMailFilterForm(FilterForm): + orders = { + 'date': 'd', + '-date': '-d', + } + query = forms.CharField( + label=_('Search email address or subject'), + widget=forms.TextInput(attrs={ + 'placeholder': _('Search email address or subject'), + }), + required=False + ) + event = forms.ModelChoiceField( + queryset=Event.objects.none(), + label=_('Event'), + empty_label=_('All events'), + required=False, + ) + status = forms.ChoiceField( + label=_('Status'), + choices=[ + ('', _('All')), + *OutgoingMail.STATUS_CHOICES, + ], + required=False + ) + + def __init__(self, *args, **kwargs): + request = kwargs.pop('request') + super().__init__(*args, **kwargs) + self.fields['event'].queryset = request.organizer.events.all() + + def filter_qs(self, qs): + fdata = self.cleaned_data + + if fdata.get('query'): + query = fdata.get('query') + qs = qs.filter( + Q(to__containsstring=query.lower()) + | Q(cc__containsstring=query.lower()) + | Q(bcc__containsstring=query.lower()) + | Q(subject__icontains=query) + ) + + if fdata.get('event'): + qs = qs.filter(event=fdata['event']) + + if fdata.get('status'): + qs = qs.filter(status=fdata['status']) + + if fdata.get('ordering'): + qs = qs.order_by(self.get_order_by()) + else: + qs = qs.order_by("-created", "-pk") + + return qs diff --git a/src/pretix/control/forms/organizer.py b/src/pretix/control/forms/organizer.py index 1f10385af7..a314b889a2 100644 --- a/src/pretix/control/forms/organizer.py +++ b/src/pretix/control/forms/organizer.py @@ -585,6 +585,7 @@ class MailSettingsForm(SettingsForm): help_text=''.join([ str(_("All emails will be sent to this address as a Bcc copy.")), str(_("You can specify multiple recipients separated by commas.")), + str(_("Sensitive emails like password resets will not be sent in Bcc.")), ]), validators=[multimail_validate], required=False, diff --git a/src/pretix/control/logdisplay.py b/src/pretix/control/logdisplay.py index 02ccce2e4c..8f8f5b291e 100644 --- a/src/pretix/control/logdisplay.py +++ b/src/pretix/control/logdisplay.py @@ -699,6 +699,8 @@ class CoreUserImpersonatedLogEntryType(UserImpersonatedLogEntryType): 'pretix.organizer.export.schedule.deleted': _('A scheduled export has been deleted.'), 'pretix.organizer.export.schedule.executed': _('A scheduled export has been executed.'), 'pretix.organizer.export.schedule.failed': _('A scheduled export has failed: {reason}.'), + 'pretix.organizer.outgoingmails.retried': _('Failed emails have been scheduled to be retried.'), + 'pretix.organizer.outgoingmails.aborted': _('Queued emails have been aborted.'), 'pretix.giftcards.acceptance.added': _('Gift card acceptance for another organizer has been added.'), 'pretix.giftcards.acceptance.removed': _('Gift card acceptance for another organizer has been removed.'), 'pretix.giftcards.acceptance.acceptor.invited': _('A new gift card acceptor has been invited.'), diff --git a/src/pretix/control/navigation.py b/src/pretix/control/navigation.py index c3a0cbb8f9..a6e3209788 100644 --- a/src/pretix/control/navigation.py +++ b/src/pretix/control/navigation.py @@ -679,6 +679,15 @@ def get_organizer_navigation(request): 'active': (url.url_name == 'organizer.datasync.failedjobs'), }]) + nav.append({ + 'label': _('Outgoing emails'), + 'url': reverse('control:organizer.outgoingmails', kwargs={ + 'organizer': request.organizer.slug, + }), + 'active': 'organizer.outgoingmail' in url.url_name, + 'icon': 'send', + }) + merge_in(nav, sorted( sum((list(a[1]) for a in nav_organizer.send(request.organizer, request=request, organizer=request.organizer)), []), diff --git a/src/pretix/control/templates/pretixcontrol/organizers/outgoing_mail.html b/src/pretix/control/templates/pretixcontrol/organizers/outgoing_mail.html new file mode 100644 index 0000000000..fc3ea19108 --- /dev/null +++ b/src/pretix/control/templates/pretixcontrol/organizers/outgoing_mail.html @@ -0,0 +1,222 @@ +{% extends "pretixcontrol/organizers/base.html" %} +{% load i18n %} +{% load bootstrap3 %} +{% load urlreplace %} +{% load icon %} +{% load compress %} +{% load static %} +{% block inner %} +

+ {% trans "Outgoing email" %} +

+
+
+

{% trans "Email details" %}

+
+
+
+
+ +
+
{% trans "From" context "email" %}
+
{{ mail.sender }}
+
{% trans "To" context "email" %}
+
{{ mail.to|join:", " }}
+ {% if mail.cc %} +
{% trans "Cc" context "email" %}
+
{{ mail.cc|join:", " }}
+ {% endif %} + {% if mail.bcc %} +
{% trans "Bcc" context "email" %}
+
{{ mail.bcc|join:", " }}
+ {% endif %} +
{% trans "Subject" %}
+
{{ mail.subject }}
+
{% trans "Status" %}
+
+ {% if mail.status == "queued" %} + {% icon "clock-o" %} {% trans "queued" %} + {% elif mail.status == "inflight" %} + {% icon "send" %} {% trans "being sent" %} + {% elif mail.status == "awaiting_retry" %} + {% icon "repeat" %} {% trans "will be retried" %} + {% elif mail.status == "failed" %} + {% icon "warning" %} {% trans "failed" %} + {% elif mail.status == "bounced" %} + {% icon "exclamation-circle" %} {% trans "bounced" %} + {% elif mail.status == "withheld" %} + {% icon "ban" %} {% trans "withheld" %} + {% elif mail.status == "aborted" %} + {% icon "ban" %} {% trans "aborted" %} + {% elif mail.status == "sent" %} + {% icon "check" %} {% trans "sent" %} + {% endif %} +
+
{% trans "Creation" %}
+
{{ mail.created|date:"SHORT_DATETIME_FORMAT" }}
+ {% if mail.sent %} +
{% trans "Sent" %}
+
{{ mail.sent|date:"SHORT_DATETIME_FORMAT" }}
+ {% endif %} + {% if mail.retry_after and mail.status == "awaiting_retry" %} +
{% trans "Next attempt (estimate)" %}
+
{{ mail.retry_after|date:"SHORT_DATETIME_FORMAT" }}
+ {% endif %} + {% if mail.event %} +
{% trans "Event" %}
+
+ + {{ mail.event }} + +
+ {% endif %} + {% if mail.order %} +
{% trans "Order" %}
+
+ + {{ mail.order.code }}{% if mail.orderposition %}- + {{ mail.orderposition.positionid }}{% endif %} +
+ {% endif %} + {% if mail.customer %} +
{% trans "Customer" %}
+
+ {% icon "user fa-fw" %} + + {{ mail.customer }} + +
+ {% endif %} +
+
+ {% if mail.actual_attachments %} +
+ {% trans "Attachments" %}
+
    + {% for a in mail.actual_attachments %} +
  • + {% if a.type == "text/calendar" %} + {% icon "calendar-plus-o fa-fw" %} + {% elif a.type == "application/pdf" %} + {% icon "file-pdf-o fa-fw" %} + {% elif "image/" in a.type %} + {% icon "file-image-o fa-fw" %} + {% elif "msword" in a.type or "document" in a.type %} + {% icon "file-word-o fa-fw" %} + {% elif "excel" in a.type or "spreadsheet" in a.type %} + {% icon "file-excel-o fa-fw" %} + {% elif "powerpoint" in a.type or "presentation" in a.type %} + {% icon "file-powerpoint-o fa-fw" %} + {% elif "pkpass" in a.type %} + {% icon "qrcode fa-fw" %} + {% else %} + {% icon "file-o fa-fw" %} + {% endif %} + {{ a.name }} + + ({{ a.size|filesizeformat }}) + +
    • + {% endfor %} +
+
+ {% endif %} +
+
+
+
+ + +
+ {% if mail.is_failed %} +
+ + {{ mail.error }} + + 
{{ mail.error_detail }}
+
+ {% endif %} + {% if mail.body_html %} +
+ {% if mail.sensitive %} +
+

+ {% icon "eye-slash fa-4x" %} +

+

+ {% blocktrans trimmed %} + Sensitive content not shown for security reasons + {% endblocktrans %} +

+
+ {% else %} + {{ data_url|json_script:"mail_body_html" }} + {% endif %} +
+ {% endif %} +
+ {% if mail.sensitive %} +
+

+ {% icon "eye-slash fa-4x" %} +

+

+ {% blocktrans trimmed %} + Sensitive content not shown for security reasons + {% endblocktrans %} +

+
+ {% else %} + 
{{ mail.body_plain }}
+ {% endif %} +
+
+ 
{% for k, v in mail.headers.items %}{{ k }}: {{ v }}
{% endfor %}
+

+ {% trans "Additional headers will be added by the mail server and are not visible here." %} +

+
+
+ +
+ {% compress js %} + + {% endcompress %} +{% endblock %} diff --git a/src/pretix/control/templates/pretixcontrol/organizers/outgoing_mails.html b/src/pretix/control/templates/pretixcontrol/organizers/outgoing_mails.html new file mode 100644 index 0000000000..5de235cd95 --- /dev/null +++ b/src/pretix/control/templates/pretixcontrol/organizers/outgoing_mails.html @@ -0,0 +1,185 @@ +{% extends "pretixcontrol/organizers/base.html" %} +{% load i18n %} +{% load bootstrap3 %} +{% load urlreplace %} +{% load icon %} +{% block inner %} +

+ {% trans "Outgoing emails" %} +

+

+ {% blocktrans trimmed with days=days %} + This is an overview of all emails sent by your organizer account in the last {{ days }} days. + {% endblocktrans %} +

+ {% if mails|length == 0 and not filter_form.filtered %} +
+

+ {% blocktrans trimmed %} + You haven't sent any emails recently. + {% endblocktrans %} +

+
+ {% else %} +
+
+

{% trans "Filter" %}

+
+
+
+
+ {% bootstrap_field filter_form.query %} +
+
+ {% bootstrap_field filter_form.status %} +
+
+ {% bootstrap_field filter_form.event %} +
+
+
+ +
+
+
+
+ {% csrf_token %} + {% for field in filter_form %} + {{ field.as_hidden }} + {% endfor %} +
+ + + + + + + + + + + + {% if page_obj.paginator.num_pages > 1 %} + + + + + {% endif %} + + + {% for m in mails %} + + + + + + + + + + {% endfor %} + +
+ + {% trans "Subject" %}{% trans "Recipients" %}{% trans "Context" %}{% trans "Status" %}{% trans "Date" %} + + +
+ + + + {{ m.subject }} + + {% if m.sensitive %} + {% icon "eye-slash" %} + {% endif %} + + {{ m.to|join:", " }} + {% if m.cc %} +
{% trans "Cc" context "email" %}: {{ m.cc|join:", " }} + {% endif %} + {% if m.bcc %} +
{% trans "Bcc" context "email" %}: {{ m.bcc|join:", " }} + {% endif %} + 		+ {% if m.event %} +
+ {% icon "calendar fa-fw" %} + + {{ m.event }} + +
+ {% endif %} + {% if m.order %} +
+ {% icon "shopping-cart fa-fw" %} + + {{ m.order.code }}{% if m.orderposition %}-{{ m.orderposition.positionid }}{% endif %} +
+ {% endif %} + {% if m.customer %} +
+ {% icon "user fa-fw" %} + + {{ m.customer }} + +
+ {% endif %} + 		+ {% if m.status == "queued" %} + {% icon "clock-o" %} {% trans "queued" %} + {% elif m.status == "inflight" %} + {% icon "send" %} {% trans "being sent" %} + {% elif m.status == "awaiting_retry" %} + {% icon "repeat" %} {% trans "will be retried" %} + {% elif m.status == "failed" %} + {% icon "warning" %} {% trans "failed" %} + {% elif m.status == "bounced" %} + {% icon "exclamation-circle" %} {% trans "bounced" %} + {% elif m.status == "withheld" %} + {% icon "ban" %} {% trans "withheld" %} + {% elif m.status == "aborted" %} + {% icon "ban" %} {% trans "aborted" %} + {% elif m.status == "sent" %} + {% icon "check" %} {% trans "sent" %} + {% endif %} + + {{ m.created|date:"SHORT_DATETIME_FORMAT" }} + {% if m.sent %} +
+ {% trans "Sent:" %} {{ m.sent|date:"SHORT_DATETIME_FORMAT" }} + {% endif %} + 		+ {% icon "eye" %} +
+
+
+ + +
+
+ {% include "pretixcontrol/pagination.html" %} + {% endif %} +{% endblock %} diff --git a/src/pretix/control/urls.py b/src/pretix/control/urls.py index d310572e30..9d92c90815 100644 --- a/src/pretix/control/urls.py +++ b/src/pretix/control/urls.py @@ -38,8 +38,9 @@ from django.views.generic.base import RedirectView from pretix.control.views import ( auth, checkin, dashboards, datasync, discounts, event, geo, - global_settings, item, main, modelimport, oauth, orders, organizer, pdf, - search, shredder, subevents, typeahead, user, users, vouchers, waitinglist, + global_settings, item, mail, main, modelimport, oauth, orders, organizer, + pdf, search, shredder, subevents, typeahead, user, users, vouchers, + waitinglist, ) urlpatterns = [ @@ -240,6 +241,9 @@ urlpatterns = [ name='organizer.gate.edit'), re_path(r'^organizer/(?P[^/]+)/gate/(?P[^/]+)/delete$', organizer.GateDeleteView.as_view(), name='organizer.gate.delete'), + re_path(r'^organizer/(?P[^/]+)/outgoingmails$', mail.OutgoingMailListView.as_view(), name='organizer.outgoingmails'), + re_path(r'^organizer/(?P[^/]+)/outgoingmail/bulk_action$', mail.OutgoingMailBulkAction.as_view(), name='organizer.outgoingmails.bulk_action'), + re_path(r'^organizer/(?P[^/]+)/outgoingmail/(?P[0-9]+)/$', mail.OutgoingMailDetailView.as_view(), name='organizer.outgoingmail'), re_path(r'^organizer/(?P[^/]+)/teams$', organizer.TeamListView.as_view(), name='organizer.teams'), re_path(r'^organizer/(?P[^/]+)/team/add$', organizer.TeamCreateView.as_view(), name='organizer.team.add'), re_path(r'^organizer/(?P[^/]+)/team/(?P[^/]+)/$', organizer.TeamMemberView.as_view(), diff --git a/src/pretix/control/views/auth.py b/src/pretix/control/views/auth.py index 237246c7a3..7daa426505 100644 --- a/src/pretix/control/views/auth.py +++ b/src/pretix/control/views/auth.py @@ -66,7 +66,6 @@ from pretix.base.forms.auth import ( ) from pretix.base.metrics import pretix_failed_logins, pretix_successful_logins from pretix.base.models import TeamInvite, U2FDevice, User, WebAuthnDevice -from pretix.base.services.mail import SendMailException from pretix.helpers.http import get_client_ip, redirect_to_url from pretix.helpers.security import handle_login_source @@ -347,9 +346,6 @@ class Forgot(TemplateView): except User.DoesNotExist: logger.warning('Backend password reset for unregistered e-mail \"' + email + '\" requested.') - except SendMailException: - logger.exception('Sending password reset email to \"' + email + '\" failed.') - except RepeatedResetDenied: pass diff --git a/src/pretix/control/views/mail.py b/src/pretix/control/views/mail.py new file mode 100644 index 0000000000..6b775961ea --- /dev/null +++ b/src/pretix/control/views/mail.py @@ -0,0 +1,183 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-today pretix GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +import base64 +import logging + +from django.conf import settings +from django.contrib import messages +from django.core.exceptions import BadRequest +from django.db import transaction +from django.shortcuts import get_object_or_404, redirect +from django.urls import reverse +from django.utils.functional import cached_property +from django.utils.translation import ngettext +from django.views import View +from django.views.generic import DetailView, ListView + +from pretix.base.middleware import _merge_csp, _parse_csp, _render_csp +from pretix.base.models import OutgoingMail +from pretix.base.services.mail import mail_send_task +from pretix.control.forms.filter import OutgoingMailFilterForm +from pretix.control.permissions import OrganizerPermissionRequiredMixin +from pretix.control.views.organizer import OrganizerDetailViewMixin + +logger = logging.getLogger(__name__) + + +class OutgoingMailQueryMixin: + + @cached_property + def request_data(self): + if self.request.method == "POST": + d = self.request.POST + else: + d = self.request.GET + d = d.copy() + return d + + @cached_property + def filter_form(self): + return OutgoingMailFilterForm( + data=self.request_data, + request=self.request, + ) + + def get_queryset(self): + qs = self.request.organizer.outgoing_mails.select_related( + 'event', 'order', 'orderposition', 'customer' + ) + + if 'outgoingmail' in self.request_data and '__ALL' not in self.request_data: + qs = qs.filter( + id__in=self.request_data.getlist('outgoingmail') + ) + elif self.request.method == 'GET' or '__ALL' in self.request_data: + if self.filter_form.is_valid(): + qs = self.filter_form.filter_qs(qs) + else: + raise BadRequest("No mails selected") + + return qs + + +class OutgoingMailListView(OutgoingMailQueryMixin, OrganizerDetailViewMixin, OrganizerPermissionRequiredMixin, ListView): + model = OutgoingMail + template_name = 'pretixcontrol/organizers/outgoing_mails.html' + # Assume "the highest" permission level for now because emails could belog to any event, order, or customer. + # We plan to add a special permissoin in the future + permission = 'can_change_organizer_settings' + context_object_name = 'mails' + paginate_by = 100 + + def get_context_data(self, **kwargs): + ctx = super().get_context_data(**kwargs) + ctx['filter_form'] = self.filter_form + ctx['days'] = int(settings.OUTGOING_MAIL_RETENTION / (24 * 3600)) + return ctx + + +class OutgoingMailDetailView(OrganizerDetailViewMixin, OrganizerPermissionRequiredMixin, DetailView): + model = OutgoingMail + template_name = 'pretixcontrol/organizers/outgoing_mail.html' + permission = 'can_change_organizer_settings' + context_object_name = 'mail' + + def get_object(self, queryset=None): + return get_object_or_404(OutgoingMail, organizer=self.request.organizer, pk=self.kwargs.get('mail')) + + def dispatch(self, request, *args, **kwargs): + response = super().dispatch(request, *args, **kwargs) + if 'Content-Security-Policy' in response: + h = _parse_csp(response['Content-Security-Policy']) + else: + h = {} + csps = { + 'frame-src': ['data:'], + } + _merge_csp(h, csps) + response['Content-Security-Policy'] = _render_csp(h) + return response + + def get_context_data(self, **kwargs): + ctx = super().get_context_data(**kwargs) + if self.object.body_html: + ctx['data_url'] = "data:text/html;charset=utf-8;base64," + base64.b64encode(self.object.body_html.encode()).decode() + return ctx + + +class OutgoingMailBulkAction(OutgoingMailQueryMixin, OrganizerPermissionRequiredMixin, OrganizerDetailViewMixin, View): + permission = 'can_change_organizer_settings' + + @transaction.atomic + def post(self, request, *args, **kwargs): + if request.POST.get('action') == 'retry': + ids = set( + self.get_queryset().filter(status__in=OutgoingMail.STATUS_LIST_RETRYABLE).values_list("pk", flat=True) + ) + with transaction.atomic(): + OutgoingMail.objects.filter(pk__in=ids).update( + status=OutgoingMail.STATUS_QUEUED, + sent=None, + ) + self.request.organizer.log_action( + 'pretix.organizer.outgoingmails.retried', user=self.request.user, data={ + 'mails': list(ids) + }, save=False + ) + for i in ids: + mail_send_task.apply_async(kwargs={"outgoing_mail": i}) + + messages.success(request, ngettext( + "A retry of one email was scheduled.", + "A retry of {num} emails was scheduled.", + len(ids), + ).format(num=len(ids))) + elif request.POST.get('action') == 'abort': + ids = set( + self.get_queryset().filter( + status__in=(OutgoingMail.STATUS_QUEUED, OutgoingMail.STATUS_AWAITING_RETRY) + ).values_list("pk", flat=True) + ) + with transaction.atomic(): + OutgoingMail.objects.filter(pk__in=ids).update( + status=OutgoingMail.STATUS_ABORTED, + sent=None, + ) + self.request.organizer.log_action( + 'pretix.organizer.outgoingmails.aborted', user=self.request.user, data={ + 'mails': list(ids) + }, save=False + ) + for i in ids: + mail_send_task.apply_async(kwargs={"outgoing_mail": i}) + + messages.success(request, ngettext( + "One email was aborted and will not be sent.", + "{num} emails were aborted and will not be sent.", + len(ids), + ).format(num=len(ids))) + return redirect(self.get_success_url()) + + def get_success_url(self) -> str: + return reverse('control:organizer.outgoingmails', kwargs={ + 'organizer': self.request.organizer.slug, + }) diff --git a/src/pretix/control/views/orders.py b/src/pretix/control/views/orders.py index 77fe326849..cf98be74a9 100644 --- a/src/pretix/control/views/orders.py +++ b/src/pretix/control/views/orders.py @@ -98,9 +98,7 @@ from pretix.base.services.invoices import ( invoice_qualified, regenerate_invoice, transmit_invoice, ) from pretix.base.services.locking import LockTimeoutException -from pretix.base.services.mail import ( - SendMailException, prefix_subject, render_mail, -) +from pretix.base.services.mail import prefix_subject, render_mail from pretix.base.services.orders import ( OrderChangeManager, OrderError, approve_order, cancel_order, deny_order, extend_order, mark_order_expired, mark_order_refunded, @@ -1066,10 +1064,6 @@ class OrderPaymentConfirm(OrderView): messages.error(self.request, str(e)) except PaymentException as e: messages.error(self.request, str(e)) - except SendMailException: - messages.warning(self.request, - _('The payment has been marked as complete, but we were unable to send a ' - 'confirmation mail.')) else: messages.success(self.request, _('The payment has been marked as complete.')) else: @@ -1540,9 +1534,6 @@ class OrderTransition(OrderView): 'message': str(e) }) messages.error(self.request, str(e)) - except SendMailException: - messages.warning(self.request, _('The order has been marked as paid, but we were unable to send a ' - 'confirmation mail.')) else: messages.success(self.request, _('The payment has been created successfully.')) elif self.order.cancel_allowed() and to == 'c': @@ -1781,15 +1772,11 @@ class OrderResendLink(OrderView): permission = 'can_change_orders' def post(self, *args, **kwargs): - try: - if 'position' in kwargs: - p = get_object_or_404(self.order.positions, pk=kwargs['position']) - p.resend_link(user=self.request.user) - else: - self.order.resend_link(user=self.request.user) - except SendMailException: - messages.error(self.request, _('There was an error sending the mail. Please try again later.')) - return redirect(self.get_order_url()) + if 'position' in kwargs: + p = get_object_or_404(self.order.positions, pk=kwargs['position']) + p.resend_link(user=self.request.user) + else: + self.order.resend_link(user=self.request.user) messages.success(self.request, _('The email has been queued to be sent.')) return redirect(self.get_order_url()) @@ -2433,24 +2420,18 @@ class OrderSendMail(EventPermissionRequiredMixin, OrderViewMixin, FormView): } return self.get(self.request, *self.args, **self.kwargs) else: - try: - order.send_mail( - form.cleaned_data['subject'], email_template, - email_context, 'pretix.event.order.email.custom_sent', - self.request.user, auto_email=False, - attach_tickets=form.cleaned_data.get('attach_tickets', False), - invoices=form.cleaned_data.get('attach_invoices', []), - attach_other_files=[a for a in [ - self.request.event.settings.get('mail_attachment_new_order', as_type=str, default='')[len('file://'):] - ] if a] if form.cleaned_data.get('attach_new_order', False) else [], - ) - messages.success(self.request, - _('Your message has been queued and will be sent to {}.'.format(order.email))) - except SendMailException: - messages.error( - self.request, - _('Failed to send mail to the following user: {}'.format(order.email)) - ) + order.send_mail( + form.cleaned_data['subject'], email_template, + email_context, 'pretix.event.order.email.custom_sent', + self.request.user, auto_email=False, + attach_tickets=form.cleaned_data.get('attach_tickets', False), + invoices=form.cleaned_data.get('attach_invoices', []), + attach_other_files=[a for a in [ + self.request.event.settings.get('mail_attachment_new_order', as_type=str, default='')[len('file://'):] + ] if a] if form.cleaned_data.get('attach_new_order', False) else [], + ) + messages.success(self.request, + _('Your message has been queued and will be sent to {}.'.format(order.email))) return super(OrderSendMail, self).form_valid(form) def get_success_url(self): @@ -2503,23 +2484,19 @@ class OrderPositionSendMail(OrderSendMail): } return self.get(self.request, *self.args, **self.kwargs) else: - try: - position.send_mail( - form.cleaned_data['subject'], - email_template, - email_context, - 'pretix.event.order.position.email.custom_sent', - self.request.user, - attach_tickets=form.cleaned_data.get('attach_tickets', False), - attach_other_files=[a for a in [ - self.request.event.settings.get('mail_attachment_new_order', as_type=str, default='')[len('file://'):] - ] if a] if form.cleaned_data.get('attach_new_order', False) else [], - ) - messages.success(self.request, - _('Your message has been queued and will be sent to {}.'.format(position.attendee_email))) - except SendMailException: - messages.error(self.request, - _('Failed to send mail to the following user: {}'.format(position.attendee_email))) + position.send_mail( + form.cleaned_data['subject'], + email_template, + email_context, + 'pretix.event.order.position.email.custom_sent', + self.request.user, + attach_tickets=form.cleaned_data.get('attach_tickets', False), + attach_other_files=[a for a in [ + self.request.event.settings.get('mail_attachment_new_order', as_type=str, default='')[len('file://'):] + ] if a] if form.cleaned_data.get('attach_new_order', False) else [], + ) + messages.success(self.request, + _('Your message has been queued and will be sent to {}.'.format(position.attendee_email))) return super(OrderSendMail, self).form_valid(form) diff --git a/src/pretix/control/views/organizer.py b/src/pretix/control/views/organizer.py index 0cf66555cf..25977e9d8f 100644 --- a/src/pretix/control/views/organizer.py +++ b/src/pretix/control/views/organizer.py @@ -103,7 +103,7 @@ from pretix.base.plugins import ( PLUGIN_LEVEL_ORGANIZER, ) from pretix.base.services.export import multiexport, scheduled_organizer_export -from pretix.base.services.mail import SendMailException, mail, prefix_subject +from pretix.base.services.mail import mail, prefix_subject from pretix.base.signals import register_multievent_data_exporters from pretix.base.templatetags.rich_text import markdown_compile_email from pretix.base.views.tasks import AsyncAction @@ -1037,24 +1037,21 @@ class TeamMemberView(OrganizerDetailViewMixin, OrganizerPermissionRequiredMixin, return ctx def _send_invite(self, instance): - try: - mail( - instance.email, - _('pretix account invitation'), - 'pretixcontrol/email/invitation.txt', - { - 'user': self, - 'organizer': self.request.organizer.name, - 'team': instance.team.name, - 'url': build_global_uri('control:auth.invite', kwargs={ - 'token': instance.token - }) - }, - event=None, - locale=self.request.LANGUAGE_CODE - ) - except SendMailException: - pass # Already logged + mail( + instance.email, + _('pretix account invitation'), + 'pretixcontrol/email/invitation.txt', + { + 'user': self, + 'organizer': self.request.organizer.name, + 'team': instance.team.name, + 'url': build_global_uri('control:auth.invite', kwargs={ + 'token': instance.token + }) + }, + event=None, + locale=self.request.LANGUAGE_CODE + ) @transaction.atomic def post(self, request, *args, **kwargs): @@ -3027,6 +3024,7 @@ class CustomerDetailView(OrganizerDetailViewMixin, OrganizerPermissionRequiredMi locale=self.customer.locale, customer=self.customer, organizer=self.request.organizer, + sensitive=True, ) messages.success( self.request, diff --git a/src/pretix/control/views/users.py b/src/pretix/control/views/users.py index ec35d378f2..207d852e22 100644 --- a/src/pretix/control/views/users.py +++ b/src/pretix/control/views/users.py @@ -41,7 +41,6 @@ from hijack import signals from pretix.base.auth import get_auth_backends from pretix.base.models import User -from pretix.base.services.mail import SendMailException from pretix.control.forms.filter import UserFilterForm from pretix.control.forms.users import UserEditForm from pretix.control.permissions import AdministratorPermissionRequiredMixin @@ -139,11 +138,7 @@ class UserResetView(AdministratorPermissionRequiredMixin, RecentAuthenticationRe def post(self, request, *args, **kwargs): self.object = get_object_or_404(User, pk=self.kwargs.get("id")) - try: - self.object.send_password_reset() - except SendMailException: - messages.error(request, _('There was an error sending the mail. Please try again later.')) - return redirect(self.get_success_url()) + self.object.send_password_reset() self.object.log_action('pretix.control.auth.user.forgot_password.mail_sent', user=request.user) diff --git a/src/pretix/helpers/database.py b/src/pretix/helpers/database.py index 6caf7499f6..1289f4f1af 100644 --- a/src/pretix/helpers/database.py +++ b/src/pretix/helpers/database.py @@ -25,7 +25,7 @@ from django.conf import settings from django.core.exceptions import FieldDoesNotExist, ImproperlyConfigured from django.db import connection, transaction from django.db.models import ( - Aggregate, Expression, F, Field, Lookup, OrderBy, Value, + Aggregate, Expression, F, Field, JSONField, Lookup, OrderBy, Value, ) from django.utils.functional import lazy @@ -154,6 +154,19 @@ class NotEqual(Lookup): return '%s <> %s' % (lhs, rhs), params +@JSONField.register_lookup +class ContainsString(Lookup): + lookup_name = 'containsstring' + + def as_sql(self, compiler, connection): + if connection.vendor != "postgresql": + raise NotImplementedError("Lookup in JSON Array not supported on this database") + lhs, lhs_params = self.process_lhs(compiler, connection) + rhs, rhs_params = self.process_rhs(compiler, connection) + params = lhs_params + rhs_params + return '%s ? %s' % (lhs, rhs), params + + class PostgresWindowFrame(Expression): template = "%(frame_type)s BETWEEN %(start)s AND %(end)s" diff --git a/src/pretix/helpers/security.py b/src/pretix/helpers/security.py index baa8f3a159..2e5b545b57 100644 --- a/src/pretix/helpers/security.py +++ b/src/pretix/helpers/security.py @@ -32,7 +32,7 @@ from django_countries.fields import Country from geoip2.errors import AddressNotFoundError from pretix.base.i18n import language -from pretix.base.services.mail import SendMailException, mail +from pretix.base.services.mail import mail from pretix.helpers.http import get_client_ip from pretix.helpers.urls import build_absolute_uri @@ -159,21 +159,18 @@ def handle_login_source(user, request): }) if user.known_login_sources.count() > 1: # Do not send on first login or first login after introduction of this feature: - try: - with language(user.locale): - mail( - user.email, - _('Login from new source detected'), - 'pretixcontrol/email/login_notice.txt', - { - 'source': src, - 'country': Country(str(country)).name if country else _('Unknown country'), - 'instance': settings.PRETIX_INSTANCE_NAME, - 'url': build_absolute_uri('control:user.settings') - }, - event=None, - user=user, - locale=user.locale - ) - except SendMailException: - pass # Not much we can do + with language(user.locale): + mail( + user.email, + _('Login from new source detected'), + 'pretixcontrol/email/login_notice.txt', + { + 'source': src, + 'country': Country(str(country)).name if country else _('Unknown country'), + 'instance': settings.PRETIX_INSTANCE_NAME, + 'url': build_absolute_uri('control:user.settings') + }, + event=None, + user=user, + locale=user.locale + ) diff --git a/src/pretix/plugins/banktransfer/tasks.py b/src/pretix/plugins/banktransfer/tasks.py index 464fa6b16b..1d2144668c 100644 --- a/src/pretix/plugins/banktransfer/tasks.py +++ b/src/pretix/plugins/banktransfer/tasks.py @@ -56,7 +56,6 @@ from pretix.base.models import ( ) from pretix.base.payment import PaymentException from pretix.base.services.locking import LockTimeoutException -from pretix.base.services.mail import SendMailException from pretix.base.services.orders import change_payment_provider from pretix.base.services.tasks import TransactionAwareTask from pretix.celery_app import app @@ -72,13 +71,10 @@ def notify_incomplete_payment(o: Order): email_context = get_email_context(event=o.event, order=o, pending_sum=o.pending_sum) email_subject = o.event.settings.mail_subject_order_incomplete_payment - try: - o.send_mail( - email_subject, email_template, email_context, - 'pretix.event.order.email.expire_warning_sent' - ) - except SendMailException: - logger.exception('Reminder email could not be sent') + o.send_mail( + email_subject, email_template, email_context, + 'pretix.event.order.email.expire_warning_sent' + ) def cancel_old_payments(order): @@ -288,9 +284,6 @@ def _handle_transaction(trans: BankTransaction, matches: tuple, regex_match_to_s except Quota.QuotaExceededException: # payment confirmed but order status could not be set, no longer problem of this plugin cancel_old_payments(order) - except SendMailException: - # payment confirmed but order status could not be set, no longer problem of this plugin - cancel_old_payments(order) else: cancel_old_payments(order) diff --git a/src/pretix/plugins/banktransfer/views.py b/src/pretix/plugins/banktransfer/views.py index 29926aa2cb..9733e32034 100644 --- a/src/pretix/plugins/banktransfer/views.py +++ b/src/pretix/plugins/banktransfer/views.py @@ -58,7 +58,6 @@ from localflavor.generic.forms import BICFormField, IBANFormField from pretix.base.forms.widgets import DatePickerWidget from pretix.base.models import Event, Order, OrderPayment, OrderRefund, Quota -from pretix.base.services.mail import SendMailException from pretix.base.settings import SettingsSandbox from pretix.base.templatetags.money import money_filter from pretix.control.permissions import ( @@ -160,11 +159,6 @@ class ActionView(View): p.confirm(user=self.request.user) except Quota.QuotaExceededException: pass - except SendMailException: - return JsonResponse({ - 'status': 'error', - 'message': _('Problem sending email.') - }) trans.state = BankTransaction.STATE_VALID trans.save() trans.order.payments.filter( diff --git a/src/pretix/plugins/paypal/payment.py b/src/pretix/plugins/paypal/payment.py index 63cb035014..9acc5ae4b2 100644 --- a/src/pretix/plugins/paypal/payment.py +++ b/src/pretix/plugins/paypal/payment.py @@ -57,7 +57,6 @@ from pretix.base.decimal import round_decimal from pretix.base.forms import SecretKeySettingsField from pretix.base.models import Event, Order, OrderPayment, OrderRefund, Quota from pretix.base.payment import BasePaymentProvider, PaymentException -from pretix.base.services.mail import SendMailException from pretix.base.settings import SettingsSandbox from pretix.multidomain.urlreverse import build_absolute_uri from pretix.plugins.paypal.api import Api @@ -468,9 +467,6 @@ class Paypal(BasePaymentProvider): payment_obj.confirm() except Quota.QuotaExceededException as e: raise PaymentException(str(e)) - - except SendMailException: - messages.warning(request, _('There was an error sending the confirmation mail.')) return None def payment_pending_render(self, request, payment) -> str: diff --git a/src/pretix/plugins/paypal2/payment.py b/src/pretix/plugins/paypal2/payment.py index 84a0e5dbae..6f1d3774b9 100644 --- a/src/pretix/plugins/paypal2/payment.py +++ b/src/pretix/plugins/paypal2/payment.py @@ -54,7 +54,6 @@ from pretix.base.forms import SecretKeySettingsField from pretix.base.forms.questions import guess_country from pretix.base.models import Event, Order, OrderPayment, OrderRefund, Quota from pretix.base.payment import BasePaymentProvider, PaymentException -from pretix.base.services.mail import SendMailException from pretix.base.settings import SettingsSandbox from pretix.helpers import OF_SELF from pretix.helpers.urls import build_absolute_uri as build_global_uri @@ -821,9 +820,6 @@ class PaypalMethod(BasePaymentProvider): payment.confirm() except Quota.QuotaExceededException as e: raise PaymentException(str(e)) - - except SendMailException: - messages.warning(request, _('There was an error sending the confirmation mail.')) finally: if 'payment_paypal_oid' in request.session: del request.session['payment_paypal_oid'] diff --git a/src/pretix/plugins/sendmail/models.py b/src/pretix/plugins/sendmail/models.py index 0f5733493c..8235eec9c0 100644 --- a/src/pretix/plugins/sendmail/models.py +++ b/src/pretix/plugins/sendmail/models.py @@ -38,7 +38,6 @@ from pretix.base.models import ( fields, ) from pretix.base.models.base import LoggingMixin -from pretix.base.services.mail import SendMailException class ScheduledMail(models.Model): @@ -180,13 +179,10 @@ class ScheduledMail(models.Model): invoice_address=ia, event_or_subevent=self.subevent or e, ) - try: - o.send_mail(self.rule.subject, self.rule.template, email_ctx, - attach_ical=self.rule.attach_ical, - log_entry_type='pretix.plugins.sendmail.rule.order.email.sent') - o_sent = True - except SendMailException: - ... # ¯\_(ツ)_/¯ + o.send_mail(self.rule.subject, self.rule.template, email_ctx, + attach_ical=self.rule.attach_ical, + log_entry_type='pretix.plugins.sendmail.rule.order.email.sent') + o_sent = True if send_to_attendees: if not self.rule.all_products: @@ -195,31 +191,28 @@ class ScheduledMail(models.Model): positions = [p for p in positions if p.subevent_id == self.subevent_id] for p in positions: - try: - if p.attendee_email and (p.attendee_email != o.email or not o_sent): - email_ctx = get_email_context( - event=e, - order=o, - invoice_address=ia, - position=p, - event_or_subevent=self.subevent or e, - ) - p.send_mail(self.rule.subject, self.rule.template, email_ctx, - attach_ical=self.rule.attach_ical, - log_entry_type='pretix.plugins.sendmail.rule.order.position.email.sent') - elif not o_sent and o.email: - email_ctx = get_email_context( - event=e, - order=o, - invoice_address=ia, - event_or_subevent=self.subevent or e, - ) - o.send_mail(self.rule.subject, self.rule.template, email_ctx, - attach_ical=self.rule.attach_ical, - log_entry_type='pretix.plugins.sendmail.rule.order.email.sent') - o_sent = True - except SendMailException: - ... # ¯\_(ツ)_/¯ + if p.attendee_email and (p.attendee_email != o.email or not o_sent): + email_ctx = get_email_context( + event=e, + order=o, + invoice_address=ia, + position=p, + event_or_subevent=self.subevent or e, + ) + p.send_mail(self.rule.subject, self.rule.template, email_ctx, + attach_ical=self.rule.attach_ical, + log_entry_type='pretix.plugins.sendmail.rule.order.position.email.sent') + elif not o_sent and o.email: + email_ctx = get_email_context( + event=e, + order=o, + invoice_address=ia, + event_or_subevent=self.subevent or e, + ) + o.send_mail(self.rule.subject, self.rule.template, email_ctx, + attach_ical=self.rule.attach_ical, + log_entry_type='pretix.plugins.sendmail.rule.order.email.sent') + o_sent = True self.last_successful_order_id = o.pk diff --git a/src/pretix/plugins/sendmail/tasks.py b/src/pretix/plugins/sendmail/tasks.py index 172b1e8c4c..1c5f5e03ee 100644 --- a/src/pretix/plugins/sendmail/tasks.py +++ b/src/pretix/plugins/sendmail/tasks.py @@ -41,7 +41,7 @@ from pretix.base.i18n import language from pretix.base.models import ( CachedFile, Checkin, Event, InvoiceAddress, Order, User, ) -from pretix.base.services.mail import SendMailException, mail +from pretix.base.services.mail import mail from pretix.base.services.tasks import ProfiledEventTask from pretix.celery_app import app from pretix.helpers.format import format_map @@ -61,7 +61,6 @@ def send_mails_to_orders(event: Event, user: int, subject: dict, message: dict, recipients: str, filter_checkins: bool, not_checked_in: bool, checkin_lists: list, attachments: list = None, attach_tickets: bool = False, attach_ical: bool = False) -> None: - failures = [] user = User.objects.get(pk=user) if user else None subject = LazyI18nString(subject) message = LazyI18nString(message) @@ -121,70 +120,64 @@ def send_mails_to_orders(event: Event, user: int, subject: dict, message: dict, if subevents_to and p.subevent.date_from >= subevents_to: continue - try: - with language(o.locale, event.settings.region): - email_context = get_email_context(event=event, order=o, invoice_address=ia, position=p) - mail( - p.attendee_email, - subject, - message, - email_context, - event, - locale=o.locale, - order=o, - position=p, - attach_tickets=attach_tickets, - attach_ical=attach_ical, - attach_cached_files=attachments - ) - o.log_action( - 'pretix.plugins.sendmail.order.email.sent.attendee', - user=user, - data={ - 'position': p.positionid, - 'subject': format_map(subject.localize(o.locale), email_context), - 'message': format_map(message.localize(o.locale), email_context), - 'recipient': p.attendee_email, - 'attach_tickets': attach_tickets, - 'attach_ical': attach_ical, - 'attach_other_files': [], - 'attach_cached_files': attachments_for_log, - } - ) - except SendMailException: - failures.append(p.attendee_email) - - if send_to_order and o.email: - try: with language(o.locale, event.settings.region): - email_context = get_email_context(event=event, order=o, invoice_address=ia) + email_context = get_email_context(event=event, order=o, invoice_address=ia, position=p) mail( - o.email, + p.attendee_email, subject, message, email_context, event, locale=o.locale, order=o, + position=p, attach_tickets=attach_tickets, attach_ical=attach_ical, - attach_cached_files=attachments, + attach_cached_files=attachments ) o.log_action( - 'pretix.plugins.sendmail.order.email.sent', + 'pretix.plugins.sendmail.order.email.sent.attendee', user=user, data={ + 'position': p.positionid, 'subject': format_map(subject.localize(o.locale), email_context), 'message': format_map(message.localize(o.locale), email_context), - 'recipient': o.email, + 'recipient': p.attendee_email, 'attach_tickets': attach_tickets, 'attach_ical': attach_ical, 'attach_other_files': [], 'attach_cached_files': attachments_for_log, } ) - except SendMailException: - failures.append(o.email) + + if send_to_order and o.email: + with language(o.locale, event.settings.region): + email_context = get_email_context(event=event, order=o, invoice_address=ia) + mail( + o.email, + subject, + message, + email_context, + event, + locale=o.locale, + order=o, + attach_tickets=attach_tickets, + attach_ical=attach_ical, + attach_cached_files=attachments, + ) + o.log_action( + 'pretix.plugins.sendmail.order.email.sent', + user=user, + data={ + 'subject': format_map(subject.localize(o.locale), email_context), + 'message': format_map(message.localize(o.locale), email_context), + 'recipient': o.email, + 'attach_tickets': attach_tickets, + 'attach_ical': attach_ical, + 'attach_other_files': [], + 'attach_cached_files': attachments_for_log, + } + ) for chunk in _chunks(objects, 1000): orders = Order.objects.filter(pk__in=chunk, event=event) diff --git a/src/pretix/plugins/stripe/payment.py b/src/pretix/plugins/stripe/payment.py index b87a4efdb7..9900882c8f 100644 --- a/src/pretix/plugins/stripe/payment.py +++ b/src/pretix/plugins/stripe/payment.py @@ -71,7 +71,6 @@ from pretix.base.payment import ( BasePaymentProvider, PaymentException, WalletQueries, ) from pretix.base.plugins import get_all_plugins -from pretix.base.services.mail import SendMailException from pretix.base.settings import SettingsSandbox from pretix.helpers import OF_SELF from pretix.helpers.countries import CachedCountries @@ -1000,9 +999,6 @@ class StripeMethod(BasePaymentProvider): payment.confirm() except Quota.QuotaExceededException as e: raise PaymentException(str(e)) - - except SendMailException: - raise PaymentException(_('There was an error sending the confirmation mail.')) elif intent.status == 'processing': if request: messages.warning(request, _('Your payment is pending completion. We will inform you as soon as the ' diff --git a/src/pretix/presale/checkoutflow.py b/src/pretix/presale/checkoutflow.py index 0629af00f0..980b728612 100644 --- a/src/pretix/presale/checkoutflow.py +++ b/src/pretix/presale/checkoutflow.py @@ -1654,11 +1654,6 @@ class ConfirmStep(CartMixin, AsyncAction, TemplateFlowStep): order = Order.objects.get(id=value) return self.get_order_url(order) - def get_error_message(self, exception): - if exception.__class__.__name__ == 'SendMailException': - return _('There was an error sending the confirmation mail. Please try again later.') - return super().get_error_message(exception) - def get_error_url(self): return self.get_step_url(self.request) diff --git a/src/pretix/presale/views/customer.py b/src/pretix/presale/views/customer.py index fa0dd884b7..7552c6c0c9 100644 --- a/src/pretix/presale/views/customer.py +++ b/src/pretix/presale/views/customer.py @@ -325,6 +325,7 @@ class ResetPasswordView(FormView): locale=customer.locale, customer=customer, organizer=self.request.organizer, + sensitive=True, ) messages.success( self.request, diff --git a/src/pretix/presale/views/order.py b/src/pretix/presale/views/order.py index ef57ae47ee..990adeb4f5 100644 --- a/src/pretix/presale/views/order.py +++ b/src/pretix/presale/views/order.py @@ -77,7 +77,6 @@ from pretix.base.services.invoices import ( generate_cancellation, generate_invoice, invoice_pdf, invoice_pdf_task, invoice_qualified, ) -from pretix.base.services.mail import SendMailException from pretix.base.services.orders import ( OrderChangeManager, OrderError, _try_auto_refund, cancel_order, change_payment_provider, error_messages, @@ -656,10 +655,7 @@ class OrderPayChangeMethod(EventViewMixin, OrderDetailMixin, TemplateView): amount=Decimal('0.00'), fee=None ) - try: - p.confirm() - except SendMailException: - pass + p.confirm() else: p._mark_order_paid( payment_refund_sum=self.order.payment_refund_sum diff --git a/src/pretix/presale/views/user.py b/src/pretix/presale/views/user.py index a95118408e..7ff3f46380 100644 --- a/src/pretix/presale/views/user.py +++ b/src/pretix/presale/views/user.py @@ -32,8 +32,6 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under the License. -import logging - from django.conf import settings from django.contrib import messages from django.utils.functional import cached_property @@ -42,7 +40,7 @@ from django.views import View from django.views.generic import TemplateView from pretix.base.email import get_email_context -from pretix.base.services.mail import INVALID_ADDRESS, SendMailException, mail +from pretix.base.services.mail import INVALID_ADDRESS, mail from pretix.helpers.http import redirect_to_url from pretix.multidomain.urlreverse import eventreverse from pretix.presale.forms.user import ResendLinkForm @@ -83,13 +81,7 @@ class ResendLinkView(EventViewMixin, TemplateView): subject = self.request.event.settings.mail_subject_resend_all_links template = self.request.event.settings.mail_text_resend_all_links context = get_email_context(event=self.request.event, orders=orders) - try: - mail(user, subject, template, context, event=self.request.event, locale=self.request.LANGUAGE_CODE) - except SendMailException: - logger = logging.getLogger('pretix.presale.user') - logger.exception('A mail resending order links to {} could not be sent.'.format(user)) - messages.error(self.request, _('We have trouble sending emails right now, please check back later.')) - return self.get(request, *args, **kwargs) + mail(user, subject, template, context, event=self.request.event, locale=self.request.LANGUAGE_CODE) messages.success(self.request, _('If there were any orders by this user, they will receive an email with their order codes.')) return redirect_to_url(eventreverse(self.request.event, 'presale:event.index')) diff --git a/src/pretix/settings.py b/src/pretix/settings.py index 627275449a..f02c840e68 100644 --- a/src/pretix/settings.py +++ b/src/pretix/settings.py @@ -855,6 +855,8 @@ COUNTRIES_OVERRIDE = { DATA_UPLOAD_MAX_NUMBER_FIELDS = 25000 DATA_UPLOAD_MAX_MEMORY_SIZE = 10 * 1024 * 1024 # 10 MB +OUTGOING_MAIL_RETENTION = 14 * 24 * 3600 # 14 days in seonds + # File sizes are in MiB FILE_UPLOAD_MAX_SIZE_IMAGE = 1024 * 1024 * config.getint("pretix_file_upload", "max_size_image", fallback=10) FILE_UPLOAD_MAX_SIZE_FAVICON = 1024 * 1024 * config.getint("pretix_file_upload", "max_size_favicon", fallback=1) diff --git a/src/pretix/static/pretixcontrol/js/ui/outgoingmail.js b/src/pretix/static/pretixcontrol/js/ui/outgoingmail.js new file mode 100644 index 0000000000..f6457440ad --- /dev/null +++ b/src/pretix/static/pretixcontrol/js/ui/outgoingmail.js @@ -0,0 +1,43 @@ +function is_sandbox_supported() { + const iframe = document.createElement('iframe'); + return 'sandbox' in iframe; +} + +function safe_render(url, parent) { + // Estimate the height that prevents the user from having to scroll on two levels to see the full email + const height = ( + window.innerHeight - parent.parent().get(0).getBoundingClientRect().top - document.querySelector("footer").getBoundingClientRect().height - 20 + ) + "px"; + + const iframe = ( + // Per the HTML spec, a data: URL in an iframe is treated as its own origin: + // https://github.com/whatwg/html/pull/1756 + // It is unclear, if Firefox complies, and the behaviour around data URLs is quite wild: + // https://github.com/whatwg/html/issues/12091 + // Together with the sandbox attribute disallowing all JavaScript, and the fact + // that we sanitize the HTML before we even save it to the database, this should + // still be the safest way to render HTML in the context of our backend. + $("