diff --git a/src/pretix/base/email.py b/src/pretix/base/email.py
new file mode 100644
index 0000000000..c8f311d3c9
--- /dev/null
+++ b/src/pretix/base/email.py
@@ -0,0 +1,21 @@
+from smtplib import SMTPRecipientsRefused, SMTPSenderRefused
+
+from django.core.mail.backends.smtp import EmailBackend
+
+
+class CustomSMTPBackend(EmailBackend):
+
+    def test(self, from_addr):
+        try:
+            self.open()
+            self.connection.ehlo_or_helo_if_needed()
+            self.connection.rcpt("test@example.org")
+            (code, resp) = self.connection.mail(from_addr, [])
+            if code != 250:
+                raise SMTPSenderRefused(code, resp, from_addr)
+            senderrs = {}
+            (code, resp) = self.connection.rcpt('')
+            if (code != 250) and (code != 251):
+                raise SMTPRecipientsRefused(senderrs)
+        finally:
+            self.close()
diff --git a/src/pretix/base/models/event.py b/src/pretix/base/models/event.py
index 1e7762452e..465e5a9671 100644
--- a/src/pretix/base/models/event.py
+++ b/src/pretix/base/models/event.py
@@ -4,7 +4,6 @@ from datetime import datetime
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.core.mail import get_connection
-from django.core.mail.backends.smtp import EmailBackend
 from django.core.validators import RegexValidator
 from django.db import models
 from django.template.defaultfilters import date as _date
@@ -12,6 +11,7 @@ from django.utils.functional import cached_property
 from django.utils.timezone import now
 from django.utils.translation import ugettext_lazy as _
 
+from pretix.base.email import CustomSMTPBackend
 from pretix.base.i18n import I18nCharField
 from pretix.base.models.base import LoggedModel
 from pretix.base.settings import SettingsProxy
@@ -188,13 +188,13 @@ class Event(LoggedModel):
 
     def get_mail_backend(self, force_custom=False):
         if self.settings.smtp_use_custom or force_custom:
-            return EmailBackend(host=self.settings.smtp_host,
-                                port=self.settings.smtp_port,
-                                username=self.settings.smtp_username,
-                                password=self.settings.smtp_password,
-                                use_tls=self.settings.smtp_use_tls,
-                                use_ssl=self.settings.smtp_use_ssl,
-                                fail_silently=False)
+            return CustomSMTPBackend(host=self.settings.smtp_host,
+                                     port=self.settings.smtp_port,
+                                     username=self.settings.smtp_username,
+                                     password=self.settings.smtp_password,
+                                     use_tls=self.settings.smtp_use_tls,
+                                     use_ssl=self.settings.smtp_use_ssl,
+                                     fail_silently=False)
         else:
             return get_connection(fail_silently=False)
 
diff --git a/src/pretix/control/views/event.py b/src/pretix/control/views/event.py
index 0aad84cc60..6fe9c469ab 100644
--- a/src/pretix/control/views/event.py
+++ b/src/pretix/control/views/event.py
@@ -240,7 +240,7 @@ class MailSettings(EventPermissionRequiredMixin, FormView):
             if request.POST.get('test', '0').strip() == '1':
                 backend = self.request.event.get_mail_backend(force_custom=True)
                 try:
-                    backend.open()
+                    backend.test(self.request.event.settings.mail_from)
                 except Exception as e:
                     messages.warning(self.request, _('An error occured while contacting the SMTP server: %s') % str(e))
                 else:
@@ -251,8 +251,6 @@ class MailSettings(EventPermissionRequiredMixin, FormView):
                         messages.success(self.request, _('We\'ve been able to contact the SMTP server you configured. '
                                                          'Remember to check the "use custom SMTP server" checkbox, '
                                                          'otherwise your SMTP server will not be used.'))
-                finally:
-                    backend.close()
             else:
                 messages.success(self.request, _('Your changes have been saved.'))
             return redirect(self.get_success_url())