diff --git a/src/pretix/testutils/__init__.py b/src/pretix/testutils/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/src/pretix/testutils/settings.py b/src/pretix/testutils/settings.py
new file mode 100644
index 0000000000..a0166d63f3
--- /dev/null
+++ b/src/pretix/testutils/settings.py
@@ -0,0 +1,32 @@
+import atexit
+import os
+import tempfile
+
+from pretix.settings import *  # NOQA
+
+tmpdir = tempfile.TemporaryDirectory()
+DATA_DIR = tmpdir.name
+LOG_DIR = os.path.join(DATA_DIR, 'logs')
+MEDIA_ROOT = os.path.join(DATA_DIR, 'media')
+
+atexit.register(tmpdir.cleanup)
+
+EMAIL_BACKEND = 'django.core.mail.outbox'
+
+COMPRESS_ENABLED = COMPRESS_OFFLINE = False
+
+PASSWORD_HASHERS = ['django.contrib.auth.hashers.MD5PasswordHasher']
+
+# Disable celery
+CELERY_ALWAYS_EAGER = True
+HAS_CELERY = False
+
+# Don't use redis
+SESSION_ENGINE = "django.contrib.sessions.backends.db"
+HAS_REDIS = False
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        'LOCATION': 'unique-snowflake',
+    }
+}
diff --git a/src/pytest.ini b/src/pytest.ini
index 77fb163990..929815d975 100644
--- a/src/pytest.ini
+++ b/src/pytest.ini
@@ -1,3 +1,2 @@
 [pytest]
 DJANGO_SETTINGS_MODULE=tests.settings
-
diff --git a/src/setup.cfg b/src/setup.cfg
index 6a95fc4d08..874f8cf4b8 100644
--- a/src/setup.cfg
+++ b/src/setup.cfg
@@ -1,7 +1,7 @@
 [flake8]
 ignore = N802,W503,E402
 max-line-length = 160
-exclude = migrations,.ropeproject,static,mt940.py,_static,build,make_testdata.py,tests/settings.py,pretix/base/models/__init__.py
+exclude = migrations,.ropeproject,static,mt940.py,_static,build,make_testdata.py,*/testutils/settings.py,tests/settings.py,pretix/base/models/__init__.py
 max-complexity = 11
 
 [isort]
diff --git a/src/tests/base/test_settings.py b/src/tests/base/test_settings.py
index ef50b197ed..e3b7305f95 100644
--- a/src/tests/base/test_settings.py
+++ b/src/tests/base/test_settings.py
@@ -2,6 +2,7 @@ from datetime import date, datetime, time
 from decimal import Decimal
 
 from django.core.files import File
+from django.core.files.storage import default_storage
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import TestCase
 from django.utils.timezone import now
@@ -161,6 +162,7 @@ class SettingsTestCase(TestCase):
 
     def test_serialize_file(self):
         val = SimpleUploadedFile("sample_invalid_image.jpg", b"file_content", content_type="image/jpeg")
+        default_storage.save(val.name, val)
         self.event.settings.set('test', val)
         self.event.settings._flush()
         self.assertIsInstance(self.event.settings.get('test', as_type=File), File)
@@ -168,6 +170,7 @@ class SettingsTestCase(TestCase):
 
     def test_detect_file_value(self):
         val = SimpleUploadedFile("sample_invalid_image.jpg", b"file_content", content_type="image/jpeg")
+        default_storage.save(val.name, val)
         self.event.settings.set('test', val)
         self.event.settings._flush()
         self.assertIsInstance(self.event.settings.get('test'), File)
diff --git a/src/tests/settings.py b/src/tests/settings.py
index a3b43e212b..ff57ee5ebc 100644
--- a/src/tests/settings.py
+++ b/src/tests/settings.py
@@ -1,31 +1,9 @@
 import os
+from pretix.testutils.settings import *  # NOQA
 
-from pretix.settings import *  # NOQA
 
 TEST_DIR = os.path.dirname(__file__)
 
 TEMPLATES[0]['DIRS'].append(os.path.join(TEST_DIR, 'templates'))  # NOQA
 
 INSTALLED_APPS.append('tests.testdummy')  # NOQA
-
-MEDIA_ROOT = os.path.join(TEST_DIR, 'media')
-
-EMAIL_BACKEND = 'django.core.mail.outbox'
-
-COMPRESS_ENABLED = COMPRESS_OFFLINE = False
-
-PASSWORD_HASHERS = ['django.contrib.auth.hashers.MD5PasswordHasher']
-
-# Disable celery
-CELERY_ALWAYS_EAGER = True
-HAS_CELERY = False
-
-# Don't use redis
-SESSION_ENGINE = "django.contrib.sessions.backends.db"
-HAS_REDIS = False
-CACHES = {
-    'default': {
-        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-        'LOCATION': 'unique-snowflake',
-    }
-}