diff --git a/src/pretix/base/middleware.py b/src/pretix/base/middleware.py
index 1c38347046..bfcb3103d4 100644
--- a/src/pretix/base/middleware.py
+++ b/src/pretix/base/middleware.py
@@ -266,6 +266,22 @@ def _merge_csp(a, b):
 
 
 class SecurityMiddleware(MiddlewareMixin):
+    SAFE_TYPES = (
+        # CSP policies are only used for:
+        # - HTML and SVG in top-level contexts
+        # - SVG or JS Workers delivered in embedded contexts
+        # See: https://www.w3.org/TR/CSP2/#which-policy-applies
+        # Therefore, we can save bandwidth on not including our (sometimes huge) policy
+        # on API responses or CSS. We do however include it with other types as a precaution
+        # (whitelist instead of blacklist) and we also do not whitelist JavaScript in
+        # we ever add service workers to not break the protection of this feature:
+        # https://www.w3.org/TR/CSP2/#sandboxing-and-workers
+        'application/json',
+        'text/css',
+        # We used to skip CSP for PDF since it was necessary for inline previews in Safari,
+        # but at the moment it does not seem to be an issue to just send it.
+    )
+
     def process_response(self, request, resp):
         if settings.DEBUG and resp.status_code >= 400:
             # Don't use CSP on debug error page as it breaks of Django's fancy error
@@ -277,6 +293,11 @@ class SecurityMiddleware(MiddlewareMixin):
         # https://github.com/pretix/pretix/issues/765
         resp['P3P'] = 'CP=\"ALL DSP COR CUR ADM TAI OUR IND COM NAV INT\"'
 
+        if "Content-Type" in resp and resp["Content-Type"].split(";")[0] in self.SAFE_TYPES:
+            if 'Content-Security-Policy' in resp:
+                del resp['Content-Security-Policy']
+            return resp
+
         if not getattr(resp, '_csp_ignore', False):
             resp['Content-Security-Policy'] = _render_csp(self._build_csp(request, resp))
         elif 'Content-Security-Policy' in resp:
diff --git a/src/pretix/control/views/pdf.py b/src/pretix/control/views/pdf.py
index e8c5d855d7..fc935df80e 100644
--- a/src/pretix/control/views/pdf.py
+++ b/src/pretix/control/views/pdf.py
@@ -70,7 +70,6 @@ class BaseEditorView(EventPermissionRequiredMixin, TemplateView):
         if 'placeholders' in request.GET:
             return self.get_placeholders_help(request)
         resp = super().get(request, *args, **kwargs)
-        resp._csp_ignore = True
         return resp
 
     def get_placeholders_help(self, request):
diff --git a/src/pretix/plugins/paypal2/views.py b/src/pretix/plugins/paypal2/views.py
index 49c75208f1..56b118230d 100644
--- a/src/pretix/plugins/paypal2/views.py
+++ b/src/pretix/plugins/paypal2/views.py
@@ -162,7 +162,6 @@ class XHRView(View):
 
         paypal_order = prov._create_paypal_order(request, None, cart_total)
         r = JsonResponse(paypal_order.dict() if paypal_order else {})
-        r._csp_ignore = True
         return r
 
 
diff --git a/src/pretix/presale/views/theme.py b/src/pretix/presale/views/theme.py
index 045addd93f..69dc7454cb 100644
--- a/src/pretix/presale/views/theme.py
+++ b/src/pretix/presale/views/theme.py
@@ -48,7 +48,6 @@ def theme_css(request, **kwargs):
     obj = getattr(request, "event", request.organizer)
     css = get_theme_vars_css(obj, widget=False)
     resp = HttpResponse(css, content_type="text/css")
-    resp._csp_ignore = True
     resp["Access-Control-Allow-Origin"] = "*"
     if "version" in request.GET:
         resp["Expires"] = http_date(time.time() + 3600 * 24 * 30)
diff --git a/src/pretix/presale/views/widget.py b/src/pretix/presale/views/widget.py
index 498ec30c3e..90f441963b 100644
--- a/src/pretix/presale/views/widget.py
+++ b/src/pretix/presale/views/widget.py
@@ -164,7 +164,6 @@ def widget_css(request, version, **kwargs):
     css = f"/* v{version} */\n" + theme_css + widget_css
 
     resp = FileResponse(css, content_type='text/css')
-    resp._csp_ignore = True
     resp['Access-Control-Allow-Origin'] = '*'
     return resp
 
@@ -246,7 +245,6 @@ def widget_js(request, version, lang, **kwargs):
     cached_js = cache.get(cache_prefix)
     if cached_js and not settings.DEBUG:
         resp = HttpResponse(cached_js, content_type='text/javascript')
-        resp._csp_ignore = True
         resp['Access-Control-Allow-Origin'] = '*'
         return resp
 
@@ -278,7 +276,6 @@ def widget_js(request, version, lang, **kwargs):
             gs.settings.set(checksum_key, checksum)
             cache.set(cache_prefix, data, 3600 * 4)
         resp = HttpResponse(data, content_type='text/javascript')
-    resp._csp_ignore = True
     resp['Access-Control-Allow-Origin'] = '*'
     return resp
 
@@ -414,7 +411,6 @@ class WidgetAPIProductList(EventListMixin, View):
         self.post_process(data)
         resp = JsonResponse(data)
         resp['Access-Control-Allow-Origin'] = '*'
-        resp._csp_ignore = True
         return resp
 
     def get(self, request, *args, **kwargs):