harmonize date-format for checkin CSV-export

Currently translated at 100.0% (5093 of 5093 strings)

Translation: pretix/pretix
Translate-URL: https://translate.pretix.eu/projects/pretix/pretix/de_Informal/

powered by weblate

.github/dependabot.yml

@@ -9,6 +9,7 @@ updates:
     directory: "/src"
     schedule:
       interval: "daily"
+    versioning-strategy: increase
   - package-ecosystem: "npm"
     directory: "/src/pretix/static/npm_dir"
     schedule:

.github/workflows/docs.yml

@@ -14,16 +14,22 @@ on:
       - 'src/pretix/static/**'
       - 'src/tests/**'
 
+permissions:
+  contents: read  #  to fetch code (actions/checkout)
+
+env:
+  FORCE_COLOR: 1
+
 jobs:
   spelling:
     name: Spellcheck
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.11
       - uses: actions/cache@v1
         with:
           path: ~/.cache/pip
@@ -31,7 +37,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Install system packages
-        run: sudo apt update && sudo apt install enchant hunspell aspell-en
+        run: sudo apt update && sudo apt install enchant-2 hunspell aspell-en
       - name: Install Dependencies
         run: pip3 install -Ur requirements.txt
         working-directory: ./doc

.github/workflows/strings.yml

@@ -12,16 +12,22 @@ on:
       - 'doc/**'
       - 'src/pretix/locale/**'
 
+permissions:
+  contents: read  #  to fetch code (actions/checkout)
+
+env:
+  FORCE_COLOR: 1
+
 jobs:
   compile:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Check gettext syntax
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.11
       - uses: actions/cache@v1
         with:
           path: ~/.cache/pip
@@ -40,14 +46,14 @@ jobs:
         run: python manage.py compilejsi18n
         working-directory: ./src
   spelling:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Spellcheck
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.11
       - uses: actions/cache@v1
         with:
           path: ~/.cache/pip
@@ -55,7 +61,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Install system packages
-        run: sudo apt update && sudo apt install enchant hunspell hunspell-de-de aspell-en aspell-de
+        run: sudo apt update && sudo apt install enchant-2 hunspell hunspell-de-de aspell-en aspell-de
       - name: Install Dependencies
         run: pip3 install -e ".[dev]"
         working-directory: ./src

.github/workflows/style.yml

@@ -12,16 +12,22 @@ on:
       - 'src/pretix/locale/**'
       - 'src/pretix/static/**'
 
+permissions:
+  contents: read  #  to fetch code (actions/checkout)
+
+env:
+  FORCE_COLOR: 1
+
 jobs:
   isort:
     name: isort
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.11
       - uses: actions/cache@v1
         with:
           path: ~/.cache/pip
@@ -36,13 +42,13 @@ jobs:
         working-directory: ./src
   flake:
     name: flake8
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.11
       - uses: actions/cache@v1
         with:
           path: ~/.cache/pip
@@ -57,13 +63,13 @@ jobs:
         working-directory: ./src
   licenseheader:
     name: licenseheaders
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.11
       - name: Install Dependencies
         run: pip3 install licenseheaders
       - name: Run licenseheaders

.github/workflows/tests.yml

@@ -12,28 +12,34 @@ on:
       - 'doc/**'
       - 'src/pretix/locale/**'
 
+permissions:
+  contents: read  #  to fetch code (actions/checkout)
+
+env:
+  FORCE_COLOR: 1
+
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Tests
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9"]
+        python-version: ["3.9", "3.10", "3.11"]
         database: [sqlite, postgres, mysql]
         exclude:
           - database: mysql
-            python-version: "3.8"
+            python-version: "3.9"
           - database: mysql
+            python-version: "3.11"
+          - database: sqlite
             python-version: "3.9"
           - database: sqlite
-            python-version: "3.7"
-          - database: sqlite
-            python-version: "3.8"
+            python-version: "3.10"
     steps:
       - uses: actions/checkout@v2
       - uses: getong/mariadb-action@v1.1
         with:
-          mariadb version: '10.4'
+          mariadb version: '10.10'
           mysql database: 'pretix'
           mysql root password: ''
         if: matrix.database == 'mysql'
@@ -55,9 +61,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Install system dependencies
-        run: sudo apt update && sudo apt install gettext mariadb-client-10.3
+        run: sudo apt update && sudo apt install gettext mariadb-client
       - name: Install Python dependencies
-        run: pip3 install -e ".[dev]" mysqlclient psycopg2-binary
+        run: pip3 install --ignore-requires-python -e ".[dev]" mysqlclient psycopg2-binary  # We ignore that flake8 needs newer python as we don't run flake8 during tests
         working-directory: ./src
       - name: Run checks
         run: python manage.py check
@@ -75,5 +81,6 @@ jobs:
         uses: codecov/codecov-action@v1
         with:
           file: src/coverage.xml
+          token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true
-        if: matrix.database == 'postgres' && matrix.python-version == '3.8'
+        if: matrix.database == 'postgres' && matrix.python-version == '3.11'

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-bullseye
+FROM python:3.11-bullseye
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

doc/admin/config.rst

@@ -117,6 +117,9 @@ Example::
 ``loglevel``
     Set console and file log level (``DEBUG``, ``INFO``, ``WARNING``, ``ERROR`` or ``CRITICAL``). Defaults to ``INFO``.
 
+``request_id_header``
+    Specifies the name of a header that should be used for logging request IDs. Off by default.
+
 Locale settings
 ---------------
 
@@ -138,7 +141,7 @@ Database settings
 Example::
 
     [database]
-    backend=mysql
+    backend=postgresql
     name=pretix
     user=pretix
     password=abcd
@@ -146,7 +149,7 @@ Example::
     port=3306
 
 ``backend``
-    One of ``mysql``, ``sqlite3``, ``oracle`` and ``postgresql``.
+    One of ``mysql`` (deprecated), ``sqlite3`` and ``postgresql``.
     Default: ``sqlite3``.
 
     If you use MySQL, be sure to create your database using
@@ -160,7 +163,7 @@ Example::
     Connection details for the database connection. Empty by default.
 
 ``galera``
-    Indicates if the database backend is a MySQL/MariaDB Galera cluster and
+    (Deprecated) Indicates if the database backend is a MySQL/MariaDB Galera cluster and
     turns on some optimizations/special case handlers. Default: ``False``
 
 .. _`config-replica`:
@@ -191,7 +194,7 @@ Example::
 
     [urls]
     media=/media/
-    static=/media/
+    static=/static/
 
 ``media``
     The URL to be used to serve user-uploaded content. You should not need to modify
@@ -396,9 +399,9 @@ The two ``transport_options`` entries can be omitted in most cases.
 If they are present they need to be a valid JSON dictionary.
 For possible entries in that dictionary see the `Celery documentation`_.
 
-To use redis with sentinels set the broker or backend to ``sentinel://sentinel_host_1:26379;sentinal_host_2:26379/0``
+To use redis with sentinels set the broker or backend to ``sentinel://sentinel_host_1:26379;sentinel_host_2:26379/0``
 and the respective transport_options to ``{"master_name":"mymaster"}``.
-If your redis instances behind the sentinel have a password use ``sentinel://:my_password@sentinel_host_1:26379;sentinal_host_2:26379/0``.
+If your redis instances behind the sentinel have a password use ``sentinel://:my_password@sentinel_host_1:26379;sentinel_host_2:26379/0``.
 If your redis sentinels themselves have a password set the transport_options to ``{"master_name":"mymaster","sentinel_kwargs":{"password":"my_password"}}``.
 
 Sentry

doc/admin/index.rst

@@ -14,4 +14,5 @@ This documentation is for everyone who wants to install pretix on a server.
    maintainance
    scaling
    errors
+   mysql2postgres
    indexes

doc/admin/installation/docker_smallscale.rst

@@ -14,7 +14,7 @@ This has some trade-offs in terms of performance and isolation but allows a rath
              get it right. If you're not feeling comfortable managing a Linux server, check out our hosting and service
              offers at `pretix.eu`_.
 
-We tested this guide on the Linux distribution **Debian 8.0** but it should work very similar on other
+We tested this guide on the Linux distribution **Debian 11.0** but it should work very similar on other
 modern distributions, especially on all systemd-based ones.
 
 Requirements
@@ -26,7 +26,7 @@ installation guides):
 * `Docker`_
 * A SMTP server to send out mails, e.g. `Postfix`_ on your machine or some third-party server you have credentials for
 * A HTTP reverse proxy, e.g. `nginx`_ or Apache to allow HTTPS connections
-* A `PostgreSQL`_ 9.6+, `MySQL`_ 5.7+, or MariaDB 10.2.7+ database server
+* A `PostgreSQL`_ 9.6+ database server
 * A `redis`_ server
 
 We also recommend that you use a firewall, although this is not a pretix-specific recommendation. If you're new to
@@ -58,9 +58,6 @@ directory writable to the user that runs pretix inside the docker container::
 Database
 --------
 
-.. warning:: **Please use PostgreSQL for all new installations**. If you need to go for MySQL, make sure you run
-             **MySQL 5.7 or newer** or **MariaDB 10.2.7 or newer**.
-
 Next, we need a database and a database user. We can create these with any kind of database managing tool or directly on
 our database's shell. Please make sure that UTF8 is used as encoding for the best compatibility. You can check this with
 the following command::
@@ -86,13 +83,6 @@ Restart PostgreSQL after you changed these files::
 
 If you have a firewall running, you should also make sure that port 5432 is reachable from the ``172.17.0.1/16`` subnet.
 
-For MySQL, you can either also use network-based connections or mount the ``/var/run/mysqld/mysqld.sock`` socket into the docker container.
-When using MySQL, make sure you set the character set of the database to ``utf8mb4``, e.g. like this::
-
-    mysql > CREATE DATABASE pretix DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
-
-You will also need to make sure that ``sql_mode`` in your ``my.cnf`` file does **not** include ``ONLY_FULL_GROUP_BY``.
-
 Redis
 -----
 
@@ -152,15 +142,13 @@ Fill the configuration file ``/etc/pretix/pretix.cfg`` with the following conten
     trust_x_forwarded_proto=on
 
     [database]
-    ; Replace postgresql with mysql for MySQL
     backend=postgresql
     name=pretix
     user=pretix
     ; Replace with the password you chose above
     password=*********
     ; In most docker setups, 172.17.0.1 is the address of the docker host. Adjust
-    ; this to wherever your database is running, e.g. the name of a linked container
-    ; or of a mounted MySQL socket.
+    ; this to wherever your database is running, e.g. the name of a linked container.
     host=172.17.0.1
 
     [mail]
@@ -212,8 +200,6 @@ named ``/etc/systemd/system/pretix.service`` with the following content::
     [Install]
     WantedBy=multi-user.target
 
-When using MySQL and socket mounting, you'll need the additional flag ``-v /var/run/mysqld:/var/run/mysqld`` in the command.
-
 You can now run the following commands
 to enable and start the service::
 
@@ -339,7 +325,6 @@ workers, e.g. ``docker run … taskworker -Q notifications --concurrency 32``.
 .. _nginx: https://botleg.com/stories/https-with-lets-encrypt-and-nginx/
 .. _Let's Encrypt: https://letsencrypt.org/
 .. _pretix.eu: https://pretix.eu/
-.. _MySQL: https://dev.mysql.com/doc/refman/5.7/en/linux-installation-apt-repo.html
 .. _PostgreSQL: https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-20-04
 .. _redis: https://blog.programster.org/debian-8-install-redis-server/
 .. _ufw: https://en.wikipedia.org/wiki/Uncomplicated_Firewall

doc/admin/installation/manual_smallscale.rst

@@ -12,7 +12,7 @@ solution with many things readily set-up, look at :ref:`dockersmallscale`.
              get it right. If you're not feeling comfortable managing a Linux server, check out our hosting and service
              offers at `pretix.eu`_.
 
-We tested this guide on the Linux distribution **Debian 10.0** but it should work very similar on other
+We tested this guide on the Linux distribution **Debian 11.6** but it should work very similar on other
 modern distributions, especially on all systemd-based ones.
 
 Requirements
@@ -23,7 +23,7 @@ installation guides):
 
 * A SMTP server to send out mails, e.g. `Postfix`_ on your machine or some third-party server you have credentials for
 * A HTTP reverse proxy, e.g. `nginx`_ or Apache to allow HTTPS connections
-* A `PostgreSQL`_ 9.6+, `MySQL`_ 5.7+, or MariaDB 10.2.7+ database server
+* A `PostgreSQL`_ 11+ database server
 * A `redis`_ server
 * A `nodejs`_ installation
 
@@ -47,9 +47,6 @@ In this guide, all code lines prepended with a ``#`` symbol are commands that yo
 Database
 --------
 
-.. warning:: **Please use PostgreSQL for all new installations**. If you need to go for MySQL, make sure you run
-             **MySQL 5.7 or newer** or **MariaDB 10.2.7 or newer**.
-
 Having the database server installed, we still need a database and a database user. We can create these with any kind
 of database managing tool or directly on our database's shell. Please make sure that UTF8 is used as encoding for the
 best compatibility. You can check this with the following command::
@@ -61,12 +58,6 @@ For PostgreSQL database creation, we would do::
     # sudo -u postgres createuser pretix
     # sudo -u postgres createdb -O pretix pretix
 
-When using MySQL, make sure you set the character set of the database to ``utf8mb4``, e.g. like this::
-
-    mysql > CREATE DATABASE pretix DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
-
-You will also need to make sure that ``sql_mode`` in your ``my.cnf`` file does **not** include ``ONLY_FULL_GROUP_BY``.
-
 Package dependencies
 --------------------
 
@@ -74,7 +65,7 @@ To build and run pretix, you will need the following debian packages::
 
     # apt-get install git build-essential python-dev python3-venv python3 python3-pip \
                       python3-dev libxml2-dev libxslt1-dev libffi-dev zlib1g-dev libssl-dev \
-                      gettext libpq-dev libmariadb-dev libjpeg-dev libopenjp2-7-dev
+                      gettext libpq-dev libjpeg-dev libopenjp2-7-dev
 
 Config file
 -----------
@@ -97,16 +88,12 @@ Fill the configuration file ``/etc/pretix/pretix.cfg`` with the following conten
     trust_x_forwarded_proto=on
 
     [database]
-    ; For MySQL, replace with "mysql"
     backend=postgresql
     name=pretix
     user=pretix
-    ; For MySQL, enter the user password. For PostgreSQL on the same host,
-    ; we don't need one because we can use peer authentification if our
-    ; PostgreSQL user matches our unix user.
+    ; For PostgreSQL on the same host, we don't need a password because we can use
+    ; peer authentication if our PostgreSQL user matches our unix user.
     password=
-    ; For MySQL, use local socket, e.g. /var/run/mysqld/mysqld.sock
-    ; For a remote host, supply an IP address
     ; For local postgres authentication, you can leave it empty
     host=
 
@@ -140,11 +127,7 @@ We now install pretix, its direct dependencies and gunicorn::
 
     (venv)$ pip3 install pretix gunicorn
 
-If you're running MySQL, also install the client library::
-
-    (venv)$ pip3 install mysqlclient
-
-Note that you need Python 3.7 or newer. You can find out your Python version using ``python -V``.
+Note that you need Python 3.9 or newer. You can find out your Python version using ``python -V``.
 
 We also need to create a data directory::
 
@@ -318,12 +301,32 @@ example::
     (venv)$ python -m pretix rebuild
     # systemctl restart pretix-web pretix-worker
 
+System updates
+--------------
+
+After system updates, such as updates to a new Ubuntu or Debian release, you might be using a new Python version.
+That's great, but requires some adjustments. First, adjust any old version paths in your nginx configuration file.
+Then, re-create your Python environment::
+
+    $ source /var/pretix/venv/bin/activate
+    (venv)$ pip3 freeze > /tmp/pip-backup.txt
+    $ rm -rf /var/pretix/venv
+    $ python3 -m venv /var/pretix/venv
+    $ source /var/pretix/venv/bin/activate
+    (venv)$ pip3 install -U pip wheel setuptools
+    (venv)$ pip3 install -r /tmp/pip-backup.txt
+
+Then, proceed like after any plugin installation::
+
+    (venv)$ python -m pretix migrate
+    (venv)$ python -m pretix rebuild
+    (venv)$ python -m pretix updatestyles
+    # systemctl restart pretix-web pretix-worker
 
 .. _Postfix: https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-postfix-as-a-send-only-smtp-server-on-ubuntu-16-04
 .. _nginx: https://botleg.com/stories/https-with-lets-encrypt-and-nginx/
 .. _Let's Encrypt: https://letsencrypt.org/
 .. _pretix.eu: https://pretix.eu/
-.. _MySQL: https://dev.mysql.com/doc/refman/5.7/en/linux-installation-apt-repo.html
 .. _PostgreSQL: https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-20-04
 .. _redis: https://blog.programster.org/debian-8-install-redis-server/
 .. _ufw: https://en.wikipedia.org/wiki/Uncomplicated_Firewall

doc/admin/maintainance.rst

@@ -17,11 +17,11 @@ Backups
 There are essentially two things which you should create backups of:
 
 Database
-    Your SQL database (MySQL or PostgreSQL). This is critical and you should **absolutely
-    always create automatic backups of your database**. There are tons of tutorials on the
-    internet on how to do this, and the exact process depends on the choice of your database.
-    For MySQL, see ``mysqldump`` and for PostgreSQL, see the ``pg_dump`` tool. You probably
-    want to create a cronjob that does the backups for you on a regular schedule.
+    Your SQL database. This is critical and you should **absolutely always create automatic
+    backups of your database**. There are tons of tutorials on the internet on how to do this,
+    and the exact process depends on the choice of your database. For PostgreSQL, see the
+    ``pg_dump`` tool. You probably want to create a cronjob that does the backups for you on a
+    regular schedule.
 
 Data directory
     The data directory of your pretix configuration might contain some things that you should

doc/admin/mysql2postgres.rst

@@ -0,0 +1,148 @@
+.. highlight:: none
+
+Migrating from MySQL/MariaDB to PostgreSQL
+==========================================
+
+Our recommended database for all production installations is PostgreSQL. Support for MySQL/MariaDB will be removed in
+pretix 5.0.
+
+In order to follow this guide, your pretix installation needs to be a version that fully supports MySQL/MariaDB. If you
+already upgraded to pretix 5.0, downgrade back to the last 4.x release using ``pip``.
+
+.. note:: We have tested this guide carefully, but we can't assume any liability for its correctness. The data loss
+          risk should be low as long as pretix is not running while you do the migration. If you are a pretix Enterprise
+          customer, feel free to reach out in advance if you want us to support you along the way.
+
+Update database schema
+----------------------
+
+Before you start, make sure your database schema is up to date::
+
+    # sudo -u pretix -s
+    $ source /var/pretix/venv/bin/activate
+    (venv)$ python -m pretix migrate
+
+Install PostgreSQL
+------------------
+
+Now, install and set up a PostgreSQL server. For a local installation on Debian or Ubuntu, use::
+
+    # apt install postgresql
+
+Having the database server installed, we still need a database and a database user. We can create these with any kind
+of database managing tool or directly on our database's shell. Please make sure that UTF8 is used as encoding for the
+best compatibility. You can check this with the following command::
+
+    # sudo -u postgres psql -c 'SHOW SERVER_ENCODING'
+
+Without Docker
+""""""""""""""
+
+For our standard manual installation, create the database and user like this::
+
+    # sudo -u postgres createuser pretix
+    # sudo -u postgres createdb -O pretix pretix
+
+With Docker
+"""""""""""
+
+For our standard docker installation, create the database and user like this::
+
+    # sudo -u postgres createuser -P pretix
+    # sudo -u postgres createdb -O pretix pretix
+
+Make sure that your database listens on the network. If PostgreSQL on the same same host as docker, but not inside a docker container, we recommend that you just listen on the Docker interface by changing the following line in ``/etc/postgresql/<version>/main/postgresql.conf``::
+
+    listen_addresses = 'localhost,172.17.0.1'
+
+You also need to add a new line to ``/etc/postgresql/<version>/main/pg_hba.conf`` to allow network connections to this user and database::
+
+    host    pretix          pretix          172.17.0.1/16           md5
+
+Restart PostgreSQL after you changed these files::
+
+    # systemctl restart postgresql
+
+If you have a firewall running, you should also make sure that port 5432 is reachable from the ``172.17.0.1/16`` subnet.
+
+Of course, instead of all this you can also run a PostgreSQL docker container and link it to the pretix container.
+
+Stop pretix
+-----------
+
+To prevent any more changes to your data, stop pretix from running::
+
+    # systemctl stop pretix-web pretix-worker
+
+Change configuration
+--------------------
+
+Change the database configuration in your ``/etc/pretix/pretix.cfg`` file::
+
+    [database]
+    backend=postgresql
+    name=pretix
+    user=pretix
+    password=  ; only required for docker or remote database, can be kept empty for local auth
+    host=      ; set to 172.17.0.1 in docker setup, keep empty for local auth
+
+
+Create database schema
+-----------------------
+
+To create the schema in your new PostgreSQL database, use the following commands::
+
+    # sudo -u pretix -s
+    $ source /var/pretix/venv/bin/activate
+    (venv)$ python -m pretix migrate
+
+
+Migrate your data
+-----------------
+
+Install ``pgloader``::
+
+    # apt install pgloader
+
+Create a new file ``/tmp/pretix.load``, replacing the MySQL and PostgreSQL connection strings with the correct user names, passwords, and/or database names::
+
+    LOAD DATABASE
+        FROM mysql://pretix:password@localhost/pretix  -- replace with mysql://username:password@hostname/dbname
+        INTO postgresql:///pretix                      -- replace with dbname
+
+    WITH data only, include no drop, truncate, disable triggers,
+         create no indexes, drop indexes, reset sequences
+
+    ALTER SCHEMA 'pretix' RENAME TO 'public'           -- replace pretix with the name of the MySQL database
+
+    ALTER TABLE NAMES MATCHING ~/.*/
+        SET SCHEMA 'public'
+
+    SET timezone TO '+00:00'
+
+    SET PostgreSQL PARAMETERS
+         maintenance_work_mem to '128MB',
+         work_mem to '12MB';
+
+Then, run::
+
+    # sudo -u postgres pgloader /tmp/pretix.load
+
+The output should end with a table summarizing the results for every table. You can ignore warnings about type casts
+and missing constraints.
+
+Afterwards, delete the file again::
+
+    # rm -rf /tmp/pretix.load
+
+Start pretix
+------------
+
+Now, restart pretix. Maybe stop your MySQL server as a verification step that you are no longer using it::
+
+    # systemctl stop mariadb
+    # systemctl start pretix-web pretix-worker
+
+And you're done! After you've verified everything has been copied correctly, you can delete the old MySQL database.
+
+.. note:: Don't forget to update your backup process to back up your PostgreSQL database instead of your MySQL database now.

doc/admin/scaling.rst

@@ -42,7 +42,7 @@ A pretix installation usually consists of the following components which run per
 
 * ``pretix-worker`` is a Celery-based application that processes tasks that should be run asynchronously outside of the web application process.
 
-* A **SQL database** keeps all the important data and processes the actual transactions. We recommend using PostgreSQL, but MySQL/MariaDB works as well.
+* A **PostgreSQL database** keeps all the important data and processes the actual transactions.
 
 * A **web server** that terminates TLS and HTTP connections and forwards them to ``pretix-web``. In some cases, e.g. when serving static files, the web servers might return a response directly. We recommend using ``nginx``.
 
@@ -74,7 +74,7 @@ We recommend reading up on tuning your web server for high concurrency. For ngin
 processes and the number of connections each worker process accepts. Double-check that TLS session caching works, because TLS
 handshakes can get really expensive.
 
-During a traffic peak, your web server will be able to make us of more CPU resources, while memory usage will stay comparatively low,
+During a traffic peak, your web server will be able to make use of more CPU resources, while memory usage will stay comparatively low,
 so if you invest in more hardware here, invest in more and faster CPU cores.
 
 Make sure that pretix' static files (such as CSS and JavaScript assets) as well as user-uploaded media files (event logos, etc)

doc/api/fundamentals.rst

@@ -48,10 +48,11 @@ Possible permissions are:
 Compatibility
 -------------
 
-We currently see pretix' API as a beta-stage feature. We therefore do not give any guarantees
-for compatibility between feature releases of pretix (such as 1.5 and 1.6). However, as always,
-we try not to break things when we don't need to. Any backwards-incompatible changes will be
-prominently noted in the release notes.
+We try to avoid any breaking changes to our API to avoid hassle on your end. If possible, we'll
+build new features in a way that keeps all pre-existing API usage unchanged. In some cases,
+this might not be possible or only possible with restrictions. In these case, any
+backwards-incompatible changes will be prominently noted in the "Changes to the REST API"
+section of our release notes. If possible, we will announce them multiple releases in advance.
 
 We treat the following types of changes as *backwards-compatible* so we ask you to make sure
 that your clients can deal with them properly:
@@ -60,6 +61,7 @@ that your clients can deal with them properly:
 * Support of new HTTP methods for a given API endpoint
 * Support of new query parameters for a given API endpoint
 * New fields contained in API responses
+* New possible values of enumeration-like fields
 * Response body structure or message texts on failed requests (``4xx``, ``5xx`` response codes)
 
 We treat the following types of changes as *backwards-incompatible*:
@@ -190,6 +192,9 @@ Relative date         *either* String in ISO 8601  ``"2017-12-27"``,
 File                  URL in responses, ``file:``  ``"https://…"``, ``"file:…"``
                       specifiers in requests
                       (see below).
+Date range            *either* two dates separated ``2022-03-18/2022-03-23``, ``2022-03-18/``,
+                      by ``/`` *or* the name of a  ``/2022-03-23``, ``week_this``, ``week_next``,
+                      defined range.               ``month_this``
 ===================== ============================ ===================================
 
 Query parameters

doc/api/resources/carts.rst

@@ -43,10 +43,6 @@ seat                                  objects                    The assigned se
 └ seat_guid                           string                     Identifier of the seat within the seating plan
 ===================================== ========================== =======================================================
 
-.. versionchanged:: 3.0
-
-   This ``seat`` attribute has been added.
-
 .. versionchanged:: 4.14
 
    This ``is_bundled`` attribute has been added and the cart creation endpoints have been updated.

doc/api/resources/checkinlists.rst

@@ -39,23 +39,6 @@ exit_all_at                           datetime                   Automatically c
 addon_match                           boolean                    If ``true``, tickets on this list can be redeemed by scanning their parent ticket if this still leads to an unambiguous match.
 ===================================== ========================== =======================================================
 
-.. versionchanged:: 3.9
-
-    The ``subevent`` attribute may now be ``null`` inside event series. The ``allow_multiple_entries``,
-    ``allow_entry_after_exit``, and ``rules`` attributes have been added.
-
-.. versionchanged:: 3.11
-
-    The ``subevent_match`` and ``exclude`` query parameters have been added.
-
-.. versionchanged:: 3.12
-
-    The ``exit_all_at`` attribute has been added.
-
-.. versionchanged:: 3.17
-
-    The ``ends_after`` and ``expand`` query parameters have been added.
-
 .. versionchanged:: 4.12
 
     The ``addon_match`` attribute has been added.
@@ -115,6 +98,8 @@ Endpoints
    :query string ends_after: Exclude all check-in lists attached to a sub-event that is already in the past at the given time.
    :query string expand: Expand a field into a full object. Currently only ``subevent`` is supported. Can be passed multiple times.
    :query string exclude: Exclude a field from the output, e.g. ``checkin_count``. Can be used as a performance optimization. Can be passed multiple times.
+   :query string ordering: Manually set the ordering of results. Valid fields to be used are ``id``, ``name``, and ``subevent__date_from``,
+                           Default: ``subevent__date_from,name``
    :param organizer: The ``slug`` field of the organizer to fetch
    :param event: The ``slug`` field of the event to fetch
    :statuscode 200: no error

doc/api/resources/events.rst

@@ -52,34 +52,9 @@ sales_channels                        list                       A list of sales
 ===================================== ========================== =======================================================
 
 
-.. versionchanged:: 3.3
-
-   The attributes ``geo_lat`` and ``geo_lon`` have been added.
-
-.. versionchanged:: 3.4
-
-   The attribute ``timezone`` has been added.
-
-.. versionchanged:: 3.7
-
-   The attribute ``item_meta_properties`` has been added.
-
-.. versionchanged:: 3.12
-
-   The attribute ``valid_keys`` has been added.
-
-.. versionchanged:: 3.14
-
-    The attribute ``sales_channels`` has been added.
-
-
 Endpoints
 ---------
 
-.. versionchanged:: 3.3
-
-    The events resource can now be filtered by meta data attributes.
-
 .. versionchanged:: 4.0
 
     The ``clone_from`` parameter has been added to the event creation endpoint.
@@ -567,10 +542,6 @@ information about the properties.
 .. warning:: This API is intended for advanced users. Even though we take care to validate your input, you will be
              able to break your event using this API by creating situations of conflicting settings. Please take care.
 
-.. versionchanged:: 3.6
-
-   Initial support for settings has been added to the API.
-
 .. http:get:: /api/v1/organizers/(organizer)/events/(event)/settings/
 
    Get current values of event settings.

doc/api/resources/exporters.rst

@@ -6,10 +6,6 @@ Data exporters
 pretix and it's plugins include a number of data exporters that allow you to bulk download various data from pretix in
 different formats. This page shows you how to use these exporters through the API.
 
-.. versionchanged:: 3.13
-
-   This feature has been added to the API.
-
 .. warning::
 
    While we consider the methods listed on this page to be a stable API, the availability and specific input field

doc/api/resources/giftcards.rst

@@ -40,10 +40,6 @@ text                                  string                     Custom text of
 Endpoints
 ---------
 
-.. versionadded:: 3.14
-
-   The transaction list endpoint was added.
-
 .. http:get:: /api/v1/organizers/(organizer)/giftcards/
 
    Returns a list of all gift cards issued by a given organizer.
@@ -257,10 +253,6 @@ Endpoints
         "value": "15.37"
       }
 
-   .. versionchanged:: 3.5
-
-      This endpoint now returns status code ``409`` if the transaction would lead to a negative gift card value.
-
    :param organizer: The ``slug`` field of the organizer to modify
    :param id: The ``id`` field of the gift card to modify
    :query boolean include_accepted: Also show gift cards issued by other organizers that are accepted by this organizer.

doc/api/resources/invoices.rst

@@ -108,16 +108,6 @@ internal_reference                    string                     Customer's refe
 ===================================== ========================== =======================================================
 
 
-.. versionchanged:: 3.4
-
-   The attribute ``lines.number`` has been added.
-
-.. versionchanged:: 3.17
-
-   The attribute ``invoice_to_*``, ``invoice_from_*``, ``custom_field``, ``lines.item``, ``lines.variation``, ``lines.event_date_from``,
-   ``lines.event_date_to``, and ``lines.attendee_name`` have been added.
-   ``refers`` now returns an invoice number including the prefix.
-
 .. versionchanged:: 4.1
 
    The attributes ``fee_type`` and ``fee_internal_type`` have been added.

doc/api/resources/item_variations.rst

@@ -43,8 +43,13 @@ available_until                       datetime                   The last date t
 hide_without_voucher                  boolean                    If ``true``, this variation is only shown during the voucher
                                                                  redemption process, but not in the normal shop
                                                                  frontend.
+meta_data                             object                     Values set for event-specific meta data parameters.
 ===================================== ========================== =======================================================
 
+.. versionchanged:: 4.16
+
+   The ``meta_data`` attribute has been added.
+
 Endpoints
 ---------
 
@@ -94,6 +99,7 @@ Endpoints
             "default_price": "223.00",
             "price": 223.0,
             "original_price": null,
+            "meta_data": {}
           },
           {
             "id": 3,
@@ -108,7 +114,8 @@ Endpoints
             "description": {},
             "position": 1,
             "default_price": null,
-            "price": 15.0
+            "price": 15.0,
+            "meta_data": {}
           }
         ]
       }
@@ -161,7 +168,8 @@ Endpoints
         "available_until": null,
         "hide_without_voucher": false,
         "description": null,
-        "position": 0
+        "position": 0,
+        "meta_data": {}
       }
 
    :param organizer: The ``slug`` field of the organizer to fetch
@@ -198,7 +206,8 @@ Endpoints
         "available_until": null,
         "hide_without_voucher": false,
         "description": null,
-        "position": 0
+        "position": 0,
+        "meta_data": {}
       }
 
    **Example response**:
@@ -225,7 +234,8 @@ Endpoints
         "available_until": null,
         "hide_without_voucher": false,
         "description": null,
-        "position": 0
+        "position": 0,
+        "meta_data": {}
       }
 
    :param organizer: The ``slug`` field of the organizer of the event/item to create a variation for
@@ -283,7 +293,8 @@ Endpoints
         "available_until": null,
         "hide_without_voucher": false,
         "description": null,
-        "position": 1
+        "position": 1,
+        "meta_data": {}
       }
 
    :param organizer: The ``slug`` field of the organizer to modify

doc/api/resources/items.rst

@@ -35,6 +35,12 @@ tax_rule                              integer                    The internal ID
 admission                             boolean                    ``true`` for items that grant admission to the event
                                                                  (such as primary tickets) and ``false`` for others
                                                                  (such as add-ons or merchandise).
+personalized                          boolean                    ``true`` for items that require personalization according
+                                                                 to event settings. Only affects system-level fields, not
+                                                                 custom questions. Currently only allowed for products with
+                                                                 ``admission`` set to ``true``. For backwards compatibility,
+                                                                 when creating new items and this field is not given, it defaults
+                                                                 to the same value as ``admission``.
 position                              integer                    An integer, used for sorting
 picture                               file                       A product picture to be displayed in the shop
                                                                  (can be ``null``).
@@ -123,6 +129,7 @@ variations                            list of objects            A list with one
 ├ hide_without_voucher                boolean                    If ``true``, this variation is only shown during the voucher
                                                                  redemption process, but not in the normal shop
                                                                  frontend.
+├ meta_data                           object                     Values set for event-specific meta data parameters.
 └ position                            integer                    An integer, used for sorting
 addons                                list of objects            Definition of add-ons that can be chosen for this item.
                                                                  Only writable during creation,
@@ -146,14 +153,6 @@ bundles                               list of objects            Definition of b
 meta_data                             object                     Values set for event-specific meta data parameters.
 ===================================== ========================== =======================================================
 
-.. versionchanged:: 3.7
-
-   The attribute ``meta_data`` has been added.
-
-.. versionchanged:: 3.10
-
-   The attribute ``multi_allowed`` has been added to ``addons``.
-
 .. versionchanged:: 4.0
 
    The attributes ``require_membership``, ``require_membership_types``, ``grant_membership_type``, ``grant_membership_duration_like_event``,
@@ -163,6 +162,10 @@ meta_data                             object                     Values set for
 
    The attributes ``require_membership_hidden`` attribute has been added.
 
+.. versionchanged:: 4.16
+
+   The ``variations[x].meta_data`` attribute has been added. The ``personalized`` attribute has been added.
+
 Notes
 -----
 
@@ -216,6 +219,7 @@ Endpoints
             "tax_rate": "0.00",
             "tax_rule": 1,
             "admission": false,
+            "personalized": false,
             "issue_giftcard": false,
             "meta_data": {},
             "position": 0,
@@ -255,6 +259,7 @@ Endpoints
                  "available_until": null,
                  "hide_without_voucher": false,
                  "description": null,
+                 "meta_data": {},
                  "position": 0
               },
               {
@@ -270,6 +275,7 @@ Endpoints
                  "available_until": null,
                  "hide_without_voucher": false,
                  "description": null,
+                 "meta_data": {},
                  "position": 1
               }
             ],
@@ -330,6 +336,7 @@ Endpoints
         "tax_rate": "0.00",
         "tax_rule": 1,
         "admission": false,
+        "personalized": false,
         "issue_giftcard": false,
         "meta_data": {},
         "position": 0,
@@ -369,6 +376,7 @@ Endpoints
              "available_from": null,
              "available_until": null,
              "hide_without_voucher": false,
+             "meta_data": {},
              "position": 0
           },
           {
@@ -384,6 +392,7 @@ Endpoints
              "available_until": null,
              "hide_without_voucher": false,
              "description": null,
+             "meta_data": {},
              "position": 1
           }
         ],
@@ -425,6 +434,7 @@ Endpoints
         "tax_rate": "0.00",
         "tax_rule": 1,
         "admission": false,
+        "personalized": false,
         "issue_giftcard": false,
         "meta_data": {},
         "position": 0,
@@ -463,6 +473,7 @@ Endpoints
              "available_until": null,
              "hide_without_voucher": false,
              "description": null,
+             "meta_data": {},
              "position": 0
           },
           {
@@ -478,6 +489,7 @@ Endpoints
              "available_until": null,
              "hide_without_voucher": false,
              "description": null,
+             "meta_data": {},
              "position": 1
           }
         ],
@@ -507,6 +519,7 @@ Endpoints
         "tax_rate": "0.00",
         "tax_rule": 1,
         "admission": false,
+        "personalized": false,
         "issue_giftcard": false,
         "meta_data": {},
         "position": 0,
@@ -546,6 +559,7 @@ Endpoints
              "available_until": null,
              "hide_without_voucher": false,
              "description": null,
+             "meta_data": {},
              "position": 0
           },
           {
@@ -561,6 +575,7 @@ Endpoints
              "available_until": null,
              "hide_without_voucher": false,
              "description": null,
+             "meta_data": {},
              "position": 1
           }
         ],
@@ -621,6 +636,7 @@ Endpoints
         "tax_rate": "0.00",
         "tax_rule": 1,
         "admission": false,
+        "personalized": false,
         "issue_giftcard": false,
         "meta_data": {},
         "position": 0,
@@ -660,6 +676,7 @@ Endpoints
              "available_until": null,
              "hide_without_voucher": false,
              "description": null,
+             "meta_data": {},
              "position": 0
           },
           {
@@ -675,6 +692,7 @@ Endpoints
              "available_until": null,
              "hide_without_voucher": false,
              "description": null,
+             "meta_data": {},
              "position": 1
           }
         ],

doc/api/resources/orders.rst

@@ -98,30 +98,6 @@ last_modified                         datetime                   Last modificati
 ===================================== ========================== =======================================================
 
 
-.. versionchanged:: 3.5
-
-   The ``order.fees.canceled`` attribute has been added.
-
-.. versionchanged:: 3.8
-
-   The ``reactivate`` operation has been added.
-
-.. versionchanged:: 3.10
-
-   The ``search`` query parameter has been added.
-
-.. versionchanged:: 3.11
-
-   The ``exclude`` and ``subevent_after`` query parameter has been added.
-
-.. versionchanged:: 3.13
-
-   The ``subevent_before`` query parameter has been added.
-
-.. versionchanged:: 3.14
-
-   The ``phone`` attribute has been added.
-
 .. versionchanged:: 4.0
 
    The ``customer`` attribute has been added.
@@ -142,6 +118,10 @@ last_modified                         datetime                   Last modificati
 
    The ``order.fees.id`` attribute has been added.
 
+.. versionchanged:: 4.15
+
+   The ``include`` query parameter has been added.
+
 
 .. _order-position-resource:
 
@@ -178,6 +158,7 @@ tax_rule                              integer                    The ID of the u
 secret                                string                     Secret code printed on the tickets for validation
 addon_to                              integer                    Internal ID of the position this position is an add-on for (or ``null``)
 subevent                              integer                    ID of the date inside an event series this position belongs to (or ``null``).
+discount                              integer                    ID of a discount that has been used during the creation of this position in some way (or ``null``).
 pseudonymization_id                   string                     A random ID, e.g. for use in lead scanning apps
 checkins                              list of objects            List of **successful** check-ins with this ticket
 ├ id                                  integer                    Internal ID of the check-in event
@@ -205,27 +186,6 @@ pdf_data                              object                     Data object req
                                                                  ``pdf_data=true`` query parameter to your request.
 ===================================== ========================== =======================================================
 
-.. versionchanged:: 3.3
-
-  The ``url`` of a ticket ``download`` can now also return a ``text/uri-list`` instead of a file. See
-  :ref:`order-position-ticket-download` for details.
-
-.. versionchanged:: 3.5
-
-  The attribute ``canceled`` has been added.
-
-.. versionchanged:: 3.8
-
-  The attributes ``company``, ``street``, ``zipcode``, ``city``, ``country``, and ``state`` have been added.
-
-.. versionchanged:: 3.9
-
-  The ``checkin.type`` attribute has been added.
-
-.. versionchanged:: 3.16
-
-   Answers to file questions are now returned as an URL.
-
 .. _order-payment-resource:
 
 Order payment resource
@@ -272,15 +232,20 @@ created                               datetime                   Date and time o
 comment                               string                     Reason for refund (shown to the customer in some cases, can be ``null``).
 execution_date                        datetime                   Date and time of completion of this refund (or ``null``)
 provider                              string                     Identification string of the payment provider
+details                               object                     Refund-specific information. This is a dictionary
+                                                                 with various fields that can be different between
+                                                                 payment providers, versions, payment states, etc. If
+                                                                 you read this field, you always need to be able to
+                                                                 deal with situations where values that you expect are
+                                                                 missing. Mostly, the field contains various IDs that
+                                                                 can be used for matching with other systems. If a
+                                                                 payment provider does not implement this feature,
+                                                                 the object is empty.
 ===================================== ========================== =======================================================
 
 List of all orders
 ------------------
 
-.. versionchanged:: 3.5
-
-   The ``include_canceled_positions`` and ``include_canceled_fees`` query parameters have been added.
-
 .. http:get:: /api/v1/organizers/(organizer)/events/(event)/orders/
 
    Returns a list of all orders within a given event.
@@ -371,6 +336,7 @@ List of all orders
                 "secret": "z3fsn8jyufm5kpk768q69gkbyr5f4h6w",
                 "addon_to": null,
                 "subevent": null,
+                "discount": null,
                 "pseudonymization_id": "MQLJvANO3B",
                 "seat": null,
                 "checkins": [
@@ -447,6 +413,7 @@ List of all orders
    :query datetime subevent_after: Only return orders that contain a ticket for a subevent taking place after the given date. This is an exclusive after, and it considers the **end** of the subevent (or its start, if the end is not set).
    :query datetime subevent_before: Only return orders that contain a ticket for a subevent taking place after the given date. This is an exclusive before, and it considers the **start** of the subevent.
    :query string exclude: Exclude a field from the output, e.g. ``fees`` or ``positions.downloads``. Can be used as a performance optimization. Can be passed multiple times.
+   :query string include: Include only the given field in the output, e.g. ``fees`` or ``positions.downloads``. Can be used as a performance optimization. Can be passed multiple times. ``include`` is applied before ``exclude``, so ``exclude`` takes precedence.
    :param organizer: The ``slug`` field of the organizer to fetch
    :param event: The ``slug`` field of the event to fetch
    :resheader X-Page-Generated: The server time at the beginning of the operation. If you're using this API to fetch
@@ -458,10 +425,6 @@ List of all orders
 Fetching individual orders
 --------------------------
 
-.. versionchanged:: 3.5
-
-   The ``include_canceled_positions`` and ``include_canceled_fees`` query parameters have been added.
-
 .. http:get:: /api/v1/organizers/(organizer)/events/(event)/orders/(code)/
 
    Returns information on one order, identified by its order code.
@@ -546,6 +509,7 @@ Fetching individual orders
             "secret": "z3fsn8jyufm5kpk768q69gkbyr5f4h6w",
             "addon_to": null,
             "subevent": null,
+            "discount": null,
             "pseudonymization_id": "MQLJvANO3B",
             "seat": null,
             "checkins": [
@@ -1035,10 +999,6 @@ Creating orders
 Order state operations
 ----------------------
 
-.. versionchanged:: 3.12
-
-   The ``mark_paid`` operation now takes a ``send_email`` parameter.
-
 .. http:post:: /api/v1/organizers/(organizer)/events/(event)/orders/(code)/mark_paid/
 
    Marks a pending or expired order as successfully paid.
@@ -1440,10 +1400,6 @@ Sending e-mails
 List of all order positions
 ---------------------------
 
-.. versionchanged:: 3.5
-
-   The ``include_canceled_positions`` and ``include_canceled_fees`` query parameters have been added.
-
 .. http:get:: /api/v1/organizers/(organizer)/events/(event)/orderpositions/
 
    Returns a list of all order positions within a given event.
@@ -1487,6 +1443,7 @@ List of all order positions
             "tax_rule": null,
             "tax_value": "0.00",
             "secret": "z3fsn8jyufm5kpk768q69gkbyr5f4h6w",
+            "discount": null,
             "pseudonymization_id": "MQLJvANO3B",
             "seat": null,
             "addon_to": null,
@@ -1597,6 +1554,7 @@ Fetching individual positions
         "secret": "z3fsn8jyufm5kpk768q69gkbyr5f4h6w",
         "addon_to": null,
         "subevent": null,
+        "discount": null,
         "pseudonymization_id": "MQLJvANO3B",
         "seat": null,
         "checkins": [
@@ -1696,10 +1654,6 @@ Order position ticket download
 Manipulating individual positions
 ---------------------------------
 
-.. versionchanged:: 3.15
-
-   The ``PATCH`` method has been added for individual positions.
-
 .. versionchanged:: 4.8
 
    The ``PATCH`` method now supports changing items, variations, subevents, seats, prices, and tax rules.
@@ -2006,14 +1960,6 @@ otherwise, such as splitting an order or changing fees.
 Order payment endpoints
 -----------------------
 
-.. versionchanged:: 3.6
-
-   Payments can now be created through the API.
-
-.. versionchanged:: 3.12
-
-   The ``confirm`` operation now takes a ``send_email`` parameter.
-
 .. http:get:: /api/v1/organizers/(organizer)/events/(event)/orders/(code)/payments/
 
    Returns a list of all payments for an order.
@@ -2319,6 +2265,7 @@ Order refund endpoints
             "created": "2017-12-01T10:00:00Z",
             "execution_date": "2017-12-04T12:13:12Z",
             "comment": "Cancellation",
+            "details": {},
             "provider": "banktransfer"
           }
         ]
@@ -2362,6 +2309,7 @@ Order refund endpoints
         "created": "2017-12-01T10:00:00Z",
         "execution_date": "2017-12-04T12:13:12Z",
         "comment": "Cancellation",
+        "details": {},
         "provider": "banktransfer"
       }
 
@@ -2419,6 +2367,7 @@ Order refund endpoints
         "created": "2017-12-01T10:00:00Z",
         "execution_date": null,
         "comment": "Cancellation",
+        "details": {},
         "provider": "manual"
       }
 
@@ -2548,10 +2497,6 @@ Revoked ticket secrets
 
 With some non-default ticket secret generation methods, a list of revoked ticket secrets is required for proper validation.
 
-.. versionchanged:: 3.12
-
-   Added revocation lists.
-
 .. http:get:: /api/v1/organizers/(organizer)/events/(event)/revokedsecrets/
 
    Returns a list of all revoked secrets within a given event.

doc/api/resources/organizers.rst

@@ -109,10 +109,6 @@ information about the properties.
 .. warning:: This API is intended for advanced users. Even though we take care to validate your input, you will be
              able to break your shops using this API by creating situations of conflicting settings. Please take care.
 
-.. versionchanged:: 3.14
-
-   Initial support for settings has been added to the API.
-
 .. http:get:: /api/v1/organizers/(organizer)/settings/
 
    Get current values of organizer settings.

doc/api/resources/questions.rst

@@ -76,26 +76,9 @@ dependency_value                      string                     An old version
                                                                  for one value. **Deprecated.**
 ===================================== ========================== =======================================================
 
-.. versionchanged:: 3.5
-
-  The attribute ``help_text`` has been added.
-
-.. versionchanged:: 3.14
-
-  The attributes ``valid_*`` have been added.
-
-.. versionchanged:: 3.18
-
-  The attribute ``valid_file_portrait`` have been added.
-
 Endpoints
 ---------
 
-.. versionchanged:: 1.15
-
-   The questions endpoint has been extended by the filter queries ``ask_during_checkin``, ``requred``, and
-   ``identifier``.
-
 .. http:get:: /api/v1/organizers/(organizer)/events/(event)/questions/
 
    Returns a list of all questions within a given event.

doc/api/resources/quotas.rst

@@ -36,10 +36,6 @@ available_number                      integer                    Number of avail
                                                                  slightly out of date. ``null`` means unlimited.
 ===================================== ========================== =======================================================
 
-.. versionchanged:: 3.10
-
-   The attribute ``release_after_exit`` has been added.
-
 .. versionchanged:: 4.1
 
     The ``with_availability`` query parameter has been added.

doc/api/resources/subevents.rst

@@ -59,29 +59,13 @@ seat_category_mapping                 object                     An object mappi
 last_modified                         datetime                   Last modification of this object
 ===================================== ========================== =======================================================
 
-.. versionchanged:: 3.3
+.. versionchanged:: 4.15
 
-   The attributes ``geo_lat`` and ``geo_lon`` have been added.
-
-.. versionchanged:: 3.10
-
-   The ``disabled`` attribute has been added to ``item_price_overrides`` and ``variation_price_overrides``.
-
-.. versionchanged:: 3.12
-
-   The ``last_modified`` attribute has been added.
-
-.. versionchanged:: 3.18
-
-   The ``available_from``/``available_until`` attributes have been added to ``item_price_overrides`` and ``variation_price_overrides``.
+    The ``search`` query parameter has been added to filter sub-events by their name or location in any language.
 
 Endpoints
 ---------
 
-.. versionchanged:: 3.3
-
-    The sub-events resource can now be filtered by meta data attributes.
-
 .. versionchanged:: 4.1
 
     The ``with_availability_for`` parameter has been added.
@@ -147,6 +131,7 @@ Endpoints
    :query is_future: If set to ``true`` (``false``), only events that happen currently or in the future are (not) returned.
    :query is_past: If set to ``true`` (``false``), only events that are over are (not) returned.
    :query ends_after: If set to a date and time, only events that happen during of after the given time are returned.
+   :query search: Only return events matching a given search query.
    :param organizer: The ``slug`` field of a valid organizer
    :param event: The ``slug`` field of the main event
    :query datetime modified_since: Only return objects that have changed since the given date. Be careful: This does not

doc/api/resources/vouchers.rst

@@ -19,6 +19,8 @@ max_usages                            integer                    The maximum num
                                                                  redeemed (default: 1).
 redeemed                              integer                    The number of times this voucher already has been
                                                                  redeemed.
+min_usages                            integer                    The minimum number of times this voucher must be
+                                                                 redeemed on first usage (default: 1).
 valid_until                           datetime                   The voucher expiration date (or ``null``).
 block_quota                           boolean                    If ``true``, quota is blocked for this voucher.
 allow_ignore_quota                    boolean                    If ``true``, this voucher can be redeemed even if a
@@ -48,10 +50,6 @@ show_hidden_items                     boolean                    Only if set to
 ===================================== ========================== =======================================================
 
 
-.. versionchanged:: 3.4
-
-   The attribute ``seat`` has been added.
-
 Endpoints
 ---------
 

doc/api/resources/waitinglist.rst

@@ -30,12 +30,6 @@ subevent                              integer                    ID of the date
 ===================================== ========================== =======================================================
 
 
-.. versionchanged:: 1.15
-
-   The write operations ``POST``, ``PATCH``, ``PUT``, and ``DELETE`` have been added as well as a method to send out
-   vouchers.
-
-
 Endpoints
 ---------
 

doc/development/api/exporter.rst

@@ -76,6 +76,10 @@ The exporter class
 
       This is an abstract attribute, you **must** override this!
 
+   .. autoattribute:: description
+
+   .. autoattribute:: category
+
    .. autoattribute:: export_form_fields
 
    .. automethod:: render

doc/development/api/general.rst

@@ -38,7 +38,7 @@ Frontend
 
 
 .. automodule:: pretix.presale.signals
-   :members: order_info, order_info_top, order_meta_from_request, order_source_from_request
+   :members: order_info, order_info_top, order_meta_from_request
 
 Request flow
 """"""""""""

doc/development/api/payment.rst

@@ -126,6 +126,8 @@ The provider class
 
    .. automethod:: api_payment_details
 
+   .. automethod:: api_refund_details
+
    .. automethod:: matching_id
 
    .. automethod:: shred_payment_info
@@ -136,6 +138,10 @@ The provider class
 
    .. autoattribute:: is_meta
 
+   .. autoattribute:: execute_payment_needs_user
+
+   .. autoattribute:: multi_use_supported
+
    .. autoattribute:: test_mode_message
 
    .. autoattribute:: requires_invoice_immediately

doc/development/implementation/permissions.rst

@@ -184,11 +184,6 @@ Most of these methods work identically on :class:`pretix.base.models.TeamAPIToke
 Staff sessions
 --------------
 
-.. versionchanged:: 1.14
-
-   In 1.14, the ``User.is_superuser`` attribute has been deprecated and statically set to return ``False``. Staff
-   sessions have been newly introduced.
-
 System administrators of a pretix instance are identified by the ``is_staff`` attribute on the user model. By default,
 the regular permission rules apply for users with ``is_staff = True``. The only difference is that such users can
 temporarily turn on "staff mode" via a button in the user interface that grants them **all permissions** as long as

doc/plugins/digital.rst

@@ -91,6 +91,7 @@ Field                                 Type                       Description
 ===================================== ========================== =======================================================
 id                                    integer                    Internal content ID
 title                                 multi-lingual string       The content title (required)
+internal_name                         string                     An optional name that is only used in the backend
 content_type                          string                     The type of content, valid values are ``webinar``, ``video``, ``livestream``, ``link``, ``file``
 url                                   string                     The location of the digital content
 file                                  file                       A downloadable file. Either ``url`` or ``file`` must be ``null``.

doc/plugins/ticketoutputpdf.rst

@@ -17,9 +17,13 @@ Field                                 Type                       Description
 id                                    integer                    Internal layout ID
 name                                  string                     Internal layout description
 default                               boolean                    ``true`` if this is the default layout
-layout                                object                     Layout specification for libpretixprint
+layout                                list                       Dynamic layout specification. Each list element
+                                                                 corresponds to one dynamic element of the layout.
+                                                                 The current version of the schema in use can be found
+                                                                 `here`_.
+                                                                 Submitting invalid content can lead to application errors.
 background                            URL                        Background PDF file
-item_assignments                      list of objects            Products this layout is assigned to
+item_assignments                      list of objects            Products this layout is assigned to (currently read-only)
 ├ sales_channel                       string                     Sales channel (defaults to ``web``).
 └ item                                integer                    Item ID
 ===================================== ========================== =======================================================
@@ -58,7 +62,7 @@ Endpoints
             "name": "Default layout",
             "default": true,
             "layout": {…},
-            "background": {},
+            "background": null,
             "item_assignments": []
           }
         ]
@@ -96,7 +100,7 @@ Endpoints
         "name": "Default layout",
         "default": true,
         "layout": {…},
-        "background": {},
+        "background": null,
         "item_assignments": []
       }
 
@@ -147,3 +151,122 @@ Endpoints
    :statuscode 200: no error
    :statuscode 401: Authentication failure
    :statuscode 403: The requested organizer does not exist **or** you have no permission to view it.
+
+.. http:post:: /api/v1/organizers/(organizer)/events/(event)/ticketlayouts/
+
+   Creates a new ticket layout
+
+   **Example request**:
+
+   .. sourcecode:: http
+
+      POST /api/v1/organizers/bigevents/events/sampleconf/ticketlayouts/ HTTP/1.1
+      Host: pretix.eu
+      Accept: application/json, text/javascript
+      Content-Type: application/json
+
+      {
+        "name": "Default layout",
+        "default": true,
+        "layout": […],
+        "background": null,
+        "item_assignments": []
+      }
+
+   **Example response**:
+
+   .. sourcecode:: http
+
+      HTTP/1.1 201 Created
+      Vary: Accept
+      Content-Type: application/json
+
+      {
+        "id": 1,
+        "name": "Default layout",
+        "default": true,
+        "layout": […],
+        "background": null,
+        "item_assignments": []
+      }
+
+   :param organizer: The ``slug`` field of the organizer of the event to create a layout for
+   :param event: The ``slug`` field of the event to create a layout for
+   :statuscode 201: no error
+   :statuscode 400: The layout could not be created due to invalid submitted data.
+   :statuscode 401: Authentication failure
+   :statuscode 403: The requested organizer/event does not exist **or** you have no permission to create this resource.
+
+.. http:patch:: /api/v1/organizers/(organizer)/events/(event)/ticketlayouts/(id)/
+
+   Update a layout. You can also use ``PUT`` instead of ``PATCH``. With ``PUT``, you have to provide all fields of
+   the resource, other fields will be reset to default. With ``PATCH``, you only need to provide the fields that you
+   want to change.
+
+   **Example request**:
+
+   .. sourcecode:: http
+
+      PATCH /api/v1/organizers/bigevents/events/sampleconf/ticketlayouts/1/ HTTP/1.1
+      Host: pretix.eu
+      Accept: application/json, text/javascript
+      Content-Type: application/json
+      Content-Length: 94
+
+      {
+        "name": "Default layout"
+      }
+
+   **Example response**:
+
+   .. sourcecode:: http
+
+      HTTP/1.1 200 OK
+      Vary: Accept
+      Content-Type: application/json
+
+      {
+        "id": 1,
+        "name": "Default layout",
+        "default": true,
+        "layout": […],
+        "background": null,
+        "item_assignments": []
+      }
+
+   :param organizer: The ``slug`` field of the organizer to modify
+   :param event: The ``slug`` field of the event to modify
+   :param id: The ``id`` field of the layout to modify
+   :statuscode 200: no error
+   :statuscode 400: The layout could not be modified due to invalid submitted data
+   :statuscode 401: Authentication failure
+   :statuscode 403: The requested organizer/event does not exist **or** you have no permission to change this resource.
+
+.. http:delete:: /api/v1/organizers/(organizer)/events/(event)/ticketlayouts/(id)/
+
+   Delete a layout.
+
+   **Example request**:
+
+   .. sourcecode:: http
+
+      DELETE /api/v1/organizers/bigevents/events/sampleconf/ticketlayouts/1/ HTTP/1.1
+      Host: pretix.eu
+      Accept: application/json, text/javascript
+
+   **Example response**:
+
+   .. sourcecode:: http
+
+      HTTP/1.1 204 No Content
+      Vary: Accept
+
+   :param organizer: The ``slug`` field of the organizer to modify
+   :param event: The ``slug`` field of the event to modify
+   :param id: The ``id`` field of the layout to delete
+   :statuscode 204: no error
+   :statuscode 401: Authentication failure
+   :statuscode 403: The requested organizer/event does not exist **or** you have no permission to delete this resource.
+
+
+.. _here: https://github.com/pretix/pretix/blob/master/src/pretix/static/schema/pdf-layout.schema.json

doc/spelling_wordlist.txt

@@ -97,6 +97,7 @@ overpayment
 param
 passphrase
 percental
+personalization
 pluggable
 positionid
 pre

doc/user/events/widget.rst

@@ -447,8 +447,4 @@ Hosted or pretix Enterprise are active, you can pass the following fields:
     </script>
 
 
-.. versionchanged:: 3.6
-
-   Dynamically opening the widget has been added in pretix 3.6.
-
 .. _Let's Encrypt: https://letsencrypt.org/

src/pretix/__init__.py

@@ -19,4 +19,4 @@
 # You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
 # <https://www.gnu.org/licenses/>.
 #
-__version__ = "4.14.1.dev0"
+__version__ = "4.17.0.dev0"

src/pretix/api/auth/devicesecurity.py

@@ -196,6 +196,7 @@ class PretixPosSecurityProfile(AllowListSecurityProfile):
         ('POST', 'plugins:pretix_posbackend:posdebuglogentry-bulk-create'),
         ('GET', 'plugins:pretix_posbackend:poscashier-list'),
         ('POST', 'plugins:pretix_posbackend:stripeterminal.token'),
+        ('POST', 'plugins:pretix_posbackend:stripeterminal.paymentintent'),
         ('PUT', 'plugins:pretix_posbackend:file.upload'),
         ('GET', 'api-v1:revokedsecrets-list'),
         ('GET', 'api-v1:event.settings'),

src/pretix/api/exception.py

@@ -19,11 +19,17 @@
 # You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
 # <https://www.gnu.org/licenses/>.
 #
+import logging
+
+import ujson
+from rest_framework import exceptions
 from rest_framework.response import Response
 from rest_framework.views import exception_handler, status
 
 from pretix.base.services.locking import LockTimeoutException
 
+logger = logging.getLogger(__name__)
+
 
 def custom_exception_handler(exc, context):
     response = exception_handler(exc, context)
@@ -37,4 +43,7 @@ def custom_exception_handler(exc, context):
             }
         )
 
+    if isinstance(exc, exceptions.APIException):
+        logger.info(f'API Exception [{exc.status_code}]: {ujson.dumps(exc.detail)}')
+
     return response

src/pretix/api/middleware.py

@@ -32,6 +32,7 @@ from rest_framework import status
 
 from pretix.api.models import ApiCall
 from pretix.base.models import Organizer
+from pretix.helpers import OF_SELF
 
 
 class IdempotencyMiddleware:
@@ -56,7 +57,7 @@ class IdempotencyMiddleware:
         idempotency_key = request.headers.get('X-Idempotency-Key', '')
 
         with transaction.atomic():
-            call, created = ApiCall.objects.select_for_update().get_or_create(
+            call, created = ApiCall.objects.select_for_update(of=OF_SELF).get_or_create(
                 auth_hash=auth_hash,
                 idempotency_key=idempotency_key,
                 defaults={

src/pretix/api/migrations/0009_auto_20221217_1847.py

@@ -0,0 +1,77 @@
+# Generated by Django 3.2.16 on 2022-12-17 18:47
+
+import uuid
+
+import django.db.models.deletion
+import oauth2_provider.generators
+import oauth2_provider.models
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pretixbase', '0226_itemvariationmetavalue'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('pretixapi', '0008_webhookcallretry'),
+    ]
+    run_before = [
+        ('oauth2_provider', '0002_auto_20190406_1805'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='oauthapplication',
+            name='algorithm',
+            field=models.CharField(default='', max_length=5),
+        ),
+        migrations.AddField(
+            model_name='oauthgrant',
+            name='claims',
+            field=models.TextField(default=''),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='oauthgrant',
+            name='code_challenge',
+            field=models.CharField(default='', max_length=128),
+        ),
+        migrations.AddField(
+            model_name='oauthgrant',
+            name='code_challenge_method',
+            field=models.CharField(default='', max_length=10),
+        ),
+        migrations.AddField(
+            model_name='oauthgrant',
+            name='nonce',
+            field=models.CharField(default='', max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='oauthapplication',
+            name='client_secret',
+            field=oauth2_provider.models.ClientSecretField(db_index=True, default=oauth2_provider.generators.generate_client_secret, max_length=255),
+        ),
+        migrations.CreateModel(
+            name='OAuthIDToken',
+            fields=[
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
+                ('jti', models.UUIDField(default=uuid.uuid4, unique=True)),
+                ('expires', models.DateTimeField()),
+                ('scope', models.TextField()),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('application', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.OAUTH2_PROVIDER_APPLICATION_MODEL)),
+                ('organizers', models.ManyToManyField(to='pretixbase.Organizer')),
+                ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='pretixapi_oauthidtoken', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.AddField(
+            model_name='oauthaccesstoken',
+            name='id_token',
+            field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='access_token', to='pretixapi.oauthidtoken'),
+        ),
+    ]

src/pretix/api/models.py

@@ -29,8 +29,8 @@ from oauth2_provider.generators import (
     generate_client_id, generate_client_secret,
 )
 from oauth2_provider.models import (
-    AbstractAccessToken, AbstractApplication, AbstractGrant,
-    AbstractRefreshToken,
+    AbstractAccessToken, AbstractApplication, AbstractGrant, AbstractIDToken,
+    AbstractRefreshToken, ClientSecretField,
 )
 from oauth2_provider.validators import URIValidator
 
@@ -46,7 +46,7 @@ class OAuthApplication(AbstractApplication):
         verbose_name=_("Client ID"),
         max_length=100, unique=True, default=generate_client_id, db_index=True
     )
-    client_secret = models.CharField(
+    client_secret = ClientSecretField(
         verbose_name=_("Client secret"),
         max_length=255, blank=False, default=generate_client_secret, db_index=True
     )
@@ -67,12 +67,26 @@ class OAuthGrant(AbstractGrant):
     redirect_uri = models.CharField(max_length=2500)  # Only 255 in AbstractGrant, which caused problems
 
 
+class OAuthIDToken(AbstractIDToken):
+    application = models.ForeignKey(
+        OAuthApplication, on_delete=models.CASCADE,
+    )
+    organizers = models.ManyToManyField('pretixbase.Organizer')
+
+
 class OAuthAccessToken(AbstractAccessToken):
     source_refresh_token = models.OneToOneField(
         # unique=True implied by the OneToOneField
         'OAuthRefreshToken', on_delete=models.SET_NULL, blank=True, null=True,
         related_name="refreshed_access_token"
     )
+    id_token = models.OneToOneField(
+        OAuthIDToken,
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+        related_name="access_token",
+    )
     application = models.ForeignKey(
         OAuthApplication, on_delete=models.CASCADE, blank=True, null=True,
     )

src/pretix/api/serializers/cart.py

@@ -237,12 +237,14 @@ class CartPositionCreateSerializer(BaseCartPositionCreateSerializer):
             for addon_data in addons_data:
                 addon_data['addon_to'] = cp
                 addon_data['is_bundled'] = False
+                addon_data['cart_id'] = cp.cart_id
                 super().create(addon_data)
 
         if bundled_data:
             for bundle_data in bundled_data:
                 bundle_data['addon_to'] = cp
                 bundle_data['is_bundled'] = True
+                bundle_data['cart_id'] = cp.cart_id
                 super().create(bundle_data)
 
         return cp

src/pretix/api/serializers/event.py

@@ -411,7 +411,8 @@ class CloneEventSerializer(EventSerializer):
         has_subevents = validated_data.pop('has_subevents', None)
         tz = validated_data.pop('timezone', None)
         sales_channels = validated_data.pop('sales_channels', None)
-        new_event = super().create(validated_data)
+        date_admission = validated_data.pop('date_admission', None)
+        new_event = super().create({**validated_data, 'plugins': None})
 
         event = Event.objects.filter(slug=self.context['event'], organizer=self.context['organizer'].pk).first()
         new_event.copy_data_from(event)
@@ -426,6 +427,10 @@ class CloneEventSerializer(EventSerializer):
             new_event.sales_channels = sales_channels
         if has_subevents is not None:
             new_event.has_subevents = has_subevents
+        if has_subevents is not None:
+            new_event.has_subevents = has_subevents
+        if date_admission is not None:
+            new_event.date_admission = date_admission
         new_event.save()
         if tz:
             new_event.settings.timezone = tz
@@ -755,6 +760,9 @@ class EventSettingsSerializer(SettingsSerializer):
         'invoice_logo_image',
         'cancel_allow_user',
         'cancel_allow_user_until',
+        'cancel_allow_user_unpaid_keep',
+        'cancel_allow_user_unpaid_keep_fees',
+        'cancel_allow_user_unpaid_keep_percentage',
         'cancel_allow_user_paid',
         'cancel_allow_user_paid_until',
         'cancel_allow_user_paid_keep',

src/pretix/api/serializers/exporters.py

@@ -22,8 +22,10 @@
 from django import forms
 from django.http import QueryDict
 from rest_framework import serializers
+from rest_framework.exceptions import ValidationError
 
 from pretix.base.exporter import OrganizerLevelExportMixin
+from pretix.base.timeframes import DateFrameField, SerializerDateFrameField
 
 
 class FormFieldWrapperField(serializers.Field):
@@ -142,6 +144,12 @@ class JobRunSerializer(serializers.Serializer):
                     allow_null=not v.required,
                     validators=v.validators,
                 )
+            elif isinstance(v, DateFrameField):
+                self.fields[k] = SerializerDateFrameField(
+                    required=v.required,
+                    allow_null=not v.required,
+                    validators=v.validators,
+                )
             else:
                 self.fields[k] = FormFieldWrapperField(form_field=v, required=v.required, allow_null=not v.required)
 
@@ -151,5 +159,40 @@ class JobRunSerializer(serializers.Serializer):
         for k, v in self.fields.items():
             if isinstance(v, serializers.ManyRelatedField) and k not in data:
                 data[k] = []
+
+        for fk in self.fields.keys():
+            # Backwards compatibility for exports that used to take e.g. (date_from, date_to) or (event_date_from, event_date_to)
+            # and now only take date_range.
+            if fk.endswith("_range") and isinstance(self.fields[fk], SerializerDateFrameField) and fk not in data:
+                if fk.replace("_range", "_from") in data:
+                    d_from = data.pop(fk.replace("_range", "_from"))
+                    if d_from:
+                        d_from = serializers.DateField().to_internal_value(d_from)
+                else:
+                    d_from = None
+                if fk.replace("_range", "_to") in data:
+                    d_to = data.pop(fk.replace("_range", "_to"))
+                    if d_to:
+                        d_to = serializers.DateField().to_internal_value(d_to)
+                else:
+                    d_to = None
+                data[fk] = f'{d_from.isoformat() if d_from else ""}/{d_to.isoformat() if d_to else ""}'
+
         data = super().to_internal_value(data)
         return data
+
+    def is_valid(self, raise_exception=False):
+        super().is_valid(raise_exception=raise_exception)
+
+        fields_keys = set(self.fields.keys())
+        input_keys = set(self.initial_data.keys())
+
+        additional_fields = input_keys - fields_keys
+
+        if bool(additional_fields):
+            self._errors['fields'] = ['Additional fields not allowed: {}.'.format(list(additional_fields))]
+
+        if self._errors and raise_exception:
+            raise ValidationError(self.errors)
+
+        return not bool(self._errors)

src/pretix/api/serializers/item.py

@@ -47,13 +47,14 @@ from pretix.api.serializers.fields import UploadedFileField
 from pretix.api.serializers.i18n import I18nAwareModelSerializer
 from pretix.base.models import (
     Item, ItemAddOn, ItemBundle, ItemCategory, ItemMetaValue, ItemVariation,
-    Question, QuestionOption, Quota,
+    ItemVariationMetaValue, Question, QuestionOption, Quota,
 )
 
 
 class InlineItemVariationSerializer(I18nAwareModelSerializer):
     price = serializers.DecimalField(read_only=True, decimal_places=2, max_digits=10,
                                      coerce_to_string=True)
+    meta_data = MetaDataField(required=False, source='*')
 
     class Meta:
         model = ItemVariation
@@ -61,16 +62,23 @@ class InlineItemVariationSerializer(I18nAwareModelSerializer):
                   'position', 'default_price', 'price', 'original_price', 'require_approval',
                   'require_membership', 'require_membership_types',
                   'require_membership_hidden', 'available_from', 'available_until',
-                  'sales_channels', 'hide_without_voucher',)
+                  'sales_channels', 'hide_without_voucher', 'meta_data')
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.fields['require_membership_types'].queryset = lazy(lambda: self.context['event'].organizer.membership_types.all(), QuerySet)
 
+    def validate_meta_data(self, value):
+        for key in value['meta_data'].keys():
+            if key not in self.parent.parent.item_meta_properties:
+                raise ValidationError(_('Item meta data property \'{name}\' does not exist.').format(name=key))
+        return value
+
 
 class ItemVariationSerializer(I18nAwareModelSerializer):
     price = serializers.DecimalField(read_only=True, decimal_places=2, max_digits=10,
                                      coerce_to_string=True)
+    meta_data = MetaDataField(required=False, source='*')
 
     class Meta:
         model = ItemVariation
@@ -78,12 +86,67 @@ class ItemVariationSerializer(I18nAwareModelSerializer):
                   'position', 'default_price', 'price', 'original_price', 'require_approval',
                   'require_membership', 'require_membership_types',
                   'require_membership_hidden', 'available_from', 'available_until',
-                  'sales_channels', 'hide_without_voucher',)
+                  'sales_channels', 'hide_without_voucher', 'meta_data')
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.fields['require_membership_types'].queryset = self.context['event'].organizer.membership_types.all()
 
+    @transaction.atomic
+    def create(self, validated_data):
+        meta_data = validated_data.pop('meta_data', None)
+        require_membership_types = validated_data.pop('require_membership_types', [])
+        variation = ItemVariation.objects.create(**validated_data)
+
+        if require_membership_types:
+            variation.require_membership_types.add(*require_membership_types)
+
+        # Meta data
+        if meta_data is not None:
+            for key, value in meta_data.items():
+                ItemVariationMetaValue.objects.create(
+                    property=self.item_meta_properties.get(key),
+                    value=value,
+                    variation=variation
+                )
+        return variation
+
+    @cached_property
+    def item_meta_properties(self):
+        return {
+            p.name: p for p in self.context['request'].event.item_meta_properties.all()
+        }
+
+    def validate_meta_data(self, value):
+        for key in value['meta_data'].keys():
+            if key not in self.item_meta_properties:
+                raise ValidationError(_('Item meta data property \'{name}\' does not exist.').format(name=key))
+        return value
+
+    def update(self, instance, validated_data):
+        meta_data = validated_data.pop('meta_data', None)
+        variation = super().update(instance, validated_data)
+
+        # Meta data
+        if meta_data is not None:
+            current = {mv.property: mv for mv in variation.meta_values.select_related('property')}
+            for key, value in meta_data.items():
+                prop = self.item_meta_properties.get(key)
+                if prop in current:
+                    current[prop].value = value
+                    current[prop].save()
+                else:
+                    variation.meta_values.create(
+                        property=self.item_meta_properties.get(key),
+                        value=value
+                    )
+
+            for prop, current_object in current.items():
+                if prop.name not in meta_data:
+                    current_object.delete()
+
+        return variation
+
 
 class InlineItemBundleSerializer(serializers.ModelSerializer):
     class Meta:
@@ -171,7 +234,7 @@ class ItemSerializer(I18nAwareModelSerializer):
     class Meta:
         model = Item
         fields = ('id', 'category', 'name', 'internal_name', 'active', 'sales_channels', 'description',
-                  'default_price', 'free_price', 'tax_rate', 'tax_rule', 'admission',
+                  'default_price', 'free_price', 'tax_rate', 'tax_rule', 'admission', 'personalized',
                   'position', 'picture', 'available_from', 'available_until',
                   'require_voucher', 'hide_without_voucher', 'allow_cancel', 'require_bundling',
                   'min_per_order', 'max_per_order', 'checkin_attention', 'has_variations', 'variations',
@@ -184,6 +247,8 @@ class ItemSerializer(I18nAwareModelSerializer):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.fields['default_price'].allow_null = False
+        self.fields['default_price'].required = True
         if not self.read_only:
             self.fields['require_membership_types'].queryset = self.context['event'].organizer.membership_types.all()
             self.fields['grant_membership_type'].queryset = self.context['event'].organizer.membership_types.all()
@@ -197,6 +262,15 @@ class ItemSerializer(I18nAwareModelSerializer):
         Item.clean_per_order(data.get('min_per_order'), data.get('max_per_order'))
         Item.clean_available(data.get('available_from'), data.get('available_until'))
 
+        if data.get('personalized') and not data.get('admission'):
+            raise ValidationError(_('Only admission products can currently be personalized.'))
+
+        if data.get('admission') and 'personalized' not in data and not self.instance:
+            # Backwards compatibility
+            data['personalized'] = True
+        elif 'admission' in data and not data['admission']:
+            data['personalized'] = False
+
         if data.get('issue_giftcard'):
             if data.get('tax_rule') and data.get('tax_rule').rate > 0:
                 raise ValidationError(
@@ -261,9 +335,19 @@ class ItemSerializer(I18nAwareModelSerializer):
 
         for variation_data in variations_data:
             require_membership_types = variation_data.pop('require_membership_types', [])
+            var_meta_data = variation_data.pop('meta_data', {})
             v = ItemVariation.objects.create(item=item, **variation_data)
             if require_membership_types:
                 v.require_membership_types.add(*require_membership_types)
+
+            if var_meta_data is not None:
+                for key, value in var_meta_data.items():
+                    ItemVariationMetaValue.objects.create(
+                        property=self.item_meta_properties.get(key),
+                        value=value,
+                        variation=v
+                    )
+
         for addon_data in addons_data:
             ItemAddOn.objects.create(base_item=item, **addon_data)
         for bundle_data in bundles_data:

src/pretix/api/serializers/order.py

@@ -29,6 +29,7 @@ import pycountry
 from django.conf import settings
 from django.core.files import File
 from django.db.models import F, Q
+from django.utils.encoding import force_str
 from django.utils.timezone import now
 from django.utils.translation import gettext_lazy
 from django_countries.fields import Country
@@ -61,14 +62,25 @@ from pretix.base.services.pricing import (
 )
 from pretix.base.settings import COUNTRIES_WITH_STATE_IN_ADDRESS
 from pretix.base.signals import register_ticket_outputs
+from pretix.helpers.countries import CachedCountries
 from pretix.multidomain.urlreverse import build_absolute_uri
 
 logger = logging.getLogger(__name__)
 
 
 class CompatibleCountryField(serializers.Field):
+    countries = CachedCountries()
+    default_error_messages = {
+        'invalid_choice': gettext_lazy('"{input}" is not a valid choice.')
+    }
+
     def to_internal_value(self, data):
-        return {self.field_name: Country(data)}
+        country = self.countries.alpha2(data)
+        if data and not country:
+            country = self.countries.by_name(force_str(data))
+            if not country:
+                self.fail("invalid_choice", input=data)
+        return {self.field_name: Country(country)}
 
     def to_representation(self, instance: InvoiceAddress):
         if instance.country:
@@ -106,6 +118,10 @@ class InvoiceAddressSerializer(I18nAwareModelSerializer):
             raise ValidationError(
                 {'name': ['Do not specify name if you specified name_parts.']}
             )
+
+        if data.get('name_parts') and not isinstance(data.get('name_parts'), dict):
+            raise ValidationError({'name_parts': ['Invalid data type']})
+
         if data.get('name_parts') and '_scheme' not in data.get('name_parts'):
             data['name_parts']['_scheme'] = self.context['request'].event.settings.name_scheme
 
@@ -359,10 +375,19 @@ class PdfDataSerializer(serializers.Field):
             for k, v in ev._cached_meta_data.items():
                 res['meta:' + k] = v
 
-            if not hasattr(instance.item, '_cached_meta_data'):
-                instance.item._cached_meta_data = instance.item.meta_data
-            for k, v in instance.item._cached_meta_data.items():
-                res['itemmeta:' + k] = v
+            if instance.variation_id:
+                print(instance, instance.variation, instance.variation_id, instance.item)
+                if not hasattr(instance.variation, '_cached_meta_data'):
+                    instance.variation.item = instance.item  # saves some database lookups
+                    instance.variation._cached_meta_data = instance.variation.meta_data
+                print(instance.variation._cached_meta_data.items())
+                for k, v in instance.variation._cached_meta_data.items():
+                    res['itemmeta:' + k] = v
+            else:
+                if not hasattr(instance.item, '_cached_meta_data'):
+                    instance.item._cached_meta_data = instance.item.meta_data
+                for k, v in instance.item._cached_meta_data.items():
+                    res['itemmeta:' + k] = v
 
             res['images'] = {}
 
@@ -410,13 +435,13 @@ class OrderPositionSerializer(I18nAwareModelSerializer):
     class Meta:
         model = OrderPosition
         fields = ('id', 'order', 'positionid', 'item', 'variation', 'price', 'attendee_name', 'attendee_name_parts',
-                  'company', 'street', 'zipcode', 'city', 'country', 'state',
+                  'company', 'street', 'zipcode', 'city', 'country', 'state', 'discount',
                   'attendee_email', 'voucher', 'tax_rate', 'tax_value', 'secret', 'addon_to', 'subevent', 'checkins',
                   'downloads', 'answers', 'tax_rule', 'pseudonymization_id', 'pdf_data', 'seat', 'canceled')
         read_only_fields = (
             'id', 'order', 'positionid', 'item', 'variation', 'price', 'voucher', 'tax_rate', 'tax_value', 'secret',
             'addon_to', 'subevent', 'checkins', 'downloads', 'answers', 'tax_rule', 'pseudonymization_id', 'pdf_data',
-            'seat', 'canceled'
+            'seat', 'canceled', 'discount',
         )
 
     def __init__(self, *args, **kwargs):
@@ -553,12 +578,22 @@ class OrderPaymentSerializer(I18nAwareModelSerializer):
                   'details')
 
 
+class RefundDetailsField(serializers.Field):
+    def to_representation(self, value: OrderRefund):
+        pp = value.payment_provider
+        if not pp:
+            return {}
+        return pp.api_refund_details(value)
+
+
 class OrderRefundSerializer(I18nAwareModelSerializer):
     payment = SlugRelatedField(slug_field='local_id', read_only=True)
+    details = RefundDetailsField(source='*', allow_null=True, read_only=True)
 
     class Meta:
         model = OrderRefund
-        fields = ('local_id', 'state', 'source', 'amount', 'payment', 'created', 'execution_date', 'comment', 'provider')
+        fields = ('local_id', 'state', 'source', 'amount', 'payment', 'created', 'execution_date', 'comment', 'provider',
+                  'details')
 
 
 class OrderURLField(serializers.URLField):
@@ -600,6 +635,32 @@ class OrderSerializer(I18nAwareModelSerializer):
         if not self.context['pdf_data']:
             self.fields['positions'].child.fields.pop('pdf_data', None)
 
+        includes = set(self.context['include'])
+        if includes:
+            for fname, field in list(self.fields.items()):
+                if fname in includes:
+                    continue
+                elif hasattr(field, 'child'):  # Nested list serializers
+                    found_any = False
+                    for childfname, childfield in list(field.child.fields.items()):
+                        if f'{fname}.{childfname}' not in includes:
+                            field.child.fields.pop(childfname)
+                        else:
+                            found_any = True
+                    if not found_any:
+                        self.fields.pop(fname)
+                elif isinstance(field, serializers.Serializer):  # Nested serializers
+                    found_any = False
+                    for childfname, childfield in list(field.fields.items()):
+                        if f'{fname}.{childfname}' not in includes:
+                            field.fields.pop(childfname)
+                        else:
+                            found_any = True
+                    if not found_any:
+                        self.fields.pop(fname)
+                else:
+                    self.fields.pop(fname)
+
         for exclude_field in self.context['exclude']:
             p = exclude_field.split('.')
             if p[0] in self.fields:
@@ -721,7 +782,7 @@ class OrderPositionCreateSerializer(I18nAwareModelSerializer):
     class Meta:
         model = OrderPosition
         fields = ('positionid', 'item', 'variation', 'price', 'attendee_name', 'attendee_name_parts', 'attendee_email',
-                  'company', 'street', 'zipcode', 'city', 'country', 'state',
+                  'company', 'street', 'zipcode', 'city', 'country', 'state', 'is_bundled',
                   'secret', 'addon_to', 'subevent', 'answers', 'seat', 'voucher')
 
     def __init__(self, *args, **kwargs):
@@ -784,6 +845,10 @@ class OrderPositionCreateSerializer(I18nAwareModelSerializer):
             raise ValidationError(
                 {'attendee_name': ['Do not specify attendee_name if you specified attendee_name_parts.']}
             )
+
+        if data.get('attendee_name_parts') and not isinstance(data.get('attendee_name_parts'), dict):
+            raise ValidationError({'attendee_name_parts': ['Invalid data type']})
+
         if data.get('attendee_name_parts') and '_scheme' not in data.get('attendee_name_parts'):
             data['attendee_name_parts']['_scheme'] = self.context['request'].event.settings.name_scheme
 
@@ -1378,7 +1443,7 @@ class OrderCreateSerializer(I18nAwareModelSerializer):
                 state=OrderPayment.PAYMENT_STATE_CREATED
             )
 
-        order.create_transactions(is_new=True, fees=fees, positions=pos_map.values(), source=self.context['source'])
+        order.create_transactions(is_new=True, fees=fees, positions=pos_map.values())
         return order
 
 

src/pretix/api/serializers/orderchange.py

@@ -158,12 +158,14 @@ class OrderPositionInfoPatchSerializer(serializers.ModelSerializer):
                 a.question_id: a for a in instance.answers.all()
             }
             for answ_data in answers_data:
+                if not answ_data.get('answer'):
+                    continue
                 options = answ_data.pop('options', [])
                 if answ_data['question'].pk in qs_seen:
                     raise ValidationError(f'Question {answ_data["question"]} was sent twice.')
                 if answ_data['question'].pk in answercache:
                     a = answercache[answ_data['question'].pk]
-                    if isinstance(answ_data['answer'], File):
+                    if isinstance(answ_data.get('answer'), File):
                         a.file.save(answ_data['answer'].name, answ_data['answer'], save=False)
                         a.answer = 'file://' + a.file.name
                     elif a.answer.startswith('file://') and answ_data['answer'] == "file:keep":
@@ -173,7 +175,7 @@ class OrderPositionInfoPatchSerializer(serializers.ModelSerializer):
                             setattr(a, attr, value)
                     a.save()
                 else:
-                    if isinstance(answ_data['answer'], File):
+                    if isinstance(answ_data.get('answer'), File):
                         an = answ_data.pop('answer')
                         a = instance.answers.create(**answ_data, answer='')
                         a.file.save(os.path.basename(an.name), an, save=False)

src/pretix/api/serializers/organizer.py

@@ -79,6 +79,13 @@ class CustomerSerializer(I18nAwareModelSerializer):
             validated_data['external_identifier'] = instance.external_identifier
         return super().update(instance, validated_data)
 
+    def validate(self, data):
+        if data.get('name_parts') and not isinstance(data.get('name_parts'), dict):
+            raise ValidationError({'name_parts': ['Invalid data type']})
+        if data.get('name_parts') and '_scheme' not in data.get('name_parts'):
+            data['name_parts']['_scheme'] = self.context['request'].organizer.settings.name_scheme
+        return data
+
 
 class CustomerCreateSerializer(CustomerSerializer):
     send_email = serializers.BooleanField(default=False, required=False, allow_null=True)

src/pretix/api/serializers/voucher.py

@@ -61,7 +61,7 @@ class VoucherSerializer(I18nAwareModelSerializer):
 
     class Meta:
         model = Voucher
-        fields = ('id', 'code', 'max_usages', 'redeemed', 'valid_until', 'block_quota',
+        fields = ('id', 'code', 'max_usages', 'redeemed', 'min_usages', 'valid_until', 'block_quota',
                   'allow_ignore_quota', 'price_mode', 'value', 'item', 'variation', 'quota',
                   'tag', 'comment', 'subevent', 'show_hidden_items', 'seat')
         read_only_fields = ('id', 'redeemed')

src/pretix/api/urls.py

@@ -35,7 +35,8 @@
 import importlib
 
 from django.apps import apps
-from django.conf.urls import include, re_path
+from django.conf.urls import re_path
+from django.urls import include
 from rest_framework import routers
 
 from pretix.api.views import cart

src/pretix/api/utils.py

@@ -1,14 +0,0 @@
-from pretix.api.models import OAuthAccessToken
-from pretix.base.models import Device, TeamAPIToken
-
-
-def get_api_source(request):
-    if isinstance(request.auth, Device):
-        return "pretix.api", f"device:{request.auth.pk}"
-    elif isinstance(request.auth, TeamAPIToken):
-        return "pretix.api", f"token:{request.auth.pk}"
-    elif isinstance(request.auth, OAuthAccessToken):
-        return "pretix.api", f"oauth.app:{request.auth.application.pk}"
-    elif request.user.is_authenticated:
-        return "pretix.api", f"user:{request.user.pk}"
-    return "pretix.api", None

src/pretix/api/views/cart.py

@@ -92,6 +92,11 @@ class CartPositionViewSet(CreateModelMixin, DestroyModelMixin, viewsets.ReadOnly
     def perform_create(self, serializer):
         raise NotImplementedError()
 
+    @transaction.atomic()
+    def perform_destroy(self, instance):
+        instance.addons.all().delete()
+        instance.delete()
+
     def _require_locking(self, quota_diff, voucher_use_diff, seat_diff):
         if voucher_use_diff or seat_diff:
             # If any vouchers or seats are used, we lock to make sure we don't redeem them to often

src/pretix/api/views/checkin.py

@@ -93,8 +93,10 @@ with scopes_disabled():
 class CheckinListViewSet(viewsets.ModelViewSet):
     serializer_class = CheckinListSerializer
     queryset = CheckinList.objects.none()
-    filter_backends = (DjangoFilterBackend,)
+    filter_backends = (DjangoFilterBackend, RichOrderingFilter)
     filterset_class = CheckinListFilter
+    ordering = ('subevent__date_from', 'name', 'id')
+    ordering_fields = ('subevent__date_from', 'id', 'name',)
 
     def _get_permission_name(self, request):
         if request.path.endswith('/failed_checkins/'):
@@ -682,7 +684,7 @@ class CheckinListPositionViewSet(viewsets.ReadOnlyModelViewSet):
     serializer_class = CheckinListOrderPositionSerializer
     queryset = OrderPosition.all.none()
     filter_backends = (ExtendedBackend, RichOrderingFilter)
-    ordering = (F('attendee_name_cached').asc(nulls_last=True), 'positionid')
+    ordering = (F('attendee_name_cached').asc(nulls_last=True), 'pk')
     ordering_fields = (
         'order__code', 'order__datetime', 'positionid', 'attendee_name',
         'last_checked_in', 'order__email',

src/pretix/api/views/event.py

@@ -33,6 +33,7 @@
 # License for the specific language governing permissions and limitations under the License.
 
 import django_filters
+from django.conf import settings
 from django.db import transaction
 from django.db.models import Prefetch, ProtectedError, Q
 from django.utils.timezone import now
@@ -241,13 +242,17 @@ class EventViewSet(viewsets.ModelViewSet):
             except Event.DoesNotExist:
                 raise ValidationError('Event to copy from was not found')
 
+        # Ensure that .installed() is only called when we NOT clone
+        plugins = serializer.validated_data.pop('plugins', None)
+        serializer.validated_data['plugins'] = None
+
         new_event = serializer.save(organizer=self.request.organizer)
 
         if copy_from:
             new_event.copy_data_from(copy_from)
 
-            if 'plugins' in serializer.validated_data:
-                new_event.set_active_plugins(serializer.validated_data['plugins'])
+            if plugins is not None:
+                new_event.set_active_plugins(plugins)
             if 'is_public' in serializer.validated_data:
                 new_event.is_public = serializer.validated_data['is_public']
             if 'testmode' in serializer.validated_data:
@@ -256,12 +261,17 @@ class EventViewSet(viewsets.ModelViewSet):
                 new_event.sales_channels = serializer.validated_data['sales_channels']
             if 'has_subevents' in serializer.validated_data:
                 new_event.has_subevents = serializer.validated_data['has_subevents']
+            if 'date_admission' in serializer.validated_data:
+                new_event.date_admission = serializer.validated_data['date_admission']
             new_event.save()
             if 'timezone' in serializer.validated_data:
                 new_event.settings.timezone = serializer.validated_data['timezone']
         else:
             serializer.instance.set_defaults()
 
+            new_event.set_active_plugins(plugins if plugins is not None else settings.PRETIX_PLUGINS_DEFAULT.split(','))
+            new_event.save(update_fields=['plugins'])
+
         serializer.instance.log_action(
             'pretix.event.added',
             user=self.request.user,
@@ -322,6 +332,7 @@ with scopes_disabled():
         ends_after = django_filters.rest_framework.IsoDateTimeFilter(method='ends_after_qs')
         modified_since = django_filters.IsoDateTimeFilter(field_name='last_modified', lookup_expr='gte')
         sales_channel = django_filters.rest_framework.CharFilter(method='sales_channel_qs')
+        search = django_filters.rest_framework.CharFilter(method='search_qs')
 
         class Meta:
             model = SubEvent
@@ -357,6 +368,12 @@ with scopes_disabled():
         def sales_channel_qs(self, queryset, name, value):
             return queryset.filter(event__sales_channels__contains=value)
 
+        def search_qs(self, queryset, name, value):
+            return queryset.filter(
+                Q(name__icontains=i18ncomp(value))
+                | Q(location__icontains=i18ncomp(value))
+            )
+
 
 class SubEventViewSet(ConditionalListView, viewsets.ModelViewSet):
     serializer_class = SubEventSerializer

src/pretix/api/views/item.py

@@ -84,7 +84,9 @@ class ItemViewSet(ConditionalListView, viewsets.ModelViewSet):
 
     def get_queryset(self):
         return self.request.event.items.select_related('tax_rule').prefetch_related(
-            'variations', 'addons', 'bundles', 'meta_values'
+            'variations', 'addons', 'bundles', 'meta_values', 'meta_values__property',
+            'variations__meta_values', 'variations__meta_values__property',
+            'require_membership_types', 'variations__require_membership_types',
         ).all()
 
     def perform_create(self, serializer):
@@ -147,7 +149,11 @@ class ItemVariationViewSet(viewsets.ModelViewSet):
         return get_object_or_404(Item, pk=self.kwargs['item'], event=self.request.event)
 
     def get_queryset(self):
-        return self.item.variations.all()
+        return self.item.variations.all().prefetch_related(
+            'meta_values',
+            'meta_values__property',
+            'require_membership_types'
+        )
 
     def get_serializer_context(self):
         ctx = super().get_serializer_context()

src/pretix/api/views/order.py

@@ -61,13 +61,14 @@ from pretix.api.serializers.orderchange import (
     OrderPositionCreateForExistingOrderSerializer,
     OrderPositionInfoPatchSerializer,
 )
-from pretix.api.utils import get_api_source
+from pretix.api.views import RichOrderingFilter
 from pretix.base.i18n import language
 from pretix.base.models import (
     CachedCombinedTicket, CachedTicket, Checkin, Device, EventMetaValue,
-    Invoice, InvoiceAddress, ItemMetaValue, Order, OrderFee, OrderPayment,
-    OrderPosition, OrderRefund, Quota, SubEvent, SubEventMetaValue, TaxRule,
-    TeamAPIToken, generate_secret,
+    Invoice, InvoiceAddress, ItemMetaValue, ItemVariation,
+    ItemVariationMetaValue, Order, OrderFee, OrderPayment, OrderPosition,
+    OrderRefund, Quota, SubEvent, SubEventMetaValue, TaxRule, TeamAPIToken,
+    generate_secret,
 )
 from pretix.base.models.orders import QuestionAnswer, RevokedTicketSecret
 from pretix.base.payment import PaymentException
@@ -191,7 +192,7 @@ class OrderViewSet(viewsets.ModelViewSet):
         ctx['event'] = self.request.event
         ctx['pdf_data'] = self.request.query_params.get('pdf_data', 'false') == 'true'
         ctx['exclude'] = self.request.query_params.getlist('exclude')
-        ctx['source'] = get_api_source(self.request)
+        ctx['include'] = self.request.query_params.getlist('include')
         return ctx
 
     def get_queryset(self):
@@ -232,7 +233,9 @@ class OrderViewSet(viewsets.ModelViewSet):
                     Prefetch('item', queryset=self.request.event.items.prefetch_related(
                         Prefetch('meta_values', ItemMetaValue.objects.select_related('property'), to_attr='meta_values_cached')
                     )),
-                    'variation',
+                    Prefetch('variation', queryset=ItemVariation.objects.prefetch_related(
+                        Prefetch('meta_values', ItemVariationMetaValue.objects.select_related('property'), to_attr='meta_values_cached')
+                    )),
                     'answers', 'answers__options', 'answers__question',
                     'item__category',
                     'addon_to__answers', 'addon_to__answers__options', 'addon_to__answers__question',
@@ -392,8 +395,7 @@ class OrderViewSet(viewsets.ModelViewSet):
                 oauth_application=request.auth.application if isinstance(request.auth, OAuthAccessToken) else None,
                 send_mail=send_mail,
                 email_comment=comment,
-                cancellation_fee=cancellation_fee,
-                source=get_api_source(request),
+                cancellation_fee=cancellation_fee
             )
         except OrderError as e:
             return Response(
@@ -417,7 +419,6 @@ class OrderViewSet(viewsets.ModelViewSet):
                 order,
                 user=request.user if request.user.is_authenticated else None,
                 auth=request.auth if isinstance(request.auth, (Device, TeamAPIToken, OAuthAccessToken)) else None,
-                source=get_api_source(request),
             )
         except OrderError as e:
             return Response(
@@ -437,7 +438,6 @@ class OrderViewSet(viewsets.ModelViewSet):
                 user=request.user if request.user.is_authenticated else None,
                 auth=request.auth if isinstance(request.auth, (Device, TeamAPIToken, OAuthAccessToken)) else None,
                 send_mail=send_mail,
-                source=get_api_source(request),
             )
         except Quota.QuotaExceededException as e:
             return Response({'detail': str(e)}, status=status.HTTP_400_BAD_REQUEST)
@@ -458,7 +458,6 @@ class OrderViewSet(viewsets.ModelViewSet):
                 auth=request.auth if isinstance(request.auth, (Device, TeamAPIToken, OAuthAccessToken)) else None,
                 send_mail=send_mail,
                 comment=comment,
-                source=get_api_source(request),
             )
         except OrderError as e:
             return Response({'detail': str(e)}, status=status.HTTP_400_BAD_REQUEST)
@@ -497,7 +496,6 @@ class OrderViewSet(viewsets.ModelViewSet):
             order,
             user=request.user if request.user.is_authenticated else None,
             auth=request.auth,
-            source=get_api_source(request),
         )
         return self.retrieve(request, [], **kwargs)
 
@@ -515,7 +513,6 @@ class OrderViewSet(viewsets.ModelViewSet):
             order,
             user=request.user if request.user.is_authenticated else None,
             auth=(request.auth if isinstance(request.auth, (TeamAPIToken, OAuthAccessToken, Device)) else None),
-            source=get_api_source(request),
         )
         return self.retrieve(request, [], **kwargs)
 
@@ -687,28 +684,33 @@ class OrderViewSet(viewsets.ModelViewSet):
                 )
                 if order.require_approval:
                     email_template = request.event.settings.mail_text_order_placed_require_approval
+                    subject_template = request.event.settings.mail_subject_order_placed_require_approval
                     log_entry = 'pretix.event.order.email.order_placed_require_approval'
                     email_attendees = False
                 elif free_flow:
                     email_template = request.event.settings.mail_text_order_free
+                    subject_template = request.event.settings.mail_subject_order_free
                     log_entry = 'pretix.event.order.email.order_free'
                     email_attendees = request.event.settings.mail_send_order_free_attendee
                     email_attendees_template = request.event.settings.mail_text_order_free_attendee
+                    subject_attendees_template = request.event.settings.mail_subject_order_free_attendee
                 else:
                     email_template = request.event.settings.mail_text_order_placed
+                    subject_template = request.event.settings.mail_subject_order_placed
                     log_entry = 'pretix.event.order.email.order_placed'
                     email_attendees = request.event.settings.mail_send_order_placed_attendee
                     email_attendees_template = request.event.settings.mail_text_order_placed_attendee
+                    subject_attendees_template = request.event.settings.mail_subject_order_placed_attendee
 
                 _order_placed_email(
-                    request.event, order, payment.payment_provider if payment else None, email_template,
-                    log_entry, invoice, payment, is_free=free_flow
+                    request.event, order, email_template, subject_template,
+                    log_entry, invoice, [payment] if payment else [], is_free=free_flow
                 )
                 if email_attendees:
                     for p in order.positions.all():
                         if p.addon_to_id is None and p.attendee_email and p.attendee_email != order.email:
-                            _order_placed_email_attendee(request.event, order, p, email_attendees_template, log_entry,
-                                                         is_free=free_flow)
+                            _order_placed_email_attendee(request.event, order, p, email_attendees_template, subject_attendees_template,
+                                                         log_entry, is_free=free_flow)
 
                 if not free_flow and order.status == Order.STATUS_PAID and payment:
                     payment._send_paid_mail(invoice, None, '')
@@ -938,7 +940,7 @@ with scopes_disabled():
 class OrderPositionViewSet(viewsets.ModelViewSet):
     serializer_class = OrderPositionSerializer
     queryset = OrderPosition.all.none()
-    filter_backends = (DjangoFilterBackend, OrderingFilter)
+    filter_backends = (DjangoFilterBackend, RichOrderingFilter)
     ordering = ('order__datetime', 'positionid')
     ordering_fields = ('order__code', 'order__datetime', 'positionid', 'attendee_name', 'order__status',)
     filterset_class = OrderPositionFilter
@@ -1000,7 +1002,11 @@ class OrderPositionViewSet(viewsets.ModelViewSet):
                                 Prefetch('meta_values', ItemMetaValue.objects.select_related('property'),
                                          to_attr='meta_values_cached')
                             )),
-                            'variation', 'answers', 'answers__options', 'answers__question',
+                            Prefetch('variation', queryset=self.request.event.items.prefetch_related(
+                                Prefetch('meta_values', ItemVariationMetaValue.objects.select_related('property'),
+                                         to_attr='meta_values_cached')
+                            )),
+                            'answers', 'answers__options', 'answers__question',
                             'item__category',
                             Prefetch('subevent', queryset=self.request.event.subevents.prefetch_related(
                                 Prefetch('meta_values', to_attr='meta_values_cached',
@@ -1492,8 +1498,7 @@ class PaymentViewSet(CreateModelMixin, viewsets.ReadOnlyModelViewSet):
                 if mark_refunded:
                     mark_order_refunded(payment.order,
                                         user=self.request.user if self.request.user.is_authenticated else None,
-                                        auth=self.request.auth,
-                                        source=get_api_source(self.request))
+                                        auth=self.request.auth)
                 else:
                     payment.order.status = Order.STATUS_PENDING
                     payment.order.set_expires(
@@ -1566,7 +1571,7 @@ class RefundViewSet(CreateModelMixin, viewsets.ReadOnlyModelViewSet):
             mark_refunded = request.data.get('mark_canceled', False)
         if mark_refunded:
             mark_order_refunded(refund.order, user=self.request.user if self.request.user.is_authenticated else None,
-                                auth=self.request.auth, source=get_api_source(self.request))
+                                auth=self.request.auth)
         elif not (refund.order.status == Order.STATUS_PAID and refund.order.pending_sum <= 0):
             refund.order.status = Order.STATUS_PENDING
             refund.order.set_expires(
@@ -1613,13 +1618,23 @@ class RefundViewSet(CreateModelMixin, viewsets.ReadOnlyModelViewSet):
                 user=request.user if request.user.is_authenticated else None,
                 auth=request.auth
             )
+
+            if r.state in (OrderRefund.REFUND_STATE_DONE, OrderRefund.REFUND_STATE_CANCELED, OrderRefund.REFUND_STATE_FAILED):
+                r.order.log_action(
+                    f'pretix.event.order.refund.{r.state}', {
+                        'local_id': r.local_id,
+                        'provider': r.provider,
+                    },
+                    user=request.user if request.user.is_authenticated else None,
+                    auth=request.auth
+                )
+
             if mark_refunded:
                 try:
                     mark_order_refunded(
                         r.order,
                         user=request.user if request.user.is_authenticated else None,
                         auth=(request.auth if request.auth else None),
-                        source=get_api_source(self.request),
                     )
                 except OrderError as e:
                     raise ValidationError(str(e))

src/pretix/api/views/organizer.py

@@ -51,6 +51,7 @@ from pretix.base.models import (
     User,
 )
 from pretix.base.settings import SETTINGS_AFFECTING_CSS
+from pretix.helpers import OF_SELF
 from pretix.helpers.dicts import merge_dicts
 from pretix.presale.style import regenerate_organizer_css
 
@@ -178,7 +179,7 @@ class GiftCardViewSet(viewsets.ModelViewSet):
     def perform_update(self, serializer):
         if 'include_accepted' in self.request.GET:
             raise PermissionDenied("Accepted gift cards cannot be updated, use transact instead.")
-        GiftCard.objects.select_for_update().get(pk=self.get_object().pk)
+        GiftCard.objects.select_for_update(of=OF_SELF).get(pk=self.get_object().pk)
         old_value = serializer.instance.value
         value = serializer.validated_data.pop('value')
         inst = serializer.save(secret=serializer.instance.secret, currency=serializer.instance.currency,
@@ -196,7 +197,7 @@ class GiftCardViewSet(viewsets.ModelViewSet):
     @action(detail=True, methods=["POST"])
     @transaction.atomic()
     def transact(self, request, **kwargs):
-        gc = GiftCard.objects.select_for_update().get(pk=self.get_object().pk)
+        gc = GiftCard.objects.select_for_update(of=OF_SELF).get(pk=self.get_object().pk)
         value = serializers.DecimalField(max_digits=10, decimal_places=2).to_internal_value(
             request.data.get('value')
         )

src/pretix/api/views/upload.py

@@ -21,6 +21,7 @@
 #
 import datetime
 
+from django.core.exceptions import ValidationError as DjangoValidationError
 from django.utils.timezone import now
 from oauth2_provider.contrib.rest_framework import OAuth2Authentication
 from rest_framework.authentication import SessionAuthentication
@@ -33,6 +34,9 @@ from pretix.api.auth.device import DeviceTokenAuthentication
 from pretix.api.auth.permission import AnyAuthenticatedClientPermission
 from pretix.api.auth.token import TeamTokenAuthentication
 from pretix.base.models import CachedFile
+from pretix.helpers.images import (
+    IMAGE_TYPES, validate_uploaded_file_for_valid_image,
+)
 
 ALLOWED_TYPES = {
     'image/gif': {'.gif'},
@@ -61,6 +65,13 @@ class UploadView(APIView):
                 name=file_obj.name,
                 type=content_type
             ))
+
+        if content_type in IMAGE_TYPES:
+            try:
+                validate_uploaded_file_for_valid_image(file_obj)
+            except DjangoValidationError as e:
+                raise ValidationError(e.message)
+
         cf = CachedFile.objects.create(
             expires=now() + datetime.timedelta(days=1),
             date=now(),

src/pretix/api/webhooks.py

@@ -42,6 +42,7 @@ from pretix.base.models import LogEntry
 from pretix.base.services.tasks import ProfiledTask, TransactionAwareTask
 from pretix.base.signals import periodic_task
 from pretix.celery_app import app
+from pretix.helpers import OF_SELF
 
 logger = logging.getLogger(__name__)
 _ALL_EVENTS = None
@@ -502,7 +503,8 @@ def manually_retry_all_calls(webhook_id: int):
         webhook = WebHook.objects.get(id=webhook_id)
     with scope(organizer=webhook.organizer), transaction.atomic():
         for whcr in webhook.retries.select_for_update(
-            skip_locked=connection.features.has_select_for_update_skip_locked
+            skip_locked=connection.features.has_select_for_update_skip_locked,
+            of=OF_SELF
         ):
             send_webhook.apply_async(
                 args=(whcr.logentry_id, whcr.action_type, whcr.webhook_id, whcr.retry_count),
@@ -515,7 +517,8 @@ def manually_retry_all_calls(webhook_id: int):
 def schedule_webhook_retries_on_celery(sender, **kwargs):
     with transaction.atomic():
         for whcr in WebHookCallRetry.objects.select_for_update(
-            skip_locked=connection.features.has_select_for_update_skip_locked
+            skip_locked=connection.features.has_select_for_update_skip_locked,
+            of=OF_SELF
         ).filter(retry_not_before__lt=now()):
             send_webhook.apply_async(
                 args=(whcr.logentry_id, whcr.action_type, whcr.webhook_id, whcr.retry_count),

src/pretix/base/addressvalidation.py

@@ -0,0 +1,225 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020 Raphael Michel and contributors
+# Copyright (C) 2020-2021 rami.io GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+from collections import defaultdict
+
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext_lazy as _
+from localflavor.ar.forms import ARPostalCodeField
+from localflavor.at.forms import ATZipCodeField
+from localflavor.au.forms import AUPostCodeField
+from localflavor.be.forms import BEPostalCodeField
+from localflavor.br.forms import BRZipCodeField
+from localflavor.ca.forms import CAPostalCodeField
+from localflavor.ch.forms import CHZipCodeField
+from localflavor.cn.forms import CNPostCodeField
+from localflavor.cu.forms import CUPostalCodeField
+from localflavor.cz.forms import CZPostalCodeField
+from localflavor.de.forms import DEZipCodeField
+from localflavor.dk.forms import DKPostalCodeField
+from localflavor.ee.forms import EEZipCodeField
+from localflavor.es.forms import ESPostalCodeField
+from localflavor.fi.forms import FIZipCodeField
+from localflavor.fr.forms import FRZipCodeField
+from localflavor.gb.forms import GBPostcodeField
+from localflavor.gr.forms import GRPostalCodeField
+from localflavor.hr.forms import HRPostalCodeField
+from localflavor.ie.forms import EircodeField
+from localflavor.il.forms import ILPostalCodeField
+from localflavor.in_.forms import INZipCodeField
+from localflavor.ir.forms import IRPostalCodeField
+from localflavor.is_.is_postalcodes import IS_POSTALCODES
+from localflavor.it.forms import ITZipCodeField
+from localflavor.jp.forms import JPPostalCodeField
+from localflavor.lt.forms import LTPostalCodeField
+from localflavor.lv.forms import LVPostalCodeField
+from localflavor.ma.forms import MAPostalCodeField
+from localflavor.mt.forms import MTPostalCodeField
+from localflavor.mx.forms import MXZipCodeField
+from localflavor.nl.forms import NLZipCodeField
+from localflavor.no.forms import NOZipCodeField
+from localflavor.nz.forms import NZPostCodeField
+from localflavor.pk.forms import PKPostCodeField
+from localflavor.pl.forms import PLPostalCodeField
+from localflavor.pt.forms import PTZipCodeField
+from localflavor.ro.forms import ROPostalCodeField
+from localflavor.ru.forms import RUPostalCodeField
+from localflavor.se.forms import SEPostalCodeField
+from localflavor.sg.forms import SGPostCodeField
+from localflavor.si.si_postalcodes import SI_POSTALCODES
+from localflavor.sk.forms import SKPostalCodeField
+from localflavor.tr.forms import TRPostalCodeField
+from localflavor.ua.forms import UAPostalCodeField
+from localflavor.us.forms import USZipCodeField
+from localflavor.za.forms import ZAPostCodeField
+
+from pretix.base.settings import COUNTRIES_WITH_STATE_IN_ADDRESS
+
+_validator_classes = defaultdict(list)
+
+COUNTRIES_WITH_STREET_ZIPCODE_AND_CITY_REQUIRED = {
+    # We don't presume this for countries we don't have knowledge about, there are countries in the
+    # world e.g. without zipcodes
+    'AR', 'AT', 'AU', 'BE', 'BR', 'CA', 'CH', 'CN', 'CU', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR',
+    'GB', 'GR', 'HR', 'IE', 'IL', 'IN', 'IR', 'IS', 'IT', 'JP', 'LT', 'LV', 'MA', 'MT', 'MX', 'NL',
+    'NO', 'NZ', 'PK', 'PL', 'PT', 'RO', 'RU', 'SE', 'SG', 'SI', 'SK', 'TR', 'UA', 'US', 'ZA',
+}
+
+
+def validate_address(address: dict, all_optional=False):
+    """
+    :param address: A dictionary with at least the entries ``street``, ``zipcode``, ``city``, ``country``,
+                    ``state``
+    :return: The dictionary, possibly with changes
+    """
+    if not address.get('street') and not address.get('zipcode') and not address.get('city'):
+        # Consider the actual address part to be empty, no further validation necessary, if the
+        # address should be required, it's the callers job to validate that at least one of these
+        # fields is filled
+        return address
+
+    if not address.get('country'):
+        raise ValidationError({'country': [_('This field is required.')]})
+
+    if str(address['country']) in COUNTRIES_WITH_STATE_IN_ADDRESS and not address.get('state') and not all_optional:
+        raise ValidationError({'state': [_('This field is required.')]})
+
+    if str(address['country']) in COUNTRIES_WITH_STREET_ZIPCODE_AND_CITY_REQUIRED and not all_optional:
+        for f in ('street', 'zipcode', 'city'):
+            if not address.get(f):
+                raise ValidationError({f: [_('This field is required.')]})
+
+    for klass in _validator_classes[str(address['country'])]:
+        validator = klass()
+        try:
+            if address.get('zipcode'):
+                address['zipcode'] = validator.validate_zipcode(address['zipcode'])
+        except ValidationError as e:
+            raise ValidationError({'zipcode': list(e)})
+
+    return address
+
+
+def register_validator_for(country):
+    def inner(klass):
+        _validator_classes[country].append(klass)
+        return klass
+
+    return inner
+
+
+class BaseValidator:
+    required_fields = []
+
+    def validate_zipcode(self, value):
+        return value
+
+
+"""
+Currently, mostly have validators that are auto-generated from django-localflavor
+but custom ones can be added like this:
+
+@register_validator_for('DE')
+class DEValidator(BaseValidator):
+    def validate_zipcode(value):
+        return value
+
+In the future, we can also add additional methods to validate that e.g. a city
+is plausible for a given zip code.
+"""
+
+_zip_code_fields = {
+    'AR': ARPostalCodeField,
+    'AT': ATZipCodeField,
+    'AU': AUPostCodeField,
+    'BE': BEPostalCodeField,
+    'BR': BRZipCodeField,
+    'CA': CAPostalCodeField,
+    'CH': CHZipCodeField,
+    'CN': CNPostCodeField,
+    'CU': CUPostalCodeField,
+    'CZ': CZPostalCodeField,
+    'DE': DEZipCodeField,
+    'DK': DKPostalCodeField,
+    'EE': EEZipCodeField,
+    'ES': ESPostalCodeField,
+    'FI': FIZipCodeField,
+    'FR': FRZipCodeField,
+    'GB': GBPostcodeField,
+    'GR': GRPostalCodeField,
+    'HR': HRPostalCodeField,
+    'IE': EircodeField,
+    'IL': ILPostalCodeField,
+    'IN': INZipCodeField,
+    'IR': IRPostalCodeField,
+    'IT': ITZipCodeField,
+    'JP': JPPostalCodeField,
+    'LT': LTPostalCodeField,
+    'LV': LVPostalCodeField,
+    'MA': MAPostalCodeField,
+    'MT': MTPostalCodeField,
+    'MX': MXZipCodeField,
+    'NL': NLZipCodeField,
+    'NO': NOZipCodeField,
+    'NZ': NZPostCodeField,
+    'PK': PKPostCodeField,
+    'PL': PLPostalCodeField,
+    'PT': PTZipCodeField,
+    'RO': ROPostalCodeField,
+    'RU': RUPostalCodeField,
+    'SE': SEPostalCodeField,
+    'SG': SGPostCodeField,
+    'SK': SKPostalCodeField,
+    'TR': TRPostalCodeField,
+    'UA': UAPostalCodeField,
+    'US': USZipCodeField,
+    'ZA': ZAPostCodeField,
+}
+
+
+def _generate_class_from_zipcode_field(field_class):
+    class _GeneratedValidator(BaseValidator):
+        def validate_zipcode(self, value):
+            return field_class().clean(value)
+    return _GeneratedValidator
+
+
+for cc, field_class in _zip_code_fields.items():
+    register_validator_for(cc)(_generate_class_from_zipcode_field(field_class))
+
+
+@register_validator_for('IS')
+class ISValidator(BaseValidator):
+    def validate_zipcode(self, value):
+        if value not in [entry[0] for entry in IS_POSTALCODES]:
+            raise ValidationError(_('Enter a postal code in the format XXX.'), code='invalid')
+        return value
+
+
+@register_validator_for('SI')
+class SIValidator(BaseValidator):
+    def validate_zipcode(self, value):
+        try:
+            if int(value) not in [entry[0] for entry in SI_POSTALCODES]:
+                raise ValidationError(_('Enter a postal code in the format XXXX.'), code='invalid')
+        except ValueError:
+            raise ValidationError(_('Enter a postal code in the format XXXX.'), code='invalid')
+        return value

src/pretix/base/email.py

@@ -320,13 +320,18 @@ def get_email_context(**kwargs):
     return ctx
 
 
-def _placeholder_payment(order, payment):
-    if not payment:
-        return None
-    if 'payment' in inspect.signature(payment.payment_provider.order_pending_mail_render).parameters:
-        return str(payment.payment_provider.order_pending_mail_render(order, payment))
+def _placeholder_payments(order, payments):
+    d = []
+    for payment in payments:
+        if 'payment' in inspect.signature(payment.payment_provider.order_pending_mail_render).parameters:
+            d.append(str(payment.payment_provider.order_pending_mail_render(order, payment)))
+        else:
+            d.append(str(payment.payment_provider.order_pending_mail_render(order)))
+    d = [line for line in d if line.strip()]
+    if d:
+        return '\n\n'.join(d)
     else:
-        return str(payment.payment_provider.order_pending_mail_render(order))
+        return ''
 
 
 def get_best_name(position_or_address, parts=False):
@@ -376,6 +381,14 @@ def base_placeholders(sender, **kwargs):
         SimpleFunctionalMailTextPlaceholder(
             'currency', ['event'], lambda event: event.currency, lambda event: event.currency
         ),
+        SimpleFunctionalMailTextPlaceholder(
+            'order_email', ['order'], lambda order: order.email, 'john@example.org'
+        ),
+        SimpleFunctionalMailTextPlaceholder(
+            'invoice_number', ['invoice'],
+            lambda invoice: invoice.full_invoice_no,
+            f'{sender.settings.invoice_numbers_prefix or (sender.slug.upper() + "-")}00000'
+        ),
         SimpleFunctionalMailTextPlaceholder(
             'refund_amount', ['event_or_subevent', 'refund_amount'],
             lambda event_or_subevent, refund_amount: LazyCurrencyNumber(refund_amount, event_or_subevent.currency),
@@ -507,20 +520,20 @@ def base_placeholders(sender, **kwargs):
             lambda event: (event if not event.has_subevents or not event.subevents.exists() else event.subevents.first()).get_date_from_display()
         ),
         SimpleFunctionalMailTextPlaceholder(
-            'url_remove', ['waiting_list_entry', 'event'],
-            lambda waiting_list_entry, event: build_absolute_uri(
+            'url_remove', ['waiting_list_voucher', 'event'],
+            lambda waiting_list_voucher, event: build_absolute_uri(
                 event, 'presale:event.waitinglist.remove'
-            ) + '?voucher=' + waiting_list_entry.voucher.code,
+            ) + '?voucher=' + waiting_list_voucher.code,
             lambda event: build_absolute_uri(
                 event,
                 'presale:event.waitinglist.remove',
             ) + '?voucher=68CYU2H6ZTP3WLK5',
         ),
         SimpleFunctionalMailTextPlaceholder(
-            'url', ['waiting_list_entry', 'event'],
-            lambda waiting_list_entry, event: build_absolute_uri(
+            'url', ['waiting_list_voucher', 'event'],
+            lambda waiting_list_voucher, event: build_absolute_uri(
                 event, 'presale:event.redeem'
-            ) + '?voucher=' + waiting_list_entry.voucher.code,
+            ) + '?voucher=' + waiting_list_voucher.code,
             lambda event: build_absolute_uri(
                 event,
                 'presale:event.redeem',
@@ -575,7 +588,7 @@ def base_placeholders(sender, **kwargs):
             _('Sample Admission Ticket')
         ),
         SimpleFunctionalMailTextPlaceholder(
-            'code', ['waiting_list_entry'], lambda waiting_list_entry: waiting_list_entry.voucher.code,
+            'code', ['waiting_list_voucher'], lambda waiting_list_voucher: waiting_list_voucher.code,
             '68CYU2H6ZTP3WLK5'
         ),
         SimpleFunctionalMailTextPlaceholder(
@@ -617,7 +630,7 @@ def base_placeholders(sender, **kwargs):
             _('An individual text with a reason can be inserted here.'),
         ),
         SimpleFunctionalMailTextPlaceholder(
-            'payment_info', ['order', 'payment'], _placeholder_payment,
+            'payment_info', ['order', 'payments'], _placeholder_payments,
             _('The amount has been charged to your card.'),
         ),
         SimpleFunctionalMailTextPlaceholder(

src/pretix/base/exporter.py

@@ -36,7 +36,7 @@ import io
 import tempfile
 from collections import OrderedDict, namedtuple
 from decimal import Decimal
-from typing import Tuple
+from typing import Optional, Tuple
 
 import pytz
 from defusedcsv import csv
@@ -84,6 +84,27 @@ class BaseExporter:
         """
         raise NotImplementedError()  # NOQA
 
+    @property
+    def description(self) -> str:
+        """
+        A description for this exporter.
+        """
+        return ""
+
+    @property
+    def category(self) -> Optional[str]:
+        """
+        A category name for this exporter, or ``None``.
+        """
+        return None
+
+    @property
+    def featured(self) -> bool:
+        """
+        If ``True``, this exporter will be highlighted.
+        """
+        return False
+
     @property
     def identifier(self) -> str:
         """

src/pretix/base/exporters/answers.py

@@ -39,7 +39,7 @@ from zipfile import ZipFile
 
 from django import forms
 from django.dispatch import receiver
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import gettext_lazy as _, pgettext_lazy
 
 from pretix.base.models import QuestionAnswer
 
@@ -49,7 +49,10 @@ from ..signals import register_data_exporters
 
 class AnswerFilesExporter(BaseExporter):
     identifier = 'answerfiles'
-    verbose_name = _('Answers to file upload questions')
+    verbose_name = _('Question answer file uploads')
+    category = pgettext_lazy('export_category', 'Order data')
+    description = _('Download a ZIP file including all files that have been uploaded by your customers while creating '
+                    'an order.')
 
     @property
     def export_form_fields(self):

src/pretix/base/exporters/customers.py

@@ -36,7 +36,7 @@ from collections import OrderedDict
 
 from django.dispatch import receiver
 from django.utils.timezone import get_current_timezone
-from django.utils.translation import gettext as _, gettext_lazy
+from django.utils.translation import gettext as _, gettext_lazy, pgettext_lazy
 
 from pretix.base.settings import PERSON_NAME_SCHEMES
 
@@ -48,6 +48,8 @@ class CustomerListExporter(OrganizerLevelExportMixin, ListExporter):
     identifier = 'customerlist'
     verbose_name = gettext_lazy('Customer accounts')
     organizer_required_permission = 'can_manage_customers'
+    category = pgettext_lazy('export_category', 'Customer accounts')
+    description = gettext_lazy('Download a spreadsheet of all currently registered customer accounts.')
 
     @property
     def additional_form_fields(self):

src/pretix/base/exporters/dekodi.py

@@ -23,22 +23,24 @@ import json
 from collections import OrderedDict
 from decimal import Decimal
 
-import dateutil
-from django import forms
 from django.core.serializers.json import DjangoJSONEncoder
 from django.dispatch import receiver
-from django.utils.translation import gettext, gettext_lazy
+from django.utils.timezone import now
+from django.utils.translation import gettext, gettext_lazy, pgettext_lazy
 
 from pretix.base.i18n import language
 from pretix.base.models import Invoice, OrderPayment
 
 from ..exporter import BaseExporter
 from ..signals import register_data_exporters
+from ..timeframes import DateFrameField, resolve_timeframe_to_dates_inclusive
 
 
 class DekodiNREIExporter(BaseExporter):
     identifier = 'dekodi_nrei'
     verbose_name = 'dekodi NREI (JSON)'
+    category = pgettext_lazy('export_category', 'Invoices')
+    description = gettext_lazy("Download invoices in a format that can be used by the dekodi NREI conversion software.")
 
     # Specification: http://manuals.dekodi.de/nexuspub/schnittstellenbuch/
 
@@ -113,7 +115,7 @@ class DekodiNREIExporter(BaseExporter):
                         'PTNo14': p.info_data.get('reference') or '',
                         'PTNo15': p.full_id or '',
                     })
-            elif p.provider.startswith('stripe'):
+            elif p.provider and p.provider.startswith('stripe'):
                 src = p.info_data.get("source", p.info_data)
                 payments.append({
                     'PTID': '81',
@@ -192,17 +194,12 @@ class DekodiNREIExporter(BaseExporter):
     def render(self, form_data):
         qs = self.event.invoices.select_related('order').prefetch_related('lines', 'lines__subevent')
 
-        if form_data.get('date_from'):
-            date_value = form_data.get('date_from')
-            if isinstance(date_value, str):
-                date_value = dateutil.parser.parse(date_value).date()
-            qs = qs.filter(date__gte=date_value)
-
-        if form_data.get('date_to'):
-            date_value = form_data.get('date_to')
-            if isinstance(date_value, str):
-                date_value = dateutil.parser.parse(date_value).date()
-            qs = qs.filter(date__lte=date_value)
+        if form_data.get('date_range'):
+            d_start, d_end = resolve_timeframe_to_dates_inclusive(now(), form_data['date_range'], self.timezone)
+            if d_start:
+                qs = qs.filter(date__gte=d_start)
+            if d_end:
+                qs = qs.filter(date__lte=d_end)
 
         jo = {
             'Format': 'NREI',
@@ -218,22 +215,14 @@ class DekodiNREIExporter(BaseExporter):
     def export_form_fields(self):
         return OrderedDict(
             [
-                ('date_from',
-                 forms.DateField(
-                     label=gettext_lazy('Start date'),
-                     widget=forms.DateInput(attrs={'class': 'datepickerfield'}),
+                ('date_range',
+                 DateFrameField(
+                     label=gettext_lazy('Date range'),
+                     include_future_frames=False,
                      required=False,
-                     help_text=gettext_lazy('Only include invoices issued on or after this date. Note that the invoice date does '
+                     help_text=gettext_lazy('Only include invoices issued in this time frame. Note that the invoice date does '
                                             'not always correspond to the order or payment date.')
                  )),
-                ('date_to',
-                 forms.DateField(
-                     label=gettext_lazy('End date'),
-                     widget=forms.DateInput(attrs={'class': 'datepickerfield'}),
-                     required=False,
-                     help_text=gettext_lazy('Only include invoices issued on or before this date. Note that the invoice date '
-                                            'does not always correspond to the order or payment date.')
-                 )),
             ]
         )
 

src/pretix/base/exporters/events.py

@@ -35,7 +35,7 @@
 from django.dispatch import receiver
 from django.utils.formats import date_format
 from django.utils.functional import cached_property
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import gettext_lazy as _, pgettext_lazy
 
 from ...control.forms.filter import get_all_payment_providers
 from ..exporter import ListExporter
@@ -45,6 +45,8 @@ from ..signals import register_multievent_data_exporters
 class EventDataExporter(ListExporter):
     identifier = 'eventdata'
     verbose_name = _('Event data')
+    category = pgettext_lazy('export_category', 'Event data')
+    description = _('Download a spreadsheet with information on all events in this organizer account.')
 
     @cached_property
     def providers(self):

src/pretix/base/exporters/invoices.py

@@ -38,13 +38,15 @@ from collections import OrderedDict
 from decimal import Decimal
 from zipfile import ZipFile
 
-import dateutil.parser
 from django import forms
 from django.db.models import CharField, Exists, F, OuterRef, Q, Subquery, Sum
 from django.dispatch import receiver
 from django.utils.formats import date_format
 from django.utils.functional import cached_property
-from django.utils.translation import gettext, gettext_lazy as _, pgettext
+from django.utils.timezone import now
+from django.utils.translation import (
+    gettext, gettext_lazy as _, pgettext, pgettext_lazy,
+)
 
 from pretix.base.models import Invoice, InvoiceLine, OrderPayment
 
@@ -57,30 +59,24 @@ from ..services.invoices import invoice_pdf_task
 from ..signals import (
     register_data_exporters, register_multievent_data_exporters,
 )
+from ..timeframes import DateFrameField, resolve_timeframe_to_dates_inclusive
 
 
 class InvoiceExporterMixin:
+    category = pgettext_lazy('export_category', 'Invoices')
 
     @property
     def invoice_exporter_form_fields(self):
         return OrderedDict(
             [
-                ('date_from',
-                 forms.DateField(
-                     label=_('Start date'),
-                     widget=forms.DateInput(attrs={'class': 'datepickerfield'}),
+                ('date_range',
+                 DateFrameField(
+                     label=_('Date range'),
+                     include_future_frames=False,
                      required=False,
-                     help_text=_('Only include invoices issued on or after this date. Note that the invoice date does '
+                     help_text=_('Only include invoices issued in this time frame. Note that the invoice date does '
                                  'not always correspond to the order or payment date.')
                  )),
-                ('date_to',
-                 forms.DateField(
-                     label=_('End date'),
-                     widget=forms.DateInput(attrs={'class': 'datepickerfield'}),
-                     required=False,
-                     help_text=_('Only include invoices issued on or before this date. Note that the invoice date '
-                                 'does not always correspond to the order or payment date.')
-                 )),
                 ('payment_provider',
                  forms.ChoiceField(
                      label=_('Payment provider'),
@@ -112,16 +108,12 @@ class InvoiceExporterMixin:
                 )
             )
             qs = qs.filter(has_payment_with_provider=1)
-        if form_data.get('date_from'):
-            date_value = form_data.get('date_from')
-            if isinstance(date_value, str):
-                date_value = dateutil.parser.parse(date_value).date()
-            qs = qs.filter(date__gte=date_value)
-        if form_data.get('date_to'):
-            date_value = form_data.get('date_to')
-            if isinstance(date_value, str):
-                date_value = dateutil.parser.parse(date_value).date()
-            qs = qs.filter(date__lte=date_value)
+        if form_data.get('date_range'):
+            d_start, d_end = resolve_timeframe_to_dates_inclusive(now(), form_data['date_range'], self.timezone)
+            if d_start:
+                qs = qs.filter(date__gte=d_start)
+            if d_end:
+                qs = qs.filter(date__lte=d_end)
 
         return qs
 
@@ -129,6 +121,7 @@ class InvoiceExporterMixin:
 class InvoiceExporter(InvoiceExporterMixin, BaseExporter):
     identifier = 'invoices'
     verbose_name = _('All invoices')
+    description = _('Download all invoices created by the system as a ZIP file of PDF files.')
 
     def render(self, form_data: dict, output_file=None):
         qs = self.invoices_queryset(form_data).filter(shredded=False)
@@ -180,6 +173,10 @@ class InvoiceExporter(InvoiceExporterMixin, BaseExporter):
 class InvoiceDataExporter(InvoiceExporterMixin, MultiSheetListExporter):
     identifier = 'invoicedata'
     verbose_name = _('Invoice data')
+    description = _('Download a spreadsheet with the data of all invoices created by the system. The spreadsheet '
+                    'includes two sheets, one with a line for every invoice, and one with a line for every position of '
+                    'every invoice.')
+    featured = True
 
     @property
     def additional_form_fields(self):

src/pretix/base/exporters/items.py

@@ -22,14 +22,14 @@
 from django.db.models import Prefetch
 from django.dispatch import receiver
 from django.utils.formats import date_format
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import gettext_lazy as _, pgettext_lazy
 from openpyxl.styles import Alignment
 from openpyxl.utils import get_column_letter
 
 from ...helpers.safe_openpyxl import SafeCell
 from ..channels import get_all_sales_channels
 from ..exporter import ListExporter
-from ..models import ItemMetaValue
+from ..models import ItemMetaValue, ItemVariation, ItemVariationMetaValue
 from ..signals import register_data_exporters
 
 
@@ -48,6 +48,8 @@ def _min(a1, a2):
 class ItemDataExporter(ListExporter):
     identifier = 'itemdata'
     verbose_name = _('Product data')
+    category = pgettext_lazy('export_category', 'Product data')
+    description = _('Download a spreadsheet with details about all products and variations.')
 
     def iterate_list(self, form_data):
         locales = self.event.settings.locales
@@ -73,6 +75,7 @@ class ItemDataExporter(ListExporter):
             _("Free price input"),
             _("Sales tax"),
             _("Is an admission ticket"),
+            _("Personalized ticket"),
             _("Generate tickets"),
             _("Waiting list"),
             _("Available from"),
@@ -106,18 +109,27 @@ class ItemDataExporter(ListExporter):
         yield row
 
         for i in self.event.items.prefetch_related(
-            'variations',
             Prefetch(
                 'meta_values',
                 ItemMetaValue.objects.select_related('property'),
                 to_attr='meta_values_cached'
-            )
+            ),
+            Prefetch(
+                'variations',
+                queryset=ItemVariation.objects.prefetch_related(
+                    Prefetch(
+                        'meta_values',
+                        ItemVariationMetaValue.objects.select_related('property'),
+                        to_attr='meta_values_cached'
+                    ),
+                ),
+            ),
         ).select_related('category', 'tax_rule'):
-            m = i.meta_data
             vars = list(i.variations.all())
 
             if vars:
                 for v in vars:
+                    m = v.meta_data
                     row = [
                         i.pk,
                         v.pk,
@@ -135,6 +147,7 @@ class ItemDataExporter(ListExporter):
                         _("Yes") if i.free_price else "",
                         str(i.tax_rule) if i.tax_rule else "",
                         _("Yes") if i.admission else "",
+                        _("Yes") if i.personalized else "",
                         _("Yes") if i.generate_tickets else (_("Default") if i.generate_tickets is None else ""),
                         _("Yes") if i.allow_waitinglist else "",
                         date_format(_max(i.available_from, v.available_from).astimezone(self.timezone),
@@ -160,6 +173,7 @@ class ItemDataExporter(ListExporter):
                     yield row
 
             else:
+                m = i.meta_data
                 row = [
                     i.pk,
                     "",
@@ -177,6 +191,7 @@ class ItemDataExporter(ListExporter):
                     _("Yes") if i.free_price else "",
                     str(i.tax_rule) if i.tax_rule else "",
                     _("Yes") if i.admission else "",
+                    _("Yes") if i.personalized else "",
                     _("Yes") if i.generate_tickets else (_("Default") if i.generate_tickets is None else ""),
                     _("Yes") if i.allow_waitinglist else "",
                     date_format(i.available_from.astimezone(self.timezone),

src/pretix/base/exporters/json.py

@@ -36,15 +36,22 @@ import json
 from decimal import Decimal
 
 from django.core.serializers.json import DjangoJSONEncoder
+from django.db.models import Prefetch
 from django.dispatch import receiver
+from django.utils.functional import lazy
+from django.utils.translation import gettext, gettext_lazy, pgettext_lazy
 
 from ..exporter import BaseExporter
+from ..models import ItemMetaValue, ItemVariation, ItemVariationMetaValue
 from ..signals import register_data_exporters
 
 
 class JSONExporter(BaseExporter):
     identifier = 'json'
-    verbose_name = 'Order data (JSON)'
+    verbose_name = lazy(lambda *args: gettext('Order data') + ' (JSON)', str)()
+    category = pgettext_lazy('export_category', 'Order data')
+    description = gettext_lazy('Download a structured JSON representation of all orders. This might be useful for the '
+                               'import in third-party systems.')
 
     def render(self, form_data):
         jo = {
@@ -76,6 +83,7 @@ class JSONExporter(BaseExporter):
                         'tax_rate': item.tax_rule.rate if item.tax_rule else Decimal('0.00'),
                         'tax_name': str(item.tax_rule.name) if item.tax_rule else None,
                         'admission': item.admission,
+                        'personalized': item.personalized,
                         'active': item.active,
                         'sales_channels': item.sales_channels,
                         'description': str(item.description),
@@ -106,9 +114,26 @@ class JSONExporter(BaseExporter):
                                 'available_from': variation.available_from,
                                 'available_until': variation.available_until,
                                 'hide_without_voucher': variation.hide_without_voucher,
+                                'meta_data': variation.meta_data,
                             } for variation in item.variations.all()
                         ]
-                    } for item in self.event.items.select_related('tax_rule').prefetch_related('variations')
+                    } for item in self.event.items.select_related('tax_rule').prefetch_related(
+                        Prefetch(
+                            'meta_values',
+                            ItemMetaValue.objects.select_related('property'),
+                            to_attr='meta_values_cached'
+                        ),
+                        Prefetch(
+                            'variations',
+                            queryset=ItemVariation.objects.prefetch_related(
+                                Prefetch(
+                                    'meta_values',
+                                    ItemVariationMetaValue.objects.select_related('property'),
+                                    to_attr='meta_values_cached'
+                                ),
+                            ),
+                        ),
+                    )
                 ],
                 'questions': [
                     {

src/pretix/base/exporters/mail.py

@@ -36,7 +36,7 @@ from collections import OrderedDict
 
 from django import forms
 from django.dispatch import receiver
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import gettext_lazy as _, pgettext_lazy
 
 from pretix.base.models import OrderPosition
 
@@ -50,6 +50,8 @@ from ..signals import (
 class MailExporter(BaseExporter):
     identifier = 'mailaddrs'
     verbose_name = _('Email addresses (text file)')
+    category = pgettext_lazy('export_category', 'Order data')
+    description = _("Download a text file with all email addresses collected either from buyers or from ticket holders.")
 
     def render(self, form_data: dict):
         qs = Order.objects.filter(event__in=self.events, status__in=form_data['status']).prefetch_related('event')

src/pretix/base/exporters/orderlist.py

@@ -33,10 +33,8 @@
 # License for the specific language governing permissions and limitations under the License.
 
 from collections import OrderedDict
-from datetime import date, datetime, time
 from decimal import Decimal
 
-import dateutil
 import pytz
 from django import forms
 from django.db.models import (
@@ -46,8 +44,10 @@ from django.db.models import (
 from django.db.models.functions import Coalesce
 from django.dispatch import receiver
 from django.utils.functional import cached_property
-from django.utils.timezone import get_current_timezone, make_aware, now
-from django.utils.translation import gettext as _, gettext_lazy, pgettext
+from django.utils.timezone import get_current_timezone, now
+from django.utils.translation import (
+    gettext as _, gettext_lazy, pgettext, pgettext_lazy,
+)
 
 from pretix.base.models import (
     GiftCard, GiftCardTransaction, Invoice, InvoiceAddress, Order,
@@ -63,14 +63,24 @@ from ...helpers.iter import chunked_iterable
 from ..exporter import (
     ListExporter, MultiSheetListExporter, OrganizerLevelExportMixin,
 )
+from ..forms.widgets import SplitDateTimePickerWidget
 from ..signals import (
     register_data_exporters, register_multievent_data_exporters,
 )
+from ..timeframes import (
+    DateFrameField,
+    resolve_timeframe_to_datetime_start_inclusive_end_exclusive,
+)
 
 
 class OrderListExporter(MultiSheetListExporter):
     identifier = 'orderlist'
     verbose_name = gettext_lazy('Order data')
+    category = pgettext_lazy('export_category', 'Order data')
+    description = gettext_lazy('Download a spreadsheet of all orders. The spreadsheet will include three sheets, one '
+                               'with a line for every order, one with a line for every order position, and one with '
+                               'a line for every additional fee charged in an order.')
+    featured = True
 
     @cached_property
     def providers(self):
@@ -105,41 +115,25 @@ class OrderListExporter(MultiSheetListExporter):
                  initial=False,
                  required=False
              )),
-            ('date_from',
-             forms.DateField(
-                 label=_('Start date'),
-                 widget=forms.DateInput(attrs={'class': 'datepickerfield'}),
+            ('date_range',
+             DateFrameField(
+                 label=_('Date range'),
+                 include_future_frames=False,
                  required=False,
-                 help_text=_('Only include orders created on or after this date.')
+                 help_text=_('Only include orders created within this date range.')
              )),
-            ('date_to',
-             forms.DateField(
-                 label=_('End date'),
-                 widget=forms.DateInput(attrs={'class': 'datepickerfield'}),
+            ('event_date_range',
+             DateFrameField(
+                 label=_('Event date'),
+                 include_future_frames=True,
                  required=False,
-                 help_text=_('Only include orders created on or before this date.')
-             )),
-            ('event_date_from',
-             forms.DateField(
-                 label=_('Start event date'),
-                 widget=forms.DateInput(attrs={'class': 'datepickerfield'}),
-                 required=False,
-                 help_text=_('Only include orders including at least one ticket for a date on or after this date. '
-                             'Will also include other dates in case of mixed orders!')
-             )),
-            ('event_date_to',
-             forms.DateField(
-                 label=_('End event date'),
-                 widget=forms.DateInput(attrs={'class': 'datepickerfield'}),
-                 required=False,
-                 help_text=_('Only include orders including at least one ticket for a date on or before this date. '
+                 help_text=_('Only include orders including at least one ticket for a date in this range. '
                              'Will also include other dates in case of mixed orders!')
              )),
         ]
         d = OrderedDict(d)
         if not self.is_multievent and not self.event.has_subevents:
-            del d['event_date_from']
-            del d['event_date_to']
+            del d['event_date_range']
         return d
 
     def _get_all_payment_methods(self, qs):
@@ -182,45 +176,27 @@ class OrderListExporter(MultiSheetListExporter):
         annotations = {}
         filters = {}
 
-        if form_data.get('date_from'):
-            date_value = form_data.get('date_from')
-            if not isinstance(date_value, date):
-                date_value = dateutil.parser.parse(date_value).date()
-            datetime_value = make_aware(datetime.combine(date_value, time(0, 0, 0)), self.timezone)
+        if form_data.get('date_range'):
+            dt_start, dt_end = resolve_timeframe_to_datetime_start_inclusive_end_exclusive(now(), form_data['date_range'], self.timezone)
+            if dt_start:
+                filters[f'{rel}datetime__gte'] = dt_start
+            if dt_end:
+                filters[f'{rel}datetime__lt'] = dt_end
 
-            filters[f'{rel}datetime__gte'] = datetime_value
-
-        if form_data.get('date_to'):
-            date_value = form_data.get('date_to')
-            if not isinstance(date_value, date):
-                date_value = dateutil.parser.parse(date_value).date()
-            datetime_value = make_aware(datetime.combine(date_value, time(23, 59, 59, 999999)), self.timezone)
-
-            filters[f'{rel}datetime__lte'] = datetime_value
-
-        if form_data.get('event_date_from'):
-            date_value = form_data.get('event_date_from')
-            if not isinstance(date_value, date):
-                date_value = dateutil.parser.parse(date_value).date()
-            datetime_value = make_aware(datetime.combine(date_value, time(0, 0, 0)), self.timezone)
-
-            annotations['event_date_max'] = Case(
-                When(**{f'{rel}event__has_subevents': True}, then=Max(f'{rel}all_positions__subevent__date_from')),
-                default=F(f'{rel}event__date_from'),
-            )
-            filters['event_date_max__gte'] = datetime_value
-
-        if form_data.get('event_date_to'):
-            date_value = form_data.get('event_date_to')
-            if not isinstance(date_value, date):
-                date_value = dateutil.parser.parse(date_value).date()
-            datetime_value = make_aware(datetime.combine(date_value, time(23, 59, 59, 999999)), self.timezone)
-
-            annotations['event_date_min'] = Case(
-                When(**{f'{rel}event__has_subevents': True}, then=Min(f'{rel}all_positions__subevent__date_from')),
-                default=F(f'{rel}event__date_from'),
-            )
-            filters['event_date_min__lte'] = datetime_value
+        if form_data.get('event_date_range'):
+            dt_start, dt_end = resolve_timeframe_to_datetime_start_inclusive_end_exclusive(now(), form_data['event_date_range'], self.timezone)
+            if dt_start:
+                annotations['event_date_max'] = Case(
+                    When(**{f'{rel}event__has_subevents': True}, then=Max(f'{rel}all_positions__subevent__date_from')),
+                    default=F(f'{rel}event__date_from'),
+                )
+                filters['event_date_max__gte'] = dt_start
+            if dt_end:
+                annotations['event_date_min'] = Case(
+                    When(**{f'{rel}event__has_subevents': True}, then=Min(f'{rel}all_positions__subevent__date_from')),
+                    default=F(f'{rel}event__date_from'),
+                )
+                filters['event_date_min__lt'] = dt_end
 
         if filters:
             return qs.annotate(**annotations).filter(**filters)
@@ -303,6 +279,8 @@ class OrderListExporter(MultiSheetListExporter):
             for id, vn in payment_methods:
                 headers.append(_('Paid by {method}').format(method=vn))
 
+        # get meta_data labels from first cached event
+        headers += next(iter(self.event_object_cache.values())).meta_data.keys()
         yield headers
 
         full_fee_sum_cache = {
@@ -416,6 +394,7 @@ class OrderListExporter(MultiSheetListExporter):
                         payment_sum_cache.get((order.id, id), Decimal('0.00')) -
                         refund_sum_cache.get((order.id, id), Decimal('0.00'))
                     )
+            row += self.event_object_cache[order.event_id].meta_data.values()
             yield row
 
     def iterate_fees(self, form_data: dict):
@@ -465,6 +444,9 @@ class OrderListExporter(MultiSheetListExporter):
 
         headers.append(_('External customer ID'))
         headers.append(_('Payment providers'))
+
+        # get meta_data labels from first cached event
+        headers += next(iter(self.event_object_cache.values())).meta_data.keys()
         yield headers
 
         yield self.ProgressSetTotal(total=qs.count())
@@ -512,6 +494,7 @@ class OrderListExporter(MultiSheetListExporter):
                 str(self.providers.get(p, p)) for p in sorted(set((op.payment_providers or '').split(',')))
                 if p and p != 'free'
             ]))
+            row += self.event_object_cache[order.event_id].meta_data.values()
             yield row
 
     def iterate_positions(self, form_data: dict):
@@ -533,6 +516,7 @@ class OrderListExporter(MultiSheetListExporter):
             'order', 'order__invoice_address', 'order__customer', 'item', 'variation',
             'voucher', 'tax_rule'
         ).prefetch_related(
+            'subevent', 'subevent__meta_values',
             'answers', 'answers__question', 'answers__options'
         )
         if form_data['paid_only']:
@@ -624,6 +608,10 @@ class OrderListExporter(MultiSheetListExporter):
             _('Payment providers'),
         ]
 
+        # get meta_data labels from first cached event
+        meta_data_labels = next(iter(self.event_object_cache.values())).meta_data.keys()
+        if has_subevents:
+            headers += meta_data_labels
         yield headers
 
         all_ids = list(base_qs.order_by('order__datetime', 'positionid').values_list('pk', flat=True))
@@ -747,6 +735,12 @@ class OrderListExporter(MultiSheetListExporter):
                     str(self.providers.get(p, p)) for p in sorted(set((op.payment_providers or '').split(',')))
                     if p and p != 'free'
                 ]))
+
+                if has_subevents:
+                    if op.subevent:
+                        row += op.subevent.meta_data.values()
+                    else:
+                        row += [''] * len(meta_data_labels)
                 yield row
 
     def get_filename(self):
@@ -758,7 +752,10 @@ class OrderListExporter(MultiSheetListExporter):
 
 class PaymentListExporter(ListExporter):
     identifier = 'paymentlist'
-    verbose_name = gettext_lazy('Order payments and refunds')
+    verbose_name = gettext_lazy('Payments and refunds')
+    category = pgettext_lazy('export_category', 'Order data')
+    description = gettext_lazy('Download a spreadsheet of all payments or refunds of every order.')
+    featured = True
 
     @property
     def additional_form_fields(self):
@@ -837,6 +834,8 @@ class PaymentListExporter(ListExporter):
 class QuotaListExporter(ListExporter):
     identifier = 'quotalist'
     verbose_name = gettext_lazy('Quota availabilities')
+    category = pgettext_lazy('export_category', 'Product data')
+    description = gettext_lazy('Download a spreadsheet of all quotas including their current availability.')
 
     def iterate_list(self, form_data):
         has_subevents = self.event.has_subevents
@@ -890,21 +889,17 @@ class GiftcardTransactionListExporter(OrganizerLevelExportMixin, ListExporter):
     identifier = 'giftcardtransactionlist'
     verbose_name = gettext_lazy('Gift card transactions')
     organizer_required_permission = 'can_manage_gift_cards'
+    category = pgettext_lazy('export_category', 'Gift cards')
+    description = gettext_lazy('Download a spreadsheet of all gift card transactions.')
 
     @property
     def additional_form_fields(self):
         d = [
-            ('date_from',
-             forms.DateField(
-                 label=_('Start date'),
-                 widget=forms.DateInput(attrs={'class': 'datepickerfield'}),
-                 required=False,
-             )),
-            ('date_to',
-             forms.DateField(
-                 label=_('End date'),
-                 widget=forms.DateInput(attrs={'class': 'datepickerfield'}),
-                 required=False,
+            ('date_range',
+             DateFrameField(
+                 label=_('Date range'),
+                 include_future_frames=False,
+                 required=False
              )),
         ]
         d = OrderedDict(d)
@@ -915,22 +910,12 @@ class GiftcardTransactionListExporter(OrganizerLevelExportMixin, ListExporter):
             card__issuer=self.organizer,
         ).order_by('datetime').select_related('card', 'order', 'order__event')
 
-        if form_data.get('date_from'):
-            date_value = form_data.get('date_from')
-            if isinstance(date_value, str):
-                date_value = dateutil.parser.parse(date_value).date()
-            qs = qs.filter(
-                datetime__gte=make_aware(datetime.combine(date_value, time(0, 0, 0)), self.timezone)
-            )
-
-        if form_data.get('date_to'):
-            date_value = form_data.get('date_to')
-            if isinstance(date_value, str):
-                date_value = dateutil.parser.parse(date_value).date()
-
-            qs = qs.filter(
-                datetime__lte=make_aware(datetime.combine(date_value, time(23, 59, 59, 999999)), self.timezone)
-            )
+        if form_data.get('date_range'):
+            dt_start, dt_end = resolve_timeframe_to_datetime_start_inclusive_end_exclusive(now(), form_data['date_range'], self.timezone)
+            if dt_start:
+                qs = qs.filter(datetime__gte=dt_start)
+            if dt_end:
+                qs = qs.filter(datetime__lt=dt_end)
 
         headers = [
             _('Gift card code'),
@@ -960,6 +945,8 @@ class GiftcardTransactionListExporter(OrganizerLevelExportMixin, ListExporter):
 class GiftcardRedemptionListExporter(ListExporter):
     identifier = 'giftcardredemptionlist'
     verbose_name = gettext_lazy('Gift card redemptions')
+    category = pgettext_lazy('export_category', 'Order data')
+    description = gettext_lazy('Download a spreadsheet of all payments or refunds that involve gift cards.')
 
     def iterate_list(self, form_data):
         payments = OrderPayment.objects.filter(
@@ -1005,14 +992,18 @@ class GiftcardListExporter(OrganizerLevelExportMixin, ListExporter):
     identifier = 'giftcardlist'
     verbose_name = gettext_lazy('Gift cards')
     organizer_required_permission = 'can_manage_gift_cards'
+    category = pgettext_lazy('export_category', 'Gift cards')
+    description = gettext_lazy('Download a spreadsheet of all gift cards including their current value.')
 
     @property
     def additional_form_fields(self):
         return OrderedDict(
             [
-                ('date', forms.DateTimeField(
+                ('date', forms.SplitDateTimeField(
                     label=_('Show value at'),
-                    initial=now(),
+                    required=False,
+                    widget=SplitDateTimePickerWidget(),
+                    help_text=_('Defaults to the time of report.')
                 )),
                 ('testmode', forms.ChoiceField(
                     label=_('Test mode'),
@@ -1040,12 +1031,13 @@ class GiftcardListExporter(OrganizerLevelExportMixin, ListExporter):
         )
 
     def iterate_list(self, form_data):
+        d = form_data.get('date') or now()
         s = GiftCardTransaction.objects.filter(
             card=OuterRef('pk'),
-            datetime__lte=form_data['date']
+            datetime__lte=d
         ).order_by().values('card').annotate(s=Sum('value')).values('s')
         qs = self.organizer.issued_gift_cards.filter(
-            issuance__lte=form_data['date']
+            issuance__lte=d
         ).annotate(
             cached_value=Coalesce(Subquery(s), Decimal('0.00')),
         ).order_by('issuance').prefetch_related(
@@ -1060,11 +1052,11 @@ class GiftcardListExporter(OrganizerLevelExportMixin, ListExporter):
         if form_data.get('state') == 'empty':
             qs = qs.filter(cached_value=0)
         elif form_data.get('state') == 'valid_value':
-            qs = qs.exclude(cached_value=0).filter(Q(expires__isnull=True) | Q(expires__gte=form_data['date']))
+            qs = qs.exclude(cached_value=0).filter(Q(expires__isnull=True) | Q(expires__gte=d))
         elif form_data.get('state') == 'expired_value':
-            qs = qs.exclude(cached_value=0).filter(expires__lt=form_data['date'])
+            qs = qs.exclude(cached_value=0).filter(expires__lt=d)
         elif form_data.get('state') == 'expired':
-            qs = qs.filter(expires__lt=form_data['date'])
+            qs = qs.filter(expires__lt=d)
 
         headers = [
             _('Gift card code'),

src/pretix/base/exporters/waitinglist.py

@@ -39,6 +39,8 @@ from ..signals import (
 class WaitingListExporter(ListExporter):
     identifier = 'waitinglist'
     verbose_name = _('Waiting list')
+    category = pgettext_lazy('export_category', 'Waiting list')
+    description = _('Download a spread sheet with all your waiting list data.')
 
     # map selected status to label and queryset-filter
     status_filters = [

src/pretix/base/forms/questions.py

@@ -135,6 +135,10 @@ class NamePartsWidget(forms.MultiWidget):
             data.append(value.get(fname, ""))
         if '_legacy' in value and not data[-1]:
             data[-1] = value.get('_legacy', '')
+        elif not any(d for d in data) and '_scheme' in value:
+            scheme = PERSON_NAME_SCHEMES[value['_scheme']]
+            data[-1] = scheme['concatenation'](value).strip()
+
         return data
 
     def render(self, name: str, value, attrs=None, renderer=None) -> str:
@@ -527,7 +531,7 @@ class PortraitImageField(SizeValidationMixin, ExtValidationMixin, forms.FileFiel
                     code='aspect_ratio_not_3_by_4',
                 )
         except Exception as exc:
-            logger.exception('foo')
+            logger.exception('Could not parse image')
             # Pillow doesn't recognize it as an image.
             if isinstance(exc, ValidationError):
                 raise
@@ -571,7 +575,7 @@ class BaseQuestionsForm(forms.Form):
 
         add_fields = {}
 
-        if item.admission and event.settings.attendee_names_asked:
+        if item.ask_attendee_data and event.settings.attendee_names_asked:
             add_fields['attendee_name_parts'] = NamePartsFormField(
                 max_length=255,
                 required=event.settings.attendee_names_required and not self.all_optional,
@@ -580,7 +584,7 @@ class BaseQuestionsForm(forms.Form):
                 label=_('Attendee name'),
                 initial=(cartpos.attendee_name_parts if cartpos else orderpos.attendee_name_parts),
             )
-        if item.admission and event.settings.attendee_emails_asked:
+        if item.ask_attendee_data and event.settings.attendee_emails_asked:
             add_fields['attendee_email'] = forms.EmailField(
                 required=event.settings.attendee_emails_required and not self.all_optional,
                 label=_('Attendee email'),
@@ -591,7 +595,7 @@ class BaseQuestionsForm(forms.Form):
                     }
                 )
             )
-        if item.admission and event.settings.attendee_company_asked:
+        if item.ask_attendee_data and event.settings.attendee_company_asked:
             add_fields['company'] = forms.CharField(
                 required=event.settings.attendee_company_required and not self.all_optional,
                 label=_('Company'),
@@ -599,7 +603,7 @@ class BaseQuestionsForm(forms.Form):
                 initial=(cartpos.company if cartpos else orderpos.company),
             )
 
-        if item.admission and event.settings.attendee_addresses_asked:
+        if item.ask_attendee_data and event.settings.attendee_addresses_asked:
             add_fields['street'] = forms.CharField(
                 required=event.settings.attendee_addresses_required and not self.all_optional,
                 label=_('Address'),
@@ -915,6 +919,7 @@ class BaseQuestionsForm(forms.Form):
 
 class BaseInvoiceAddressForm(forms.ModelForm):
     vat_warning = False
+    address_validation = False
 
     class Meta:
         model = InvoiceAddress
@@ -1050,6 +1055,9 @@ class BaseInvoiceAddressForm(forms.ModelForm):
                 v.widget.attrs['autocomplete'] = 'section-invoice billing ' + v.widget.attrs.get('autocomplete', '')
 
     def clean(self):
+        from pretix.base.addressvalidation import \
+            validate_address  # local import to prevent impact on startup time
+
         data = self.cleaned_data
         if not data.get('is_business'):
             data['company'] = ''
@@ -1065,9 +1073,8 @@ class BaseInvoiceAddressForm(forms.ModelForm):
         if 'vat_id' in self.changed_data or not data.get('vat_id'):
             self.instance.vat_id_validated = False
 
-        if data.get('city') and data.get('country') and str(data['country']) in COUNTRIES_WITH_STATE_IN_ADDRESS:
-            if not data.get('state'):
-                self.add_error('state', _('This field is required.'))
+        if self.address_validation:
+            self.cleaned_data = data = validate_address(data, self.all_optional)
 
         self.instance.name_parts = data.get('name_parts')
 

src/pretix/base/invoice.py

@@ -23,6 +23,7 @@ import logging
 from collections import defaultdict
 from decimal import Decimal
 from io import BytesIO
+from itertools import groupby
 from typing import Tuple
 
 import bleach
@@ -241,6 +242,12 @@ class BaseReportlabInvoiceRenderer(BaseInvoiceRenderer):
         buffer.seek(0)
         return 'invoice.pdf', 'application/pdf', buffer.read()
 
+    def _clean_text(self, text, tags=None):
+        return bleach.clean(
+            text,
+            tags=tags or []
+        ).strip().replace('<br>', '<br />').replace('\n', '<br />\n')
+
 
 class ClassicInvoiceRenderer(BaseReportlabInvoiceRenderer):
     identifier = 'classic'
@@ -265,7 +272,7 @@ class ClassicInvoiceRenderer(BaseReportlabInvoiceRenderer):
     invoice_to_top = 52 * mm
 
     def _draw_invoice_to(self, canvas):
-        p = Paragraph(bleach.clean(self.invoice.address_invoice_to, tags=[]).strip().replace('\n', '<br />\n'),
+        p = Paragraph(self._clean_text(self.invoice.address_invoice_to),
                       style=self.stylesheet['Normal'])
         p.wrapOn(canvas, self.invoice_to_width, self.invoice_to_height)
         p_size = p.wrap(self.invoice_to_width, self.invoice_to_height)
@@ -278,7 +285,7 @@ class ClassicInvoiceRenderer(BaseReportlabInvoiceRenderer):
 
     def _draw_invoice_from(self, canvas):
         p = Paragraph(
-            bleach.clean(self.invoice.full_invoice_from, tags=[]).strip().replace('\n', '<br />\n'),
+            self._clean_text(self.invoice.full_invoice_from),
             style=self.stylesheet['InvoiceFrom']
         )
         p.wrapOn(canvas, self.invoice_from_width, self.invoice_from_height)
@@ -473,8 +480,8 @@ class ClassicInvoiceRenderer(BaseReportlabInvoiceRenderer):
         if self.invoice.custom_field:
             story.append(Paragraph(
                 '{}: {}'.format(
-                    bleach.clean(str(self.invoice.event.settings.invoice_address_custom_field), tags=[]).strip().replace('\n', '<br />\n'),
-                    bleach.clean(self.invoice.custom_field, tags=[]).strip().replace('\n', '<br />\n'),
+                    self._clean_text(str(self.invoice.event.settings.invoice_address_custom_field)),
+                    self._clean_text(self.invoice.custom_field),
                 ),
                 self.stylesheet['Normal']
             ))
@@ -482,7 +489,7 @@ class ClassicInvoiceRenderer(BaseReportlabInvoiceRenderer):
         if self.invoice.internal_reference:
             story.append(Paragraph(
                 pgettext('invoice', 'Customer reference: {reference}').format(
-                    reference=bleach.clean(self.invoice.internal_reference, tags=[]).strip().replace('\n', '<br />\n'),
+                    reference=self._clean_text(self.invoice.internal_reference),
                 ),
                 self.stylesheet['Normal']
             ))
@@ -490,20 +497,20 @@ class ClassicInvoiceRenderer(BaseReportlabInvoiceRenderer):
         if self.invoice.invoice_to_vat_id:
             story.append(Paragraph(
                 pgettext('invoice', 'Customer VAT ID') + ': ' +
-                bleach.clean(self.invoice.invoice_to_vat_id, tags=[]).replace("\n", "<br />\n"),
+                self._clean_text(self.invoice.invoice_to_vat_id),
                 self.stylesheet['Normal']
             ))
 
         if self.invoice.invoice_to_beneficiary:
             story.append(Paragraph(
                 pgettext('invoice', 'Beneficiary') + ':<br />' +
-                bleach.clean(self.invoice.invoice_to_beneficiary, tags=[]).replace("\n", "<br />\n"),
+                self._clean_text(self.invoice.invoice_to_beneficiary),
                 self.stylesheet['Normal']
             ))
 
         if self.invoice.introductory_text:
             story.append(Paragraph(
-                self.invoice.introductory_text,
+                self._clean_text(self.invoice.introductory_text, tags=['br']),
                 self.stylesheet['Normal']
             ))
             story.append(Spacer(1, 10 * mm))
@@ -554,31 +561,47 @@ class ClassicInvoiceRenderer(BaseReportlabInvoiceRenderer):
                 pgettext('invoice', 'Amount'),
             )]
 
+        def _group_key(line):
+            return (line.description, line.tax_rate, line.tax_name, line.net_value, line.gross_value, line.subevent_id,
+                    line.event_date_from, line.event_date_to)
+
         total = Decimal('0.00')
-        for line in self.invoice.lines.all():
+        for (description, tax_rate, tax_name, net_value, gross_value, *ignored), lines in groupby(self.invoice.lines.all(), key=_group_key):
+            lines = list(lines)
             if has_taxes:
+                if len(lines) > 1:
+                    single_price_line = pgettext('invoice', 'Single price: {net_price} net / {gross_price} gross').format(
+                        net_price=money_filter(net_value, self.invoice.event.currency),
+                        gross_price=money_filter(gross_value, self.invoice.event.currency),
+                    )
+                    description = description + "\n" + single_price_line
                 tdata.append((
                     Paragraph(
-                        bleach.clean(line.description, tags=['br']).strip().replace('<br>', '<br/>').replace('\n', '<br />\n'),
+                        self._clean_text(description, tags=['br']),
                         self.stylesheet['Normal']
                     ),
-                    "1",
-                    localize(line.tax_rate) + " %",
-                    money_filter(line.net_value, self.invoice.event.currency),
-                    money_filter(line.gross_value, self.invoice.event.currency),
+                    str(len(lines)),
+                    localize(tax_rate) + " %",
+                    money_filter(net_value * len(lines), self.invoice.event.currency),
+                    money_filter(gross_value * len(lines), self.invoice.event.currency),
                 ))
             else:
+                if len(lines) > 1:
+                    single_price_line = pgettext('invoice', 'Single price: {price}').format(
+                        price=money_filter(gross_value, self.invoice.event.currency),
+                    )
+                    description = description + "\n" + single_price_line
                 tdata.append((
                     Paragraph(
-                        bleach.clean(line.description, tags=['br']).strip().replace('<br>', '<br/>').replace('\n', '<br />\n'),
+                        self._clean_text(description, tags=['br']),
                         self.stylesheet['Normal']
                     ),
-                    "1",
-                    money_filter(line.gross_value, self.invoice.event.currency),
+                    str(len(lines)),
+                    money_filter(gross_value * len(lines), self.invoice.event.currency),
                 ))
-            taxvalue_map[line.tax_rate, line.tax_name] += line.tax_value
-            grossvalue_map[line.tax_rate, line.tax_name] += line.gross_value
-            total += line.gross_value
+            taxvalue_map[tax_rate, tax_name] += (gross_value - net_value) * len(lines)
+            grossvalue_map[tax_rate, tax_name] += gross_value * len(lines)
+            total += gross_value * len(lines)
 
         if has_taxes:
             tdata.append([
@@ -640,7 +663,7 @@ class ClassicInvoiceRenderer(BaseReportlabInvoiceRenderer):
 
         if self.invoice.additional_text:
             story.append(Paragraph(
-                self.invoice.additional_text,
+                self._clean_text(self.invoice.additional_text, tags=['br']),
                 self.stylesheet['Normal']
             ))
             story.append(Spacer(1, 5 * mm))
@@ -777,7 +800,7 @@ class Modern1Renderer(ClassicInvoiceRenderer):
         if not self.invoice.invoice_from:
             return
         c = [
-            bleach.clean(l, tags=[]).strip().replace('\n', '<br />\n')
+            self._clean_text(l)
             for l in self.invoice.address_invoice_from.strip().split('\n')
         ]
         p = Paragraph(' · '.join(c), style=self.stylesheet['Sender'])

src/pretix/base/management/commands/export.py

@@ -103,6 +103,8 @@ class Command(BaseCommand):
 
             with language(locale), override(timezone):
                 for receiver, response in signal_result:
+                    if not response:
+                        return None
                     ex = response(e, o, report_status)
                     if ex.identifier == options['export_provider']:
                         params = json.loads(options.get('parameters') or '{}')

src/pretix/base/management/commands/runperiodic.py

@@ -79,9 +79,9 @@ class Command(BaseCommand):
                 if settings.SENTRY_ENABLED:
                     from sentry_sdk import capture_exception
                     capture_exception(err)
-                    self.stdout.write(self.style.ERROR(f'ERROR runperiodic {str(err)}\n'))
+                    self.stdout.write(self.style.ERROR(f'ERROR {name}: {str(err)}\n'))
                 else:
-                    self.stdout.write(self.style.ERROR(f'ERROR runperiodic {str(err)}\n'))
+                    self.stdout.write(self.style.ERROR(f'ERROR {name}: {str(err)}\n'))
                     traceback.print_exc()
             else:
                 if options.get('verbosity') > 1:

src/pretix/base/middleware.py

@@ -30,7 +30,6 @@ from django.urls import get_script_prefix
 from django.utils import timezone, translation
 from django.utils.cache import patch_vary_headers
 from django.utils.deprecation import MiddlewareMixin
-from django.utils.translation import LANGUAGE_SESSION_KEY
 from django.utils.translation.trans_real import (
     check_for_language, get_supported_language_variant, language_code_re,
     parse_accept_lang_header,
@@ -128,12 +127,7 @@ def get_language_from_user_settings(request: HttpRequest) -> str:
             return lang_code
 
 
-def get_language_from_session_or_cookie(request: HttpRequest) -> str:
-    if hasattr(request, 'session'):
-        lang_code = request.session.get(LANGUAGE_SESSION_KEY)
-        if lang_code in _supported and lang_code is not None and check_for_language(lang_code):
-            return lang_code
-
+def get_language_from_cookie(request: HttpRequest) -> str:
     lang_code = request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME)
     try:
         return get_supported_language_variant(lang_code)
@@ -187,14 +181,14 @@ def get_language_from_request(request: HttpRequest) -> str:
         return (
             get_language_from_user_settings(request)
             or get_language_from_customer_settings(request)
-            or get_language_from_session_or_cookie(request)
+            or get_language_from_cookie(request)
             or get_language_from_browser(request)
             or get_language_from_event(request)
             or get_default_language()
         )
     else:
         return (
-            get_language_from_session_or_cookie(request)
+            get_language_from_cookie(request)
             or get_language_from_customer_settings(request)
             or get_language_from_user_settings(request)
             or get_language_from_browser(request)
@@ -224,6 +218,11 @@ def _merge_csp(a, b):
         if k not in a:
             a[k] = b[k]
 
+    for k, v in a.items():
+        if "'unsafe-inline'" in v:
+            # If we need unsafe-inline, drop any hashes or nonce as they will be ignored otherwise
+            a[k] = [i for i in v if not i.startswith("'nonce-") and not i.startswith("'sha-")]
+
 
 class SecurityMiddleware(MiddlewareMixin):
     CSP_EXEMPT = (
@@ -301,7 +300,7 @@ class SecurityMiddleware(MiddlewareMixin):
             resp['Content-Security-Policy'] = _render_csp(h).format(static=staticdomain, dynamic=dynamicdomain,
                                                                     media=mediadomain)
             for k, v in h.items():
-                h[k] = ' '.join(v).format(static=staticdomain, dynamic=dynamicdomain, media=mediadomain).split(' ')
+                h[k] = sorted(set(' '.join(v).format(static=staticdomain, dynamic=dynamicdomain, media=mediadomain).split(' ')))
             resp['Content-Security-Policy'] = _render_csp(h)
         elif 'Content-Security-Policy' in resp:
             del resp['Content-Security-Policy']

src/pretix/base/migrations/0223_auto_20221019_0950.py

@@ -1,23 +0,0 @@
-# Generated by Django 3.2.2 on 2022-10-19 09:50
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('pretixbase', '0222_alter_question_unique_together'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='transaction',
-            name='source_identifier',
-            field=models.CharField(db_index=True, max_length=190, null=True),
-        ),
-        migrations.AddField(
-            model_name='transaction',
-            name='source_type',
-            field=models.CharField(db_index=True, max_length=190, null=True),
-        ),
-    ]

src/pretix/base/migrations/0223_voucher_min_usages.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.12 on 2022-10-12 09:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pretixbase', '0222_alter_question_unique_together'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='voucher',
+            name='min_usages',
+            field=models.PositiveIntegerField(default=1),
+        ),
+    ]

src/pretix/base/migrations/0224_eventmetaproperty_filter_allowed.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-11-14 11:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pretixbase', '0223_voucher_min_usages'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='eventmetaproperty',
+            name='filter_allowed',
+            field=models.BooleanField(default=True),
+        ),
+    ]

src/pretix/base/migrations/0225_orderpayment_process_initiated.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-11-17 15:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pretixbase', '0224_eventmetaproperty_filter_allowed'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='orderpayment',
+            name='process_initiated',
+            field=models.BooleanField(null=True),
+        ),
+    ]

src/pretix/base/migrations/0226_itemvariationmetavalue.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.16 on 2022-12-09 10:06
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+import pretix.base.models.base
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pretixbase', '0225_orderpayment_process_initiated'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ItemVariationMetaValue',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False)),
+                ('value', models.TextField()),
+                ('property', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='variation_values', to='pretixbase.itemmetaproperty')),
+                ('variation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='meta_values', to='pretixbase.itemvariation')),
+            ],
+            options={
+                'unique_together': {('variation', 'property')},
+            },
+            bases=(models.Model, pretix.base.models.base.LoggingMixin),
+        ),
+    ]

src/pretix/base/migrations/0227_item_personalized.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.2.16 on 2022-12-21 08:59
+
+from django.db import migrations, models
+
+
+def item_set_personalized(apps, schema_editor):
+    # We cannot really know if a position was bundled or an add-on, but we can at least guess
+    Item = apps.get_model("pretixbase", "Item")
+    Item.objects.filter(admission=True).update(personalized=True)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('pretixbase', '0226_itemvariationmetavalue'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='item',
+            name='personalized',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.RunPython(
+            item_set_personalized,
+            migrations.RunPython.noop,
+        ),
+    ]

src/pretix/base/migrations/0228_scheduledeventexport_scheduledorganizerexport.py

@@ -0,0 +1,68 @@
+# Generated by Django 3.2.16 on 2023-01-18 11:57
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+import pretix.base.models.base
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pretixbase', '0227_item_personalized'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ScheduledOrganizerExport',
+            fields=[
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
+                ('export_identifier', models.CharField(max_length=190)),
+                ('export_form_data', models.JSONField(default=dict)),
+                ('locale', models.CharField(max_length=250)),
+                ('mail_additional_recipients', models.TextField()),
+                ('mail_additional_recipients_cc', models.TextField()),
+                ('mail_additional_recipients_bcc', models.TextField()),
+                ('mail_subject', models.CharField(max_length=250)),
+                ('mail_template', models.TextField()),
+                ('schedule_rrule', models.TextField(null=True)),
+                ('schedule_rrule_time', models.TimeField()),
+                ('schedule_next_run', models.DateTimeField(blank=True, null=True)),
+                ('error_counter', models.IntegerField(default=0)),
+                ('error_last_message', models.TextField(null=True)),
+                ('timezone', models.CharField(default='UTC', max_length=100)),
+                ('organizer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='scheduled_exports', to='pretixbase.organizer')),
+                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(models.Model, pretix.base.models.base.LoggingMixin),
+        ),
+        migrations.CreateModel(
+            name='ScheduledEventExport',
+            fields=[
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
+                ('export_identifier', models.CharField(max_length=190)),
+                ('export_form_data', models.JSONField(default=dict)),
+                ('locale', models.CharField(max_length=250)),
+                ('mail_additional_recipients', models.TextField()),
+                ('mail_additional_recipients_cc', models.TextField()),
+                ('mail_additional_recipients_bcc', models.TextField()),
+                ('mail_subject', models.CharField(max_length=250)),
+                ('mail_template', models.TextField()),
+                ('schedule_rrule', models.TextField(null=True)),
+                ('schedule_rrule_time', models.TimeField()),
+                ('schedule_next_run', models.DateTimeField(blank=True, null=True)),
+                ('error_counter', models.IntegerField(default=0)),
+                ('error_last_message', models.TextField(null=True)),
+                ('event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='scheduled_exports', to='pretixbase.event')),
+                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(models.Model, pretix.base.models.base.LoggingMixin),
+        ),
+    ]

src/pretix/base/models/__init__.py

@@ -30,12 +30,13 @@ from .event import (
     Event, Event_SettingsStore, EventLock, EventMetaProperty, EventMetaValue,
     SubEvent, SubEventMetaValue, generate_invite_token,
 )
+from .exports import ScheduledEventExport, ScheduledOrganizerExport
 from .giftcards import GiftCard, GiftCardAcceptance, GiftCardTransaction
 from .invoices import Invoice, InvoiceLine, invoice_filename
 from .items import (
     Item, ItemAddOn, ItemBundle, ItemCategory, ItemMetaProperty, ItemMetaValue,
-    ItemVariation, Question, QuestionOption, Quota, SubEventItem,
-    SubEventItemVariation, itempicture_upload_to,
+    ItemVariation, ItemVariationMetaValue, Question, QuestionOption, Quota,
+    SubEventItem, SubEventItemVariation, itempicture_upload_to,
 )
 from .log import LogEntry
 from .memberships import Membership, MembershipType

src/pretix/base/models/checkin.py

@@ -36,13 +36,17 @@ from datetime import timedelta
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.db import models
-from django.db.models import Exists, F, Max, OuterRef, Q, Subquery
+from django.db.models import (
+    Count, Exists, F, Max, OuterRef, Q, Subquery, Value, Window,
+)
+from django.db.models.expressions import RawSQL
 from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _, pgettext_lazy
 from django_scopes import ScopedManager, scopes_disabled
 
 from pretix.base.models import LoggedModel
 from pretix.base.models.fields import MultiStringField
+from pretix.helpers import PostgresWindowFrame
 
 
 class CheckinList(LoggedModel):
@@ -95,15 +99,18 @@ class CheckinList(LoggedModel):
     class Meta:
         ordering = ('subevent__date_from', 'name')
 
-    @property
-    def positions(self):
+    def positions_query(self, ignore_status=False):
         from . import Order, OrderPosition
 
-        qs = OrderPosition.objects.filter(
+        qs = OrderPosition.all.filter(
             order__event=self.event,
-            order__status__in=[Order.STATUS_PAID, Order.STATUS_PENDING] if self.include_pending else [
-                Order.STATUS_PAID],
         )
+        if not ignore_status:
+            qs = qs.filter(
+                canceled=False,
+                order__status__in=[Order.STATUS_PAID, Order.STATUS_PENDING] if self.include_pending else [Order.STATUS_PAID],
+            )
+
         if self.subevent_id:
             qs = qs.filter(subevent_id=self.subevent_id)
         if not self.all_products:
@@ -111,36 +118,90 @@ class CheckinList(LoggedModel):
         return qs
 
     @property
-    def positions_inside(self):
-        return self.positions.annotate(
-            last_entry=Subquery(
-                Checkin.objects.filter(
-                    position_id=OuterRef('pk'),
-                    list_id=self.pk,
-                    type=Checkin.TYPE_ENTRY,
-                ).order_by().values('position_id').annotate(
-                    m=Max('datetime')
-                ).values('m')
-            ),
-            last_exit=Subquery(
-                Checkin.objects.filter(
-                    position_id=OuterRef('pk'),
-                    list_id=self.pk,
-                    type=Checkin.TYPE_EXIT,
-                ).order_by().values('position_id').annotate(
-                    m=Max('datetime')
-                ).values('m')
-            ),
-        ).filter(
-            Q(last_entry__isnull=False)
-            & Q(
-                Q(last_exit__isnull=True) | Q(last_exit__lt=F('last_entry'))
+    def positions(self):
+        return self.positions_query(ignore_status=False)
+
+    @scopes_disabled()
+    def positions_inside_query(self, ignore_status=False, at_time=None):
+        if at_time is None:
+            c_q = []
+        else:
+            c_q = [Q(datetime__lt=at_time)]
+
+        if "postgresql" not in settings.DATABASES["default"]["ENGINE"]:
+            # Use a simple approach that works on all databases
+            qs = self.positions_query(ignore_status=ignore_status).annotate(
+                last_entry=Subquery(
+                    Checkin.objects.filter(
+                        *c_q,
+                        position_id=OuterRef('pk'),
+                        list_id=self.pk,
+                        type=Checkin.TYPE_ENTRY,
+                    ).order_by().values('position_id').annotate(
+                        m=Max('datetime')
+                    ).values('m')
+                ),
+                last_exit=Subquery(
+                    Checkin.objects.filter(
+                        *c_q,
+                        position_id=OuterRef('pk'),
+                        list_id=self.pk,
+                        type=Checkin.TYPE_EXIT,
+                    ).order_by().values('position_id').annotate(
+                        m=Max('datetime')
+                    ).values('m')
+                ),
+            ).filter(
+                Q(last_entry__isnull=False)
+                & Q(
+                    Q(last_exit__isnull=True) | Q(last_exit__lt=F('last_entry'))
+                )
+            )
+            return qs
+
+        # Use the PostgreSQL-specific query using Window functions, which is a lot faster.
+        # On a real-world example with ~100k tickets, of which ~17k are checked in, we observed
+        # a speed-up from 29s (old) to a few hundred milliseconds (new)!
+        # Why is this so much faster? The regular query get's PostgreSQL all busy with filtering
+        # the tickets both by their belonging the event and checkin status at the same time, while
+        # this query just iterates over all successful checkins on the list, and -- by the power
+        # of window functions -- asks "is this an entry that is followed by no exit?". Then we
+        # dedupliate by position and count it up.
+        cl = self
+        base_q, base_params = (
+            Checkin.all.filter(*c_q, successful=True, list=cl)
+            .annotate(
+                cnt_exists_after=Window(
+                    expression=Count("position_id", filter=Q(type=Value("exit"))),
+                    partition_by=[F("position_id"), F("list_id")],
+                    order_by=F("datetime").asc(),
+                    frame=PostgresWindowFrame(
+                        "ROWS", start="1 following", end="unbounded following"
+                    ),
+                )
+            )
+            .values("position_id", "type", "datetime", "cnt_exists_after")
+            .query.sql_with_params()
+        )
+        return self.positions_query(ignore_status=ignore_status).filter(
+            pk__in=RawSQL(
+                f"""
+                SELECT "position_id"
+                FROM ({str(base_q)}) s
+                WHERE "type" = %s AND "cnt_exists_after" = 0
+                GROUP BY "position_id"
+                """,
+                [*base_params, Checkin.TYPE_ENTRY]
             )
         )
 
+    @property
+    def positions_inside(self):
+        return self.positions_inside_query(None)
+
     @property
     def inside_count(self):
-        return self.positions_inside.count()
+        return self.positions_inside_query(None).count()
 
     @property
     @scopes_disabled()

src/pretix/base/models/customers.py

@@ -262,7 +262,7 @@ class Customer(LoggedModel):
         ) + '?id=' + self.identifier + '&token=' + token
         mail(
             self.email,
-            _('Activate your account at {organizer}').format(organizer=self.organizer.name),
+            self.organizer.settings.mail_subject_customer_registration,
             self.organizer.settings.mail_text_customer_registration,
             ctx,
             locale=self.locale,

src/pretix/base/models/discount.py

@@ -28,6 +28,7 @@ from typing import Dict, Optional, Tuple
 from django.core.exceptions import ValidationError
 from django.core.validators import MinValueValidator
 from django.db import models
+from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _, pgettext_lazy
 from django_scopes import ScopedManager
 
@@ -198,6 +199,14 @@ class Discount(LoggedModel):
             'subevent_mode': self.subevent_mode,
         })
 
+    def is_available_by_time(self, now_dt=None) -> bool:
+        now_dt = now_dt or now()
+        if self.available_from and self.available_from > now_dt:
+            return False
+        if self.available_until and self.available_until < now_dt:
+            return False
+        return True
+
     def _apply_min_value(self, positions, idx_group, result):
         if self.condition_min_value and sum(positions[idx][2] for idx in idx_group) < self.condition_min_value:
             return

src/pretix/base/models/event.py

@@ -36,7 +36,7 @@ import logging
 import os
 import string
 import uuid
-from collections import OrderedDict
+from collections import Counter, OrderedDict, defaultdict
 from datetime import datetime, time, timedelta
 from operator import attrgetter
 from urllib.parse import urljoin
@@ -340,64 +340,104 @@ class EventMixin:
             )
         )
 
-    @cached_property
+    @property
     def best_availability_state(self):
+        return self.best_availability[0]
+
+    @property
+    def best_availability_is_low(self):
+        """
+        Returns ``True`` if the availability of tickets in this event is lower than the percentage
+        given in setting ``low_availability_percentage``.
+        """
+        if not self.settings.low_availability_percentage:
+            return False
+        ba = self.best_availability
+        if ba[1] is None or not ba[2]:
+            return False
+
+        percentage = ba[1] / ba[2] * 100
+        return percentage < self.settings.low_availability_percentage
+
+    @cached_property
+    def best_availability(self):
+        """
+        Returns a 3-tuple of
+
+        - The availability state of this event (one of the ``Quota.AVAILABILITY_*`` constants)
+        - The number of tickets currently available (or ``None``)
+        - The number of tickets "originally" available (or ``None``)
+
+        This can only be called on objects obtained through a queryset that has been passed through ``.annotated()``.
+        """
         from .items import Quota
 
         if not hasattr(self, 'active_quotas'):
             raise TypeError("Call this only if you fetched the subevents via Event/SubEvent.annotated()")
-        items_available = set()
-        vars_available = set()
-        items_reserved = set()
-        vars_reserved = set()
-        items_gone = set()
-        vars_gone = set()
-        items_disabled = set()
-        vars_disabled = set()
 
         if hasattr(self, 'disabled_items'):  # SubEventItem
             items_disabled = set(self.disabled_items.split(","))
+        else:
+            items_disabled = set()
 
         if hasattr(self, 'disabled_vars'):  # SubEventItemVariation
             vars_disabled = set(self.disabled_vars.split(","))
+        else:
+            vars_disabled = set()
 
+        # Compute the availability of all quotas and build a item→quotas mapping with all non-disabled items
         r = getattr(self, '_quota_cache', {})
+        quotas_for_item = defaultdict(list)
+        quotas_for_variation = defaultdict(list)
         for q in self.active_quotas:
-            res = r[q] if q in r else q.availability(allow_cache=True)
+            if q not in r:
+                r[q] = q.availability(allow_cache=True)
 
-            if res[0] == Quota.AVAILABILITY_OK:
-                if q.active_items:
-                    items_available.update(q.active_items.split(","))
-                if q.active_variations:
-                    vars_available.update(q.active_variations.split(","))
-            elif res[0] == Quota.AVAILABILITY_RESERVED:
-                if q.active_items:
-                    items_reserved.update(q.active_items.split(","))
-                if q.active_variations:
-                    vars_available.update(q.active_variations.split(","))
-            elif res[0] < Quota.AVAILABILITY_RESERVED:
-                if q.active_items:
-                    items_gone.update(q.active_items.split(","))
-                if q.active_variations:
-                    vars_gone.update(q.active_variations.split(","))
+            if q.active_items:
+                for item_id in q.active_items.split(","):
+                    if item_id not in items_disabled:
+                        quotas_for_item[item_id].append(q)
+            if q.active_variations:
+                for var_id in q.active_variations.split(","):
+                    if var_id not in vars_disabled:
+                        quotas_for_variation[var_id].append(q)
 
-        items_available -= items_disabled
-        items_reserved -= items_disabled
-        items_gone -= items_disabled
-        vars_available -= vars_disabled
-        vars_reserved -= vars_disabled
-        vars_gone -= vars_disabled
+        if not self.active_quotas or (not quotas_for_item and not quotas_for_variation):
+            # No item is enabled for this event, treat the event as "unknown"
+            return None, None, None
 
-        if not self.active_quotas or (
-            not items_available and not items_reserved and not items_gone and not vars_gone and not vars_available and not vars_reserved
-        ):
-            return None
+        # We iterate over all items and variations and keep track of
+        # - `best_state_found` - the best availability state we have seen so far. If one item is available, the event is available!
+        # - `num_tickets_found` - the number of tickets currently available in total. We sum up all the items and variations, but keep
+        #   track of them per-quota in `quota_used_for_found_tickets` to make sure we don't count the same tickets twice if two or more
+        #   items share the same quota
+        # - `num_tickets_possible` - basically the same thing, just with the total size of quotas instead of their currently availability
+        #   since we need that for the percentage calculation
+        best_state_found = Quota.AVAILABILITY_GONE
+        num_tickets_found = 0
+        num_tickets_possible = 0
+        quota_used_for_found_tickets = Counter()
+        quota_used_for_possible_tickets = Counter()
+        for quota_list in list(quotas_for_item.values()) + list(quotas_for_variation.values()):
+            worst_state_for_ticket = min(r[q][0] for q in quota_list)
+            quotas_that_are_not_unlimited = [q for q in quota_list if q.size is not None]
+            if not quotas_that_are_not_unlimited:
+                # We found an unlimited ticket, no more need to do anything else
+                return Quota.AVAILABILITY_OK, None, None
 
-        if items_available - items_reserved - items_gone or vars_available - vars_reserved - vars_gone:
-            return Quota.AVAILABILITY_OK
-        if items_reserved - items_gone or vars_reserved - vars_gone:
-            return Quota.AVAILABILITY_RESERVED
-        return Quota.AVAILABILITY_GONE
+            if worst_state_for_ticket == Quota.AVAILABILITY_OK:
+                availability_of_this = min(max(0, r[q][1] - quota_used_for_found_tickets[q]) for q in quotas_that_are_not_unlimited)
+                num_tickets_found += availability_of_this
+                for q in quota_list:
+                    quota_used_for_found_tickets[q] += availability_of_this
+
+            possible_of_this = min(max(0, q.size - quota_used_for_possible_tickets[q]) for q in quotas_that_are_not_unlimited)
+            num_tickets_possible += possible_of_this
+            for q in quota_list:
+                quota_used_for_possible_tickets[q] += possible_of_this
+
+            best_state_found = max(best_state_found, worst_state_for_ticket)
+        return best_state_found, num_tickets_found, num_tickets_possible
 
     def free_seats(self, ignore_voucher=None, sales_channel='web', include_blocked=False):
         qs_annotated = self._seats(ignore_voucher=ignore_voucher)
@@ -590,6 +630,8 @@ class Event(EventMixin, LoggedModel):
         self.settings.event_list_type = 'calendar'
         self.settings.invoice_email_attachment = True
         self.settings.name_scheme = 'given_family'
+        self.settings.payment_banktransfer_invoice_immediately = True
+        self.settings.low_availability_percentage = 10
 
     @property
     def social_image(self):
@@ -631,6 +673,7 @@ class Event(EventMixin, LoggedModel):
             return super().presale_has_ended
 
     def delete_all_orders(self, really=False):
+        from .checkin import Checkin
         from .orders import (
             OrderFee, OrderPayment, OrderPosition, OrderRefund, Transaction,
         )
@@ -644,6 +687,7 @@ class Event(EventMixin, LoggedModel):
         OrderFee.objects.filter(order__event=self).delete()
         OrderRefund.objects.filter(order__event=self).delete()
         OrderPayment.objects.filter(order__event=self).delete()
+        Checkin.objects.filter(list__event=self).delete()
         self.orders.all().delete()
 
     def save(self, *args, **kwargs):
@@ -727,7 +771,7 @@ class Event(EventMixin, LoggedModel):
         from ..signals import event_copy_data
         from . import (
             Discount, Item, ItemAddOn, ItemBundle, ItemCategory, ItemMetaValue,
-            Question, Quota,
+            ItemVariationMetaValue, Question, Quota,
         )
 
         #  Note: avoid self.set_active_plugins(), it causes trouble e.g. for the badges plugin.
@@ -803,12 +847,18 @@ class Event(EventMixin, LoggedModel):
                 v.item = i
                 v.save(force_insert=True)
 
-        for imv in ItemMetaValue.objects.filter(item__event=other).prefetch_related('item', 'property'):
+        for imv in ItemMetaValue.objects.filter(item__event=other):
             imv.pk = None
-            imv.property = item_meta_properties_map[imv.property.pk]
+            imv.property = item_meta_properties_map[imv.property_id]
             imv.item = item_map[imv.item.pk]
             imv.save(force_insert=True)
 
+        for imv in ItemVariationMetaValue.objects.filter(variation__item__event=other):
+            imv.pk = None
+            imv.property = item_meta_properties_map[imv.property_id]
+            imv.variation = variation_map[imv.variation_id]
+            imv.save(force_insert=True)
+
         for ia in ItemAddOn.objects.filter(base_item__event=other).prefetch_related('base_item', 'addon_category'):
             ia.pk = None
             ia.base_item = item_map[ia.base_item.pk]
@@ -1579,6 +1629,11 @@ class EventMetaProperty(LoggedModel):
         verbose_name=_("Valid values"),
         help_text=_("If you keep this empty, any value is allowed. Otherwise, enter one possible value per line.")
     )
+    filter_allowed = models.BooleanField(
+        default=True, verbose_name=_("Can be used for filtering"),
+        help_text=_("This field will be shown to filter events or reports in the backend, and it can also be used "
+                    "for hidden filter parameters in the frontend (e.g. using the widget).")
+    )
 
     def full_clean(self, exclude=None, validate_unique=True):
         super().full_clean(exclude, validate_unique)

src/pretix/base/models/exports.py

@@ -0,0 +1,139 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020 Raphael Michel and contributors
+# Copyright (C) 2020-2021 rami.io GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+from datetime import datetime, timedelta
+
+import pytz
+from dateutil.rrule import rrulestr
+from django.conf import settings
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db import models
+from django.utils.timezone import make_aware, now
+from django.utils.translation import gettext_lazy as _
+
+from pretix.base.models import LoggedModel
+from pretix.base.validators import RRuleValidator, multimail_validate
+
+
+class AbstractScheduledExport(LoggedModel):
+    id = models.BigAutoField(primary_key=True)
+
+    export_identifier = models.CharField(
+        max_length=190,
+        verbose_name=_("Export"),
+    )
+    export_form_data = models.JSONField(
+        default=dict,
+        encoder=DjangoJSONEncoder,
+    )
+
+    owner = models.ForeignKey(
+        "pretixbase.User",
+        on_delete=models.PROTECT,
+    )
+    locale = models.CharField(
+        verbose_name=_('Language'),
+        max_length=250
+    )
+
+    mail_additional_recipients = models.TextField(
+        verbose_name=_('Additional recipients'),
+        null=False, blank=True, validators=[multimail_validate],
+        help_text=_("You can specify multiple recipients separated by commas.")
+    )
+    mail_additional_recipients_cc = models.TextField(
+        verbose_name=_('Additional recipients (Cc)'),
+        null=False, blank=True, validators=[multimail_validate],
+        help_text=_("You can specify multiple recipients separated by commas.")
+    )
+    mail_additional_recipients_bcc = models.TextField(
+        verbose_name=_('Additional recipients (Bcc)'),
+        null=False, blank=True, validators=[multimail_validate],
+        help_text=_("You can specify multiple recipients separated by commas.")
+    )
+    mail_subject = models.CharField(
+        verbose_name=_('Subject'),
+        max_length=250
+    )
+    mail_template = models.TextField(
+        verbose_name=_('Message'),
+    )
+
+    schedule_rrule = models.TextField(
+        null=True, blank=True, validators=[RRuleValidator()]
+    )
+    schedule_rrule_time = models.TimeField(
+        verbose_name=_("Requested start time"),
+        help_text=_("The actual start time might be delayed depending on system load."),
+    )
+    schedule_next_run = models.DateTimeField(null=True, blank=True)
+
+    error_counter = models.IntegerField(default=0)
+    error_last_message = models.TextField(null=True, blank=True)
+
+    class Meta:
+        abstract = True
+
+    def __str__(self):
+        return self.mail_subject
+
+    def compute_next_run(self):
+        tz = self.tz
+        r = rrulestr(self.schedule_rrule)
+
+        base_dt = now().astimezone(tz).replace(tzinfo=None)
+        if now().astimezone(tz).time() < self.schedule_rrule_time:
+            base_dt -= timedelta(days=1)
+
+        new_d = r.after(base_dt, inc=False)
+        if not new_d:
+            self.schedule_next_run = None
+            return
+
+        try:
+            self.schedule_next_run = make_aware(datetime.combine(new_d.date(), self.schedule_rrule_time), tz)
+        except pytz.exceptions.AmbiguousTimeError:
+            self.schedule_next_run = make_aware(datetime.combine(new_d.date(), self.schedule_rrule_time), tz, is_dst=False)
+        except pytz.exceptions.NonExistentTimeError:
+            self.schedule_next_run = make_aware(datetime.combine(new_d.date(), self.schedule_rrule_time) + timedelta(hours=1), tz)
+
+
+class ScheduledEventExport(AbstractScheduledExport):
+    event = models.ForeignKey(
+        "pretixbase.Event", on_delete=models.CASCADE, related_name="scheduled_exports"
+    )
+
+    @property
+    def tz(self):
+        return self.event.timezone
+
+
+class ScheduledOrganizerExport(AbstractScheduledExport):
+    organizer = models.ForeignKey(
+        "pretixbase.Organizer", on_delete=models.CASCADE, related_name="scheduled_exports"
+    )
+    timezone = models.CharField(max_length=100,
+                                default=settings.TIME_ZONE,
+                                verbose_name=_('Timezone'))
+
+    @property
+    def tz(self):
+        return pytz.timezone(self.timezone)

src/pretix/base/models/items.py

@@ -62,6 +62,7 @@ from pretix.base.models.base import LoggedModel
 from pretix.base.models.fields import MultiStringField
 from pretix.base.models.tax import TaxedPrice
 
+from ...helpers.images import ImageSizeValidator
 from .event import Event, SubEvent
 
 
@@ -310,6 +311,8 @@ class Item(LoggedModel):
     :type tax_rate: decimal.Decimal
     :param admission: ``True``, if this item allows persons to enter the event (as opposed to e.g. merchandise)
     :type admission: bool
+    :param personalized: ``True``, if attendee information should be collected for this ticket
+    :type personalized: bool
     :param picture: A product picture to be shown next to the product description
     :type picture: File
     :param available_from: The date this product goes on sale
@@ -396,8 +399,14 @@ class Item(LoggedModel):
     admission = models.BooleanField(
         verbose_name=_("Is an admission ticket"),
         help_text=_(
-            'Whether or not buying this product allows a person to enter '
-            'your event'
+            'Whether or not buying this product allows a person to enter your event'
+        ),
+        default=False
+    )
+    personalized = models.BooleanField(
+        verbose_name=_("Is a personalized ticket"),
+        help_text=_(
+            'Whether or not buying this product allows to enter attendee information'
         ),
         default=False
     )
@@ -421,7 +430,8 @@ class Item(LoggedModel):
     picture = models.ImageField(
         verbose_name=_("Product picture"),
         null=True, blank=True, max_length=255,
-        upload_to=itempicture_upload_to
+        upload_to=itempicture_upload_to,
+        validators=[ImageSizeValidator()]
     )
     available_from = models.DateTimeField(
         verbose_name=_("Available from"),
@@ -578,21 +588,22 @@ class Item(LoggedModel):
             return self.event.settings.show_quota_left
         return self.show_quota_left
 
+    @property
+    def ask_attendee_data(self):
+        return self.admission and self.personalized
+
     def tax(self, price=None, base_price_is='auto', currency=None, invoice_address=None, override_tax_rate=None, include_bundled=False):
         price = price if price is not None else self.default_price
 
-        if not self.tax_rule:
-            t = TaxedPrice(gross=price, net=price, tax=Decimal('0.00'),
-                           rate=Decimal('0.00'), name='')
-        else:
-            t = self.tax_rule.tax(price, base_price_is=base_price_is, invoice_address=invoice_address,
-                                  override_tax_rate=override_tax_rate, currency=currency or self.event.currency)
-
+        bundled_sum = Decimal('0.00')
+        bundled_sum_net = Decimal('0.00')
+        bundled_sum_tax = Decimal('0.00')
         if include_bundled:
             for b in self.bundles.all():
                 if b.designated_price and b.bundled_item.tax_rule_id != self.tax_rule_id:
                     if b.bundled_variation:
-                        bprice = b.bundled_variation.tax(b.designated_price * b.count, base_price_is='gross',
+                        bprice = b.bundled_variation.tax(b.designated_price * b.count,
+                                                         base_price_is='gross',
                                                          invoice_address=invoice_address,
                                                          currency=currency)
                     else:
@@ -600,17 +611,23 @@ class Item(LoggedModel):
                                                     invoice_address=invoice_address,
                                                     base_price_is='gross',
                                                     currency=currency)
-                    if not self.tax_rule:
-                        compare_price = TaxedPrice(gross=b.designated_price * b.count, net=b.designated_price * b.count,
-                                                   tax=Decimal('0.00'), rate=Decimal('0.00'), name='')
-                    else:
-                        compare_price = self.tax_rule.tax(b.designated_price * b.count,
-                                                          override_tax_rate=override_tax_rate,
-                                                          invoice_address=invoice_address,
-                                                          currency=currency)
-                    t.net += bprice.net - compare_price.net
-                    t.tax += bprice.tax - compare_price.tax
-                    t.name = "MIXED!"
+                    bundled_sum += bprice.gross
+                    bundled_sum_net += bprice.net
+                    bundled_sum_tax += bprice.tax
+
+        if not self.tax_rule:
+            t = TaxedPrice(gross=price - bundled_sum, net=price - bundled_sum, tax=Decimal('0.00'),
+                           rate=Decimal('0.00'), name='')
+        else:
+            t = self.tax_rule.tax(price, base_price_is=base_price_is, invoice_address=invoice_address,
+                                  override_tax_rate=override_tax_rate, currency=currency or self.event.currency,
+                                  subtract_from_gross=bundled_sum)
+
+        if bundled_sum:
+            t.name = "MIXED!"
+            t.gross += bundled_sum
+            t.net += bundled_sum_net
+            t.tax += bundled_sum_tax
 
         return t
 
@@ -1005,6 +1022,16 @@ class ItemVariation(models.Model):
             return False
         return True
 
+    @property
+    def meta_data(self):
+        data = self.item.meta_data
+        if hasattr(self, 'meta_values_cached'):
+            data.update({v.property.name: v.value for v in self.meta_values_cached})
+        else:
+            data.update({v.property.name: v.value for v in self.meta_values.select_related('property').all()})
+
+        return OrderedDict((k, v) for k, v in sorted(data.items(), key=lambda k: k[0]))
+
 
 class ItemAddOn(models.Model):
     """
@@ -1381,8 +1408,10 @@ class Question(LoggedModel):
         if self.type == Question.TYPE_CHOICE:
             if isinstance(answer, QuestionOption):
                 return answer
+            if not isinstance(answer, (int, str)):
+                raise ValidationError(_('Invalid input type.'))
             q = Q(identifier=answer)
-            if isinstance(answer, int) or answer.isdigit():
+            if isinstance(answer, int) or (isinstance(answer, str) and answer.isdigit()):
                 q |= Q(pk=answer)
             o = self.options.filter(q).first()
             if not o:
@@ -1782,8 +1811,21 @@ class ItemMetaValue(LoggedModel):
     class Meta:
         unique_together = ('item', 'property')
 
-    def delete(self, *args, **kwargs):
-        super().delete(*args, **kwargs)
 
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
+class ItemVariationMetaValue(LoggedModel):
+    """
+    A meta-data value assigned to an item variation, overriding the value on the item.
+
+    :param variation: The variation this metadata is valid for
+    :type variation: ItemVariation
+    :param property: The property this value belongs to
+    :type property: ItemMetaProperty
+    :param value: The actual value
+    :type value: str
+    """
+    variation = models.ForeignKey('ItemVariation', on_delete=models.CASCADE, related_name='meta_values')
+    property = models.ForeignKey('ItemMetaProperty', on_delete=models.CASCADE, related_name='variation_values')
+    value = models.TextField()
+
+    class Meta:
+        unique_together = ('variation', 'property')

src/pretix/base/models/orders.py

@@ -79,7 +79,9 @@ from pretix.base.services.locking import LOCK_TIMEOUT, NoLockManager
 from pretix.base.settings import PERSON_NAME_SCHEMES
 from pretix.base.signals import order_gracefully_delete
 
+from ...helpers import OF_SELF
 from ...helpers.countries import CachedCountries, FastCountryField
+from ...helpers.format import format_map
 from ._transactions import (
     _fail, _transactions_mark_order_clean, _transactions_mark_order_dirty,
 )
@@ -564,17 +566,30 @@ class Order(LockModel, LoggedModel):
     @cached_property
     def user_cancel_fee(self):
         fee = Decimal('0.00')
-        if self.event.settings.cancel_allow_user_paid_keep_fees:
-            fee += self.fees.filter(
-                fee_type__in=(OrderFee.FEE_TYPE_PAYMENT, OrderFee.FEE_TYPE_SHIPPING, OrderFee.FEE_TYPE_SERVICE,
-                              OrderFee.FEE_TYPE_CANCELLATION)
-            ).aggregate(
-                s=Sum('value')
-            )['s'] or 0
-        if self.event.settings.cancel_allow_user_paid_keep_percentage:
-            fee += self.event.settings.cancel_allow_user_paid_keep_percentage / Decimal('100.0') * (self.total - fee)
-        if self.event.settings.cancel_allow_user_paid_keep:
-            fee += self.event.settings.cancel_allow_user_paid_keep
+        if self.status == Order.STATUS_PAID:
+            if self.event.settings.cancel_allow_user_paid_keep_fees:
+                fee += self.fees.filter(
+                    fee_type__in=(OrderFee.FEE_TYPE_PAYMENT, OrderFee.FEE_TYPE_SHIPPING, OrderFee.FEE_TYPE_SERVICE,
+                                  OrderFee.FEE_TYPE_CANCELLATION)
+                ).aggregate(
+                    s=Sum('value')
+                )['s'] or 0
+            if self.event.settings.cancel_allow_user_paid_keep_percentage:
+                fee += self.event.settings.cancel_allow_user_paid_keep_percentage / Decimal('100.0') * (self.total - fee)
+            if self.event.settings.cancel_allow_user_paid_keep:
+                fee += self.event.settings.cancel_allow_user_paid_keep
+        else:
+            if self.event.settings.cancel_allow_user_unpaid_keep_fees:
+                fee += self.fees.filter(
+                    fee_type__in=(OrderFee.FEE_TYPE_PAYMENT, OrderFee.FEE_TYPE_SHIPPING, OrderFee.FEE_TYPE_SERVICE,
+                                  OrderFee.FEE_TYPE_CANCELLATION)
+                ).aggregate(
+                    s=Sum('value')
+                )['s'] or 0
+            if self.event.settings.cancel_allow_user_unpaid_keep_percentage:
+                fee += self.event.settings.cancel_allow_user_unpaid_keep_percentage / Decimal('100.0') * (self.total - fee)
+            if self.event.settings.cancel_allow_user_unpaid_keep:
+                fee += self.event.settings.cancel_allow_user_unpaid_keep
         return round_decimal(min(fee, self.total), self.event.currency)
 
     @property
@@ -642,10 +657,12 @@ class Order(LockModel, LoggedModel):
         if self.user_cancel_deadline and now() > self.user_cancel_deadline:
             return False
 
-        if self.status == Order.STATUS_PAID or self.payment_refund_sum > Decimal('0.00'):
+        if self.status == Order.STATUS_PAID:
             if self.total == Decimal('0.00'):
                 return self.event.settings.cancel_allow_user
             return self.event.settings.cancel_allow_user_paid
+        elif self.payment_refund_sum > Decimal('0.00'):
+            return False
         elif self.status == Order.STATUS_PENDING:
             return self.event.settings.cancel_allow_user
         return False
@@ -792,7 +809,7 @@ class Order(LockModel, LoggedModel):
             return True
         ask_names = self.event.settings.get('attendee_names_asked', as_type=bool)
         for cp in positions:
-            if (cp.item.admission and ask_names) or cp.item.questions.all():
+            if (cp.item.ask_attendee_data and ask_names) or cp.item.questions.all():
                 return True
 
         return False  # nothing there to modify
@@ -981,7 +998,7 @@ class Order(LockModel, LoggedModel):
                          position and the attendee email will be used if available.
         """
         from pretix.base.services.mail import (
-            SendMailException, TolerantDict, mail, render_mail,
+            SendMailException, mail, render_mail,
         )
 
         if not self.email and not (position and position.attendee_email):
@@ -997,7 +1014,7 @@ class Order(LockModel, LoggedModel):
 
             try:
                 email_content = render_mail(template, context)
-                subject = str(subject).format_map(TolerantDict(context))
+                subject = format_map(subject, context)
                 mail(
                     recipient, subject, template, context,
                     self.event, self.locale, self, headers=headers, sender=sender,
@@ -1027,7 +1044,7 @@ class Order(LockModel, LoggedModel):
         with language(self.locale, self.event.settings.region):
             email_template = self.event.settings.mail_text_resend_link
             email_context = get_email_context(event=self.event, order=self)
-            email_subject = _('Your order: %(code)s') % {'code': self.code}
+            email_subject = self.event.settings.mail_subject_resend_link
             self.send_mail(
                 email_subject, email_template, email_context,
                 'pretix.event.order.email.resend', user=user, auth=auth,
@@ -1041,13 +1058,10 @@ class Order(LockModel, LoggedModel):
                 continue
             yield op
 
-    def create_transactions(self, *, source=None, is_new=False, positions=None, fees=None,
-                            dt_now=None, migrated=False, _backfill_before_cancellation=False, save=True):
+    def create_transactions(self, is_new=False, positions=None, fees=None, dt_now=None, migrated=False,
+                            _backfill_before_cancellation=False, save=True):
         dt_now = dt_now or now()
 
-        if source is not None and (not isinstance(source, tuple) or len(source) != 2 or not all(isinstance(a, str) or a is None for a in source)):
-            return ValueError("source needs to be a 2-tuple of (source_type(str), source_identifier(str))")
-
         # Count the transactions we already have
         current_transaction_count = Counter()
         if not is_new:
@@ -1092,8 +1106,6 @@ class Order(LockModel, LoggedModel):
                     tax_value=taxvalue,
                     fee_type=feetype,
                     internal_type=internaltype,
-                    source_type=source[0] if source else None,
-                    source_identifier=source[1] if source else None,
                 ))
         create.sort(key=lambda t: (0 if t.count < 0 else 1, t.positionid or 0))
         if save:
@@ -1499,6 +1511,9 @@ class OrderPayment(models.Model):
     :type info: str
     :param fee: The ``OrderFee`` object used to track the fee for this order.
     :type fee: pretix.base.models.OrderFee
+    :param process_initiated: Only for internal use inside pretix.presale to check which payments have started
+                              the execution process.
+    :type process_initiated: bool
     """
     PAYMENT_STATE_CREATED = 'created'
     PAYMENT_STATE_PENDING = 'pending'
@@ -1549,6 +1564,9 @@ class OrderPayment(models.Model):
         null=True, blank=True, related_name='payments', on_delete=models.SET_NULL
     )
     migrated = models.BooleanField(default=False)
+    process_initiated = models.BooleanField(
+        null=True  # null = created before this field was introduced
+    )
 
     objects = ScopedManager(organizer='order__event__organizer')
 
@@ -1578,7 +1596,7 @@ class OrderPayment(models.Model):
         return self.order.event.get_payment_providers(cached=True).get(self.provider)
 
     @transaction.atomic()
-    def _mark_paid_inner(self, force, count_waitinglist, user, auth, ignore_date=False, overpaid=False, source=None):
+    def _mark_paid_inner(self, force, count_waitinglist, user, auth, ignore_date=False, overpaid=False):
         from pretix.base.signals import order_paid
         can_be_paid = self.order._can_be_paid(count_waitinglist=count_waitinglist, ignore_date=ignore_date, force=force)
         if can_be_paid is not True:
@@ -1601,9 +1619,7 @@ class OrderPayment(models.Model):
             self.order.log_action('pretix.event.order.overpaid', {}, user=user, auth=auth)
         order_paid.send(self.order.event, order=self.order)
         if status_change:
-            self.order.create_transactions(
-                source=source or ('pretix.payment', None),
-            )
+            self.order.create_transactions()
 
     def fail(self, info=None, user=None, auth=None, log_data=None):
         """
@@ -1613,13 +1629,13 @@ class OrderPayment(models.Model):
         been marked as paid.
         """
         with transaction.atomic():
-            locked_instance = OrderPayment.objects.select_for_update().get(pk=self.pk)
+            locked_instance = OrderPayment.objects.select_for_update(of=OF_SELF).get(pk=self.pk)
             if locked_instance.state not in (OrderPayment.PAYMENT_STATE_CREATED, OrderPayment.PAYMENT_STATE_PENDING):
                 # Race condition detected, this payment is already confirmed
                 logger.info('Failed payment {} but ignored due to likely race condition.'.format(
                     self.full_id,
                 ))
-                return
+                return False
 
             if isinstance(info, str):
                 locked_instance.info = info
@@ -1635,9 +1651,10 @@ class OrderPayment(models.Model):
             'info': info,
             'data': log_data,
         }, user=user, auth=auth)
+        return True
 
     def confirm(self, count_waitinglist=True, send_mail=True, force=False, user=None, auth=None, mail_text='',
-                ignore_date=False, lock=True, payment_date=None, source=None):
+                ignore_date=False, lock=True, payment_date=None, generate_invoice=True):
         """
         Marks the payment as complete. If possible, this also marks the order as paid if no further
         payment is required
@@ -1658,7 +1675,7 @@ class OrderPayment(models.Model):
         :raises Quota.QuotaExceededException: if the quota is exceeded and ``force`` is ``False``
         """
         with transaction.atomic():
-            locked_instance = OrderPayment.objects.select_for_update().get(pk=self.pk)
+            locked_instance = OrderPayment.objects.select_for_update(of=OF_SELF).get(pk=self.pk)
             if locked_instance.state == self.PAYMENT_STATE_CONFIRMED:
                 # Race condition detected, this payment is already confirmed
                 logger.info('Confirmed payment {} but ignored due to likely race condition.'.format(
@@ -1701,10 +1718,10 @@ class OrderPayment(models.Model):
             return
 
         self._mark_order_paid(count_waitinglist, send_mail, force, user, auth, mail_text, ignore_date, lock, payment_sum - refund_sum,
-                              source)
+                              generate_invoice)
 
     def _mark_order_paid(self, count_waitinglist=True, send_mail=True, force=False, user=None, auth=None, mail_text='',
-                         ignore_date=False, lock=True, payment_refund_sum=0, source=None):
+                         ignore_date=False, lock=True, payment_refund_sum=0, allow_generate_invoice=True):
         from pretix.base.services.invoices import (
             generate_invoice, invoice_qualified,
         )
@@ -1718,10 +1735,10 @@ class OrderPayment(models.Model):
 
         with lockfn():
             self._mark_paid_inner(force, count_waitinglist, user, auth, overpaid=payment_refund_sum > self.order.total,
-                                  ignore_date=ignore_date, source=source)
+                                  ignore_date=ignore_date)
 
         invoice = None
-        if invoice_qualified(self.order):
+        if invoice_qualified(self.order) and allow_generate_invoice:
             invoices = self.order.invoices.filter(is_cancellation=False).count()
             cancellations = self.order.invoices.filter(is_cancellation=True).count()
             gen_invoice = (
@@ -1746,8 +1763,8 @@ class OrderPayment(models.Model):
 
         with language(self.order.locale, self.order.event.settings.region):
             email_template = self.order.event.settings.mail_text_order_paid_attendee
+            email_subject = self.order.event.settings.mail_subject_order_paid_attendee
             email_context = get_email_context(event=self.order.event, order=self.order, position=position)
-            email_subject = _('Event registration confirmed: %(code)s') % {'code': self.order.code}
             try:
                 position.send_mail(
                     email_subject, email_template, email_context,
@@ -1764,8 +1781,8 @@ class OrderPayment(models.Model):
 
         with language(self.order.locale, self.order.event.settings.region):
             email_template = self.order.event.settings.mail_text_order_paid
+            email_subject = self.order.event.settings.mail_subject_order_paid
             email_context = get_email_context(event=self.order.event, order=self.order, payment_info=mail_text)
-            email_subject = _('Payment received for your order: %(code)s') % {'code': self.order.code}
             try:
                 self.order.send_mail(
                     email_subject, email_template, email_context,
@@ -2400,7 +2417,7 @@ class OrderPosition(AbstractPosition):
         :param attach_ical: Attach relevant ICS files
         """
         from pretix.base.services.mail import (
-            SendMailException, TolerantDict, mail, render_mail,
+            SendMailException, mail, render_mail,
         )
 
         if not self.attendee_email:
@@ -2413,7 +2430,7 @@ class OrderPosition(AbstractPosition):
             recipient = self.attendee_email
             try:
                 email_content = render_mail(template, context)
-                subject = str(subject).format_map(TolerantDict(context))
+                subject = format_map(subject, context)
                 mail(
                     recipient, subject, template, context,
                     self.event, self.order.locale, order=self.order, headers=headers, sender=sender,
@@ -2445,7 +2462,7 @@ class OrderPosition(AbstractPosition):
         with language(self.order.locale, self.order.event.settings.region):
             email_template = self.event.settings.mail_text_resend_link
             email_context = get_email_context(event=self.order.event, order=self.order, position=self)
-            email_subject = _('Your event registration: %(code)s') % {'code': self.order.code}
+            email_subject = self.event.settings.mail_subject_resend_link
             self.send_mail(
                 email_subject, email_template, email_context,
                 'pretix.event.order.email.resend', user=user, auth=auth,
@@ -2491,8 +2508,6 @@ class Transaction(models.Model):
 
     :param id: ID of the transaction
     :param order: Order the transaction belongs to
-    :param source_type: Functionality that caused the transaction to be created, usually the name of a module or plugin
-    :param source_identifier: Identifier of the entity that caused the transaction to be created, as defined by the module or plugin noted in ``source_type``.
     :param datetime: Date and time of the transaction
     :param migrated: Whether this object was reconstructed because the order was created before transactions where introduced
     :param positionid: Affected Position ID, in case this transaction represents a change in an order position
@@ -2515,12 +2530,6 @@ class Transaction(models.Model):
         related_name='transactions',
         on_delete=models.PROTECT
     )
-    source_type = models.CharField(
-        max_length=190, db_index=True, null=True, blank=True
-    )
-    source_identifier = models.CharField(
-        max_length=190, db_index=True, null=True, blank=True
-    )
     created = models.DateTimeField(
         auto_now_add=True,
         db_index=True,
@@ -2681,7 +2690,7 @@ class CartPosition(AbstractPosition):
         category_key = (self.item.category.position, self.item.category.id) if self.item.category_id is not None else (0, 0)
         item_key = self.item.position, self.item_id
         variation_key = (self.variation.position, self.variation.id) if self.variation_id is not None else (0, 0)
-        line_key = (self.price, (self.voucher_id or 0), (self.seat.sorting_rank if self.seat_id else None), self.pk)
+        line_key = (self.price, (self.voucher_id or 0), (self.seat.sorting_rank if self.seat_id else 0), self.pk)
         sort_key = subevent_key + category_key + item_key + variation_key + line_key
 
         if self.addon_to_id:
@@ -2738,6 +2747,7 @@ class CartPosition(AbstractPosition):
             tax_rule=self.item.tax_rule,
             invoice_address=invoice_address,
             bundled_sum=sum([b.price_after_voucher for b in bundled_positions]),
+            is_bundled=self.is_bundled,
         )
         if line_price.gross != self.line_price_gross or line_price.rate != self.tax_rate:
             self.line_price_gross = line_price.gross

src/pretix/base/models/tax.py

@@ -23,6 +23,7 @@ import json
 from decimal import Decimal
 
 from django.core.exceptions import ValidationError
+from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
 from django.utils.formats import localize
 from django.utils.translation import gettext_lazy as _, pgettext
@@ -149,7 +150,15 @@ class TaxRule(LoggedModel):
     rate = models.DecimalField(
         max_digits=10,
         decimal_places=2,
-        verbose_name=_("Tax rate")
+        validators=[
+            MaxValueValidator(
+                limit_value=Decimal("100.00"),
+            ),
+            MinValueValidator(
+                limit_value=Decimal("0.00"),
+            ),
+        ],
+        verbose_name=_("Tax rate"),
     )
     price_includes_tax = models.BooleanField(
         verbose_name=_("The configured product prices include the tax amount"),

src/pretix/base/models/vouchers.py

@@ -137,6 +137,8 @@ class Voucher(LoggedModel):
     :type max_usages: int
     :param redeemed: The number of times this voucher already has been redeemed
     :type redeemed: int
+    :param min_usages: The minimum number of times this voucher must be redeemed
+    :type min_usages: int
     :param valid_until: The expiration date of this voucher (optional)
     :type valid_until: datetime
     :param block_quota: If set to true, this voucher will reserve quota for its holder
@@ -199,6 +201,14 @@ class Voucher(LoggedModel):
         verbose_name=_("Redeemed"),
         default=0
     )
+    min_usages = models.PositiveIntegerField(
+        verbose_name=_("Minimum usages"),
+        help_text=_("If set to more than one, the voucher must be redeemed for this many products when it is used for "
+                    "the first time. On later usages, it can also be used for lower numbers of products. Note that "
+                    "this means that the total number of usages in some cases can be lower than this limit, e.g. in "
+                    "case of cancellations."),
+        default=1
+    )
     budget = models.DecimalField(
         verbose_name=_("Maximum discount budget"),
         help_text=_("This is the maximum monetary amount that will be discounted using this voucher across all usages. "
@@ -350,6 +360,10 @@ class Voucher(LoggedModel):
                     'redeemed': redeemed
                 }
             )
+        if data.get('max_usages', 1) < data.get('min_usages', 1):
+            raise ValidationError(
+                _('The maximum number of usages may not be lower than the minimum number of usages.'),
+            )
 
     @staticmethod
     def clean_subevent(data, event):
@@ -464,7 +478,7 @@ class Voucher(LoggedModel):
         if quota:
             raise ValidationError(_('You need to choose a specific product if you select a seat.'))
 
-        if data.get('max_usages', 1) > 1:
+        if data.get('max_usages', 1) > 1 or data.get('min_usages', 1) > 1:
             raise ValidationError(_('Seat-specific vouchers can only be used once.'))
 
         if item and seat.product != item:
@@ -567,6 +581,10 @@ class Voucher(LoggedModel):
         else:
             return bool(subevent.seating_plan) if subevent else self.event.seating_plan
 
+    @property
+    def min_usages_remaining(self):
+        return max(1, self.min_usages - self.redeemed)
+
     @classmethod
     def annotate_budget_used_orders(cls, qs):
         opq = OrderPosition.objects.filter(

src/pretix/base/models/waitinglist.py

@@ -20,6 +20,7 @@
 # <https://www.gnu.org/licenses/>.
 #
 from datetime import timedelta
+from typing import Any, Dict, Union
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import models, transaction
@@ -27,14 +28,16 @@ from django.db.models import F, Q, Sum
 from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _, pgettext_lazy
 from django_scopes import ScopedManager
+from i18nfield.strings import LazyI18nString
 from phonenumber_field.modelfields import PhoneNumberField
 
 from pretix.base.email import get_email_context
 from pretix.base.i18n import language
-from pretix.base.models import Voucher
-from pretix.base.services.mail import mail
+from pretix.base.models import User, Voucher
+from pretix.base.services.mail import SendMailException, mail, render_mail
 from pretix.base.settings import PERSON_NAME_SCHEMES
 
+from ...helpers.format import format_map
 from .base import LoggedModel
 from .event import Event, SubEvent
 from .items import Item, ItemVariation
@@ -213,15 +216,74 @@ class WaitingListEntry(LoggedModel):
             self.voucher = v
             self.save()
 
+        self.send_mail(
+            self.event.settings.mail_subject_waiting_list,
+            self.event.settings.mail_text_waiting_list,
+            get_email_context(
+                event=self.event,
+                waiting_list_entry=self,
+                waiting_list_voucher=v,
+                event_or_subevent=self.subevent or self.event,
+            ),
+            user=user,
+            auth=auth,
+        )
+
+    def send_mail(self, subject: Union[str, LazyI18nString], template: Union[str, LazyI18nString],
+                  context: Dict[str, Any]=None, log_entry_type: str='pretix.waitinglist.email.sent',
+                  user: User=None, headers: dict=None, sender: str=None, auth=None, auto_email=True,
+                  attach_other_files: list=None, attach_cached_files: list=None):
+        """
+        Sends an email to the entry's contact address.
+
+        * Call ``pretix.base.services.mail.mail`` with useful values for the ``event``, ``locale``, and ``recipient``
+          parameters.
+
+        * Create a ``LogEntry`` with the email contents.
+
+        :param subject: Subject of the email
+        :param template: LazyI18nString or template filename, see ``pretix.base.services.mail.mail`` for more details
+        :param context: Dictionary to use for rendering the template
+        :param log_entry_type: Key to be used for the log entry
+        :param user: Administrative user who triggered this mail to be sent
+        :param headers: Dictionary with additional mail headers
+        :param sender: Custom email sender.
+        """
+        if not self.email:
+            return
+
+        for k, v in self.event.meta_data.items():
+            context['meta_' + k] = v
+
         with language(self.locale, self.event.settings.region):
-            mail(
-                self.email,
-                _('You have been selected from the waitinglist for {event}').format(event=str(self.event)),
-                self.event.settings.mail_text_waiting_list,
-                get_email_context(event=self.event, waiting_list_entry=self),
-                self.event,
-                locale=self.locale
-            )
+            recipient = self.email
+
+            try:
+                email_content = render_mail(template, context)
+                subject = format_map(subject, context)
+                mail(
+                    recipient, subject, template, context,
+                    self.event,
+                    self.locale,
+                    headers=headers,
+                    sender=sender,
+                    auto_email=auto_email,
+                    attach_other_files=attach_other_files,
+                    attach_cached_files=attach_cached_files,
+                )
+            except SendMailException:
+                raise
+            else:
+                self.log_action(
+                    log_entry_type,
+                    user=user,
+                    auth=auth,
+                    data={
+                        'subject': subject,
+                        'message': email_content,
+                        'recipient': recipient,
+                    }
+                )
 
     @staticmethod
     def clean_itemvar(event, item, variation):

src/pretix/base/payment.py

@@ -63,14 +63,15 @@ from pretix.base.models import (
     OrderRefund, Quota,
 )
 from pretix.base.reldate import RelativeDateField, RelativeDateWrapper
-from pretix.base.services.cart import get_fees
 from pretix.base.settings import SettingsSandbox
 from pretix.base.signals import register_payment_providers
 from pretix.base.templatetags.money import money_filter
 from pretix.base.templatetags.rich_text import rich_text
+from pretix.helpers import OF_SELF
 from pretix.helpers.countries import CachedCountries
+from pretix.helpers.format import format_map
 from pretix.helpers.money import DecimalTextInput
-from pretix.multidomain.urlreverse import build_absolute_uri, eventreverse
+from pretix.multidomain.urlreverse import build_absolute_uri
 from pretix.presale.views import get_cart, get_cart_total
 from pretix.presale.views.cart import cart_session, get_or_create_cart_id
 
@@ -138,6 +139,50 @@ class BasePaymentProvider:
         """
         return self.settings.get('_enabled', as_type=bool)
 
+    @property
+    def multi_use_supported(self) -> bool:
+        """
+        Returns whether or whether not this payment provider supports being used multiple times in the same
+        checkout, or in addition to a different payment provider. This is usually only useful for payment providers
+        that represent gift cards, i.e. payment methods with an upper limit per payment instrument that can usually
+        be combined with other instruments.
+
+        If you set this property to ``True``, the behavior of how pretix interacts with your payment provider changes
+        and you will need to respect the following rules:
+
+        - ``payment_form_render`` must not depend on session state, it must always allow a user to add a new payment.
+           Editing a payment is not possible, but pretix will give users an option to delete it.
+
+        - Returning ``True`` from ``checkout_prepare`` is no longer enough. Instead, you must *also* call
+          ``pretix.base.services.cart.add_payment_to_cart(request, provider, min_value, max_value, info_data)``
+          to add the payment to the session. You are still allowed to do a redirect from ``checkout_prepare`` and then
+          call this function upon return.
+
+        - Unlike in the general case, when ``checkout_prepare`` is called, the ``cart['total']`` parameter will _not yet_
+          include payment fees charged by your provider as we don't yet know the amount of the charge, so you need to
+          take care of that yourself when setting your maximum amount.
+
+        - ``payment_is_valid_session`` will not be called during checkout, don't rely on it. If you called
+          ``add_payment_to_cart``, we'll trust the payment is okay and your next chance to change that will be
+          ``execute_payment``.
+
+        The changed behavior currently only affects the behavior during initial checkout (i.e. ``checkout_prepare``),
+        for ``payment_prepare`` the regular behavior applies and you are expected to just modify the amount of the
+        ``OrderPayment`` object if you need to.
+        """
+        return False
+
+    @property
+    def execute_payment_needs_user(self) -> bool:
+        """
+        Set this to ``True`` if your ``execute_payment`` function needs to be triggered by a user request, i.e. either
+        needs the ``request`` object or might require a browser redirect. If this is ``False``, you will not receive
+        a ``request`` and may not redirect since execute_payment might be called server-side. You should ensure that
+        your ``execute_payment`` method has a limited execution time (i.e. by using ``timeout`` for all external calls)
+        and handles all error cases appropriately.
+        """
+        return True
+
     @property
     def test_mode_message(self) -> str:
         """
@@ -281,16 +326,6 @@ class BasePaymentProvider:
                  help_text=_('Users will not be able to choose this payment provider after the given date.'),
                  required=False,
              )),
-            ('_invoice_text',
-             I18nFormField(
-                 label=_('Text on invoices'),
-                 help_text=_('Will be printed just below the payment figures and above the closing text on invoices. '
-                             'This will only be used if the invoice is generated before the order is paid. If the '
-                             'invoice is generated later, it will show a text stating that it has already been paid.'),
-                 required=False,
-                 widget=I18nTextarea,
-                 widget_kwargs={'attrs': {'rows': '2'}}
-             )),
             ('_total_min',
              forms.DecimalField(
                  label=_('Minimum order total'),
@@ -338,6 +373,16 @@ class BasePaymentProvider:
                              'above!').format(docs_url='https://docs.pretix.eu/en/latest/user/payments/fees.html'),
                  required=False
              )),
+            ('_invoice_text',
+             I18nFormField(
+                 label=_('Text on invoices'),
+                 help_text=_('Will be printed just below the payment figures and above the closing text on invoices. '
+                             'This will only be used if the invoice is generated before the order is paid. If the '
+                             'invoice is generated later, it will show a text stating that it has already been paid.'),
+                 required=False,
+                 widget=I18nTextarea,
+                 widget_kwargs={'attrs': {'rows': '2'}}
+             )),
             ('_restricted_countries',
              forms.MultipleChoiceField(
                  label=_('Restrict to countries'),
@@ -574,7 +619,7 @@ class BasePaymentProvider:
         ctx = {'request': request, 'form': form}
         return template.render(ctx)
 
-    def checkout_confirm_render(self, request, order: Order=None) -> str:
+    def checkout_confirm_render(self, request, order: Order=None, info_data: dict=None) -> str:
         """
         If the user has successfully filled in their payment data, they will be redirected
         to a confirmation page which lists all details of their order for a final review.
@@ -584,7 +629,9 @@ class BasePaymentProvider:
         In most cases, this should include a short summary of the user's input and
         a short explanation on how the payment process will continue.
 
+        :param request: The current HTTP request.
         :param order: Only set when this is a change to a new payment method for an existing order.
+        :param info_data: The ``info_data`` dictionary you set during ``add_payment_to_cart`` (only filled if ``multi_use_supported`` is set)
         """
         raise NotImplementedError()  # NOQA
 
@@ -618,6 +665,10 @@ class BasePaymentProvider:
         .. IMPORTANT:: If this is called, the user has not yet confirmed their order.
            You may NOT do anything which actually moves money.
 
+        Note: The behavior of this method changes significantly when you set
+              ``multi_use_supported``. Please refer to the ``multi_use_supported`` documentation
+              for more information.
+
         :param cart: This dictionary contains at least the following keys:
 
             positions:
@@ -657,9 +708,9 @@ class BasePaymentProvider:
         You will be passed an :py:class:`pretix.base.models.OrderPayment` object that contains
         the amount of money that should be paid.
 
-        If you need any special behavior, you can return a string
-        containing the URL the user will be redirected to. If you are done with your process
-        you should return the user to the order's detail page.
+        If you need any special behavior, you can return a string containing the URL the user will be redirected to.
+        If you are done with your process you should return the user to the order's detail page. Redirection is not
+        allowed if you set ``execute_payment_needs_user`` to ``True``.
 
         If the payment is completed, you should call ``payment.confirm()``. Please note that this might
         raise a ``Quota.QuotaExceededException`` if (and only if) the payment term of this order is over and
@@ -671,7 +722,7 @@ class BasePaymentProvider:
 
         On errors, you should raise a ``PaymentException``.
 
-        :param order: The order object
+        :param request: A HTTP request, except if ``execute_payment_needs_user`` is ``False``
         :param payment: An ``OrderPayment`` instance
         """
         return None
@@ -877,6 +928,15 @@ class BasePaymentProvider:
         """
         return {}
 
+    def api_refund_details(self, refund: OrderRefund):
+        """
+        Will be called to populate the ``details`` parameter of the refund in the REST API.
+
+        :param refund: The refund in question.
+        :return: A serializable dictionary
+        """
+        return {}
+
     def matching_id(self, payment: OrderPayment):
         """
         Will be called to get an ID for matching this payment when comparing pretix records with records of an external
@@ -896,6 +956,7 @@ class FreeOrderProvider(BasePaymentProvider):
     is_implicit = True
     is_enabled = True
     identifier = "free"
+    execute_payment_needs_user = False
 
     def checkout_confirm_render(self, request: HttpRequest) -> str:
         return _("No payment is required as this order only includes products which are free of charge.")
@@ -959,6 +1020,9 @@ class BoxOfficeProvider(BasePaymentProvider):
             "payment_data": payment.info_data.get('payment_data', {}),
         }
 
+    def api_refund_details(self, refund: OrderRefund):
+        return self.api_payment_details(refund)
+
     def payment_control_render(self, request, payment) -> str:
         if not payment.info:
             return
@@ -979,6 +1043,7 @@ class BoxOfficeProvider(BasePaymentProvider):
 class ManualPayment(BasePaymentProvider):
     identifier = 'manual'
     verbose_name = _('Manual payment')
+    execute_payment_needs_user = False
 
     @property
     def test_mode_message(self):
@@ -1059,12 +1124,12 @@ class ManualPayment(BasePaymentProvider):
         }
 
     def order_pending_mail_render(self, order, payment) -> str:
-        msg = str(self.settings.get('email_instructions', as_type=LazyI18nString)).format_map(self.format_map(order, payment))
+        msg = format_map(self.settings.get('email_instructions', as_type=LazyI18nString), self.format_map(order, payment))
         return msg
 
     def payment_pending_render(self, request, payment) -> str:
         return rich_text(
-            str(self.settings.get('pending_description', as_type=LazyI18nString)).format_map(self.format_map(payment.order, payment))
+            format_map(self.settings.get('pending_description', as_type=LazyI18nString), self.format_map(payment.order, payment))
         )
 
 
@@ -1119,18 +1184,42 @@ class OffsettingProvider(BasePaymentProvider):
 
 class GiftCardPayment(BasePaymentProvider):
     identifier = "giftcard"
-    verbose_name = _("Gift card")
     priority = 10
+    multi_use_supported = True
+    execute_payment_needs_user = False
+    verbose_name = _("Gift card")
+
+    @property
+    def public_name(self) -> str:
+        return str(self.settings.get("public_name", as_type=LazyI18nString)) or _(
+            "Gift card"
+        )
 
     @property
     def settings_form_fields(self):
-        f = super().settings_form_fields
+        fields = [
+            (
+                "public_name",
+                I18nFormField(
+                    label=_("Payment method name"), widget=I18nTextInput, required=False
+                ),
+            ),
+            (
+                "public_description",
+                I18nFormField(
+                    label=_("Payment method description"), widget=I18nTextarea, required=False
+                ),
+            ),
+        ]
+
+        f = OrderedDict(fields + list(super().settings_form_fields.items()))
         del f['_fee_abs']
         del f['_fee_percent']
         del f['_fee_reverse_calc']
         del f['_total_min']
         del f['_total_max']
         del f['_invoice_text']
+        f.move_to_end("_enabled", last=False)
         return f
 
     @property
@@ -1144,10 +1233,14 @@ class GiftCardPayment(BasePaymentProvider):
         return super().order_change_allowed(order) and self.event.organizer.has_gift_cards
 
     def payment_form_render(self, request: HttpRequest, total: Decimal) -> str:
-        return get_template('pretixcontrol/giftcards/checkout.html').render({})
+        return get_template('pretixcontrol/giftcards/checkout.html').render({
+            'request': request,
+        })
 
-    def checkout_confirm_render(self, request) -> str:
-        return get_template('pretixcontrol/giftcards/checkout_confirm.html').render({})
+    def checkout_confirm_render(self, request, order=None, info_data=None) -> str:
+        return get_template('pretixcontrol/giftcards/checkout_confirm.html').render({
+            'info_data': info_data,
+        })
 
     def refund_control_render(self, request, refund) -> str:
         from .models import GiftCard
@@ -1191,6 +1284,9 @@ class GiftCardPayment(BasePaymentProvider):
             }
         }
 
+    def api_refund_details(self, refund: OrderRefund):
+        return self.api_payment_details(refund)
+
     def payment_partial_refund_supported(self, payment: OrderPayment) -> bool:
         return True
 
@@ -1198,6 +1294,8 @@ class GiftCardPayment(BasePaymentProvider):
         return True
 
     def checkout_prepare(self, request: HttpRequest, cart: Dict[str, Any]) -> Union[bool, str, None]:
+        from pretix.base.services.cart import add_payment_to_cart
+
         for p in get_cart(request):
             if p.item.issue_giftcard:
                 messages.error(request, _("You cannot pay with gift cards when buying a gift card."))
@@ -1206,7 +1304,7 @@ class GiftCardPayment(BasePaymentProvider):
         cs = cart_session(request)
         try:
             gc = self.event.organizer.accepted_gift_cards.get(
-                secret=request.POST.get("giftcard")
+                secret=request.POST.get("giftcard").strip()
             )
             if gc.currency != self.event.currency:
                 messages.error(request, _("This gift card does not support this currency."))
@@ -1223,34 +1321,22 @@ class GiftCardPayment(BasePaymentProvider):
             if gc.value <= Decimal("0.00"):
                 messages.error(request, _("All credit on this gift card has been used."))
                 return
-            if 'gift_cards' not in cs:
-                cs['gift_cards'] = []
-            elif gc.pk in cs['gift_cards']:
-                messages.error(request, _("This gift card is already used for your payment."))
-                return
-            cs['gift_cards'] = cs['gift_cards'] + [gc.pk]
 
-            total = sum(p.total for p in cart['positions'])
-            # Recompute fees. Some plugins, e.g. pretix-servicefees, change their fee schedule if a gift card is
-            # applied.
-            fees = get_fees(
-                self.event, request, total, cart['invoice_address'], cs.get('payment'),
-                cart['raw']
+            for p in cs.get('payments', []):
+                if p['provider'] == self.identifier and p['info_data']['gift_card'] == gc.pk:
+                    messages.error(request, _("This gift card is already used for your payment."))
+                    return
+
+            add_payment_to_cart(
+                request,
+                self,
+                max_value=gc.value,
+                info_data={
+                    'gift_card': gc.pk,
+                    'gift_card_secret': gc.secret,
+                }
             )
-            total += sum([f.value for f in fees])
-            remainder = total
-            if remainder > Decimal('0.00'):
-                del cs['payment']
-                messages.success(request, _("Your gift card has been applied, but {} still need to be paid. Please select a payment method.").format(
-                    money_filter(remainder, self.event.currency)
-                ))
-            else:
-                messages.success(request, _("Your gift card has been applied."))
-
-            kwargs = {'step': 'payment'}
-            if request.resolver_match and 'cart_namespace' in request.resolver_match.kwargs:
-                kwargs['cart_namespace'] = request.resolver_match.kwargs['cart_namespace']
-            return eventreverse(self.event, 'presale:event.checkout', kwargs=kwargs)
+            return True
         except GiftCard.DoesNotExist:
             if self.event.vouchers.filter(code__iexact=request.POST.get("giftcard")).exists():
                 messages.warning(request, _("You entered a voucher instead of a gift card. Vouchers can only be entered on the first page of the shop below "
@@ -1268,7 +1354,7 @@ class GiftCardPayment(BasePaymentProvider):
 
         try:
             gc = self.event.organizer.accepted_gift_cards.get(
-                secret=request.POST.get("giftcard")
+                secret=request.POST.get("giftcard").strip()
             )
             if gc.currency != self.event.currency:
                 messages.error(request, _("This gift card does not support this currency."))
@@ -1287,6 +1373,7 @@ class GiftCardPayment(BasePaymentProvider):
                 return
             payment.info_data = {
                 'gift_card': gc.pk,
+                'gift_card_secret': gc.secret,
                 'retry': True
             }
             payment.amount = min(payment.amount, gc.value)
@@ -1294,7 +1381,7 @@ class GiftCardPayment(BasePaymentProvider):
 
             return True
         except GiftCard.DoesNotExist:
-            if self.event.vouchers.filter(code__iexact=request.POST.get("giftcard")).exists():
+            if self.event.vouchers.filter(code__iexact=request.POST.get("giftcard").strip()).exists():
                 messages.warning(request, _("You entered a voucher instead of a gift card. Vouchers can only be entered on the first page of the shop below "
                                             "the product selection."))
             else:
@@ -1302,37 +1389,46 @@ class GiftCardPayment(BasePaymentProvider):
         except GiftCard.MultipleObjectsReturned:
             messages.error(request, _("This gift card can not be redeemed since its code is not unique. Please contact the organizer of this event."))
 
-    def execute_payment(self, request: HttpRequest, payment: OrderPayment) -> str:
-        # This method will only be called when retrying payments, e.g. after a payment_prepare call. It is not called
-        # during the order creation phase because this payment provider is a special case.
-        for p in payment.order.positions.all():  # noqa - just a safeguard
+    def execute_payment(self, request: HttpRequest, payment: OrderPayment, is_early_special_case=False) -> str:
+        for p in payment.order.positions.all():
             if p.item.issue_giftcard:
                 raise PaymentException(_("You cannot pay with gift cards when buying a gift card."))
 
         gcpk = payment.info_data.get('gift_card')
-        if not gcpk or not payment.info_data.get('retry'):
+        if not gcpk:
             raise PaymentException("Invalid state, should never occur.")
-        with transaction.atomic():
-            gc = GiftCard.objects.select_for_update().get(pk=gcpk)
-            if gc.currency != self.event.currency:  # noqa - just a safeguard
-                raise PaymentException(_("This gift card does not support this currency."))
-            if not gc.accepted_by(self.event.organizer):  # noqa - just a safeguard
-                raise PaymentException(_("This gift card is not accepted by this event organizer."))
-            if payment.amount > gc.value:  # noqa - just a safeguard
-                raise PaymentException(_("This gift card was used in the meantime. Please try again."))
-            if gc.expires and gc.expires < now():  # noqa - just a safeguard
-                messages.error(request, _("This gift card is no longer valid."))
-                return
-            trans = gc.transactions.create(
-                value=-1 * payment.amount,
-                order=payment.order,
-                payment=payment
-            )
-            payment.info_data = {
-                'gift_card': gc.pk,
-                'transaction_id': trans.pk,
-            }
-            payment.confirm()
+        try:
+            with transaction.atomic():
+                try:
+                    gc = GiftCard.objects.select_for_update(of=OF_SELF).get(pk=gcpk)
+                except GiftCard.DoesNotExist:
+                    raise PaymentException(_("This gift card does not support this currency."))
+                if gc.currency != self.event.currency:  # noqa - just a safeguard
+                    raise PaymentException(_("This gift card does not support this currency."))
+                if not gc.accepted_by(self.event.organizer):
+                    raise PaymentException(_("This gift card is not accepted by this event organizer."))
+                if payment.amount > gc.value:
+                    raise PaymentException(_("This gift card was used in the meantime. Please try again."))
+                if gc.testmode and not payment.order.testmode:
+                    raise PaymentException(_("This gift card can only be used in test mode."))
+                if not gc.testmode and payment.order.testmode:
+                    raise PaymentException(_("Only test gift cards can be used in test mode."))
+                if gc.expires and gc.expires < now():
+                    raise PaymentException(_("This gift card is no longer valid."))
+
+                trans = gc.transactions.create(
+                    value=-1 * payment.amount,
+                    order=payment.order,
+                    payment=payment
+                )
+                payment.info_data = {
+                    'gift_card': gc.pk,
+                    'transaction_id': trans.pk,
+                }
+                payment.confirm(send_mail=not is_early_special_case, generate_invoice=not is_early_special_case)
+        except PaymentException as e:
+            payment.fail(info={'error': str(e)})
+            raise e
 
     def payment_is_valid_session(self, request: HttpRequest) -> bool:
         return True

src/pretix/base/pdf.py

@@ -35,22 +35,27 @@
 import copy
 import hashlib
 import itertools
+import json
 import logging
 import os
 import re
 import subprocess
 import tempfile
+import unicodedata
 import uuid
 from collections import OrderedDict
 from functools import partial
 from io import BytesIO
 
+import jsonschema
 from arabic_reshaper import ArabicReshaper
 from bidi.algorithm import get_display
 from django.conf import settings
 from django.contrib.staticfiles import finders
+from django.core.exceptions import ValidationError
 from django.db.models import Max, Min
 from django.dispatch import receiver
+from django.utils.deconstruct import deconstructible
 from django.utils.formats import date_format
 from django.utils.functional import SimpleLazyObject
 from django.utils.html import conditional_escape
@@ -739,12 +744,14 @@ class Renderer:
 
         if o['content'] == 'other' or o['content'] == 'other_i18n':
             if o['content'] == 'other_i18n':
-                text = str(LazyI18nString(o['text_i18n']))
+                text = str(LazyI18nString(o.get('text_i18n', {})))
             else:
-                text = o['text']
+                text = o.get('text', '')
 
             def replace(x):
                 if x.group(1).startswith('itemmeta:'):
+                    if op.variation_id:
+                        return op.variation.meta_data.get(x.group(1)[9:]) or ''
                     return op.item.meta_data.get(x.group(1)[9:]) or ''
                 elif x.group(1).startswith('meta:'):
                     return ev.meta_data.get(x.group(1)[5:]) or ''
@@ -765,6 +772,8 @@ class Renderer:
             return re.sub(r'\{([a-zA-Z0-9:_]+)\}', replace, text)
 
         elif o['content'].startswith('itemmeta:'):
+            if op.variation_id:
+                return op.variation.meta_data.get(o['content'][9:]) or ''
             return op.item.meta_data.get(o['content'][9:]) or ''
 
         elif o['content'].startswith('meta:'):
@@ -827,6 +836,13 @@ class Renderer:
         if o['italic']:
             font += ' I'
 
+        try:
+            ad = getAscentDescent(font, float(o['fontsize']))
+        except KeyError:  # font not known, fall back
+            logger.warning(f'Use of unknown font "{font}"')
+            font = 'Open Sans'
+            ad = getAscentDescent(font, float(o['fontsize']))
+
         align_map = {
             'left': TA_LEFT,
             'center': TA_CENTER,
@@ -846,6 +862,10 @@ class Renderer:
             self._get_text_content(op, order, o) or "",
         ).replace("\n", "<br/>\n").replace("-", "-&hairsp;")
 
+        # reportlab does not support unicode combination characters
+        # It's important we do this before we use ArabicReshaper
+        text = unicodedata.normalize("NFKC", text)
+
         # reportlab does not support RTL, ligature-heavy scripts like Arabic. Therefore, we use ArabicReshaper
         # to resolve all ligatures and python-bidi to switch RTL texts.
         try:
@@ -856,7 +876,6 @@ class Renderer:
         p = Paragraph(text, style=style)
         w, h = p.wrapOn(canvas, float(o['width']) * mm, 1000 * mm)
         # p_size = p.wrap(float(o['width']) * mm, 1000 * mm)
-        ad = getAscentDescent(font, float(o['fontsize']))
         canvas.saveState()
         # The ascent/descent offsets here are not really proven to be correct, they're just empirical values to get
         # reportlab render similarly to browser canvas.
@@ -961,3 +980,22 @@ class Renderer:
             output.write(outbuffer)
             outbuffer.seek(0)
             return outbuffer
+
+
+@deconstructible
+class PdfLayoutValidator:
+    def __call__(self, value):
+        if not isinstance(value, dict):
+            try:
+                val = json.loads(value)
+            except ValueError:
+                raise ValidationError(_('Your layout file is not a valid JSON file.'))
+        else:
+            val = value
+        with open(finders.find('schema/pdf-layout.schema.json'), 'r') as f:
+            schema = json.loads(f.read())
+        try:
+            jsonschema.validate(val, schema)
+        except jsonschema.ValidationError as e:
+            e = str(e).replace('%', '%%')
+            raise ValidationError(_('Your layout file is not a valid layout. Error message: {}').format(e))

src/pretix/base/plugins.py

@@ -65,7 +65,14 @@ def get_all_plugins(event=None) -> List[type]:
     )
 
 
-class PluginConfig(AppConfig):
+class PluginConfigMeta(type):
+    def __getattribute__(cls, item):
+        if item == "default" and cls is PluginConfig:
+            return False
+        return super().__getattribute__(item)
+
+
+class PluginConfig(AppConfig, metaclass=PluginConfigMeta):
     IGNORE = False
 
     def __init__(self, *args, **kwargs):