Fix precision issue

Co-authored-by: pajowu <engelhardt@pretix.eu>

src/pretix/api/auth/device.py

@@ -20,8 +20,11 @@
 # <https://www.gnu.org/licenses/>.
 #
 import logging
+from datetime import timedelta
 
 from django.contrib.auth.models import AnonymousUser
+from django.db import DatabaseError
+from django.utils.timezone import now
 from django_scopes import scopes_disabled
 from rest_framework import exceptions
 from rest_framework.authentication import TokenAuthentication
@@ -30,6 +33,7 @@ from pretix.api.auth.devicesecurity import (
     FullAccessSecurityProfile, get_all_security_profiles,
 )
 from pretix.base.models import Device
+from pretix.base.models.devices import DeviceLastSeen
 
 logger = logging.getLogger(__name__)
 
@@ -42,7 +46,7 @@ class DeviceTokenAuthentication(TokenAuthentication):
         model = self.get_model()
         try:
             with scopes_disabled():
-                device = model.objects.select_related('organizer').get(api_token=key)
+                device = model.objects.select_related('organizer', 'last_seen').get(api_token=key)
         except model.DoesNotExist:
             raise exceptions.AuthenticationFailed('Invalid token.')
 
@@ -53,6 +57,7 @@ class DeviceTokenAuthentication(TokenAuthentication):
             logging.warning(f'Connection attempt of revoked device {device.pk}.')
             raise exceptions.AuthenticationFailed('Device access has been revoked.')
 
+        self._update_last_seen(device)
         return AnonymousUser(), device
 
     def authenticate(self, request):
@@ -63,3 +68,22 @@ class DeviceTokenAuthentication(TokenAuthentication):
             if not profile.is_allowed(request):
                 raise exceptions.PermissionDenied('Request denied by device security profile.')
         return r
+
+    def _update_last_seen(self, device: Device):
+        try:
+            try:
+                last_seen_obj = device.last_seen
+            except DeviceLastSeen.DoesNotExist:
+                # First request from device, create model, ignore result. Use get_or_create to be safe
+                # against concurrent create requests
+                DeviceLastSeen.objects.get_or_create(device=device, last_seen=now())
+            else:
+                if now() - last_seen_obj.last_seen < timedelta(seconds=10):
+                    # We don't need to know the last seen info of a device to more precision than this,
+                    # so we can avoid some database writes if the device is bursting a lot of requests.
+                    return
+                last_seen_obj.last_seen = now()
+                last_seen_obj.save(update_fields=["last_seen"])
+        except DatabaseError:
+            # Do not stop the request from happening
+            logger.exception("Database error while updating last_seen")

src/pretix/base/migrations/0307_devicelastseen.py

@@ -0,0 +1,43 @@
+# Generated by Django 5.2.16 on 2026-08-05 08:00
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+import pretix.helpers.database
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("pretixbase", "0306_alter_eventmetaproperty_unique_together"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DeviceLastSeen",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False
+                    ),
+                ),
+                ("last_seen", models.DateTimeField(auto_now=True)),
+                (
+                    "device",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="pretixbase.device",
+                    ),
+                ),
+            ],
+        ),
+        migrations.AddIndex(
+            model_name="devicelastseen",
+            index=pretix.helpers.database.BrinIndexIgnoredOnSQLite(
+                models.F("last_seen"),
+                autosummarize=True,
+                name="pretixbase_device_last_seen",
+            ),
+        ),
+    ]

src/pretix/base/models/devices.py

@@ -32,6 +32,7 @@ from pretix.base.models import LoggedModel
 from pretix.base.permissions import (
     AnyPermissionOf, assert_valid_event_permission,
 )
+from pretix.helpers import BrinIndexIgnoredOnSQLite
 
 
 @scopes_disabled()
@@ -287,3 +288,22 @@ class Device(LoggedModel):
             return self.get_events_with_any_permission()
         else:
             return self.organizer.events.none()
+
+
+class DeviceLastSeen(models.Model):
+    # This is a separate model since we expect it to get A LOT of writes and PostgreSQL always
+    # writes full rows and then needs to update all indexes on the row, so this is going to save a
+    # lot of write traffic on the databse
+    device = models.OneToOneField("Device", on_delete=models.CASCADE, related_name="last_seen")
+    last_seen = models.DateTimeField(auto_now=True)
+
+    class Meta:
+        indexes = [
+            BrinIndexIgnoredOnSQLite(
+                # BRIN indexes are highly efficient on lots of updates, especially of chronological data
+                # and especially if we later want to query them by range, as we likely want to.
+                "last_seen",
+                name="pretixbase_device_last_seen",
+                autosummarize=True
+            )
+        ]

src/pretix/base/services/placeholders.py

@@ -801,11 +801,10 @@ def get_available_placeholders(event, base_parameters, rich=False):
     return params
 
 
-def get_sample_context(event, context_parameters, rich=True):
+def prepare_sample_context_for_preview(placeholder_to_sample):
     context_dict = {}
     lbl = _('This value will be replaced based on dynamic parameters.')
-    for k, v in get_available_placeholders(event, context_parameters, rich=rich).items():
-        sample = v.render_sample(event)
+    for k, sample in placeholder_to_sample.items():
         if isinstance(sample, PlainHtmlAlternativeString):
             context_dict[k] = PlainHtmlAlternativeString(
                 '<{el} class="placeholder" title="{title}">{plain}</{el}>'.format(
@@ -830,3 +829,12 @@ def get_sample_context(event, context_parameters, rich=True):
                 escape(sample)
             ))
     return context_dict
+
+
+def get_sample_context(event, context_parameters, rich=True):
+    return prepare_sample_context_for_preview(
+        {
+            k: v.render_sample(event)
+            for k, v in get_available_placeholders(event, context_parameters, rich=rich).items()
+        }
+    )

src/pretix/base/templatetags/money.py

@@ -20,6 +20,7 @@
 # <https://www.gnu.org/licenses/>.
 #
 from decimal import ROUND_HALF_UP, Decimal
+from typing import Optional
 
 from babel import Locale, UnknownLocaleError
 from babel.numbers import format_currency
@@ -35,32 +36,32 @@ register = template.Library()
 
 
 @register.filter("money")
-def money_filter(value: Decimal, arg='', hide_currency=False):
-    if isinstance(value, (float, int)):
+def money_filter(value: Optional[Decimal | float | int | str], arg='', hide_currency=False):
+    if isinstance(value, (float, int, str)):
+        if value == '':
+            return value
         value = Decimal(value)
     if value is None:
         value = Decimal('0.00')
     if not isinstance(value, Decimal):
-        if value == '':
-            return value
         raise TypeError("Invalid data type passed to money filter: %r" % type(value))
     if not arg:
         raise ValueError("No currency passed.")
     arg = arg.upper()
 
-    places = settings.CURRENCY_PLACES.get(arg, 2)
-    rounded = value.quantize(Decimal('1') / 10 ** places, ROUND_HALF_UP)
-    if places < 2 and rounded != value:
-        # We display decimal places even if we shouldn't for this currency if rounding
-        # would make the numbers incorrect. If this branch executes, it's likely a bug in
-        # pretix, but we won't show wrong numbers!
-        if hide_currency:
-            return floatformat(value, "2g")
-        else:
-            return '{} {}'.format(arg, floatformat(value, "2g"))
+    if value.normalize().as_tuple().exponent < -9:
+        # Heuristic: It's unlikely we'll ever see values of less than 0.000000001 in any currency. Therefore, if we
+        # do see them, we very likely deal with a floating point error. This happens mostly in dev mode when computations
+        # are made in SQLite, which uses REAL precision, but it can also happen when we naively pass a float from Python
+        # land to this filter (even though it should not happen).
+        value = value.quantize(Decimal('1e-9'), ROUND_HALF_UP).normalize()
+
+    currency_places = settings.CURRENCY_PLACES.get(arg, 2)
+    required_places = -value.normalize().as_tuple().exponent
+    render_places = max(currency_places, required_places)
 
     if hide_currency:
-        return floatformat(value, f"{places}g")
+        return floatformat(value, f"{render_places}g")
 
     try:
         locale = Locale(get_babel_locale())
@@ -68,14 +69,29 @@ def money_filter(value: Decimal, arg='', hide_currency=False):
         locale = "en"
 
     try:
-        return format_currency(value, arg, locale=locale)
+        return format_currency(
+            value,
+            arg,
+            locale=locale,
+            # We only allow Babel to restrict the digits to the digits defined by the currency if this does not remove any
+            # precision in case we have sub-currency precision (which we shouldn't have in most places, but it's still
+            # better than showing wrong data). Note: Weird precision effects can occur after in-database arithmetic
+            # on SQLite, since SQLite does not have fixed-decimal computation.
+            currency_digits=currency_places >= required_places,
+            decimal_quantization=currency_places >= required_places,
+        )
     except:
-        return '{} {}'.format(arg, floatformat(value, f"{places}g"))
+        return '{} {}'.format(arg, floatformat(value, f"{render_places}g"))
+
+
+@register.filter("money_without_currency")
+def money_filter_without_currency(value: Optional[Decimal | float | int | str], arg=''):
+    return money_filter(value, arg, hide_currency=True)
 
 
 @register.filter("money_numberfield")
-def money_numberfield_filter(value: Decimal, arg=''):
-    if isinstance(value, (float, int)):
+def money_numberfield_filter(value: Optional[Decimal | float | int | str], arg=''):
+    if isinstance(value, (float, int, str)):
         value = Decimal(value)
     if not isinstance(value, Decimal):
         raise TypeError("Invalid data type passed to money filter: %r" % type(value))
@@ -87,15 +103,28 @@ def money_numberfield_filter(value: Decimal, arg=''):
 
 
 @register.filter(is_safe=True)
-def tax_rate_format(number):
+def tax_rate_format(number: Optional[Decimal | float | int | str]):
     """
     Display a Decimal to its significant decimal places, used for tax rates.
     """
-    assert isinstance(number, Decimal)
+    if isinstance(number, (float, int, str)):
+        if number == '':
+            return number
+        number = Decimal(number)
+    if number is None:
+        number = Decimal('0.00')
+    if not isinstance(number, Decimal):
+        raise TypeError("Invalid data type passed to tax rate format filter: %r" % type(number))
+    if number.normalize().as_tuple().exponent < -9:
+        # Heuristic: It's unlikely we'll ever see values of less than 0.000000001 in any currency. Therefore, if we
+        # do see them, we very likely deal with a floating point error. This happens mostly in dev mode when computations
+        # are made in SQLite, which uses REAL precision, but it can also happen when we naively pass a float from Python
+        # land to this filter (even though it should not happen).
+        number = number.quantize(Decimal('1e-9'), ROUND_HALF_UP).normalize()
     return mark_safe(
         formats.number_format(
-            number.normalize(),
-            -number.as_tuple().exponent,
+            number,
+            -number.normalize().as_tuple().exponent,
             use_l10n=True,
             force_grouping=False,
         )

src/pretix/control/views/organizer.py

@@ -108,6 +108,9 @@ from pretix.base.services.export import (
     init_organizer_exporters, multiexport, scheduled_organizer_export,
 )
 from pretix.base.services.mail import mail, prefix_subject
+from pretix.base.services.placeholders import (
+    prepare_sample_context_for_preview,
+)
 from pretix.base.templatetags.rich_text import markdown_compile_email
 from pretix.base.views.tasks import AsyncAction
 from pretix.control.forms.exports import ScheduledOrganizerExportForm
@@ -345,16 +348,11 @@ class MailSettingsPreview(OrganizerPermissionRequiredMixin, View):
 
     # get all supported placeholders with dummy values
     def placeholders(self, item):
-        ctx = {}
-        for p, s in MailSettingsForm(obj=self.request.organizer)._get_sample_context(
-                MailSettingsForm.base_context[item]).items():
-            if s.strip().startswith('*'):
-                ctx[p] = s
-            else:
-                ctx[p] = '<span class="placeholder" title="{}">{}</span>'.format(
-                    _('This value will be replaced based on dynamic parameters.'),
-                    s
-                )
+        ctx = prepare_sample_context_for_preview(
+            MailSettingsForm(obj=self.request.organizer)._get_sample_context(
+                MailSettingsForm.base_context[item]
+            )
+        )
         return self.SafeDict(ctx)
 
     def post(self, request, *args, **kwargs):

src/pretix/helpers/database.py

@@ -22,6 +22,7 @@
 import contextlib
 
 from django.conf import settings
+from django.contrib.postgres.indexes import BrinIndex
 from django.core.exceptions import FieldDoesNotExist, ImproperlyConfigured
 from django.db import connection, transaction
 from django.db.models import (
@@ -285,3 +286,21 @@ def get_deterministic_ordering(model, ordering):
             # on the primary key to provide total ordering.
             ordering.append("-pk")
     return ordering
+
+
+class IgnoreOnSQLiteMixin:
+    # Mixin to allow defining PostgreSQL-specific indexes that will just not be created
+    # on SQLite. SQLite is supported for testing only anyways!
+    def create_sql(self, model, schema_editor, *args, **kwargs):
+        if "sqlite" in settings.DATABASES["default"]["ENGINE"]:
+            return ""
+        return super().create_sql(model, schema_editor, *args, **kwargs)
+
+    def remove_sql(self, model, schema_editor, **kwargs):
+        if "sqlite" in settings.DATABASES["default"]["ENGINE"]:
+            return ""
+        return super().remove_sql(model, schema_editor, **kwargs)
+
+
+class BrinIndexIgnoredOnSQLite(IgnoreOnSQLiteMixin, BrinIndex):
+    pass

src/tests/api/test_deviceauth.py

@@ -20,13 +20,16 @@
 # <https://www.gnu.org/licenses/>.
 #
 import base64
+from datetime import datetime, timezone
 
 import pytest
 from cryptography.hazmat.primitives.asymmetric import padding
 from cryptography.hazmat.primitives.serialization import load_pem_private_key
 from django_scopes import scopes_disabled
+from freezegun import freeze_time
 
 from pretix.base.models import Device
+from pretix.base.models.devices import DeviceLastSeen
 
 
 @pytest.fixture
@@ -386,3 +389,26 @@ def test_device_info_key_sets(device_client, device: Device):
         base64.b64decode(ks['diversification_key']),
         padding.PKCS1v15()
     )
+
+
+@pytest.mark.django_db
+def test_update_last_seen(device_client, device: Device):
+    assert not DeviceLastSeen.objects.exists()
+
+    with freeze_time("2020-01-10T14:30:00+00:00"):
+        resp = device_client.get('/api/v1/device/info')
+        assert resp.status_code == 200
+        assert device.last_seen.last_seen == datetime(2020, 1, 10, 14, 30, tzinfo=timezone.utc)
+
+    with freeze_time("2020-01-10T14:30:05+00:00"):
+        resp = device_client.get('/api/v1/device/info')
+        assert resp.status_code == 200
+        # No update, interal too short
+        device.last_seen.refresh_from_db()
+        assert device.last_seen.last_seen == datetime(2020, 1, 10, 14, 30, tzinfo=timezone.utc)
+
+    with freeze_time("2020-01-10T14:30:30+00:00"):
+        resp = device_client.get('/api/v1/device/info')
+        assert resp.status_code == 200
+        device.last_seen.refresh_from_db()
+        assert device.last_seen.last_seen == datetime(2020, 1, 10, 14, 30, 30, tzinfo=timezone.utc)

src/tests/base/test_templatetag.py

@@ -26,7 +26,7 @@ from django.template import Context, Template
 from django.test import RequestFactory
 from django.utils import translation
 
-from pretix.base.templatetags.money import money_filter
+from pretix.base.templatetags.money import money_filter, tax_rate_format
 
 TEMPLATE_REPLACE_PAGE = Template(
     "{% load urlreplace %}{% url_replace request 'page' 3 %}"
@@ -60,7 +60,9 @@ def test_urlreplace_replace_parameter():
     "locale,amount,currency,expected",
     [
         ("en", None, "USD", "$0.00"),
+        ("en", "", "USD", ""),
         ("en", 1000000, "USD", "$1,000,000.00"),
+        ("en", 2.23, "USD", "$2.23"),
         ("en", Decimal("1000.00"), "USD", "$1,000.00"),
         ("de", Decimal("1.23"), "EUR", "1,23" + NBSP + "€"),
         ("de", Decimal("1000.00"), "EUR", "1.000,00" + NBSP + "€"),
@@ -70,11 +72,14 @@ def test_urlreplace_replace_parameter():
 
         # unknown currency
         ("de", Decimal("1234.56"), "FOO", "1.234,56" + NBSP + "FOO"),
-        ("de", Decimal("1234.567"), "FOO", "1.234,57" + NBSP + "FOO"),
+        ("de", Decimal("1234.567"), "FOO", "1.234,567" + NBSP + "FOO"),
 
-        # rounding errors
-        ("de", Decimal("1.234"), "EUR", "1,23" + NBSP + "€"),
-        ("de", Decimal("1023.1"), "JPY", "JPY 1.023,10"),
+        # deal with precision that is higher than the currency
+        ("de", Decimal("1.234"), "EUR", "1,234" + NBSP + "€"),
+        ("de", 1.234, "EUR", "1,234" + NBSP + "€"),
+        ("de", Decimal("1.2340"), "EUR", "1,234" + NBSP + "€"),
+        ("de", Decimal("1.2300"), "EUR", "1,23" + NBSP + "€"),
+        ("de", Decimal("1023.1"), "JPY", "1.023,10" + NBSP + "¥"),
     ]
 )
 def test_money_filter(locale, amount, currency, expected):
@@ -98,9 +103,31 @@ def test_money_filter(locale, amount, currency, expected):
     [
         ("de", Decimal("1000.00"), "EUR", "1.000,00"),
         ("en", Decimal("1000.00"), "EUR", "1,000.00"),
-        ("de", Decimal("1023.1"), "JPY", "1.023,10"),
+        ("de", Decimal("1023.1"), "JPY", "1.023,1"),
     ]
 )
 def test_money_filter_hidecurrency(locale, amount, currency, expected):
     translation.activate(locale)
     assert money_filter(amount, currency, hide_currency=True) == expected
+
+
+@pytest.mark.parametrize(
+    "locale,rate,expected",
+    [
+        ("de", Decimal("2.00"), "2"),
+        ("de", Decimal("2.50"), "2,5"),
+        ("de", Decimal("2.2340"), "2,234"),
+        ("en", Decimal("2.00"), "2"),
+        ("en", Decimal("2.50"), "2.5"),
+        ("en", Decimal("4.3e7"), "43000000"),
+        ("en", Decimal("2.2340"), "2.234"),
+        ("en", "2.23", "2.23"),
+        ("en", 2.23, "2.23"),
+        ("en", 2, "2"),
+        ("en", "", ""),
+        ("en", None, "0"),
+    ]
+)
+def test_tax_rate_format(locale, rate, expected):
+    translation.activate(locale)
+    assert tax_rate_format(rate) == expected