Money representation in templates: Allow more precision (#6454)

* Money representation in templates: Allow more precision

When rendering money in templates, we used to have the following logic:

- When the decimal places fit the currency, render with Babel
- When they don't, e.g. we stored 123.67 JPY, even though there are no
  fractional Yens, render without Babel with a custom format, but render
  the fractional Yens because we'd rather *show* wrong data and make the
  bug obvious than hide it.

However, we only did that up to a prevision of two places, we never
showed more. This is still sufficient for core pretix, but we have
plugins that need to operate in fractional cents. Also, we CAN render
everything through babel for consistent formatting.

There is one **risk**: This might cause weird results on SQLite. Since
SQLite has no concept of precise decimal math, results of in-SQL
computations can sometimes experience floating point errors and show
with A LOT of decimal palces. This used to be invisible since the UI
performed the rounding. With this PR – not any more. We'll need to see
how annoying it is, but it should only affect development mode.

This PR also fixes a bug in tax_rate_format that for some reason did not
do what it was supposed to do, even though I tested it back then, weird.
Might even be a Python version thing?

* Update src/pretix/base/templatetags/money.py

Co-authored-by: pajowu <engelhardt@pretix.eu>

* Apply suggestion from @pajowu

Co-authored-by: pajowu <engelhardt@pretix.eu>

* Fix typing stuff

* Fix precision issue

---------

Co-authored-by: pajowu <engelhardt@pretix.eu>

src/pretix/base/invoicing/email.py

@@ -71,8 +71,8 @@ class EmailTransmissionType(TransmissionType):
 
     def transmission_info_to_form_data(self, transmission_info: dict) -> dict:
         return {
-            "transmission_email_other": bool(transmission_info.get("transmission_email_address")) if transmission_info else False,
-            "transmission_email_address": transmission_info.get("transmission_email_address") if transmission_info else "",
+            "transmission_email_other": bool(transmission_info.get("transmission_email_address")),
+            "transmission_email_address": transmission_info.get("transmission_email_address"),
         }
 
     def form_data_to_transmission_info(self, form_data: dict) -> dict:

src/pretix/base/pdf.py

@@ -801,6 +801,18 @@ def generate_compressed_addon_list(op, order, event, only_checked_in=False):
     return addonlist
 
 
+def get_sizebox(page: pypdf.PageObject):
+    mediabox = page.mediabox
+    cropbox = page.cropbox
+
+    return pypdf.generic.RectangleObject((
+        max(mediabox[0], cropbox[0]),
+        max(mediabox[1], cropbox[1]),
+        min(mediabox[2], cropbox[2]),
+        min(mediabox[3], cropbox[3]),
+    ))
+
+
 class Renderer:
 
     def __init__(self, event, layout, background_file):
@@ -1153,11 +1165,10 @@ class Renderer:
                 elif o['type'] == "poweredby":
                     self._draw_poweredby(canvas, op, o)
                 if self.bg_pdf:
-                    page_size = (
-                        self.bg_pdf.pages[0].mediabox[2] - self.bg_pdf.pages[0].mediabox[0],
-                        self.bg_pdf.pages[0].mediabox[3] - self.bg_pdf.pages[0].mediabox[1]
-                    )
-                    if self.bg_pdf.pages[0].get('/Rotate') in (90, 270):
+                    first_page = self.bg_pdf.pages[0]
+                    sizebox = get_sizebox(first_page)
+                    page_size = (sizebox.width, sizebox.height)
+                    if first_page.rotation in (90, 270):
                         # swap dimensions due to pdf being rotated
                         page_size = page_size[::-1]
                     canvas.setPageSize(page_size)
@@ -1312,14 +1323,18 @@ def merge_background(fg_pdf: PdfWriter, bg_pdf: PdfWriter, out_file, compress):
 
 
 def _merge_with_correct_page_media_box(output: pypdf.PdfWriter, fg_page: pypdf.PageObject, bg_page: pypdf.PageObject):
-    if bg_page.rotation != 0:
-        bg_page.transfer_rotation_to_content()
-    media_box = bg_page.mediabox
+    """
+    Adds fg_page to output, merging bg_page behind it.
+
+    If bg_page has a non-zero mergebox/cropbox or is rotated via /Rotate, a transformation is applied to fix this."""
     trsf = pypdf.Transformation()
-    if media_box.bottom != 0:
-        trsf = trsf.translate(0, -media_box.bottom)
-    if media_box.left != 0:
-        trsf = trsf.translate(-media_box.left, 0)
+    if bg_page.rotation != 0:
+        trsf = trsf.rotate(-bg_page.rotation)
+
+    mb = get_sizebox(bg_page)
+    pt1 = trsf.apply_on(mb.lower_left)
+    pt2 = trsf.apply_on(mb.upper_right)
+    trsf = trsf.translate(-min(pt1[0], pt2[0]), -min(pt1[1], pt2[1]))
 
     fg_page = output.add_page(fg_page)
     fg_page.merge_transformed_page(bg_page, trsf, over=False, expand=False)

src/pretix/base/templatetags/money.py

@@ -20,6 +20,7 @@
 # <https://www.gnu.org/licenses/>.
 #
 from decimal import ROUND_HALF_UP, Decimal
+from typing import Optional
 
 from babel import Locale, UnknownLocaleError
 from babel.numbers import format_currency
@@ -35,32 +36,32 @@ register = template.Library()
 
 
 @register.filter("money")
-def money_filter(value: Decimal, arg='', hide_currency=False):
-    if isinstance(value, (float, int)):
+def money_filter(value: Optional[Decimal | float | int | str], arg='', hide_currency=False):
+    if isinstance(value, (float, int, str)):
+        if value == '':
+            return value
         value = Decimal(value)
     if value is None:
         value = Decimal('0.00')
     if not isinstance(value, Decimal):
-        if value == '':
-            return value
         raise TypeError("Invalid data type passed to money filter: %r" % type(value))
     if not arg:
         raise ValueError("No currency passed.")
     arg = arg.upper()
 
-    places = settings.CURRENCY_PLACES.get(arg, 2)
-    rounded = value.quantize(Decimal('1') / 10 ** places, ROUND_HALF_UP)
-    if places < 2 and rounded != value:
-        # We display decimal places even if we shouldn't for this currency if rounding
-        # would make the numbers incorrect. If this branch executes, it's likely a bug in
-        # pretix, but we won't show wrong numbers!
-        if hide_currency:
-            return floatformat(value, "2g")
-        else:
-            return '{} {}'.format(arg, floatformat(value, "2g"))
+    if value.normalize().as_tuple().exponent < -9:
+        # Heuristic: It's unlikely we'll ever see values of less than 0.000000001 in any currency. Therefore, if we
+        # do see them, we very likely deal with a floating point error. This happens mostly in dev mode when computations
+        # are made in SQLite, which uses REAL precision, but it can also happen when we naively pass a float from Python
+        # land to this filter (even though it should not happen).
+        value = value.quantize(Decimal('1e-9'), ROUND_HALF_UP).normalize()
+
+    currency_places = settings.CURRENCY_PLACES.get(arg, 2)
+    required_places = -value.normalize().as_tuple().exponent
+    render_places = max(currency_places, required_places)
 
     if hide_currency:
-        return floatformat(value, f"{places}g")
+        return floatformat(value, f"{render_places}g")
 
     try:
         locale = Locale(get_babel_locale())
@@ -68,14 +69,29 @@ def money_filter(value: Decimal, arg='', hide_currency=False):
         locale = "en"
 
     try:
-        return format_currency(value, arg, locale=locale)
+        return format_currency(
+            value,
+            arg,
+            locale=locale,
+            # We only allow Babel to restrict the digits to the digits defined by the currency if this does not remove any
+            # precision in case we have sub-currency precision (which we shouldn't have in most places, but it's still
+            # better than showing wrong data). Note: Weird precision effects can occur after in-database arithmetic
+            # on SQLite, since SQLite does not have fixed-decimal computation.
+            currency_digits=currency_places >= required_places,
+            decimal_quantization=currency_places >= required_places,
+        )
     except:
-        return '{} {}'.format(arg, floatformat(value, f"{places}g"))
+        return '{} {}'.format(arg, floatformat(value, f"{render_places}g"))
+
+
+@register.filter("money_without_currency")
+def money_filter_without_currency(value: Optional[Decimal | float | int | str], arg=''):
+    return money_filter(value, arg, hide_currency=True)
 
 
 @register.filter("money_numberfield")
-def money_numberfield_filter(value: Decimal, arg=''):
-    if isinstance(value, (float, int)):
+def money_numberfield_filter(value: Optional[Decimal | float | int | str], arg=''):
+    if isinstance(value, (float, int, str)):
         value = Decimal(value)
     if not isinstance(value, Decimal):
         raise TypeError("Invalid data type passed to money filter: %r" % type(value))
@@ -87,15 +103,28 @@ def money_numberfield_filter(value: Decimal, arg=''):
 
 
 @register.filter(is_safe=True)
-def tax_rate_format(number):
+def tax_rate_format(number: Optional[Decimal | float | int | str]):
     """
     Display a Decimal to its significant decimal places, used for tax rates.
     """
-    assert isinstance(number, Decimal)
+    if isinstance(number, (float, int, str)):
+        if number == '':
+            return number
+        number = Decimal(number)
+    if number is None:
+        number = Decimal('0.00')
+    if not isinstance(number, Decimal):
+        raise TypeError("Invalid data type passed to tax rate format filter: %r" % type(number))
+    if number.normalize().as_tuple().exponent < -9:
+        # Heuristic: It's unlikely we'll ever see values of less than 0.000000001 in any currency. Therefore, if we
+        # do see them, we very likely deal with a floating point error. This happens mostly in dev mode when computations
+        # are made in SQLite, which uses REAL precision, but it can also happen when we naively pass a float from Python
+        # land to this filter (even though it should not happen).
+        number = number.quantize(Decimal('1e-9'), ROUND_HALF_UP).normalize()
     return mark_safe(
         formats.number_format(
-            number.normalize(),
-            -number.as_tuple().exponent,
+            number,
+            -number.normalize().as_tuple().exponent,
             use_l10n=True,
             force_grouping=False,
         )

src/tests/base/test_templatetag.py

@@ -26,7 +26,7 @@ from django.template import Context, Template
 from django.test import RequestFactory
 from django.utils import translation
 
-from pretix.base.templatetags.money import money_filter
+from pretix.base.templatetags.money import money_filter, tax_rate_format
 
 TEMPLATE_REPLACE_PAGE = Template(
     "{% load urlreplace %}{% url_replace request 'page' 3 %}"
@@ -60,7 +60,9 @@ def test_urlreplace_replace_parameter():
     "locale,amount,currency,expected",
     [
         ("en", None, "USD", "$0.00"),
+        ("en", "", "USD", ""),
         ("en", 1000000, "USD", "$1,000,000.00"),
+        ("en", 2.23, "USD", "$2.23"),
         ("en", Decimal("1000.00"), "USD", "$1,000.00"),
         ("de", Decimal("1.23"), "EUR", "1,23" + NBSP + "€"),
         ("de", Decimal("1000.00"), "EUR", "1.000,00" + NBSP + "€"),
@@ -70,11 +72,14 @@ def test_urlreplace_replace_parameter():
 
         # unknown currency
         ("de", Decimal("1234.56"), "FOO", "1.234,56" + NBSP + "FOO"),
-        ("de", Decimal("1234.567"), "FOO", "1.234,57" + NBSP + "FOO"),
+        ("de", Decimal("1234.567"), "FOO", "1.234,567" + NBSP + "FOO"),
 
-        # rounding errors
-        ("de", Decimal("1.234"), "EUR", "1,23" + NBSP + "€"),
-        ("de", Decimal("1023.1"), "JPY", "JPY 1.023,10"),
+        # deal with precision that is higher than the currency
+        ("de", Decimal("1.234"), "EUR", "1,234" + NBSP + "€"),
+        ("de", 1.234, "EUR", "1,234" + NBSP + "€"),
+        ("de", Decimal("1.2340"), "EUR", "1,234" + NBSP + "€"),
+        ("de", Decimal("1.2300"), "EUR", "1,23" + NBSP + "€"),
+        ("de", Decimal("1023.1"), "JPY", "1.023,10" + NBSP + "¥"),
     ]
 )
 def test_money_filter(locale, amount, currency, expected):
@@ -98,9 +103,31 @@ def test_money_filter(locale, amount, currency, expected):
     [
         ("de", Decimal("1000.00"), "EUR", "1.000,00"),
         ("en", Decimal("1000.00"), "EUR", "1,000.00"),
-        ("de", Decimal("1023.1"), "JPY", "1.023,10"),
+        ("de", Decimal("1023.1"), "JPY", "1.023,1"),
     ]
 )
 def test_money_filter_hidecurrency(locale, amount, currency, expected):
     translation.activate(locale)
     assert money_filter(amount, currency, hide_currency=True) == expected
+
+
+@pytest.mark.parametrize(
+    "locale,rate,expected",
+    [
+        ("de", Decimal("2.00"), "2"),
+        ("de", Decimal("2.50"), "2,5"),
+        ("de", Decimal("2.2340"), "2,234"),
+        ("en", Decimal("2.00"), "2"),
+        ("en", Decimal("2.50"), "2.5"),
+        ("en", Decimal("4.3e7"), "43000000"),
+        ("en", Decimal("2.2340"), "2.234"),
+        ("en", "2.23", "2.23"),
+        ("en", 2.23, "2.23"),
+        ("en", 2, "2"),
+        ("en", "", ""),
+        ("en", None, "0"),
+    ]
+)
+def test_tax_rate_format(locale, rate, expected):
+    translation.activate(locale)
+    assert tax_rate_format(rate) == expected