Support for IDNA domain names

pyproject.toml

@@ -62,6 +62,7 @@ dependencies = [
   "drf_ujson2==1.7.*",
   "geoip2==4.*",
   "importlib_metadata==6.6.*",  # Polyfill, we can probably drop this once we require Python 3.10+
+  "idna",
   "isoweek",
   "jsonschema",
   "kombu==5.2.*",

src/pretix/base/email.py

@@ -420,7 +420,7 @@ def base_placeholders(sender, **kwargs):
                     'order': 'F8VVL',
                     'secret': '6zzjnumtsx136ddy',
                     'hash': '98kusd8ofsj8dnkd'
-                }
+                },
             ),
         ),
         SimpleFunctionalMailTextPlaceholder(

src/pretix/base/templatetags/rich_text.py

@@ -36,6 +36,7 @@ import re
 import urllib.parse
 
 import bleach
+import idna
 import markdown
 from bleach import DEFAULT_CALLBACKS
 from bleach.linkifier import build_email_re, build_url_re
@@ -121,6 +122,12 @@ def safelink_callback(attrs, new=False):
     return attrs
 
 
+def idna_decode_safe(src):
+    v = idna.decode(src)
+
+    return v
+
+
 def truelink_callback(attrs, new=False):
     """
     Tries to prevent "phishing" attacks in which a link looks like it points to a safe place but instead
@@ -136,20 +143,40 @@ def truelink_callback(attrs, new=False):
 
         <a href="https://maps.google.com/location/foo">https://maps.google.com</a>
     """
-    text = re.sub(r'[^a-zA-Z0-9.\-/_ ]', '', attrs.get('_text'))  # clean up link text
+    text = re.sub(r'[^a-zA-Z0-9:.\-/_ ]', '', attrs.get('_text'))  # clean up link text
     url = attrs.get((None, 'href'), '/')
     href_url = urllib.parse.urlparse(url)
+    strip_http = False
     if (None, 'href') in attrs and URL_RE.match(text) and href_url.scheme not in ('tel', 'mailto'):
+        if URL_RE.match(attrs.get('_text').strip()):  # maybe we cleaned up too much
+            text = attrs.get('_text').strip()
         # link text looks like a url
         if text.startswith('//'):
             text = 'https:' + text
         elif not text.startswith('http'):
+            strip_http = True
             text = 'https://' + text
 
         text_url = urllib.parse.urlparse(text)
-        if text_url.netloc != href_url.netloc or not href_url.path.startswith(href_url.path):
+
+        if href_url.netloc.startswith('xn--'):
+            href_netloc_nice = idna_decode_safe(href_url.netloc)
+        else:
+            href_netloc_nice = href_url.netloc
+
+        if text_url.netloc not in (href_url.netloc, href_netloc_nice) or not href_url.path.startswith(text_url.path):
             # link text contains an URL that has a different base than the actual URL
             attrs['_text'] = attrs[None, 'href']
+            text_url = href_url
+
+        if text_url.netloc.startswith('xn--'):
+            # Show punicode nicely
+            text_netloc_nice = idna_decode_safe(text_url.netloc)
+            url = text_url._replace(netloc=text_netloc_nice, scheme=href_url.scheme).geturl()
+            if strip_http:
+                url = url[len(href_url.scheme + '://'):]
+            attrs['_text'] = url
+
     return attrs
 
 

src/pretix/control/templates/pretixcontrol/event/index.html

@@ -18,7 +18,7 @@
     </h1>
     <div class="helper-space-below">
         {% trans "Shop URL:" %}
-        <span id="shop_url" class="text-muted">{% abseventurl request.event "presale:event.index" %}</span>
+        <span id="shop_url" class="text-muted">{% humanabseventurl request.event "presale:event.index" %}</span>
         <button type="button" class="btn btn-default btn-xs btn-clipboard js-only" data-clipboard-target="#shop_url">
             <i class="fa fa-clipboard" aria-hidden="true"></i>
             <span class="sr-only">{% trans "Copy to clipboard" %}</span>

src/pretix/control/templates/pretixcontrol/vouchers/detail.html

@@ -43,7 +43,7 @@
                             <label class="col-md-3 control-label" for="id_url">{% trans "Voucher link" %}</label>
                             <div class="col-md-9">
                                 <input type="text" name="url"
-                                        value="{% abseventurl request.event "presale:event.redeem" %}?voucher={{ voucher.code|urlencode }}{% if voucher.subevent_id %}&subevent={{ voucher.subevent_id }}{% endif %}"
+                                        value="{% humanabseventurl request.event "presale:event.redeem" %}?voucher={{ voucher.code|urlencode }}{% if voucher.subevent_id %}&subevent={{ voucher.subevent_id }}{% endif %}"
                                         class="form-control"
                                         id="id_url" readonly>
                             </div>

src/pretix/control/views/event.py

@@ -1513,7 +1513,7 @@ class EventQRCode(EventPermissionRequiredMixin, View):
     permission = 'can_change_event_settings'
 
     def get(self, request, *args, filetype, **kwargs):
-        url = build_absolute_uri(request.event, 'presale:event.index')
+        url = build_absolute_uri(request.event, 'presale:event.index', human_readable=True)
 
         qr = qrcode.QRCode(
             version=1,

src/pretix/multidomain/templatetags/eventurl.py

@@ -33,9 +33,10 @@ register = template.Library()
 
 
 class EventURLNode(URLNode):
-    def __init__(self, event, view_name, kwargs, asvar, absolute):
+    def __init__(self, event, view_name, kwargs, asvar, absolute, human):
         self.event = event
         self.absolute = absolute
+        self.human = human
         super().__init__(view_name, [], kwargs, asvar)
 
     def render(self, context):
@@ -49,7 +50,7 @@ class EventURLNode(URLNode):
         url = ''
         try:
             if self.absolute:
-                url = build_absolute_uri(event, view_name, kwargs=kwargs)
+                url = build_absolute_uri(event, view_name, kwargs=kwargs, human_readable=self.human)
             else:
                 url = eventreverse(event, view_name, kwargs=kwargs)
         except NoReverseMatch:
@@ -66,7 +67,7 @@ class EventURLNode(URLNode):
 
 
 @register.tag
-def eventurl(parser, token, absolute=False):
+def eventurl(parser, token, absolute=False, human=False):
     """
     Similar to {% url %} in the same way that eventreverse() is similar to reverse().
 
@@ -95,7 +96,7 @@ def eventurl(parser, token, absolute=False):
             else:
                 raise TemplateSyntaxError('Event urls only have keyword arguments.')
 
-    return EventURLNode(event, viewname, kwargs, asvar, absolute)
+    return EventURLNode(event, viewname, kwargs, asvar, absolute, human)
 
 
 @register.tag
@@ -106,3 +107,13 @@ def abseventurl(parser, token):
     Returns an absolute URL.
     """
     return eventurl(parser, token, absolute=True)
+
+
+@register.tag
+def humanabseventurl(parser, token):
+    """
+    Similar to {% url %} in the same way that eventreverse() is similar to reverse().
+
+    Returns an absolute URL that is intended to be read by a human.
+    """
+    return eventurl(parser, token, absolute=True, human=False)

src/pretix/multidomain/urlreverse.py

@@ -32,8 +32,9 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under the License.
 
-from urllib.parse import urljoin, urlsplit
+from urllib.parse import urljoin, urlsplit, urlparse
 
+import idna
 from django.conf import settings
 from django.db.models import Q
 from django.urls import reverse
@@ -171,8 +172,18 @@ def eventreverse(obj, name, kwargs=None):
     return url
 
 
-def build_absolute_uri(obj, urlname, kwargs=None):
+def build_absolute_uri(obj, urlname, kwargs=None, human_readable=True):
     reversedurl = eventreverse(obj, urlname, kwargs)
     if '://' in reversedurl:
         return reversedurl
-    return urljoin(settings.SITE_URL, reversedurl)
+    url = urljoin(settings.SITE_URL, reversedurl)
+
+    if human_readable and 'xn--' in url:
+        try:
+            u = urlparse(url)
+            netloc = idna.encode(u.netloc).decode()
+            url = u._replace(netloc=netloc).geturl()
+        except Exception:
+            pass
+
+    return url

src/pretix/presale/ical.py

@@ -47,11 +47,11 @@ def get_public_ical(events):
         event = ev if isinstance(ev, Event) else ev.event
         tz = pytz.timezone(event.settings.timezone)
         if isinstance(ev, Event):
-            url = build_absolute_uri(event, 'presale:event.index')
+            url = build_absolute_uri(event, 'presale:event.index', human_readable=True)
         else:
             url = build_absolute_uri(event, 'presale:event.index', {
                 'subevent': ev.pk
-            })
+            }, human_readable=True)
 
         vevent = cal.add('vevent')
         vevent.add('summary').value = str(ev.name)
@@ -122,11 +122,11 @@ def get_private_icals(event, positions):
     evs = set(p.subevent or event for p in positions)
     for ev in evs:
         if isinstance(ev, Event):
-            url = build_absolute_uri(event, 'presale:event.index')
+            url = build_absolute_uri(event, 'presale:event.index', human_readable=True)
         else:
             url = build_absolute_uri(event, 'presale:event.index', {
                 'subevent': ev.pk
-            })
+            }, human_readable=True)
 
         if event.settings.mail_attach_ical_description:
             ctx = get_email_context(event=event, event_or_subevent=ev)

src/tests/base/test_rich_text.py

@@ -30,6 +30,21 @@ from pretix.base.templatetags.rich_text import (
     # Test link detection
     ("google.com",
      '<a href="http://google.com" rel="noopener" target="_blank">google.com</a>'),
+    ("https://google.com",
+     '<a href="https://google.com" rel="noopener" target="_blank">https://google.com</a>'),
+    # Test IDNA conversion
+    ("https://xn--dmin-moa0i.com",
+     '<a href="https://xn--dmin-moa0i.com" rel="noopener" target="_blank">https://dömäin.com</a>'),
+    ("xn--dmin-moa0i.com",
+     '<a href="http://xn--dmin-moa0i.com" rel="noopener" target="_blank">dömäin.com</a>'),
+    ("[dömäin.com](https://xn--dmin-moa0i.com)",
+     '<a href="https://xn--dmin-moa0i.com" rel="noopener" target="_blank">dömäin.com</a>'),
+    ("https://dömäin.com",
+     '<a href="https://dömäin.com" rel="noopener" target="_blank">https://dömäin.com</a>'),
+    ("[https://dömäin.com](https://xn--dmin-moa0i.com)",
+     '<a href="https://xn--dmin-moa0i.com" rel="noopener" target="_blank">https://dömäin.com</a>'),
+    ("[xn--dmin-moa0i.com](https://xn--dmin-moa0i.com)",
+     '<a href="https://xn--dmin-moa0i.com" rel="noopener" target="_blank">dömäin.com</a>'),
     # Test abslink_callback
     ("[Call](tel:+12345)",
      '<a href="tel:+12345" rel="nofollow">Call</a>'),
@@ -54,6 +69,10 @@ from pretix.base.templatetags.rich_text import (
      '<a href="https://goodsite.com.evilsite.com" rel="noopener" target="_blank">https://goodsite.com.evilsite.com</a>'),
     ('<a href="https://evilsite.com/deep/path">evilsite.com</a>',
      '<a href="https://evilsite.com/deep/path" rel="noopener" target="_blank">evilsite.com</a>'),
+    ('<a href="https://evilsite.com/deep/path">evilsite.com/deep</a>',
+     '<a href="https://evilsite.com/deep/path" rel="noopener" target="_blank">evilsite.com/deep</a>'),
+    ('<a href="https://evilsite.com/deep/path">evilsite.com/other</a>',
+     '<a href="https://evilsite.com/deep/path" rel="noopener" target="_blank">https://evilsite.com/deep/path</a>'),
     ('<a>broken</a>', '<a>broken</a>'),
 ])
 def test_linkify_abs(link):