Support for IDNA domain names

PRETIXEU-8DX)

.gitlab-ci.yml

@@ -22,13 +22,14 @@ pypi:
         - source env/bin/activate
         - pip install -U pip wheel setuptools check-manifest twine
         - XDG_CACHE_HOME=/cache pip3 install -e ".[dev]"
-        - cd src
         - python setup.py sdist
         - pip install dist/pretix-*.tar.gz
         - python -m pretix migrate
         - python -m pretix check
-        - check-manifest
+        - cd src
         - make npminstall
+        - cd ..
+        - check-manifest
         - python setup.py sdist bdist_wheel
         - twine check dist/*
         - twine upload dist/*

MANIFEST.in

@@ -1,5 +1,6 @@
 include LICENSE
 include README.rst
+include src/Makefile
 global-include *.proto
 recursive-include src/pretix/static *
 recursive-include src/pretix/static.dist *
@@ -31,3 +32,16 @@ recursive-include src/pretix/plugins/returnurl/templates *
 recursive-include src/pretix/plugins/returnurl/static *
 recursive-include src/pretix/plugins/webcheckin/templates *
 recursive-include src/pretix/plugins/webcheckin/static *
+ recursive-include src *.cfg
+ recursive-include src *.csv
+ recursive-include src *.gitkeep
+ recursive-include src *.jpg
+ recursive-include src *.json
+ recursive-include src *.py
+ recursive-include src *.svg
+ recursive-include src *.txt
+ recursive-include src Makefile
+
+recursive-exclude doc *
+recursive-exclude deployment *
+recursive-exclude res *

pyproject.toml

@@ -62,6 +62,7 @@ dependencies = [
   "drf_ujson2==1.7.*",
   "geoip2==4.*",
   "importlib_metadata==6.6.*",  # Polyfill, we can probably drop this once we require Python 3.10+
+  "idna",
   "isoweek",
   "jsonschema",
   "kombu==5.2.*",

setup.cfg

@@ -0,0 +1,40 @@
+[check-manifest]
+ignore =
+    env/**
+    doc/*
+    deployment/*
+    res/*
+    src/.update-locales
+    src/Makefile
+    src/manage.py
+    src/pretix/icons/*
+    src/pretix/static.dist/**
+    src/pretix/static/jsi18n/**
+    src/requirements.txt
+    src/requirements/*
+    src/tests/*
+    src/tests/api/*
+    src/tests/base/*
+    src/tests/control/*
+    src/tests/testdummy/*
+    src/tests/templates/*
+    src/tests/presale/*
+    src/tests/doc/*
+    src/tests/helpers/*
+    src/tests/media/*
+    src/tests/multidomain/*
+    src/tests/plugins/*
+    src/tests/plugins/badges/*
+    src/tests/plugins/banktransfer/*
+    src/tests/plugins/paypal/*
+    src/tests/plugins/paypal2/*
+    src/tests/plugins/pretixdroid/*
+    src/tests/plugins/stripe/*
+    src/tests/plugins/sendmail/*
+    src/tests/plugins/ticketoutputpdf/*
+    .*
+    CODE_OF_CONDUCT.md
+    CONTRIBUTING.md
+    Dockerfile
+    SECURITY.md
+

src/pretix/__init__.py

@@ -19,4 +19,4 @@
 # You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
 # <https://www.gnu.org/licenses/>.
 #
-__version__ = "4.19.0"
+__version__ = "4.20.0.dev0"

src/pretix/base/email.py

@@ -420,7 +420,7 @@ def base_placeholders(sender, **kwargs):
                     'order': 'F8VVL',
                     'secret': '6zzjnumtsx136ddy',
                     'hash': '98kusd8ofsj8dnkd'
-                }
+                },
             ),
         ),
         SimpleFunctionalMailTextPlaceholder(

src/pretix/base/models/orders.py

@@ -830,7 +830,7 @@ class Order(LockModel, LoggedModel):
     @property
     def is_expired_by_time(self):
         return (
-            self.status == Order.STATUS_PENDING and self.expires < now()
+            self.status == Order.STATUS_PENDING and not self.require_approval and self.expires < now()
             and not self.event.settings.get('payment_term_expire_automatically')
         )
 

src/pretix/base/templatetags/rich_text.py

@@ -36,6 +36,7 @@ import re
 import urllib.parse
 
 import bleach
+import idna
 import markdown
 from bleach import DEFAULT_CALLBACKS
 from bleach.linkifier import build_email_re, build_url_re
@@ -121,6 +122,12 @@ def safelink_callback(attrs, new=False):
     return attrs
 
 
+def idna_decode_safe(src):
+    v = idna.decode(src)
+
+    return v
+
+
 def truelink_callback(attrs, new=False):
     """
     Tries to prevent "phishing" attacks in which a link looks like it points to a safe place but instead
@@ -136,20 +143,40 @@ def truelink_callback(attrs, new=False):
 
         <a href="https://maps.google.com/location/foo">https://maps.google.com</a>
     """
-    text = re.sub(r'[^a-zA-Z0-9.\-/_ ]', '', attrs.get('_text'))  # clean up link text
+    text = re.sub(r'[^a-zA-Z0-9:.\-/_ ]', '', attrs.get('_text'))  # clean up link text
     url = attrs.get((None, 'href'), '/')
     href_url = urllib.parse.urlparse(url)
+    strip_http = False
     if (None, 'href') in attrs and URL_RE.match(text) and href_url.scheme not in ('tel', 'mailto'):
+        if URL_RE.match(attrs.get('_text').strip()):  # maybe we cleaned up too much
+            text = attrs.get('_text').strip()
         # link text looks like a url
         if text.startswith('//'):
             text = 'https:' + text
         elif not text.startswith('http'):
+            strip_http = True
             text = 'https://' + text
 
         text_url = urllib.parse.urlparse(text)
-        if text_url.netloc != href_url.netloc or not href_url.path.startswith(href_url.path):
+
+        if href_url.netloc.startswith('xn--'):
+            href_netloc_nice = idna_decode_safe(href_url.netloc)
+        else:
+            href_netloc_nice = href_url.netloc
+
+        if text_url.netloc not in (href_url.netloc, href_netloc_nice) or not href_url.path.startswith(text_url.path):
             # link text contains an URL that has a different base than the actual URL
             attrs['_text'] = attrs[None, 'href']
+            text_url = href_url
+
+        if text_url.netloc.startswith('xn--'):
+            # Show punicode nicely
+            text_netloc_nice = idna_decode_safe(text_url.netloc)
+            url = text_url._replace(netloc=text_netloc_nice, scheme=href_url.scheme).geturl()
+            if strip_http:
+                url = url[len(href_url.scheme + '://'):]
+            attrs['_text'] = url
+
     return attrs
 
 

src/pretix/control/templates/pretixcontrol/boxoffice/payment.html

@@ -13,20 +13,25 @@
     {% elif payment_info.payment_type == "terminal_zvt" %}
         <dt>{% trans "Payment provider" %}</dt>
         <dd>{% trans "ZVT Terminal" %}</dd>
-        <dt>{% trans "Trace number" context "terminal_zvt" %}</dt>
-        <dd>{{ payment_info.payment_data.traceNumber }}</dd>
-        <dt>{% trans "Payment type" context "terminal_zvt" %}</dt>
-        <dd>{{ payment_info.payment_data.paymentType }}</dd>
-        <dt>{% trans "Additional text" context "terminal_zvt" %}</dt>
-        <dd>{{ payment_info.payment_data.additionalText }}</dd>
-        <dt>{% trans "Turnover number" context "terminal_zvt" %}</dt>
-        <dd>{{ payment_info.payment_data.turnoverNumber }}</dd>
-        <dt>{% trans "Receipt number" context "terminal_zvt" %}</dt>
-        <dd>{{ payment_info.payment_data.receiptNumber }}</dd>
-        <dt>{% trans "Card type" context "terminal_zvt" %}</dt>
-        <dd>{{ payment_info.payment_data.cardName|default_if_none:payment_info.payment_data.cardType }}</dd>
-        <dt>{% trans "Card expiration" context "terminal_zvt" %}</dt>
-        <dd>{{ payment_info.payment_data.expiry }}</dd>
+        {% if payment_info.payment_data.source == "manual" %}
+            <dt>{% trans "Confirmation mode" context "terminal_zvt" %}</dt>
+            <dd>{{ payment_info.payment_data.source }}</dd>
+        {% else %}
+            <dt>{% trans "Trace number" context "terminal_zvt" %}</dt>
+            <dd>{{ payment_info.payment_data.traceNumber }}</dd>
+            <dt>{% trans "Payment type" context "terminal_zvt" %}</dt>
+            <dd>{{ payment_info.payment_data.paymentType }}</dd>
+            <dt>{% trans "Additional text" context "terminal_zvt" %}</dt>
+            <dd>{{ payment_info.payment_data.additionalText }}</dd>
+            <dt>{% trans "Turnover number" context "terminal_zvt" %}</dt>
+            <dd>{{ payment_info.payment_data.turnoverNumber }}</dd>
+            <dt>{% trans "Receipt number" context "terminal_zvt" %}</dt>
+            <dd>{{ payment_info.payment_data.receiptNumber }}</dd>
+            <dt>{% trans "Card type" context "terminal_zvt" %}</dt>
+            <dd>{{ payment_info.payment_data.cardName|default_if_none:payment_info.payment_data.cardType }}</dd>
+            <dt>{% trans "Card expiration" context "terminal_zvt" %}</dt>
+            <dd>{{ payment_info.payment_data.expiry }}</dd>
+        {% endif %}
     {% elif payment_info.payment_type == "sumup" %}
         <dt>{% trans "Payment provider" %}</dt>
         <dd>SumUp</dd>

src/pretix/control/templates/pretixcontrol/event/index.html

@@ -18,7 +18,7 @@
     </h1>
     <div class="helper-space-below">
         {% trans "Shop URL:" %}
-        <span id="shop_url" class="text-muted">{% abseventurl request.event "presale:event.index" %}</span>
+        <span id="shop_url" class="text-muted">{% humanabseventurl request.event "presale:event.index" %}</span>
         <button type="button" class="btn btn-default btn-xs btn-clipboard js-only" data-clipboard-target="#shop_url">
             <i class="fa fa-clipboard" aria-hidden="true"></i>
             <span class="sr-only">{% trans "Copy to clipboard" %}</span>

src/pretix/control/templates/pretixcontrol/vouchers/detail.html

@@ -43,7 +43,7 @@
                             <label class="col-md-3 control-label" for="id_url">{% trans "Voucher link" %}</label>
                             <div class="col-md-9">
                                 <input type="text" name="url"
-                                        value="{% abseventurl request.event "presale:event.redeem" %}?voucher={{ voucher.code|urlencode }}{% if voucher.subevent_id %}&subevent={{ voucher.subevent_id }}{% endif %}"
+                                        value="{% humanabseventurl request.event "presale:event.redeem" %}?voucher={{ voucher.code|urlencode }}{% if voucher.subevent_id %}&subevent={{ voucher.subevent_id }}{% endif %}"
                                         class="form-control"
                                         id="id_url" readonly>
                             </div>

src/pretix/control/views/event.py

@@ -1513,7 +1513,7 @@ class EventQRCode(EventPermissionRequiredMixin, View):
     permission = 'can_change_event_settings'
 
     def get(self, request, *args, filetype, **kwargs):
-        url = build_absolute_uri(request.event, 'presale:event.index')
+        url = build_absolute_uri(request.event, 'presale:event.index', human_readable=True)
 
         qr = qrcode.QRCode(
             version=1,

src/pretix/multidomain/templatetags/eventurl.py

@@ -33,9 +33,10 @@ register = template.Library()
 
 
 class EventURLNode(URLNode):
-    def __init__(self, event, view_name, kwargs, asvar, absolute):
+    def __init__(self, event, view_name, kwargs, asvar, absolute, human):
         self.event = event
         self.absolute = absolute
+        self.human = human
         super().__init__(view_name, [], kwargs, asvar)
 
     def render(self, context):
@@ -49,7 +50,7 @@ class EventURLNode(URLNode):
         url = ''
         try:
             if self.absolute:
-                url = build_absolute_uri(event, view_name, kwargs=kwargs)
+                url = build_absolute_uri(event, view_name, kwargs=kwargs, human_readable=self.human)
             else:
                 url = eventreverse(event, view_name, kwargs=kwargs)
         except NoReverseMatch:
@@ -66,7 +67,7 @@ class EventURLNode(URLNode):
 
 
 @register.tag
-def eventurl(parser, token, absolute=False):
+def eventurl(parser, token, absolute=False, human=False):
     """
     Similar to {% url %} in the same way that eventreverse() is similar to reverse().
 
@@ -95,7 +96,7 @@ def eventurl(parser, token, absolute=False):
             else:
                 raise TemplateSyntaxError('Event urls only have keyword arguments.')
 
-    return EventURLNode(event, viewname, kwargs, asvar, absolute)
+    return EventURLNode(event, viewname, kwargs, asvar, absolute, human)
 
 
 @register.tag
@@ -106,3 +107,13 @@ def abseventurl(parser, token):
     Returns an absolute URL.
     """
     return eventurl(parser, token, absolute=True)
+
+
+@register.tag
+def humanabseventurl(parser, token):
+    """
+    Similar to {% url %} in the same way that eventreverse() is similar to reverse().
+
+    Returns an absolute URL that is intended to be read by a human.
+    """
+    return eventurl(parser, token, absolute=True, human=False)

src/pretix/multidomain/urlreverse.py

@@ -32,8 +32,9 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under the License.
 
-from urllib.parse import urljoin, urlsplit
+from urllib.parse import urljoin, urlsplit, urlparse
 
+import idna
 from django.conf import settings
 from django.db.models import Q
 from django.urls import reverse
@@ -171,8 +172,18 @@ def eventreverse(obj, name, kwargs=None):
     return url
 
 
-def build_absolute_uri(obj, urlname, kwargs=None):
+def build_absolute_uri(obj, urlname, kwargs=None, human_readable=True):
     reversedurl = eventreverse(obj, urlname, kwargs)
     if '://' in reversedurl:
         return reversedurl
-    return urljoin(settings.SITE_URL, reversedurl)
+    url = urljoin(settings.SITE_URL, reversedurl)
+
+    if human_readable and 'xn--' in url:
+        try:
+            u = urlparse(url)
+            netloc = idna.encode(u.netloc).decode()
+            url = u._replace(netloc=netloc).geturl()
+        except Exception:
+            pass
+
+    return url

src/pretix/plugins/paypal2/templates/pretixplugins/paypal2/control.html

@@ -21,6 +21,17 @@
             {% endfor %}
         {% endif %}
         {% if payment_info.purchase_units.0.payments %}
+            <dt>{% trans "Capture status" %}</dt>
+            <dd>
+                {% if payment_info.purchase_units.0.payments.captures.0.status_details.reason in "PENDING_REVIEW, BUYER_COMPLAINT" %}
+                    <span class="fa fa-warning fa-danger" data-toggle="tooltip" title="{% trans "This payment is being reviewed by PayPal. Until the review is lifted, the money will not be disbursed and the order remain in its pending state." %}"></span>
+                {% endif %}
+                {{ payment_info.purchase_units.0.payments.captures.0.status }}
+                {% if payment_info.purchase_units.0.payments.captures.0.status_details.reason %}
+                    ({{ payment_info.purchase_units.0.payments.captures.0.status_details.reason }})
+                {% endif %}
+            </dd>
+
             <dt>{% trans "Last update" %}</dt>
             <dd>{{ payment_info.purchase_units.0.payments.captures.0.update_time }}</dd>
             <dt>{% trans "Total value" %}</dt>

src/pretix/presale/ical.py

@@ -47,11 +47,11 @@ def get_public_ical(events):
         event = ev if isinstance(ev, Event) else ev.event
         tz = pytz.timezone(event.settings.timezone)
         if isinstance(ev, Event):
-            url = build_absolute_uri(event, 'presale:event.index')
+            url = build_absolute_uri(event, 'presale:event.index', human_readable=True)
         else:
             url = build_absolute_uri(event, 'presale:event.index', {
                 'subevent': ev.pk
-            })
+            }, human_readable=True)
 
         vevent = cal.add('vevent')
         vevent.add('summary').value = str(ev.name)
@@ -122,11 +122,11 @@ def get_private_icals(event, positions):
     evs = set(p.subevent or event for p in positions)
     for ev in evs:
         if isinstance(ev, Event):
-            url = build_absolute_uri(event, 'presale:event.index')
+            url = build_absolute_uri(event, 'presale:event.index', human_readable=True)
         else:
             url = build_absolute_uri(event, 'presale:event.index', {
                 'subevent': ev.pk
-            })
+            }, human_readable=True)
 
         if event.settings.mail_attach_ical_description:
             ctx = get_email_context(event=event, event_or_subevent=ev)

src/pretix/presale/templates/pretixpresale/event/checkout_base.html

@@ -13,7 +13,7 @@
 {% endblock %}
 {% block content %}
 <aside aria-label="{% trans "Your cart" %}">
-    <details class="panel panel-default cart" {% if "open_cart" in request.GET %}open{% endif %}>
+    <details class="panel panel-default cart{% if "open_cart" not in request.GET %} sneak-peek{% endif %}" {% if "open_cart" in request.GET %}open{% endif %}>
         <summary class="panel-heading">
             <h2 class="panel-title">
                 <span>
@@ -32,6 +32,11 @@
                 </span>
             </h2>
         </summary>
+        {% if "open_cart" not in request.GET %}
+        <p class="sneak-peek-trigger">
+            <button type="button" class="btn btn-default">{% trans "Show full cart" %}</button>
+        </p>
+        {% endif %}
         <div>
             <div class="panel-body">
                 {% include "pretixpresale/event/fragment_cart.html" with cart=cart event=request.event %}

src/pretix/static/pretixbase/js/asynctask.js

@@ -20,7 +20,7 @@ function async_task_check() {
     );
 }
 
-function async_task_check_callback(data, jqXHR, status) {
+function async_task_check_callback(data, textStatus, jqXHR) {
     "use strict";
     if (data.ready && data.redirect) {
         if (async_task_is_download && data.success) {

src/pretix/static/pretixbase/js/details.js

@@ -7,6 +7,10 @@ setup_collapsible_details = function (el) {
             return true;
         }
         var $details = $(this).closest("details");
+        if ($details.hasClass('sneak-peek')) {
+            // if sneak-peek is active, needs to be handled differently
+            return true;
+        }
         var isOpen = $details.prop("open");
         var $detailsNotSummary = $details.children(':not(summary)');
         if ($detailsNotSummary.is(':animated')) {
@@ -27,6 +31,10 @@ setup_collapsible_details = function (el) {
         e.preventDefault();
         return false;
     }).keyup(function (event) {
+        if ($details.hasClass('sneak-peek')) {
+            // if sneak-peek is active, needs to be handled differently
+            return true;
+        }
         if (32 == event.keyCode || (13 == event.keyCode && !isOpera)) {
             // Space or Enter is pressed — trigger the `click` event on the `summary` element
             // Opera already seems to trigger the `click` event when Enter is pressed

src/pretix/static/pretixpresale/js/ui/main.js

@@ -288,6 +288,30 @@ $(function () {
 
     $("#ajaxerr").on("click", ".ajaxerr-close", ajaxErrDialog.hide);
 
+    $('details.sneak-peek:not([open])').each(function() {
+        this.open = true;
+        var $elements = $("> :not(summary)", this).show().filter(':not(.sneak-peek-trigger)').attr('aria-hidden', 'true');
+
+        var container = this;
+        var trigger = $('summary, .sneak-peek-trigger button', container);
+        function onclick(e) {
+            e.preventDefault();
+
+            container.addEventListener('transitionend', function() {
+                $(container).removeClass('sneak-peek');
+                container.style.removeProperty('height');
+            }, {once: true});
+            container.style.height = container.scrollHeight + 'px';
+            $('.sneak-peek-trigger', container).fadeOut(function() {
+                $(this).remove();
+            });
+            $elements.removeAttr('aria-hidden');
+
+            trigger.off('click', onclick);
+        }
+        trigger.on('click', onclick);
+    });
+
     // Copy answers
     $(".js-copy-answers").click(function (e) {
         e.preventDefault();

src/pretix/static/pretixpresale/scss/main.scss

@@ -398,6 +398,27 @@ details.details-open .panel-heading .collapse-indicator,
     transform: rotate(180deg);
 }
 
+details.sneak-peek {
+    position: relative;
+    height: 11em;
+    overflow: hidden;
+    transition: height .5s;
+}
+.sneak-peek-trigger {
+    display: grid;
+    justify-content: center;
+    align-content: center;
+    width: 100%;
+    height: 4.5em;
+    margin: 0;
+    padding: 15px;
+    background: linear-gradient(0deg, rgba(248,248,248,.9) 44%, rgba(248,248,248,0) 100%);
+    position: absolute;
+    bottom: 0;
+    left: 0;
+    z-index: 10;
+}
+
 form.download-btn-form {
     display: inline;
 }

src/setup.cfg

@@ -93,34 +93,3 @@ phrases =
     Stripe Connect
 chunkers = enchant.tokenize.HTMLChunker
 filters = PythonFormatFilter,enchant.tokenize.URLFilter,HTMLFilter
-
-[check-manifest]
-ignore =
-    .update-locales
-    Makefile
-    manage.py
-    pretix/icons/*
-	pretix/static.dist/**
-    pretix/static/jsi18n/**
-    requirements.txt
-    requirements/*
-    tests/*
-    tests/api/*
-	tests/base/*
-	tests/control/*
-	tests/testdummy/*
-	tests/templates/*
-	tests/presale/*
-	tests/doc/*
-	tests/helpers/*
-	tests/media/*
-	tests/multidomain/*
-	tests/plugins/*
-	tests/plugins/badges/*
-	tests/plugins/banktransfer/*
-	tests/plugins/paypal/*
-	tests/plugins/paypal2/*
-	tests/plugins/pretixdroid/*
-	tests/plugins/stripe/*
-	tests/plugins/sendmail/*
-	tests/plugins/ticketoutputpdf/*

src/tests/base/test_rich_text.py

@@ -30,6 +30,21 @@ from pretix.base.templatetags.rich_text import (
     # Test link detection
     ("google.com",
      '<a href="http://google.com" rel="noopener" target="_blank">google.com</a>'),
+    ("https://google.com",
+     '<a href="https://google.com" rel="noopener" target="_blank">https://google.com</a>'),
+    # Test IDNA conversion
+    ("https://xn--dmin-moa0i.com",
+     '<a href="https://xn--dmin-moa0i.com" rel="noopener" target="_blank">https://dömäin.com</a>'),
+    ("xn--dmin-moa0i.com",
+     '<a href="http://xn--dmin-moa0i.com" rel="noopener" target="_blank">dömäin.com</a>'),
+    ("[dömäin.com](https://xn--dmin-moa0i.com)",
+     '<a href="https://xn--dmin-moa0i.com" rel="noopener" target="_blank">dömäin.com</a>'),
+    ("https://dömäin.com",
+     '<a href="https://dömäin.com" rel="noopener" target="_blank">https://dömäin.com</a>'),
+    ("[https://dömäin.com](https://xn--dmin-moa0i.com)",
+     '<a href="https://xn--dmin-moa0i.com" rel="noopener" target="_blank">https://dömäin.com</a>'),
+    ("[xn--dmin-moa0i.com](https://xn--dmin-moa0i.com)",
+     '<a href="https://xn--dmin-moa0i.com" rel="noopener" target="_blank">dömäin.com</a>'),
     # Test abslink_callback
     ("[Call](tel:+12345)",
      '<a href="tel:+12345" rel="nofollow">Call</a>'),
@@ -54,6 +69,10 @@ from pretix.base.templatetags.rich_text import (
      '<a href="https://goodsite.com.evilsite.com" rel="noopener" target="_blank">https://goodsite.com.evilsite.com</a>'),
     ('<a href="https://evilsite.com/deep/path">evilsite.com</a>',
      '<a href="https://evilsite.com/deep/path" rel="noopener" target="_blank">evilsite.com</a>'),
+    ('<a href="https://evilsite.com/deep/path">evilsite.com/deep</a>',
+     '<a href="https://evilsite.com/deep/path" rel="noopener" target="_blank">evilsite.com/deep</a>'),
+    ('<a href="https://evilsite.com/deep/path">evilsite.com/other</a>',
+     '<a href="https://evilsite.com/deep/path" rel="noopener" target="_blank">https://evilsite.com/deep/path</a>'),
     ('<a>broken</a>', '<a>broken</a>'),
 ])
 def test_linkify_abs(link):