Invoice export: Add transmission-specific data (Z#23239732)

* Event calendar: consider events without products "not available" for filtering

Also, drop the help text of the flag that is no longer accurate.

* Update src/pretix/presale/views/organizer.py

Co-authored-by: Richard Schreiber <schreiber@pretix.eu>

---------

Co-authored-by: Richard Schreiber <schreiber@pretix.eu>

src/pretix/base/exporters/invoices.py

@@ -54,6 +54,7 @@ from ...control.forms.filter import get_all_payment_providers
 from ...helpers import GroupConcat
 from ...helpers.iter import chunked_iterable
 from ..exporter import BaseExporter, MultiSheetListExporter
+from ..invoicing.transmission import get_transmission_types
 from ..services.export import ExportError
 from ..services.invoices import invoice_pdf_task
 from ..signals import (
@@ -197,7 +198,7 @@ class InvoiceDataExporter(InvoiceExporterMixin, MultiSheetListExporter):
     def iterate_sheet(self, form_data, sheet):
         _ = gettext
         if sheet == 'invoices':
-            yield [
+            headers = [
                 _('Invoice number'),
                 _('Date'),
                 _('Order code'),
@@ -230,8 +231,18 @@ class InvoiceDataExporter(InvoiceExporterMixin, MultiSheetListExporter):
                 _('Total value (without taxes)'),
                 _('Payment matching IDs'),
                 _('Payment providers'),
+                _('Transmission type'),
+                _('Transmission status'),
+                _('Transmission date'),
             ]
 
+            transmission_types = get_transmission_types()
+            for tt in transmission_types:
+                for c in tt.describe_info_columns():
+                    headers.append(str(tt.verbose_name) + ': ' + str(c))
+
+            yield headers
+
             p_providers = OrderPayment.objects.filter(
                 order=OuterRef('order'),
                 state__in=(OrderPayment.PAYMENT_STATE_CONFIRMED, OrderPayment.PAYMENT_STATE_REFUNDED,
@@ -242,7 +253,7 @@ class InvoiceDataExporter(InvoiceExporterMixin, MultiSheetListExporter):
                 'm'
             ).order_by()
 
-            base_qs = self.invoices_queryset(form_data)\
+            base_qs = self.invoices_queryset(form_data)
 
             qs = base_qs.select_related(
                 'order', 'refers'
@@ -280,7 +291,7 @@ class InvoiceDataExporter(InvoiceExporterMixin, MultiSheetListExporter):
                                 if mid:
                                     pmis.append(mid)
                     pmi = '\n'.join(pmis)
-                    yield [
+                    line = [
                         i.full_invoice_no,
                         date_format(i.date, "SHORT_DATE_FORMAT"),
                         i.order.code,
@@ -315,8 +326,20 @@ class InvoiceDataExporter(InvoiceExporterMixin, MultiSheetListExporter):
                         ', '.join([
                             str(self.providers.get(p, p)) for p in sorted(set((i.payment_providers or '').split(',')))
                             if p and p != 'free'
-                        ])
+                        ]),
+                        i.transmission_type_instance.verbose_name,
+                        i.get_transmission_status_display(),
+                        date_format(i.transmission_date, "SHORT_DATETIME_FORMAT") if i.transmission_date else "",
                     ]
+                    for tt in transmission_types:
+                        if tt.identifier == i.transmission_type:
+                            described = dict(tt.describe_info(i.invoice_to_transmission_info, i.invoice_to_country, i.invoice_to_is_business))
+                            for c in tt.describe_info_columns():
+                                line.append(described.get(c, ""))
+                        else:
+                            for c in tt.describe_info_columns():
+                                line.append("")
+                    yield line
         elif sheet == 'lines':
             yield [
                 _('Invoice number'),

src/pretix/base/forms/questions.py

@@ -899,7 +899,7 @@ class BaseQuestionsForm(forms.Form):
             field.widget.attrs['data-question-dependency-values'] = escapejson_attr(json.dumps(q.dependency_values))
             if q.type != 'M':
                 field.widget.attrs['required'] = q.required and not self.all_optional
-                field._required = q.required and not self.all_optional
+            field._required = q.required and not self.all_optional
             field.required = False
         return field
 
@@ -1497,7 +1497,8 @@ class BaseInvoiceAddressForm(forms.ModelForm):
             pass  # Skip re-validation if it is validated
         elif self.validate_vat_id and vat_id_applicable:
             try:
-                normalized_id = validate_vat_id(data.get('vat_id'), str(data.get('country')))
+                requester_id = self.request.event.settings.invoice_address_from_vat_id
+                normalized_id = validate_vat_id(data.get('vat_id'), str(data.get('country')), requester_id)
                 self.instance.vat_id_validated = bool(normalized_id)
                 self.instance.vat_id = data['vat_id'] = normalized_id
             except VATIDFinalError as e:

src/pretix/base/invoicing/transmission.py

@@ -107,6 +107,9 @@ class TransmissionType:
     def transmission_info_to_form_data(self, transmission_info: dict) -> dict:
         return transmission_info
 
+    def describe_info_columns(self):
+        return [f.label for f in self.invoice_address_form_fields.values()]
+
     def describe_info(self, transmission_info: dict, country: Country, is_business: bool):
         form_data = self.transmission_info_to_form_data(transmission_info)
         data = []

src/pretix/base/services/tax.py

@@ -343,6 +343,66 @@ def _validate_vat_id_EU(vat_id, country_code):
         return vat_id
 
 
+def _validate_vat_id_EU_fallback_germany(vat_id, country_code, requester_id):
+    # We can skip most static validation checks because _validate_vat_id_EU always runs before
+    vat_id = normalize_vat_id(vat_id, country_code)
+
+    # The VIES service of the European commission is overused and down due to rate limits A LOT. There is another
+    # API by German BZSt, but it only works if the requester is German and the requested is not.
+    # https://www.bzst.de/DE/Unternehmen/Identifikationsnummern/Umsatzsteuer-Identifikationsnummer/AuslaendischeUSt-IdNr/auslaendische_ust_idnr_node.html
+    try:
+        r = requests.post(
+            "https://api.evatr.vies.bzst.de/app/v1/abfrage",
+            json={
+                "anfragendeUstid": requester_id,
+                "angefragteUstid": vat_id,
+            },
+            timeout=10,
+        )
+        d = r.json()
+        if r.status_code == 200:
+            if d['status'] in ('evatr-0000', 'evatr-2008'):
+                # evatr-0000: Die angefragte Ust-IdNr. ist zum Anfragezeitpunkt gültig.
+                # evatr-2008: Die angefragte Ust-IdNr. ist zum Anfragezeitpunkt gültig.
+                #             Für die qualifizierte Bestätigungsanfrage liegt einer Besonderheit vor.
+                #             Für Rückfragen wenden Sie sich an das BZSt.
+                return vat_id
+            # evatr-2002: Die angefragte USt-IdNr. ist zum Anfragezeitpunkt nicht gültig.
+            #             Sie ist erst gültig ab dem Datum im Feld gueltigAb.
+            # evatr-2006: Die angefragte Ust-IdNr. ist zum Anfragezeitpunkt nicht gültig.
+            #             Sie war gültig im Zeitraum, der durch die Werte in den Feldern gueltigAb und gueltigBis beschrieben ist.
+            raise VATIDFinalError(error_messages['invalid'])
+        elif r.status_code == 400:
+            if d['status'] in ('evatr-0002', 'evatr-0004', 'evatr-0008'):
+                # evatr-0002: Mindestens eins der Pflichtfelder ist nicht besetzt.
+                # evatr-0004: Die anfragende DE Ust-IdNr. ist syntaktisch falsch. Sie passt nicht in das deutsche Erzeugungsschema.
+                # evatr-0008: Die maximale Anzahl von qualifizierten Bestätigungsabfragen für diese Session wurde erreicht.
+                #             Bitte starten Sie erneut mit einer einfachen Bestätigungsabfrage.
+                raise VATIDTemporaryError(error_messages['unavailable'])
+            # evatr-0005: Die angegebene angefragte Ust-IdNr. ist syntaktisch falsch.
+            # evatr-0012: Die angefrage USt-IdNr. ist syntaktisch falsch. Sie passt nicht in das Erzeugungsschema.
+            # evatr-2003: Das angegebene Länderkennzeichen der angefragten USt-IdNr. ist nicht gültig.
+            raise VATIDFinalError(error_messages['invalid'])
+        elif r.status_code == 403:
+            # evatr-0006: Die anfragende DE USt-IdNr. ist nicht berechtigt eine DE Ust-IdNr. anzufragen.
+            # evatr-0007: Fehlerhafter Aufruf.
+            raise VATIDTemporaryError(error_messages['unavailable'])
+        elif r.status_code == 404:
+            if d['status'] in ('evatr-2005'):
+                # evatr-2005: Die angegebene eigene DE Ust-IdNr. ist zum Anfragezeitpunkt nicht gültig.
+                raise VATIDTemporaryError(error_messages['unavailable'])
+            # evatr-2001: Die angefragte USt-IdNr. ist zum Anfragezeitpunkt nicht vergeben.
+            raise VATIDFinalError(error_messages['invalid'])
+        else:  # 500, 503
+            raise VATIDTemporaryError(error_messages['unavailable'])
+    except requests.RequestException:
+        logger.exception('VAT ID checking failed for country {}'.format(country_code))
+        raise VATIDTemporaryError(error_messages['unavailable'])
+    except ValueError:  # JSON parsing failed
+        logger.exception('VAT ID checking failed for country {}'.format(country_code))
+        raise VATIDTemporaryError(error_messages['unavailable'])
+
+
 def _validate_vat_id_CH(vat_id, country_code):
     if vat_id[:3] != 'CHE':
         raise VATIDFinalError(error_messages['country_mismatch'])
@@ -394,12 +454,18 @@ def _validate_vat_id_CH(vat_id, country_code):
         return vat_id
 
 
-def validate_vat_id(vat_id, country_code):
+def validate_vat_id(vat_id, country_code, requester_id=None):
     if not vat_id:
         return vat_id
     country_code = str(country_code)
     if is_eu_country(country_code):
-        return _validate_vat_id_EU(vat_id, country_code)
+        try:
+            return _validate_vat_id_EU(vat_id, country_code)
+        except VATIDTemporaryError:
+            if requester_id and requester_id.startswith("DE") and not vat_id.startswith("DE"):
+                return _validate_vat_id_EU_fallback_germany(vat_id, country_code, requester_id)
+            else:
+                raise
     elif country_code == 'CH':
         return _validate_vat_id_CH(vat_id, country_code)
     elif country_code == 'NO':

src/pretix/base/settings.py

@@ -1930,8 +1930,6 @@ DEFAULTS = {
         'serializer_class': serializers.BooleanField,
         'form_kwargs': dict(
             label=_("Hide all unavailable dates from calendar or list views"),
-            help_text=_("This option currently only affects the calendar of this event series, not the organizer-wide "
-                        "calendar.")
         )
     },
     'event_calendar_future_only': {

src/pretix/control/views/orders.py

@@ -1646,7 +1646,8 @@ class OrderCheckVATID(OrderView):
                 return redirect(self.get_order_url())
 
             try:
-                normalized_id = validate_vat_id(ia.vat_id, str(ia.country))
+                requester_id = self.request.event.settings.invoice_address_from_vat_id
+                normalized_id = validate_vat_id(ia.vat_id, str(ia.country), requester_id)
                 with transaction.atomic():
                     ia.vat_id_validated = True
                     ia.vat_id = normalized_id

src/pretix/presale/views/event.py

@@ -79,7 +79,7 @@ from pretix.presale.signals import seatingframe_html_head
 from pretix.presale.views.organizer import (
     EventListMixin, add_subevents_for_days, days_for_template,
     filter_qs_by_attr, filter_subevents_with_plugins, has_before_after,
-    weeks_for_template,
+    should_hide_subevent, weeks_for_template,
 )
 
 from . import (
@@ -443,12 +443,10 @@ class EventIndex(EventViewMixin, EventListMixin, CartMixin, TemplateView):
                 )
             )
             subevents = filter_subevents_with_plugins(list(subevents), self.request.sales_channel)
-            context['subevent_list'] = subevents
-            if self.request.event.settings.event_list_available_only and not voucher:
-                context['subevent_list'] = [
-                    se for se in subevents
-                    if not se.presale_has_ended and (se.best_availability_state is None or se.best_availability_state >= Quota.AVAILABILITY_RESERVED)
-                ]
+            context['subevent_list'] = [
+                se for se in subevents
+                if not should_hide_subevent(self.request.event.settings, se, voucher)
+            ]
             context['visible_events'] = len(subevents) > 0
         return context
 

src/pretix/presale/views/organizer.py

@@ -601,6 +601,32 @@ def filter_subevents_with_plugins(subevents, sales_channel=None):
     return subevents
 
 
+def should_hide_subevent(settings, subevent, voucher=None):
+    hide = False
+    if settings.event_list_available_only:
+        hide = (
+            # Presale is over → the subevent is not available → hide
+            subevent.presale_has_ended or
+            # Not a single product is available on this sales channel → hide
+            # Note that means there could be products which are ignored for calendar availability (Quota.ignore_for_event_availability)
+            # or products only visible with a voucher. However, for customers with these scenarios, the event_list_available_only
+            # makes only very little sense as it would never do anything, so the flag can just be removed -- or the products should
+            # be made visible so people know why there are no products. In case a voucher is already entered on the calendar view,
+            # this is already respected and subevents are shown correctly.
+            subevent.best_availability_state is None or
+            (
+                # Sold out → hide, unless we have a voucher active that can bypass all quotas
+                (not voucher or not voucher.allow_ignore_quota) and
+                subevent.best_availability_state < Quota.AVAILABILITY_RESERVED
+            )
+        )
+
+    if settings.event_calendar_future_only:
+        if (subevent.date_to or subevent.date_from) < time_machine_now():
+            hide = True
+    return hide
+
+
 def add_subevents_for_days(qs, before, after, ebd, timezones, sales_channel, event=None, cart_namespace=None,
                            voucher=None):
     qs = qs.filter(active=True, is_public=True).filter(
@@ -640,19 +666,8 @@ def add_subevents_for_days(qs, before, after, ebd, timezones, sales_channel, eve
             kwargs['cart_namespace'] = cart_namespace
 
         s = event.settings if event else se.event.settings
-
-        if s.event_list_available_only:
-            hide = se.presale_has_ended or (
-                (not voucher or not voucher.allow_ignore_quota) and
-                se.best_availability_state is not None and
-                se.best_availability_state < Quota.AVAILABILITY_RESERVED
-            )
-            if hide:
-                continue
-
-        if s.event_calendar_future_only:
-            if (se.date_to or se.date_from) < time_machine_now():
-                continue
+        if should_hide_subevent(s, se, voucher):
+            continue
 
         timezones.add(s.timezone)
         tz = ZoneInfo(s.timezone)

src/pretix/presale/views/widget.py

@@ -75,7 +75,7 @@ from pretix.presale.views.cart import get_or_create_cart_id
 from pretix.presale.views.organizer import (
     EventListMixin, add_events_for_days, add_subevents_for_days,
     days_for_template, filter_qs_by_attr, filter_subevents_with_plugins,
-    weeks_for_template,
+    should_hide_subevent, weeks_for_template,
 )
 
 logger = logging.getLogger(__name__)
@@ -757,14 +757,10 @@ class WidgetAPIProductList(EventListMixin, View):
                         evs = evs[:limit]
 
                 tz = request.event.timezone
-                if self.request.event.settings.event_list_available_only:
-                    evs = [
-                        se for se in evs
-                        if not se.presale_has_ended and (
-                            se.best_availability_state is not None and
-                            se.best_availability_state >= Quota.AVAILABILITY_RESERVED
-                        )
-                    ]
+                evs = [
+                    se for se in evs
+                    if not should_hide_subevent(self.request.event.settings, se)
+                ]
 
                 data['events'] = [
                     {

src/pretix/static/pretixpresale/js/widget/widget.js

@@ -345,7 +345,7 @@ Vue.component('pricebox', {
         + '       :min="display_price_nonlocalized" :value="suggested_price_nonlocalized" :name="field_name"'
         + '       step="any" v-bind:aria-labelledby="aria_labelledby" v-bind:aria-describedby="price_desc_id">'
         + '</div>'
-        + '<small class="pretix-widget-pricebox-tax" :id="price_desc_id" v-if="price.rate != \'0\' && price.gross != \'0.00\'">'
+        + '<small class="pretix-widget-pricebox-tax" :id="price_desc_id" v-if="show_taxline">'
         + '{{ taxline }}'
         + '</small>'
         + '</div>'),
@@ -422,6 +422,10 @@ Vue.component('pricebox', {
                 return '<span class="pretix-widget-pricebox-currency">' + this.$root.currency + "</span> " + this.display_price;
             }
         },
+        show_taxline: function () {
+            // rate can either be "0.00" or "0" => parseFloat to check
+            return Number.parseFloat(this.price.rate) && Number.parseFloat(this.price.gross);
+        },
         taxline: function () {
             if (this.$root.display_net_prices) {
                 if (this.price.includes_mixed_tax_rate) {

src/pretix/static/pretixpresale/widget/src/components/PriceBox.vue

@@ -86,7 +86,8 @@ const taxline = computed(() => {
 	}
 })
 
-const showTaxline = computed(() => props.price.rate !== '0' && props.price.gross !== '0.00')
+// rate can either be "0.00" or "0" => parseFloat to check
+const showTaxline = computed(() => Number.parseFloat(props.price.rate) && Number.parseFloat(props.price.gross))
 </script>
 <template lang="pug">
 .pretix-widget-pricebox