Database consistency checks

src/pretix/base/apps.py

@@ -46,7 +46,7 @@ class PretixBaseConfig(AppConfig):
         from . import invoice  # NOQA
         from . import notifications  # NOQA
         from . import email  # NOQA
-        from .services import auth, checkin, export, mail, tickets, cart, orderimport, orders, invoices, cleanup, update_check, quotas, notifications, vouchers  # NOQA
+        from .services import auth, checkin, checks, export, mail, tickets, cart, orderimport, orders, invoices, cleanup, update_check, quotas, notifications, vouchers  # NOQA
         from .models import _transactions  # NOQA
         from django.conf import settings
 

src/pretix/base/migrations/0201_check.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.4 on 2021-10-29 09:58
+
+from django.db import migrations, models
+
+import pretix.base.models.base
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pretixbase', '0200_transaction'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Check',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('result', models.CharField(max_length=190)),
+                ('check_type', models.CharField(max_length=190)),
+                ('log', models.TextField()),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(models.Model, pretix.base.models.base.LoggingMixin),
+        ),
+    ]

src/pretix/base/models/__init__.py

@@ -23,6 +23,7 @@ from ..settings import GlobalSettingsObject_SettingsStore
 from .auth import U2FDevice, User, WebAuthnDevice
 from .base import CachedFile, LoggedModel, cachedfile_name
 from .checkin import Checkin, CheckinList
+from .checks import Check
 from .customers import Customer
 from .devices import Device, Gate
 from .event import (

src/pretix/base/models/checks.py

@@ -0,0 +1,54 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020 Raphael Michel and contributors
+# Copyright (C) 2020-2021 rami.io GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+
+# This file is based on an earlier version of pretix which was released under the Apache License 2.0. The full text of
+# the Apache License 2.0 can be obtained at <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# This file may have since been changed and any changes are released under the terms of AGPLv3 as described above. A
+# full history of changes and contributors is available at <https://github.com/pretix/pretix>.
+#
+# This file contains Apache-licensed contributions copyrighted by: Jakob Schnell
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the Apache License 2.0 is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under the License.
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+from pretix.base.models import LoggedModel
+
+
+class Check(LoggedModel):
+    RESULT_OK = 'ok'
+    RESULT_WARNING = 'warning'
+    RESULT_ERROR = 'error'
+
+    RESULTS = (
+        (RESULT_OK, _('OK')),
+        (RESULT_WARNING, _('Warning')),
+        (RESULT_ERROR, _('Error')),
+    )
+
+    created = models.DateTimeField(auto_now_add=True)
+    result = models.CharField(max_length=190, choices=RESULTS)
+    check_type = models.CharField(max_length=190)
+    log = models.TextField()

src/pretix/base/services/checks.py

@@ -0,0 +1,191 @@
+#
+# This file is part of pretix (Community Edition).
+#
+# Copyright (C) 2014-2020 Raphael Michel and contributors
+# Copyright (C) 2020-2021 rami.io GmbH and contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
+# Public License as published by the Free Software Foundation in version 3 of the License.
+#
+# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
+# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
+# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
+# this file, see <https://pretix.eu/about/en/license>.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+#
+import logging
+from datetime import timedelta
+
+from django.db import models
+from django.db.models import Case, F, OuterRef, Q, Subquery, Sum, Value, When
+from django.db.models.functions import Cast, Coalesce, StrIndex, Substr
+from django.dispatch import receiver
+from django.utils.timezone import now
+from django_scopes import scopes_disabled
+
+from pretix.base.models import Check, Invoice, Order, OrderFee, OrderPosition
+from pretix.base.models.orders import Transaction
+from pretix.base.signals import periodic_task
+from pretix.celery_app import app
+from pretix.helpers.periodic import minimum_interval
+
+logger = logging.getLogger(__name__)
+
+
+def check_order_transactions():
+    qs = Order.objects.annotate(
+        position_total=Coalesce(
+            Subquery(
+                OrderPosition.objects.filter(
+                    order=OuterRef('pk')
+                ).order_by().values('order').annotate(p=Sum('price')).values('p'),
+                output_field=models.DecimalField(decimal_places=2, max_digits=10)
+            ), Value(0), output_field=models.DecimalField(decimal_places=2, max_digits=10)
+        ),
+        fee_total=Coalesce(
+            Subquery(
+                OrderFee.objects.filter(
+                    order=OuterRef('pk')
+                ).order_by().values('order').annotate(p=Sum('value')).values('p'),
+                output_field=models.DecimalField(decimal_places=2, max_digits=10)
+            ), Value(0), output_field=models.DecimalField(decimal_places=2, max_digits=10)
+        ),
+        tx_total=Coalesce(
+            Subquery(
+                Transaction.objects.filter(
+                    order=OuterRef('pk')
+                ).order_by().values('order').annotate(p=Sum(F('price') * F('count'))).values('p'),
+                output_field=models.DecimalField(decimal_places=2, max_digits=10)
+            ), Value(0), output_field=models.DecimalField(decimal_places=2, max_digits=10)
+        ),
+    ).annotate(
+        correct_total=Case(
+            When(Q(status=Order.STATUS_CANCELED) | Q(status=Order.STATUS_EXPIRED) | Q(require_approval=True),
+                 then=0),
+            default=F('position_total') + F('fee_total'),
+            output_field=models.DecimalField(decimal_places=2, max_digits=10)
+        ),
+    ).exclude(
+        tx_total=F('correct_total')
+    ).select_related('event')
+    for o in qs:
+        yield [
+            Check.RESULT_ERROR,
+            f'Order {o.full_code} has a wrong total: Status is {o.status} and sum of positions and fees is '
+            f'{o.position_total + o.fee_total}, so sum of transactions should be {o.correct_total} but is {o.tx_total}'
+        ]
+    yield [
+        Check.RESULT_OK,
+        'Check completed.'
+    ]
+
+
+def check_order_total():
+    qs = Order.objects.annotate(
+        position_total=Coalesce(
+            Subquery(
+                OrderPosition.objects.filter(
+                    order=OuterRef('pk')
+                ).order_by().values('order').annotate(p=Sum('price')).values('p'),
+                output_field=models.DecimalField(decimal_places=2, max_digits=10)
+            ), Value(0), output_field=models.DecimalField(decimal_places=2, max_digits=10)
+        ),
+        fee_total=Coalesce(
+            Subquery(
+                OrderFee.objects.filter(
+                    order=OuterRef('pk')
+                ).order_by().values('order').annotate(p=Sum('value')).values('p'),
+                output_field=models.DecimalField(decimal_places=2, max_digits=10)
+            ), Value(0), output_field=models.DecimalField(decimal_places=2, max_digits=10)
+        ),
+    ).exclude(
+        total=F('position_total') + F('fee_total'),
+    ).select_related('event')
+    for o in qs:
+        if o.total != o.position_total + o.fee_total:
+            yield [
+                Check.RESULT_ERROR,
+                f'Order {o.full_code} has a wrong total: Sum of positions and fees is '
+                f'{o.position_total + o.fee_total}, but total is {o.total}'
+            ]
+    yield [
+        Check.RESULT_OK,
+        'Check completed.'
+    ]
+
+
+def check_invoice_gaps():
+    group_qs = Invoice.objects.annotate(
+        sub_prefix=Substr('invoice_no', 1, StrIndex('invoice_no', Value('-'))),
+    ).order_by().values(
+        'organizer', 'prefix', 'sub_prefix', 'organizer__slug'
+    )
+    for g in group_qs:
+        numbers = Invoice.objects.filter(
+            prefix=g['prefix'], organizer=g['organizer']
+        )
+        if g['sub_prefix']:
+            numbers = numbers.filter(invoice_no__startswith=g['sub_prefix']).alias(
+                real_number=Cast(Substr('invoice_no', StrIndex('invoice_no', Value('-')) + 1), models.IntegerField())
+            ).order_by('real_number')
+        else:
+            numbers = numbers.exclude(invoice_no__contains='-').order_by('invoice_no')
+
+        numbers = list(numbers.values_list('invoice_no', flat=True))
+        previous_n = "(initial state)"
+        previous_numeric_part = 0
+        for n in numbers:
+            numeric_part = int(n.split("-")[-1])
+            if numeric_part != previous_numeric_part + 1:
+                print(g)
+                yield [
+                    Check.RESULT_WARNING,
+                    f'Organizer {g["organizer__slug"]}, prefix {g["prefix"]}, invoice {n} follows on {previous_n} with gap'
+                ]
+            previous_n = n
+            previous_numeric_part = numeric_part
+
+    yield [
+        Check.RESULT_OK,
+        'Check completed.'
+    ]
+
+
+@app.task()
+@scopes_disabled()
+def run_default_consistency_checks():
+    check_functions = [
+        ('pretix.orders.transactions', check_order_transactions),
+        ('pretix.orders.total', check_order_total),
+        ('pretix.invoices.gaps', check_invoice_gaps),
+    ]
+    for check_type, check_function in check_functions:
+        r = Check.RESULT_OK
+        log = []
+        try:
+            for result, logline in check_function():
+                if result == Check.RESULT_WARNING and r == Check.RESULT_OK:
+                    r = Check.RESULT_WARNING
+                elif result == Check.RESULT_ERROR:
+                    r = Check.RESULT_ERROR
+                log.append(f'[{result}] {logline}')
+        except Exception as e:
+            logger.exception('Could not run consistency check')
+            r = Check.RESULT_ERROR
+            log.append(f'[error] Check aborted: {e}')
+
+        Check.objects.create(result=r, check_type=check_type, log='\n'.join(log))
+
+    Check.objects.filter(created__lt=now() - timedelta(days=90)).delete()
+
+
+@receiver(signal=periodic_task)
+@minimum_interval(minutes_after_success=24 * 60)
+def periodic_consistency_checks(sender, **kwargs):
+    run_default_consistency_checks.apply_async()

src/pretix/control/navigation.py

@@ -422,6 +422,11 @@ def get_global_navigation(request):
                     'url': reverse('control:global.license'),
                     'active': (url.url_name == 'global.license'),
                 },
+                {
+                    'label': _('Consistency check'),
+                    'url': reverse('control:global.consistency'),
+                    'active': (url.url_name == 'global.consistency'),
+                },
             ]
         })
 

src/pretix/control/templates/pretixcontrol/global_consistency.html

@@ -0,0 +1,33 @@
+{% extends "pretixcontrol/global_settings_base.html" %}
+{% load i18n %}
+{% load bootstrap3 %}
+
+{% block inner %}
+    <h2>{% trans "Consistency checks" %}</h2>
+    <div class="table-responsive">
+
+        <table class="table">
+            <thead>
+            <tr>
+                <th>{% trans "Date" %}</th>
+                <th>{% trans "Check type" %}</th>
+                <th>{% trans "Result" %}</th>
+            </tr>
+            </thead>
+            <tbody>
+
+            {% for r in results %}
+                <tr class="{% if r.result == "error" %}danger{% elif r.result == "warning" %}warning{% endif %}">
+                    <td>{{ r.created|date:"SHORT_DATETIME_FORMAT" }}</td>
+                    <td>{{ r.check_type }}</td>
+                    <td>{{ r.get_result_display }}</td>
+                <td>
+                    <a href="{% url "control:global.consistency.detail" pk=r.pk %}" class="btn btn-default">{% trans "Show log" %}</a>
+                </td>
+                </tr>
+            {% endfor %}
+            </tbody>
+
+        </table>
+    </div>
+{% endblock %}

src/pretix/control/templates/pretixcontrol/global_consistency_detail.html

@@ -0,0 +1,11 @@
+{% extends "pretixcontrol/global_settings_base.html" %}
+{% load i18n %}
+{% load bootstrap3 %}
+
+{% block inner %}
+    <h2>{% trans "Consistency check" %}</h2>
+    <p>{{ result.created|date:"SHORT_DATETIME_FORMAT" }}</p>
+    <p>{{ result.check_type }}</p>
+    <p>{{ result.get_result_display }}</p>
+    <pre>{{ result.log }}</pre>
+{% endblock %}

src/pretix/control/urls.py

@@ -55,6 +55,8 @@ urlpatterns = [
     re_path(r'^global/settings/$', global_settings.GlobalSettingsView.as_view(), name='global.settings'),
     re_path(r'^global/update/$', global_settings.UpdateCheckView.as_view(), name='global.update'),
     re_path(r'^global/license/$', global_settings.LicenseCheckView.as_view(), name='global.license'),
+    re_path(r'^global/consistency/$', global_settings.ConsistencyCheckListView.as_view(), name='global.consistency'),
+    re_path(r'^global/consistency/(?P<pk>\d+)/$', global_settings.ConsistencyCheckDetailView.as_view(), name='global.consistency.detail'),
     re_path(r'^global/message/$', global_settings.MessageView.as_view(), name='global.message'),
     re_path(r'^logdetail/$', global_settings.LogDetailView.as_view(), name='global.logdetail'),
     re_path(r'^logdetail/payment/$', global_settings.PaymentDetailView.as_view(), name='global.paymentdetail'),

src/pretix/control/views/global_settings.py

@@ -39,9 +39,9 @@ from django.shortcuts import get_object_or_404, redirect, reverse
 from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _
 from django.views import View
-from django.views.generic import FormView, TemplateView
+from django.views.generic import FormView, ListView, TemplateView, DetailView
 
-from pretix.base.models import LogEntry, OrderPayment, OrderRefund
+from pretix.base.models import Check, LogEntry, OrderPayment, OrderRefund
 from pretix.base.services.update_check import check_result_table, update_check
 from pretix.base.settings import GlobalSettingsObject
 from pretix.control.forms.global_settings import (
@@ -265,3 +265,15 @@ class LicenseCheckView(StaffMemberRequiredMixin, FormView):
                 ))
 
         return res
+
+
+class ConsistencyCheckListView(AdministratorPermissionRequiredMixin, ListView):
+    queryset = Check.objects.order_by('-created').defer('log')
+    context_object_name = 'results'
+    template_name = 'pretixcontrol/global_consistency.html'
+
+
+class ConsistencyCheckDetailView(AdministratorPermissionRequiredMixin, DetailView):
+    queryset = Check.objects.all()
+    context_object_name = 'result'
+    template_name = 'pretixcontrol/global_consistency_detail.html'