From e2dad8fdcc4031575a0ab338ddd4aa5b6e5f4ffe Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Fri, 29 Oct 2021 12:02:00 +0200 Subject: [PATCH] Database consistency checks --- src/pretix/base/apps.py | 2 +- src/pretix/base/migrations/0201_check.py | 29 +++ src/pretix/base/models/__init__.py | 1 + src/pretix/base/models/checks.py | 54 +++++ src/pretix/base/services/checks.py | 191 ++++++++++++++++++ src/pretix/control/navigation.py | 5 + .../pretixcontrol/global_consistency.html | 33 +++ .../global_consistency_detail.html | 11 + src/pretix/control/urls.py | 2 + src/pretix/control/views/global_settings.py | 16 +- 10 files changed, 341 insertions(+), 3 deletions(-) create mode 100644 src/pretix/base/migrations/0201_check.py create mode 100644 src/pretix/base/models/checks.py create mode 100644 src/pretix/base/services/checks.py create mode 100644 src/pretix/control/templates/pretixcontrol/global_consistency.html create mode 100644 src/pretix/control/templates/pretixcontrol/global_consistency_detail.html diff --git a/src/pretix/base/apps.py b/src/pretix/base/apps.py index 435eff51e..1fbc161c8 100644 --- a/src/pretix/base/apps.py +++ b/src/pretix/base/apps.py @@ -46,7 +46,7 @@ class PretixBaseConfig(AppConfig): from . import invoice # NOQA from . import notifications # NOQA from . import email # NOQA - from .services import auth, checkin, export, mail, tickets, cart, orderimport, orders, invoices, cleanup, update_check, quotas, notifications, vouchers # NOQA + from .services import auth, checkin, checks, export, mail, tickets, cart, orderimport, orders, invoices, cleanup, update_check, quotas, notifications, vouchers # NOQA from .models import _transactions # NOQA from django.conf import settings diff --git a/src/pretix/base/migrations/0201_check.py b/src/pretix/base/migrations/0201_check.py new file mode 100644 index 000000000..b8db4c12e --- /dev/null +++ b/src/pretix/base/migrations/0201_check.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.4 on 2021-10-29 09:58 + +from django.db import migrations, models + +import pretix.base.models.base + + +class Migration(migrations.Migration): + + dependencies = [ + ('pretixbase', '0200_transaction'), + ] + + operations = [ + migrations.CreateModel( + name='Check', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True)), + ('result', models.CharField(max_length=190)), + ('check_type', models.CharField(max_length=190)), + ('log', models.TextField()), + ], + options={ + 'abstract': False, + }, + bases=(models.Model, pretix.base.models.base.LoggingMixin), + ), + ] diff --git a/src/pretix/base/models/__init__.py b/src/pretix/base/models/__init__.py index d5f47b71c..fd694446d 100644 --- a/src/pretix/base/models/__init__.py +++ b/src/pretix/base/models/__init__.py @@ -23,6 +23,7 @@ from ..settings import GlobalSettingsObject_SettingsStore from .auth import U2FDevice, User, WebAuthnDevice from .base import CachedFile, LoggedModel, cachedfile_name from .checkin import Checkin, CheckinList +from .checks import Check from .customers import Customer from .devices import Device, Gate from .event import ( diff --git a/src/pretix/base/models/checks.py b/src/pretix/base/models/checks.py new file mode 100644 index 000000000..7243f3661 --- /dev/null +++ b/src/pretix/base/models/checks.py @@ -0,0 +1,54 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# + +# This file is based on an earlier version of pretix which was released under the Apache License 2.0. The full text of +# the Apache License 2.0 can be obtained at . +# +# This file may have since been changed and any changes are released under the terms of AGPLv3 as described above. A +# full history of changes and contributors is available at . +# +# This file contains Apache-licensed contributions copyrighted by: Jakob Schnell +# +# Unless required by applicable law or agreed to in writing, software distributed under the Apache License 2.0 is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under the License. +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from pretix.base.models import LoggedModel + + +class Check(LoggedModel): + RESULT_OK = 'ok' + RESULT_WARNING = 'warning' + RESULT_ERROR = 'error' + + RESULTS = ( + (RESULT_OK, _('OK')), + (RESULT_WARNING, _('Warning')), + (RESULT_ERROR, _('Error')), + ) + + created = models.DateTimeField(auto_now_add=True) + result = models.CharField(max_length=190, choices=RESULTS) + check_type = models.CharField(max_length=190) + log = models.TextField() diff --git a/src/pretix/base/services/checks.py b/src/pretix/base/services/checks.py new file mode 100644 index 000000000..cca7bd323 --- /dev/null +++ b/src/pretix/base/services/checks.py @@ -0,0 +1,191 @@ +# +# This file is part of pretix (Community Edition). +# +# Copyright (C) 2014-2020 Raphael Michel and contributors +# Copyright (C) 2020-2021 rami.io GmbH and contributors +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation in version 3 of the License. +# +# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are +# applicable granting you additional permissions and placing additional restrictions on your usage of this software. +# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive +# this file, see . +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +# +import logging +from datetime import timedelta + +from django.db import models +from django.db.models import Case, F, OuterRef, Q, Subquery, Sum, Value, When +from django.db.models.functions import Cast, Coalesce, StrIndex, Substr +from django.dispatch import receiver +from django.utils.timezone import now +from django_scopes import scopes_disabled + +from pretix.base.models import Check, Invoice, Order, OrderFee, OrderPosition +from pretix.base.models.orders import Transaction +from pretix.base.signals import periodic_task +from pretix.celery_app import app +from pretix.helpers.periodic import minimum_interval + +logger = logging.getLogger(__name__) + + +def check_order_transactions(): + qs = Order.objects.annotate( + position_total=Coalesce( + Subquery( + OrderPosition.objects.filter( + order=OuterRef('pk') + ).order_by().values('order').annotate(p=Sum('price')).values('p'), + output_field=models.DecimalField(decimal_places=2, max_digits=10) + ), Value(0), output_field=models.DecimalField(decimal_places=2, max_digits=10) + ), + fee_total=Coalesce( + Subquery( + OrderFee.objects.filter( + order=OuterRef('pk') + ).order_by().values('order').annotate(p=Sum('value')).values('p'), + output_field=models.DecimalField(decimal_places=2, max_digits=10) + ), Value(0), output_field=models.DecimalField(decimal_places=2, max_digits=10) + ), + tx_total=Coalesce( + Subquery( + Transaction.objects.filter( + order=OuterRef('pk') + ).order_by().values('order').annotate(p=Sum(F('price') * F('count'))).values('p'), + output_field=models.DecimalField(decimal_places=2, max_digits=10) + ), Value(0), output_field=models.DecimalField(decimal_places=2, max_digits=10) + ), + ).annotate( + correct_total=Case( + When(Q(status=Order.STATUS_CANCELED) | Q(status=Order.STATUS_EXPIRED) | Q(require_approval=True), + then=0), + default=F('position_total') + F('fee_total'), + output_field=models.DecimalField(decimal_places=2, max_digits=10) + ), + ).exclude( + tx_total=F('correct_total') + ).select_related('event') + for o in qs: + yield [ + Check.RESULT_ERROR, + f'Order {o.full_code} has a wrong total: Status is {o.status} and sum of positions and fees is ' + f'{o.position_total + o.fee_total}, so sum of transactions should be {o.correct_total} but is {o.tx_total}' + ] + yield [ + Check.RESULT_OK, + 'Check completed.' + ] + + +def check_order_total(): + qs = Order.objects.annotate( + position_total=Coalesce( + Subquery( + OrderPosition.objects.filter( + order=OuterRef('pk') + ).order_by().values('order').annotate(p=Sum('price')).values('p'), + output_field=models.DecimalField(decimal_places=2, max_digits=10) + ), Value(0), output_field=models.DecimalField(decimal_places=2, max_digits=10) + ), + fee_total=Coalesce( + Subquery( + OrderFee.objects.filter( + order=OuterRef('pk') + ).order_by().values('order').annotate(p=Sum('value')).values('p'), + output_field=models.DecimalField(decimal_places=2, max_digits=10) + ), Value(0), output_field=models.DecimalField(decimal_places=2, max_digits=10) + ), + ).exclude( + total=F('position_total') + F('fee_total'), + ).select_related('event') + for o in qs: + if o.total != o.position_total + o.fee_total: + yield [ + Check.RESULT_ERROR, + f'Order {o.full_code} has a wrong total: Sum of positions and fees is ' + f'{o.position_total + o.fee_total}, but total is {o.total}' + ] + yield [ + Check.RESULT_OK, + 'Check completed.' + ] + + +def check_invoice_gaps(): + group_qs = Invoice.objects.annotate( + sub_prefix=Substr('invoice_no', 1, StrIndex('invoice_no', Value('-'))), + ).order_by().values( + 'organizer', 'prefix', 'sub_prefix', 'organizer__slug' + ) + for g in group_qs: + numbers = Invoice.objects.filter( + prefix=g['prefix'], organizer=g['organizer'] + ) + if g['sub_prefix']: + numbers = numbers.filter(invoice_no__startswith=g['sub_prefix']).alias( + real_number=Cast(Substr('invoice_no', StrIndex('invoice_no', Value('-')) + 1), models.IntegerField()) + ).order_by('real_number') + else: + numbers = numbers.exclude(invoice_no__contains='-').order_by('invoice_no') + + numbers = list(numbers.values_list('invoice_no', flat=True)) + previous_n = "(initial state)" + previous_numeric_part = 0 + for n in numbers: + numeric_part = int(n.split("-")[-1]) + if numeric_part != previous_numeric_part + 1: + print(g) + yield [ + Check.RESULT_WARNING, + f'Organizer {g["organizer__slug"]}, prefix {g["prefix"]}, invoice {n} follows on {previous_n} with gap' + ] + previous_n = n + previous_numeric_part = numeric_part + + yield [ + Check.RESULT_OK, + 'Check completed.' + ] + + +@app.task() +@scopes_disabled() +def run_default_consistency_checks(): + check_functions = [ + ('pretix.orders.transactions', check_order_transactions), + ('pretix.orders.total', check_order_total), + ('pretix.invoices.gaps', check_invoice_gaps), + ] + for check_type, check_function in check_functions: + r = Check.RESULT_OK + log = [] + try: + for result, logline in check_function(): + if result == Check.RESULT_WARNING and r == Check.RESULT_OK: + r = Check.RESULT_WARNING + elif result == Check.RESULT_ERROR: + r = Check.RESULT_ERROR + log.append(f'[{result}] {logline}') + except Exception as e: + logger.exception('Could not run consistency check') + r = Check.RESULT_ERROR + log.append(f'[error] Check aborted: {e}') + + Check.objects.create(result=r, check_type=check_type, log='\n'.join(log)) + + Check.objects.filter(created__lt=now() - timedelta(days=90)).delete() + + +@receiver(signal=periodic_task) +@minimum_interval(minutes_after_success=24 * 60) +def periodic_consistency_checks(sender, **kwargs): + run_default_consistency_checks.apply_async() diff --git a/src/pretix/control/navigation.py b/src/pretix/control/navigation.py index 9cd416037..5fe47fed2 100644 --- a/src/pretix/control/navigation.py +++ b/src/pretix/control/navigation.py @@ -422,6 +422,11 @@ def get_global_navigation(request): 'url': reverse('control:global.license'), 'active': (url.url_name == 'global.license'), }, + { + 'label': _('Consistency check'), + 'url': reverse('control:global.consistency'), + 'active': (url.url_name == 'global.consistency'), + }, ] }) diff --git a/src/pretix/control/templates/pretixcontrol/global_consistency.html b/src/pretix/control/templates/pretixcontrol/global_consistency.html new file mode 100644 index 000000000..51f1de981 --- /dev/null +++ b/src/pretix/control/templates/pretixcontrol/global_consistency.html @@ -0,0 +1,33 @@ +{% extends "pretixcontrol/global_settings_base.html" %} +{% load i18n %} +{% load bootstrap3 %} + +{% block inner %} +

{% trans "Consistency checks" %}

+
+ + + + + + + + + + + + {% for r in results %} + + + + + + + {% endfor %} + + +
{% trans "Date" %}{% trans "Check type" %}{% trans "Result" %}
{{ r.created|date:"SHORT_DATETIME_FORMAT" }}{{ r.check_type }}{{ r.get_result_display }} + {% trans "Show log" %} +
+
+{% endblock %} diff --git a/src/pretix/control/templates/pretixcontrol/global_consistency_detail.html b/src/pretix/control/templates/pretixcontrol/global_consistency_detail.html new file mode 100644 index 000000000..a1724d828 --- /dev/null +++ b/src/pretix/control/templates/pretixcontrol/global_consistency_detail.html @@ -0,0 +1,11 @@ +{% extends "pretixcontrol/global_settings_base.html" %} +{% load i18n %} +{% load bootstrap3 %} + +{% block inner %} +

{% trans "Consistency check" %}

+

{{ result.created|date:"SHORT_DATETIME_FORMAT" }}

+

{{ result.check_type }}

+

{{ result.get_result_display }}

+ 
{{ result.log }}
+{% endblock %} diff --git a/src/pretix/control/urls.py b/src/pretix/control/urls.py index da14f08f2..7801537bf 100644 --- a/src/pretix/control/urls.py +++ b/src/pretix/control/urls.py @@ -55,6 +55,8 @@ urlpatterns = [ re_path(r'^global/settings/$', global_settings.GlobalSettingsView.as_view(), name='global.settings'), re_path(r'^global/update/$', global_settings.UpdateCheckView.as_view(), name='global.update'), re_path(r'^global/license/$', global_settings.LicenseCheckView.as_view(), name='global.license'), + re_path(r'^global/consistency/$', global_settings.ConsistencyCheckListView.as_view(), name='global.consistency'), + re_path(r'^global/consistency/(?P\d+)/$', global_settings.ConsistencyCheckDetailView.as_view(), name='global.consistency.detail'), re_path(r'^global/message/$', global_settings.MessageView.as_view(), name='global.message'), re_path(r'^logdetail/$', global_settings.LogDetailView.as_view(), name='global.logdetail'), re_path(r'^logdetail/payment/$', global_settings.PaymentDetailView.as_view(), name='global.paymentdetail'), diff --git a/src/pretix/control/views/global_settings.py b/src/pretix/control/views/global_settings.py index c893234eb..a492fd148 100644 --- a/src/pretix/control/views/global_settings.py +++ b/src/pretix/control/views/global_settings.py @@ -39,9 +39,9 @@ from django.shortcuts import get_object_or_404, redirect, reverse from django.utils.timezone import now from django.utils.translation import gettext_lazy as _ from django.views import View -from django.views.generic import FormView, TemplateView +from django.views.generic import FormView, ListView, TemplateView, DetailView -from pretix.base.models import LogEntry, OrderPayment, OrderRefund +from pretix.base.models import Check, LogEntry, OrderPayment, OrderRefund from pretix.base.services.update_check import check_result_table, update_check from pretix.base.settings import GlobalSettingsObject from pretix.control.forms.global_settings import ( @@ -265,3 +265,15 @@ class LicenseCheckView(StaffMemberRequiredMixin, FormView): )) return res + + +class ConsistencyCheckListView(AdministratorPermissionRequiredMixin, ListView): + queryset = Check.objects.order_by('-created').defer('log') + context_object_name = 'results' + template_name = 'pretixcontrol/global_consistency.html' + + +class ConsistencyCheckDetailView(AdministratorPermissionRequiredMixin, DetailView): + queryset = Check.objects.all() + context_object_name = 'result' + template_name = 'pretixcontrol/global_consistency_detail.html'