CGM_Public/pretix_cgo
1
0
Fork 0
forked from CGM_Public/pretix_original
Code Pull Requests Actions Activity

Scheduled exports (#3033)

Browse Source
This commit is contained in:
Raphael Michel
2023-01-19 11:46:30 +01:00
committed by GitHub
parent 0bb5af191b
commit 19d1a8de71
36 changed files with 2461 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
src/pretix/base/migrations/0228_scheduledeventexport_scheduledorganizerexport.py Normal file
View File

@@ -0,0 +1,68 @@
# Generated by Django 3.2.16 on 2023-01-18 11:57
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
import pretix.base.models.base
class Migration(migrations.Migration):
dependencies = [
('pretixbase', '0227_item_personalized'),
]
operations = [
migrations.CreateModel(
name='ScheduledOrganizerExport',
fields=[
('id', models.BigAutoField(primary_key=True, serialize=False)),
('export_identifier', models.CharField(max_length=190)),
('export_form_data', models.JSONField(default=dict)),
('locale', models.CharField(max_length=250)),
('mail_additional_recipients', models.TextField()),
('mail_additional_recipients_cc', models.TextField()),
('mail_additional_recipients_bcc', models.TextField()),
('mail_subject', models.CharField(max_length=250)),
('mail_template', models.TextField()),
('schedule_rrule', models.TextField(null=True)),
('schedule_rrule_time', models.TimeField()),
('schedule_next_run', models.DateTimeField(blank=True, null=True)),
('error_counter', models.IntegerField(default=0)),
('error_last_message', models.TextField(null=True)),
('timezone', models.CharField(default='UTC', max_length=100)),
('organizer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='scheduled_exports', to='pretixbase.organizer')),
('owner', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
],
options={
'abstract': False,
},
bases=(models.Model, pretix.base.models.base.LoggingMixin),
),
migrations.CreateModel(
name='ScheduledEventExport',
fields=[
('id', models.BigAutoField(primary_key=True, serialize=False)),
('export_identifier', models.CharField(max_length=190)),
('export_form_data', models.JSONField(default=dict)),
('locale', models.CharField(max_length=250)),
('mail_additional_recipients', models.TextField()),
('mail_additional_recipients_cc', models.TextField()),
('mail_additional_recipients_bcc', models.TextField()),
('mail_subject', models.CharField(max_length=250)),
('mail_template', models.TextField()),
('schedule_rrule', models.TextField(null=True)),
('schedule_rrule_time', models.TimeField()),
('schedule_next_run', models.DateTimeField(blank=True, null=True)),
('error_counter', models.IntegerField(default=0)),
('error_last_message', models.TextField(null=True)),
('event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='scheduled_exports', to='pretixbase.event')),
('owner', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
],
options={
'abstract': False,
},
bases=(models.Model, pretix.base.models.base.LoggingMixin),
),
]

1
src/pretix/base/models/__init__.py
View File

@@ -30,6 +30,7 @@ from .event import (
Event, Event_SettingsStore, EventLock, EventMetaProperty, EventMetaValue,
SubEvent, SubEventMetaValue, generate_invite_token,
)
from .exports import ScheduledEventExport, ScheduledOrganizerExport
from .giftcards import GiftCard, GiftCardAcceptance, GiftCardTransaction
from .invoices import Invoice, InvoiceLine, invoice_filename
from .items import (

130
src/pretix/base/models/exports.py Normal file
View File

@@ -0,0 +1,130 @@
#
# This file is part of pretix (Community Edition).
#
# Copyright (C) 2014-2020 Raphael Michel and contributors
# Copyright (C) 2020-2021 rami.io GmbH and contributors
#
# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General
# Public License as published by the Free Software Foundation in version 3 of the License.
#
# ADDITIONAL TERMS APPLY: Pursuant to Section 7 of the GNU Affero General Public License, additional terms are
# applicable granting you additional permissions and placing additional restrictions on your usage of this software.
# Please refer to the pretix LICENSE file to obtain the full terms applicable to this work. If you did not receive
# this file, see <https://pretix.eu/about/en/license>.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
# <https://www.gnu.org/licenses/>.
#
from datetime import datetime, timedelta
import pytz
from dateutil.rrule import rrulestr
from django.conf import settings
from django.db import models
from django.utils.timezone import make_aware, now
from django.utils.translation import gettext_lazy as _
from pretix.base.models import LoggedModel
from pretix.base.validators import RRuleValidator, multimail_validate
class AbstractScheduledExport(LoggedModel):
id = models.BigAutoField(primary_key=True)
export_identifier = models.CharField(
max_length=190,
verbose_name=_("Export"),
)
export_form_data = models.JSONField(default=dict)
owner = models.ForeignKey(
"pretixbase.User",
on_delete=models.PROTECT,
)
locale = models.CharField(
verbose_name=_('Language'),
max_length=250
)
mail_additional_recipients = models.TextField(
verbose_name=_('Additional recipients'),
null=False, blank=True, validators=[multimail_validate],
help_text=_("You can specify multiple recipients separated by commas.")
)
mail_additional_recipients_cc = models.TextField(
verbose_name=_('Additional recipients (Cc)'),
null=False, blank=True, validators=[multimail_validate],
help_text=_("You can specify multiple recipients separated by commas.")
)
mail_additional_recipients_bcc = models.TextField(
verbose_name=_('Additional recipients (Bcc)'),
null=False, blank=True, validators=[multimail_validate],
help_text=_("You can specify multiple recipients separated by commas.")
)
mail_subject = models.CharField(
verbose_name=_('Subject'),
max_length=250
)
mail_template = models.TextField(
verbose_name=_('Message'),
)
schedule_rrule = models.TextField(
null=True, blank=True, validators=[RRuleValidator()]
)
schedule_rrule_time = models.TimeField(
verbose_name=_("Requested start time"),
help_text=_("The actual start time might be delayed depending on system load."),
)
schedule_next_run = models.DateTimeField(null=True, blank=True)
error_counter = models.IntegerField(default=0)
error_last_message = models.TextField(null=True, blank=True)
class Meta:
abstract = True
def __str__(self):
return self.mail_subject
def compute_next_run(self):
tz = self.tz
r = rrulestr(self.schedule_rrule)
new_d = r.after(now().astimezone(tz).replace(tzinfo=None), inc=False)
if not new_d:
self.schedule_next_run = None
return
try:
self.schedule_next_run = make_aware(datetime.combine(new_d.date(), self.schedule_rrule_time), tz)
except pytz.exceptions.AmbiguousTimeError:
self.schedule_next_run = make_aware(datetime.combine(new_d.date(), self.schedule_rrule_time), tz, is_dst=False)
except pytz.exceptions.NonExistentTimeError:
self.schedule_next_run = make_aware(datetime.combine(new_d.date(), self.schedule_rrule_time) + timedelta(hours=1), tz)
class ScheduledEventExport(AbstractScheduledExport):
event = models.ForeignKey(
"pretixbase.Event", on_delete=models.CASCADE, related_name="scheduled_exports"
)
@property
def tz(self):
return self.event.timezone
class ScheduledOrganizerExport(AbstractScheduledExport):
organizer = models.ForeignKey(
"pretixbase.Organizer", on_delete=models.CASCADE, related_name="scheduled_exports"
)
timezone = models.CharField(max_length=100,
default=settings.TIME_ZONE,
verbose_name=_('Timezone'))
@property
def tz(self):
return pytz.timezone(self.timezone)

232
src/pretix/base/services/export.py
View File

@@ -19,31 +19,48 @@
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
# <https://www.gnu.org/licenses/>.
#
from typing import Any, Dict
import logging
from datetime import timedelta
from typing import Any, Dict, Union
from celery.exceptions import MaxRetriesExceededError
from django.conf import settings
from django.core.files.base import ContentFile
from django.utils.timezone import override
from django.dispatch import receiver
from django.utils.timezone import now, override
from django.utils.translation import gettext
from django_scopes import scopes_disabled
from i18nfield.strings import LazyI18nString
from pretix.base.email import get_email_context
from pretix.base.exporter import OrganizerLevelExportMixin
from pretix.base.i18n import LazyLocaleException, language
from pretix.base.models import (
CachedFile, Device, Event, Organizer, TeamAPIToken, User, cachedfile_name,
CachedFile, Device, Event, Organizer, ScheduledEventExport, TeamAPIToken,
User, cachedfile_name,
)
from pretix.base.models.exports import ScheduledOrganizerExport
from pretix.base.services.mail import mail
from pretix.base.services.tasks import (
ProfiledEventTask, ProfiledOrganizerUserTask,
EventTask, OrganizerTask, ProfiledEventTask, ProfiledOrganizerUserTask,
)
from pretix.base.signals import (
register_data_exporters, register_multievent_data_exporters,
periodic_task, register_data_exporters, register_multievent_data_exporters,
)
from pretix.celery_app import app
from pretix.helpers.urls import build_absolute_uri
logger = logging.getLogger(__name__)
class ExportError(LazyLocaleException):
pass
class ExportEmptyError(ExportError):
pass
@app.task(base=ProfiledEventTask, throws=(ExportError,), bind=True)
def export(self, event: Event, fileid: str, provider: str, form_data: Dict[str, Any]) -> None:
def set_progress(val):
@@ -56,7 +73,7 @@ def export(self, event: Event, fileid: str, provider: str, form_data: Dict[str,
file = CachedFile.objects.get(id=fileid)
with language(event.settings.locale, event.settings.region), override(event.settings.timezone):
responses = register_data_exporters.send(event)
for receiver, response in responses:
for recv, response in responses:
if not response:
continue
ex = response(event, event.organizer, set_progress)
@@ -106,16 +123,16 @@ def multiexport(self, organizer: Organizer, user: User, device: int, token: int,
timezone = organizer.settings.timezone or settings.TIME_ZONE
region = organizer.settings.region
with language(locale, region), override(timezone):
if form_data.get('events') is not None:
if form_data.get('events') is not None and not form_data.get('all_events'):
if isinstance(form_data['events'][0], str):
events = allowed_events.filter(slug__in=form_data.get('events'), organizer=organizer)
else:
events = allowed_events.filter(pk__in=form_data.get('events'))
events = allowed_events.filter(pk__in=form_data.get('events'), organizer=organizer)
else:
events = allowed_events
events = allowed_events.filter(organizer=organizer)
responses = register_multievent_data_exporters.send(organizer)
for receiver, response in responses:
for recv, response in responses:
if not response:
continue
ex = response(events, organizer, set_progress)
@@ -138,3 +155,198 @@ def multiexport(self, organizer: Organizer, user: User, device: int, token: int,
f = ContentFile(data)
file.file.save(cachedfile_name(file, file.filename), f)
return file.pk
def _run_scheduled_export(schedule, context: Union[Event, Organizer], exporter, config_url, retry_func, has_permission):
with language(schedule.locale, context.settings.region), override(schedule.tz):
file = CachedFile(web_download=False)
file.date = now()
file.expires = now() + timedelta(hours=24)
file.save()
def _handle_error(msg, soft=False):
context.log_action(
'pretix.event.export.schedule.failed',
data={
'id': schedule.id,
'export_identifier': schedule.export_identifier,
'export_form_data': schedule.export_form_data,
'reason': msg,
'soft': soft,
}
)
if schedule.owner.is_active:
mail(
email=schedule.owner.email,
subject=gettext('Export failed'),
template='pretixbase/email/export_failed.txt',
context={
'configuration_url': config_url,
'reason': msg,
'soft': soft,
},
event=context if isinstance(context, Event) else None,
organizer=context.organizer if isinstance(context, Event) else context,
locale=schedule.locale,
)
if not soft:
schedule.error_counter += 1
schedule.error_last_message = msg
schedule.save(update_fields=['error_counter', 'error_last_message'])
if not has_permission:
_handle_error(gettext('Permission denied.'))
return
try:
if not exporter:
raise ExportError("Export type not found.")
d = exporter.render(schedule.export_form_data)
if d is None:
raise ExportEmptyError(
gettext('Your export did not contain any data.')
)
file.filename, file.type, data = d
filesize = len(data)
if filesize > 20 * 1024 * 1024: # 20 MB
raise ExportError(
gettext('Your exported data exceeded the size limit for scheduled exports.')
)
f = ContentFile(data)
file.file.save(cachedfile_name(file, file.filename), f)
except ExportEmptyError as e:
_handle_error(str(e), soft=True)
except ExportError as e:
_handle_error(str(e), soft=False)
except Exception:
logger.exception("Scheduled export failed.")
try:
retry_func()
except MaxRetriesExceededError:
_handle_error('Internal Error')
else:
schedule.error_counter = 0
schedule.save(update_fields=['error_counter'])
mail(
email=[schedule.owner.email] + [r for r in schedule.mail_additional_recipients.split(",") if r],
cc=[r for r in schedule.mail_additional_recipients_cc.split(",") if r],
bcc=[r for r in schedule.mail_additional_recipients_bcc.split(",") if r],
subject=schedule.mail_subject,
template=LazyI18nString(schedule.mail_template),
context=get_email_context(event=context) if isinstance(context, Event) else {},
organizer=context.organizer if isinstance(context, Event) else context,
locale=schedule.locale,
attach_cached_files=[file],
)
context.log_action(
'pretix.event.export.schedule.executed',
data={
'id': schedule.id,
'export_identifier': schedule.export_identifier,
'export_form_data': schedule.export_form_data,
'result_file_size': filesize,
'result_file_name': file.file.name,
}
)
@app.task(base=OrganizerTask, bind=True, max_retries=5, default_retry_delay=120)
def scheduled_organizer_export(self, organizer: Organizer, schedule: int) -> None:
schedule = organizer.scheduled_exports.get(pk=schedule)
allowed_events = schedule.owner.get_events_with_permission('can_view_orders')
if schedule.export_form_data.get('events') is not None and not schedule.export_form_data.get('all_events'):
if isinstance(schedule.export_form_data['events'][0], str):
events = allowed_events.filter(slug__in=schedule.export_form_data.get('events'), organizer=organizer)
else:
events = allowed_events.filter(pk__in=schedule.export_form_data.get('events'), organizer=organizer)
else:
events = allowed_events.filter(organizer=organizer)
responses = register_multievent_data_exporters.send(organizer)
exporter = None
for recv, response in responses:
if not response:
continue
ex = response(events, organizer)
if ex.identifier == schedule.export_identifier:
exporter = ex
break
has_permission = schedule.owner.is_active
if isinstance(exporter, OrganizerLevelExportMixin):
if not schedule.owner.has_organizer_permission(organizer, exporter.organizer_required_permission):
has_permission = False
_run_scheduled_export(
schedule,
organizer,
exporter,
build_absolute_uri(
'control:organizer.export',
kwargs={
'organizer': organizer.slug,
}
) + f'?identifier={schedule.export_identifier}&scheduled={schedule.pk}',
self.retry,
has_permission,
)
@app.task(base=EventTask, bind=True, max_retries=5, default_retry_delay=120)
def scheduled_event_export(self, event: Event, schedule: int) -> None:
schedule = event.scheduled_exports.get(pk=schedule)
responses = register_data_exporters.send(event)
exporter = None
for recv, response in responses:
if not response:
continue
ex = response(event, event.organizer)
if ex.identifier == schedule.export_identifier:
exporter = ex
break
has_permission = schedule.owner.is_active and schedule.owner.has_event_permission(event.organizer, event, 'can_view_orders')
_run_scheduled_export(
schedule,
event,
exporter,
build_absolute_uri(
'control:event.orders.export',
kwargs={
'event': event.slug,
'organizer': event.organizer.slug,
}
) + f'?identifier={schedule.export_identifier}&scheduled={schedule.pk}',
self.retry,
has_permission,
)
@receiver(signal=periodic_task)
@scopes_disabled()
def run_scheduled_exports(sender, **kwargs):
qs = ScheduledEventExport.objects.filter(
schedule_next_run__lt=now(),
error_counter__lt=5,
).select_related('event')
for s in qs:
scheduled_event_export.apply_async(kwargs={
'event': s.event_id,
'schedule': s.pk,
})
s.compute_next_run()
s.save(update_fields=['schedule_next_run'])
qs = ScheduledOrganizerExport.objects.filter(
schedule_next_run__lt=now(),
error_counter__lt=5,
).select_related('organizer')
for s in qs:
scheduled_organizer_export.apply_async(kwargs={
'organizer': s.organizer_id,
'schedule': s.pk,
})
s.compute_next_run()
s.save(update_fields=['schedule_next_run'])

9
src/pretix/base/services/mail.py
View File

@@ -100,7 +100,7 @@ def mail(email: Union[str, Sequence[str]], subject: str, template: Union[str, La
position: OrderPosition = None, *, headers: dict = None, sender: str = None, organizer: Organizer = None,
customer: Customer = None, invoices: Sequence = None, attach_tickets=False, auto_email=True, user=None,
attach_ical=False, attach_cached_files: Sequence = None, attach_other_files: list=None,
plain_text_only=False, no_order_links=False):
plain_text_only=False, no_order_links=False, cc: Sequence[str]=None, bcc: Sequence[str]=None):
"""
Sends out an email to a user. The mail will be sent synchronously or asynchronously depending on the installation.
@@ -211,7 +211,7 @@ def mail(email: Union[str, Sequence[str]], subject: str, template: Union[str, La
subject = raw_subject = str(subject).replace('\n', ' ').replace('\r', '')[:900]
signature = ""
bcc = []
bcc = list(bcc or [])
settings_holder = event or organizer
@@ -305,6 +305,7 @@ def mail(email: Union[str, Sequence[str]], subject: str, template: Union[str, La
send_task = mail_send_task.si(
to=[email] if isinstance(email, str) else list(email),
cc=cc,
bcc=bcc,
subject=subject,
body=body_plain,
@@ -357,11 +358,11 @@ class CustomEmail(EmailMultiAlternatives):
@app.task(base=TransactionAwareTask, bind=True, acks_late=True)
def mail_send_task(self, *args, to: List[str], subject: str, body: str, html: str, sender: str,
event: int = None, position: int = None, headers: dict = None, bcc: List[str] = None,
event: int = None, position: int = None, headers: dict = None, cc: List[str] = None, bcc: List[str] = None,
invoices: List[int] = None, order: int = None, attach_tickets=False, user=None,
organizer=None, customer=None, attach_ical=False, attach_cached_files: List[int] = None,
attach_other_files: List[str] = None) -> bool:
email = CustomEmail(subject, body, sender, to=to, bcc=bcc, headers=headers)
email = CustomEmail(subject, body, sender, to=to, cc=cc, bcc=bcc, headers=headers)
if html is not None:
html_message = SafeMIMEMultipart(_subtype='related', encoding=settings.DEFAULT_CHARSET)
html_with_cid, cid_images = replace_images_with_cid_paths(html)

25
src/pretix/base/services/tasks.py
View File

@@ -109,6 +109,31 @@ class EventTask(app.Task):
return ret
class OrganizerTask(app.Task):
def __call__(self, *args, **kwargs):
if 'organizer_id' in kwargs:
organizer_id = kwargs.get('organizer_id')
with scopes_disabled():
organizer = Organizer.objects.get(pk=organizer_id)
del kwargs['organizer_id']
kwargs['organizer'] = organizer
elif 'organizer' in kwargs:
organizer_id = kwargs.get('organizer')
with scopes_disabled():
organizer = Organizer.objects.get(pk=organizer_id)
kwargs['organizer'] = organizer
else:
args = list(args)
organizer_id = args[0]
with scopes_disabled():
organizer = Organizer.objects.get(pk=organizer_id)
args[0] = organizer
with scope(organizer=organizer):
ret = super().__call__(*args, **kwargs)
return ret
class OrganizerUserTask(app.Task):
def __call__(self, *args, **kwargs):
organizer_id = kwargs['organizer']

12
src/pretix/base/templates/pretixbase/email/export_failed.txt Normal file
View File

@@ -0,0 +1,12 @@
{% load i18n %}
{% trans "Your export failed." %}
{% trans "Reason:" %} {{ reason }}
{% if not soft %}
{% trans "If your export fails five times in a row, it will no longer be sent." %}
{% endif %}
{% trans "Configuration link:" %}
{{ configuration_url }}

26
src/pretix/base/validators.py
View File

@@ -19,6 +19,12 @@
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
# <https://www.gnu.org/licenses/>.
#
from dateutil.rrule import rrulestr
from django.conf import settings
from django.core.exceptions import ValidationError
from django.core.validators import validate_email
from django.utils.deconstruct import deconstructible
from django.utils.translation import gettext_lazy as _
# This file is based on an earlier version of pretix which was released under the Apache License 2.0. The full text of
# the Apache License 2.0 can be obtained at <http://www.apache.org/licenses/LICENSE-2.0>.
@@ -32,11 +38,6 @@
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under the License.
from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.deconstruct import deconstructible
from django.utils.translation import gettext_lazy as _
class BanlistValidator:
@@ -101,3 +102,18 @@ class EmailBanlistValidator(BanlistValidator):
banlist = [
settings.PRETIX_EMAIL_NONE_VALUE,
]
def multimail_validate(val):
s = val.split(',')
for part in s:
validate_email(part.strip())
return s
class RRuleValidator:
def __call__(self, value):
try:
rrulestr(value)
except Exception:
raise ValidationError("Not a valid rrule.")