From 33ace8554b5aa1baab77fc439763b2fd8590366e Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Fri, 12 Jan 2024 10:58:26 +0100 Subject: [PATCH] Check-in list export: Constant-memory implementation --- src/pretix/plugins/checkinlists/exporters.py | 222 ++++++++++--------- 1 file changed, 116 insertions(+), 106 deletions(-) diff --git a/src/pretix/plugins/checkinlists/exporters.py b/src/pretix/plugins/checkinlists/exporters.py index 096eaa11d5..529ab7bd95 100644 --- a/src/pretix/plugins/checkinlists/exporters.py +++ b/src/pretix/plugins/checkinlists/exporters.py @@ -63,6 +63,7 @@ from pretix.base.timeframes import ( ) from pretix.control.forms.widgets import Select2 from pretix.helpers.filenames import safe_for_filename +from pretix.helpers.iter import chunked_iterable from pretix.helpers.templatetags.jsonfield import JSONExtract from pretix.plugins.reports.exporters import ReportlabExportMixin @@ -156,7 +157,7 @@ class CheckInListMixin(BaseExporter): return d - def _get_queryset(self, cl, form_data): + def _get_queryset(self, cl, form_data, prefetch=True): cqs = Checkin.objects.filter( position_id=OuterRef('pk'), list_id=cl.pk @@ -179,9 +180,11 @@ class CheckInListMixin(BaseExporter): auto_checked_in=Exists( Checkin.objects.filter(position_id=OuterRef('pk'), list_id=cl.pk, auto_checked_in=True) ) - ).prefetch_related( - 'answers', 'answers__question', 'addon_to__answers', 'addon_to__answers__question' ).select_related('order', 'item', 'variation', 'addon_to', 'order__invoice_address', 'voucher', 'seat') + if prefetch: + qs = qs.prefetch_related( + 'answers', 'answers__question', 'addon_to__answers', 'addon_to__answers__question' + ) if form_data.get('status'): s = form_data.get('status') @@ -465,7 +468,7 @@ class CSVCheckinList(CheckInListMixin, ListExporter): questions = list(Question.objects.filter(event=self.event, id__in=form_data['questions'])) - qs = self._get_queryset(cl, form_data) + base_qs = self._get_queryset(cl, form_data, prefetch=False) name_scheme = PERSON_NAME_SCHEMES[self.event.settings.name_scheme] headers = [ @@ -518,124 +521,131 @@ class CSVCheckinList(CheckInListMixin, ListExporter): ] yield headers - yield self.ProgressSetTotal(total=qs.count()) + qs = base_qs.prefetch_related( + 'answers', 'answers__question', 'addon_to__answers', 'addon_to__answers__question' + ) - for op in qs: - try: - ia = op.order.invoice_address - except InvoiceAddress.DoesNotExist: - ia = InvoiceAddress() + all_ids = list(base_qs.values_list('pk', flat=True)) + yield self.ProgressSetTotal(total=len(all_ids)) - last_checked_in = None - if isinstance(op.last_checked_in, str): # SQLite - last_checked_in = dateutil.parser.parse(op.last_checked_in) - elif op.last_checked_in: - last_checked_in = op.last_checked_in - if last_checked_in and not is_aware(last_checked_in): - last_checked_in = make_aware(last_checked_in, timezone.utc) + for ids in chunked_iterable(all_ids, 1000): + ops = sorted(qs.filter(id__in=ids).order_by(), key=lambda k: ids.index(k.pk)) + for op in ops: + try: + ia = op.order.invoice_address + except InvoiceAddress.DoesNotExist: + ia = InvoiceAddress() - last_checked_out = None - if isinstance(op.last_checked_out, str): # SQLite - last_checked_out = dateutil.parser.parse(op.last_checked_out) - elif op.last_checked_out: - last_checked_out = op.last_checked_out - if last_checked_out and not is_aware(last_checked_out): - last_checked_out = make_aware(last_checked_out, timezone.utc) + last_checked_in = None + if isinstance(op.last_checked_in, str): # SQLite + last_checked_in = dateutil.parser.parse(op.last_checked_in) + elif op.last_checked_in: + last_checked_in = op.last_checked_in + if last_checked_in and not is_aware(last_checked_in): + last_checked_in = make_aware(last_checked_in, timezone.utc) - row = [ - op.order.code, - op.attendee_name or (op.addon_to.attendee_name if op.addon_to else '') or ia.name, - ] - if len(name_scheme['fields']) > 1: - for k, label, w in name_scheme['fields']: - v = ( - op.attendee_name_parts or - (op.addon_to.attendee_name_parts if op.addon_to else {}) or - ia.name_parts - ).get(k, '') - if k == "salutation": - v = pgettext("person_name_salutation", v) + last_checked_out = None + if isinstance(op.last_checked_out, str): # SQLite + last_checked_out = dateutil.parser.parse(op.last_checked_out) + elif op.last_checked_out: + last_checked_out = op.last_checked_out + if last_checked_out and not is_aware(last_checked_out): + last_checked_out = make_aware(last_checked_out, timezone.utc) - row.append(v) - row += [ - str(op.item) + (" – " + str(op.variation.value) if op.variation else ""), - op.price, - date_format(last_checked_in.astimezone(self.event.timezone), 'SHORT_DATETIME_FORMAT') - if last_checked_in else '', - date_format(last_checked_out.astimezone(self.event.timezone), 'SHORT_DATETIME_FORMAT') - if last_checked_out else '', - _('Yes') if op.auto_checked_in else _('No'), - ] - if cl.include_pending: - row.append(_('Yes') if op.order.status == Order.STATUS_PAID else _('No')) - if form_data['secrets']: - row.append(op.secret) - row.append(op.attendee_email or (op.addon_to.attendee_email if op.addon_to else '') or op.order.email or '') - row.append(str(op.order.phone) if op.order.phone else '') - if self.event.has_subevents: - row.append(str(op.subevent.name)) - row.append(date_format(op.subevent.date_from.astimezone(self.event.timezone), 'SHORT_DATETIME_FORMAT')) - if op.subevent.date_to: - row.append( - date_format(op.subevent.date_to.astimezone(self.event.timezone), 'SHORT_DATETIME_FORMAT') - ) - else: - row.append('') - acache = {} - if op.addon_to: - for a in op.addon_to.answers.all(): + row = [ + op.order.code, + op.attendee_name or (op.addon_to.attendee_name if op.addon_to else '') or ia.name, + ] + if len(name_scheme['fields']) > 1: + for k, label, w in name_scheme['fields']: + v = ( + op.attendee_name_parts or + (op.addon_to.attendee_name_parts if op.addon_to else {}) or + ia.name_parts + ).get(k, '') + if k == "salutation": + v = pgettext("person_name_salutation", v) + + row.append(v) + row += [ + str(op.item) + (" – " + str(op.variation.value) if op.variation else ""), + op.price, + date_format(last_checked_in.astimezone(self.event.timezone), 'SHORT_DATETIME_FORMAT') + if last_checked_in else '', + date_format(last_checked_out.astimezone(self.event.timezone), 'SHORT_DATETIME_FORMAT') + if last_checked_out else '', + _('Yes') if op.auto_checked_in else _('No'), + ] + if cl.include_pending: + row.append(_('Yes') if op.order.status == Order.STATUS_PAID else _('No')) + if form_data['secrets']: + row.append(op.secret) + row.append(op.attendee_email or (op.addon_to.attendee_email if op.addon_to else '') or op.order.email or '') + row.append(str(op.order.phone) if op.order.phone else '') + if self.event.has_subevents: + row.append(str(op.subevent.name)) + row.append(date_format(op.subevent.date_from.astimezone(self.event.timezone), 'SHORT_DATETIME_FORMAT')) + if op.subevent.date_to: + row.append( + date_format(op.subevent.date_to.astimezone(self.event.timezone), 'SHORT_DATETIME_FORMAT') + ) + else: + row.append('') + acache = {} + if op.addon_to: + for a in op.addon_to.answers.all(): + # We do not want to localize Date, Time and Datetime question answers, as those can lead + # to difficulties parsing the data (for example 2019-02-01 may become Février, 2019 01 in French). + if a.question.type in Question.UNLOCALIZED_TYPES: + acache[a.question_id] = a.answer + else: + acache[a.question_id] = str(a) + for a in op.answers.all(): # We do not want to localize Date, Time and Datetime question answers, as those can lead # to difficulties parsing the data (for example 2019-02-01 may become Février, 2019 01 in French). if a.question.type in Question.UNLOCALIZED_TYPES: acache[a.question_id] = a.answer else: acache[a.question_id] = str(a) - for a in op.answers.all(): - # We do not want to localize Date, Time and Datetime question answers, as those can lead - # to difficulties parsing the data (for example 2019-02-01 may become Février, 2019 01 in French). - if a.question.type in Question.UNLOCALIZED_TYPES: - acache[a.question_id] = a.answer + for q in questions: + row.append(acache.get(q.pk, '')) + + row.append(op.company or ia.company) + row.append(op.voucher.code if op.voucher else "") + row.append(op.order.datetime.astimezone(self.event.timezone).strftime('%Y-%m-%d')) + row.append(op.order.datetime.astimezone(self.event.timezone).strftime('%H:%M:%S')) + row.append(_('Yes') if op.require_checkin_attention else _('No')) + row.append(op.order.comment or "") + + if op.seat: + row += [ + op.seat.seat_guid, + str(op.seat), + op.seat.zone_name, + op.seat.row_name, + op.seat.seat_number, + ] else: - acache[a.question_id] = str(a) - for q in questions: - row.append(acache.get(q.pk, '')) + row += ['', '', '', '', ''] - row.append(op.company or ia.company) - row.append(op.voucher.code if op.voucher else "") - row.append(op.order.datetime.astimezone(self.event.timezone).strftime('%Y-%m-%d')) - row.append(op.order.datetime.astimezone(self.event.timezone).strftime('%H:%M:%S')) - row.append(_('Yes') if op.require_checkin_attention else _('No')) - row.append(op.order.comment or "") - - if op.seat: row += [ - op.seat.seat_guid, - str(op.seat), - op.seat.zone_name, - op.seat.row_name, - op.seat.seat_number, + _('Yes') if op.blocked else '', + date_format(op.valid_from, 'SHORT_DATETIME_FORMAT') if op.valid_from else '', + date_format(op.valid_until, 'SHORT_DATETIME_FORMAT') if op.valid_until else '', + ] + if (op.street or op.zipcode or op.city): + address = op + else: + address = ia + row += [ + address.street or '', + address.zipcode or '', + address.city or '', + address.country if address.country else '', + address.state or '', ] - else: - row += ['', '', '', '', ''] - row += [ - _('Yes') if op.blocked else '', - date_format(op.valid_from, 'SHORT_DATETIME_FORMAT') if op.valid_from else '', - date_format(op.valid_until, 'SHORT_DATETIME_FORMAT') if op.valid_until else '', - ] - if (op.street or op.zipcode or op.city): - address = op - else: - address = ia - row += [ - address.street or '', - address.zipcode or '', - address.city or '', - address.country if address.country else '', - address.state or '', - ] - - yield row + yield row def get_filename(self): return '{}_checkin_{}'.format(self.event.slug, safe_for_filename(self.cl.name))