Fix memory usage in exporters by using chunked iterators

2026-08-27 13:24:41 +00:00 · 2020-07-23 20:39:49 +02:00
parent bff1041878
commit fc5c3caf66
3 changed files with 179 additions and 142 deletions
@@ -0,0 +1,10 @@
+import itertools
+
+
+def chunked_iterable(iterable, size):
+    it = iter(iterable)
+    while True:
+        chunk = tuple(itertools.islice(it, size))
+        if not chunk:
+            break
+        yield chunk