From 2f0f611d12b5bb24146fd3a9e403a095b80521cb Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Mon, 20 Jan 2014 23:44:46 +0100 Subject: [PATCH] dratically improve collection zip packaqe streaming thanks to zipstream Conflicts: telemeta/views/collection.py --- telemeta/views/collection.py | 64 +++++++++++++----------------------- 1 file changed, 22 insertions(+), 42 deletions(-) diff --git a/telemeta/views/collection.py b/telemeta/views/collection.py index a0aa61cb..0adb015e 100644 --- a/telemeta/views/collection.py +++ b/telemeta/views/collection.py @@ -167,7 +167,7 @@ class CollectionPackageView(View): def get_object(self): return MediaCollection.objects.get(public_id=self.kwargs['public_id']) - def get_stream(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): """ Stream a ZIP file of collection data without loading the whole file into memory. @@ -179,11 +179,28 @@ class CollectionPackageView(View): import zipstream z = zipstream.ZipFile() - collection = MediaCollection.objects.get(public_id=public_id) - z.write(collection.code) + cache_data = TelemetaCache(settings.TELEMETA_DATA_CACHE_DIR) + collection = self.get_object() + serializer = CollectionSerializer(collection) + + data = serializer.get_xml().encode("utf-8") + filename = collection.public_id + '.xml' + cache_data.write_bin(data, filename) + path = cache_data.dir + os.sep + filename + z.write(path, collection.public_id + os.sep + filename) + for item in collection.items.all(): - z.write(item.file.path) + filename = item.file.path.split(os.sep)[-1] + z.write(item.file.path, arcname=collection.public_id + os.sep + filename) + marker_view = MarkerView() + markers = marker_view.get_markers(item.id) + if markers: + data = json.dumps(markers) + filename = item.code + '.json' + cache_data.write_bin(data, filename) + path = cache_data.dir + os.sep + filename + z.write(path, arcname=collection.public_id + os.sep + filename) try: from django.http import StreamingHttpResponse @@ -194,44 +211,7 @@ class CollectionPackageView(View): response['Content-Disposition'] = "attachment; filename=%s.%s" % \ (item.code, 'zip') return response - + @method_decorator(login_required) def dispatch(self, *args, **kwargs): return super(CollectionPackageView, self).dispatch(*args, **kwargs) - - - - """ - Create a ZIP file on disk and transmit it in chunks of 8KB, - without loading the whole file into memory. A similar approach can - be used for large dynamic PDF files. - """ - - collection = self.get_object() - tmp_dir = getattr(settings, "FILE_UPLOAD_TEMP_DIR") - if not tmp_dir: - tmp_dir = '/tmp' - temp = tempfile.TemporaryFile(prefix=tmp_dir+os.sep) - archive = zipfile.ZipFile(temp, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) - serializer = CollectionSerializer(collection) - archive.writestr('%s/%s%s' % (collection.code, collection.code, '.xml'), - serializer.get_xml().encode("utf-8")) - - for item in collection.items.all(): - if item.file: - ext = os.path.splitext(item.file.path)[1] - archive.write(item.file.path, '%s/%s%s' % (collection.code, item.code, ext)) - marker_view = MarkerView() - markers = marker_view.get_markers(item.id) - if markers: - archive.writestr('%s/%s%s' % (collection.code, item.code, '.json'), json.dumps(markers)) - - archive.close() - wrapper = FixedFileWrapper(temp) - response = HttpResponse(wrapper, content_type='application/zip') - - response['Content-Disposition'] = "attachment; filename=%s.%s" % \ - (collection.code, 'zip') - response['Content-Length'] = temp.tell() - temp.seek(0) - return response -- 2.39.5