From: Guillaume Pellerin Date: Fri, 9 Jan 2026 10:20:32 +0000 (+0100) Subject: use s3 to manage script.file, add script.size, cleanup X-Git-Url: https://git.parisson.com/?a=commitdiff_plain;h=7ea7ed58de0aa231f160705c1a7e1f97b852b1b4;p=teleforma.git use s3 to manage script.file, add script.size, cleanup --- diff --git a/poetry.lock b/poetry.lock index cce6a245..4f5e13ba 100644 --- a/poetry.lock +++ b/poetry.lock @@ -788,13 +788,13 @@ django = ">1.11,<4.0" [[package]] name = "django-storages" -version = "1.14.4" +version = "1.14.6" description = "Support for many storage backends in Django" optional = false python-versions = ">=3.7" files = [ - {file = "django-storages-1.14.4.tar.gz", hash = "sha256:69aca94d26e6714d14ad63f33d13619e697508ee33ede184e462ed766dc2a73f"}, - {file = "django_storages-1.14.4-py3-none-any.whl", hash = "sha256:d61930acb4a25e3aebebc6addaf946a3b1df31c803a6bf1af2f31c9047febaa3"}, + {file = "django_storages-1.14.6-py3-none-any.whl", hash = "sha256:11b7b6200e1cb5ffcd9962bd3673a39c7d6a6109e8096f0e03d46fab3d3aabd9"}, + {file = "django_storages-1.14.6.tar.gz", hash = "sha256:7a25ce8f4214f69ac9c7ce87e2603887f7ae99326c316bc8d2d75375e09341c9"}, ] [package.dependencies] @@ -805,7 +805,7 @@ Django = ">=3.2" azure = ["azure-core (>=1.13)", "azure-storage-blob (>=12)"] boto3 = ["boto3 (>=1.4.4)"] dropbox = ["dropbox (>=7.2.1)"] -google = ["google-cloud-storage (>=1.27)"] +google = ["google-cloud-storage (>=1.36.1)"] libcloud = ["apache-libcloud"] s3 = ["boto3 (>=1.4.4)"] sftp = ["paramiko (>=1.15)"] diff --git a/teleforma/exam/admin.py b/teleforma/exam/admin.py index 385d7a48..aa63c9c4 100644 --- a/teleforma/exam/admin.py +++ b/teleforma/exam/admin.py @@ -69,12 +69,12 @@ class ScriptAdmin(admin.ModelAdmin): def file_size(self, instance): if instance.file: - if os.path.exists(instance.file.path): - return filesizeformat(os.stat(instance.file.path).st_size) - else: - return '0' + try: + return filesizeformat(instance.size) + except: + return 'undefined' else: - return '0' + return 'none' def force_resubmit(self, request, queryset): for script in queryset.all(): diff --git a/teleforma/exam/migrations/0004_auto_20260109_1116.py b/teleforma/exam/migrations/0004_auto_20260109_1116.py new file mode 100644 index 00000000..f1cfbfc1 --- /dev/null +++ b/teleforma/exam/migrations/0004_auto_20260109_1116.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.25 on 2026-01-09 11:16 + +from django.db import migrations, models +import storages.backends.s3 + + +class Migration(migrations.Migration): + + dependencies = [ + ('exam', '0003_alter_quota_session'), + ] + + operations = [ + migrations.AddField( + model_name='script', + name='size', + field=models.IntegerField(blank=True, null=True, verbose_name='size'), + ), + migrations.AlterField( + model_name='script', + name='file', + field=models.FileField(blank=True, max_length=1024, storage=storages.backends.s3.S3Storage, upload_to='scripts/%Y/%m/%d', verbose_name='PDF file'), + ), + ] diff --git a/teleforma/exam/models.py b/teleforma/exam/models.py index ddc3865e..baf8b304 100755 --- a/teleforma/exam/models.py +++ b/teleforma/exam/models.py @@ -282,6 +282,7 @@ class Script(BaseResource): date_rejected = models.DateTimeField( _('date rejected'), null=True, blank=True) url = models.CharField(_('URL'), max_length=2048, blank=True) + size = models.IntegerField(_('size'), blank=True, null=True) @property def title(self): @@ -402,21 +403,6 @@ class Script(BaseResource): if not self.url: self.url = settings.MEDIA_URL + str(new_rel) - @property - def safe_url(self): - domain = Site.objects.get_current().domain - url = self.url - url = url.replace( - 'http://e-learning.crfpa.pre-barreau.com', '//' + domain) - return urllib.quote(url) - - def unquoted_url(self): - domain = Site.objects.get_current().domain - url = self.url - url = url.replace( - 'http://e-learning.crfpa.pre-barreau.com', '//' + domain) - return url - def has_annotations_file(self): """ check if an annotations file exists. Then use this file, otherwise use the new db implementation @@ -431,17 +417,11 @@ class Script(BaseResource): # self.save() def submit(self): - self.box_upload_done = 0 - if not self.file: self.auto_reject('no file') return - if not os.path.exists(self.file.path): - self.auto_reject('file not found') - return - - mime_type = guess_mimetypes(self.file.path) + mime_type = guess_mimetypes(self.file.url) if mime_type: if not 'pdf' in mime_type: self.auto_reject('wrong format') @@ -450,14 +430,11 @@ class Script(BaseResource): self.auto_reject('wrong format') return - if os.stat(self.file.path).st_size > settings.TELEFORMA_EXAM_SCRIPT_MAX_SIZE: + if self.file.size > settings.TELEFORMA_EXAM_SCRIPT_MAX_SIZE: self.auto_reject('file too large') return if not self.status == 0 and self.file: - # if not self.box_uuid: - # self.uuid_link() - # self.box_upload() if not self.corrector: self.auto_set_corrector() @@ -497,18 +474,10 @@ def set_file_properties(sender, instance, **kwargs): if instance.file: trig_save = False if not instance.mime_type: - mime_type = guess_mimetypes(instance.file.path) - if mime_type: - instance.mime_type = guess_mimetypes(instance.file.path) - trig_save = True - # HOTFIX - else: - instance.mime_type = 'application/pdf' - if not instance.sha1: - instance.sha1 = sha1sum_file(instance.file.path) + instance.mime_type = guess_mimetypes(instance.file.url) trig_save = True - if not instance.url: - instance.uuid_link() + if not instance.size: + instance.size = instance.file.size trig_save = True if not instance.corrector: instance.submit() @@ -516,13 +485,6 @@ def set_file_properties(sender, instance, **kwargs): if trig_save: super(sender, instance).save() - # if hasattr(instance, 'image'): - # if not instance.image: - # path = cache_path + os.sep + instance.uuid + '.jpg' - # command = 'convert ' + instance.file.path + ' ' + path - # os.system(command) - # instance.image = path - post_save.connect(set_file_properties, sender=Script, dispatch_uid="script_post_save") diff --git a/teleforma/exam/templates/exam/script_detail.html b/teleforma/exam/templates/exam/script_detail.html index 11e9a916..fa849d64 100644 --- a/teleforma/exam/templates/exam/script_detail.html +++ b/teleforma/exam/templates/exam/script_detail.html @@ -146,12 +146,7 @@

-{# {% if script.has_annotations_file %}#} -{# #} -{# {% else %}#} - {% pdfannotator url=script.unquoted_url uuid=script.uuid %} -{# {% endif %}#} + {% pdfannotator url=script.file.url uuid=script.uuid %}
diff --git a/teleforma/exam/views.py b/teleforma/exam/views.py index d857a4d1..6566b68e 100755 --- a/teleforma/exam/views.py +++ b/teleforma/exam/views.py @@ -5,6 +5,7 @@ import datetime import io import json +import urllib import numpy as np from django.conf import settings @@ -175,6 +176,7 @@ class ScriptView(ScriptMixinView, CourseAccessMixin, UpdateView): def serve(self, request, pk, streaming=False): script = self.get_object() + print(script) bucket = False if settings.TELEFORMA_EXAM_USE_S3: @@ -202,7 +204,9 @@ class ScriptDownloadView(ScriptMixinView, CourseAccessMixin): script = get_object_or_404(Script, id=kwargs['pk']) # existing PDF - existing_pdf = PdfReader(script.file.path) + remote_file = urllib.request.urlopen(script.file.url).read() + memory_file = io.BytesIO(remote_file) + existing_pdf = PdfReader(memory_file) width = existing_pdf.pages[0].mediabox.width height = existing_pdf.pages[0].mediabox.height @@ -290,6 +294,10 @@ class ScriptDownloadView(ScriptMixinView, CourseAccessMixin): response = HttpResponse(content_type='application/pdf') response['Content-Disposition'] = 'attachment; filename="%s"' % script.file.name output.write(response) + + remote_file = None + memory_file = None + return response