From 4a79bae884db67061d48c5f9be508bba9d55e874 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Wed, 24 Apr 2024 09:07:28 +0200 Subject: [PATCH] option to use S3 object storage for private documents --- app/local_settings.py.sample | 6 -- app/settings.py | 11 +++ poetry.lock | 132 ++++++++++++++++++++++++++++++++++- pyproject.toml | 2 + teleforma/admin.py | 3 +- teleforma/models/core.py | 32 ++++++--- 6 files changed, 168 insertions(+), 18 deletions(-) diff --git a/app/local_settings.py.sample b/app/local_settings.py.sample index 054934a0..1ecfb00c 100644 --- a/app/local_settings.py.sample +++ b/app/local_settings.py.sample @@ -68,9 +68,3 @@ DEFAULT_TO_EMAIL = 'recipient@recipient.org' # default recipient, for your tests EMAIL_SUBJECT_PREFIX = "[PREFIX]" # prefix title in email SITE_TITLE = 'Your Site' SITE_TAGLINE = 'This is a Mezzo site' - -AUTHENTICATION_BACKENDS = ( - # "organization.core.backend.OrganizationLDAPBackend", - "mezzanine.core.auth_backends.MezzanineBackend", - "guardian.backends.ObjectPermissionBackend", -) diff --git a/app/settings.py b/app/settings.py index 0189154b..a635051b 100644 --- a/app/settings.py +++ b/app/settings.py @@ -136,6 +136,8 @@ MEDIA_ROOT = '/srv/media/' #MEDIA_URL = 'http://pre-barreau.com/archives/' MEDIA_URL = '/media/' +DEFAULT_FILE_STORAGE='django.core.files.storage.FileSystemStorage' + # Absolute path to the directory static files should be collected to. # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. @@ -210,6 +212,7 @@ INSTALLED_APPS = ( 'pdfannotator', 'rest_framework', 'rest_framework.authtoken', + 'storages', ) @@ -266,6 +269,14 @@ TELEFORMA_EXAM_MAX_SESSIONS = 99 TELEFORMA_EXAM_SCRIPT_MAX_SIZE = 31457280 TELEFORMA_EXAM_SCRIPT_SERVICE_URL = '/webviewer/teleforma.html' TELEFORMA_PRIVATE_DOCUMENTS_MODE = False +TELEFORMA_PRIVATE_MEDIA_USE_S3 = False + +AWS_ACCESS_KEY_ID="" +AWS_SECRET_ACCESS_KEY="" +AWS_STORAGE_BUCKET_NAME="" +AWS_S3_REGION_NAME="" +AWS_S3_ENDPOINT_URL="" +AWS_S3_ENCRYPTION=True EMAIL_HOST = 'angus.parisson.com' DEFAULT_FROM_EMAIL = 'crfpa@pre-barreau.com' diff --git a/poetry.lock b/poetry.lock index 09b0fe74..b99ccfd0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -153,6 +153,49 @@ files = [ [package.dependencies] jxmlease = "*" +[[package]] +name = "boto3" +version = "1.34.89" +description = "The AWS SDK for Python" +category = "main" +optional = false +python-versions = ">=3.8" +files = [ + {file = "boto3-1.34.89-py3-none-any.whl", hash = "sha256:f9166f485d64b012d46acd212fb29a45b195a85ff66a645b05b06d9f7572af36"}, + {file = "boto3-1.34.89.tar.gz", hash = "sha256:e0940e43810fe82f5b77442c751491fcc2768af7e7c3e8c15ea158e1ca9b586c"}, +] + +[package.dependencies] +botocore = ">=1.34.89,<1.35.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.10.0,<0.11.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.34.89" +description = "Low-level, data-driven core of boto 3." +category = "main" +optional = false +python-versions = ">=3.8" +files = [ + {file = "botocore-1.34.89-py3-none-any.whl", hash = "sha256:35205ed7db13058a3f7114c28e93058a8ff1490dfc6a5b5dff9c581c738fbf59"}, + {file = "botocore-1.34.89.tar.gz", hash = "sha256:6624b69bcdf2c5d0568b7bc9cbac13e605f370e7ea06710c61e2e2dc76831141"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = [ + {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, + {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}, +] + +[package.extras] +crt = ["awscrt (==0.20.9)"] + [[package]] name = "cairocffi" version = "1.6.1" @@ -682,6 +725,31 @@ files = [ [package.dependencies] django = ">1.11,<4.0" +[[package]] +name = "django-storages" +version = "1.14.2" +description = "Support for many storage backends in Django" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "django-storages-1.14.2.tar.gz", hash = "sha256:51b36af28cc5813b98d5f3dfe7459af638d84428c8df4a03990c7d74d1bea4e5"}, + {file = "django_storages-1.14.2-py3-none-any.whl", hash = "sha256:1db759346b52ada6c2efd9f23d8241ecf518813eb31db9e2589207174f58f6ad"}, +] + +[package.dependencies] +boto3 = {version = ">=1.4.4", optional = true, markers = "extra == \"s3\""} +Django = ">=3.2" + +[package.extras] +azure = ["azure-core (>=1.13)", "azure-storage-blob (>=12)"] +boto3 = ["boto3 (>=1.4.4)"] +dropbox = ["dropbox (>=7.2.1)"] +google = ["google-cloud-storage (>=1.27)"] +libcloud = ["apache-libcloud"] +s3 = ["boto3 (>=1.4.4)"] +sftp = ["paramiko (>=1.15)"] + [[package]] name = "django-tinymce" version = "3.3.0" @@ -1074,6 +1142,18 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "jxmlease" version = "1.0.3" @@ -1491,6 +1571,21 @@ files = [ doc = ["sphinx", "sphinx_rtd_theme"] test = ["flake8", "isort", "pytest"] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + [[package]] name = "python-dotenv" version = "1.0.1" @@ -1663,6 +1758,24 @@ idna = {version = "*", optional = true, markers = "extra == \"idna2008\""} [package.extras] idna2008 = ["idna"] +[[package]] +name = "s3transfer" +version = "0.10.1" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.8" +files = [ + {file = "s3transfer-0.10.1-py3-none-any.whl", hash = "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d"}, + {file = "s3transfer-0.10.1.tar.gz", hash = "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19"}, +] + +[package.dependencies] +botocore = ">=1.33.2,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] + [[package]] name = "service-identity" version = "24.1.0" @@ -1898,6 +2011,23 @@ files = [ {file = "Unidecode-1.2.0.tar.gz", hash = "sha256:8d73a97d387a956922344f6b74243c2c6771594659778744b2dbdaad8f6b727d"}, ] +[[package]] +name = "urllib3" +version = "1.26.18" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "urllib3" version = "2.2.1" @@ -2308,4 +2438,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "e4c0fb2ee0b9e0f2d378d1f2b6c57a4e6c2789419f70025ef40ba226bcecab24" +content-hash = "6fb77c2e4082ac217d649e1f65e655efd6c739b0b3c5a97278d7b5d33f8464e4" diff --git a/pyproject.toml b/pyproject.toml index cf6cbd8c..f39bea98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,8 @@ redis = "3.5.3" uwsgi = "2.0.25.1" uvicorn = {version = "0.18.1", extras = ["standard"]} httpx = "0.23.3" +django-storages = {extras = ["s3"], version = "^1.14.2"} +boto3 = "^1.34.89" [build-system] diff --git a/teleforma/admin.py b/teleforma/admin.py index 5ccf49b3..89cd4add 100644 --- a/teleforma/admin.py +++ b/teleforma/admin.py @@ -23,7 +23,7 @@ from .exam.admin import QuotaInline from .models.appointment import (Appointment, AppointmentJury, AppointmentPeriod, AppointmentSlot) from .models.core import (Conference, ConferencePublication, Course, CourseType, Department, Document, - DocumentSimple, DocumentType, LiveStream, Media, + DocumentSimple, DocumentType, LiveStream, Media, DocumentPrivate, MediaTranscoded, Organization, Period, Professor, Room, StreamingServer) from .models.crfpa import (IEJ, Corrector, Discount, Home, NewsItem, @@ -537,6 +537,7 @@ admin.site.register(IEJ) admin.site.register(Document, DocumentAdmin) admin.site.register(DocumentSimple) admin.site.register(DocumentType) +admin.site.register(DocumentPrivate) admin.site.register(Media, MediaAdmin) admin.site.register(Room) admin.site.register(User, UserProfileAdmin) diff --git a/teleforma/models/core.py b/teleforma/models/core.py index 3cf902e5..1cf1096d 100644 --- a/teleforma/models/core.py +++ b/teleforma/models/core.py @@ -41,10 +41,13 @@ import string import random import requests import asyncio +import tempfile import django.db.models as models from django.conf import settings from django.contrib.auth.models import User +from django.core.files import File +from django.core.files.storage import FileSystemStorage from django.core.paginator import InvalidPage from django.db import models from django.forms.fields import FileField @@ -61,6 +64,8 @@ from sorl.thumbnail import default as sorl_default from pypdf import PdfWriter import httpx +from storages.backends.s3boto3 import S3Boto3Storage + HAS_TELEMETA = False if 'telemeta' in settings.INSTALLED_APPS: @@ -111,6 +116,12 @@ STATUS_CHOICES = ( WEIGHT_CHOICES = get_nint_choices(5) +if settings.TELEFORMA_PRIVATE_MEDIA_USE_S3: + private_storage = S3Boto3Storage +else: + private_storage = FileSystemStorage + + def get_random_hash(): hash = random.getrandbits(128) return "%032x" % hash @@ -852,28 +863,29 @@ class DocumentPrivate(MediaBase): null=True, blank=True, on_delete=models.CASCADE) user = models.ForeignKey(User, related_name="private_documents", verbose_name=_('user'), null=True, blank=True, on_delete=models.CASCADE) - file = models.FileField(_('file'), upload_to='private_documents/%Y/%m/%d', db_column="filename", - max_length=1024, blank=True) + file = models.FileField(_('file'), upload_to='private/documents/%Y/%m/%d', db_column="filename", + max_length=1024, blank=True, storage=private_storage) + class Meta(MetaCore): db_table = app_label + '_' + 'document_private' ordering = ['-date_added'] def save(self, **kwargs): + temp_file = None if "pdf" in self.document.mime_type: writer = PdfWriter(clone_from=self.document.file.path) writer.add_metadata({"/Downloader": self.user.username}) writer.add_metadata({"/Publisher": settings.TELEFORMA_ORGANIZATION}) writer.add_metadata({"/Copyright": settings.TELEFORMA_ORGANIZATION}) filename = self.document.file.name.split(os.sep)[-1] - today = datetime.date.today() - root = settings.MEDIA_ROOT + "private/documents/%s/%s/%s/" % (today.year, today.month, today.day) - if not os.path.exists(root): - os.makedirs(root) - path = root + self.user.username + "--" + filename - with open(path, "wb") as f: - writer.write(f) - self.file = path + name = self.user.username + "--" + filename + temp_file = tempfile.SpooledTemporaryFile() + writer.write(temp_file) + self.file = File(temp_file, name=name) super(DocumentPrivate, self).save(**kwargs) + if temp_file: + temp_file.close() + del temp_file class DocumentSimple(MediaBase): -- 2.39.5