]> git.parisson.com Git - teleforma.git/commitdiff
option to use S3 object storage for private documents
authorGuillaume Pellerin <guillaume.pellerin@free.fr>
Wed, 24 Apr 2024 07:07:28 +0000 (09:07 +0200)
committerGuillaume Pellerin <guillaume.pellerin@free.fr>
Wed, 24 Apr 2024 07:07:28 +0000 (09:07 +0200)
app/local_settings.py.sample
app/settings.py
poetry.lock
pyproject.toml
teleforma/admin.py
teleforma/models/core.py

index 054934a06f0971c03afa405246a19ab56f741edf..1ecfb00c34b6c6c2286df936a3b26becadc27dfc 100644 (file)
@@ -68,9 +68,3 @@ DEFAULT_TO_EMAIL = 'recipient@recipient.org' # default recipient, for your tests
 EMAIL_SUBJECT_PREFIX = "[PREFIX]" # prefix title in email
 SITE_TITLE = 'Your Site'
 SITE_TAGLINE = 'This is a Mezzo site'
-
-AUTHENTICATION_BACKENDS = (
-    # "organization.core.backend.OrganizationLDAPBackend",
-    "mezzanine.core.auth_backends.MezzanineBackend",
-    "guardian.backends.ObjectPermissionBackend",
-)
index 0189154b3abc718419464252b1b74880c276ff63..a635051b873f6964ac35a886e87845b568f712e1 100644 (file)
@@ -136,6 +136,8 @@ MEDIA_ROOT = '/srv/media/'
 #MEDIA_URL = 'http://pre-barreau.com/archives/'
 MEDIA_URL = '/media/'
 
+DEFAULT_FILE_STORAGE='django.core.files.storage.FileSystemStorage'
+
 # Absolute path to the directory static files should be collected to.
 # Don't put anything in this directory yourself; store your static files
 # in apps' "static/" subdirectories and in STATICFILES_DIRS.
@@ -210,6 +212,7 @@ INSTALLED_APPS = (
     'pdfannotator',
     'rest_framework',
     'rest_framework.authtoken',
+    'storages',
 )
 
 
@@ -266,6 +269,14 @@ TELEFORMA_EXAM_MAX_SESSIONS = 99
 TELEFORMA_EXAM_SCRIPT_MAX_SIZE = 31457280
 TELEFORMA_EXAM_SCRIPT_SERVICE_URL = '/webviewer/teleforma.html'
 TELEFORMA_PRIVATE_DOCUMENTS_MODE = False
+TELEFORMA_PRIVATE_MEDIA_USE_S3 = False
+
+AWS_ACCESS_KEY_ID=""
+AWS_SECRET_ACCESS_KEY=""
+AWS_STORAGE_BUCKET_NAME=""
+AWS_S3_REGION_NAME=""
+AWS_S3_ENDPOINT_URL=""
+AWS_S3_ENCRYPTION=True
 
 EMAIL_HOST = 'angus.parisson.com'
 DEFAULT_FROM_EMAIL = 'crfpa@pre-barreau.com'
index 09b0fe74ded6a9743575e4ccf272ccfe83515ea7..b99ccfd0561d58fe73962bec8ce83ea7097fc38e 100644 (file)
@@ -153,6 +153,49 @@ files = [
 [package.dependencies]
 jxmlease = "*"
 
+[[package]]
+name = "boto3"
+version = "1.34.89"
+description = "The AWS SDK for Python"
+category = "main"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "boto3-1.34.89-py3-none-any.whl", hash = "sha256:f9166f485d64b012d46acd212fb29a45b195a85ff66a645b05b06d9f7572af36"},
+    {file = "boto3-1.34.89.tar.gz", hash = "sha256:e0940e43810fe82f5b77442c751491fcc2768af7e7c3e8c15ea158e1ca9b586c"},
+]
+
+[package.dependencies]
+botocore = ">=1.34.89,<1.35.0"
+jmespath = ">=0.7.1,<2.0.0"
+s3transfer = ">=0.10.0,<0.11.0"
+
+[package.extras]
+crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
+
+[[package]]
+name = "botocore"
+version = "1.34.89"
+description = "Low-level, data-driven core of boto 3."
+category = "main"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "botocore-1.34.89-py3-none-any.whl", hash = "sha256:35205ed7db13058a3f7114c28e93058a8ff1490dfc6a5b5dff9c581c738fbf59"},
+    {file = "botocore-1.34.89.tar.gz", hash = "sha256:6624b69bcdf2c5d0568b7bc9cbac13e605f370e7ea06710c61e2e2dc76831141"},
+]
+
+[package.dependencies]
+jmespath = ">=0.7.1,<2.0.0"
+python-dateutil = ">=2.1,<3.0.0"
+urllib3 = [
+    {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""},
+    {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""},
+]
+
+[package.extras]
+crt = ["awscrt (==0.20.9)"]
+
 [[package]]
 name = "cairocffi"
 version = "1.6.1"
@@ -682,6 +725,31 @@ files = [
 [package.dependencies]
 django = ">1.11,<4.0"
 
+[[package]]
+name = "django-storages"
+version = "1.14.2"
+description = "Support for many storage backends in Django"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "django-storages-1.14.2.tar.gz", hash = "sha256:51b36af28cc5813b98d5f3dfe7459af638d84428c8df4a03990c7d74d1bea4e5"},
+    {file = "django_storages-1.14.2-py3-none-any.whl", hash = "sha256:1db759346b52ada6c2efd9f23d8241ecf518813eb31db9e2589207174f58f6ad"},
+]
+
+[package.dependencies]
+boto3 = {version = ">=1.4.4", optional = true, markers = "extra == \"s3\""}
+Django = ">=3.2"
+
+[package.extras]
+azure = ["azure-core (>=1.13)", "azure-storage-blob (>=12)"]
+boto3 = ["boto3 (>=1.4.4)"]
+dropbox = ["dropbox (>=7.2.1)"]
+google = ["google-cloud-storage (>=1.27)"]
+libcloud = ["apache-libcloud"]
+s3 = ["boto3 (>=1.4.4)"]
+sftp = ["paramiko (>=1.15)"]
+
 [[package]]
 name = "django-tinymce"
 version = "3.3.0"
@@ -1074,6 +1142,18 @@ MarkupSafe = ">=2.0"
 [package.extras]
 i18n = ["Babel (>=2.7)"]
 
+[[package]]
+name = "jmespath"
+version = "1.0.1"
+description = "JSON Matching Expressions"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"},
+    {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
+]
+
 [[package]]
 name = "jxmlease"
 version = "1.0.3"
@@ -1491,6 +1571,21 @@ files = [
 doc = ["sphinx", "sphinx_rtd_theme"]
 test = ["flake8", "isort", "pytest"]
 
+[[package]]
+name = "python-dateutil"
+version = "2.9.0.post0"
+description = "Extensions to the standard Python datetime module"
+category = "main"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
+files = [
+    {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"},
+    {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"},
+]
+
+[package.dependencies]
+six = ">=1.5"
+
 [[package]]
 name = "python-dotenv"
 version = "1.0.1"
@@ -1663,6 +1758,24 @@ idna = {version = "*", optional = true, markers = "extra == \"idna2008\""}
 [package.extras]
 idna2008 = ["idna"]
 
+[[package]]
+name = "s3transfer"
+version = "0.10.1"
+description = "An Amazon S3 Transfer Manager"
+category = "main"
+optional = false
+python-versions = ">= 3.8"
+files = [
+    {file = "s3transfer-0.10.1-py3-none-any.whl", hash = "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d"},
+    {file = "s3transfer-0.10.1.tar.gz", hash = "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19"},
+]
+
+[package.dependencies]
+botocore = ">=1.33.2,<2.0a.0"
+
+[package.extras]
+crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"]
+
 [[package]]
 name = "service-identity"
 version = "24.1.0"
@@ -1898,6 +2011,23 @@ files = [
     {file = "Unidecode-1.2.0.tar.gz", hash = "sha256:8d73a97d387a956922344f6b74243c2c6771594659778744b2dbdaad8f6b727d"},
 ]
 
+[[package]]
+name = "urllib3"
+version = "1.26.18"
+description = "HTTP library with thread-safe connection pooling, file post, and more."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+files = [
+    {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"},
+    {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"},
+]
+
+[package.extras]
+brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
+secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
+socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
+
 [[package]]
 name = "urllib3"
 version = "2.2.1"
@@ -2308,4 +2438,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "e4c0fb2ee0b9e0f2d378d1f2b6c57a4e6c2789419f70025ef40ba226bcecab24"
+content-hash = "6fb77c2e4082ac217d649e1f65e655efd6c739b0b3c5a97278d7b5d33f8464e4"
index cf6cbd8cccb7092d232b1a671f6f4734b691ea74..f39bea9893bfa3db54df9012cd06362ecc7f7276 100644 (file)
@@ -39,6 +39,8 @@ redis = "3.5.3"
 uwsgi = "2.0.25.1"
 uvicorn = {version = "0.18.1", extras = ["standard"]}
 httpx = "0.23.3"
+django-storages = {extras = ["s3"], version = "^1.14.2"}
+boto3 = "^1.34.89"
 
 
 [build-system]
index 5ccf49b3a9af5fd826606bb42265a7f9fad9bb1c..89cd4add5bbb82587e827b8e9bdb8dcad1d0c62e 100644 (file)
@@ -23,7 +23,7 @@ from .exam.admin import QuotaInline
 from .models.appointment import (Appointment, AppointmentJury,
                                  AppointmentPeriod, AppointmentSlot)
 from .models.core import (Conference, ConferencePublication, Course, CourseType, Department, Document,
-                          DocumentSimple, DocumentType, LiveStream, Media,
+                          DocumentSimple, DocumentType, LiveStream, Media, DocumentPrivate,
                           MediaTranscoded, Organization, Period, Professor,
                           Room, StreamingServer)
 from .models.crfpa import (IEJ, Corrector, Discount, Home, NewsItem,
@@ -537,6 +537,7 @@ admin.site.register(IEJ)
 admin.site.register(Document, DocumentAdmin)
 admin.site.register(DocumentSimple)
 admin.site.register(DocumentType)
+admin.site.register(DocumentPrivate)
 admin.site.register(Media, MediaAdmin)
 admin.site.register(Room)
 admin.site.register(User, UserProfileAdmin)
index 3cf902e52f3f1c1bb12d6f419b05bb80c3842eb1..1cf1096df802fc0afe17dc9204c67d85bd68da3b 100644 (file)
@@ -41,10 +41,13 @@ import string
 import random
 import requests
 import asyncio
+import tempfile
 
 import django.db.models as models
 from django.conf import settings
 from django.contrib.auth.models import User
+from django.core.files import File
+from django.core.files.storage import FileSystemStorage
 from django.core.paginator import InvalidPage
 from django.db import models
 from django.forms.fields import FileField
@@ -61,6 +64,8 @@ from sorl.thumbnail import default as sorl_default
 from pypdf import PdfWriter
 import httpx
 
+from storages.backends.s3boto3 import S3Boto3Storage
+
 
 HAS_TELEMETA = False
 if 'telemeta' in settings.INSTALLED_APPS:
@@ -111,6 +116,12 @@ STATUS_CHOICES = (
 WEIGHT_CHOICES = get_nint_choices(5)
 
 
+if settings.TELEFORMA_PRIVATE_MEDIA_USE_S3:
+    private_storage = S3Boto3Storage
+else:
+    private_storage = FileSystemStorage
+
+
 def get_random_hash():
     hash = random.getrandbits(128)
     return "%032x" % hash
@@ -852,28 +863,29 @@ class DocumentPrivate(MediaBase):
                                 null=True, blank=True, on_delete=models.CASCADE)
     user = models.ForeignKey(User, related_name="private_documents", verbose_name=_('user'),
                                 null=True, blank=True, on_delete=models.CASCADE)
-    file = models.FileField(_('file'), upload_to='private_documents/%Y/%m/%d', db_column="filename",
-                            max_length=1024, blank=True)
+    file = models.FileField(_('file'), upload_to='private/documents/%Y/%m/%d', db_column="filename",
+                            max_length=1024, blank=True, storage=private_storage)
+
     class Meta(MetaCore):
         db_table = app_label + '_' + 'document_private'
         ordering = ['-date_added']
 
     def save(self, **kwargs):
+        temp_file = None
         if "pdf" in self.document.mime_type:
             writer = PdfWriter(clone_from=self.document.file.path)
             writer.add_metadata({"/Downloader": self.user.username})
             writer.add_metadata({"/Publisher": settings.TELEFORMA_ORGANIZATION})
             writer.add_metadata({"/Copyright": settings.TELEFORMA_ORGANIZATION})
             filename = self.document.file.name.split(os.sep)[-1]
-            today = datetime.date.today()
-            root = settings.MEDIA_ROOT + "private/documents/%s/%s/%s/" % (today.year, today.month, today.day)
-            if not os.path.exists(root):
-                os.makedirs(root)
-            path = root + self.user.username + "--" + filename
-            with open(path, "wb") as f:
-                writer.write(f)
-            self.file = path
+            name = self.user.username + "--" + filename
+            temp_file = tempfile.SpooledTemporaryFile()
+            writer.write(temp_file)
+            self.file = File(temp_file, name=name)
         super(DocumentPrivate, self).save(**kwargs)
+        if temp_file:
+            temp_file.close()
+            del temp_file
 
 
 class DocumentSimple(MediaBase):