From 14b5041bd1714a6042cbbd9643985ad92da74f88 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Mon, 27 Apr 2015 18:35:53 +0200 Subject: [PATCH] update and cleanup import commands, add more options --- .../management/commands/telemeta-cleanup.py | 17 ++- .../telemeta-import-collection-from-dir.py | 114 ++++++++++++++++++ ... telemeta-import-collection-from-files.py} | 4 +- ...=> telemeta-import-collection-from-url.py} | 0 ...crem-import.py => telemeta-import-crem.py} | 22 ++-- ...-formats.py => telemeta-import-formats.py} | 0 .../telemeta-import-items-without-copy.py | 2 +- .../commands/telemeta-import-items.py | 43 ------- .../commands/telemeta-test-import.py | 65 ---------- 9 files changed, 137 insertions(+), 130 deletions(-) create mode 100644 telemeta/management/commands/telemeta-import-collection-from-dir.py rename telemeta/management/commands/{telemeta-media-import.py => telemeta-import-collection-from-files.py} (98%) rename telemeta/management/commands/{telemeta-import-items-from-url.py => telemeta-import-collection-from-url.py} (100%) rename telemeta/management/commands/{telemeta-crem-import.py => telemeta-import-crem.py} (94%) rename telemeta/management/commands/{telemeta-init-formats.py => telemeta-import-formats.py} (100%) delete mode 100644 telemeta/management/commands/telemeta-import-items.py delete mode 100644 telemeta/management/commands/telemeta-test-import.py diff --git a/telemeta/management/commands/telemeta-cleanup.py b/telemeta/management/commands/telemeta-cleanup.py index 35f47a8e..1e9be8c4 100644 --- a/telemeta/management/commands/telemeta-cleanup.py +++ b/telemeta/management/commands/telemeta-cleanup.py @@ -9,24 +9,21 @@ import codecs class Command(BaseCommand): help = "Cleanup DB : multiple analyses, data cache, export cache, etc.." - args = "None" - cache_data = None - cache_export = None + args = "cache" + cache_data = TelemetaCache(settings.TELEMETA_DATA_CACHE_DIR) + cache_export = TelemetaCache(settings.TELEMETA_EXPORT_CACHE_DIR) def handle(self, *args, **options): - if 'cache' in args: - self.cache_data = TelemetaCache(settings.TELEMETA_DATA_CACHE_DIR) - self.cache_export = TelemetaCache(settings.TELEMETA_EXPORT_CACHE_DIR) - print "Cleaning all cache..." - items = MediaItem.objects.all() a_counter = 0 - print 'Cleaning multiple analyses per item...' + print 'cleaning multiple analyses per item...' for item in items: - if self.cache_data and self.cache_export: + if 'cache' in args:: + print 'cleaning cache...' self.cache_data.delete_item_data(item.code) self.cache_export.delete_item_data(item.code) + analyses = MediaItemAnalysis.objects.filter(item=item) ids = [] for analysis in analyses: diff --git a/telemeta/management/commands/telemeta-import-collection-from-dir.py b/telemeta/management/commands/telemeta-import-collection-from-dir.py new file mode 100644 index 00000000..38b3e774 --- /dev/null +++ b/telemeta/management/commands/telemeta-import-collection-from-dir.py @@ -0,0 +1,114 @@ +from optparse import make_option +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.core.files.base import ContentFile +from django.contrib.auth.models import User +from telemeta.models import * +from telemeta.util.unaccent import unaccent +import os + + +try: + from django.utils.text import slugify +except ImportError: + def slugify(string): + killed_chars = re.sub('[\(\),]', '', string) + return re.sub(' ', '_', killed_chars) + +def beautify(string): + return os.path.splitext(string)[0].replace('_',' ') + + +class Command(BaseCommand): + help = "import media files from a directory into a collection" + media_root = os.path.normpath(settings.MEDIA_ROOT) + + option_list = BaseCommand.option_list + ( + make_option('-d', '--dry-run', + action='store_true', + dest='dry-run', + help='Do NOT write anything'), + make_option('-f', '--force', + action='store_true', + dest='force', + help='Force overwrite data'), + make_option('-s', '--source', + dest='source_dir', + help='define the source directory'), + make_option('-l', '--log', + dest='log', + help='define log file'), + make_option('-p', '--pattern', + dest='pattern', + help='define the pattern'), + make_option('-u', '--username', + dest='user', + help='define the username'), + make_option('-c', '--collection-code', + action='store', + dest='collection_code', + default='default', + metavar = '', + help='collection code'), + make_option('-t', '--collection-title', + action='store', + dest='collection_title', + default='default', + metavar = '', + help='collection title'), + + ) + + def write_file(self, item, media): + filename = media.split(os.sep)[-1] + if os.path.exists(media): + if not item.file or self.force: + if not self.dry_run: + if not self.media_root in self.source_dir: + print "file not in MEDIA_ROOT, copying..." + f = open(media, 'r') + file_content = ContentFile(f.read()) + item.file.save(filename, file_content) + f.close() + else: + print "file in MEDIA_ROOT, linking..." + path = media[len(self.media_root)+1:] + item.file = path + item.save() + if self.user: + item.set_revision(self.user) + + def handle(self, *args, **options): + self.source_dir = os.path.abspath(options.get('source_dir')) + self.collection_code = options.get('collection_code') + self.collection_title = options.get('collection_title') + self.dry_run = options.get('dry-run') + self.user = None + users = User.objects.filter(username=options.get('username')) + if users: + self.user = users[0] + + collections = MediaCollection.objects.filter(code=self.collection_code) + if not collections: + collection = MediaCollection(code=self.collection_code, title=self.collection_code) + collection.public_access = 'full' + collection.save() + print 'Collection created: ' + self.collection_code + else: + collection = collections[0] + print 'Using collection: ' + collection.code + + for root, dirs, files in os.walk(self.source_dir): + for filename in files: + path = root + os.sep + filename + filename_pre, ext = os.path.splitext(filename) + item_code = collection.code + '_' + filename_pre + item, c = MediaItem.objects.get_or_create(collection=collection, code=item_code) + if c: + item.title = filename_pre + item.public_access = 'full' + self.write_file(item, path) + item.save() + print 'item created: ' + item.code + else: + print 'item already exists: ' + item.code diff --git a/telemeta/management/commands/telemeta-media-import.py b/telemeta/management/commands/telemeta-import-collection-from-files.py similarity index 98% rename from telemeta/management/commands/telemeta-media-import.py rename to telemeta/management/commands/telemeta-import-collection-from-files.py index c73d4f2f..c22c9c0b 100644 --- a/telemeta/management/commands/telemeta-media-import.py +++ b/telemeta/management/commands/telemeta-import-collection-from-files.py @@ -43,9 +43,9 @@ class Command(BaseCommand): def handle(self, *args, **options): if len(args) < 1: return - if options['title']: + if options['title']: self.title = options['title'] - if options['code']: + if options['code']: self.code = options['code'] for file in args: self.urls.append('file://' + file) diff --git a/telemeta/management/commands/telemeta-import-items-from-url.py b/telemeta/management/commands/telemeta-import-collection-from-url.py similarity index 100% rename from telemeta/management/commands/telemeta-import-items-from-url.py rename to telemeta/management/commands/telemeta-import-collection-from-url.py diff --git a/telemeta/management/commands/telemeta-crem-import.py b/telemeta/management/commands/telemeta-import-crem.py similarity index 94% rename from telemeta/management/commands/telemeta-crem-import.py rename to telemeta/management/commands/telemeta-import-crem.py index 77593df0..f73cd402 100644 --- a/telemeta/management/commands/telemeta-crem-import.py +++ b/telemeta/management/commands/telemeta-import-crem.py @@ -51,14 +51,14 @@ class Logger: class Command(BaseCommand): - - """Import CREM collections from collection directories containing media files + + """Import CREM collections from collection directories containing media files and eventually a XLS files representing the relation between old codes and new codes """ - help = "import CREM collections" - args = 'source_dir pattern log_file' + help = "import CREM collections (special usecase)" admin_email = 'webmaster@parisson.com' + media_root = settings.MEDIA_ROOT option_list = BaseCommand.option_list + ( make_option('-d', '--dry-run', @@ -86,10 +86,14 @@ class Command(BaseCommand): if os.path.exists(media): if not item.file or self.force: if not self.dry_run: - f = open(media, 'r') - file_content = ContentFile(f.read()) - item.file.save(filename, file_content) - f.close() + if not self.media_root in self.source_dir: + f = open(media, 'r') + file_content = ContentFile(f.read()) + item.file.save(filename, file_content) + f.close() + else: + path = media[len(self.media_root)+1:] + item.file = path item.save() item.set_revision(self.user) else: @@ -112,7 +116,7 @@ class Command(BaseCommand): self.domain = Site.objects.all()[0].domain self.user = User.objects.filter(username='admin')[0] self.collections = os.listdir(self.source_dir) - + collections = [] for collection in self.collections: collection_dir = self.source_dir + os.sep + collection diff --git a/telemeta/management/commands/telemeta-init-formats.py b/telemeta/management/commands/telemeta-import-formats.py similarity index 100% rename from telemeta/management/commands/telemeta-init-formats.py rename to telemeta/management/commands/telemeta-import-formats.py diff --git a/telemeta/management/commands/telemeta-import-items-without-copy.py b/telemeta/management/commands/telemeta-import-items-without-copy.py index 714d2f53..1f80aeb8 100644 --- a/telemeta/management/commands/telemeta-import-items-without-copy.py +++ b/telemeta/management/commands/telemeta-import-items-without-copy.py @@ -25,7 +25,7 @@ class Command(BaseCommand): collection_code = args[-2] import_dir = os.path.abspath(args[-1]) media_dir = os.path.normpath(settings.MEDIA_ROOT) - + if not media_dir in import_dir: sys.exit('This directory is not in the MEDIA_ROOT directory') diff --git a/telemeta/management/commands/telemeta-import-items.py b/telemeta/management/commands/telemeta-import-items.py deleted file mode 100644 index 3d619556..00000000 --- a/telemeta/management/commands/telemeta-import-items.py +++ /dev/null @@ -1,43 +0,0 @@ -from optparse import make_option -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from django.core.files.base import ContentFile -from telemeta.models import * -from telemeta.util.unaccent import unaccent -import os - - -class Command(BaseCommand): - help = "import media files from a directory to a collection" - args = "collection_code media_dir" - - def handle(self, *args, **options): - collection_code = args[-2] - media_dir = args[-1] - - collections = MediaCollection.objects.filter(code=collection_code) - if not collections: - collection = MediaCollection(code=collection_code, title=collection_code) - collection.public_access = 'full' - collection.save() - print 'collection created: ' + collection_code - else: - collection = collections[0] - print 'using collection: ' + collection.code - - for root, dirs, files in os.walk(media_dir): - for filename in files: - path = root + os.sep + filename - filename_pre, ext = os.path.splitext(filename) - items = MediaItem.objects.filter(code=filename_pre) - if not items: - item = MediaItem(collection=collection, code=filename_pre) - item.title = filename_pre - f = open(path, 'r') - file_content = ContentFile(f.read()) - item.file.save(filename, file_content) - item.public_access = 'full' - item.save() - print 'item created: ' + item.code - else: - print 'item already exists: ' + items[0].code diff --git a/telemeta/management/commands/telemeta-test-import.py b/telemeta/management/commands/telemeta-test-import.py deleted file mode 100644 index b2d8ede6..00000000 --- a/telemeta/management/commands/telemeta-test-import.py +++ /dev/null @@ -1,65 +0,0 @@ -from optparse import make_option -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from django.core.files.base import ContentFile -from telemeta.models import * -from telemeta.util.unaccent import unaccent -from telemeta.cache import TelemetaCache -import urllib - -class Command(BaseCommand): - help = "Test: download and import a test item" - args = "absolute paths of a local audio files" - code = 'test' - title = 'test' - urls = ['http://files.parisson.com/telemeta/tests/media/sweep.mp3', - 'http://files.parisson.com/telemeta/tests/media/sweep.wav', - 'http://files.parisson.com/telemeta/tests/media/test.ogg', - 'http://files.parisson.com/telemeta/tests/media/test.flac', - 'http://files.parisson.com/telemeta/tests/media/test4.mp3', - 'http://files.parisson.com/telemeta/tests/media/test5.wav', - 'http://files.parisson.com/telemeta/tests/media/test6.wav'] - - cache_data = TelemetaCache(settings.TELEMETA_DATA_CACHE_DIR) - cache_export = TelemetaCache(settings.TELEMETA_EXPORT_CACHE_DIR) - - def handle(self, *args, **options): - if args: - self.urls = [] - for file in args: - self.urls.append('file://' + file) - - collections = MediaCollection.objects.filter(code=self.code) - if not collections: - collection = MediaCollection(code=self.code, title=self.title) - collection.public_access = 'full' - collection.save() - else: - collection = collections[0] - - for url in self.urls: - code = url.split('/')[-1] - code = code.replace(' ', '_') - items = MediaItem.objects.filter(code=code) - if not items: - item = MediaItem(collection=collection, code=code, title=code) - item.save() - else: - print 'cleanup' - item = items[0] - self.cache_data.delete_item_data(code) - self.cache_export.delete_item_data(code) - flags = MediaItemTranscodingFlag.objects.filter(item=item) - analyses = MediaItemAnalysis.objects.filter(item=item) - for flag in flags: - flag.delete() - for analysis in analyses: - analysis.delete() - - print 'downloading: ' + url - file = urllib.urlopen(url) - file_content = ContentFile(file.read()) - item.file.save(code, file_content) - item.public_access = 'full' - item.save() - print 'item created: ' + code -- 2.39.5