From: Guillaume Pellerin Date: Mon, 27 Apr 2015 16:35:53 +0000 (+0200) Subject: update and cleanup import commands, add more options X-Git-Tag: 1.6a~4^2~107 X-Git-Url: https://git.parisson.com/?a=commitdiff_plain;h=14b5041bd1714a6042cbbd9643985ad92da74f88;p=telemeta.git update and cleanup import commands, add more options --- diff --git a/telemeta/management/commands/telemeta-cleanup.py b/telemeta/management/commands/telemeta-cleanup.py index 35f47a8e..1e9be8c4 100644 --- a/telemeta/management/commands/telemeta-cleanup.py +++ b/telemeta/management/commands/telemeta-cleanup.py @@ -9,24 +9,21 @@ import codecs class Command(BaseCommand): help = "Cleanup DB : multiple analyses, data cache, export cache, etc.." - args = "None" - cache_data = None - cache_export = None + args = "cache" + cache_data = TelemetaCache(settings.TELEMETA_DATA_CACHE_DIR) + cache_export = TelemetaCache(settings.TELEMETA_EXPORT_CACHE_DIR) def handle(self, *args, **options): - if 'cache' in args: - self.cache_data = TelemetaCache(settings.TELEMETA_DATA_CACHE_DIR) - self.cache_export = TelemetaCache(settings.TELEMETA_EXPORT_CACHE_DIR) - print "Cleaning all cache..." - items = MediaItem.objects.all() a_counter = 0 - print 'Cleaning multiple analyses per item...' + print 'cleaning multiple analyses per item...' for item in items: - if self.cache_data and self.cache_export: + if 'cache' in args:: + print 'cleaning cache...' self.cache_data.delete_item_data(item.code) self.cache_export.delete_item_data(item.code) + analyses = MediaItemAnalysis.objects.filter(item=item) ids = [] for analysis in analyses: diff --git a/telemeta/management/commands/telemeta-crem-import.py b/telemeta/management/commands/telemeta-crem-import.py deleted file mode 100644 index 77593df0..00000000 --- a/telemeta/management/commands/telemeta-crem-import.py +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# Copyright (C) 2010 Guillaume Pellerin -# All rights reserved. -# -# This software is licensed as described in the file COPYING, which -# you should have received as part of this distribution. The terms -# are also available at http://svn.parisson.org/telemeta/TelemetaLicense. -# -# Author: Guillaume Pellerin -# - -import logging -import codecs -import os -import sys -import csv -import logging -import datetime -from optparse import make_option - -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from django.contrib.auth.models import User -from django.core.management import setup_environ -from django.core.files.base import ContentFile -from django.contrib.auth.models import User -from django.contrib.sites.models import Site -from django.template.defaultfilters import slugify - -from telemeta.models import * -from telemeta.util.unaccent import unaccent - - -class Logger: - - def __init__(self, file): - self.logger = logging.getLogger('myapp') - self.hdlr = logging.FileHandler(file) - self.formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') - self.hdlr.setFormatter(self.formatter) - self.logger.addHandler(self.hdlr) - self.logger.setLevel(logging.INFO) - - def info(self, prefix, message): - self.logger.info(' ' + prefix + ' : ' + message.decode('utf8')) - - def error(self, prefix, message): - self.logger.error(prefix + ' : ' + message.decode('utf8')) - - -class Command(BaseCommand): - - """Import CREM collections from collection directories containing media files - and eventually a XLS files representing the relation between old codes and new codes - """ - - help = "import CREM collections" - args = 'source_dir pattern log_file' - admin_email = 'webmaster@parisson.com' - - option_list = BaseCommand.option_list + ( - make_option('-d', '--dry-run', - action='store_true', - dest='dry-run', - help='Do NOT write anything'), - make_option('-f', '--force', - action='store_true', - dest='force', - help='Force overwrite data'), - make_option('-s', '--source', - dest='source_dir', - help='define the source directory'), - make_option('-l', '--log', - dest='log', - help='define log file'), - make_option('-p', '--pattern', - dest='pattern', - help='define the pattern'), - ) - - - def write_file(self, item, media): - filename = media.split(os.sep)[-1] - if os.path.exists(media): - if not item.file or self.force: - if not self.dry_run: - f = open(media, 'r') - file_content = ContentFile(f.read()) - item.file.save(filename, file_content) - f.close() - item.save() - item.set_revision(self.user) - else: - msg = item.code + " : pas d'écriture, utiliser l'option --write " - self.logger.info('item', msg) - else: - msg = item.code + ' : fichier ' + item.file.name + ' deja inscrit dans la base de donnees et pas de forcage !' - self.logger.info('item', msg) - else: - msg = item.code + ' : fichier audio ' + filename + ' inexistant dans le dossier !' - self.logger.error('item', msg) - - def handle(self, *args, **kwargs): - self.logger = Logger(kwargs.get('log')) - self.pattern = kwargs.get('pattern') - self.source_dir = kwargs.get('source_dir') - self.dry_run = kwargs.get('dry-run') - self.force = kwargs.get('force') - - self.domain = Site.objects.all()[0].domain - self.user = User.objects.filter(username='admin')[0] - self.collections = os.listdir(self.source_dir) - - collections = [] - for collection in self.collections: - collection_dir = self.source_dir + os.sep + collection - collection_files = os.listdir(collection_dir) - - - if not '/.' in collection_dir and self.pattern in collection_dir: - collection_name = collection.split(os.sep)[-1] - collections.append(collection_name) - c = MediaCollection.objects.filter(code=collection_name) - - if not c and collection + '.csv' in collection_files: - msg = collection + ' collection NON présente dans la base de données, SORTIE ' - self.logger.error(collection, msg) - sys.exit(msg) - elif not c: - msg = 'collection NON présente dans la base de données, CREATION ' - self.logger.info(collection, msg) - if not self.dry_run: - c = MediaCollection(code=collection_name, title=collection_name) - c.save() - c.set_revision(self.user) - else: - msg = 'collection présente dans la base de données, SELECTION' - self.logger.info(collection, msg) - - for collection in collections: - collection_dir = self.source_dir + os.sep + collection - collection_name = collection - collection_files = os.listdir(collection_dir) - msg = '************************ ' + collection + ' ******************************' - self.logger.info(collection, msg[:70]) - csv_file = '' - rows = {} - - if collection + '.csv' in collection_files: - csv_file = self.source_dir + os.sep + collection + os.sep + collection + '.csv' - csv_data = csv.reader(open(csv_file), delimiter=';') - for row in csv_data: - rows[row[1].strip()] = row[0].strip() - msg = collection + ' import du fichier CSV de la collection' - self.logger.info(collection, msg[:70]) - else: - msg = collection + ' pas de fichier CSV dans la collection' - self.logger.info(collection, msg[:70]) - - c = MediaCollection.objects.filter(code=collection_name) - if not c: - if not self.dry_run: - c = MediaCollection(code=collection_name) - c.save() - msg = ' collection NON présente dans la BDD, CREATION ' - self.logger.info(c.code, msg) - else: - c = c[0] - msg = ' id = '+str(c.id) - self.logger.info(c.code, msg) - - audio_files = [] - for file in collection_files: - ext = ['WAV', 'wav'] - if file.split('.')[-1] in ext and file[0] != '.': - audio_files.append(file) - - audio_files.sort() - nb_items = c.items.count() - counter = 0 - - for file in audio_files: - code = file.split('.')[0] - wav_file = self.source_dir + os.sep + collection + os.sep + file - - if len(audio_files) <= nb_items: - items = MediaItem.objects.filter(code=code) - - old_ref = '' - if code in rows and not items: - old_ref = rows[code] - items = MediaItem.objects.filter(old_code=old_ref) - - if items: - item = items[0] - msg = code + ' : ' + item.old_code + ' : Cas 1 ou 2 : id = ' + str(item.id) - self.logger.info('item', msg) - item.code = code - else: - item = MediaItem(code=code, collection=c) - msg = code + ' : ' + old_ref + ' : Cas 1 ou 2 : item NON présent dans la base de données, CREATION' - self.logger.info('item', msg) - - self.write_file(item, wav_file) - - elif nb_items == 1 and len(audio_files) > 1: - if counter == 0: - msg = code + ' : Cas 3a : item n°01 présent dans la base de données, PASSE' - self.logger.info('item', msg) - else: - item = MediaItem(code=code, collection=c) - msg = code + ' : Cas 3a : item NON présent dans la base de données, CREATION' - self.logger.info('item', msg) - self.write_file(item, wav_file) - - elif nb_items > 1 and nb_items < len(audio_files): - msg = code + ' : Cas 3b : nb items < nb de fichiers audio, PAS de creation' - self.logger.info('item', msg) - - counter += 1 - - msg = 'Liste des URLs des collections importées :' - self.logger.info('INFO', msg) - for collection in collections: - msg = 'http://'+self.domain+'/archives/collections/'+collection - self.logger.info(collection, msg) - - diff --git a/telemeta/management/commands/telemeta-import-collection-from-dir.py b/telemeta/management/commands/telemeta-import-collection-from-dir.py new file mode 100644 index 00000000..38b3e774 --- /dev/null +++ b/telemeta/management/commands/telemeta-import-collection-from-dir.py @@ -0,0 +1,114 @@ +from optparse import make_option +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.core.files.base import ContentFile +from django.contrib.auth.models import User +from telemeta.models import * +from telemeta.util.unaccent import unaccent +import os + + +try: + from django.utils.text import slugify +except ImportError: + def slugify(string): + killed_chars = re.sub('[\(\),]', '', string) + return re.sub(' ', '_', killed_chars) + +def beautify(string): + return os.path.splitext(string)[0].replace('_',' ') + + +class Command(BaseCommand): + help = "import media files from a directory into a collection" + media_root = os.path.normpath(settings.MEDIA_ROOT) + + option_list = BaseCommand.option_list + ( + make_option('-d', '--dry-run', + action='store_true', + dest='dry-run', + help='Do NOT write anything'), + make_option('-f', '--force', + action='store_true', + dest='force', + help='Force overwrite data'), + make_option('-s', '--source', + dest='source_dir', + help='define the source directory'), + make_option('-l', '--log', + dest='log', + help='define log file'), + make_option('-p', '--pattern', + dest='pattern', + help='define the pattern'), + make_option('-u', '--username', + dest='user', + help='define the username'), + make_option('-c', '--collection-code', + action='store', + dest='collection_code', + default='default', + metavar = '', + help='collection code'), + make_option('-t', '--collection-title', + action='store', + dest='collection_title', + default='default', + metavar = '', + help='collection title'), + + ) + + def write_file(self, item, media): + filename = media.split(os.sep)[-1] + if os.path.exists(media): + if not item.file or self.force: + if not self.dry_run: + if not self.media_root in self.source_dir: + print "file not in MEDIA_ROOT, copying..." + f = open(media, 'r') + file_content = ContentFile(f.read()) + item.file.save(filename, file_content) + f.close() + else: + print "file in MEDIA_ROOT, linking..." + path = media[len(self.media_root)+1:] + item.file = path + item.save() + if self.user: + item.set_revision(self.user) + + def handle(self, *args, **options): + self.source_dir = os.path.abspath(options.get('source_dir')) + self.collection_code = options.get('collection_code') + self.collection_title = options.get('collection_title') + self.dry_run = options.get('dry-run') + self.user = None + users = User.objects.filter(username=options.get('username')) + if users: + self.user = users[0] + + collections = MediaCollection.objects.filter(code=self.collection_code) + if not collections: + collection = MediaCollection(code=self.collection_code, title=self.collection_code) + collection.public_access = 'full' + collection.save() + print 'Collection created: ' + self.collection_code + else: + collection = collections[0] + print 'Using collection: ' + collection.code + + for root, dirs, files in os.walk(self.source_dir): + for filename in files: + path = root + os.sep + filename + filename_pre, ext = os.path.splitext(filename) + item_code = collection.code + '_' + filename_pre + item, c = MediaItem.objects.get_or_create(collection=collection, code=item_code) + if c: + item.title = filename_pre + item.public_access = 'full' + self.write_file(item, path) + item.save() + print 'item created: ' + item.code + else: + print 'item already exists: ' + item.code diff --git a/telemeta/management/commands/telemeta-import-collection-from-files.py b/telemeta/management/commands/telemeta-import-collection-from-files.py new file mode 100644 index 00000000..c22c9c0b --- /dev/null +++ b/telemeta/management/commands/telemeta-import-collection-from-files.py @@ -0,0 +1,91 @@ +from optparse import make_option +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.core.files.base import ContentFile +from telemeta.models import * +from telemeta.util.unaccent import unaccent +from telemeta.cache import TelemetaCache +import urllib + +try: + from django.utils.text import slugify +except ImportError: + def slugify(string): + killed_chars = re.sub('[\(\),]', '', string) + return re.sub(' ', '_', killed_chars) + +def beautify(string): + return os.path.splitext(string)[0].replace('_',' ') + +class Command(BaseCommand): + args = "<media_file1 [media_file2 ...]>" + help = "Download and import a media item" + option_list = BaseCommand.option_list + ( + make_option('--collection-code', + action='store', + dest='code', + default='default', + metavar = '<code>', + help='collection code'), + make_option('--collection-title', + action='store', + dest='title', + default='default', + metavar = '<title>', + help='collection title'), + ) + + cache_data = TelemetaCache(settings.TELEMETA_DATA_CACHE_DIR) + cache_export = TelemetaCache(settings.TELEMETA_EXPORT_CACHE_DIR) + + urls = [] + + def handle(self, *args, **options): + if len(args) < 1: + return + if options['title']: + self.title = options['title'] + if options['code']: + self.code = options['code'] + for file in args: + self.urls.append('file://' + file) + + collections = MediaCollection.objects.filter(code=self.code) + if not collections: + # create a new collection + collection = MediaCollection(code=self.code, title=self.title) + collection.public_access = 'full' + collection.save() + else: + collection = collections[0] + + for url in self.urls: + basename = os.path.basename(url) + code = slugify(basename) + title = beautify(basename) + items = MediaItem.objects.filter(code=code) + if not items: + item = MediaItem(collection=collection, code=code, title=title) + item.save() + else: + print 'cleaning up', code + item = items[0] + self.cache_data.delete_item_data(code) + self.cache_export.delete_item_data(code) + flags = MediaItemTranscodingFlag.objects.filter(item=item) + analyses = MediaItemAnalysis.objects.filter(item=item) + for flag in flags: + flag.delete() + for analysis in analyses: + analysis.delete() + + print 'fetching: ' + url + file = urllib.urlopen(url) + file_content = ContentFile(file.read()) + item.title = title + item.file.save(code, file_content) + item.public_access = 'full' + item.save() + print 'item created: ', collection, code + + print 'done importing', len(self.urls), 'items' diff --git a/telemeta/management/commands/telemeta-import-collection-from-url.py b/telemeta/management/commands/telemeta-import-collection-from-url.py new file mode 100644 index 00000000..99145737 --- /dev/null +++ b/telemeta/management/commands/telemeta-import-collection-from-url.py @@ -0,0 +1,45 @@ +from optparse import make_option +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.core.files.base import ContentFile +from telemeta.models import * +from telemeta.util.unaccent import unaccent +from telemeta.util.url import URLMediaParser +import os + + +class Command(BaseCommand): + help = "import media files from a URL directory into a collection with a given prefix" + args = "collection_code URL" + + def handle(self, *args, **options): + collection_code = args[0] + prefix = args[1] + url = args[2] + + parser = URLMediaParser(url) + urls = parser.get_urls() + collections = MediaCollection.objects.filter(code=collection_code) + if not collections: + collection = MediaCollection(code=collection_code, title=collection_code) + collection.public_access = 'full' + collection.save() + print 'collection created: ' + collection_code + else: + collection = collections[0] + print 'using collection: ' + collection.code + + for url in urls: + filename = url.split('/')[-1] + name, ext = os.path.splitext(filename) + items = MediaItem.objects.filter(code=name) + if not items and prefix in name: + code = collection.code + '_' + name + item = MediaItem(collection=collection, code=code) + item.title = name + item.url = url + item.public_access = 'full' + item.save() + print 'item created: ' + item.code + else: + print 'item already exists: ' + items[0].code diff --git a/telemeta/management/commands/telemeta-import-crem.py b/telemeta/management/commands/telemeta-import-crem.py new file mode 100644 index 00000000..f73cd402 --- /dev/null +++ b/telemeta/management/commands/telemeta-import-crem.py @@ -0,0 +1,234 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Copyright (C) 2010 Guillaume Pellerin +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://svn.parisson.org/telemeta/TelemetaLicense. +# +# Author: Guillaume Pellerin <yomguy@parisson.com> +# + +import logging +import codecs +import os +import sys +import csv +import logging +import datetime +from optparse import make_option + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.contrib.auth.models import User +from django.core.management import setup_environ +from django.core.files.base import ContentFile +from django.contrib.auth.models import User +from django.contrib.sites.models import Site +from django.template.defaultfilters import slugify + +from telemeta.models import * +from telemeta.util.unaccent import unaccent + + +class Logger: + + def __init__(self, file): + self.logger = logging.getLogger('myapp') + self.hdlr = logging.FileHandler(file) + self.formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + self.hdlr.setFormatter(self.formatter) + self.logger.addHandler(self.hdlr) + self.logger.setLevel(logging.INFO) + + def info(self, prefix, message): + self.logger.info(' ' + prefix + ' : ' + message.decode('utf8')) + + def error(self, prefix, message): + self.logger.error(prefix + ' : ' + message.decode('utf8')) + + +class Command(BaseCommand): + + """Import CREM collections from collection directories containing media files + and eventually a XLS files representing the relation between old codes and new codes + """ + + help = "import CREM collections (special usecase)" + admin_email = 'webmaster@parisson.com' + media_root = settings.MEDIA_ROOT + + option_list = BaseCommand.option_list + ( + make_option('-d', '--dry-run', + action='store_true', + dest='dry-run', + help='Do NOT write anything'), + make_option('-f', '--force', + action='store_true', + dest='force', + help='Force overwrite data'), + make_option('-s', '--source', + dest='source_dir', + help='define the source directory'), + make_option('-l', '--log', + dest='log', + help='define log file'), + make_option('-p', '--pattern', + dest='pattern', + help='define the pattern'), + ) + + + def write_file(self, item, media): + filename = media.split(os.sep)[-1] + if os.path.exists(media): + if not item.file or self.force: + if not self.dry_run: + if not self.media_root in self.source_dir: + f = open(media, 'r') + file_content = ContentFile(f.read()) + item.file.save(filename, file_content) + f.close() + else: + path = media[len(self.media_root)+1:] + item.file = path + item.save() + item.set_revision(self.user) + else: + msg = item.code + " : pas d'écriture, utiliser l'option --write " + self.logger.info('item', msg) + else: + msg = item.code + ' : fichier ' + item.file.name + ' deja inscrit dans la base de donnees et pas de forcage !' + self.logger.info('item', msg) + else: + msg = item.code + ' : fichier audio ' + filename + ' inexistant dans le dossier !' + self.logger.error('item', msg) + + def handle(self, *args, **kwargs): + self.logger = Logger(kwargs.get('log')) + self.pattern = kwargs.get('pattern') + self.source_dir = kwargs.get('source_dir') + self.dry_run = kwargs.get('dry-run') + self.force = kwargs.get('force') + + self.domain = Site.objects.all()[0].domain + self.user = User.objects.filter(username='admin')[0] + self.collections = os.listdir(self.source_dir) + + collections = [] + for collection in self.collections: + collection_dir = self.source_dir + os.sep + collection + collection_files = os.listdir(collection_dir) + + + if not '/.' in collection_dir and self.pattern in collection_dir: + collection_name = collection.split(os.sep)[-1] + collections.append(collection_name) + c = MediaCollection.objects.filter(code=collection_name) + + if not c and collection + '.csv' in collection_files: + msg = collection + ' collection NON présente dans la base de données, SORTIE ' + self.logger.error(collection, msg) + sys.exit(msg) + elif not c: + msg = 'collection NON présente dans la base de données, CREATION ' + self.logger.info(collection, msg) + if not self.dry_run: + c = MediaCollection(code=collection_name, title=collection_name) + c.save() + c.set_revision(self.user) + else: + msg = 'collection présente dans la base de données, SELECTION' + self.logger.info(collection, msg) + + for collection in collections: + collection_dir = self.source_dir + os.sep + collection + collection_name = collection + collection_files = os.listdir(collection_dir) + msg = '************************ ' + collection + ' ******************************' + self.logger.info(collection, msg[:70]) + csv_file = '' + rows = {} + + if collection + '.csv' in collection_files: + csv_file = self.source_dir + os.sep + collection + os.sep + collection + '.csv' + csv_data = csv.reader(open(csv_file), delimiter=';') + for row in csv_data: + rows[row[1].strip()] = row[0].strip() + msg = collection + ' import du fichier CSV de la collection' + self.logger.info(collection, msg[:70]) + else: + msg = collection + ' pas de fichier CSV dans la collection' + self.logger.info(collection, msg[:70]) + + c = MediaCollection.objects.filter(code=collection_name) + if not c: + if not self.dry_run: + c = MediaCollection(code=collection_name) + c.save() + msg = ' collection NON présente dans la BDD, CREATION ' + self.logger.info(c.code, msg) + else: + c = c[0] + msg = ' id = '+str(c.id) + self.logger.info(c.code, msg) + + audio_files = [] + for file in collection_files: + ext = ['WAV', 'wav'] + if file.split('.')[-1] in ext and file[0] != '.': + audio_files.append(file) + + audio_files.sort() + nb_items = c.items.count() + counter = 0 + + for file in audio_files: + code = file.split('.')[0] + wav_file = self.source_dir + os.sep + collection + os.sep + file + + if len(audio_files) <= nb_items: + items = MediaItem.objects.filter(code=code) + + old_ref = '' + if code in rows and not items: + old_ref = rows[code] + items = MediaItem.objects.filter(old_code=old_ref) + + if items: + item = items[0] + msg = code + ' : ' + item.old_code + ' : Cas 1 ou 2 : id = ' + str(item.id) + self.logger.info('item', msg) + item.code = code + else: + item = MediaItem(code=code, collection=c) + msg = code + ' : ' + old_ref + ' : Cas 1 ou 2 : item NON présent dans la base de données, CREATION' + self.logger.info('item', msg) + + self.write_file(item, wav_file) + + elif nb_items == 1 and len(audio_files) > 1: + if counter == 0: + msg = code + ' : Cas 3a : item n°01 présent dans la base de données, PASSE' + self.logger.info('item', msg) + else: + item = MediaItem(code=code, collection=c) + msg = code + ' : Cas 3a : item NON présent dans la base de données, CREATION' + self.logger.info('item', msg) + self.write_file(item, wav_file) + + elif nb_items > 1 and nb_items < len(audio_files): + msg = code + ' : Cas 3b : nb items < nb de fichiers audio, PAS de creation' + self.logger.info('item', msg) + + counter += 1 + + msg = 'Liste des URLs des collections importées :' + self.logger.info('INFO', msg) + for collection in collections: + msg = 'http://'+self.domain+'/archives/collections/'+collection + self.logger.info(collection, msg) + + diff --git a/telemeta/management/commands/telemeta-import-formats.py b/telemeta/management/commands/telemeta-import-formats.py new file mode 100644 index 00000000..5e2faace --- /dev/null +++ b/telemeta/management/commands/telemeta-import-formats.py @@ -0,0 +1,27 @@ +from optparse import make_option +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.contrib.auth.models import User +from django.template.defaultfilters import slugify +from telemeta.models import * +from telemeta.util.unaccent import unaccent +import logging +import codecs + +class Command(BaseCommand): + help = "Init original formats from a txt file (see files in example/init/" + args = "path" + admin_email = 'webmaster@parisson.com' + + def handle(self, *args, **options): + path = args[0] + file = open(path, 'r') + for format in file.readlines(): + if not PhysicalFormat.objects.filter(value=format): + format = PhysicalFormat(value=format) + format.save() + + + + + diff --git a/telemeta/management/commands/telemeta-import-items-from-url.py b/telemeta/management/commands/telemeta-import-items-from-url.py deleted file mode 100644 index 99145737..00000000 --- a/telemeta/management/commands/telemeta-import-items-from-url.py +++ /dev/null @@ -1,45 +0,0 @@ -from optparse import make_option -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from django.core.files.base import ContentFile -from telemeta.models import * -from telemeta.util.unaccent import unaccent -from telemeta.util.url import URLMediaParser -import os - - -class Command(BaseCommand): - help = "import media files from a URL directory into a collection with a given prefix" - args = "collection_code URL" - - def handle(self, *args, **options): - collection_code = args[0] - prefix = args[1] - url = args[2] - - parser = URLMediaParser(url) - urls = parser.get_urls() - collections = MediaCollection.objects.filter(code=collection_code) - if not collections: - collection = MediaCollection(code=collection_code, title=collection_code) - collection.public_access = 'full' - collection.save() - print 'collection created: ' + collection_code - else: - collection = collections[0] - print 'using collection: ' + collection.code - - for url in urls: - filename = url.split('/')[-1] - name, ext = os.path.splitext(filename) - items = MediaItem.objects.filter(code=name) - if not items and prefix in name: - code = collection.code + '_' + name - item = MediaItem(collection=collection, code=code) - item.title = name - item.url = url - item.public_access = 'full' - item.save() - print 'item created: ' + item.code - else: - print 'item already exists: ' + items[0].code diff --git a/telemeta/management/commands/telemeta-import-items-without-copy.py b/telemeta/management/commands/telemeta-import-items-without-copy.py index 714d2f53..1f80aeb8 100644 --- a/telemeta/management/commands/telemeta-import-items-without-copy.py +++ b/telemeta/management/commands/telemeta-import-items-without-copy.py @@ -25,7 +25,7 @@ class Command(BaseCommand): collection_code = args[-2] import_dir = os.path.abspath(args[-1]) media_dir = os.path.normpath(settings.MEDIA_ROOT) - + if not media_dir in import_dir: sys.exit('This directory is not in the MEDIA_ROOT directory') diff --git a/telemeta/management/commands/telemeta-import-items.py b/telemeta/management/commands/telemeta-import-items.py deleted file mode 100644 index 3d619556..00000000 --- a/telemeta/management/commands/telemeta-import-items.py +++ /dev/null @@ -1,43 +0,0 @@ -from optparse import make_option -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from django.core.files.base import ContentFile -from telemeta.models import * -from telemeta.util.unaccent import unaccent -import os - - -class Command(BaseCommand): - help = "import media files from a directory to a collection" - args = "collection_code media_dir" - - def handle(self, *args, **options): - collection_code = args[-2] - media_dir = args[-1] - - collections = MediaCollection.objects.filter(code=collection_code) - if not collections: - collection = MediaCollection(code=collection_code, title=collection_code) - collection.public_access = 'full' - collection.save() - print 'collection created: ' + collection_code - else: - collection = collections[0] - print 'using collection: ' + collection.code - - for root, dirs, files in os.walk(media_dir): - for filename in files: - path = root + os.sep + filename - filename_pre, ext = os.path.splitext(filename) - items = MediaItem.objects.filter(code=filename_pre) - if not items: - item = MediaItem(collection=collection, code=filename_pre) - item.title = filename_pre - f = open(path, 'r') - file_content = ContentFile(f.read()) - item.file.save(filename, file_content) - item.public_access = 'full' - item.save() - print 'item created: ' + item.code - else: - print 'item already exists: ' + items[0].code diff --git a/telemeta/management/commands/telemeta-init-formats.py b/telemeta/management/commands/telemeta-init-formats.py deleted file mode 100644 index 5e2faace..00000000 --- a/telemeta/management/commands/telemeta-init-formats.py +++ /dev/null @@ -1,27 +0,0 @@ -from optparse import make_option -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from django.contrib.auth.models import User -from django.template.defaultfilters import slugify -from telemeta.models import * -from telemeta.util.unaccent import unaccent -import logging -import codecs - -class Command(BaseCommand): - help = "Init original formats from a txt file (see files in example/init/" - args = "path" - admin_email = 'webmaster@parisson.com' - - def handle(self, *args, **options): - path = args[0] - file = open(path, 'r') - for format in file.readlines(): - if not PhysicalFormat.objects.filter(value=format): - format = PhysicalFormat(value=format) - format.save() - - - - - diff --git a/telemeta/management/commands/telemeta-media-import.py b/telemeta/management/commands/telemeta-media-import.py deleted file mode 100644 index c73d4f2f..00000000 --- a/telemeta/management/commands/telemeta-media-import.py +++ /dev/null @@ -1,91 +0,0 @@ -from optparse import make_option -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from django.core.files.base import ContentFile -from telemeta.models import * -from telemeta.util.unaccent import unaccent -from telemeta.cache import TelemetaCache -import urllib - -try: - from django.utils.text import slugify -except ImportError: - def slugify(string): - killed_chars = re.sub('[\(\),]', '', string) - return re.sub(' ', '_', killed_chars) - -def beautify(string): - return os.path.splitext(string)[0].replace('_',' ') - -class Command(BaseCommand): - args = "<media_file1 [media_file2 ...]>" - help = "Download and import a media item" - option_list = BaseCommand.option_list + ( - make_option('--collection-code', - action='store', - dest='code', - default='default', - metavar = '<code>', - help='collection code'), - make_option('--collection-title', - action='store', - dest='title', - default='default', - metavar = '<title>', - help='collection title'), - ) - - cache_data = TelemetaCache(settings.TELEMETA_DATA_CACHE_DIR) - cache_export = TelemetaCache(settings.TELEMETA_EXPORT_CACHE_DIR) - - urls = [] - - def handle(self, *args, **options): - if len(args) < 1: - return - if options['title']: - self.title = options['title'] - if options['code']: - self.code = options['code'] - for file in args: - self.urls.append('file://' + file) - - collections = MediaCollection.objects.filter(code=self.code) - if not collections: - # create a new collection - collection = MediaCollection(code=self.code, title=self.title) - collection.public_access = 'full' - collection.save() - else: - collection = collections[0] - - for url in self.urls: - basename = os.path.basename(url) - code = slugify(basename) - title = beautify(basename) - items = MediaItem.objects.filter(code=code) - if not items: - item = MediaItem(collection=collection, code=code, title=title) - item.save() - else: - print 'cleaning up', code - item = items[0] - self.cache_data.delete_item_data(code) - self.cache_export.delete_item_data(code) - flags = MediaItemTranscodingFlag.objects.filter(item=item) - analyses = MediaItemAnalysis.objects.filter(item=item) - for flag in flags: - flag.delete() - for analysis in analyses: - analysis.delete() - - print 'fetching: ' + url - file = urllib.urlopen(url) - file_content = ContentFile(file.read()) - item.title = title - item.file.save(code, file_content) - item.public_access = 'full' - item.save() - print 'item created: ', collection, code - - print 'done importing', len(self.urls), 'items' diff --git a/telemeta/management/commands/telemeta-test-import.py b/telemeta/management/commands/telemeta-test-import.py deleted file mode 100644 index b2d8ede6..00000000 --- a/telemeta/management/commands/telemeta-test-import.py +++ /dev/null @@ -1,65 +0,0 @@ -from optparse import make_option -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from django.core.files.base import ContentFile -from telemeta.models import * -from telemeta.util.unaccent import unaccent -from telemeta.cache import TelemetaCache -import urllib - -class Command(BaseCommand): - help = "Test: download and import a test item" - args = "absolute paths of a local audio files" - code = 'test' - title = 'test' - urls = ['http://files.parisson.com/telemeta/tests/media/sweep.mp3', - 'http://files.parisson.com/telemeta/tests/media/sweep.wav', - 'http://files.parisson.com/telemeta/tests/media/test.ogg', - 'http://files.parisson.com/telemeta/tests/media/test.flac', - 'http://files.parisson.com/telemeta/tests/media/test4.mp3', - 'http://files.parisson.com/telemeta/tests/media/test5.wav', - 'http://files.parisson.com/telemeta/tests/media/test6.wav'] - - cache_data = TelemetaCache(settings.TELEMETA_DATA_CACHE_DIR) - cache_export = TelemetaCache(settings.TELEMETA_EXPORT_CACHE_DIR) - - def handle(self, *args, **options): - if args: - self.urls = [] - for file in args: - self.urls.append('file://' + file) - - collections = MediaCollection.objects.filter(code=self.code) - if not collections: - collection = MediaCollection(code=self.code, title=self.title) - collection.public_access = 'full' - collection.save() - else: - collection = collections[0] - - for url in self.urls: - code = url.split('/')[-1] - code = code.replace(' ', '_') - items = MediaItem.objects.filter(code=code) - if not items: - item = MediaItem(collection=collection, code=code, title=code) - item.save() - else: - print 'cleanup' - item = items[0] - self.cache_data.delete_item_data(code) - self.cache_export.delete_item_data(code) - flags = MediaItemTranscodingFlag.objects.filter(item=item) - analyses = MediaItemAnalysis.objects.filter(item=item) - for flag in flags: - flag.delete() - for analysis in analyses: - analysis.delete() - - print 'downloading: ' + url - file = urllib.urlopen(url) - file_content = ContentFile(file.read()) - item.file.save(code, file_content) - item.public_access = 'full' - item.save() - print 'item created: ' + code