From 908ab06c0647af0cdce371c6f06f50232e2d0d36 Mon Sep 17 00:00:00 2001 From: yomguy Date: Wed, 21 Dec 2011 13:20:16 +0100 Subject: [PATCH] add telemeta-test-import script, partially fix MySQL requests when using SQLite, add sqlite sandbox --- example/sandbox_sqlite/__init__.py | 0 example/sandbox_sqlite/manage.py | 11 ++ example/sandbox_sqlite/settings.py | 129 ++++++++++++++++++ example/sandbox_sqlite/urls.py | 29 ++++ .../commands/telemeta-test-import.py | 58 ++++++++ telemeta/models/query.py | 118 ++++++++-------- telemeta/web/base.py | 5 +- 7 files changed, 293 insertions(+), 57 deletions(-) create mode 100644 example/sandbox_sqlite/__init__.py create mode 100755 example/sandbox_sqlite/manage.py create mode 100644 example/sandbox_sqlite/settings.py create mode 100644 example/sandbox_sqlite/urls.py create mode 100644 telemeta/management/commands/telemeta-test-import.py diff --git a/example/sandbox_sqlite/__init__.py b/example/sandbox_sqlite/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/example/sandbox_sqlite/manage.py b/example/sandbox_sqlite/manage.py new file mode 100755 index 00000000..bcdd55e2 --- /dev/null +++ b/example/sandbox_sqlite/manage.py @@ -0,0 +1,11 @@ +#!/usr/bin/python +from django.core.management import execute_manager +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) + sys.exit(1) + +if __name__ == "__main__": + execute_manager(settings) diff --git a/example/sandbox_sqlite/settings.py b/example/sandbox_sqlite/settings.py new file mode 100644 index 00000000..4d8e6166 --- /dev/null +++ b/example/sandbox_sqlite/settings.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +# Django settings for sandbox project. + +import os + +DEBUG = True +TEMPLATE_DEBUG = DEBUG + +ADMINS = ( + ('Guillaume Pellerin', 'yomguy@parisson.com'), +) + +MANAGERS = ADMINS + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. + 'NAME': 'sandbox.sql', # Or path to database file if using sqlite3. + 'USER': '', # Not used with sqlite3. + 'PASSWORD': '', # Not used with sqlite3. + 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. + 'PORT': '', # Set to empty string for default. Not used with sqlite3. + } +} + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# On Unix systems, a value of None will cause Django to use the same +# timezone as the operating system. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = 'Europe/Paris' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +#LANGUAGE_CODE = 'fr_FR' +LANGUAGES = [ ('fr', 'French'), + ('en', 'English'), +] + +SITE_ID = 1 + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True + +# If you set this to False, Django will not format dates, numbers and +# calendars according to the current locale +USE_L10N = True + +# Absolute path to the directory that holds media. +# Example: "/home/media/media.lawrence.com/" +MEDIA_ROOT = os.path.join(os.path.dirname(__file__), 'media/') + +if not os.path.exists(MEDIA_ROOT): + os.mkdir(MEDIA_ROOT) + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash if there is a path component (optional in other cases). +# Examples: "http://media.lawrence.com", "http://example.com/media/" +MEDIA_URL = '/media/' + +# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a +# trailing slash. +# Examples: "http://foo.com/media/", "/media/". +ADMIN_MEDIA_PREFIX = 'http://localhost/django/media/' + +# Make this unique, and don't share it with anybody. +SECRET_KEY = 'a8l7%06wr2k+3=%#*#@#rvop2mmzko)44%7k(zx%lls^ihm9^5' + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', +# 'django.template.loaders.eggs.Loader', +) + + +MIDDLEWARE_CLASSES = ( + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.locale.LocaleMiddleware', +) + +ROOT_URLCONF = 'sandbox_sqlite.urls' + +TEMPLATE_DIRS = ( + # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. + '', +) + +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'django.contrib.admin', + 'telemeta', + 'jsonrpc', +) + +TEMPLATE_CONTEXT_PROCESSORS = ( + 'django.core.context_processors.request', + 'django.contrib.auth.context_processors.auth', +) + +TELEMETA_ORGANIZATION = 'Parisson' +TELEMETA_SUBJECTS = ('test', 'telemeta', 'sandbox') +TELEMETA_DESCRIPTION = "Telemeta TEST sandbox" +TELEMETA_GMAP_KEY = 'ABQIAAAArg7eSfnfTkBRma8glnGrlxRVbMrhnNNvToCbZQtWdaMbZTA_3RRGObu5PDoiBImgalVnnLU2yN4RMA' +TELEMETA_CACHE_DIR = MEDIA_ROOT + 'cache' +TELEMETA_EXPORT_CACHE_DIR = TELEMETA_CACHE_DIR + "/export" +TELEMETA_DATA_CACHE_DIR = TELEMETA_CACHE_DIR + "/data" + +TELEMETA_DOWNLOAD_ENABLED = True +TELEMETA_STREAMING_FORMATS = ('mp3', 'ogg') +TELEMETA_PUBLIC_ACCESS_PERIOD = 51 +AUTH_PROFILE_MODULE = 'telemeta.userprofile' +SESSION_EXPIRE_AT_BROWSER_CLOSE = False + +EMAIL_HOST = 'localhost' +DEFAULT_FROM_EMAIL = 'webmaster@parisson.com' diff --git a/example/sandbox_sqlite/urls.py b/example/sandbox_sqlite/urls.py new file mode 100644 index 00000000..6dd41922 --- /dev/null +++ b/example/sandbox_sqlite/urls.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from django.conf.urls.defaults import * + +# Uncomment the next two lines to enable the admin: +from django.contrib import admin +admin.autodiscover() + +js_info_dict = { + 'packages': ('telemeta',), +} + +urlpatterns = patterns('', + # Example: + # (r'^sandbox/', include('sandbox.foo.urls')), + + # Uncomment the admin/doc line below and add 'django.contrib.admindocs' + # to INSTALLED_APPS to enable admin documentation: + # (r'^admin/doc/', include('django.contrib.admindocs.urls')), + + # Uncomment the next line to enable the admin: + (r'^admin/django/', include(admin.site.urls)), + + # Telemeta + (r'^', include('telemeta.urls')), + + # Languages + (r'^i18n/', include('django.conf.urls.i18n')), + (r'^jsi18n/$', 'django.views.i18n.javascript_catalog', js_info_dict), +) diff --git a/telemeta/management/commands/telemeta-test-import.py b/telemeta/management/commands/telemeta-test-import.py new file mode 100644 index 00000000..f0739d06 --- /dev/null +++ b/telemeta/management/commands/telemeta-test-import.py @@ -0,0 +1,58 @@ +from optparse import make_option +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.core.files.base import ContentFile +from telemeta.models import * +from telemeta.util.unaccent import unaccent +from telemeta.cache import TelemetaCache +import urllib + +class Command(BaseCommand): + help = "Test: download and import a test item" + args = "absolute paths of a local audio files" + code = 'test' + title = 'test' + urls = ['http://files.parisson.com/telemeta/tests/media/sweep.mp3', + 'http://files.parisson.com/telemeta/tests/media/test4.mp3', + 'http://files.parisson.com/telemeta/tests/media/test5.wav'] + cache_data = TelemetaCache(settings.TELEMETA_DATA_CACHE_DIR) + cache_export = TelemetaCache(settings.TELEMETA_EXPORT_CACHE_DIR) + + def handle(self, *args, **options): + if args: + self.urls = [] + for file in args: + self.urls.append('file://' + file) + + collections = MediaCollection.objects.filter(code=self.code) + if not collections: + collection = MediaCollection(code=self.code, title=self.title) + collection.save() + else: + collection = collections[0] + + for url in self.urls: + code = url.split('/')[-1] + items = MediaItem.objects.filter(code=code) + if not items: + item = MediaItem(collection=collection, code=code, title=code) + item.save() + else: + print 'cleanup' + item = items[0] + self.cache_data.delete_item_data(code) + self.cache_export.delete_item_data(code) + flags = MediaItemTranscodingFlag.objects.filter(item=item) + analyses = MediaItemAnalysis.objects.filter(item=item) + for flag in flags: + flag.delete() + for analysis in analyses: + analysis.delete() + + print 'downloading: ' + url + file = urllib.urlopen(url) + file_content = ContentFile(file.read()) + item.file.save(code, file_content) + item.public_access = 'full' + item.save() + print 'item created: ' + code diff --git a/telemeta/models/query.py b/telemeta/models/query.py index 3bb8a22c..2c8fe22e 100644 --- a/telemeta/models/query.py +++ b/telemeta/models/query.py @@ -35,19 +35,22 @@ # David LIPSZYC # Guillaume Pellerin +from django.conf import settings from django.db.models import Q, Max, Min from telemeta.models.core import * from telemeta.util.unaccent import unaccent, unaccent_icmp from telemeta.models.enum import EthnicGroup import re +engine = settings.DATABASES['default']['ENGINE'] + class MediaItemQuerySet(CoreQuerySet): "Base class for all media item query sets" - + def quick_search(self, pattern): "Perform a quick search on code, title and collector name" pattern = pattern.strip() - + # from telemeta.models.media import MediaItem # mod = MediaItem() # fields = mod.to_dict() @@ -60,49 +63,49 @@ class MediaItemQuerySet(CoreQuerySet): q = ( Q(code__contains=pattern) | Q(old_code__contains=pattern) | - word_search_q('title', pattern) | - word_search_q('comment', pattern) | + word_search_q('title', pattern) | + word_search_q('comment', pattern) | self.by_fuzzy_collector_q(pattern) ) - + return self.filter(q) - def without_collection(self): + def without_collection(self): "Find items which do not belong to any collection" return self.extra( where = ["collection_id NOT IN (SELECT id FROM media_collections)"]); def by_public_id(self, public_id): "Find items by public_id" - return self.filter(public_id=public_id) + return self.filter(public_id=public_id) def by_recording_date(self, from_date, to_date = None): "Find items by recording date" if to_date is None: return (self.filter(recorded_from_date__lte=from_date, recorded_to_date__gte=from_date)) else : - return (self.filter(Q(recorded_from_date__range=(from_date, to_date)) + return (self.filter(Q(recorded_from_date__range=(from_date, to_date)) | Q(recorded_to_date__range=(from_date, to_date)))) def by_title(self, pattern): "Find items by title" # to (sort of) sync with models.media.MediaItem.get_title() - return self.filter(word_search_q("title", pattern) | + return self.filter(word_search_q("title", pattern) | (Q(title="") & word_search_q("collection__title", pattern))) def by_publish_year(self, from_year, to_year = None): "Find items by publishing year" if to_year is None: to_year = from_year - return self.filter(collection__year_published__range=(from_year, to_year)) + return self.filter(collection__year_published__range=(from_year, to_year)) def by_change_time(self, from_time = None, until_time = None): - "Find items by last change time" + "Find items by last change time" return self._by_change_time('item', from_time, until_time) def by_location(self, location): "Find items by location" return self.filter(location__in=location.apparented()) - + @staticmethod def __name_cmp(obj1, obj2): return unaccent_icmp(obj1.name, obj2.name) @@ -133,19 +136,19 @@ class MediaItemQuerySet(CoreQuerySet): grouped[continent] = [] grouped[continent].append(country) - + keys = grouped.keys() keys.sort(self.__name_cmp) ordered = [] for c in keys: grouped[c].sort(self.__name_cmp) ordered.append({'continent': c, 'countries': grouped[c]}) - + countries = ordered else: countries.sort(self.__name_cmp) - - return countries + + return countries def virtual(self, *args): qs = self @@ -154,30 +157,32 @@ class MediaItemQuerySet(CoreQuerySet): from telemeta.models import Location for f in args: if f == 'apparent_collector': - related.append('collection') - qs = qs.extra(select={f: - 'IF(collector_from_collection, ' - 'IF(media_collections.collector_is_creator, ' - 'media_collections.creator, ' - 'media_collections.collector),' - 'media_items.collector)'}) + if not 'sqlite3' in engine: + related.append('collection') + qs = qs.extra(select={f: + 'IF(collector_from_collection, ' + 'IF(media_collections.collector_is_creator, ' + 'media_collections.creator, ' + 'media_collections.collector),' + 'media_items.collector)'}) elif f == 'country_or_continent': related.append('location') - qs = qs.extra(select={f: - 'IF(locations.type = ' + str(Location.COUNTRY) + ' ' - 'OR locations.type = ' + str(Location.CONTINENT) + ',' - 'locations.name, ' - '(SELECT l2.name FROM location_relations AS r INNER JOIN locations AS l2 ' - 'ON r.ancestor_location_id = l2.id ' - 'WHERE r.location_id = media_items.location_id AND l2.type = ' + str(Location.COUNTRY) + ' LIMIT 1))' - }) + if not 'sqlite3' in engine: + qs = qs.extra(select={f: + 'IF(locations.type = ' + str(Location.COUNTRY) + ' ' + 'OR locations.type = ' + str(Location.CONTINENT) + ',' + 'locations.name, ' + '(SELECT l2.name FROM location_relations AS r INNER JOIN locations AS l2 ' + 'ON r.ancestor_location_id = l2.id ' + 'WHERE r.location_id = media_items.location_id AND l2.type = ' + str(Location.COUNTRY) + ' LIMIT 1))' + }) else: raise Exception("Unsupported virtual field: %s" % f) if related: qs = qs.select_related(*related) - return qs + return qs def ethnic_groups(self): ids = self.filter(ethnic_group__isnull=False).values('ethnic_group'); @@ -185,17 +190,17 @@ class MediaItemQuerySet(CoreQuerySet): @staticmethod def by_fuzzy_collector_q(pattern): - return (word_search_q('collection__creator', pattern) | - word_search_q('collection__collector', pattern) | + return (word_search_q('collection__creator', pattern) | + word_search_q('collection__collector', pattern) | word_search_q('collector', pattern)) def by_fuzzy_collector(self, pattern): return self.filter(self.by_fuzzy_collector_q(pattern)) - + def sound(self): return self.filter(file__contains='/') - - + + class MediaItemManager(CoreManager): "Manage media items queries" @@ -213,7 +218,7 @@ class MediaItemManager(CoreManager): def without_collection(self, *args, **kwargs): return self.get_query_set().without_collection(*args, **kwargs) - without_collection.__doc__ = MediaItemQuerySet.without_collection.__doc__ + without_collection.__doc__ = MediaItemQuerySet.without_collection.__doc__ def by_recording_date(self, *args, **kwargs): return self.get_query_set().by_recording_date(*args, **kwargs) @@ -229,16 +234,16 @@ class MediaItemManager(CoreManager): def by_change_time(self, *args, **kwargs): return self.get_query_set().by_change_time(*args, **kwargs) - by_change_time.__doc__ = MediaItemQuerySet.by_change_time.__doc__ + by_change_time.__doc__ = MediaItemQuerySet.by_change_time.__doc__ def by_location(self, *args, **kwargs): return self.get_query_set().by_location(*args, **kwargs) - by_location.__doc__ = MediaItemQuerySet.by_location.__doc__ + by_location.__doc__ = MediaItemQuerySet.by_location.__doc__ def sound(self, *args, **kwargs): return self.get_query_set().sound(*args, **kwargs) - sound.__doc__ = MediaItemQuerySet.sound.__doc__ - + sound.__doc__ = MediaItemQuerySet.sound.__doc__ + class MediaCollectionQuerySet(CoreQuerySet): @@ -259,20 +264,20 @@ class MediaCollectionQuerySet(CoreQuerySet): def by_location(self, location): "Find collections by location" return self.filter(items__location__in=location.apparented()).distinct() - + def by_recording_year(self, from_year, to_year=None): "Find collections by recording year" if to_year is None: return (self.filter(recorded_from_year__lte=from_year, recorded_to_year__gte=from_year)) else: - return (self.filter(Q(recorded_from_year__range=(from_year, to_year)) | + return (self.filter(Q(recorded_from_year__range=(from_year, to_year)) | Q(recorded_to_year__range=(from_year, to_year)))) def by_publish_year(self, from_year, to_year=None): "Find collections by publishing year" if to_year is None: to_year = from_year - return self.filter(year_published__range=(from_year, to_year)) + return self.filter(year_published__range=(from_year, to_year)) def by_ethnic_group(self, group): "Find collections by ethnic group" @@ -286,12 +291,13 @@ class MediaCollectionQuerySet(CoreQuerySet): qs = self for f in args: if f == 'apparent_collector': - qs = qs.extra(select={f: 'IF(media_collections.collector_is_creator, ' + if not 'sqlite3' in engine: + qs = qs.extra(select={f: 'IF(media_collections.collector_is_creator, ' 'media_collections.creator, media_collections.collector)'}) else: raise Exception("Unsupported virtual field: %s" % f) - return qs + return qs def recording_year_range(self): from_max = self.aggregate(Max('recorded_from_year'))['recorded_from_year__max'] @@ -300,7 +306,7 @@ class MediaCollectionQuerySet(CoreQuerySet): from_min = self.filter(recorded_from_year__gt=0).aggregate(Min('recorded_from_year'))['recorded_from_year__min'] to_min = self.filter(recorded_to_year__gt=0).aggregate(Min('recorded_to_year'))['recorded_to_year__min'] - year_min = min(from_min, to_min) + year_min = min(from_min, to_min) if not year_max: year_max = year_min @@ -324,8 +330,8 @@ class MediaCollectionQuerySet(CoreQuerySet): def sound(self): return self.filter(items__file__contains='/').distinct() - - + + class MediaCollectionManager(CoreManager): "Manage collection queries" @@ -360,16 +366,16 @@ class MediaCollectionManager(CoreManager): def by_change_time(self, *args, **kwargs): return self.get_query_set().by_change_time(*args, **kwargs) by_change_time.__doc__ = MediaCollectionQuerySet.by_change_time.__doc__ - + @staticmethod def __name_cmp(obj1, obj2): return unaccent_icmp(obj1.name, obj2.name) def sound(self, *args, **kwargs): return self.get_query_set().sound(*args, **kwargs) - sound.__doc__ = MediaCollectionQuerySet.sound.__doc__ - - + sound.__doc__ = MediaCollectionQuerySet.sound.__doc__ + + class LocationQuerySet(CoreQuerySet): __flatname_map = None @@ -404,9 +410,9 @@ class LocationManager(CoreManager): def by_flatname(self, *args, **kwargs): return self.get_query_set().by_flatname(*args, **kwargs) - by_flatname.__doc__ = LocationQuerySet.by_flatname.__doc__ + by_flatname.__doc__ = LocationQuerySet.by_flatname.__doc__ def flatname_map(self, *args, **kwargs): return self.get_query_set().flatname_map(*args, **kwargs) - flatname_map.__doc__ = LocationQuerySet.flatname_map.__doc__ + flatname_map.__doc__ = LocationQuerySet.flatname_map.__doc__ diff --git a/telemeta/web/base.py b/telemeta/web/base.py index d9fcb6b2..96993370 100644 --- a/telemeta/web/base.py +++ b/telemeta/web/base.py @@ -880,7 +880,10 @@ class ItemView(object): if not extension in mapping.unavailable_extensions: proc = encoder(audio) proc.set_metadata(metadata) - proc.write_metadata() + try: + proc.write_metadata() + except: + pass response = HttpResponse(stream_from_file(audio), mimetype = mime_type) else: media = self.cache_export.dir + os.sep + file -- 2.39.5