From: olivier <> Date: Mon, 1 Feb 2010 18:11:39 +0000 (+0000) Subject: improve and optimize location models structure and querying X-Git-Tag: 1.1~568 X-Git-Url: https://git.parisson.com/?a=commitdiff_plain;h=e1d0274b3975e1e51d77b9bd75e2c112861e0e09;p=telemeta.git improve and optimize location models structure and querying --- diff --git a/telemeta/models/__init__.py b/telemeta/models/__init__.py index 09106a8b..3b4367b0 100644 --- a/telemeta/models/__init__.py +++ b/telemeta/models/__init__.py @@ -36,43 +36,3 @@ from telemeta.models.crem import * #MediaCollection, MediaItem, MediaPart, Revision, \ # PhysicalFormat, PublishingStatus -from django.db.models.signals import post_syncdb - -def syncdb_callback(sender, **kwargs): - from django.db import connection - import _mysql_exceptions - cursor = connection.cursor() - print "Creating MySQL stored procedure" - try: - cursor.execute("DROP FUNCTION IF EXISTS telemeta_location_ascendant") - except _mysql_exceptions.Warning: - pass - try: - cursor.execute(""" - CREATE FUNCTION telemeta_location_ascendant(loc CHAR(150), asc_type CHAR(16)) - RETURNS CHAR(150) - READS SQL DATA - BEGIN - DECLARE t, n CHAR(150); - DECLARE c INT; - SELECT COUNT(*) INTO c FROM locations WHERE name = loc; - IF c = 0 THEN - RETURN NULL; - END IF; - SELECT name, type INTO n, t FROM locations WHERE name = loc; - WHILE t <> asc_type DO - SELECT COUNT(*) INTO c FROM location_relations WHERE location_name = n; - IF c = 0 THEN - RETURN NULL; - END IF; - SELECT parent_location_name INTO n FROM location_relations WHERE location_name = n LIMIT 1; - SELECT type INTO t FROM locations WHERE name = n; - END WHILE; - RETURN n; - END""") - except _mysql_exceptions.Warning: - pass - -post_syncdb.connect(syncdb_callback) - - diff --git a/telemeta/models/crem.py b/telemeta/models/crem.py index d5618133..4797c692 100755 --- a/telemeta/models/crem.py +++ b/telemeta/models/crem.py @@ -219,13 +219,11 @@ class MediaCollection(MediaResource): def countries(self): "Return the countries of the items" countries = [] - items = self.items.all() - for item in items: - if item.location: - country = item.location.country() - if country and not country in countries: + for item in self.items.filter(location__isnull=False): + for country in item.location.countries(): + if not country in countries: countries.append(country) - + countries.sort(self.__name_cmp) return countries @@ -291,8 +289,7 @@ class MediaItem(MediaResource): approx_duration = DurationField(_('approximative duration')) recorded_from_date = DateField(_('recording date (from)')) recorded_to_date = DateField(_('recording date (until)')) - location = WeakForeignKey('Location', related_name="items", - db_column='location_name', verbose_name=_('location')) + location = WeakForeignKey('Location', verbose_name=_('location')) location_comment = CharField(_('location comment')) ethnic_group = WeakForeignKey('EthnicGroup', related_name="items", verbose_name=_('population / social group')) @@ -552,35 +549,37 @@ class PlaylistResource(ModelCore): db_table = 'playlist_resources' class Location(ModelCore): - "Item location" - TYPE_CHOICES = (('country', 'country'), ('continent', 'continent'), ('other', 'other')) - - name = CharField(_('name'), primary_key=True, max_length=150, required=True) - type = CharField(_('type'), choices=TYPE_CHOICES, max_length=16, required=True) - complete_type = ForeignKey('LocationType', related_name="types", verbose_name=_('complete type')) - current_name = WeakForeignKey('self', related_name="past_names", db_column="current_name", - verbose_name=_('current name')) + "Locations" + OTHER_TYPE = 0 + CONTINENT = 1 + COUNTRY = 2 + TYPE_CHOICES = ((COUNTRY, _('country')), (CONTINENT, _('continent')), (OTHER_TYPE, _('other'))) + + name = CharField(_('name'), unique=True, max_length=150, required=True) + type = IntegerField(_('type'), choices=TYPE_CHOICES, required=True, db_index=True) + complete_type = ForeignKey('LocationType', related_name="locations", verbose_name=_('complete type')) + current_location = WeakForeignKey('self', related_name="past_names", + verbose_name=_('current location')) is_authoritative = BooleanField(_('authoritative')) - def parent(self): - relations = self.parent_relations.all() - if relations: - return relations[0].parent_location + objects = query.LocationManager() - return None + def items(self): + return MediaItem.objects.by_location(self) - def _by_type(self, typename): - location = self - while location and location.type != typename: - location = location.parent() + def collections(self): + return MediaCollection.objects.by_location(self) - return location + def ancestors(self): + return Location.objects.filter(descendant_relations__location=self) - def country(self): - return self._by_type('country') + def descendants(self): + return Location.objects.filter(ancestor_relations__ancestor_location=self) - def continent(self): - return self._by_type('continent') + def countries(self): + if self.type == self.COUNTRY: + return Location.objects.filter(pk=self.id) + return self.ancestors().filter(type=self.COUNTRY) class Meta(MetaCore): db_table = 'locations' @@ -588,7 +587,18 @@ class Location(ModelCore): def __unicode__(self): return self.name - def sequence(self): + def flatname(self): + if self.type != self.COUNTRY and self.type != self.CONTINENT: + raise Exceptions("Flat names are only supported for countries and continents") + + map = Location.objects.flatname_map() + for flatname in map: + if self.id == map[flatname]: + return flatname + + return None + + def sequences(self): sequence = [] location = self while location: @@ -596,22 +606,21 @@ class Location(ModelCore): location = location.parent() return sequence - def fullname(self): + def fullnames(self): return u', '.join([unicode(l) for l in self.sequence()]) class LocationType(ModelCore): - "Location type of an item location" - id = CharField(_('identifier'), max_length=64, primary_key=True, required=True) + "Location types" + code = CharField(_('identifier'), max_length=64, unique=True, required=True) name = CharField(_('name'), max_length=150, required=True) class Meta(MetaCore): db_table = 'location_types' class LocationAlias(ModelCore): - "Location other name" - location = ForeignKey('Location', related_name="aliases", db_column="location_name", - max_length=150, verbose_name=_('location')) + "Location aliases" + location = ForeignKey('Location', related_name="aliases", verbose_name=_('location')) alias = CharField(_('alias'), max_length=150, required=True) is_authoritative = BooleanField(_('authoritative')) @@ -623,15 +632,14 @@ class LocationAlias(ModelCore): unique_together = (('location', 'alias'),) class LocationRelation(ModelCore): - "Location family" - location = ForeignKey('Location', related_name="parent_relations", - db_column="location_name", max_length=150, verbose_name=_('location')) - parent_location = ForeignKey('Location', related_name="child_relations", db_column="parent_location_name", - null=True, max_length=150, verbose_name=_('parent location')) - is_authoritative = BooleanField() + "Location relations" + location = ForeignKey('Location', related_name="ancestor_relations", verbose_name=_('location')) + ancestor_location = ForeignKey('Location', related_name="descendant_relations", verbose_name=_('ancestor location')) + is_direct = BooleanField(db_index=True) class Meta(MetaCore): db_table = 'location_relations' + unique_together = ('location', 'ancestor_location') class ContextKeyword(Enumeration): "Keyword" diff --git a/telemeta/models/cremquery.py b/telemeta/models/cremquery.py index b4452ba9..fb8f0484 100644 --- a/telemeta/models/cremquery.py +++ b/telemeta/models/cremquery.py @@ -40,6 +40,7 @@ import re from django.core.exceptions import ObjectDoesNotExist from django import db import _mysql_exceptions +from telemeta.util.unaccent import unaccent_icmp, unaccent class CoreQuerySet(EnhancedQuerySet): "Base class for all query sets" @@ -103,13 +104,9 @@ class MediaCollectionQuerySet(CoreQuerySet): self.word_search_q('creator', pattern) ) - def by_country(self, country): + def by_location(self, location): "Find collections by country" - db.connection.cursor() # Need this to establish connection - country = db.connection.connection.literal(country) - return self.extra(where=["media_items.collection_id = media_collections.id", - "telemeta_location_ascendant(media_items.location_name, 'country') = %s" % country], - tables=['media_items']).distinct() + return self.filter(Q(items__location=location) | Q(items__location__in=location.descendants())).distinct() def by_continent(self, continent): "Find collections by continent" @@ -148,9 +145,9 @@ class MediaCollectionManager(CoreManager): return self.get_query_set().quick_search(*args, **kwargs) quick_search.__doc__ = MediaCollectionQuerySet.quick_search.__doc__ - def by_country(self, *args, **kwargs): - return self.get_query_set().by_country(*args, **kwargs) - by_country.__doc__ = MediaCollectionQuerySet.by_country.__doc__ + def by_location(self, *args, **kwargs): + return self.get_query_set().by_location(*args, **kwargs) + by_location.__doc__ = MediaCollectionQuerySet.by_location.__doc__ def by_continent(self, *args, **kwargs): return self.get_query_set().by_continent(*args, **kwargs) @@ -172,63 +169,42 @@ class MediaCollectionManager(CoreManager): return self.get_query_set().by_change_time(*args, **kwargs) by_change_time.__doc__ = MediaCollectionQuerySet.by_change_time.__doc__ - def stat_continents(self, order_by='nitems'): + @staticmethod + def __name_cmp(obj1, obj2): + return unaccent_icmp(obj1.name, obj2.name) + + def stat_continents(self, only_continent=None): "Return the number of collections by continents and countries as a tree" - from django.db import connection - cursor = connection.cursor() - if order_by == 'nitems': - order_by = 'items_num DESC' - elif order_by != 'country': - raise Exception("stat_continents() can only order by nitems or country") - - try: - cursor.execute(""" - SELECT telemeta_location_ascendant(location_name, 'continent') as continent, - telemeta_location_ascendant(location_name, 'country') as country, - count(*) AS items_num - FROM media_collections INNER JOIN media_items - ON media_collections.id = media_items.collection_id - GROUP BY country ORDER BY continent, """ + order_by) - except _mysql_exceptions.Warning: - pass - result_set = cursor.fetchall() + from telemeta.models import MediaItem, Location + + countries = [] + for lid in MediaItem.objects.filter(location__isnull=False).values_list('location', flat=True).distinct(): + location = Location.objects.get(pk=lid) + if not only_continent or (only_continent in location.ancestors().filter(type=Location.CONTINENT)): + for l in location.countries(): + if not l in countries: + countries.append(l) + stat = {} - for continent, country, count in result_set: - if continent and country: - if stat.has_key(continent): - stat[continent].append({'name':country, 'count':count}) - else: - stat[continent] = [{'name':country, 'count':count}] - - keys = stat.keys() - keys.sort() - ordered = [{'name': k, 'countries': stat[k]} for k in keys] - return ordered - - def list_countries(self): - "Return a 2D list of all countries with continents" - from django.db import connection - cursor = connection.cursor() + for country in countries: + count = country.collections().count() + for continent in country.ancestors().filter(type=Location.CONTINENT): + if not stat.has_key(continent): + stat[continent] = {} - cursor.execute("SELECT continent, etat FROM media_items " - "GROUP BY continent, etat ORDER BY REPLACE(etat, '\"', '')"); - return cursor.fetchall() - - def list_continents(self): - "Return a list of all continents" - - from django.db import connection - cursor = connection.cursor() - - cursor.execute("SELECT DISTINCT(name) FROM locations WHERE type = 'continent' ORDER BY name") - result_set = cursor.fetchall() - result = [] - for a, in result_set: - if a != '' and a != 'N': # CREM fix - result.append(a) + stat[continent][country] = count + + keys1 = stat.keys() + keys1.sort(self.__name_cmp) + ordered = [] + for c in keys1: + keys2 = stat[c].keys() + keys2.sort(self.__name_cmp) + sub = [{'location': d, 'count': stat[c][d]} for d in keys2] + ordered.append({'location': c, 'countries': sub}) - return result + return ordered class MediaItemQuerySet(CoreQuerySet): @@ -269,6 +245,12 @@ class MediaItemQuerySet(CoreQuerySet): def by_change_time(self, from_time = None, until_time = None): "Find items by last change time" return self._by_change_time('item', from_time, until_time) + + def by_location(self, location): + "Find items by location" + from telemeta.models import LocationRelation + descendants = LocationRelation.objects.filter(ancestor_location=location) + return self.filter(Q(location=location) | Q(location__in=descendants)) class MediaItemManager(CoreManager): "Manage media items queries" @@ -301,3 +283,42 @@ class MediaItemManager(CoreManager): return self.get_query_set().by_change_time(*args, **kwargs) by_change_time.__doc__ = MediaItemQuerySet.by_change_time.__doc__ + def by_location(self, *args, **kwargs): + return self.get_query_set().by_location(*args, **kwargs) + by_location.__doc__ = MediaItemQuerySet.by_location.__doc__ + +class LocationQuerySet(CoreQuerySet): + def by_flatname(self, flatname): + map = LocationManager.flatname_map() + return self.filter(pk=map[flatname]) + +class LocationManager(CoreManager): + __flatname_map = None + + def get_query_set(self): + "Return location query set" + return LocationQuerySet(self.model) + + @classmethod + def flatname_map(cls): + if cls.__flatname_map: + return cls.__flatname_map + + from telemeta.models import Location + map = {} + locations = Location.objects.filter(Q(type=Location.COUNTRY) | Q(type=Location.CONTINENT)) + for l in locations: + flatname = unaccent(l.name).lower() + flatname = re.sub('[^a-z]', '_', flatname) + while map.has_key(flatname): + flatname = '_' + flatname + map[flatname] = l.id + + cls.__flatname_map = map + return map + + def by_flatname(self, *args, **kwargs): + return self.get_query_set().by_flatname(*args, **kwargs) + by_flatname.__doc__ = LocationQuerySet.by_flatname.__doc__ + + diff --git a/telemeta/templates/telemeta_default/geo_continents.html b/telemeta/templates/telemeta_default/geo_continents.html index 6ba9e731..55bc8bfd 100644 --- a/telemeta/templates/telemeta_default/geo_continents.html +++ b/telemeta/templates/telemeta_default/geo_continents.html @@ -6,15 +6,15 @@ {% if continents %}