From: Guillaume Pellerin Date: Wed, 17 Apr 2013 11:26:23 +0000 (+0200) Subject: try better performance for conversation view (P. Samson) X-Git-Url: https://git.parisson.com/?a=commitdiff_plain;h=c9f818befc7d62ee020a0e626228836fd3aee297;p=django-postman.git try better performance for conversation view (P. Samson) --- diff --git a/postman/models.py b/postman/models.py index 5174af5..9bef0d8 100644 --- a/postman/models.py +++ b/postman/models.py @@ -1,17 +1,26 @@ +from __future__ import unicode_literals import hashlib from django.conf import settings -from django.contrib.auth.models import User +try: + from django.contrib.auth import get_user_model # Django 1.5 +except ImportError: + from postman.future_1_5 import get_user_model from django.core.exceptions import ValidationError from django.db import models -from django.utils.text import truncate_words +from django.db.models.query import QuerySet +try: + from django.utils.text import Truncator # Django 1.4 +except ImportError: + from postman.future_1_4 import Truncator from django.utils.translation import ugettext, ugettext_lazy as _ try: - from django.utils.timezone import now # Django 1.4 aware datetimes + from django.utils.timezone import now # Django 1.4 aware datetimes except ImportError: from datetime import datetime now = datetime.now +from postman.query import PostmanQuery from postman.urls import OPTION_MESSAGES from postman.utils import email_visitor, notify_user @@ -25,17 +34,18 @@ STATUS_CHOICES = ( (STATUS_REJECTED, _('Rejected')), ) # ordering constants -ORDER_BY_KEY = 'o' # as 'order' +ORDER_BY_KEY = 'o' # as 'order' ORDER_BY_FIELDS = { - 'f': 'sender__username', # as 'from' - 't': 'recipient__username', # as 'to' + 'f': 'sender__' + get_user_model().USERNAME_FIELD, # as 'from' + 't': 'recipient__' + get_user_model().USERNAME_FIELD, # as 'to' 's': 'subject', # as 'subject' 'd': 'sent_at', # as 'date' } -ORDER_BY_MAPPER = {'sender': 'f', 'recipient': 't', 'subject': 's', 'date': 'd'} # for templatetags usage +ORDER_BY_MAPPER = {'sender': 'f', 'recipient': 't', 'subject': 's', 'date': 'd'} # for templatetags usage + +# dbms = settings.DATABASES['default']['ENGINE'].rsplit('.', 1)[-1] +# QUOTE_CHAR = '`' if dbms == 'mysql' else '"' -dbms = settings.DATABASES['default']['ENGINE'].rsplit('.',1)[-1] -QUOTE_CHAR = '`' if dbms == 'mysql' else '"' def get_order_by(query_dict): """ @@ -44,39 +54,53 @@ def get_order_by(query_dict): Argument: ``query_dict``: a dictionary to look for a key dedicated to ordering purpose - >>> get_order_by({}) - - >>> get_order_by({ORDER_BY_KEY: 'f'}) - 'sender__username' - >>> get_order_by({ORDER_BY_KEY: 'D'}) - '-sent_at' """ if ORDER_BY_KEY in query_dict: - code = query_dict[ORDER_BY_KEY] # code may be uppercase or lowercase + code = query_dict[ORDER_BY_KEY] # code may be uppercase or lowercase order_by_field = ORDER_BY_FIELDS.get(code.lower()) if order_by_field: if code.isupper(): order_by_field = '-' + order_by_field return order_by_field + +def get_user_representation(user): + """ + Return a User representation for display, configurable through an optional setting. + """ + show_user_as = getattr(settings, 'POSTMAN_SHOW_USER_AS', None) + if isinstance(show_user_as, (unicode, str)): + attr = getattr(user, show_user_as, None) + if callable(attr): + attr = attr() + if attr: + return unicode(attr) + elif callable(show_user_as): + try: + return unicode(show_user_as(user)) + except: + pass + return unicode(user) # default value, or in case of empty attribute or exception + +# import logging +# logging.getLogger('django.db.backends').setLevel(logging.DEBUG) class MessageManager(models.Manager): """The manager for Message.""" - @property - def _last_in_thread(self): - """Return the latest message id for each conversation.""" - return self.filter(thread__isnull=False).values('thread').annotate(models.Max('pk'))\ - .values_list('pk__max', flat=True).order_by() + # @property + # def _last_in_thread(self): + # """Return the latest message id for each conversation.""" + # return self.filter(thread__isnull=False).values('thread').annotate(models.Max('pk'))\ + # .values_list('pk__max', flat=True).order_by() def _folder(self, related, filters, option=None, order_by=None): """Base code, in common to the folders.""" + qs = self.all() if option == OPTION_MESSAGES else QuerySet(self.model, PostmanQuery(self.model), using=self._db) if related: - qs = self.select_related(*related) - else: - qs = self.all() + qs = qs.select_related(*related) if order_by: qs = qs.order_by(order_by) - if isinstance(filters, (list,tuple)): + if isinstance(filters, (list, tuple)): lookups = models.Q() for filter in filters: lookups |= models.Q(**filter) @@ -88,12 +112,25 @@ class MessageManager(models.Manager): # should not be necessary. Otherwise add: # .extra(select={'count': 'SELECT 1'}) else: - return qs.filter( - models.Q(id__in=self._last_in_thread.filter(lookups)) | models.Q(lookups, thread__isnull=True) - ).extra(select={'count': QUOTE_CHAR.join([ - 'SELECT COUNT(*) FROM ', 'postman_message', ' T' - ' WHERE T.', 'thread_id', ' = ', 'postman_message', '.', 'thread_id', ' ' - ])}) + qs = qs.extra(select={'count': '{0}.count'.format(qs.query.pm_alias_prefix)}) + qs.query.pm_set_extra(table=( + self.filter(lookups, thread_id__isnull=True).extra(select={'count': 0})\ + .values_list('id', 'count').order_by(), + self.filter(lookups, thread_id__isnull=False).values('thread').annotate(id=models.Max('pk'), count=models.Count('pk'))\ + .values_list('id', 'count').order_by(), + )) + return qs + # from rqs import PostmanRawQuery, PostmanRawQuerySet + # raw_query = 'select *, 12 as count from postman_message' + # query = PostmanRawQuery(sql=raw_query, using=Message.objects.db) + # rqs = PostmanRawQuerySet(raw_query=raw_query, model=Message, query=query) + # return rqs + # return qs.filter( + # models.Q(id__in=self._last_in_thread.filter(lookups)) | models.Q(lookups, thread__isnull=True) + # ).extra(select={'count': QUOTE_CHAR.join([ + # 'SELECT COUNT(*) FROM ', 'postman_message', ' T' + # ' WHERE T.', 'thread_id', ' = ', 'postman_message', '.', 'thread_id', ' ' + # ])}) # For single message, 'count' is returned as 0. Should be acceptable if known. # If not, replace "COUNT(*)" by "1+COUNT(*)" and add: # ' AND T."id" <> T."thread_id"' @@ -119,7 +156,7 @@ class MessageManager(models.Manager): """ return self.inbox(user, related=False, option=OPTION_MESSAGES).filter(read_at__isnull=True).count() - + def sent(self, user, **kwargs): """ Return all messages sent by a user but not marked as archived or deleted. @@ -137,7 +174,7 @@ class MessageManager(models.Manager): """ Return messages belonging to a user and marked as archived. """ - related = ('sender','recipient') + related = ('sender', 'recipient') filters = ({ 'recipient': user, 'recipient_archived': True, @@ -154,7 +191,7 @@ class MessageManager(models.Manager): """ Return messages belonging to a user and marked as deleted. """ - related = ('sender','recipient') + related = ('sender', 'recipient') filters = ({ 'recipient': user, 'recipient_deleted_at__isnull': False, @@ -169,7 +206,7 @@ class MessageManager(models.Manager): """ Return message/conversation for display. """ - return self.select_related('sender','recipient').filter( + return self.select_related('sender', 'recipient').filter( filter, (models.Q(recipient=user) & models.Q(moderation_status=STATUS_ACCEPTED)) | models.Q(sender=user), ).order_by('sent_at') @@ -184,7 +221,7 @@ class MessageManager(models.Manager): """ Return messages matching a filter AND being visible to a user as the sender. """ - return self.filter(filter, sender=user) # any status is fine + return self.filter(filter, sender=user) # any status is fine def perms(self, user): """ @@ -206,6 +243,7 @@ class MessageManager(models.Manager): read_at__isnull=True, ).update(read_at=now()) + class Message(models.Model): """ A message between a User and another User or an AnonymousUser. @@ -215,9 +253,9 @@ class Message(models.Model): subject = models.CharField(_("subject"), max_length=SUBJECT_MAX_LENGTH) body = models.TextField(_("body"), blank=True) - sender = models.ForeignKey(User, related_name='sent_messages', null=True, blank=True, verbose_name=_("sender")) - recipient = models.ForeignKey(User, related_name='received_messages', null=True, blank=True, verbose_name=_("recipient")) - email = models.EmailField(_("visitor"), blank=True) # instead of either sender or recipient, for an AnonymousUser + sender = models.ForeignKey(get_user_model(), related_name='sent_messages', null=True, blank=True, verbose_name=_("sender")) + recipient = models.ForeignKey(get_user_model(), related_name='received_messages', null=True, blank=True, verbose_name=_("recipient")) + email = models.EmailField(_("visitor"), blank=True) # instead of either sender or recipient, for an AnonymousUser parent = models.ForeignKey('self', related_name='next_messages', null=True, blank=True, verbose_name=_("parent message")) thread = models.ForeignKey('self', related_name='child_messages', null=True, blank=True, verbose_name=_("root message")) sent_at = models.DateTimeField(_("sent at"), default=now) @@ -229,7 +267,7 @@ class Message(models.Model): recipient_deleted_at = models.DateTimeField(_("deleted by recipient at"), null=True, blank=True) # moderation fields moderation_status = models.CharField(_("status"), max_length=1, choices=STATUS_CHOICES, default=STATUS_PENDING) - moderation_by = models.ForeignKey(User, related_name='moderated_messages', + moderation_by = models.ForeignKey(get_user_model(), related_name='moderated_messages', null=True, blank=True, verbose_name=_("moderator")) moderation_date = models.DateTimeField(_("moderated at"), null=True, blank=True) moderation_reason = models.CharField(_("rejection reason"), max_length=120, blank=True) @@ -242,7 +280,7 @@ class Message(models.Model): ordering = ['-sent_at', '-id'] def __unicode__(self): - return u"{0}>{1}:{2}".format(self.obfuscated_sender, self.obfuscated_recipient, truncate_words(self.subject,5)) + return "{0}>{1}:{2}".format(self.obfuscated_sender, self.obfuscated_recipient, Truncator(self.subject).words(5)) @models.permalink def get_absolute_url(self): @@ -280,12 +318,12 @@ class Message(models.Model): """ email = self.email digest = hashlib.md5(email + settings.SECRET_KEY).hexdigest() - shrunken_digest = '..'.join((digest[:4], digest[-4:])) # 32 characters is too long and is useless + shrunken_digest = '..'.join((digest[:4], digest[-4:])) # 32 characters is too long and is useless bits = email.split('@') - if len(bits) <> 2: - return u'' + if len(bits) != 2: + return '' domain = bits[1] - return '@'.join((shrunken_digest, domain.rsplit('.',1)[0])) # leave off the TLD to gain some space + return '@'.join((shrunken_digest, domain.rsplit('.', 1)[0])) # leave off the TLD to gain some space def admin_sender(self): """ @@ -307,7 +345,7 @@ class Message(models.Model): def obfuscated_sender(self): """Return the sender either as a username or as an undisclosed email.""" if self.sender: - return unicode(self.sender) + return get_user_representation(self.sender) else: return self._obfuscated_email() @@ -331,7 +369,7 @@ class Message(models.Model): def obfuscated_recipient(self): """Return the recipient either as a username or as an undisclosed email.""" if self.recipient: - return unicode(self.recipient) + return get_user_representation(self.recipient) else: return self._obfuscated_email() @@ -353,7 +391,7 @@ class Message(models.Model): def clean_moderation(self, initial_status, user=None): """Adjust automatically some fields, according to status workflow.""" - if self.moderation_status <> initial_status: + if self.moderation_status != initial_status: self.moderation_date = now() self.moderation_by = user if self.is_rejected(): @@ -385,7 +423,7 @@ class Message(models.Model): def update_parent(self, initial_status): """Update the parent to actualize its response state.""" - if self.moderation_status <> initial_status: + if self.moderation_status != initial_status: parent = self.parent if self.is_accepted(): # keep the very first date; no need to do differently @@ -467,7 +505,7 @@ class Message(models.Model): reasons.append(reason) if auto is None and percents: average = float(sum(percents)) / len(percents) - final_reason = ', '.join([r for i,r in enumerate(reasons) if r and not r.isspace() and percents[i] < 50]) + final_reason = ', '.join([r for i, r in enumerate(reasons) if r and not r.isspace() and percents[i] < 50]) auto = average >= 50 if auto is None: auto = getattr(settings, 'POSTMAN_AUTO_MODERATE_AS', None) @@ -477,6 +515,7 @@ class Message(models.Model): self.moderation_status = STATUS_REJECTED self.moderation_reason = final_reason + class PendingMessageManager(models.Manager): """The manager for PendingMessage.""" @@ -484,6 +523,7 @@ class PendingMessageManager(models.Manager): """Filter to get only pending objects.""" return super(PendingMessageManager, self).get_query_set().filter(moderation_status=STATUS_PENDING) + class PendingMessage(Message): """ A proxy to Message, focused on pending objects to accept or reject. diff --git a/postman/query.py b/postman/query.py new file mode 100644 index 0000000..92252cd --- /dev/null +++ b/postman/query.py @@ -0,0 +1,98 @@ +from __future__ import unicode_literals +import new +from types import MethodType + +from django.db.models.sql.query import Query + +class Proxy(object): + """ + Code base for an instance proxy. + """ + + def __init__(self, target): + self._target = target + + def __getattr__(self, name): + target = self._target + f = getattr(target, name) + if isinstance(f, MethodType): + # print name,'()' + return new.instancemethod(f.im_func, self, target.__class__) + else: + # print name + return f + + def __setattr__(self, name, value): + # print name, '=', value + if name != '_target': + setattr(self._target, name, value) + else: + object.__setattr__(self, name, value) + +class CompilerProxy(Proxy): + """ + A proxy to a compiler. + """ + + # @Override + def as_sql(self, *args, **kwargs): + sql, params = self._target.as_sql(*args, **kwargs) + # mimics compiler.py/SQLCompiler/get_from_clause() and as_sql() + qn = self.quote_name_unless_alias + qn2 = self.connection.ops.quote_name + alias = self.query.tables[0] + name, alias, join_type, lhs, lhs_col, col, nullable = self.query.alias_map[alias] + alias_str = (alias != name and ' {0}'.format(alias) or '') + clause = 'FROM {0}{1}'.format(qn(name), alias_str) + index = sql.index(clause) + len(clause) + extra_table, extra_params = self.union(self.query.pm_get_extra()) + # print extra_table, extra_params + new_sql = [ + sql[:index], + ' {0} ({1}) {2} ON ({3}.{4} = {2}.{5})'.format( + self.query.INNER, extra_table, self.query.pm_alias_prefix, qn(alias), qn2('id'), qn2('id')), + ] + if index < len(sql): + new_sql.append(sql[index:]) + new_sql = ''.join(new_sql) + # print new_sql + return new_sql, extra_params + params + + def union(self, querysets): + """ + Join several querysets by a UNION clause. Returns the SQL string and the list of parameters. + """ + result_sql, result_params = [], [] + for qs in querysets: + sql, params = qs.query.sql_with_params() + result_sql.append(sql) + result_params.extend(params) + return ' UNION '.join(result_sql), tuple(result_params) + +class PostmanQuery(Query): + """ + A custom SQL query. + """ + pm_alias_prefix = 'PM' + + # @Override + def __init__(self, *args, **kwargs): + super(PostmanQuery, self).__init__(*args, **kwargs) + self._pm_table = None + + # @Override + def clone(self, *args, **kwargs): + obj = super(PostmanQuery, self).clone(*args, **kwargs) + obj._pm_table = self._pm_table + return obj + + # @Override + def get_compiler(self, *args, **kwargs): + compiler = super(PostmanQuery, self).get_compiler(*args, **kwargs) + return CompilerProxy(compiler) + + def pm_set_extra(self, table): + self._pm_table = table + + def pm_get_extra(self): + return self._pm_table