From cd6ff8c248f9bb91956f80911d9e27afac91dd85 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Sun, 27 Mar 2016 23:47:02 +0200 Subject: [PATCH] rename files, add more compose env files, prepare saved_searches --- app/deploy/app.sh | 55 ++++++ app/deploy/{apt-app.list => apt.list} | 0 app/deploy/install_linux.py | 169 ------------------ app/deploy/my.cnf | 56 ------ app/deploy/nginx-dev.conf | 28 +++ app/deploy/{nginx-app.conf => nginx.conf} | 0 app/deploy/start_app.sh | 50 ------ app/deploy/{celery_app.sh => worker.sh} | 2 +- app/sandbox/settings.py | 3 +- app/{celery_app.py => worker.py} | 0 docker-compose.yml | 26 ++- env/build.yml | 28 +++ .../tm-docker-compose.env => env/debug.env | 0 env/debug.yml | 34 ++++ env/dev.yml | 37 ++++ env/prod.env | 19 ++ requirements-dev.txt | 1 + telemeta/models/system.py | 10 +- telemeta/views/haystack_search.py | 28 ++- 19 files changed, 231 insertions(+), 315 deletions(-) create mode 100644 app/deploy/app.sh rename app/deploy/{apt-app.list => apt.list} (100%) delete mode 100755 app/deploy/install_linux.py delete mode 100644 app/deploy/my.cnf create mode 100644 app/deploy/nginx-dev.conf rename app/deploy/{nginx-app.conf => nginx.conf} (100%) delete mode 100644 app/deploy/start_app.sh rename app/deploy/{celery_app.sh => worker.sh} (88%) rename app/{celery_app.py => worker.py} (100%) create mode 100644 env/build.yml rename app/sandbox/tm-docker-compose.env => env/debug.env (100%) create mode 100644 env/debug.yml create mode 100644 env/dev.yml create mode 100644 env/prod.env diff --git a/app/deploy/app.sh b/app/deploy/app.sh new file mode 100644 index 00000000..02aba8e8 --- /dev/null +++ b/app/deploy/app.sh @@ -0,0 +1,55 @@ +#!/bin/sh + +# paths +app='/srv/app' +manage=$app'/manage.py' +wsgi=$app'/wsgi.py' +static='/srv/static/' +media='/srv/media/' +src='/srv/src/' + +# uwsgi params +port=8000 +processes=8 +threads=8 +autoreload=3 +uid='www-data' +gid='www-data' + +# stating apps +pip install -U django==1.6.11 django-environ redis django-angular +pip install -e git+https://github.com/django-haystack/saved_searches.git#egg=saved_searches-2.0.0-alpha + +# waiting for other services +sh $app/deploy/wait.sh + +# waiting for available database +python $app/wait.py + +# django init +python $manage syncdb --noinput +python $manage migrate --noinput +python $manage bower_install -- --allow-root +python $manage collectstatic --noinput +python $manage telemeta-create-admin-user +python $manage telemeta-create-boilerplate + +if [ $DEBUG = "False" ] +then + python $manage update_index --workers $processes & +fi + +if [ $1 = "--runserver" ] +then + python $manage runserver 0.0.0.0:8000 +else + # static files auto update + watchmedo shell-command --patterns="*.js;*.css" --recursive \ + --command='python '$manage' collectstatic --noinput' $src & + + # app start + uwsgi --socket :$port --wsgi-file $wsgi --chdir $app --master \ + --processes $processes --threads $threads \ + --uid $uid --gid $gid \ + --py-autoreload $autoreload +fi diff --git a/app/deploy/apt-app.list b/app/deploy/apt.list similarity index 100% rename from app/deploy/apt-app.list rename to app/deploy/apt.list diff --git a/app/deploy/install_linux.py b/app/deploy/install_linux.py deleted file mode 100755 index da6d7bef..00000000 --- a/app/deploy/install_linux.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/python - -import os, sys -import argparse - -sysvinit_script = """ -#!/bin/sh - -### BEGIN INIT INFO -# Provides: %s -# Required-Start: docker -# Required-Stop: docker -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Docker Services -### END INIT INFO - -set -e - -PROJECT_NAME=%s -YAMLFILE=%s -OPTS="-f $YAMLFILE -p $PROJECT_NAME" -UPOPTS="-d --no-recreate --no-build --no-deps" - -. /lib/lsb/init-functions - -case "$1" in - start) - log_daemon_msg "Starting $PROJECT_NAME composition" "$PROJECT_NAME" || true - if su -c "docker-compose $OPTS up $UPOPTS > /dev/null 2>&1" root ; then - log_end_msg 0 || true - else - log_end_msg 1 || true - fi - ;; - - stop) - log_daemon_msg "Stopping $PROJECT_NAME composition" "$PROJECT_NAME" || true - if su -c "docker-compose $OPTS stop > /dev/null 2>&1" root; then - log_end_msg 0 || true - else - log_end_msg 1 || true - fi - ;; - - reload|force-reload) - log_daemon_msg "Reloading $PROJECT_NAME composition" "$PROJECT_NAME" || true - if docker-compose $OPTS up $UPOPTS > /dev/null 2>&1 ; then - log_end_msg 0 || true - else - log_end_msg 1 || true - fi - ;; - - restart|try-restart) - log_daemon_msg "Restarting $PROJECT_NAME composition" "$PROJECT_NAME" || true - if docker-compose $OPTS stop > /dev/null 2>&1; docker-compose $OPTS up $UPOPTS > /dev/null 2>&1 ; then - log_end_msg 0 || true - else - log_end_msg 1 || true - fi - ;; - - status) - docker-compose $OPTS ps && exit 0 || exit $? - ;; - - *) - log_action_msg "Usage: /etc/init.d/$PROJECT_NAME {start|stop|reload|force-reload|restart|try-restart|status}" || true - exit 1 - ;; -esac - -exit 0 -""" - -systemd_service = """ -[Unit] -Description=%s composition -Requires=docker.service -After=docker.service - -[Service] -ExecStart=%s -f %s up -d -ExecStop=%s -f %s stop - -[Install] -WantedBy=local.target -""" - - -class DockerComposeDaemonInstall(object): - - vcs_types = ['git', 'svn', 'hg'] - docker = '/etc/init.d/docker' - docker_compose = '/usr/local/bin/docker-compose' - - def __init__(self, path=None, init_type='sysvinit'): - self.init_type = init_type - - self.local_path = os.path.dirname(os.path.realpath(__file__)) - if not path or not os.path.isdir(path): - self.root = self.get_root(self.local_path) - else: - self.root = os.path.abspath(path) - - if self.root[-1] == os.sep: - self.name = self.root.split(os.sep)[-2].lower() - else: - self.name = self.root.split(os.sep)[-1].lower() - - self.conf = self.root + os.sep + 'docker-compose.yml' - - def is_root(self, path): - content = os.listdir(path) - for vcs_type in self.vcs_types: - if '.' + vcs_type in content: - return True - return False - - def get_root(self, path): - while not self.is_root(path): - path = os.sep.join(path.split(os.sep)[:-1]) - if not path: - raise ValueError('This is not a versioned repository, please give the root directory of the app as the first argument.') - return path - - def install_docker(self): - if not os.path.exists(self.docker): - print 'Installing docker first...' - os.system('wget -qO- https://get.docker.com/ | sh') - if not os.path.exists(self.docker_compose): - print 'Installing docker-compose...' - os.system('pip install docker-compose') - - def install_daemon_sysvinit(self): - service = '/etc/init.d/' + self.name - print 'Writing sysvinit script in ' + service - script = sysvinit_script % (self.name, self.name, self.conf) - f = open(service, 'w') - f.write(script) - f.close() - os.system('chmod 755 ' + service) - os.system('update-rc.d ' + self.name + ' defaults') - - def install_daemon_systemd(self): - service = '/lib/systemd/system/' + self.name + '.service' - print 'Writing systemd service in ' + service - conf = systemd_service % (self.name, self.docker_compose, self.conf, self.docker_compose, self.conf) - f = open(service, 'w') - f.write(rules) - f.close() - os.system('systemctl enable ' + service) - os.system('systemctl daemon-reload') - - def run(self): - print 'Installing ' + self.name + ' composition as a daemon...' - self.install_docker() - if self.init_type == 'sysvinit': - self.install_daemon_sysvinit() - elif self.init_type == 'systemd': - self.install_daemon_systemd() - print 'Done' - - -if __name__ == '__main__': - path = sys.argv[-1] - install = DockerComposeDaemonInstall(path) - install.run() diff --git a/app/deploy/my.cnf b/app/deploy/my.cnf deleted file mode 100644 index 469e6401..00000000 --- a/app/deploy/my.cnf +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -# -# The MySQL Community Server configuration file. -# -# For explanations see -# http://dev.mysql.com/doc/mysql/en/server-system-variables.html - -[client] -port = 3306 -socket = /var/run/mysqld/mysqld.sock - -[mysqld_safe] -pid-file = /var/run/mysqld/mysqld.pid -socket = /var/run/mysqld/mysqld.sock -nice = 0 - -[mysqld] -skip-host-cache -skip-name-resolve -user = mysql -pid-file = /var/run/mysqld/mysqld.pid -socket = /var/run/mysqld/mysqld.sock -port = 3306 -basedir = /usr -datadir = /var/lib/mysql -tmpdir = /tmp -lc-messages-dir = /usr/share/mysql -explicit_defaults_for_timestamp - -# Instead of skip-networking the default is now to listen only on -# localhost which is more compatible and is not less secure. -#bind-address = 127.0.0.1 - -#log-error = /var/log/mysql/error.log - -# Disabling symbolic-links is recommended to prevent assorted security risks -symbolic-links=0 - -# * IMPORTANT: Additional settings that can override those from this file! -# The files must end with '.cnf', otherwise they'll be ignored. -# -!includedir /etc/mysql/conf.d/ diff --git a/app/deploy/nginx-dev.conf b/app/deploy/nginx-dev.conf new file mode 100644 index 00000000..164f71da --- /dev/null +++ b/app/deploy/nginx-dev.conf @@ -0,0 +1,28 @@ +server_tokens off; + +server { + listen 80; + server_name nginx; + charset utf-8; + + access_log /var/log/nginx/app-access.log; + error_log /var/log/nginx/app-error.log; + + # max upload size + client_max_body_size 4096M; # adjust to taste + + # phpmyadmin + location /phpmyadmin/ { + proxy_pass http://phpmyadmin/; + #proxy_set_header Host $http_host; + #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + #proxy_set_header X-Forwarded-Server $http_host; + #proxy_redirect / /phpmyadmin/; + #proxy_cookie_path / /phpmyadmin/; + } + + location / { + uwsgi_pass app:8000; + include /etc/nginx/uwsgi_params; + } +} diff --git a/app/deploy/nginx-app.conf b/app/deploy/nginx.conf similarity index 100% rename from app/deploy/nginx-app.conf rename to app/deploy/nginx.conf diff --git a/app/deploy/start_app.sh b/app/deploy/start_app.sh deleted file mode 100644 index b349f9b0..00000000 --- a/app/deploy/start_app.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh - -# paths -app='/srv/app' -manage=$app'/manage.py' -wsgi=$app'/wsgi.py' -static='/srv/static/' -media='/srv/media/' -src='/srv/src/' - -# uwsgi params -port=8000 -processes=8 -threads=8 -autoreload=3 -uid='www-data' -gid='www-data' - -# stating apps -pip install django-environ redis - -# waiting for other services -sh $app/deploy/wait.sh - -# waiting for available database -python $app/wait.py - -# django init -python $manage syncdb --noinput -python $manage migrate --noinput -python $manage bower_install -- --allow-root -python $manage collectstatic --noinput -python $manage telemeta-create-admin-user -python $manage telemeta-create-boilerplate - -if [ ! -f $app/.init ]; then - chown www-data:www-data $media - python $manage update_index --workers $processes & - touch $app/.init -fi - -# static files auto update -watchmedo shell-command --patterns="*.js;*.css" --recursive \ - --command='python '$manage' collectstatic --noinput' $src & - -# app start -uwsgi --socket :$port --wsgi-file $wsgi --chdir $app --master \ - --processes $processes --threads $threads \ - --uid $uid --gid $gid \ - --py-autoreload $autoreload diff --git a/app/deploy/celery_app.sh b/app/deploy/worker.sh similarity index 88% rename from app/deploy/celery_app.sh rename to app/deploy/worker.sh index 9fdcbc1d..5a1139a8 100644 --- a/app/deploy/celery_app.sh +++ b/app/deploy/worker.sh @@ -14,4 +14,4 @@ sh $app/deploy/wait.sh # Starting celery worker with the --autoreload option will enable the worker to watch for file system changes # This is an experimental feature intended for use in development only # see http://celery.readthedocs.org/en/latest/userguide/workers.html#autoreloading -python $manage celery worker --autoreload -A celery_app +python $manage celery worker --autoreload -A worker diff --git a/app/sandbox/settings.py b/app/sandbox/settings.py index ef1113be..e9dc02cf 100644 --- a/app/sandbox/settings.py +++ b/app/sandbox/settings.py @@ -171,6 +171,7 @@ INSTALLED_APPS = ( 'haystack', 'djangobower', 'djng', + 'saved_searches', ) TEMPLATE_CONTEXT_PROCESSORS = ( @@ -297,7 +298,7 @@ CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_ACCEPT_CONTENT = ['application/json'] -from celery_app import app +from worker import app HAYSTACK_CONNECTIONS = { 'default': { diff --git a/app/celery_app.py b/app/worker.py similarity index 100% rename from app/celery_app.py rename to app/worker.py diff --git a/docker-compose.yml b/docker-compose.yml index 239a86a1..89cdd3c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,17 +4,17 @@ # This file is part of Telemeta. -# TimeSide is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 2 of the License, or # (at your option) any later version. -# Telemeta is distributed in the hope that it will be useful, +# TimeSide is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. -# You should have received a copy of the GNU General Public License +# You should have received a copy of the GNU Affero General Public License # along with TimeSide. If not, see . # Authors: @@ -38,7 +38,7 @@ db: volumes_from: - data env_file: - - app/sandbox/tm-docker-compose.env + - env/prod.env broker: image: redis @@ -50,14 +50,11 @@ app: build: . volumes: - ./app/:/srv/app - - ./telemeta/:/srv/src/telemeta/telemeta volumes_from: - data env_file: - - app/sandbox/tm-docker-compose.env - command: /bin/sh deploy/start_app.sh - ports: - - "9001:9000" + - env/prod.env + command: /bin/sh deploy/app.sh links: - broker - db @@ -68,8 +65,8 @@ worker: volumes_from: - app env_file: - - app/sandbox/tm-docker-compose.env - command: /bin/sh deploy/celery_app.sh + - env/prod.env + command: /bin/sh deploy/worker.sh links: - broker - db @@ -79,8 +76,7 @@ nginx: ports: - "8000:80" volumes: - - ./app/deploy/nginx-app.conf:/etc/nginx/conf.d/default.conf - - ./var/log/nginx/:/var/log/nginx + - ./app/deploy/nginx.conf:/etc/nginx/conf.d/default.conf volumes_from: - data links: diff --git a/env/build.yml b/env/build.yml new file mode 100644 index 00000000..3ee1e4c0 --- /dev/null +++ b/env/build.yml @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2015-2016 Parisson SARL + +# This file is part of Telemeta. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. + +# TimeSide is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with TimeSide. If not, see . + +# Authors: +# Guillaume Pellerin +# Thomas Fillon + +app: + build: . + +worker: + build: . diff --git a/app/sandbox/tm-docker-compose.env b/env/debug.env similarity index 100% rename from app/sandbox/tm-docker-compose.env rename to env/debug.env diff --git a/env/debug.yml b/env/debug.yml new file mode 100644 index 00000000..7a5301ab --- /dev/null +++ b/env/debug.yml @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2015-2016 Parisson SARL + +# This file is part of Telemeta. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. + +# TimeSide is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with TimeSide. If not, see . + +# Authors: +# Guillaume Pellerin +# Thomas Fillon + + +app: + volumes: + - ./app/:/srv/app + - ./telemeta/:/srv/src/telemeta/telemeta + env_file: + - env/debug.env + +worker: + env_file: + - env/debug.env diff --git a/env/dev.yml b/env/dev.yml new file mode 100644 index 00000000..94ef14ad --- /dev/null +++ b/env/dev.yml @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2015-2016 Parisson SARL + +# This file is part of Telemeta. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. + +# TimeSide is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with TimeSide. If not, see . + +# Authors: +# Guillaume Pellerin +# Thomas Fillon + + +app: + volumes: + - ./app/:/srv/app + - ./telemeta/:/srv/src/telemeta/telemeta + env_file: + - env/debug.env + command: /bin/sh deploy/app.sh --runserver + ports: + - 9000:8000 + +worker: + env_file: + - env/debug.env diff --git a/env/prod.env b/env/prod.env new file mode 100644 index 00000000..79b9b03c --- /dev/null +++ b/env/prod.env @@ -0,0 +1,19 @@ +# -- MYSQL DATABASE + +ENGINE=django.db.backends.mysql +MYSQL_ROOT_PASSWORD=mysecretpassword +MYSQL_DATABASE=telemeta +MYSQL_USER=telemeta +MYSQL_PASSWORD=mysecretpassword + +# -- DJANGO + +DEBUG=False + +SECRET_KEY=ghv8us2587n97dq&w$c((o5rj_$-9#d-8j#57y_a9og8wux1h7 + +# replace broker by localhost if you start your app outside docker-compose +BROKER_URL=redis://broker:6379/0 + +# If this is True, all tasks will be executed locally by blocking until the task returns. +CELERY_ALWAYS_EAGER=False diff --git a/requirements-dev.txt b/requirements-dev.txt index 764c86bf..552e7311 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ -e git+https://github.com/mariocesar/sorl-thumbnail.git@v12.2#egg=sorl-thumbnail-12.2 -e git+https://github.com/Parisson/ebooklib.git#egg=ebooklib-0.16 +-e git+https://github.com/django-haystack/saved_searches.git#egg=saved_searches-2.0.0-alpha diff --git a/telemeta/models/system.py b/telemeta/models/system.py index 73ab5551..cfc536c6 100644 --- a/telemeta/models/system.py +++ b/telemeta/models/system.py @@ -58,8 +58,7 @@ class Revision(ModelCore): @classmethod def touch(cls, element, user): "Create or update a revision" - revision = cls(element_type=element.element_type, element_id=element.pk, - user=user, change_type='create') + revision = cls(element_type=element.element_type, element_id=element.pk, user=user, change_type='create') if element.pk: try: element.__class__.objects.get(pk=element.pk) @@ -116,14 +115,11 @@ class Search(ModelCore): username = ForeignKey(User, related_name="searches", db_column="username") date = DateTimeField(_('date'), auto_now_add=True) description = CharField(_('Description')) - criteria = models.ManyToManyField(Criteria, related_name="search", - verbose_name=_('criteria'), blank=True, null=True) + criteria = models.ManyToManyField(Criteria, related_name="search", verbose_name=_('criteria'), blank=True, null=True) class Meta(MetaCore): db_table = 'searches' ordering = ['-date'] def __unicode__(self): - return ' - '.join([self.username.username, unicode(self.date), - ' - '.join([c.key for c in self.criteria.all()])]) - + return ' - '.join([self.username.username, unicode(self.date), ' - '.join([c.key for c in self.criteria.all()])]) diff --git a/telemeta/views/haystack_search.py b/telemeta/views/haystack_search.py index 3a57ff98..7509f59b 100644 --- a/telemeta/views/haystack_search.py +++ b/telemeta/views/haystack_search.py @@ -12,10 +12,13 @@ class HaystackSearch(FacetedSearchView): self.type = type self.form_class = HaySearchForm self.selected_facet = self.selected_facet_list(request.GET.getlist('selected_facets', ['a'])) + print(self.selected_facet) if request.GET.get('results_page'): self.results_per_page = int(request.GET.get('results_page')) else: self.results_per_page = 20 + self.request = request + self.save_search() return super(HaystackSearch, self).__call__(request) def get_query(self): @@ -74,6 +77,7 @@ class HaystackSearch(FacetedSearchView): extra['Radio_count'] = self.get_results().narrow('recording_context:Radio').count() extra['Video_count'] = self.get_results().narrow('media_type:Video').count() extra['Audio_count'] = self.get_results().narrow('media_type:Audio').count() + if self.type == 'item': extra['type'] = 'item' elif self.type == 'fonds': @@ -88,26 +92,18 @@ class HaystackSearch(FacetedSearchView): extra['results_page'] = self.results_per_page return extra - def save_search(self, request): - user = request.user + def save_search(self): + user = self.request.user if user: if user.is_authenticated(): search = Search(username=user) search.save() - if criteria: - for key in criteria.keys(): - value = criteria[key] - if key == 'ethnic_group': - try: - group = EthnicGroup.objects.get(value=value) - value = group.id - except: - value = '' - criter = Criteria(key=key, value=value) - criter.save() - search.criteria.add(criter) - search.save() - + q = self.get_query() + print(q) + criteria = Criteria(key=key, value=value) + criteria.save() + search.criteria.add(criter) + search.save() #def auto_complete(request): #content = SearchQuerySet().autocomplete(content_auto=request.POST.get('seatch_text', '')) -- 2.39.5