From: Guillaume Pellerin Date: Wed, 18 Mar 2015 22:09:29 +0000 (+0100) Subject: Add timeside.server and a celery based worker X-Git-Tag: 1.6a^2~48^2~5 X-Git-Url: https://git.parisson.com/?a=commitdiff_plain;h=d594c2eac2d00f563aecc6d7983bdc11b75fbbf2;p=telemeta.git Add timeside.server and a celery based worker --- diff --git a/Dockerfile b/Dockerfile index eaaa3fca..65be1b9d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,9 +26,4 @@ WORKDIR /opt/Telemeta # Install deps RUN pip install -r requirements.txt -# Sandbox setup -RUN /opt/Telemeta/examples/sandbox/manage.py syncdb --noinput -RUN /opt/Telemeta/examples/sandbox/manage.py migrate --noinput -RUN /opt/Telemeta/examples/sandbox/manage.py collectstatic --noinput - EXPOSE 8000 diff --git a/README.rst b/README.rst index a0fbb2fc..e6859cd3 100644 --- a/README.rst +++ b/README.rst @@ -166,11 +166,9 @@ The official project site is `telemeta.org `_ but you can f To get and run the lastest development version:: sudo apt-get install git - git clone https://github.com/Parisson/Telemeta.git + git clone --recursive https://github.com/Parisson/Telemeta.git cd Telemeta git checkout dev - git submodule foreach git fetch --tags - git submodule update --init --recursive sudo pip install -e . export PYTHONPATH=$PYTHONPATH:`pwd` diff --git a/docker-compose.yml b/docker-compose.yml index b196e490..205cf7f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,26 @@ -home: - image: debian:wheezy - volumes: - - ./examples/sandbox:/home/telemeta - command: /bin/true +# -*- coding: utf-8 -*- +# +# Copyright (c) 2015 Parisson SARL + +# This file is part of Telemeta. + +# TimeSide is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. + +# TimeSide is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with TimeSide. If not, see . + +# Authors: +# Guillaume Pellerin +# Thomas Fillon + static: image: debian:wheezy @@ -18,29 +36,46 @@ log: - /var/log/uwsgi command: /bin/true -# datadb: -# image: debian:wheezy -# volumes: -# - /var/lib/postgresql -# command: /bin/true +lib: + image: debian:wheezy + volumes: + - /var/lib/rabbitmq + - /var/lib/postgres + - /var/lib/mysql + command: /bin/true + +home: + image: debian:wheezy + volumes: + - ./examples/sandbox:/home/telemeta + command: /bin/true -# db: -# image: postgres -# volumes_from: -# - datadb +db: + image: mysql + volumes_from: + - lib + - log + environment: + - MYSQL_ROOT_PASSWORD=mysecretpassword + - MYSQL_DATABASE=telemeta + - MYSQL_USER=root + - MYSQL_PASSWORD=mysecretpassword -nginx: - image: nginx +rabbitmq: + image: rabbitmq:3-management ports: - - "8000:80" - volumes: - - ./examples/deploy/nginx-app.conf:/etc/nginx/conf.d/default.conf + - "15672:15672" + expose: + - "5672" + +worker: + build: . volumes_from: - - static - home - - log + command: /bin/sh /opt/TimeSide/examples/deploy/celery_app.sh links: - - app + - rabbitmq + - db app: build: . @@ -54,4 +89,22 @@ app: ports: - "9000:9000" expose: - - "80" #default runserver wsgi port \ No newline at end of file + - "8000" + links: + - rabbitmq + - db + - worker + +nginx: + image: nginx + ports: + - "8000:80" + volumes: + - ./examples/deploy/nginx-app.conf:/etc/nginx/conf.d/default.conf + volumes_from: + - static + - home + - log + links: + - app + diff --git a/examples/deploy/celery_app.sh b/examples/deploy/celery_app.sh new file mode 100644 index 00000000..a5752d42 --- /dev/null +++ b/examples/deploy/celery_app.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# paths +app_dir='/opt/Telemeta/' +sandbox_dir='/home/telemeta/' +manage=$sandbox_dir'manage.py' + +python $manage syncdb --noinput +python $manage migrate --noinput +python $manage collectstatic --noinput +python $manage timeside-create-admin-user + +# Starting celery worker with the --autoreload option will enable the worker to watch for file system changes +# This is an experimental feature intended for use in development only +# see http://celery.readthedocs.org/en/latest/userguide/workers.html#autoreloading +$manage celery worker --autoreload -A celery_app diff --git a/examples/sandbox/celery_app.py b/examples/sandbox/celery_app.py new file mode 100644 index 00000000..7e9d7f86 --- /dev/null +++ b/examples/sandbox/celery_app.py @@ -0,0 +1,24 @@ +from __future__ import absolute_import + +import os + +from celery import Celery + +from django.conf import settings + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') + +app = Celery('sandbox') + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) +# app.conf.update( +# CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend', +# ) + +@app.task(bind=True) +def debug_task(self): + print('Request: {0!r}'.format(self.request)) diff --git a/examples/sandbox/settings.py b/examples/sandbox/settings.py index 16cf0dff..d1681fb7 100644 --- a/examples/sandbox/settings.py +++ b/examples/sandbox/settings.py @@ -22,12 +22,20 @@ PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. - 'NAME': os.path.join(PROJECT_ROOT, 'sandbox.sql'), # Or path to database file if using sqlite3. - 'USER': '', # Not used with sqlite3. - 'PASSWORD': '', # Not used with sqlite3. - 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. - 'PORT': '', # Set to empty string for default. Not used with sqlite3. + # SQLite config + # 'ENGINE': 'django.db.backends.sqlite', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. + # 'NAME': os.path.join(PROJECT_ROOT, 'telemeta.sql'), # Or path to database file if using sqlite3. + # 'OPTIONS': { + # 'timeout': 60, + # } + + # MySQL config + 'ENGINE': 'django.db.backends.mysql', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. + 'USER': 'root', # Not used with sqlite3. + 'PASSWORD': 'mysecretpassword', # Not used with sqlite3. + 'NAME': 'timeside', + 'HOST': 'db', # Set to empty string for localhost. Not used with sqlite3. + 'PORT': '3306', # Set to empty string for default. Not used with sqlite3. } } @@ -138,6 +146,7 @@ INSTALLED_APPS = ( 'django_extensions', 'telemeta', 'timeside.player', + 'timeside.server', 'jsonrpc', 'south', 'sorl.thumbnail', @@ -150,6 +159,8 @@ INSTALLED_APPS = ( 'bootstrap_pagination', 'googletools', 'registration', + 'rest_framework', + 'djcelery', ) TEMPLATE_CONTEXT_PROCESSORS = ( @@ -235,26 +246,41 @@ SUIT_CONFIG = { } -# LOG_DIR = os.path.join(BASE_DIR, 'log') - -# if not os.path.exists(LOG_DIR): -# os.mkdir(LOG_DIR) - -# LOGGING = { -# 'version': 1, -# 'disable_existing_loggers': False, -# 'handlers': { -# 'file': { -# 'level': 'DEBUG', -# 'class': 'logging.FileHandler', -# 'filename': os.path.join(LOG_DIR, 'debug.log') -# }, -# }, -# 'loggers': { -# 'django.request': { -# 'handlers': ['file'], -# 'level': 'DEBUG', -# 'propagate': True, -# }, -# }, -# } +# A sample logging configuration. The only tangible logging +# performed by this configuration is to send an email to +# the site admins on every HTTP 500 error when DEBUG=False. +# See http://docs.djangoproject.com/en/dev/topics/logging for +# more details on how to customize your logging configuration. +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler' + } + }, + 'loggers': { + 'django.request': { + 'handlers': ['mail_admins'], + 'level': 'ERROR', + 'propagate': True, + }, + } +} + +# replace rabbitmq by localhost if you start your app outside docker-compose +BROKER_URL = 'amqp://guest:guest@rabbitmq//' + +CELERY_IMPORTS = ("timeside.server.tasks",) +CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend' +CELERY_TASK_SERIALIZER = "json" +CELERY_ACCEPT_CONTENT = ['application/json'] + +from celery_app import app diff --git a/telemeta/settings_base.py b/telemeta/settings_base.py deleted file mode 100644 index 1ecd79da..00000000 --- a/telemeta/settings_base.py +++ /dev/null @@ -1,189 +0,0 @@ -# -*- coding: utf-8 -*- -# Django settings for sandbox project. - -import os, sys -from django.core.urlresolvers import reverse_lazy, reverse - -DEBUG = True -TEMPLATE_DEBUG = DEBUG -sys.dont_write_bytecode = True - -ADMINS = ( - ('Guillaume Pellerin', 'yomguy@parisson.com'), -) - -MANAGERS = ADMINS - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. - 'NAME': 'sandbox.sql', # Or path to database file if using sqlite3. - 'USER': '', # Not used with sqlite3. - 'PASSWORD': '', # Not used with sqlite3. - 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. - 'PORT': '', # Set to empty string for default. Not used with sqlite3. - } -} - -# Local time zone for this installation. Choices can be found here: -# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name -# although not all choices may be available on all operating systems. -# On Unix systems, a value of None will cause Django to use the same -# timezone as the operating system. -# If running in a Windows environment this must be set to the same as your -# system time zone. -TIME_ZONE = 'Europe/Paris' - -# Language code for this installation. All choices can be found here: -# http://www.i18nguy.com/unicode/language-identifiers.html -#LANGUAGE_CODE = 'fr_FR' -LANGUAGES = [ ('fr', 'French'), - ('en', 'English'), -] - -SITE_ID = 1 - -# If you set this to False, Django will make some optimizations so as not -# to load the internationalization machinery. -USE_I18N = True - -# If you set this to False, Django will not format dates, numbers and -# calendars according to the current locale -USE_L10N = True - -# Absolute path to the directory that holds media. -# Example: "/home/media/media.lawrence.com/" -MEDIA_ROOT = os.path.join(os.path.dirname(__file__), 'media/') - -if not os.path.exists(MEDIA_ROOT): - os.mkdir(MEDIA_ROOT) - -# URL that handles the media served from MEDIA_ROOT. Make sure to use a -# trailing slash if there is a path component (optional in other cases). -# Examples: "http://media.lawrence.com", "http://example.com/media/" -MEDIA_URL = '/media/' - -# Absolute path to the directory static files should be collected to. -# Don't put anything in this directory yourself; store your static files -# in apps' "static/" subdirectories and in STATICFILES_DIRS. -# Example: "/home/media/media.lawrence.com/static/" -STATIC_ROOT = '' - -# URL prefix for static files. -# Example: "http://media.lawrence.com/static/" -STATIC_URL = '/static/' - -# Additional locations of static files -STATICFILES_DIRS = ( -# Put strings here, like "/home/html/static" or "C:/www/django/static". -# Always use forward slashes, even on Windows. -# Don't forget to use absolute paths, not relative paths. -) - -# List of finder classes that know how to find static files in -# various locations. -STATICFILES_FINDERS = ( -'django.contrib.staticfiles.finders.FileSystemFinder', -'django.contrib.staticfiles.finders.AppDirectoriesFinder', -# 'django.contrib.staticfiles.finders.DefaultStorageFinder', -) -# Make this unique, and don't share it with anybody. -SECRET_KEY = 'a8l7%06wr2k+3=%#*#@#rvop2mmzko)44%7k(zx%lls^ihm9^5' - -# List of callables that know how to import templates from various sources. -TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', -# 'django.template.loaders.eggs.Loader', -) - - -MIDDLEWARE_CLASSES = ( - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.locale.LocaleMiddleware', - # 'pagination.middleware.PaginationMiddleware', -) - -ROOT_URLCONF = 'urls' - -TEMPLATE_DIRS = ( - # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. -) - -INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.sites', - 'django.contrib.messages', - 'django.contrib.admin', - 'django.contrib.staticfiles', - 'django_extensions', - 'telemeta', - 'timeside.player', - 'jsonrpc', - 'south', - 'sorl.thumbnail', - 'timezones', - 'jqchat', - 'extra_views', - # 'breadcrumbs', - 'debug_toolbar', - 'bootstrap3', - 'bootstrap_pagination', -) - -TEMPLATE_CONTEXT_PROCESSORS = ( - 'django.core.context_processors.request', - 'django.contrib.auth.context_processors.auth', - "django.core.context_processors.i18n", - "django.core.context_processors.media", - 'django.core.context_processors.static', - 'django.contrib.messages.context_processors.messages', -) - -TELEMETA_ORGANIZATION = 'Parisson' -TELEMETA_SUBJECTS = ('test', 'telemeta', 'sandbox') -TELEMETA_DESCRIPTION = "Telemeta TEST sandbox" -TELEMETA_GMAP_KEY = 'ABQIAAAArg7eSfnfTkBRma8glnGrlxRVbMrhnNNvToCbZQtWdaMbZTA_3RRGObu5PDoiBImgalVnnLU2yN4RMA' -TELEMETA_CACHE_DIR = MEDIA_ROOT + 'cache/' -TELEMETA_EXPORT_CACHE_DIR = MEDIA_ROOT + 'export/' -TELEMETA_DATA_CACHE_DIR = TELEMETA_CACHE_DIR + "data/" - -TELEMETA_DOWNLOAD_ENABLED = True -TELEMETA_STREAMING_FORMATS = ('mp3', 'ogg') -TELEMETA_DOWNLOAD_FORMATS = ('wav', 'mp3', 'ogg', 'flac') -TELEMETA_PUBLIC_ACCESS_PERIOD = 51 -TELEMETA_DEFAULT_WAVEFORM_SIZES = ['360x130', '640x130'] - -AUTH_PROFILE_MODULE = 'telemeta.userprofile' -SESSION_EXPIRE_AT_BROWSER_CLOSE = False - -LOGIN_URL = '/login/' -LOGIN_REDIRECT_URL = '/desk/lists/' - -EMAIL_HOST = 'localhost' -DEFAULT_FROM_EMAIL = 'webmaster@parisson.com' - -TIMESIDE_DEFAULT_GRAPHER_ID = 'waveform_centroid' -TIMESIDE_AUTO_ZOOM = True - -# Settings for django-bootstrap3 -BOOTSTRAP3 = { - 'set_required': True, - 'set_placeholder': False, - 'error_css_class': 'has-error', - 'required_css_class': 'has-warning', - 'javascript_in_head': True, -} - -PAGINATION_SETTINGS = { - 'PAGE_RANGE_DISPLAYED': 10, - 'MARGIN_PAGES_DISPLAYED': 2, -} diff --git a/tests/test_import.py b/tests/test_import.py deleted file mode 100644 index 1a681e86..00000000 --- a/tests/test_import.py +++ /dev/null @@ -1,2 +0,0 @@ - -import telemeta