]> git.parisson.com Git - teleforma.git/commitdiff
Add bin scripts
authorGuillaume Pellerin <guillaume.pellerin@ircam.fr>
Tue, 5 Feb 2019 00:50:40 +0000 (01:50 +0100)
committerGuillaume Pellerin <guillaume.pellerin@ircam.fr>
Tue, 5 Feb 2019 00:50:40 +0000 (01:50 +0100)
14 files changed:
.gitignore
app/bin/app.sh [new file with mode: 0755]
app/bin/diag.sh [new file with mode: 0755]
app/bin/enumeration.sh [new file with mode: 0755]
app/bin/init.sh [new file with mode: 0755]
app/bin/install_plugins.sh [new file with mode: 0755]
app/bin/modelviz.py [new file with mode: 0755]
app/bin/notebook.sh [new file with mode: 0755]
app/bin/setup_plugins.sh [new file with mode: 0755]
app/bin/update_schema.sh [new file with mode: 0755]
app/bin/upgrade_from_1.6_to_1.7.sh [new file with mode: 0755]
app/bin/wait.sh [new file with mode: 0755]
app/bin/worker.sh [new file with mode: 0755]
docker-compose.yml

index 24a0c1a5bcc2891d73504bb4cf0a7a1387924928..13a58fe2ac4ba6c409d5adc30487d959fce5d594 100644 (file)
@@ -8,7 +8,6 @@ dist
 build
 eggs
 parts
-bin
 var
 sdist
 develop-eggs
diff --git a/app/bin/app.sh b/app/bin/app.sh
new file mode 100755 (executable)
index 0000000..84e0e24
--- /dev/null
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# paths
+app='/srv/app'
+manage=$app'/manage.py'
+wsgi=$app'/wsgi.py'
+static='/srv/static/'
+media='/srv/media/'
+lib='/srv/lib/'
+log='/var/log/uwsgi/app.log'
+
+# uwsgi params
+port=8000
+processes=8
+threads=8
+uid='www-data'
+gid='www-data'
+
+# stating apps
+# pip uninstall -y south
+# pip install -U django==1.8.18 django-registration-redux djangorestframework==3.6.4
+# pip install django-debug-toolbar==1.6
+# pip install -e git+https://github.com/Parisson/django-jqchat.git@dj1.8#egg=django-jqchat
+# pip install -e git+https://github.com/Parisson/saved_searches.git@dj1.8#egg=saved_searches-2.0.0-beta
+
+# waiting for other network services
+sh $app/bin/wait.sh
+python $manage wait-for-db
+
+if [ ! -f .init ]; then
+    python $manage migrate --noinput
+    python $manage bower_install -- --allow-root
+    touch .init
+fi
+
+# telemeta setup
+python $manage telemeta-create-admin-user
+python $manage telemeta-create-boilerplate
+python $manage telemeta-setup-enumerations
+
+
+# Delete Timeside database if it exists
+cat /srv/lib/telemeta/bin/sql/drop_timeside.sql | python $manage dbshell
+
+if [ $REINDEX = "True" ]; then
+    python $manage rebuild_index --noinput
+fi
+
+# choose dev or prod mode
+if [ "$1" = "--runserver" ]; then
+    python $manage runserver 0.0.0.0:8000
+else
+    # static files auto update
+    # watchmedo shell-command --patterns="$patterns" --recursive \
+    #     --command='python '$manage' collectstatic --noinput' $lib &
+    python $manage collectstatic --noinput
+
+    # fix media access rights
+    find $media -maxdepth 2 -path ${media}import -prune -o -type d -not -user www-data -exec chown www-data:www-data {} \;
+
+    # app start
+    uwsgi --socket :$port --wsgi-file $wsgi --chdir $app --master \
+        --processes $processes --threads $threads \
+        --uid $uid --gid $gid --logto $log --touch-reload $wsgi
+fi
diff --git a/app/bin/diag.sh b/app/bin/diag.sh
new file mode 100755 (executable)
index 0000000..44d0552
--- /dev/null
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+app="telemeta"
+dir="../../doc/devel"
+
+python modelviz.py -a > $dir/$app-all.dot
+python modelviz.py $app > $dir/$app.dot
+
+dot $dir/$app-all.dot -Tpdf -o $dir/$app-all.pdf
+dot $dir/$app.dot -Tpdf -o $dir/$app.pdf
+
+rsync -a $dir/ doc.parisson.com:/var/www/files/doc/$app/diagram/
diff --git a/app/bin/enumeration.sh b/app/bin/enumeration.sh
new file mode 100755 (executable)
index 0000000..c6a42b0
--- /dev/null
@@ -0,0 +1,4 @@
+#!/bin/bash
+ pwd
+
+   chmod 777 "enumeration/enumeration.txt" && echo "The file is now writable"
diff --git a/app/bin/init.sh b/app/bin/init.sh
new file mode 100755 (executable)
index 0000000..21bd3d5
--- /dev/null
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+app='/srv/app'
+manage=$app'/manage.py'
+
+python $manage migrate --noinput
+python $manage telemeta-create-admin-user
+python $manage telemeta-create-boilerplate
+python $manage bower_install -- --allow-root
diff --git a/app/bin/install_plugins.sh b/app/bin/install_plugins.sh
new file mode 100755 (executable)
index 0000000..b0315e9
--- /dev/null
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+plugins=/srv/lib/plugins
+
+for dir in $(ls $plugins); do
+    if [ -f $plugins/$dir/setup.py ]; then
+        pip install -e $plugins/$dir/.
+    fi
+done
diff --git a/app/bin/modelviz.py b/app/bin/modelviz.py
new file mode 100755 (executable)
index 0000000..24af062
--- /dev/null
@@ -0,0 +1,359 @@
+#!/usr/bin/env python
+
+"""Django model to DOT (Graphviz) converter
+by Antonio Cavedoni <antonio@cavedoni.org>
+
+Make sure your DJANGO_SETTINGS_MODULE is set to your project or
+place this script in the same directory of the project and call
+the script like this:
+
+$ python modelviz.py [-h] [-a] [-d] [-g] [-n] [-L <language>] [-i <model_names>] <app_label> ... <app_label> > <filename>.dot
+$ dot <filename>.dot -Tpng -o <filename>.png
+
+options:
+    -h, --help
+    show this help message and exit.
+
+    -a, --all_applications
+    show models from all applications.
+
+    -d, --disable_fields
+    don't show the class member fields.
+
+    -g, --group_models
+    draw an enclosing box around models from the same app.
+
+    -i, --include_models=User,Person,Car
+    only include selected models in graph.
+
+    -n, --verbose_names
+    use verbose_name for field and models.
+
+    -L, --language
+    specify language used for verrbose_name localization
+
+    -x, --exclude_columns
+    exclude specific column(s) from the graph.
+
+    -X, --exclude_models
+    exclude specific model(s) from the graph.
+    
+    -e, --inheritance
+    show inheritance arrows.
+"""
+__version__ = "0.9"
+__svnid__ = "$Id$"
+__license__ = "Python"
+__author__ = "Antonio Cavedoni <http://cavedoni.com/>"
+__contributors__ = [
+   "Stefano J. Attardi <http://attardi.org/>",
+   "limodou <http://www.donews.net/limodou/>",
+   "Carlo C8E Miron",
+   "Andre Campos <cahenan@gmail.com>",
+   "Justin Findlay <jfindlay@gmail.com>",
+   "Alexander Houben <alexander@houben.ch>",
+   "Bas van Oostveen <v.oostveen@gmail.com>",
+   "Joern Hees <gitdev@joernhees.de>"
+]
+
+import os
+import sys
+import getopt
+
+from django.core.management import setup_environ
+
+try:
+    import settings
+except ImportError:
+    pass
+else:
+    setup_environ(settings)
+
+from django.utils.translation import activate as activate_language
+from django.utils.safestring import mark_safe
+from django.template import Template, Context, loader
+from django.db import models
+from django.db.models import get_models
+from django.db.models.fields.related import \
+    ForeignKey, OneToOneField, ManyToManyField, RelatedField
+
+try:
+    from django.db.models.fields.generic import GenericRelation
+except ImportError:
+    from django.contrib.contenttypes.generic import GenericRelation
+
+def parse_file_or_list(arg):
+    if not arg:
+        return []
+    if not ',' in arg and os.path.isfile(arg):
+        return [e.strip() for e in open(arg).readlines()]
+    return arg.split(',')
+
+
+def generate_dot(app_labels, **kwargs):
+    disable_fields = kwargs.get('disable_fields', False)
+    include_models = parse_file_or_list(kwargs.get('include_models', ""))
+    all_applications = kwargs.get('all_applications', False)
+    use_subgraph = kwargs.get('group_models', False)
+    verbose_names = kwargs.get('verbose_names', False)
+    inheritance = kwargs.get('inheritance', False)
+    language = kwargs.get('language', None)
+    if language is not None:
+        activate_language(language)
+    exclude_columns = parse_file_or_list(kwargs.get('exclude_columns', ""))
+    exclude_models = parse_file_or_list(kwargs.get('exclude_models', ""))
+
+    def skip_field(field):
+        if exclude_columns:
+            if verbose_names and field.verbose_name:
+                if field.verbose_name in exclude_columns:
+                    return True
+            if field.name in exclude_columns:
+                return True
+        return False
+
+
+
+
+    t = loader.get_template('django_extensions/graph_models/head.html')
+    c = Context({})
+    dot = t.render(c)
+
+    apps = []
+    if all_applications:
+        apps = models.get_apps()
+
+    for app_label in app_labels:
+        app = models.get_app(app_label)
+        if not app in apps:
+            apps.append(app)
+
+    graphs = []
+    for app in apps:
+        graph = Context({
+            'name': '"%s"' % app.__name__,
+            'app_name': "%s" % '.'.join(app.__name__.split('.')[:-1]),
+            'cluster_app_name': "cluster_%s" % app.__name__.replace(".", "_"),
+            'disable_fields': disable_fields,
+            'use_subgraph': use_subgraph,
+            'models': []
+        })
+
+        appmodels = get_models(app)
+        abstract_models = []
+        for appmodel in appmodels:
+            abstract_models = abstract_models + [abstract_model for abstract_model in appmodel.__bases__ if hasattr(abstract_model, '_meta') and abstract_model._meta.abstract]
+        abstract_models = list(set(abstract_models)) # remove duplicates
+        appmodels = abstract_models + appmodels
+        
+
+        for appmodel in appmodels:
+            appmodel_abstracts = [abstract_model.__name__ for abstract_model in appmodel.__bases__ if hasattr(abstract_model, '_meta') and abstract_model._meta.abstract]
+
+            # collect all attribs of abstract superclasses
+            def getBasesAbstractFields(c):
+                _abstract_fields = []
+                for e in c.__bases__:
+                    if hasattr(e, '_meta') and e._meta.abstract:
+                        _abstract_fields.extend(e._meta.fields)
+                        _abstract_fields.extend(getBasesAbstractFields(e))
+                return _abstract_fields
+            abstract_fields = getBasesAbstractFields(appmodel)
+
+            model = {
+                'app_name': appmodel.__module__.replace(".", "_"),
+                'name': appmodel.__name__,
+                'abstracts': appmodel_abstracts,
+                'fields': [],
+                'relations': []
+            }
+
+            # consider given model name ?
+            def consider(model_name):
+                if exclude_models and model_name in exclude_models:
+                    return False
+                return not include_models or model_name in include_models
+
+            if not consider(appmodel._meta.object_name):
+                continue
+
+            if verbose_names and appmodel._meta.verbose_name:
+                model['label'] = appmodel._meta.verbose_name
+            else:
+                model['label'] = model['name']
+
+            # model attributes
+            def add_attributes(field):
+                if verbose_names and field.verbose_name:
+                    label = field.verbose_name
+                else:
+                    label = field.name
+
+                t = type(field).__name__
+                if isinstance(field, (OneToOneField, ForeignKey)):
+                    t += " ({0})".format(field.rel.field_name)
+                # TODO: ManyToManyField, GenericRelation
+
+                model['fields'].append({
+                    'name': field.name,
+                    'label': label,
+                    'type': t,
+                    'blank': field.blank,
+                    'abstract': field in abstract_fields,
+                })
+
+            # Find all the real attributes. Relations are depicted as graph edges instead of attributes
+            attributes = [field for field in appmodel._meta.local_fields if not isinstance(field, RelatedField)]
+
+            # find primary key and print it first, ignoring implicit id if other pk exists
+            pk = appmodel._meta.pk
+            if not appmodel._meta.abstract and pk in attributes:
+                add_attributes(pk)
+            for field in attributes:
+                if skip_field(field):
+                    continue
+                if not field.primary_key:
+                    add_attributes(field)
+            
+            # FIXME: actually many_to_many fields aren't saved in this model's db table, so why should we add an attribute-line for them in the resulting graph?
+            #if appmodel._meta.many_to_many:
+            #    for field in appmodel._meta.many_to_many:
+            #        if skip_field(field):
+            #            continue
+            #        add_attributes(field)
+
+            # relations
+            def add_relation(field, extras=""):
+                if verbose_names and field.verbose_name:
+                    label = field.verbose_name
+                else:
+                    label = field.name
+                    
+                # show related field name
+                if hasattr(field, 'related_query_name'):
+                    label += ' (%s)' % field.related_query_name()
+
+                _rel = {
+                    'target_app': field.rel.to.__module__.replace('.', '_'),
+                    'target': field.rel.to.__name__,
+                    'type': type(field).__name__,
+                    'name': field.name,
+                    'label': label,
+                    'arrows': extras,
+                    'needs_node': True
+                }
+                if _rel not in model['relations'] and consider(_rel['target']):
+                    model['relations'].append(_rel)
+
+            for field in appmodel._meta.local_fields:
+                if field.attname.endswith('_ptr_id'): # excluding field redundant with inheritance relation
+                    continue
+                if field in abstract_fields: # excluding fields inherited from abstract classes. they too show as local_fields
+                    continue
+                if skip_field(field):
+                    continue
+                if isinstance(field, OneToOneField):
+                    add_relation(field, '[arrowhead=none, arrowtail=none]')
+                elif isinstance(field, ForeignKey):
+                    add_relation(field, '[arrowhead=none, arrowtail=dot]')
+
+            for field in appmodel._meta.local_many_to_many:
+                if skip_field(field):
+                    continue
+                if isinstance(field, ManyToManyField):
+                    if (getattr(field, 'creates_table', False) or  # django 1.1.
+                        (hasattr(field.rel.through, '_meta') and field.rel.through._meta.auto_created)):  # django 1.2
+                        add_relation(field, '[arrowhead=dot arrowtail=dot, dir=both]')
+                    elif isinstance(field, GenericRelation):
+                        add_relation(field, mark_safe('[style="dotted", arrowhead=normal, arrowtail=normal, dir=both]'))
+            
+            if inheritance:
+                # add inheritance arrows
+                for parent in appmodel.__bases__:
+                    if hasattr(parent, "_meta"): # parent is a model
+                        l = "multi-table"
+                        if parent._meta.abstract:
+                            l = "abstract"
+                        if appmodel._meta.proxy:
+                            l = "proxy"
+                        l += r"\ninheritance"
+                        _rel = {
+                            'target_app': parent.__module__.replace(".", "_"),
+                            'target': parent.__name__,
+                            'type': "inheritance",
+                            'name': "inheritance",
+                            'label': l,
+                            'arrows': '[arrowhead=empty, arrowtail=none]',
+                            'needs_node': True
+                        }
+                        # TODO: seems as if abstract models aren't part of models.getModels, which is why they are printed by this without any attributes.
+                        if _rel not in model['relations'] and consider(_rel['target']):
+                            model['relations'].append(_rel)
+            
+            graph['models'].append(model)
+        graphs.append(graph)
+
+    nodes = []
+    for graph in graphs:
+        nodes.extend([e['name'] for e in graph['models']])
+
+    for graph in graphs:
+        # don't draw duplication nodes because of relations
+        for model in graph['models']:
+            for relation in model['relations']:
+                if relation['target'] in nodes:
+                    relation['needs_node'] = False
+        # render templates
+        t = loader.get_template('django_extensions/graph_models/body.html')
+        dot += '\n' + t.render(graph)
+
+    for graph in graphs:
+        t = loader.get_template('django_extensions/graph_models/rel.html')
+        dot += '\n' + t.render(graph)
+
+
+    t = loader.get_template('django_extensions/graph_models/tail.html')
+    c = Context({})
+    dot += '\n' + t.render(c)
+    return dot
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hadgi:L:x:X:en",
+                    ["help", "all_applications", "disable_fields", "group_models", "include_models=", "inheritance", "verbose_names", "language=", "exclude_columns=", "exclude_models="])
+    except getopt.GetoptError, error:
+        print __doc__
+        sys.exit(error)
+
+    kwargs = {}
+    for opt, arg in opts:
+        if opt in ("-h", "--help"):
+            print __doc__
+            sys.exit()
+        if opt in ("-a", "--all_applications"):
+            kwargs['all_applications'] = True
+        if opt in ("-d", "--disable_fields"):
+            kwargs['disable_fields'] = True
+        if opt in ("-g", "--group_models"):
+            kwargs['group_models'] = True
+        if opt in ("-i", "--include_models"):
+            kwargs['include_models'] = arg
+        if opt in ("-e", "--inheritance"):
+            kwargs['inheritance'] = True
+        if opt in ("-n", "--verbose-names"):
+            kwargs['verbose_names'] = True
+        if opt in ("-L", "--language"):
+            kwargs['language'] = arg
+        if opt in ("-x", "--exclude_columns"):
+            kwargs['exclude_columns'] = arg
+        if opt in ("-X", "--exclude_models"):
+            kwargs['exclude_models'] = arg
+
+    if not args and not kwargs.get('all_applications', False):
+        print __doc__
+        sys.exit()
+
+    print generate_dot(args, **kwargs)
+
+if __name__ == "__main__":
+    main()
diff --git a/app/bin/notebook.sh b/app/bin/notebook.sh
new file mode 100755 (executable)
index 0000000..562f9df
--- /dev/null
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export PYTHONPATH=$PYTHONPATH:/opt/miniconda/lib/python2.7/site-packages/:/srv/app/
+export DJANGO_SETTINGS_MODULE=settings
+
+python /srv/app/manage.py shell_plus --notebook
diff --git a/app/bin/setup_plugins.sh b/app/bin/setup_plugins.sh
new file mode 100755 (executable)
index 0000000..de55f84
--- /dev/null
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+plugins=/srv/lib/plugins
+
+apt-get update
+
+for dir in $(ls $plugins); do
+    env=$plugins/$dir/conda-environment.yml
+    if [ -f $env ]; then
+        conda env update --name root --file $env
+    fi
+    req=$plugins/$dir/debian-requirements.txt
+    if [ -f $req ]; then
+        packs=$(egrep -v "^\s*(#|$)" $req)
+        apt-get install -y --force-yes $packs
+    fi
+    if [ -f $plugins/$dir/setup.py ]; then
+        pip install -e $plugins/$dir/.
+    fi
+done
+
+apt-get clean
diff --git a/app/bin/update_schema.sh b/app/bin/update_schema.sh
new file mode 100755 (executable)
index 0000000..1b64fa3
--- /dev/null
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+./manage.py schemamigration telemeta --auto
+./manage.py migrate telemeta
diff --git a/app/bin/upgrade_from_1.6_to_1.7.sh b/app/bin/upgrade_from_1.6_to_1.7.sh
new file mode 100755 (executable)
index 0000000..af74807
--- /dev/null
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+python manage.py migrate
+python manage.py migrate contenttypes --fake-initial
+python manage.py migrate --fake-initial
+python manage.py migrate thumbnail --fake-initial
+python manage.py migrate --fake telemeta 0006
diff --git a/app/bin/wait.sh b/app/bin/wait.sh
new file mode 100755 (executable)
index 0000000..f2a93f7
--- /dev/null
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# apt-get install -y --force-yes netcat
+
+set -e
+
+host=$(env | grep _TCP_ADDR | cut -d = -f 2)
+port=$(env | grep _TCP_PORT | cut -d = -f 2)
+
+echo -n "waiting for TCP connection to $host:$port..."
+
+while ! nc -w 1 $host $port 2>/dev/null
+do
+  echo -n .
+  sleep 1
+done
+
+echo 'ok'
diff --git a/app/bin/worker.sh b/app/bin/worker.sh
new file mode 100755 (executable)
index 0000000..3eb7b40
--- /dev/null
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# paths
+app='/srv/app'
+manage=$app'/manage.py'
+wsgi=$app'/wsgi.py'
+concurrency=12
+
+# stating apps
+# pip uninstall -y south
+# pip install -U django==1.8.18 django-registration-redux djangorestframework==3.6.4
+# pip install django-debug-toolbar==1.6
+# pip install -e git+https://github.com/Parisson/django-jqchat.git@dj1.8#egg=django-jqchat
+# pip install -e git+https://github.com/Parisson/saved_searches.git@dj1.8#egg=saved_searches-2.0.0-beta
+
+# waiting for other services
+bash $app/bin/wait.sh
+
+# Starting celery worker with the --autoreload option will enable the worker to watch for file system changes
+# This is an experimental feature intended for use in development only
+# see http://celery.readthedocs.org/en/latest/userguide/workers.html#autoreloading
+python $manage celery worker --autoreload -A worker --concurrency=$concurrency
index 84d2a999e5c3a1480932a38c1e1369da6531f778..eede8a7dfa39210ac5e11ec0f7c84c84e822376b 100644 (file)
@@ -42,6 +42,18 @@ services:
     env_file:
       - env/debug.env
 
+  worker:
+    build: .
+    volumes_from:
+      - app
+      - var
+    env_file:
+      - env/debug.env
+    command: /bin/bash bin/worker.sh
+    links:
+      - broker
+      - db
+
   web:
     image: nginx
     ports: