From cdfdcd13b4de94acd4ec0b57ff722411fddd3219 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Tue, 5 Feb 2019 01:50:40 +0100 Subject: [PATCH] Add bin scripts --- .gitignore | 1 - app/bin/app.sh | 65 ++++++ app/bin/diag.sh | 12 + app/bin/enumeration.sh | 4 + app/bin/init.sh | 9 + app/bin/install_plugins.sh | 9 + app/bin/modelviz.py | 359 +++++++++++++++++++++++++++++ app/bin/notebook.sh | 6 + app/bin/setup_plugins.sh | 22 ++ app/bin/update_schema.sh | 4 + app/bin/upgrade_from_1.6_to_1.7.sh | 7 + app/bin/wait.sh | 18 ++ app/bin/worker.sh | 22 ++ docker-compose.yml | 12 + 14 files changed, 549 insertions(+), 1 deletion(-) create mode 100755 app/bin/app.sh create mode 100755 app/bin/diag.sh create mode 100755 app/bin/enumeration.sh create mode 100755 app/bin/init.sh create mode 100755 app/bin/install_plugins.sh create mode 100755 app/bin/modelviz.py create mode 100755 app/bin/notebook.sh create mode 100755 app/bin/setup_plugins.sh create mode 100755 app/bin/update_schema.sh create mode 100755 app/bin/upgrade_from_1.6_to_1.7.sh create mode 100755 app/bin/wait.sh create mode 100755 app/bin/worker.sh diff --git a/.gitignore b/.gitignore index 24a0c1a5..13a58fe2 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ dist build eggs parts -bin var sdist develop-eggs diff --git a/app/bin/app.sh b/app/bin/app.sh new file mode 100755 index 00000000..84e0e246 --- /dev/null +++ b/app/bin/app.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# paths +app='/srv/app' +manage=$app'/manage.py' +wsgi=$app'/wsgi.py' +static='/srv/static/' +media='/srv/media/' +lib='/srv/lib/' +log='/var/log/uwsgi/app.log' + +# uwsgi params +port=8000 +processes=8 +threads=8 +uid='www-data' +gid='www-data' + +# stating apps +# pip uninstall -y south +# pip install -U django==1.8.18 django-registration-redux djangorestframework==3.6.4 +# pip install django-debug-toolbar==1.6 +# pip install -e git+https://github.com/Parisson/django-jqchat.git@dj1.8#egg=django-jqchat +# pip install -e git+https://github.com/Parisson/saved_searches.git@dj1.8#egg=saved_searches-2.0.0-beta + +# waiting for other network services +sh $app/bin/wait.sh +python $manage wait-for-db + +if [ ! -f .init ]; then + python $manage migrate --noinput + python $manage bower_install -- --allow-root + touch .init +fi + +# telemeta setup +python $manage telemeta-create-admin-user +python $manage telemeta-create-boilerplate +python $manage telemeta-setup-enumerations + + +# Delete Timeside database if it exists +cat /srv/lib/telemeta/bin/sql/drop_timeside.sql | python $manage dbshell + +if [ $REINDEX = "True" ]; then + python $manage rebuild_index --noinput +fi + +# choose dev or prod mode +if [ "$1" = "--runserver" ]; then + python $manage runserver 0.0.0.0:8000 +else + # static files auto update + # watchmedo shell-command --patterns="$patterns" --recursive \ + # --command='python '$manage' collectstatic --noinput' $lib & + python $manage collectstatic --noinput + + # fix media access rights + find $media -maxdepth 2 -path ${media}import -prune -o -type d -not -user www-data -exec chown www-data:www-data {} \; + + # app start + uwsgi --socket :$port --wsgi-file $wsgi --chdir $app --master \ + --processes $processes --threads $threads \ + --uid $uid --gid $gid --logto $log --touch-reload $wsgi +fi diff --git a/app/bin/diag.sh b/app/bin/diag.sh new file mode 100755 index 00000000..44d05529 --- /dev/null +++ b/app/bin/diag.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +app="telemeta" +dir="../../doc/devel" + +python modelviz.py -a > $dir/$app-all.dot +python modelviz.py $app > $dir/$app.dot + +dot $dir/$app-all.dot -Tpdf -o $dir/$app-all.pdf +dot $dir/$app.dot -Tpdf -o $dir/$app.pdf + +rsync -a $dir/ doc.parisson.com:/var/www/files/doc/$app/diagram/ diff --git a/app/bin/enumeration.sh b/app/bin/enumeration.sh new file mode 100755 index 00000000..c6a42b04 --- /dev/null +++ b/app/bin/enumeration.sh @@ -0,0 +1,4 @@ +#!/bin/bash + pwd + + chmod 777 "enumeration/enumeration.txt" && echo "The file is now writable" diff --git a/app/bin/init.sh b/app/bin/init.sh new file mode 100755 index 00000000..21bd3d5f --- /dev/null +++ b/app/bin/init.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +app='/srv/app' +manage=$app'/manage.py' + +python $manage migrate --noinput +python $manage telemeta-create-admin-user +python $manage telemeta-create-boilerplate +python $manage bower_install -- --allow-root diff --git a/app/bin/install_plugins.sh b/app/bin/install_plugins.sh new file mode 100755 index 00000000..b0315e9f --- /dev/null +++ b/app/bin/install_plugins.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +plugins=/srv/lib/plugins + +for dir in $(ls $plugins); do + if [ -f $plugins/$dir/setup.py ]; then + pip install -e $plugins/$dir/. + fi +done diff --git a/app/bin/modelviz.py b/app/bin/modelviz.py new file mode 100755 index 00000000..24af0627 --- /dev/null +++ b/app/bin/modelviz.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python + +"""Django model to DOT (Graphviz) converter +by Antonio Cavedoni + +Make sure your DJANGO_SETTINGS_MODULE is set to your project or +place this script in the same directory of the project and call +the script like this: + +$ python modelviz.py [-h] [-a] [-d] [-g] [-n] [-L ] [-i ] ... > .dot +$ dot .dot -Tpng -o .png + +options: + -h, --help + show this help message and exit. + + -a, --all_applications + show models from all applications. + + -d, --disable_fields + don't show the class member fields. + + -g, --group_models + draw an enclosing box around models from the same app. + + -i, --include_models=User,Person,Car + only include selected models in graph. + + -n, --verbose_names + use verbose_name for field and models. + + -L, --language + specify language used for verrbose_name localization + + -x, --exclude_columns + exclude specific column(s) from the graph. + + -X, --exclude_models + exclude specific model(s) from the graph. + + -e, --inheritance + show inheritance arrows. +""" +__version__ = "0.9" +__svnid__ = "$Id$" +__license__ = "Python" +__author__ = "Antonio Cavedoni " +__contributors__ = [ + "Stefano J. Attardi ", + "limodou ", + "Carlo C8E Miron", + "Andre Campos ", + "Justin Findlay ", + "Alexander Houben ", + "Bas van Oostveen ", + "Joern Hees " +] + +import os +import sys +import getopt + +from django.core.management import setup_environ + +try: + import settings +except ImportError: + pass +else: + setup_environ(settings) + +from django.utils.translation import activate as activate_language +from django.utils.safestring import mark_safe +from django.template import Template, Context, loader +from django.db import models +from django.db.models import get_models +from django.db.models.fields.related import \ + ForeignKey, OneToOneField, ManyToManyField, RelatedField + +try: + from django.db.models.fields.generic import GenericRelation +except ImportError: + from django.contrib.contenttypes.generic import GenericRelation + +def parse_file_or_list(arg): + if not arg: + return [] + if not ',' in arg and os.path.isfile(arg): + return [e.strip() for e in open(arg).readlines()] + return arg.split(',') + + +def generate_dot(app_labels, **kwargs): + disable_fields = kwargs.get('disable_fields', False) + include_models = parse_file_or_list(kwargs.get('include_models', "")) + all_applications = kwargs.get('all_applications', False) + use_subgraph = kwargs.get('group_models', False) + verbose_names = kwargs.get('verbose_names', False) + inheritance = kwargs.get('inheritance', False) + language = kwargs.get('language', None) + if language is not None: + activate_language(language) + exclude_columns = parse_file_or_list(kwargs.get('exclude_columns', "")) + exclude_models = parse_file_or_list(kwargs.get('exclude_models', "")) + + def skip_field(field): + if exclude_columns: + if verbose_names and field.verbose_name: + if field.verbose_name in exclude_columns: + return True + if field.name in exclude_columns: + return True + return False + + + + + t = loader.get_template('django_extensions/graph_models/head.html') + c = Context({}) + dot = t.render(c) + + apps = [] + if all_applications: + apps = models.get_apps() + + for app_label in app_labels: + app = models.get_app(app_label) + if not app in apps: + apps.append(app) + + graphs = [] + for app in apps: + graph = Context({ + 'name': '"%s"' % app.__name__, + 'app_name': "%s" % '.'.join(app.__name__.split('.')[:-1]), + 'cluster_app_name': "cluster_%s" % app.__name__.replace(".", "_"), + 'disable_fields': disable_fields, + 'use_subgraph': use_subgraph, + 'models': [] + }) + + appmodels = get_models(app) + abstract_models = [] + for appmodel in appmodels: + abstract_models = abstract_models + [abstract_model for abstract_model in appmodel.__bases__ if hasattr(abstract_model, '_meta') and abstract_model._meta.abstract] + abstract_models = list(set(abstract_models)) # remove duplicates + appmodels = abstract_models + appmodels + + + for appmodel in appmodels: + appmodel_abstracts = [abstract_model.__name__ for abstract_model in appmodel.__bases__ if hasattr(abstract_model, '_meta') and abstract_model._meta.abstract] + + # collect all attribs of abstract superclasses + def getBasesAbstractFields(c): + _abstract_fields = [] + for e in c.__bases__: + if hasattr(e, '_meta') and e._meta.abstract: + _abstract_fields.extend(e._meta.fields) + _abstract_fields.extend(getBasesAbstractFields(e)) + return _abstract_fields + abstract_fields = getBasesAbstractFields(appmodel) + + model = { + 'app_name': appmodel.__module__.replace(".", "_"), + 'name': appmodel.__name__, + 'abstracts': appmodel_abstracts, + 'fields': [], + 'relations': [] + } + + # consider given model name ? + def consider(model_name): + if exclude_models and model_name in exclude_models: + return False + return not include_models or model_name in include_models + + if not consider(appmodel._meta.object_name): + continue + + if verbose_names and appmodel._meta.verbose_name: + model['label'] = appmodel._meta.verbose_name + else: + model['label'] = model['name'] + + # model attributes + def add_attributes(field): + if verbose_names and field.verbose_name: + label = field.verbose_name + else: + label = field.name + + t = type(field).__name__ + if isinstance(field, (OneToOneField, ForeignKey)): + t += " ({0})".format(field.rel.field_name) + # TODO: ManyToManyField, GenericRelation + + model['fields'].append({ + 'name': field.name, + 'label': label, + 'type': t, + 'blank': field.blank, + 'abstract': field in abstract_fields, + }) + + # Find all the real attributes. Relations are depicted as graph edges instead of attributes + attributes = [field for field in appmodel._meta.local_fields if not isinstance(field, RelatedField)] + + # find primary key and print it first, ignoring implicit id if other pk exists + pk = appmodel._meta.pk + if not appmodel._meta.abstract and pk in attributes: + add_attributes(pk) + for field in attributes: + if skip_field(field): + continue + if not field.primary_key: + add_attributes(field) + + # FIXME: actually many_to_many fields aren't saved in this model's db table, so why should we add an attribute-line for them in the resulting graph? + #if appmodel._meta.many_to_many: + # for field in appmodel._meta.many_to_many: + # if skip_field(field): + # continue + # add_attributes(field) + + # relations + def add_relation(field, extras=""): + if verbose_names and field.verbose_name: + label = field.verbose_name + else: + label = field.name + + # show related field name + if hasattr(field, 'related_query_name'): + label += ' (%s)' % field.related_query_name() + + _rel = { + 'target_app': field.rel.to.__module__.replace('.', '_'), + 'target': field.rel.to.__name__, + 'type': type(field).__name__, + 'name': field.name, + 'label': label, + 'arrows': extras, + 'needs_node': True + } + if _rel not in model['relations'] and consider(_rel['target']): + model['relations'].append(_rel) + + for field in appmodel._meta.local_fields: + if field.attname.endswith('_ptr_id'): # excluding field redundant with inheritance relation + continue + if field in abstract_fields: # excluding fields inherited from abstract classes. they too show as local_fields + continue + if skip_field(field): + continue + if isinstance(field, OneToOneField): + add_relation(field, '[arrowhead=none, arrowtail=none]') + elif isinstance(field, ForeignKey): + add_relation(field, '[arrowhead=none, arrowtail=dot]') + + for field in appmodel._meta.local_many_to_many: + if skip_field(field): + continue + if isinstance(field, ManyToManyField): + if (getattr(field, 'creates_table', False) or # django 1.1. + (hasattr(field.rel.through, '_meta') and field.rel.through._meta.auto_created)): # django 1.2 + add_relation(field, '[arrowhead=dot arrowtail=dot, dir=both]') + elif isinstance(field, GenericRelation): + add_relation(field, mark_safe('[style="dotted", arrowhead=normal, arrowtail=normal, dir=both]')) + + if inheritance: + # add inheritance arrows + for parent in appmodel.__bases__: + if hasattr(parent, "_meta"): # parent is a model + l = "multi-table" + if parent._meta.abstract: + l = "abstract" + if appmodel._meta.proxy: + l = "proxy" + l += r"\ninheritance" + _rel = { + 'target_app': parent.__module__.replace(".", "_"), + 'target': parent.__name__, + 'type': "inheritance", + 'name': "inheritance", + 'label': l, + 'arrows': '[arrowhead=empty, arrowtail=none]', + 'needs_node': True + } + # TODO: seems as if abstract models aren't part of models.getModels, which is why they are printed by this without any attributes. + if _rel not in model['relations'] and consider(_rel['target']): + model['relations'].append(_rel) + + graph['models'].append(model) + graphs.append(graph) + + nodes = [] + for graph in graphs: + nodes.extend([e['name'] for e in graph['models']]) + + for graph in graphs: + # don't draw duplication nodes because of relations + for model in graph['models']: + for relation in model['relations']: + if relation['target'] in nodes: + relation['needs_node'] = False + # render templates + t = loader.get_template('django_extensions/graph_models/body.html') + dot += '\n' + t.render(graph) + + for graph in graphs: + t = loader.get_template('django_extensions/graph_models/rel.html') + dot += '\n' + t.render(graph) + + + t = loader.get_template('django_extensions/graph_models/tail.html') + c = Context({}) + dot += '\n' + t.render(c) + return dot + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], "hadgi:L:x:X:en", + ["help", "all_applications", "disable_fields", "group_models", "include_models=", "inheritance", "verbose_names", "language=", "exclude_columns=", "exclude_models="]) + except getopt.GetoptError, error: + print __doc__ + sys.exit(error) + + kwargs = {} + for opt, arg in opts: + if opt in ("-h", "--help"): + print __doc__ + sys.exit() + if opt in ("-a", "--all_applications"): + kwargs['all_applications'] = True + if opt in ("-d", "--disable_fields"): + kwargs['disable_fields'] = True + if opt in ("-g", "--group_models"): + kwargs['group_models'] = True + if opt in ("-i", "--include_models"): + kwargs['include_models'] = arg + if opt in ("-e", "--inheritance"): + kwargs['inheritance'] = True + if opt in ("-n", "--verbose-names"): + kwargs['verbose_names'] = True + if opt in ("-L", "--language"): + kwargs['language'] = arg + if opt in ("-x", "--exclude_columns"): + kwargs['exclude_columns'] = arg + if opt in ("-X", "--exclude_models"): + kwargs['exclude_models'] = arg + + if not args and not kwargs.get('all_applications', False): + print __doc__ + sys.exit() + + print generate_dot(args, **kwargs) + +if __name__ == "__main__": + main() diff --git a/app/bin/notebook.sh b/app/bin/notebook.sh new file mode 100755 index 00000000..562f9dfd --- /dev/null +++ b/app/bin/notebook.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +export PYTHONPATH=$PYTHONPATH:/opt/miniconda/lib/python2.7/site-packages/:/srv/app/ +export DJANGO_SETTINGS_MODULE=settings + +python /srv/app/manage.py shell_plus --notebook diff --git a/app/bin/setup_plugins.sh b/app/bin/setup_plugins.sh new file mode 100755 index 00000000..de55f841 --- /dev/null +++ b/app/bin/setup_plugins.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +plugins=/srv/lib/plugins + +apt-get update + +for dir in $(ls $plugins); do + env=$plugins/$dir/conda-environment.yml + if [ -f $env ]; then + conda env update --name root --file $env + fi + req=$plugins/$dir/debian-requirements.txt + if [ -f $req ]; then + packs=$(egrep -v "^\s*(#|$)" $req) + apt-get install -y --force-yes $packs + fi + if [ -f $plugins/$dir/setup.py ]; then + pip install -e $plugins/$dir/. + fi +done + +apt-get clean diff --git a/app/bin/update_schema.sh b/app/bin/update_schema.sh new file mode 100755 index 00000000..1b64fa36 --- /dev/null +++ b/app/bin/update_schema.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +./manage.py schemamigration telemeta --auto +./manage.py migrate telemeta diff --git a/app/bin/upgrade_from_1.6_to_1.7.sh b/app/bin/upgrade_from_1.6_to_1.7.sh new file mode 100755 index 00000000..af74807e --- /dev/null +++ b/app/bin/upgrade_from_1.6_to_1.7.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +python manage.py migrate +python manage.py migrate contenttypes --fake-initial +python manage.py migrate --fake-initial +python manage.py migrate thumbnail --fake-initial +python manage.py migrate --fake telemeta 0006 diff --git a/app/bin/wait.sh b/app/bin/wait.sh new file mode 100755 index 00000000..f2a93f79 --- /dev/null +++ b/app/bin/wait.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# apt-get install -y --force-yes netcat + +set -e + +host=$(env | grep _TCP_ADDR | cut -d = -f 2) +port=$(env | grep _TCP_PORT | cut -d = -f 2) + +echo -n "waiting for TCP connection to $host:$port..." + +while ! nc -w 1 $host $port 2>/dev/null +do + echo -n . + sleep 1 +done + +echo 'ok' diff --git a/app/bin/worker.sh b/app/bin/worker.sh new file mode 100755 index 00000000..3eb7b40f --- /dev/null +++ b/app/bin/worker.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# paths +app='/srv/app' +manage=$app'/manage.py' +wsgi=$app'/wsgi.py' +concurrency=12 + +# stating apps +# pip uninstall -y south +# pip install -U django==1.8.18 django-registration-redux djangorestframework==3.6.4 +# pip install django-debug-toolbar==1.6 +# pip install -e git+https://github.com/Parisson/django-jqchat.git@dj1.8#egg=django-jqchat +# pip install -e git+https://github.com/Parisson/saved_searches.git@dj1.8#egg=saved_searches-2.0.0-beta + +# waiting for other services +bash $app/bin/wait.sh + +# Starting celery worker with the --autoreload option will enable the worker to watch for file system changes +# This is an experimental feature intended for use in development only +# see http://celery.readthedocs.org/en/latest/userguide/workers.html#autoreloading +python $manage celery worker --autoreload -A worker --concurrency=$concurrency diff --git a/docker-compose.yml b/docker-compose.yml index 84d2a999..eede8a7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,6 +42,18 @@ services: env_file: - env/debug.env + worker: + build: . + volumes_from: + - app + - var + env_file: + - env/debug.env + command: /bin/bash bin/worker.sh + links: + - broker + - db + web: image: nginx ports: -- 2.39.5