From 94850ce4451319c8854dc3385d76398d40350ae4 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Tue, 11 May 2021 09:58:42 +0200 Subject: [PATCH] cleanup, take app scripts from mezzo again --- Dockerfile | 2 +- app/bin/app.sh | 66 +++--- app/bin/diag.sh | 12 - app/bin/enumeration.sh | 4 - app/bin/init.sh | 13 +- app/bin/install_plugins.sh | 9 - app/bin/modelviz.py | 359 ----------------------------- app/bin/notebook.sh | 6 - app/bin/setup_plugins.sh | 22 -- app/bin/update_schema.sh | 4 - app/bin/upgrade_from_1.6_to_1.7.sh | 7 - app/bin/wait.sh | 19 +- app/bin/worker.sh | 22 -- 13 files changed, 43 insertions(+), 502 deletions(-) delete mode 100755 app/bin/diag.sh delete mode 100755 app/bin/enumeration.sh delete mode 100755 app/bin/install_plugins.sh delete mode 100755 app/bin/modelviz.py delete mode 100755 app/bin/notebook.sh delete mode 100755 app/bin/setup_plugins.sh delete mode 100755 app/bin/update_schema.sh delete mode 100755 app/bin/upgrade_from_1.6_to_1.7.sh delete mode 100755 app/bin/worker.sh diff --git a/Dockerfile b/Dockerfile index eb989980..167588e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM parisson/telemeta:latest-dev +FROM python2 MAINTAINER Guillaume Pellerin diff --git a/app/bin/app.sh b/app/bin/app.sh index 8fa9a6ab..45cc51b2 100755 --- a/app/bin/app.sh +++ b/app/bin/app.sh @@ -6,60 +6,54 @@ manage=$app'/manage.py' wsgi=$app'/wsgi.py' static='/srv/static/' media='/srv/media/' -lib='/srv/lib/' -log='/var/log/uwsgi/app.log' +src='/srv/src/' +uwsgi_log='/var/log/uwsgi/app.log' +debug_log='/var/log/app/debug.log' # uwsgi params port=8000 -processes=2 -threads=2 +processes=8 +threads=16 +autoreload=3 uid='www-data' gid='www-data' +# patterns='*.js;*.css;*.jpg;*.jpeg;*.gif;*.png;*.svg;*.ttf;*.eot;*.woff;*.woff2' -# stating apps -# pip uninstall -y south -# pip install -U django==1.8.18 django-registration-redux djangorestframework==3.6.4 -# pip install django-debug-toolbar==1.6 -# pip install -e git+https://github.com/Parisson/django-jqchat.git@dj1.8#egg=django-jqchat -# pip install -e git+https://github.com/Parisson/saved_searches.git@dj1.8#egg=saved_searches-2.0.0-beta +# Install a package in development mode +# without rebuidling docker image. +# You need at first checkout your sources in 'lib' folder +# in host project side, then run : +# pip install -e /srv/lib/mypackage... +#pip install mysqlclient==1.3.13 -# waiting for other network services -sh $app/bin/wait.sh -python $manage wait-for-db - -if [ ! -f .init ]; then - python $manage migrate --noinput - python $manage bower_install -- --allow-root - touch .init -fi - -# telemeta setup -python $manage telemeta-create-admin-user -python $manage telemeta-create-boilerplate -python $manage telemeta-setup-enumerations +# Install (staging) libs +# /srv/bin/build/local/setup_lib.sh +# waiting for other services +sh $app/bin/wait.sh -# Delete Timeside database if it exists -cat /srv/lib/telemeta/bin/sql/drop_timeside.sql | python $manage dbshell +# django setup +#python $manage wait-for-db -if [ $REINDEX = "True" ]; then - python $manage rebuild_index --noinput -fi +# initial setup +# if [ ! -f .init ]; then +# bash $app/bin/init.sh +# touch .init +# fi -# choose dev or prod mode +# app start if [ "$1" = "--runserver" ]; then python $manage runserver 0.0.0.0:8000 else # static files auto update # watchmedo shell-command --patterns="$patterns" --recursive \ - # --command='python '$manage' collectstatic --noinput' $lib & + # --command='python '$manage' collectstatic --noinput' $app & + python $manage collectstatic --noinput - # fix media access rights - find $media -maxdepth 2 -path ${media}import -prune -o -type d -not -user www-data -exec chown www-data:www-data {} \; + chown www-data: $debug_log - # app start uwsgi --socket :$port --wsgi-file $wsgi --chdir $app --master \ - --processes $processes --threads $threads \ - --uid $uid --gid $gid --logto $log --touch-reload $wsgi + --processes $processes --threads $threads \ + --uid $uid --gid $gid --logto $uwsgi_log --touch-reload $wsgi fi diff --git a/app/bin/diag.sh b/app/bin/diag.sh deleted file mode 100755 index 44d05529..00000000 --- a/app/bin/diag.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -app="telemeta" -dir="../../doc/devel" - -python modelviz.py -a > $dir/$app-all.dot -python modelviz.py $app > $dir/$app.dot - -dot $dir/$app-all.dot -Tpdf -o $dir/$app-all.pdf -dot $dir/$app.dot -Tpdf -o $dir/$app.pdf - -rsync -a $dir/ doc.parisson.com:/var/www/files/doc/$app/diagram/ diff --git a/app/bin/enumeration.sh b/app/bin/enumeration.sh deleted file mode 100755 index c6a42b04..00000000 --- a/app/bin/enumeration.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - pwd - - chmod 777 "enumeration/enumeration.txt" && echo "The file is now writable" diff --git a/app/bin/init.sh b/app/bin/init.sh index 21bd3d5f..1aa59bf6 100755 --- a/app/bin/init.sh +++ b/app/bin/init.sh @@ -1,9 +1,16 @@ #!/bin/bash +# paths app='/srv/app' manage=$app'/manage.py' python $manage migrate --noinput -python $manage telemeta-create-admin-user -python $manage telemeta-create-boilerplate -python $manage bower_install -- --allow-root +python $manage create-admin-user +python $manage create-default-organization +python $manage build-front + +# @todo searching every fixtures file in each folder +python $manage loaddata $app/organization/job/fixtures/organization-job.json +python $manage loaddata $app/organization/projects/fixtures/organization-projects-repositorysystems.json + +bash /srv/doc/build.sh diff --git a/app/bin/install_plugins.sh b/app/bin/install_plugins.sh deleted file mode 100755 index b0315e9f..00000000 --- a/app/bin/install_plugins.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -plugins=/srv/lib/plugins - -for dir in $(ls $plugins); do - if [ -f $plugins/$dir/setup.py ]; then - pip install -e $plugins/$dir/. - fi -done diff --git a/app/bin/modelviz.py b/app/bin/modelviz.py deleted file mode 100755 index 24af0627..00000000 --- a/app/bin/modelviz.py +++ /dev/null @@ -1,359 +0,0 @@ -#!/usr/bin/env python - -"""Django model to DOT (Graphviz) converter -by Antonio Cavedoni - -Make sure your DJANGO_SETTINGS_MODULE is set to your project or -place this script in the same directory of the project and call -the script like this: - -$ python modelviz.py [-h] [-a] [-d] [-g] [-n] [-L ] [-i ] ... > .dot -$ dot .dot -Tpng -o .png - -options: - -h, --help - show this help message and exit. - - -a, --all_applications - show models from all applications. - - -d, --disable_fields - don't show the class member fields. - - -g, --group_models - draw an enclosing box around models from the same app. - - -i, --include_models=User,Person,Car - only include selected models in graph. - - -n, --verbose_names - use verbose_name for field and models. - - -L, --language - specify language used for verrbose_name localization - - -x, --exclude_columns - exclude specific column(s) from the graph. - - -X, --exclude_models - exclude specific model(s) from the graph. - - -e, --inheritance - show inheritance arrows. -""" -__version__ = "0.9" -__svnid__ = "$Id$" -__license__ = "Python" -__author__ = "Antonio Cavedoni " -__contributors__ = [ - "Stefano J. Attardi ", - "limodou ", - "Carlo C8E Miron", - "Andre Campos ", - "Justin Findlay ", - "Alexander Houben ", - "Bas van Oostveen ", - "Joern Hees " -] - -import os -import sys -import getopt - -from django.core.management import setup_environ - -try: - import settings -except ImportError: - pass -else: - setup_environ(settings) - -from django.utils.translation import activate as activate_language -from django.utils.safestring import mark_safe -from django.template import Template, Context, loader -from django.db import models -from django.db.models import get_models -from django.db.models.fields.related import \ - ForeignKey, OneToOneField, ManyToManyField, RelatedField - -try: - from django.db.models.fields.generic import GenericRelation -except ImportError: - from django.contrib.contenttypes.generic import GenericRelation - -def parse_file_or_list(arg): - if not arg: - return [] - if not ',' in arg and os.path.isfile(arg): - return [e.strip() for e in open(arg).readlines()] - return arg.split(',') - - -def generate_dot(app_labels, **kwargs): - disable_fields = kwargs.get('disable_fields', False) - include_models = parse_file_or_list(kwargs.get('include_models', "")) - all_applications = kwargs.get('all_applications', False) - use_subgraph = kwargs.get('group_models', False) - verbose_names = kwargs.get('verbose_names', False) - inheritance = kwargs.get('inheritance', False) - language = kwargs.get('language', None) - if language is not None: - activate_language(language) - exclude_columns = parse_file_or_list(kwargs.get('exclude_columns', "")) - exclude_models = parse_file_or_list(kwargs.get('exclude_models', "")) - - def skip_field(field): - if exclude_columns: - if verbose_names and field.verbose_name: - if field.verbose_name in exclude_columns: - return True - if field.name in exclude_columns: - return True - return False - - - - - t = loader.get_template('django_extensions/graph_models/head.html') - c = Context({}) - dot = t.render(c) - - apps = [] - if all_applications: - apps = models.get_apps() - - for app_label in app_labels: - app = models.get_app(app_label) - if not app in apps: - apps.append(app) - - graphs = [] - for app in apps: - graph = Context({ - 'name': '"%s"' % app.__name__, - 'app_name': "%s" % '.'.join(app.__name__.split('.')[:-1]), - 'cluster_app_name': "cluster_%s" % app.__name__.replace(".", "_"), - 'disable_fields': disable_fields, - 'use_subgraph': use_subgraph, - 'models': [] - }) - - appmodels = get_models(app) - abstract_models = [] - for appmodel in appmodels: - abstract_models = abstract_models + [abstract_model for abstract_model in appmodel.__bases__ if hasattr(abstract_model, '_meta') and abstract_model._meta.abstract] - abstract_models = list(set(abstract_models)) # remove duplicates - appmodels = abstract_models + appmodels - - - for appmodel in appmodels: - appmodel_abstracts = [abstract_model.__name__ for abstract_model in appmodel.__bases__ if hasattr(abstract_model, '_meta') and abstract_model._meta.abstract] - - # collect all attribs of abstract superclasses - def getBasesAbstractFields(c): - _abstract_fields = [] - for e in c.__bases__: - if hasattr(e, '_meta') and e._meta.abstract: - _abstract_fields.extend(e._meta.fields) - _abstract_fields.extend(getBasesAbstractFields(e)) - return _abstract_fields - abstract_fields = getBasesAbstractFields(appmodel) - - model = { - 'app_name': appmodel.__module__.replace(".", "_"), - 'name': appmodel.__name__, - 'abstracts': appmodel_abstracts, - 'fields': [], - 'relations': [] - } - - # consider given model name ? - def consider(model_name): - if exclude_models and model_name in exclude_models: - return False - return not include_models or model_name in include_models - - if not consider(appmodel._meta.object_name): - continue - - if verbose_names and appmodel._meta.verbose_name: - model['label'] = appmodel._meta.verbose_name - else: - model['label'] = model['name'] - - # model attributes - def add_attributes(field): - if verbose_names and field.verbose_name: - label = field.verbose_name - else: - label = field.name - - t = type(field).__name__ - if isinstance(field, (OneToOneField, ForeignKey)): - t += " ({0})".format(field.rel.field_name) - # TODO: ManyToManyField, GenericRelation - - model['fields'].append({ - 'name': field.name, - 'label': label, - 'type': t, - 'blank': field.blank, - 'abstract': field in abstract_fields, - }) - - # Find all the real attributes. Relations are depicted as graph edges instead of attributes - attributes = [field for field in appmodel._meta.local_fields if not isinstance(field, RelatedField)] - - # find primary key and print it first, ignoring implicit id if other pk exists - pk = appmodel._meta.pk - if not appmodel._meta.abstract and pk in attributes: - add_attributes(pk) - for field in attributes: - if skip_field(field): - continue - if not field.primary_key: - add_attributes(field) - - # FIXME: actually many_to_many fields aren't saved in this model's db table, so why should we add an attribute-line for them in the resulting graph? - #if appmodel._meta.many_to_many: - # for field in appmodel._meta.many_to_many: - # if skip_field(field): - # continue - # add_attributes(field) - - # relations - def add_relation(field, extras=""): - if verbose_names and field.verbose_name: - label = field.verbose_name - else: - label = field.name - - # show related field name - if hasattr(field, 'related_query_name'): - label += ' (%s)' % field.related_query_name() - - _rel = { - 'target_app': field.rel.to.__module__.replace('.', '_'), - 'target': field.rel.to.__name__, - 'type': type(field).__name__, - 'name': field.name, - 'label': label, - 'arrows': extras, - 'needs_node': True - } - if _rel not in model['relations'] and consider(_rel['target']): - model['relations'].append(_rel) - - for field in appmodel._meta.local_fields: - if field.attname.endswith('_ptr_id'): # excluding field redundant with inheritance relation - continue - if field in abstract_fields: # excluding fields inherited from abstract classes. they too show as local_fields - continue - if skip_field(field): - continue - if isinstance(field, OneToOneField): - add_relation(field, '[arrowhead=none, arrowtail=none]') - elif isinstance(field, ForeignKey): - add_relation(field, '[arrowhead=none, arrowtail=dot]') - - for field in appmodel._meta.local_many_to_many: - if skip_field(field): - continue - if isinstance(field, ManyToManyField): - if (getattr(field, 'creates_table', False) or # django 1.1. - (hasattr(field.rel.through, '_meta') and field.rel.through._meta.auto_created)): # django 1.2 - add_relation(field, '[arrowhead=dot arrowtail=dot, dir=both]') - elif isinstance(field, GenericRelation): - add_relation(field, mark_safe('[style="dotted", arrowhead=normal, arrowtail=normal, dir=both]')) - - if inheritance: - # add inheritance arrows - for parent in appmodel.__bases__: - if hasattr(parent, "_meta"): # parent is a model - l = "multi-table" - if parent._meta.abstract: - l = "abstract" - if appmodel._meta.proxy: - l = "proxy" - l += r"\ninheritance" - _rel = { - 'target_app': parent.__module__.replace(".", "_"), - 'target': parent.__name__, - 'type': "inheritance", - 'name': "inheritance", - 'label': l, - 'arrows': '[arrowhead=empty, arrowtail=none]', - 'needs_node': True - } - # TODO: seems as if abstract models aren't part of models.getModels, which is why they are printed by this without any attributes. - if _rel not in model['relations'] and consider(_rel['target']): - model['relations'].append(_rel) - - graph['models'].append(model) - graphs.append(graph) - - nodes = [] - for graph in graphs: - nodes.extend([e['name'] for e in graph['models']]) - - for graph in graphs: - # don't draw duplication nodes because of relations - for model in graph['models']: - for relation in model['relations']: - if relation['target'] in nodes: - relation['needs_node'] = False - # render templates - t = loader.get_template('django_extensions/graph_models/body.html') - dot += '\n' + t.render(graph) - - for graph in graphs: - t = loader.get_template('django_extensions/graph_models/rel.html') - dot += '\n' + t.render(graph) - - - t = loader.get_template('django_extensions/graph_models/tail.html') - c = Context({}) - dot += '\n' + t.render(c) - return dot - -def main(): - try: - opts, args = getopt.getopt(sys.argv[1:], "hadgi:L:x:X:en", - ["help", "all_applications", "disable_fields", "group_models", "include_models=", "inheritance", "verbose_names", "language=", "exclude_columns=", "exclude_models="]) - except getopt.GetoptError, error: - print __doc__ - sys.exit(error) - - kwargs = {} - for opt, arg in opts: - if opt in ("-h", "--help"): - print __doc__ - sys.exit() - if opt in ("-a", "--all_applications"): - kwargs['all_applications'] = True - if opt in ("-d", "--disable_fields"): - kwargs['disable_fields'] = True - if opt in ("-g", "--group_models"): - kwargs['group_models'] = True - if opt in ("-i", "--include_models"): - kwargs['include_models'] = arg - if opt in ("-e", "--inheritance"): - kwargs['inheritance'] = True - if opt in ("-n", "--verbose-names"): - kwargs['verbose_names'] = True - if opt in ("-L", "--language"): - kwargs['language'] = arg - if opt in ("-x", "--exclude_columns"): - kwargs['exclude_columns'] = arg - if opt in ("-X", "--exclude_models"): - kwargs['exclude_models'] = arg - - if not args and not kwargs.get('all_applications', False): - print __doc__ - sys.exit() - - print generate_dot(args, **kwargs) - -if __name__ == "__main__": - main() diff --git a/app/bin/notebook.sh b/app/bin/notebook.sh deleted file mode 100755 index 562f9dfd..00000000 --- a/app/bin/notebook.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -export PYTHONPATH=$PYTHONPATH:/opt/miniconda/lib/python2.7/site-packages/:/srv/app/ -export DJANGO_SETTINGS_MODULE=settings - -python /srv/app/manage.py shell_plus --notebook diff --git a/app/bin/setup_plugins.sh b/app/bin/setup_plugins.sh deleted file mode 100755 index de55f841..00000000 --- a/app/bin/setup_plugins.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -plugins=/srv/lib/plugins - -apt-get update - -for dir in $(ls $plugins); do - env=$plugins/$dir/conda-environment.yml - if [ -f $env ]; then - conda env update --name root --file $env - fi - req=$plugins/$dir/debian-requirements.txt - if [ -f $req ]; then - packs=$(egrep -v "^\s*(#|$)" $req) - apt-get install -y --force-yes $packs - fi - if [ -f $plugins/$dir/setup.py ]; then - pip install -e $plugins/$dir/. - fi -done - -apt-get clean diff --git a/app/bin/update_schema.sh b/app/bin/update_schema.sh deleted file mode 100755 index 1b64fa36..00000000 --- a/app/bin/update_schema.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -./manage.py schemamigration telemeta --auto -./manage.py migrate telemeta diff --git a/app/bin/upgrade_from_1.6_to_1.7.sh b/app/bin/upgrade_from_1.6_to_1.7.sh deleted file mode 100755 index af74807e..00000000 --- a/app/bin/upgrade_from_1.6_to_1.7.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -python manage.py migrate -python manage.py migrate contenttypes --fake-initial -python manage.py migrate --fake-initial -python manage.py migrate thumbnail --fake-initial -python manage.py migrate --fake telemeta 0006 diff --git a/app/bin/wait.sh b/app/bin/wait.sh index f2a93f79..e9331381 100755 --- a/app/bin/wait.sh +++ b/app/bin/wait.sh @@ -1,18 +1,3 @@ -#!/bin/bash +#!/bin/sh -# apt-get install -y --force-yes netcat - -set -e - -host=$(env | grep _TCP_ADDR | cut -d = -f 2) -port=$(env | grep _TCP_PORT | cut -d = -f 2) - -echo -n "waiting for TCP connection to $host:$port..." - -while ! nc -w 1 $host $port 2>/dev/null -do - echo -n . - sleep 1 -done - -echo 'ok' +/srv/bin/misc/wait-for-it/wait-for-it.sh -h localhost -p $DB_PORT; diff --git a/app/bin/worker.sh b/app/bin/worker.sh deleted file mode 100755 index ff23a89c..00000000 --- a/app/bin/worker.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# paths -app='/srv/app' -manage=$app'/manage.py' -wsgi=$app'/wsgi.py' -concurrency=2 - -# stating apps -# pip uninstall -y south -# pip install -U django==1.8.18 django-registration-redux djangorestframework==3.6.4 -# pip install django-debug-toolbar==1.6 -# pip install -e git+https://github.com/Parisson/django-jqchat.git@dj1.8#egg=django-jqchat -# pip install -e git+https://github.com/Parisson/saved_searches.git@dj1.8#egg=saved_searches-2.0.0-beta - -# waiting for other services -bash $app/bin/wait.sh - -# Starting celery worker with the --autoreload option will enable the worker to watch for file system changes -# This is an experimental feature intended for use in development only -# see http://celery.readthedocs.org/en/latest/userguide/workers.html#autoreloading -python $manage celery worker --autoreload -A worker --concurrency=$concurrency -- 2.39.5