From 37b90f76cd2a9b8da154c2f1944f24a5d183b40a Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Wed, 15 Nov 2023 11:24:28 +0100 Subject: [PATCH] cleanup bin --- bin/build/doc.sh | 3 - bin/build/front.sh | 3 - bin/build/graph.sh | 3 - bin/build/local/messages.sh | 10 --- bin/build/local/setup_lib.sh | 14 ---- bin/build/local/setup_lib_py2.sh | 14 ---- bin/build/messages.sh | 3 - bin/build/readme.sh | 7 -- bin/misc/fix/fs_migrate.sh | 9 --- bin/misc/fix_var_perms.sh | 17 ---- bin/misc/gulp.sh | 3 - bin/misc/poll_twitter.sh | 3 - bin/misc/sql/latin1_to_utf8.py | 31 -------- bin/misc/sql/latin1_to_utf8_tester.py | 107 -------------------------- bin/misc/statifier.py | 79 ------------------- bin/misc/statifier.sh | 20 ----- 16 files changed, 326 deletions(-) delete mode 100755 bin/build/doc.sh delete mode 100755 bin/build/front.sh delete mode 100755 bin/build/graph.sh delete mode 100755 bin/build/local/messages.sh delete mode 100755 bin/build/local/setup_lib.sh delete mode 100755 bin/build/local/setup_lib_py2.sh delete mode 100755 bin/build/messages.sh delete mode 100755 bin/build/readme.sh delete mode 100755 bin/misc/fix/fs_migrate.sh delete mode 100755 bin/misc/fix_var_perms.sh delete mode 100755 bin/misc/gulp.sh delete mode 100755 bin/misc/poll_twitter.sh delete mode 100755 bin/misc/sql/latin1_to_utf8.py delete mode 100755 bin/misc/sql/latin1_to_utf8_tester.py delete mode 100644 bin/misc/statifier.py delete mode 100755 bin/misc/statifier.sh diff --git a/bin/build/doc.sh b/bin/build/doc.sh deleted file mode 100755 index abc248de..00000000 --- a/bin/build/doc.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -docker-compose run app bash /srv/doc/build.sh diff --git a/bin/build/front.sh b/bin/build/front.sh deleted file mode 100755 index c4157f53..00000000 --- a/bin/build/front.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -docker-compose exec -T app python /srv/app/manage.py build-front $1 diff --git a/bin/build/graph.sh b/bin/build/graph.sh deleted file mode 100755 index f2423137..00000000 --- a/bin/build/graph.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -docker-compose run app python /srv/app/manage.py graph_models organization_core organization-media organization_pages organization.network organization.magazine organization.projects organization.agenda organization.shop organization.job > /srv/doc/graph/mezzanine-organization.dot diff --git a/bin/build/local/messages.sh b/bin/build/local/messages.sh deleted file mode 100755 index 5fe3892c..00000000 --- a/bin/build/local/messages.sh +++ /dev/null @@ -1,10 +0,0 @@ -# docker-compose run app python manage.py compilemessages - -cd /srv - -find $(pwd) -type d -name 'locale' -print | while read f; do - cd "$f" && cd .. - echo $(pwd) - django-admin makemessages -l fr - django-admin compilemessages -done diff --git a/bin/build/local/setup_lib.sh b/bin/build/local/setup_lib.sh deleted file mode 100755 index de62e568..00000000 --- a/bin/build/local/setup_lib.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# Exit when any command fails -# We need to stop build if pip fails -set -e - -for module in `ls /srv/lib/`; do - cd /srv/lib/$module - if [ -f 'requirements.txt' ]; then - pip3 install -r requirements.txt - elif [ -f 'setup.py' ]; then - pip3 install -e . - fi -done diff --git a/bin/build/local/setup_lib_py2.sh b/bin/build/local/setup_lib_py2.sh deleted file mode 100755 index c037a260..00000000 --- a/bin/build/local/setup_lib_py2.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# Exit when any command fails -# We need to stop build if pip fails -set -e - -for module in `ls /srv/lib/`; do - cd /srv/lib/$module - if [ -f 'requirements.txt' ]; then - pip install -r requirements.txt - elif [ -f 'setup.py' ]; then - pip install -e . - fi -done diff --git a/bin/build/messages.sh b/bin/build/messages.sh deleted file mode 100755 index 9a685c81..00000000 --- a/bin/build/messages.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -docker compose exec app bash /srv/bin/build/local/messages.sh diff --git a/bin/build/readme.sh b/bin/build/readme.sh deleted file mode 100755 index a22fb8cc..00000000 --- a/bin/build/readme.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -cd "$(dirname "$0")"/../../ - -cat doc/src/overview.rst doc/src/architecture.rst doc/src/install.rst doc/src/development.rst doc/src/maintenance.rst doc/src/copyright.rst doc/src/license.rst > README.rst - -echo "Build finished. The README.rst file is up to date." diff --git a/bin/misc/fix/fs_migrate.sh b/bin/misc/fix/fs_migrate.sh deleted file mode 100755 index 9ec9fcf1..00000000 --- a/bin/misc/fix/fs_migrate.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -docker-compose stop -mv data var -mkdir var/lib -sudo mv var/postgresql var/lib -sudo mv var/external var/opt -sudo mv var/var/log var -sudo rm -rf var/var diff --git a/bin/misc/fix_var_perms.sh b/bin/misc/fix_var_perms.sh deleted file mode 100755 index f0c5b1b6..00000000 --- a/bin/misc/fix_var_perms.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -cd "$(dirname "$0")"/../../ - -# We need to chown folders so that they can be used by the $USER (with an exception on Darwin) -if ! uname -a | grep Darwin > /dev/null; then - - declare -a arr=("var/media" "var/backup") - - for folder in "${arr[@]}"; do - INFO=( $(stat -L -c "%a %G %U" $folder) ) - OWNER=${INFO[2]} - if [ "$OWNER" != "$USER" ]; then - sudo chown -R $USER $folder - fi - done -fi diff --git a/bin/misc/gulp.sh b/bin/misc/gulp.sh deleted file mode 100755 index dcff3d0b..00000000 --- a/bin/misc/gulp.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -docker-compose run app python /srv/app/manage.py gulp diff --git a/bin/misc/poll_twitter.sh b/bin/misc/poll_twitter.sh deleted file mode 100755 index fc4b929f..00000000 --- a/bin/misc/poll_twitter.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -docker-compose exec -T app python /srv/app/manage.py poll_twitter --force diff --git a/bin/misc/sql/latin1_to_utf8.py b/bin/misc/sql/latin1_to_utf8.py deleted file mode 100755 index 97f1bc10..00000000 --- a/bin/misc/sql/latin1_to_utf8.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/python3.5 - -import os -from chardet import detect -from pyutil import filereplace -srcfile = "./var/backup/mariadb.dump" -trgfile = "./var/backup/mariadb2.dump" - -# get file encoding type -def get_encoding_type(file): - with open(file, 'rb') as f: - rawdata = f.read() - return detect(rawdata)['encoding'] - -from_codec = get_encoding_type(srcfile) - -# add try: except block for reliability -try: - with open(srcfile, 'r', encoding=from_codec) as f, open(trgfile, 'w', encoding='utf-8') as e: - text = f.read() # for small files, for big use chunks - e.write(text) - - os.remove(srcfile) # remove old encoding file - os.rename(trgfile, srcfile) # rename new encoding -except UnicodeDecodeError: - print('Decode Error') -except UnicodeEncodeError: - print('Encode Error') - -# replace charset latin1 to ut8 -filereplace(srcfile,"CHARSET=latin1","CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci") \ No newline at end of file diff --git a/bin/misc/sql/latin1_to_utf8_tester.py b/bin/misc/sql/latin1_to_utf8_tester.py deleted file mode 100755 index 6aa629d3..00000000 --- a/bin/misc/sql/latin1_to_utf8_tester.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/python - -import pandas as pd -import requests -from subprocess import check_output, CalledProcessError -domain_prod = "http://brahms.ircam.fr" -domain_dev = "http://localhost:9030" -session_option = "-H 'Cookie: sessionid={0}'" -session_prod = "gd9zgu68wo3lptwfhqqkawo59w0486rz" -session_dev = "qh25niyvxetuouxwmafdpvxlc8eldp5u" - -analyse_urls = [ - # analyse - published - "/analyses/dialogue/", - "/analyses/metallics/", - "/analyses/noanoa/", - "/analyses/traiettoria/", - "/analyses/Mortuos/", - "/analyses/Stria/", - "/analyses/Prologue/", - # analyse - draft - "/analyses/Etymo/", - "/analyses/EnTrance/", - "/analyses/test/", - "/analyses/test2/", - # composer - "/witold-lutoslawski", - "/wlodzimierz-kotonski", - "/omer-hulusier", - "/necil-kazim-akses", - "/wenjing-guo", - "/rene-alix", - "/axel-borup-jrgensen", - "/per-nrgard", - "/pierre-boulez", - # events_event - "/admin/events/event/13/change/", - "/admin/events/event/23/change/", - #events_hall - "/admin/events/hall/5/change/", - #events_manifestation - "/admin/events/manifestation/3/change/", - #repertoire_analysis_definitions - "/admin/repertoire/definition/32/change/", - #robots_rule - "/admin/robots/rule/1/change/", - #robots_url - "/admin/robots/url/3/change/", - #utils_citysidney - "/admin/utils/citysidney/3/change/", - #utils_corporatebody - "/admin/utils/corporatebody/127/change/", - #utils_countrysidney - "/admin/utils/countrysidney/230/change/", - #utils_equipmentbrand - "/admin/utils/equipmentbrand/127/change/", - #utils_equipmentcategory - "admin/utils/equipmentcategory/4/change/", - #utils_equipmentreference - "/admin/utils/equipmentreference/422/change/", - #utils_error - "/admin/utils/error/1703/change/", - #utils_lang - "/admin/utils/lang/1/change/", - #utils_naturalperson - "/admin/utils/naturalperson/6791/change/", - "/admin/utils/naturalperson/6806/change/", - "/admin/utils/naturalperson/6892/change/", - #utils_personfunction - "/admin/utils/personfunction/12/change/", - #validation_fichedevalidation - "/admin/validation/fichedevalidation/74/change/", - "/admin/validation/fichedevalidation/63/change/", - #works_electronic - "/admin/works/electronic/11/change/", - "/admin/works/electronic/5/change/", - #works_filetype - "/admin/works/filetype/20/change/", - #works_version - "/admin/works/version/1345/change/", - "/admin/works/version/990/change/", - #works_versionfile - "/admin/works/versionfile/2665/change/", - "/admin/works/versionfile/9250/change/", - #works_worksidney - "/admin/works/worksidney/6970/change/", - "/admin/works/worksidney/18566/change/", - "/admin/works/worksidney/7549/change/", - "/admin/works/worksidney/25673/change/", - "/admin/works/worksidney/10575/change/", -] - -def get_curl_command(domain, url, session): - return "curl -s '{0}' {1}".format(domain + url, session_option.format(session)) - -for url in analyse_urls: - print("==========================================================================") - print("url ", url) - print("-----------------------------------------") - try: - curl_prod = get_curl_command(domain_prod, url, session_prod) - curl_dev = get_curl_command(domain_dev, url, session_dev) - check_output("diff <("+curl_prod+") <("+ curl_dev + ")", - shell=True, executable='/bin/bash', universal_newlines=True) - except CalledProcessError as e: - print(e.output, e.returncode) - diff --git a/bin/misc/statifier.py b/bin/misc/statifier.py deleted file mode 100644 index 641cd057..00000000 --- a/bin/misc/statifier.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/python3 - -import os -from optparse import OptionParser -import shutil - -class Statifier: - - def __init__(self, domain, languages): - self.domain = domain - self.languages = languages - self.replace_dict = { - } - - def wget(self, language, dir): - command = 'wget -q --mirror -p --adjust-extension --header="Accept-Language: %s" -e robots=off --base=./ -k -P %s https://%s' % (language, dir, self.domain) - # print(command) - os.system(command) - - def sed(self, dir, rule): - command = "find %s -type f -print0 | xargs -0 sed -r -i -e '%s'" % (dir, rule) - # print(command) - os.system(command) - - def fix_languages(self, language): - for root, dirs, files in os.walk(self.domain): - for filename in files: - name = os.path.splitext(filename)[0] - ext = os.path.splitext(filename)[1][1:] - if ext == 'html': - path = root + os.sep + filename - rel_root = root.split('/')[2:] - rel_root = language + '/' + '/'.join(rel_root) - - f = open(path, 'rt') - content = f.read() - f.close() - - s_in = '
' % self.domain - s_out = '' - content = content.replace(s_in, s_out) - - s_in = '' % (language, language.upper()) - s_out = '' + s_in + '' - s_out = s_out % rel_root - content = content.replace(s_in, s_out) - - s_in = '' % (language, language.upper()) - s_out = '' + s_in + '' - s_out = s_out % rel_root - content = content.replace(s_in, s_out) - - f = open(path, 'wt') - f.write(content) - f.close() - - def main(self): - for language in self.languages: - self.wget(language, self.domain) - shutil.move(self.domain + os.sep + self.domain, self.domain + os.sep + language) - - for language in self.languages: - self.fix_languages(language) - - -def main(): - parser = OptionParser() - parser.add_option("-d", "--domain", dest="domain", help="domain") - parser.add_option("-l", "--languages", - dest="languages", nargs=2, - help="languages") - (options, args) = parser.parse_args() - - s = Statifier(options.domain, options.languages) - s.main() - -if __name__ == "__main__": - main() - diff --git a/bin/misc/statifier.sh b/bin/misc/statifier.sh deleted file mode 100755 index 262c24b2..00000000 --- a/bin/misc/statifier.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -DOMAIN=$1 - -mkdir $DOMAIN -cd $DOMAIN - -LANG=en -wget -q --mirror -p --adjust-extension --header='Accept-Language: en' -e robots=off --base=./ -k -P ./ https://$DOMAIN -mv $DOMAIN $LANG -find $LANG/ -type f -print0 | xargs -0 sed -r -i -e 's,,,g' -find $LANG/ -type f -print0 | xargs -0 sed -r -i -e 's,,,g' -find $LANG/ -type f -print0 | xargs -0 sed -r -i -e 's,
,,g' - -LANG=fr -wget -q --mirror -p --adjust-extension --header='Accept-Language: fr' -e robots=off --base=./ -k -P ./ https://$DOMAIN -mv $DOMAIN $LANG -find $LANG -type f -print0 | xargs -0 sed -r -i -e 's,
,,g' -find $LANG -type f -print0 | xargs -0 sed -r -i -e 's,,,g' -find $LANG -type f -print0 | xargs -0 sed -r -i -e 's,
,,g' -- 2.39.5