From a007b8fae9084d4823aebfeae913222266fc1d01 Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Mon, 5 Jun 2023 11:07:29 +0200 Subject: [PATCH] add master processing scripts --- bin/{ => master/py}/create_thumbs.py | 0 bin/{ => master/py}/remux_fix_media.py | 2 +- bin/{ => master/py}/transcode.py | 0 bin/{ => master/py}/transcode_nonv.py | 0 bin/master/py/transcode_nv-pro.py | 75 +++++++++++++++++++ bin/{ => master/py}/transcode_nv.py | 0 bin/master/sh/backup_nile.sh | 4 + bin/master/sh/remux | 3 + bin/master/sh/reverse_ssh_tunnel | 29 +++++++ bin/master/sh/seafile_sync.sh | 6 ++ bin/master/sh/telecaster_master-ae.sh | 28 +++++++ .../sh/telecaster_master-crfpa-mp4-staging.sh | 35 +++++++++ bin/master/sh/telecaster_master-crfpa-mp4.sh | 42 +++++++++++ bin/master/sh/telecaster_master-pro.sh | 30 ++++++++ 14 files changed, 253 insertions(+), 1 deletion(-) rename bin/{ => master/py}/create_thumbs.py (100%) rename bin/{ => master/py}/remux_fix_media.py (97%) rename bin/{ => master/py}/transcode.py (100%) rename bin/{ => master/py}/transcode_nonv.py (100%) create mode 100644 bin/master/py/transcode_nv-pro.py rename bin/{ => master/py}/transcode_nv.py (100%) create mode 100755 bin/master/sh/backup_nile.sh create mode 100755 bin/master/sh/remux create mode 100755 bin/master/sh/reverse_ssh_tunnel create mode 100755 bin/master/sh/seafile_sync.sh create mode 100755 bin/master/sh/telecaster_master-ae.sh create mode 100755 bin/master/sh/telecaster_master-crfpa-mp4-staging.sh create mode 100755 bin/master/sh/telecaster_master-crfpa-mp4.sh create mode 100755 bin/master/sh/telecaster_master-pro.sh diff --git a/bin/create_thumbs.py b/bin/master/py/create_thumbs.py similarity index 100% rename from bin/create_thumbs.py rename to bin/master/py/create_thumbs.py diff --git a/bin/remux_fix_media.py b/bin/master/py/remux_fix_media.py similarity index 97% rename from bin/remux_fix_media.py rename to bin/master/py/remux_fix_media.py index f5693a6..895fc93 100644 --- a/bin/remux_fix_media.py +++ b/bin/master/py/remux_fix_media.py @@ -52,7 +52,7 @@ class FixCheckMedia(object): def hard_fix_webm(self, path): try: tmp_file = self.tmp_dir + 'out.webm ' - command = 'ffmpeg -loglevel 0 -i "'+ path + '" -vcodec libvpx -vb 500k -acodec libvorbis -aq 7 -f webm -y "' + tmp_file + '" > /dev/null' + command = 'ffmpeg -loglevel 0 -i "'+ path + '" -vcodec libvpx -vb 1500k -acodec libvorbis -aq 7 -f webm -y "' + tmp_file + '" > /dev/null' print(command) os.system(command) command = 'mv ' + tmp_file + path diff --git a/bin/transcode.py b/bin/master/py/transcode.py similarity index 100% rename from bin/transcode.py rename to bin/master/py/transcode.py diff --git a/bin/transcode_nonv.py b/bin/master/py/transcode_nonv.py similarity index 100% rename from bin/transcode_nonv.py rename to bin/master/py/transcode_nonv.py diff --git a/bin/master/py/transcode_nv-pro.py b/bin/master/py/transcode_nv-pro.py new file mode 100644 index 0000000..1c0161f --- /dev/null +++ b/bin/master/py/transcode_nv-pro.py @@ -0,0 +1,75 @@ +#!/usr/bin/python + +import os, sys, string +import logging +import datetime + + +class Logger: + """A logging object""" + + def __init__(self, file): + self.logger = logging.getLogger('myapp') + self.hdlr = logging.FileHandler(file) + self.formatter = logging.Formatter('%(asctime)s %(message)s') + self.hdlr.setFormatter(self.formatter) + self.logger.addHandler(self.hdlr) + self.logger.setLevel(logging.INFO) + + +class TelemetaTranscode(object): + """docstring for TelemetaTranscode""" + + source_formats = ['webm',] + dest_formats = { + 'mp3' : '-vn -acodec libmp3lame -aq 6', + 'mp4' : '-c:v h264_nvenc -maxrate 1100k -c:a aac -b:a 128k', + 'png' : '-ss 0:0:10', + } + + date_limit = datetime.datetime(year=2021, month=6, day=24) + + def __init__(self, args): + self.args = args + self.log_file = args[-1] + self.root_dir = args[-2] + self.logger = Logger(self.log_file) + + def get_ext_in_dir(self, extension, root): + files = os.listdir(root) + exts = [] + for f in files: + name, ext = os.path.splitext(f) + ext = ext[1:] + if not ext in exts: + exts.append(ext) + return extension in exts + + def run(self): + for root, dirs, files in os.walk(self.root_dir): + for file in files: + path = os.path.abspath(root + os.sep + file) + name, ext = os.path.splitext(file) + ext = ext[1:] + date_dir = datetime.datetime.fromtimestamp(os.path.getmtime(path)) + if ext in self.source_formats and date_dir > self.date_limit: + for format, ffmpeg_args in self.dest_formats.items(): + local_file = name + '.' + format + dest = os.path.abspath(root + os.sep + local_file) + local_files = os.listdir(root) + if not (local_file in local_files or self.get_ext_in_dir(format, root)) or '--force' in self.args: + if ext == 'webm' and format == 'ogg': + ffmpeg_args = '-vn -acodec copy' + if format == 'png': + command = '/usr/local/bin/ffmpeg -loglevel 0 ' + ffmpeg_args + ' -i "' + path + '" -frames:v 1 -y "' + dest + '"' + else: + command = '/usr/local/bin/ffmpeg -loglevel 0 -i "' + path + '" ' + ffmpeg_args + ' -y "' + dest + '"' + self.logger.logger.info(command) + if not '--dry-run' in self.args: + os.system(command) + print(command) + + +if __name__ == '__main__': + t = TelemetaTranscode(sys.argv[1:]) + t.run() diff --git a/bin/transcode_nv.py b/bin/master/py/transcode_nv.py similarity index 100% rename from bin/transcode_nv.py rename to bin/master/py/transcode_nv.py diff --git a/bin/master/sh/backup_nile.sh b/bin/master/sh/backup_nile.sh new file mode 100755 index 0000000..f38b6a6 --- /dev/null +++ b/bin/master/sh/backup_nile.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +rsync -aq /home/telecaster/archives/Pre-Barreau/ /mnt/nile/Pre-Barreau/ +#rsync -aq /home/telecaster/archives/Pre-Barreau/ admin-tech@Nile:/volume1/archives/Pre-Barreau/ diff --git a/bin/master/sh/remux b/bin/master/sh/remux new file mode 100755 index 0000000..e4e88a8 --- /dev/null +++ b/bin/master/sh/remux @@ -0,0 +1,3 @@ +#!/bin/bash + +ffmpeg -i $1 -c copy out.webm diff --git a/bin/master/sh/reverse_ssh_tunnel b/bin/master/sh/reverse_ssh_tunnel new file mode 100755 index 0000000..cd81fe6 --- /dev/null +++ b/bin/master/sh/reverse_ssh_tunnel @@ -0,0 +1,29 @@ +#!/bin/sh +# ------------------------------ +# autossh reverse tunnel on boot +# ------------------------------ +# See autossh and google for reverse ssh tunnels to see how this works + +# When this script runs it will allow you to ssh into this machine even if it is behind a firewall or has a NAT'd IP address. +# From any ssh capable machine you just type ssh -p $PORT_MIDDLEMAN_WILL_LISTEN_ON localusername@middleman + +# This is the username on your local server who has public key authentication setup at the middleman +USER_TO_SSH_IN_AS=telecaster + +# This is the username and hostname/IP address for the middleman (internet accessible server) +MIDDLEMAN_SERVER_AND_USERNAME=telecaster@jimi.parisson.com + +# The following two numbers can be whatever you want, but need to be unique if you have multiple reverse ssh tunnels +# Port that the middleman will listen on (use this value as the -p argument when sshing) +PORT_MIDDLEMAN_WILL_LISTEN_ON=22012 + +# Connection monitoring port, don't need to know this one +AUTOSSH_PORT=27012 + +# Ensures that autossh keeps trying to connect +AUTOSSH_GATETIME=0 + +export AUTOSSH_PORT AUTOSSH_GATETIME + +su -c "autossh -f -N -R *:${PORT_MIDDLEMAN_WILL_LISTEN_ON}:localhost:22 ${MIDDLEMAN_SERVER_AND_USERNAME} -oLogLevel=error -oUserKnownHostsFile=/dev/null -oStrictHostKeyChecking=no" $USER_TO_SSH_IN_AS + diff --git a/bin/master/sh/seafile_sync.sh b/bin/master/sh/seafile_sync.sh new file mode 100755 index 0000000..aece99e --- /dev/null +++ b/bin/master/sh/seafile_sync.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +seaf-cli start + +seaf-cli sync -l 063a03f2-2947-4f90-b4a1-7d1e34eb62e0 -s http://localhost:9080 -d "/home/telecaster/seafile/CRFPA" -u "telecaster@parisson.com" -p "cab2GhetGoog" + diff --git a/bin/master/sh/telecaster_master-ae.sh b/bin/master/sh/telecaster_master-ae.sh new file mode 100755 index 0000000..2d6948b --- /dev/null +++ b/bin/master/sh/telecaster_master-ae.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +media_dir=/home/telecaster/archives/Pre-Barreau/AE/ +tmp_log=/tmp/telecaster-remux-ae.log +log=/home/telecaster/log/remux-ae.log +tmp_dir=/home/telecaster/tmp/ +app_server=angus.parisson.com +media_server=angus.parisson.com +emails="webmaster@parisson.com,jeannot@parisson.com" +subject="TeleCaster remux (AE)" +year=`date '+%Y'` +#year=2020 +year2=$((year-1)) +#echo $year2 + +python ~/apps/Telemeta/scripts/transcode/remux_fix_media.py $media_dir $tmp_dir > $tmp_log + +#cat $tmp_log >> $log + +echo "$media_server:$media_dir" + +rsync -avuLKPr --delete --bwlimit=15000 --include="*/" --include="$year/**" --include="$year2/**" --exclude="*" $media_dir $media_server:$media_dir + +ssh $app_server "~/bin/teleforma_import.sh" + +python ~/apps/tools/message/mail_msg.py $subject $tmp_log $emails + + diff --git a/bin/master/sh/telecaster_master-crfpa-mp4-staging.sh b/bin/master/sh/telecaster_master-crfpa-mp4-staging.sh new file mode 100755 index 0000000..d20fae9 --- /dev/null +++ b/bin/master/sh/telecaster_master-crfpa-mp4-staging.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +year=`date '+%Y'` +media_dir=/home/telecaster/archives/Pre-Barreau/CRFPA/$year/ +backup_dir=/mnt/nile/Pre-Barreau/CRFPA/$year/ +tmp_log=/tmp/telecaster-remux-crfpa.log +log=/home/telecaster/log/remux-crfpa.log +tmp_dir=/home/telecaster/tmp/ +app_server=malcom.parisson.com +app_path=/srv/crfpa-docker-staging +media_server=malcom.parisson.com +emails="webmaster@parisson.com,jeannot@parisson.com" +subject="TeleCaster remux (CRFPA)" +bwlimit=20000 + +python ~/apps/Telemeta/scripts/transcode/remux_fix_media.py $media_dir $tmp_dir > $tmp_log + +cat $tmp_log >> $log + +rsync -uLKPr $media_dir $backup_dir + +python ~/apps/Telemeta/scripts/transcode/transcode_nv.py $backup_dir ~/log/transcode-crfpa.log + +find $backup_dir -type d -exec chmod 755 {} \; +find $backup_dir -type f -exec chmod 644 {} \; + +#rsync -auLKPr --bwlimit=$bwlimit --include="*/" --include="$year/**" --exclude="*" $backup_dir $media_server:$media_dir +#rsync -uLKPr --bwlimit=$bwlimit --include="*/" --include="$year/**" --exclude=".webm" --exclude=".ogg" --exclude="*" $backup_dir $media_server:$media_dir +rsync -auLKPr --bwlimit=$bwlimit --include="*/" --include="$year/**" --exclude="*.webm" --exclude="@eaDir" $backup_dir $media_server:$media_dir + +ssh $app_server "docker-compose -f $app_path/docker-compose.yml -f $app_path/env/prod.yml exec app /srv/app/manage.py teleforma-import-conferences-2 Pre-Barreau CRFPA /var/log/app/import_media.log" + +#python ~/apps/tools/message/mail_msg.py $subject $tmp_log $emails + + diff --git a/bin/master/sh/telecaster_master-crfpa-mp4.sh b/bin/master/sh/telecaster_master-crfpa-mp4.sh new file mode 100755 index 0000000..2cdb406 --- /dev/null +++ b/bin/master/sh/telecaster_master-crfpa-mp4.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +organization=Pre-Barreau +department=CRFPA +year=`date '+%Y'` +media_dir=/home/telecaster/archives/$organization/$department/$year/ +backup_dir=/mnt/nile/$organization/$department/$year/ +tmp_log=/tmp/telecaster-$department.log +remux_log=/home/telecaster/log/remux-$department.log +transcode_log=/home/telecaster/log/transcode-$department.log +import_log=/var/log/app/import_media.log +tmp_dir=/home/telecaster/tmp/ +app_server=malcom.parisson.com +app_path=/srv/crfpa-docker +media_server=malcom.parisson.com +emails="webmaster@parisson.com,jeannot@parisson.com" +subject="TeleCaster remux (CRFPA)" +bwlimit=20000 + +python apps/telecaster/telecaster-server/bin/remux_fix_media.py $media_dir $tmp_dir > $tmp_log + +cat $tmp_log >> $remux_log + +#rsync -uLKPr $media_dir $backup_dir + +#echo `date` >> ~/log/transcode-crfpa.log + +python ~/apps/telecaster/telecaster-server/bin/transcode_nv.py $media_dir $transcode_log + +#find $backup_dir -type d -exec chmod 755 {} \; +#find $backup_dir -type f -exec chmod 644 {} \; + +#rsync -auLKPr --bwlimit=$bwlimit --include="*/" --include="$year/**" --exclude="*" $backup_dir $media_server:$media_dir +#rsync -uLKPr --bwlimit=$bwlimit --include="*/" --include="$year/**" --exclude=".webm" --exclude=".ogg" --exclude="*" $backup_dir $media_server:$media_dir + +rsync -auLKr --bwlimit=$bwlimit --include="*/" --include="$year/**" --exclude="*.webm" --exclude="@eaDir" $media_dir $media_server:$media_dir + +ssh $app_server "docker-compose -f $app_path/docker-compose.yml -f $app_path/env/prod.yml exec -T app /srv/app/manage.py teleforma-import-conferences-2 $organization $department $import_log" + +#python ~/apps/tools/message/mail_msg.py $subject $tmp_log $emails + + diff --git a/bin/master/sh/telecaster_master-pro.sh b/bin/master/sh/telecaster_master-pro.sh new file mode 100755 index 0000000..99f4c73 --- /dev/null +++ b/bin/master/sh/telecaster_master-pro.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +day=`date '+%Y-%m-%d_%H-%M-%S'` +year=`date '+%Y'` +#year=2020 + +script_dir="/home/telecaster/kdenlive/scripts/" +server="malcom.parisson.com" +tmp_dir="/home/telecaster/tmp/" +log_dir="/home/telecaster/log/" + + +for y in `seq $(($year)) $(($year+1))`; do + echo $y + media_dir="/home/telecaster/kdenlive/Final/$y/" + server_media_dir="/mnt/prob-videos/Pre-Barreau/Pro-Barreau/Final/$y/" + archives_dir="/home/telecaster/archives/Pre-Barreau/Pro-Barreau/$y/" + + #bash ~/apps/Telemeta/scripts/kdenlive/mlt_fix_threads.sh $script_dir + #python ~/apps/Telemeta/scripts/kdenlive/mlt_process_batch.py --fading $script_dir >> $log_dir/mlt.log + python ~/apps/Telemeta/scripts/transcode/remux_fix_media.py $archives_dir $tmp_dir >> $log_dir/remux-pro.log + python ~/apps/Telemeta/scripts/transcode/transcode_nv.py $media_dir $log_dir/transcode-pro.log + + chmod -fR 664 $media_dir; chmod -fR +rX $media_dir + rsync -aquLKP --bwlimit=15000 --delete --exclude="@eaDir" $media_dir $server:$server_media_dir + + #ssh $server chmod -R 664 $server_media_dir + #ssh $server chmod -R +rX $server_media_dir + +done -- 2.39.5