From: Guillaume Pellerin <yomguy@parisson.com>
Date: Wed, 15 Feb 2012 13:43:39 +0000 (+0100)
Subject: add migration tools
X-Git-Url: https://git.parisson.com/?a=commitdiff_plain;h=cc511e9ebe81aa37f73b4fef4ea4b8c967681215;p=tools.git

add migration tools
---

diff --git a/deegger/deegger.py b/deegger/deegger.py
new file mode 100755
index 0000000..1b113ad
--- /dev/null
+++ b/deegger/deegger.py
@@ -0,0 +1,236 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright Guillaume Pellerin, 2009
+
+# <yomguy@parisson.com>
+
+# This software is a computer program whose purpose is to find cool music
+# samples over internet with the help of google search ajax api.
+
+# This software is governed by the CeCILL  license under French law and
+# abiding by the rules of distribution of free software.  You can  use,
+# modify and/ or redistribute the software under the terms of the CeCILL
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+
+# As a counterpart to the access to the source code and  rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty  and the software's author,  the holder of the
+# economic rights,  and the successive licensors  have only  limited
+# liability.
+
+# In this respect, the user's attention is drawn to the risks associated
+# with loading,  using,  modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean  that it is complicated to manipulate,  and  that  also
+# therefore means  that it is reserved for developers  and  experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and,  more generally, to use and operate it in the
+# same conditions as regards security.
+
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL license and that you accept its terms.
+
+# Author: Guillaume Pellerin <yomguy@parisson.com>
+
+import os
+import re
+import sys
+import urllib
+import simplejson
+import Queue
+from threading import Thread
+from optparse import OptionParser
+
+version = '0.3'
+
+def prog_info():
+    return """ deegger : easy media crawler through google search api
+
+ Version:
+    %s
+    
+ Depends on:
+    python, python-simplejson
+
+ Usage:
+    $ ./deegger.py FORMAT TEXT M3U_DIR
+
+ Where:
+    FORMAT is the media type you are looking for
+    TEXT is your google text query
+    M3U_DIR an output M3U playlist directory
+
+ For example:
+    ./deegger.py wav "sample" /var/www/m3u
+
+ Author:
+    Guillaume Pellerin <yomguy@parisson.com>
+
+ License:
+    CeCILL v2
+ """ % version
+
+
+class DeeGGer(object):
+
+    def __init__(self, options, args):
+        self.format = options.format
+        self.text = options.text
+        self.m3u_dir = options.output
+        if not os.path.exists(self.m3u_dir):
+            os.makedirs(self.m3u_dir)
+        self.m3u_file = self.m3u_dir + os.sep + 'deegger_' + self.text.replace('/', '_') + '.' + self.format + '.m3u'
+        self.range = 4
+        self.servers = []
+        
+        self.media_q = 'intitle:"index.of" "parent directory" "size" "last modified" "description" [snd] (%s) -inurl:(jsp|php|html|aspx|htm|cf|shtml|lyrics|index|%s|%ss) -gallery -intitle:"last modified"' % (self.format, self.format, self.format)
+        #self.media_q = 'intitle:"index.of" [snd] (%s) -inurl:(jsp|php|html|aspx|htm|cf|shtml|lyrics|index|%s|%ss) -gallery' % (self.format, self.format, self.format)
+        
+        self.query = '%s %s' % (self.text, self.media_q)
+        self.q = Queue.Queue(1)
+        self.results = Queue.Queue(1)
+
+    def run(self):
+        g = GoogleSearch(self.range, self.query)
+        self.results = g.search()
+#        print self.results
+        for result in self.results:
+            url = result['url']
+            s = UrlMediaParser(self.format, self.text, url, self.q)
+            s.start()
+        
+        self.m3u = M3UPlaylist(self.q, self.m3u_file)
+        self.m3u.start()
+        
+        self.q.join()
+        self.m3u.close()
+        
+class Producer(Thread):
+    """a Producer master thread"""
+
+    def __init__(self, q):
+        Thread.__init__(self)
+        self.q = q
+
+    def run(self):
+        i=0
+        q = self.q
+        while True:
+            q.put(i,1)
+            i+=1
+            
+class GoogleSearch(object):
+
+    def __init__(self, range, query):
+        self.range = range
+        self.query = query
+        
+    def search(self):
+        results = []
+        for j in range(0, self.range):
+            page = str(j)
+            query = urllib.urlencode({'q' : self.query, 'start': page})
+            url = 'http://ajax.googleapis.com/ajax/services/search/web?v=1.0&%s' % (query)
+            json = simplejson.loads(urllib.urlopen(url).read())
+            if json['responseData']:
+                for r in json['responseData']['results']:
+                    results.append(r)
+        return results
+
+
+class UrlMediaParser(Thread):
+
+    def __init__(self, format, text, url, q):
+        Thread.__init__(self)
+        self.format = format
+        self.text = text
+        self.url = url
+        self.q = q
+
+    def is_in_multiple_case(self, _string, text):
+        return _string in text \
+                or _string.upper() in text \
+                or _string.lower() in text \
+                or _string.capitalize() in text
+
+    def get_multiple_case_string(self, _string):
+        return _string.upper(), _string.lower() , _string.capitalize()
+
+
+    def run(self):
+        q = self.q
+        media_list = []
+        if self.url:
+            print 'deegging : ' + self.url
+            try:
+                data = urllib.urlopen(self.url).read()
+                for line in data.split("\012"):
+                    s = re.compile('href=".*\.'+ format + '"').search(line,1)
+                    if s:
+                        file_name = line[s.start():s.end()].split('"')[1]
+                        if self.is_in_multiple_case(self.text, file_name):
+                            q.put(self.url + '/' + file_name)
+            
+            except:
+                pass
+
+
+class M3UPlaylist(Thread):
+
+    def __init__(self, q, m3u_file):
+        Thread.__init__(self)
+        self.q = q
+        self.m3u_file = m3u_file
+        self.m3u = open(self.m3u_file, 'w')
+        self.m3u.write('#EXTM3U\n')
+        #self.m3u.flush()
+
+    def run(self):
+        url = self.q.get()
+        print 'adding : ' + url
+        info = '#EXTINF:'',%s' % (url +'\n')
+        self.m3u.write(info)
+        self.m3u.write(url + '\n')
+        #self.m3u.flush()
+
+    def close(self):
+        self.m3u.close()
+
+
+class Logger:
+    """A logging object"""
+    
+    def __init__(self, file):
+        import logging
+        self.logger = logging.getLogger('myapp')
+        self.hdlr = logging.FileHandler(file)
+        self.formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
+        self.hdlr.setFormatter(self.formatter)
+        self.logger.addHandler(self.hdlr)
+        self.logger.setLevel(logging.INFO)
+
+    def write(self, message):
+        self.logger.info(message)
+
+
+def main():
+    if len(sys.argv) <= 2:
+        text = prog_info()
+        sys.exit(text)
+    
+    else:
+        parser = OptionParser()
+        parser.add_option("-t", "--text", dest="text", help="set the TEXT google query", metavar="TEXT")
+        parser.add_option("-f", "--format", dest="format", help="set the format to search for" , metavar="FORMAT")
+        parser.add_option("-o", "--output", dest="output", help="set the output directory" , metavar="OUTPUT")
+        (options, args) = parser.parse_args()
+        
+        d = DeeGGer(options, args)
+        d.run()
+        
+if __name__ == '__main__':
+    main()
diff --git a/deegger/deegger_url.py b/deegger/deegger_url.py
new file mode 100755
index 0000000..066795d
--- /dev/null
+++ b/deegger/deegger_url.py
@@ -0,0 +1,14 @@
+#!/usr/bin/python
+
+import sys
+from deegger import *
+
+format = sys.argv[1]
+text = sys.argv[2]
+url = sys.argv[3]
+m3u_dir = sys.argv[4]
+m3u = M3UPlaylist(m3u_dir + os.sep + 'deegger_' + url[7:].replace('/', '_') + '.m3u')
+
+u = UrlMediaParser(format, text, {'unescapedUrl': url}, m3u)
+u.start()
+    
\ No newline at end of file
diff --git a/server-migration/migrate-apache b/server-migration/migrate-apache
new file mode 100755
index 0000000..008e4c6
--- /dev/null
+++ b/server-migration/migrate-apache
@@ -0,0 +1,15 @@
+#!/usr/bin/python
+
+import os, sys
+
+dir_old = '/home/pbackup/pypix/etc/apache2/sites-available/'
+dir_new = '/etc/apache2/sites-available/'
+items_old = os.listdir(dir_old)
+items_new = os.listdir(dir_new)
+
+for item in items_old:
+    if item in items_new:
+    	command = 'sudo diff '
+	args = dir_old+item+' '+dir_new+item
+    	print command+args
+	os.system(command+args)
diff --git a/server-migration/migrate-example b/server-migration/migrate-example
new file mode 100755
index 0000000..634af64
--- /dev/null
+++ b/server-migration/migrate-example
@@ -0,0 +1,3 @@
+# migrate-shadow my-redhat-shadow-useronly-list > shadow-filter.awk
+
+# awk -f shadow-filter.awk /etc/shadow > my-new-shadow-file-with-updated-pass
\ No newline at end of file
diff --git a/server-migration/migrate-home b/server-migration/migrate-home
new file mode 100755
index 0000000..d4b1ea2
--- /dev/null
+++ b/server-migration/migrate-home
@@ -0,0 +1,13 @@
+#!/usr/bin/python
+
+import os, sys
+
+home_old = '/home/pbackup/pypix/home/'
+home_new = '/home/'
+homes = os.listdir(home_old)
+
+for home in homes:
+    if not home == 'momo':
+    	command = 'sudo cp -ra '+home_old+home+' '+home_new
+    	print command
+	os.system(command)
diff --git a/server-migration/migrate-logs b/server-migration/migrate-logs
new file mode 100755
index 0000000..d4b1ea2
--- /dev/null
+++ b/server-migration/migrate-logs
@@ -0,0 +1,13 @@
+#!/usr/bin/python
+
+import os, sys
+
+home_old = '/home/pbackup/pypix/home/'
+home_new = '/home/'
+homes = os.listdir(home_old)
+
+for home in homes:
+    if not home == 'momo':
+    	command = 'sudo cp -ra '+home_old+home+' '+home_new
+    	print command
+	os.system(command)
diff --git a/server-migration/migrate-passwd b/server-migration/migrate-passwd
new file mode 100755
index 0000000..c7f5168
--- /dev/null
+++ b/server-migration/migrate-passwd
@@ -0,0 +1,17 @@
+#!/bin/bash
+# 
+# usage:   migrate-passwd password-file>
+#
+# remarks: the password file must be a copy of passwd that contains
+#          only the entries to be migrated.
+#          
+
+[ -f $1 ] || exit 1
+
+userlist=$(cat $1)
+
+for entry in $userlist ; do
+        username=$(echo $entry | cut -d ':' -f 1)
+        gecos=$(echo $entry | cut -d ':' -f 5)
+        adduser --force-badname --disabled-password --gecos "$gecos" "$username" > /dev/null
+done
diff --git a/server-migration/migrate-shadow b/server-migration/migrate-shadow
new file mode 100755
index 0000000..e7b0409
--- /dev/null
+++ b/server-migration/migrate-shadow
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# usage: migrate-shadow shadow-like-file>
+#
+# remarks: the shadow password file must be a copy of passwd that contains
+#          only the entries to be migrated.
+#
+
+[ -f $1 ] || exit 1
+
+userlist=$(cat $1)
+elsebranch=
+
+echo -e "BEGIN { FS=\":\" }\n{"
+
+for entry in $userlist ; do
+        username=$(echo $entry | cut -d ':' -f 1)
+        encpassword=$(echo $entry | cut -d ':' -f 2)
+        echo "${elsebranch}if ( \$0 ~ /^$username:/ ) \
+                printf \"%s:%s:%s:%s:%s:%s:%s:%s:%s\n\",\$1, \
+                \"$encpassword\", \$3,\$4,\$5,\$6,\$7,\$8,\$9;"
+        elsebranch="else "
+done
+
+echo "else print \$0;"
+echo -e "}\nEND {}"
\ No newline at end of file
diff --git a/server-migration/migrate.py b/server-migration/migrate.py
new file mode 100755
index 0000000..819b2ef
--- /dev/null
+++ b/server-migration/migrate.py
@@ -0,0 +1,41 @@
+#!/usr/bin/python
+
+import os
+ 
+
+class Migration(object):
+
+    def __init__(self, name, app, args=None):
+        self.name = name
+        self.app = app
+        self.args = args
+
+    def command(self):
+        return self.app + ' ' + self.args
+
+    def run(self):
+        os.system(self.command)
+
+
+class Migrator(object):
+
+    def __init__(self, in_host, out_host):
+        self.migrations = []
+        self.in_host = in_host
+        self.out_host = out_host
+
+    def add(self, migration):
+        self.migrations.append(migration['name'], migration['app'], migration['args'])
+
+    def run(self):
+        for migration in self.migrations:
+            migration.run()
+
+
+
+migrations = [
+    {'app': 'rsync', 'args': '-aA /etc/ momo@angus.parisson.com:/home/momo/backups/ns206309/etc/'},
+    {'app': 'rsync', 'args': '-aA /home/momo/backups/mysql/daily/ momo@angus:~/backups/ns206309/mysql/'},
+    
+    ]
+    
\ No newline at end of file
diff --git a/various/clean_user.sh b/various/clean_user.sh
index 1bb8948..12febe5 100755
--- a/various/clean_user.sh
+++ b/various/clean_user.sh
@@ -1,4 +1,4 @@
-#!bin/sh
+#!/bin/sh
 
 u=$1
 sudo deluser $u
diff --git a/various/deegger.py b/various/deegger.py
deleted file mode 100755
index 1b113ad..0000000
--- a/various/deegger.py
+++ /dev/null
@@ -1,236 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-# Copyright Guillaume Pellerin, 2009
-
-# <yomguy@parisson.com>
-
-# This software is a computer program whose purpose is to find cool music
-# samples over internet with the help of google search ajax api.
-
-# This software is governed by the CeCILL  license under French law and
-# abiding by the rules of distribution of free software.  You can  use,
-# modify and/ or redistribute the software under the terms of the CeCILL
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-
-# As a counterpart to the access to the source code and  rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty  and the software's author,  the holder of the
-# economic rights,  and the successive licensors  have only  limited
-# liability.
-
-# In this respect, the user's attention is drawn to the risks associated
-# with loading,  using,  modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean  that it is complicated to manipulate,  and  that  also
-# therefore means  that it is reserved for developers  and  experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and,  more generally, to use and operate it in the
-# same conditions as regards security.
-
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL license and that you accept its terms.
-
-# Author: Guillaume Pellerin <yomguy@parisson.com>
-
-import os
-import re
-import sys
-import urllib
-import simplejson
-import Queue
-from threading import Thread
-from optparse import OptionParser
-
-version = '0.3'
-
-def prog_info():
-    return """ deegger : easy media crawler through google search api
-
- Version:
-    %s
-    
- Depends on:
-    python, python-simplejson
-
- Usage:
-    $ ./deegger.py FORMAT TEXT M3U_DIR
-
- Where:
-    FORMAT is the media type you are looking for
-    TEXT is your google text query
-    M3U_DIR an output M3U playlist directory
-
- For example:
-    ./deegger.py wav "sample" /var/www/m3u
-
- Author:
-    Guillaume Pellerin <yomguy@parisson.com>
-
- License:
-    CeCILL v2
- """ % version
-
-
-class DeeGGer(object):
-
-    def __init__(self, options, args):
-        self.format = options.format
-        self.text = options.text
-        self.m3u_dir = options.output
-        if not os.path.exists(self.m3u_dir):
-            os.makedirs(self.m3u_dir)
-        self.m3u_file = self.m3u_dir + os.sep + 'deegger_' + self.text.replace('/', '_') + '.' + self.format + '.m3u'
-        self.range = 4
-        self.servers = []
-        
-        self.media_q = 'intitle:"index.of" "parent directory" "size" "last modified" "description" [snd] (%s) -inurl:(jsp|php|html|aspx|htm|cf|shtml|lyrics|index|%s|%ss) -gallery -intitle:"last modified"' % (self.format, self.format, self.format)
-        #self.media_q = 'intitle:"index.of" [snd] (%s) -inurl:(jsp|php|html|aspx|htm|cf|shtml|lyrics|index|%s|%ss) -gallery' % (self.format, self.format, self.format)
-        
-        self.query = '%s %s' % (self.text, self.media_q)
-        self.q = Queue.Queue(1)
-        self.results = Queue.Queue(1)
-
-    def run(self):
-        g = GoogleSearch(self.range, self.query)
-        self.results = g.search()
-#        print self.results
-        for result in self.results:
-            url = result['url']
-            s = UrlMediaParser(self.format, self.text, url, self.q)
-            s.start()
-        
-        self.m3u = M3UPlaylist(self.q, self.m3u_file)
-        self.m3u.start()
-        
-        self.q.join()
-        self.m3u.close()
-        
-class Producer(Thread):
-    """a Producer master thread"""
-
-    def __init__(self, q):
-        Thread.__init__(self)
-        self.q = q
-
-    def run(self):
-        i=0
-        q = self.q
-        while True:
-            q.put(i,1)
-            i+=1
-            
-class GoogleSearch(object):
-
-    def __init__(self, range, query):
-        self.range = range
-        self.query = query
-        
-    def search(self):
-        results = []
-        for j in range(0, self.range):
-            page = str(j)
-            query = urllib.urlencode({'q' : self.query, 'start': page})
-            url = 'http://ajax.googleapis.com/ajax/services/search/web?v=1.0&%s' % (query)
-            json = simplejson.loads(urllib.urlopen(url).read())
-            if json['responseData']:
-                for r in json['responseData']['results']:
-                    results.append(r)
-        return results
-
-
-class UrlMediaParser(Thread):
-
-    def __init__(self, format, text, url, q):
-        Thread.__init__(self)
-        self.format = format
-        self.text = text
-        self.url = url
-        self.q = q
-
-    def is_in_multiple_case(self, _string, text):
-        return _string in text \
-                or _string.upper() in text \
-                or _string.lower() in text \
-                or _string.capitalize() in text
-
-    def get_multiple_case_string(self, _string):
-        return _string.upper(), _string.lower() , _string.capitalize()
-
-
-    def run(self):
-        q = self.q
-        media_list = []
-        if self.url:
-            print 'deegging : ' + self.url
-            try:
-                data = urllib.urlopen(self.url).read()
-                for line in data.split("\012"):
-                    s = re.compile('href=".*\.'+ format + '"').search(line,1)
-                    if s:
-                        file_name = line[s.start():s.end()].split('"')[1]
-                        if self.is_in_multiple_case(self.text, file_name):
-                            q.put(self.url + '/' + file_name)
-            
-            except:
-                pass
-
-
-class M3UPlaylist(Thread):
-
-    def __init__(self, q, m3u_file):
-        Thread.__init__(self)
-        self.q = q
-        self.m3u_file = m3u_file
-        self.m3u = open(self.m3u_file, 'w')
-        self.m3u.write('#EXTM3U\n')
-        #self.m3u.flush()
-
-    def run(self):
-        url = self.q.get()
-        print 'adding : ' + url
-        info = '#EXTINF:'',%s' % (url +'\n')
-        self.m3u.write(info)
-        self.m3u.write(url + '\n')
-        #self.m3u.flush()
-
-    def close(self):
-        self.m3u.close()
-
-
-class Logger:
-    """A logging object"""
-    
-    def __init__(self, file):
-        import logging
-        self.logger = logging.getLogger('myapp')
-        self.hdlr = logging.FileHandler(file)
-        self.formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
-        self.hdlr.setFormatter(self.formatter)
-        self.logger.addHandler(self.hdlr)
-        self.logger.setLevel(logging.INFO)
-
-    def write(self, message):
-        self.logger.info(message)
-
-
-def main():
-    if len(sys.argv) <= 2:
-        text = prog_info()
-        sys.exit(text)
-    
-    else:
-        parser = OptionParser()
-        parser.add_option("-t", "--text", dest="text", help="set the TEXT google query", metavar="TEXT")
-        parser.add_option("-f", "--format", dest="format", help="set the format to search for" , metavar="FORMAT")
-        parser.add_option("-o", "--output", dest="output", help="set the output directory" , metavar="OUTPUT")
-        (options, args) = parser.parse_args()
-        
-        d = DeeGGer(options, args)
-        d.run()
-        
-if __name__ == '__main__':
-    main()
diff --git a/various/deegger_url.py b/various/deegger_url.py
deleted file mode 100755
index 066795d..0000000
--- a/various/deegger_url.py
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/usr/bin/python
-
-import sys
-from deegger import *
-
-format = sys.argv[1]
-text = sys.argv[2]
-url = sys.argv[3]
-m3u_dir = sys.argv[4]
-m3u = M3UPlaylist(m3u_dir + os.sep + 'deegger_' + url[7:].replace('/', '_') + '.m3u')
-
-u = UrlMediaParser(format, text, {'unescapedUrl': url}, m3u)
-u.start()
-    
\ No newline at end of file