import sys
import ConfigParser
import MySQLdb
+import time
from tasks.api import IDataMigrator, IDataInitializer
import tasks
migrators = ExtensionPoint(IDataMigrator)
def run_tasks(self, tasks, type, only_task = None):
- ntasks = 0
+ done = []
task = None
for t in tasks:
if only_task:
task = t
if task:
- print "Runinng %s %s\n" % (task.get_name(), type)
- task.setup(cfg, self.src_db, self.target_db)
+ sys.stdout.write("Runinng %s %s" % (task.get_name(), type))
+ start = time.time()
+ task.setup(self.cfg, self.src_db, self.target_db)
task.process()
- ntasks += 1
+ sys.stdout.write("\t(%.2fs)\n" % (time.time() - start))
+ done.append(task)
if only_task:
break
- return ntasks
+
+ return done
- def run(self, config, only_task = None):
- self.cfg = config
- self.src_db = MySQLdb.connect(user=cfg.get('src', 'user'),
- host=cfg.get('src', 'host'),
- db=cfg.get('src', 'name'),
- passwd=cfg.get('src', 'pass'),
- charset='utf8')
+ def setup(self, inifile):
+ self.cfg = ConfigParser.ConfigParser()
+ self.cfg.read(inifile)
- self.target_db = MySQLdb.connect(user=cfg.get('target', 'user'),
- host=cfg.get('target', 'host'),
- db=cfg.get('target', 'name'),
- passwd=cfg.get('target', 'pass'),
+ self.src_db = MySQLdb.connect(user=self.cfg.get('src', 'user'),
+ host=self.cfg.get('src', 'host'),
+ db=self.cfg.get('src', 'name'),
+ passwd=self.cfg.get('src', 'pass'),
charset='utf8')
+ self.target_db = MySQLdb.connect(user=self.cfg.get('target', 'user'),
+ host=self.cfg.get('target', 'host'),
+ db=self.cfg.get('target', 'name'),
+ passwd=self.cfg.get('target', 'pass'),
+ charset='utf8')
- ntasks = self.run_tasks(self.initializers, "initializer", only_task = only_task)
- ntasks += self.run_tasks(self.migrators, "migrator", only_task = only_task)
+ def run(self, only_task = None):
+ self.done = []
+ self.done.extend(self.run_tasks(self.initializers, "initializer", only_task = only_task))
+ self.done.extend(self.run_tasks(self.migrators, "migrator", only_task = only_task))
- if only_task and not ntasks:
+ if only_task and not len(self.done):
raise "No such task: %s" % only_task
-
-
+
+ def print_stats(self):
+ init = False
+ for task in self.done:
+ if len(task.stats):
+ if not init:
+ print 'Statistics:'
+ init = True
+ print '%s:' % task.get_name()
+ for s in task.stats:
+ print '\t%s:\t%s' % (s, task.stats[s])
+
if __name__ == '__main__':
if len(sys.argv) != 2 and len(sys.argv) != 3:
else:
only_task = None
- cfg = ConfigParser.ConfigParser()
- cfg.read(sys.argv[1])
-
cmpmgr = ComponentManager()
migrator = Migrator(cmpmgr)
- migrator.run(cfg, only_task)
-
+ migrator.setup(sys.argv[1])
+ migrator.run(only_task)
+ migrator.print_stats()
--- /dev/null
+import reset
+import enums
+import geoethno
+import ethnic
+import publishers
+import sys
+from telemeta.core import *
+
+class DataMigrationTask(Component):
+
+ def setup(self, cfg, src_db, target_db):
+ self.target_db = target_db
+ self.target_cursor = target_db.cursor()
+ self.src_db = src_db
+ self.src_cursor = src_db.cursor()
+ self.src_db_name = cfg.get('src', 'name')
+ self.target_db_name = cfg.get('target', 'name')
+ self.stats = {}
+
+ def step(self):
+ sys.stdout.write('.')
+ sys.stdout.flush()
class GroupedItemsManager(object):
from telemeta.core import *
from api import IDataMigrator
+from core import DataMigrationTask
-class SimpleEnumMigrator(Component):
+class SimpleEnumMigrator(DataMigrationTask):
implements(IDataMigrator)
'FormStyl generi':'generic_styles'
}
- def setup(self, cfg, src_db, target_db):
- self.target_cursor = target_db.cursor()
- self.src_db_name = cfg.get('src', 'name')
-
def get_name(self):
return "enumerations"
-
def process(self):
for src in self.map:
from telemeta.core import *
from api import IDataMigrator
-from core import GroupedItemsManager
+from core import GroupedItemsManager, DataMigrationTask
-class EthnicGroupsMigrator(Component):
+class EthnicGroupsMigrator(DataMigrationTask):
implements(IDataMigrator)
def setup(self, cfg, src_db, target_db):
- self.target_db = target_db
- self.target_cursor = target_db.cursor()
- self.src_cursor = src_db.cursor()
+ DataMigrationTask.setup(self, cfg, src_db, target_db)
self.data = GroupedItemsManager()
def get_name(self):
def process(self):
self.extract()
self.insert()
- print "Ethnic groups/aliases: %d/%d\n" % (len(self.data.groups), self.data.count_items())
+ self.stats = {
+ 'groups': len(self.data.groups),
+ 'aliases': self.data.count_items()
+ }
import sys
import xml.dom.minidom as dom
from api import IDataMigrator
+from core import DataMigrationTask
-class GeoEthnoImporter(Component):
+class GeoEthnoImporter(DataMigrationTask):
implements(IDataMigrator)
nhistoric_names = 0
def setup(self, cfg, src_db, target_db):
- self.db = target_db
- self.cursor = self.db.cursor()
+ DataMigrationTask.setup(self, cfg, src_db, target_db)
+ self.cursor = self.target_cursor
self.dom = dom.parse(cfg.get('geoethno', 'xml_file'))
self.known_types = []
self.nhistoric_names += 1
- if self.nlocations % 1000 == 0:
- sys.stdout.write('.')
- sys.stdout.flush()
+# if self.nlocations % 1000 == 0:
+# self.step()
def add_aliases(self, name, items):
for alias in items:
self.add_aliases(name, self.flatten_node_list(aliasNodes))
self.process_children(n, name)
- self.path.pop()
+ self.path.pop()
+ if len(self.path) == 1:
+ self.step()
def process(self):
self.path = []
self.process_children(self.dom.getElementsByTagName('GEOETHNO')[0], '')
- sys.stdout.write('\nGeoethno import result:\n')
- sys.stdout.write(' types: %d\n' % len(self.known_types))
- sys.stdout.write(' locations: %d\n' % self.nlocations)
- sys.stdout.write(' relations: %d\n' % self.nrelations)
- sys.stdout.write(' aliases: %d\n' % self.naliases)
- sys.stdout.write(' historical names: %d\n' % self.nhistoric_names)
-
+ self.stats = {
+ 'types': len(self.known_types),
+ 'locations': self.nlocations,
+ 'relations': self.nrelations,
+ 'aliases': self.naliases,
+ 'historical names': self.nhistoric_names
+ }
def warn(self, msg):
print u"\nWarning: %s: %s\n" % ("/".join(self.path), msg)
from telemeta.core import *
from api import IDataMigrator
-from core import GroupedItemsManager
+from core import GroupedItemsManager, DataMigrationTask
-class PublishersMigrator(Component):
+class PublishersMigrator(DataMigrationTask):
groups = {}
implements(IDataMigrator)
def setup(self, cfg, src_db, target_db):
- self.target_db = target_db
- self.target_cursor = target_db.cursor()
- self.src_cursor = src_db.cursor()
+ DataMigrationTask.setup(self, cfg, src_db, target_db)
self.data = GroupedItemsManager()
def get_name(self):
def process(self):
self.extract()
self.insert()
- print "Publishers/Collections: %d/%d\n" % (len(self.data.groups), self.data.count_items())
+ self.stats = {
+ 'publishers': len(self.data.groups),
+ 'collections': self.data.count_items()
+ }
from telemeta.core import *
from api import IDataInitializer
+from core import DataMigrationTask
-class DatabaseResetMigrator(Component):
+class DatabaseResetMigrator(DataMigrationTask):
implements(IDataInitializer)
'revisions'
]
- def setup(self, cfg, src_db, target_db):
- self.target_cursor = target_db.cursor()
-
def get_name(self):
return "reset"
projects / telemeta-data.git / commitdiff