From dbd57fee5ee7f6d76af8a52259003c0aa7839068 Mon Sep 17 00:00:00 2001 From: olivier Date: Tue, 21 Apr 2009 13:07:19 +0000 Subject: [PATCH] add task listing + cosmetic consolidation and license headers git-svn-id: http://svn.parisson.org/svn/crem@78 3bf09e05-f825-4182-b9bc-eedd7160adf0 --- trunk/import/migration/migrate.py | 89 +++++++++++++++++----- trunk/import/migration/tasks/__init__.py | 33 ++++++++ trunk/import/migration/tasks/api.py | 41 ++++++++-- trunk/import/migration/tasks/core.py | 39 ++++++++++ trunk/import/migration/tasks/enums.py | 56 +++++++++++--- trunk/import/migration/tasks/ethnic.py | 38 ++++++++- trunk/import/migration/tasks/geoethno.py | 48 ++++++++++-- trunk/import/migration/tasks/publishers.py | 39 +++++++++- trunk/import/migration/tasks/reset.py | 38 ++++++++- 9 files changed, 371 insertions(+), 50 deletions(-) diff --git a/trunk/import/migration/migrate.py b/trunk/import/migration/migrate.py index f16569f..254bd05 100644 --- a/trunk/import/migration/migrate.py +++ b/trunk/import/migration/migrate.py @@ -1,3 +1,36 @@ +# -*- coding: utf-8 -*- +# +# CREM Database migrator + +# Copyright (C) 2009 Samalyse SARL +# Author: Olivier Guilyardi + +# This software is governed by the CeCILL license under French law and +# abiding by the rules of distribution of free software. You can use, +# modify and/ or redistribute the software under the terms of the CeCILL +# license as circulated by CEA, CNRS and INRIA at the following URL +# "http://www.cecill.info". + +# As a counterpart to the access to the source code and rights to copy, +# modify and redistribute granted by the license, users are provided only +# with a limited warranty and the software's author, the holder of the +# economic rights, and the successive licensors have only limited +# liability. + +# In this respect, the user's attention is drawn to the risks associated +# with loading, using, modifying and/or developing or reproducing the +# software by the user in light of its specific status of free software, +# that may mean that it is complicated to manipulate, and that also +# therefore means that it is reserved for developers and experienced +# professionals having in-depth computer knowledge. Users are therefore +# encouraged to load and test the software's suitability as regards their +# requirements in conditions enabling the security of their systems and/or +# data to be ensured and, more generally, to use and operate it in the +# same conditions as regards security. + +# The fact that you are presently reading this means that you have had +# knowledge of the CeCILL license and that you accept its terms. + from telemeta.core import * from telemeta.core import ComponentManager import sys @@ -8,11 +41,16 @@ import time from tasks.api import IDataMigrator, IDataInitializer import tasks -class Migrator(Component): +class MigrationManager(Component): + """Establish databases connections, initialize and run migration tasks""" + initializers = ExtensionPoint(IDataInitializer) migrators = ExtensionPoint(IDataMigrator) def run_tasks(self, tasks, type, only_task = None): + """Run a set of migration tasks of a given type ('initializer', 'migrator', + ...), or the single task idenfied by its name with only_task""" + done = [] task = None for t in tasks: @@ -34,24 +72,28 @@ class Migrator(Component): return done - def setup(self, inifile): + """Setup using the provided ini file""" + self.cfg = ConfigParser.ConfigParser() self.cfg.read(inifile) - self.src_db = MySQLdb.connect(user=self.cfg.get('src', 'user'), - host=self.cfg.get('src', 'host'), - db=self.cfg.get('src', 'name'), - passwd=self.cfg.get('src', 'pass'), - charset='utf8') + self.src_db = MySQLdb.connect( + user = self.cfg.get('src', 'user'), + host = self.cfg.get('src', 'host'), + db = self.cfg.get('src', 'name'), + passwd = self.cfg.get('src', 'pass'), + charset = 'utf8') - self.target_db = MySQLdb.connect(user=self.cfg.get('target', 'user'), - host=self.cfg.get('target', 'host'), - db=self.cfg.get('target', 'name'), - passwd=self.cfg.get('target', 'pass'), - charset='utf8') + self.target_db = MySQLdb.connect( + user = self.cfg.get('target', 'user'), + host = self.cfg.get('target', 'host'), + db = self.cfg.get('target', 'name'), + passwd = self.cfg.get('target', 'pass'), + charset = 'utf8') def run(self, only_task = None): + """Run all tasks or a single one identified by its name with only_task""" self.done = [] self.done.extend(self.run_tasks(self.initializers, "initializer", only_task = only_task)) self.done.extend(self.run_tasks(self.migrators, "migrator", only_task = only_task)) @@ -59,7 +101,16 @@ class Migrator(Component): if only_task and not len(self.done): raise "No such task: %s" % only_task + def list_tasks(self): + """Generator listing available tasks as DataMigrationTask instances""" + for task in self.initializers: + yield task + + for task in self.migrators: + yield task + def print_stats(self): + """Print all stats, retrieved from the previously run tasks""" init = False for task in self.done: if len(task.stats): @@ -72,8 +123,14 @@ class Migrator(Component): if __name__ == '__main__': + manager = MigrationManager(ComponentManager()) + if len(sys.argv) != 2 and len(sys.argv) != 3: print "Usage: %s [task_name]" % sys.argv[0] + print "Tasks:" + for task in manager.list_tasks(): + print " %-16s%s" % (task.get_name() + ':', task.__doc__) + sys.exit(1) if (len(sys.argv) == 3): @@ -81,9 +138,7 @@ if __name__ == '__main__': else: only_task = None - cmpmgr = ComponentManager() - migrator = Migrator(cmpmgr) - migrator.setup(sys.argv[1]) - migrator.run(only_task) - migrator.print_stats() + manager.setup(sys.argv[1]) + manager.run(only_task) + manager.print_stats() diff --git a/trunk/import/migration/tasks/__init__.py b/trunk/import/migration/tasks/__init__.py index af6b6d9..337a207 100644 --- a/trunk/import/migration/tasks/__init__.py +++ b/trunk/import/migration/tasks/__init__.py @@ -1,3 +1,36 @@ +# -*- coding: utf-8 -*- +# +# CREM Database migrator + +# Copyright (C) 2009 Samalyse SARL +# Author: Olivier Guilyardi + +# This software is governed by the CeCILL license under French law and +# abiding by the rules of distribution of free software. You can use, +# modify and/ or redistribute the software under the terms of the CeCILL +# license as circulated by CEA, CNRS and INRIA at the following URL +# "http://www.cecill.info". + +# As a counterpart to the access to the source code and rights to copy, +# modify and redistribute granted by the license, users are provided only +# with a limited warranty and the software's author, the holder of the +# economic rights, and the successive licensors have only limited +# liability. + +# In this respect, the user's attention is drawn to the risks associated +# with loading, using, modifying and/or developing or reproducing the +# software by the user in light of its specific status of free software, +# that may mean that it is complicated to manipulate, and that also +# therefore means that it is reserved for developers and experienced +# professionals having in-depth computer knowledge. Users are therefore +# encouraged to load and test the software's suitability as regards their +# requirements in conditions enabling the security of their systems and/or +# data to be ensured and, more generally, to use and operate it in the +# same conditions as regards security. + +# The fact that you are presently reading this means that you have had +# knowledge of the CeCILL license and that you accept its terms. + import reset import enums import geoethno diff --git a/trunk/import/migration/tasks/api.py b/trunk/import/migration/tasks/api.py index 1504243..32b370f 100644 --- a/trunk/import/migration/tasks/api.py +++ b/trunk/import/migration/tasks/api.py @@ -1,15 +1,40 @@ -# Copyright (C) 2007 Samalyse SARL -# All rights reserved. +# -*- coding: utf-8 -*- # -# This software is licensed as described in the file COPYING, which -# you should have received as part of this distribution. The terms -# are also available at http://svn.parisson.org/telemeta/TelemetaLicense. -# -# Author: Olivier Guilyardi +# CREM Database migrator + +# Copyright (C) 2009 Samalyse SARL +# Author: Olivier Guilyardi + +# This software is governed by the CeCILL license under French law and +# abiding by the rules of distribution of free software. You can use, +# modify and/ or redistribute the software under the terms of the CeCILL +# license as circulated by CEA, CNRS and INRIA at the following URL +# "http://www.cecill.info". + +# As a counterpart to the access to the source code and rights to copy, +# modify and redistribute granted by the license, users are provided only +# with a limited warranty and the software's author, the holder of the +# economic rights, and the successive licensors have only limited +# liability. + +# In this respect, the user's attention is drawn to the risks associated +# with loading, using, modifying and/or developing or reproducing the +# software by the user in light of its specific status of free software, +# that may mean that it is complicated to manipulate, and that also +# therefore means that it is reserved for developers and experienced +# professionals having in-depth computer knowledge. Users are therefore +# encouraged to load and test the software's suitability as regards their +# requirements in conditions enabling the security of their systems and/or +# data to be ensured and, more generally, to use and operate it in the +# same conditions as regards security. + +# The fact that you are presently reading this means that you have had +# knowledge of the CeCILL license and that you accept its terms. from telemeta.core import * class IDataMigrationTask(Interface): + """Migration tasks interface""" def setup(cfg, src_db, target_db): """Set the migrator up""" @@ -23,9 +48,11 @@ class IDataMigrationTask(Interface): class IDataInitializer(IDataMigrationTask): + """Initialization migration tasks interface""" pass class IDataMigrator(IDataMigrationTask): + """Data migrating tasks interface""" pass diff --git a/trunk/import/migration/tasks/core.py b/trunk/import/migration/tasks/core.py index 481ab20..d9f1773 100644 --- a/trunk/import/migration/tasks/core.py +++ b/trunk/import/migration/tasks/core.py @@ -1,3 +1,36 @@ +# -*- coding: utf-8 -*- +# +# CREM Database migrator + +# Copyright (C) 2009 Samalyse SARL +# Author: Olivier Guilyardi + +# This software is governed by the CeCILL license under French law and +# abiding by the rules of distribution of free software. You can use, +# modify and/ or redistribute the software under the terms of the CeCILL +# license as circulated by CEA, CNRS and INRIA at the following URL +# "http://www.cecill.info". + +# As a counterpart to the access to the source code and rights to copy, +# modify and redistribute granted by the license, users are provided only +# with a limited warranty and the software's author, the holder of the +# economic rights, and the successive licensors have only limited +# liability. + +# In this respect, the user's attention is drawn to the risks associated +# with loading, using, modifying and/or developing or reproducing the +# software by the user in light of its specific status of free software, +# that may mean that it is complicated to manipulate, and that also +# therefore means that it is reserved for developers and experienced +# professionals having in-depth computer knowledge. Users are therefore +# encouraged to load and test the software's suitability as regards their +# requirements in conditions enabling the security of their systems and/or +# data to be ensured and, more generally, to use and operate it in the +# same conditions as regards security. + +# The fact that you are presently reading this means that you have had +# knowledge of the CeCILL license and that you accept its terms. + import sys from telemeta.core import * @@ -16,6 +49,12 @@ class DataMigrationTask(Component): sys.stdout.write('.') sys.stdout.flush() +class DataMigrator(DataMigrationTask): + pass + +class DataInitializer(DataMigrationTask): + pass + class GroupedItemsManager(object): def __init__(self): diff --git a/trunk/import/migration/tasks/enums.py b/trunk/import/migration/tasks/enums.py index 6cf32b7..ce937c1 100644 --- a/trunk/import/migration/tasks/enums.py +++ b/trunk/import/migration/tasks/enums.py @@ -1,24 +1,56 @@ # -*- coding: utf-8 -*- +# +# CREM Database migrator + +# Copyright (C) 2009 Samalyse SARL +# Author: Olivier Guilyardi + +# This software is governed by the CeCILL license under French law and +# abiding by the rules of distribution of free software. You can use, +# modify and/ or redistribute the software under the terms of the CeCILL +# license as circulated by CEA, CNRS and INRIA at the following URL +# "http://www.cecill.info". + +# As a counterpart to the access to the source code and rights to copy, +# modify and redistribute granted by the license, users are provided only +# with a limited warranty and the software's author, the holder of the +# economic rights, and the successive licensors have only limited +# liability. + +# In this respect, the user's attention is drawn to the risks associated +# with loading, using, modifying and/or developing or reproducing the +# software by the user in light of its specific status of free software, +# that may mean that it is complicated to manipulate, and that also +# therefore means that it is reserved for developers and experienced +# professionals having in-depth computer knowledge. Users are therefore +# encouraged to load and test the software's suitability as regards their +# requirements in conditions enabling the security of their systems and/or +# data to be ensured and, more generally, to use and operate it in the +# same conditions as regards security. + +# The fact that you are presently reading this means that you have had +# knowledge of the CeCILL license and that you accept its terms. from telemeta.core import * from api import IDataMigrator -from core import DataMigrationTask +from core import DataMigrator -class SimpleEnumMigrator(DataMigrationTask): +class SimpleEnumMigrator(DataMigrator): + """Fill simple id/value enumerations tables""" implements(IDataMigrator) map = { - 'Format': 'physical_formats', - 'Reedition': 'publishing_status', - 'Mode_Acqui': 'acquisition_modes', - 'Redacteur_Fiche': 'metadata_authors', - 'Saisie_Fiche': 'metadata_writers', - 'Droit_Utiliser': 'legal_rights', - 'Terrain_ou_Autr': 'recording_contexts', - 'Numerisation': 'ad_conversions', - 'Form': 'vernacular_styles', - 'FormStyl generi':'generic_styles' + 'Format': 'physical_formats', + 'Reedition': 'publishing_status', + 'Mode_Acqui': 'acquisition_modes', + 'Redacteur_Fiche': 'metadata_authors', + 'Saisie_Fiche': 'metadata_writers', + 'Droit_Utiliser': 'legal_rights', + 'Terrain_ou_Autr': 'recording_contexts', + 'Numerisation': 'ad_conversions', + 'Form': 'vernacular_styles', + 'FormStyl generi': 'generic_styles' } def get_name(self): diff --git a/trunk/import/migration/tasks/ethnic.py b/trunk/import/migration/tasks/ethnic.py index 13f5b71..34a8bc2 100644 --- a/trunk/import/migration/tasks/ethnic.py +++ b/trunk/import/migration/tasks/ethnic.py @@ -1,15 +1,47 @@ # -*- coding: utf-8 -*- +# +# CREM Database migrator + +# Copyright (C) 2009 Samalyse SARL +# Author: Olivier Guilyardi + +# This software is governed by the CeCILL license under French law and +# abiding by the rules of distribution of free software. You can use, +# modify and/ or redistribute the software under the terms of the CeCILL +# license as circulated by CEA, CNRS and INRIA at the following URL +# "http://www.cecill.info". + +# As a counterpart to the access to the source code and rights to copy, +# modify and redistribute granted by the license, users are provided only +# with a limited warranty and the software's author, the holder of the +# economic rights, and the successive licensors have only limited +# liability. + +# In this respect, the user's attention is drawn to the risks associated +# with loading, using, modifying and/or developing or reproducing the +# software by the user in light of its specific status of free software, +# that may mean that it is complicated to manipulate, and that also +# therefore means that it is reserved for developers and experienced +# professionals having in-depth computer knowledge. Users are therefore +# encouraged to load and test the software's suitability as regards their +# requirements in conditions enabling the security of their systems and/or +# data to be ensured and, more generally, to use and operate it in the +# same conditions as regards security. + +# The fact that you are presently reading this means that you have had +# knowledge of the CeCILL license and that you accept its terms. from telemeta.core import * from api import IDataMigrator -from core import GroupedItemsManager, DataMigrationTask +from core import GroupedItemsManager, DataMigrator -class EthnicGroupsMigrator(DataMigrationTask): +class EthnicGroupsMigrator(DataMigrator): + """Fill ethnic groups and aliases tables""" implements(IDataMigrator) def setup(self, cfg, src_db, target_db): - DataMigrationTask.setup(self, cfg, src_db, target_db) + super(EthnicGroupsMigrator, self).setup(cfg, src_db, target_db) self.data = GroupedItemsManager() def get_name(self): diff --git a/trunk/import/migration/tasks/geoethno.py b/trunk/import/migration/tasks/geoethno.py index 3b2326d..9026cec 100644 --- a/trunk/import/migration/tasks/geoethno.py +++ b/trunk/import/migration/tasks/geoethno.py @@ -1,10 +1,44 @@ +# -*- coding: utf-8 -*- +# +# CREM Database migrator + +# Copyright (C) 2009 Samalyse SARL +# Author: Olivier Guilyardi + +# This software is governed by the CeCILL license under French law and +# abiding by the rules of distribution of free software. You can use, +# modify and/ or redistribute the software under the terms of the CeCILL +# license as circulated by CEA, CNRS and INRIA at the following URL +# "http://www.cecill.info". + +# As a counterpart to the access to the source code and rights to copy, +# modify and redistribute granted by the license, users are provided only +# with a limited warranty and the software's author, the holder of the +# economic rights, and the successive licensors have only limited +# liability. + +# In this respect, the user's attention is drawn to the risks associated +# with loading, using, modifying and/or developing or reproducing the +# software by the user in light of its specific status of free software, +# that may mean that it is complicated to manipulate, and that also +# therefore means that it is reserved for developers and experienced +# professionals having in-depth computer knowledge. Users are therefore +# encouraged to load and test the software's suitability as regards their +# requirements in conditions enabling the security of their systems and/or +# data to be ensured and, more generally, to use and operate it in the +# same conditions as regards security. + +# The fact that you are presently reading this means that you have had +# knowledge of the CeCILL license and that you accept its terms. + from telemeta.core import * import sys import xml.dom.minidom as dom from api import IDataMigrator -from core import DataMigrationTask +from core import DataMigrator -class GeoEthnoImporter(DataMigrationTask): +class GeoEthnoImporter(DataMigrator): + """Import Geo Ethno thesaurus into location* tables""" implements(IDataMigrator) @@ -14,7 +48,7 @@ class GeoEthnoImporter(DataMigrationTask): nhistoric_names = 0 def setup(self, cfg, src_db, target_db): - DataMigrationTask.setup(self, cfg, src_db, target_db) + super(GeoEthnoImporter, self).setup(cfg, src_db, target_db) self.cursor = self.target_cursor self.dom = dom.parse(cfg.get('geoethno', 'xml_file')) self.known_types = [] @@ -106,12 +140,15 @@ class GeoEthnoImporter(DataMigrationTask): for n in terms.childNodes: if (n.nodeType == dom.Node.ELEMENT_NODE): + type = n.nodeName name = self.get_children_by_tag_name(n, 'DESCR') + if not len(name): if self.is_empty(n): continue raise Error(self, "Can't find tag 'DESCR' in %s" % str(n)) + if len(name) > 1: self.warn(u"More than one name for %s " % name[0].firstChild.nodeValue.strip()) @@ -120,8 +157,9 @@ class GeoEthnoImporter(DataMigrationTask): raise "empty 'DESCR' tag" - aliasNodes = self.get_children_by_tag_name(n, 'ALIAS') - historicNameNodes = self.get_children_by_tag_name(n, 'DESCR-HISTORIQUE') + aliasNodes = self.get_children_by_tag_name(n, 'ALIAS') + historicNameNodes = self.get_children_by_tag_name(n, 'DESCR-HISTORIQUE') + self.insert_location(name, type, parentName, self.flatten_node_list(historicNameNodes)) self.add_aliases(name, self.flatten_node_list(aliasNodes)) self.process_children(n, name) diff --git a/trunk/import/migration/tasks/publishers.py b/trunk/import/migration/tasks/publishers.py index fab36f1..2a0b929 100644 --- a/trunk/import/migration/tasks/publishers.py +++ b/trunk/import/migration/tasks/publishers.py @@ -1,16 +1,49 @@ # -*- coding: utf-8 -*- +# +# CREM Database migrator + +# Copyright (C) 2009 Samalyse SARL +# Author: Olivier Guilyardi + +# This software is governed by the CeCILL license under French law and +# abiding by the rules of distribution of free software. You can use, +# modify and/ or redistribute the software under the terms of the CeCILL +# license as circulated by CEA, CNRS and INRIA at the following URL +# "http://www.cecill.info". + +# As a counterpart to the access to the source code and rights to copy, +# modify and redistribute granted by the license, users are provided only +# with a limited warranty and the software's author, the holder of the +# economic rights, and the successive licensors have only limited +# liability. + +# In this respect, the user's attention is drawn to the risks associated +# with loading, using, modifying and/or developing or reproducing the +# software by the user in light of its specific status of free software, +# that may mean that it is complicated to manipulate, and that also +# therefore means that it is reserved for developers and experienced +# professionals having in-depth computer knowledge. Users are therefore +# encouraged to load and test the software's suitability as regards their +# requirements in conditions enabling the security of their systems and/or +# data to be ensured and, more generally, to use and operate it in the +# same conditions as regards security. + +# The fact that you are presently reading this means that you have had +# knowledge of the CeCILL license and that you accept its terms. from telemeta.core import * from api import IDataMigrator -from core import GroupedItemsManager, DataMigrationTask +from core import GroupedItemsManager, DataMigrator + +class PublishersMigrator(DataMigrator): + """Fill publishers and publisher collections tables""" -class PublishersMigrator(DataMigrationTask): groups = {} implements(IDataMigrator) def setup(self, cfg, src_db, target_db): - DataMigrationTask.setup(self, cfg, src_db, target_db) + super(PublishersMigrator, self).setup(cfg, src_db, target_db) self.data = GroupedItemsManager() def get_name(self): diff --git a/trunk/import/migration/tasks/reset.py b/trunk/import/migration/tasks/reset.py index c2a4a67..8403f69 100644 --- a/trunk/import/migration/tasks/reset.py +++ b/trunk/import/migration/tasks/reset.py @@ -1,10 +1,42 @@ # -*- coding: utf-8 -*- +# +# CREM Database migrator + +# Copyright (C) 2009 Samalyse SARL +# Author: Olivier Guilyardi + +# This software is governed by the CeCILL license under French law and +# abiding by the rules of distribution of free software. You can use, +# modify and/ or redistribute the software under the terms of the CeCILL +# license as circulated by CEA, CNRS and INRIA at the following URL +# "http://www.cecill.info". + +# As a counterpart to the access to the source code and rights to copy, +# modify and redistribute granted by the license, users are provided only +# with a limited warranty and the software's author, the holder of the +# economic rights, and the successive licensors have only limited +# liability. + +# In this respect, the user's attention is drawn to the risks associated +# with loading, using, modifying and/or developing or reproducing the +# software by the user in light of its specific status of free software, +# that may mean that it is complicated to manipulate, and that also +# therefore means that it is reserved for developers and experienced +# professionals having in-depth computer knowledge. Users are therefore +# encouraged to load and test the software's suitability as regards their +# requirements in conditions enabling the security of their systems and/or +# data to be ensured and, more generally, to use and operate it in the +# same conditions as regards security. + +# The fact that you are presently reading this means that you have had +# knowledge of the CeCILL license and that you accept its terms. from telemeta.core import * from api import IDataInitializer -from core import DataMigrationTask +from core import DataInitializer -class DatabaseResetMigrator(DataMigrationTask): +class DatabaseResetMigrator(DataInitializer): + """Empty tables of the target database""" implements(IDataInitializer) @@ -45,7 +77,7 @@ class DatabaseResetMigrator(DataMigrationTask): def get_name(self): return "reset" - + def process(self): #self.target_cursor.execute("SHOW TABLES") #tables = self.target_cursor.fetchall() -- 2.39.5