+# -*- coding: utf-8 -*-
+#
+# CREM Database migrator
+
+# Copyright (C) 2009 Samalyse SARL
+# Author: Olivier Guilyardi <olivier samalyse com>
+
+# This software is governed by the CeCILL license under French law and
+# abiding by the rules of distribution of free software. You can use,
+# modify and/ or redistribute the software under the terms of the CeCILL
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+
+# As a counterpart to the access to the source code and rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty and the software's author, the holder of the
+# economic rights, and the successive licensors have only limited
+# liability.
+
+# In this respect, the user's attention is drawn to the risks associated
+# with loading, using, modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean that it is complicated to manipulate, and that also
+# therefore means that it is reserved for developers and experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and, more generally, to use and operate it in the
+# same conditions as regards security.
+
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL license and that you accept its terms.
+
from telemeta.core import *
from telemeta.core import ComponentManager
import sys
from tasks.api import IDataMigrator, IDataInitializer
import tasks
-class Migrator(Component):
+class MigrationManager(Component):
+ """Establish databases connections, initialize and run migration tasks"""
+
initializers = ExtensionPoint(IDataInitializer)
migrators = ExtensionPoint(IDataMigrator)
def run_tasks(self, tasks, type, only_task = None):
+ """Run a set of migration tasks of a given type ('initializer', 'migrator',
+ ...), or the single task idenfied by its name with only_task"""
+
done = []
task = None
for t in tasks:
return done
-
def setup(self, inifile):
+ """Setup using the provided ini file"""
+
self.cfg = ConfigParser.ConfigParser()
self.cfg.read(inifile)
- self.src_db = MySQLdb.connect(user=self.cfg.get('src', 'user'),
- host=self.cfg.get('src', 'host'),
- db=self.cfg.get('src', 'name'),
- passwd=self.cfg.get('src', 'pass'),
- charset='utf8')
+ self.src_db = MySQLdb.connect(
+ user = self.cfg.get('src', 'user'),
+ host = self.cfg.get('src', 'host'),
+ db = self.cfg.get('src', 'name'),
+ passwd = self.cfg.get('src', 'pass'),
+ charset = 'utf8')
- self.target_db = MySQLdb.connect(user=self.cfg.get('target', 'user'),
- host=self.cfg.get('target', 'host'),
- db=self.cfg.get('target', 'name'),
- passwd=self.cfg.get('target', 'pass'),
- charset='utf8')
+ self.target_db = MySQLdb.connect(
+ user = self.cfg.get('target', 'user'),
+ host = self.cfg.get('target', 'host'),
+ db = self.cfg.get('target', 'name'),
+ passwd = self.cfg.get('target', 'pass'),
+ charset = 'utf8')
def run(self, only_task = None):
+ """Run all tasks or a single one identified by its name with only_task"""
self.done = []
self.done.extend(self.run_tasks(self.initializers, "initializer", only_task = only_task))
self.done.extend(self.run_tasks(self.migrators, "migrator", only_task = only_task))
if only_task and not len(self.done):
raise "No such task: %s" % only_task
+ def list_tasks(self):
+ """Generator listing available tasks as DataMigrationTask instances"""
+ for task in self.initializers:
+ yield task
+
+ for task in self.migrators:
+ yield task
+
def print_stats(self):
+ """Print all stats, retrieved from the previously run tasks"""
init = False
for task in self.done:
if len(task.stats):
if __name__ == '__main__':
+ manager = MigrationManager(ComponentManager())
+
if len(sys.argv) != 2 and len(sys.argv) != 3:
print "Usage: %s <config_file> [task_name]" % sys.argv[0]
+ print "Tasks:"
+ for task in manager.list_tasks():
+ print " %-16s%s" % (task.get_name() + ':', task.__doc__)
+
sys.exit(1)
if (len(sys.argv) == 3):
else:
only_task = None
- cmpmgr = ComponentManager()
- migrator = Migrator(cmpmgr)
- migrator.setup(sys.argv[1])
- migrator.run(only_task)
- migrator.print_stats()
+ manager.setup(sys.argv[1])
+ manager.run(only_task)
+ manager.print_stats()
+# -*- coding: utf-8 -*-
+#
+# CREM Database migrator
+
+# Copyright (C) 2009 Samalyse SARL
+# Author: Olivier Guilyardi <olivier samalyse com>
+
+# This software is governed by the CeCILL license under French law and
+# abiding by the rules of distribution of free software. You can use,
+# modify and/ or redistribute the software under the terms of the CeCILL
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+
+# As a counterpart to the access to the source code and rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty and the software's author, the holder of the
+# economic rights, and the successive licensors have only limited
+# liability.
+
+# In this respect, the user's attention is drawn to the risks associated
+# with loading, using, modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean that it is complicated to manipulate, and that also
+# therefore means that it is reserved for developers and experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and, more generally, to use and operate it in the
+# same conditions as regards security.
+
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL license and that you accept its terms.
+
import reset
import enums
import geoethno
-# Copyright (C) 2007 Samalyse SARL
-# All rights reserved.
+# -*- coding: utf-8 -*-
#
-# This software is licensed as described in the file COPYING, which
-# you should have received as part of this distribution. The terms
-# are also available at http://svn.parisson.org/telemeta/TelemetaLicense.
-#
-# Author: Olivier Guilyardi <olivier@samalyse.com>
+# CREM Database migrator
+
+# Copyright (C) 2009 Samalyse SARL
+# Author: Olivier Guilyardi <olivier samalyse com>
+
+# This software is governed by the CeCILL license under French law and
+# abiding by the rules of distribution of free software. You can use,
+# modify and/ or redistribute the software under the terms of the CeCILL
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+
+# As a counterpart to the access to the source code and rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty and the software's author, the holder of the
+# economic rights, and the successive licensors have only limited
+# liability.
+
+# In this respect, the user's attention is drawn to the risks associated
+# with loading, using, modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean that it is complicated to manipulate, and that also
+# therefore means that it is reserved for developers and experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and, more generally, to use and operate it in the
+# same conditions as regards security.
+
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL license and that you accept its terms.
from telemeta.core import *
class IDataMigrationTask(Interface):
+ """Migration tasks interface"""
def setup(cfg, src_db, target_db):
"""Set the migrator up"""
class IDataInitializer(IDataMigrationTask):
+ """Initialization migration tasks interface"""
pass
class IDataMigrator(IDataMigrationTask):
+ """Data migrating tasks interface"""
pass
+# -*- coding: utf-8 -*-
+#
+# CREM Database migrator
+
+# Copyright (C) 2009 Samalyse SARL
+# Author: Olivier Guilyardi <olivier samalyse com>
+
+# This software is governed by the CeCILL license under French law and
+# abiding by the rules of distribution of free software. You can use,
+# modify and/ or redistribute the software under the terms of the CeCILL
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+
+# As a counterpart to the access to the source code and rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty and the software's author, the holder of the
+# economic rights, and the successive licensors have only limited
+# liability.
+
+# In this respect, the user's attention is drawn to the risks associated
+# with loading, using, modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean that it is complicated to manipulate, and that also
+# therefore means that it is reserved for developers and experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and, more generally, to use and operate it in the
+# same conditions as regards security.
+
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL license and that you accept its terms.
+
import sys
from telemeta.core import *
sys.stdout.write('.')
sys.stdout.flush()
+class DataMigrator(DataMigrationTask):
+ pass
+
+class DataInitializer(DataMigrationTask):
+ pass
+
class GroupedItemsManager(object):
def __init__(self):
# -*- coding: utf-8 -*-
+#
+# CREM Database migrator
+
+# Copyright (C) 2009 Samalyse SARL
+# Author: Olivier Guilyardi <olivier samalyse com>
+
+# This software is governed by the CeCILL license under French law and
+# abiding by the rules of distribution of free software. You can use,
+# modify and/ or redistribute the software under the terms of the CeCILL
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+
+# As a counterpart to the access to the source code and rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty and the software's author, the holder of the
+# economic rights, and the successive licensors have only limited
+# liability.
+
+# In this respect, the user's attention is drawn to the risks associated
+# with loading, using, modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean that it is complicated to manipulate, and that also
+# therefore means that it is reserved for developers and experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and, more generally, to use and operate it in the
+# same conditions as regards security.
+
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL license and that you accept its terms.
from telemeta.core import *
from api import IDataMigrator
-from core import DataMigrationTask
+from core import DataMigrator
-class SimpleEnumMigrator(DataMigrationTask):
+class SimpleEnumMigrator(DataMigrator):
+ """Fill simple id/value enumerations tables"""
implements(IDataMigrator)
map = {
- 'Format': 'physical_formats',
- 'Reedition': 'publishing_status',
- 'Mode_Acqui': 'acquisition_modes',
- 'Redacteur_Fiche': 'metadata_authors',
- 'Saisie_Fiche': 'metadata_writers',
- 'Droit_Utiliser': 'legal_rights',
- 'Terrain_ou_Autr': 'recording_contexts',
- 'Numerisation': 'ad_conversions',
- 'Form': 'vernacular_styles',
- 'FormStyl generi':'generic_styles'
+ 'Format': 'physical_formats',
+ 'Reedition': 'publishing_status',
+ 'Mode_Acqui': 'acquisition_modes',
+ 'Redacteur_Fiche': 'metadata_authors',
+ 'Saisie_Fiche': 'metadata_writers',
+ 'Droit_Utiliser': 'legal_rights',
+ 'Terrain_ou_Autr': 'recording_contexts',
+ 'Numerisation': 'ad_conversions',
+ 'Form': 'vernacular_styles',
+ 'FormStyl generi': 'generic_styles'
}
def get_name(self):
# -*- coding: utf-8 -*-
+#
+# CREM Database migrator
+
+# Copyright (C) 2009 Samalyse SARL
+# Author: Olivier Guilyardi <olivier samalyse com>
+
+# This software is governed by the CeCILL license under French law and
+# abiding by the rules of distribution of free software. You can use,
+# modify and/ or redistribute the software under the terms of the CeCILL
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+
+# As a counterpart to the access to the source code and rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty and the software's author, the holder of the
+# economic rights, and the successive licensors have only limited
+# liability.
+
+# In this respect, the user's attention is drawn to the risks associated
+# with loading, using, modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean that it is complicated to manipulate, and that also
+# therefore means that it is reserved for developers and experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and, more generally, to use and operate it in the
+# same conditions as regards security.
+
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL license and that you accept its terms.
from telemeta.core import *
from api import IDataMigrator
-from core import GroupedItemsManager, DataMigrationTask
+from core import GroupedItemsManager, DataMigrator
-class EthnicGroupsMigrator(DataMigrationTask):
+class EthnicGroupsMigrator(DataMigrator):
+ """Fill ethnic groups and aliases tables"""
implements(IDataMigrator)
def setup(self, cfg, src_db, target_db):
- DataMigrationTask.setup(self, cfg, src_db, target_db)
+ super(EthnicGroupsMigrator, self).setup(cfg, src_db, target_db)
self.data = GroupedItemsManager()
def get_name(self):
+# -*- coding: utf-8 -*-
+#
+# CREM Database migrator
+
+# Copyright (C) 2009 Samalyse SARL
+# Author: Olivier Guilyardi <olivier samalyse com>
+
+# This software is governed by the CeCILL license under French law and
+# abiding by the rules of distribution of free software. You can use,
+# modify and/ or redistribute the software under the terms of the CeCILL
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+
+# As a counterpart to the access to the source code and rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty and the software's author, the holder of the
+# economic rights, and the successive licensors have only limited
+# liability.
+
+# In this respect, the user's attention is drawn to the risks associated
+# with loading, using, modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean that it is complicated to manipulate, and that also
+# therefore means that it is reserved for developers and experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and, more generally, to use and operate it in the
+# same conditions as regards security.
+
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL license and that you accept its terms.
+
from telemeta.core import *
import sys
import xml.dom.minidom as dom
from api import IDataMigrator
-from core import DataMigrationTask
+from core import DataMigrator
-class GeoEthnoImporter(DataMigrationTask):
+class GeoEthnoImporter(DataMigrator):
+ """Import Geo Ethno thesaurus into location* tables"""
implements(IDataMigrator)
nhistoric_names = 0
def setup(self, cfg, src_db, target_db):
- DataMigrationTask.setup(self, cfg, src_db, target_db)
+ super(GeoEthnoImporter, self).setup(cfg, src_db, target_db)
self.cursor = self.target_cursor
self.dom = dom.parse(cfg.get('geoethno', 'xml_file'))
self.known_types = []
for n in terms.childNodes:
if (n.nodeType == dom.Node.ELEMENT_NODE):
+
type = n.nodeName
name = self.get_children_by_tag_name(n, 'DESCR')
+
if not len(name):
if self.is_empty(n):
continue
raise Error(self, "Can't find tag 'DESCR' in %s" % str(n))
+
if len(name) > 1:
self.warn(u"More than one name for %s " % name[0].firstChild.nodeValue.strip())
raise "empty 'DESCR' tag"
- aliasNodes = self.get_children_by_tag_name(n, 'ALIAS')
- historicNameNodes = self.get_children_by_tag_name(n, 'DESCR-HISTORIQUE')
+ aliasNodes = self.get_children_by_tag_name(n, 'ALIAS')
+ historicNameNodes = self.get_children_by_tag_name(n, 'DESCR-HISTORIQUE')
+
self.insert_location(name, type, parentName, self.flatten_node_list(historicNameNodes))
self.add_aliases(name, self.flatten_node_list(aliasNodes))
self.process_children(n, name)
# -*- coding: utf-8 -*-
+#
+# CREM Database migrator
+
+# Copyright (C) 2009 Samalyse SARL
+# Author: Olivier Guilyardi <olivier samalyse com>
+
+# This software is governed by the CeCILL license under French law and
+# abiding by the rules of distribution of free software. You can use,
+# modify and/ or redistribute the software under the terms of the CeCILL
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+
+# As a counterpart to the access to the source code and rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty and the software's author, the holder of the
+# economic rights, and the successive licensors have only limited
+# liability.
+
+# In this respect, the user's attention is drawn to the risks associated
+# with loading, using, modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean that it is complicated to manipulate, and that also
+# therefore means that it is reserved for developers and experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and, more generally, to use and operate it in the
+# same conditions as regards security.
+
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL license and that you accept its terms.
from telemeta.core import *
from api import IDataMigrator
-from core import GroupedItemsManager, DataMigrationTask
+from core import GroupedItemsManager, DataMigrator
+
+class PublishersMigrator(DataMigrator):
+ """Fill publishers and publisher collections tables"""
-class PublishersMigrator(DataMigrationTask):
groups = {}
implements(IDataMigrator)
def setup(self, cfg, src_db, target_db):
- DataMigrationTask.setup(self, cfg, src_db, target_db)
+ super(PublishersMigrator, self).setup(cfg, src_db, target_db)
self.data = GroupedItemsManager()
def get_name(self):
# -*- coding: utf-8 -*-
+#
+# CREM Database migrator
+
+# Copyright (C) 2009 Samalyse SARL
+# Author: Olivier Guilyardi <olivier samalyse com>
+
+# This software is governed by the CeCILL license under French law and
+# abiding by the rules of distribution of free software. You can use,
+# modify and/ or redistribute the software under the terms of the CeCILL
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+
+# As a counterpart to the access to the source code and rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty and the software's author, the holder of the
+# economic rights, and the successive licensors have only limited
+# liability.
+
+# In this respect, the user's attention is drawn to the risks associated
+# with loading, using, modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean that it is complicated to manipulate, and that also
+# therefore means that it is reserved for developers and experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and, more generally, to use and operate it in the
+# same conditions as regards security.
+
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL license and that you accept its terms.
from telemeta.core import *
from api import IDataInitializer
-from core import DataMigrationTask
+from core import DataInitializer
-class DatabaseResetMigrator(DataMigrationTask):
+class DatabaseResetMigrator(DataInitializer):
+ """Empty tables of the target database"""
implements(IDataInitializer)
def get_name(self):
return "reset"
-
+
def process(self):
#self.target_cursor.execute("SHOW TABLES")
#tables = self.target_cursor.fetchall()
projects / telemeta-data.git / commitdiff