From: olivier Date: Tue, 3 Mar 2009 11:34:17 +0000 (+0000) Subject: add enums migrator and reset task X-Git-Url: https://git.parisson.com/?a=commitdiff_plain;h=802ec342eb38a9d0054712e0b92d395249bef4a7;p=telemeta-data.git add enums migrator and reset task git-svn-id: http://svn.parisson.org/svn/crem@65 3bf09e05-f825-4182-b9bc-eedd7160adf0 --- diff --git a/trunk/import/migration/api.py b/trunk/import/migration/api.py new file mode 100644 index 0000000..cca0379 --- /dev/null +++ b/trunk/import/migration/api.py @@ -0,0 +1,23 @@ +# Copyright (C) 2007 Samalyse SARL +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://svn.parisson.org/telemeta/TelemetaLicense. +# +# Author: Olivier Guilyardi + +from telemeta.core import * + +class IDataMigrator(Interface): + + def setup(cfg, src_db, target_db): + """Set the migrator up""" + pass + + def get_name(): + """"Return a meaningful name""" + + def process(): + """Run the migration task""" + diff --git a/trunk/import/migration/enums.py b/trunk/import/migration/enums.py new file mode 100644 index 0000000..87c94cb --- /dev/null +++ b/trunk/import/migration/enums.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +from telemeta.core import * +from api import IDataMigrator + +class SimpleEnumMigrator(Component): + + implements(IDataMigrator) + + map = { + 'Format': 'physical_formats', + 'Reedition': 'publishing_status', + 'Mode_Acqui': 'acquisition_modes', + 'Redacteur_Fiche': 'metadata_authors', + 'Saisie_Fiche': 'metadata_writers', + 'Droit_Utiliser': 'legal_rights', + 'Terrain_ou_Autr': 'recording_contexts', + 'Numerisation': 'ad_conversions', + 'Form': 'vernacular_styles', + 'FormStyl generi':'generic_styles' + } + + def setup(self, cfg, src_db, target_db): + self.target_cursor = target_db.cursor() + self.src_db_name = cfg.get('src', 'name') + + def get_name(self): + return "enumerations" + + + def process(self): + for src in self.map: + dest = self.map[src] + if src == 'Form': + src_field = 'Form' + else: + src_field = 'value' + + self.target_cursor.execute("INSERT INTO `" + dest + "` (value) " + + "SELECT " + src_field +" FROM " + self.src_db_name + ".`" + src + "`") + + + diff --git a/trunk/import/migration/geoethno.py b/trunk/import/migration/geoethno.py index 3ec572d..bc51b0e 100644 --- a/trunk/import/migration/geoethno.py +++ b/trunk/import/migration/geoethno.py @@ -1,20 +1,26 @@ +from telemeta.core import * import sys import xml.dom.minidom as dom -import ConfigParser -import MySQLdb +from api import IDataMigrator + +class GeoEthnoImporter(Component): + + implements(IDataMigrator) -class GeoEthnoImporter(object): nlocations = 0 nrelations = 0 naliases = 0 nhistoric_names = 0 - def __init__(self, input_file, target_db): + def setup(self, cfg, src_db, target_db): self.db = target_db self.cursor = self.db.cursor() - self.dom = dom.parse(input_file) + self.dom = dom.parse(cfg.get('geoethno', 'xml_file')) self.known_types = [] + def get_name(self): + return "geoethno" + def get_children_by_tag_name(self, node, tagName): children = [] for n in node.childNodes: @@ -141,22 +147,6 @@ class Error(Exception): def __init__(self, importer, msg): print u"\nError: %s: %s" % ("/".join(importer.path), msg) -if __name__ == '__main__': - if len(sys.argv) != 2: - print "Usage: %s " % sys.argv[0] - sys.exit(1) - - cfg = ConfigParser.ConfigParser() - cfg.read(sys.argv[1]) - db = MySQLdb.connect(user=cfg.get('target', 'user'), - host=cfg.get('target', 'host'), - db=cfg.get('target', 'name'), - passwd=cfg.get('target', 'pass'), - charset='utf8') - importer = GeoEthnoImporter(cfg.get('geoethno', 'xml_file'), db) - importer.process() - - diff --git a/trunk/import/migration/migrate.py b/trunk/import/migration/migrate.py new file mode 100644 index 0000000..4bb5fd9 --- /dev/null +++ b/trunk/import/migration/migrate.py @@ -0,0 +1,67 @@ +from telemeta.core import * +from telemeta.core import ComponentManager +import sys +import ConfigParser +import MySQLdb + +from api import IDataMigrator + +import reset +import enums +import geoethno + +class Migrator(Component): + data_migrators = ExtensionPoint(IDataMigrator) + + def run(self, config, migrator_name = None): + src_db = MySQLdb.connect(user=cfg.get('src', 'user'), + host=cfg.get('src', 'host'), + db=cfg.get('src', 'name'), + passwd=cfg.get('src', 'pass'), + charset='utf8') + + target_db = MySQLdb.connect(user=cfg.get('target', 'user'), + host=cfg.get('target', 'host'), + db=cfg.get('target', 'name'), + passwd=cfg.get('target', 'pass'), + charset='utf8') + + + item = None + for m in self.data_migrators: + if migrator_name: + if m.get_name() == migrator_name: + item = m + else: + item = m + + if item: + print "Runinng %s migrator\n" % item.get_name() + item.setup(config, src_db, target_db) + item.process() + if migrator_name: + break + + if migrator_name and not item: + raise "No such migrator: %s" % migrator_name + + + +if __name__ == '__main__': + if len(sys.argv) != 2 and len(sys.argv) != 3: + print "Usage: %s [migrator_name]" % sys.argv[0] + sys.exit(1) + + if (len(sys.argv) == 3): + migrator_name = sys.argv[2] + else: + migrator_name = None + + cfg = ConfigParser.ConfigParser() + cfg.read(sys.argv[1]) + + cmpmgr = ComponentManager() + migrator = Migrator(cmpmgr) + migrator.run(cfg, migrator_name) + + diff --git a/trunk/import/migration/migration.ini.default b/trunk/import/migration/migration.ini.default index 907337f..02bd935 100644 --- a/trunk/import/migration/migration.ini.default +++ b/trunk/import/migration/migration.ini.default @@ -4,5 +4,11 @@ pass= name=crem_target host=127.0.0.1 +[src] +user=root +pass= +name=crem_src +host=127.0.0.1 + [geoethno] xml_file=geoEthnoTelemeta.xml diff --git a/trunk/import/migration/reset.py b/trunk/import/migration/reset.py new file mode 100644 index 0000000..b0bdad1 --- /dev/null +++ b/trunk/import/migration/reset.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +from telemeta.core import * +from api import IDataMigrator + +class DatabaseResetMigrator(Component): + + implements(IDataMigrator) + + tables = [ + 'physical_formats', + 'publishing_status', + 'acquisition_modes', + 'metadata_authors', + 'metadata_writers', + 'legal_rights', + 'recording_contexts', + 'ad_conversions', + 'vernacular_styles', + 'generic_styles', + 'publishers', + 'publisher_collections', + 'location_types', + 'locations', + 'location_aliases', + 'location_relations', + 'ethnic_groups', + 'ethnic_group_aliases', + 'media_collections', + 'media_items', + 'media_parts', + 'instruments', + 'instrument_relations', + 'instrument_aliases', + 'instrument_alias_relations', + 'media_item_performances', + 'context_keywords', + 'media_item_keywords', + 'users', + 'playlists', + 'playlist_resources', + 'revisions' + ] + + def setup(self, cfg, src_db, target_db): + self.target_cursor = target_db.cursor() + + def get_name(self): + return "reset" + + def process(self): + #self.target_cursor.execute("SHOW TABLES") + #tables = self.target_cursor.fetchall() + tables = self.tables + for t in tables: + #table = t[0] + table = t + self.target_cursor.execute("DELETE FROM " + table) + self.target_cursor.execute("DESCRIBE " + table) + fields = self.target_cursor.fetchall() + for f in fields: + if f[5] == 'auto_increment': + self.target_cursor.execute("ALTER TABLE " + table + " AUTO_INCREMENT = 1") + + + +