From: olivier <olivier@3bf09e05-f825-4182-b9bc-eedd7160adf0>
Date: Wed, 22 Apr 2009 18:28:06 +0000 (+0000)
Subject: migration: fix geoethno for innodb
X-Git-Url: https://git.parisson.com/?a=commitdiff_plain;h=f8bc03354c2c0358af21b2d8060c3a2f5cc03c76;p=telemeta-data.git

migration: fix geoethno for innodb

git-svn-id: http://svn.parisson.org/svn/crem@82 3bf09e05-f825-4182-b9bc-eedd7160adf0
---

diff --git a/trunk/import/migration/tasks/geoethno.py b/trunk/import/migration/tasks/geoethno.py
index 9026cec..7009a5f 100644
--- a/trunk/import/migration/tasks/geoethno.py
+++ b/trunk/import/migration/tasks/geoethno.py
@@ -36,6 +36,8 @@ import sys
 import xml.dom.minidom as dom
 from api import IDataMigrator
 from core import DataMigrator
+from _mysql_exceptions import IntegrityError
+from MySQLdb.constants.ER import DUP_ENTRY
 
 class GeoEthnoImporter(DataMigrator):
     """Import Geo Ethno thesaurus into location* tables"""
@@ -72,6 +74,21 @@ class GeoEthnoImporter(DataMigrator):
                 flat.append(n.firstChild.nodeValue.strip())
         return flat
 
+    def replace(self, query, params = None):
+        inserted = 0
+
+        try:
+            self.cursor.execute(query, params)
+            inserted = 1
+
+        except IntegrityError, e:
+            (errno, errmsg) = e
+            if errno != DUP_ENTRY:
+                raise e
+
+        return inserted                
+        
+
     def insert_location(self, name, type, parentName, historic_names):
         if type == 'CONTINENT':
             short_type = 'continent'
@@ -82,41 +99,34 @@ class GeoEthnoImporter(DataMigrator):
 
         self.register_type(type)
 
-        self.cursor.execute("REPLACE INTO locations "+
-                            "(name, type, complete_type_id, current_name, is_authoritative) "+
-                            "VALUES (%s, %s, %s, %s, %s)", (name, short_type, type, name, 1))
-
-        self.nlocations += 1
+        self.nlocations += self.replace("INSERT INTO locations "+
+                                        "(name, type, complete_type_id, current_name, is_authoritative) "+
+                                        "VALUES (%s, %s, %s, %s, %s)", (name, short_type, type, name, 1))
 
         if (len(parentName)):
-            self.cursor.execute("REPLACE INTO location_relations "+
-                                "(location_name, parent_location_name) "+
-                                "VALUE (%s, %s)", (name, parentName))
-
-            self.nrelations += 1
+            self.nrelations += self.replace("INSERT INTO location_relations "+
+                                            "(location_name, parent_location_name) "+
+                                            "VALUE (%s, %s)", (name, parentName))
 
         for hname in historic_names:
-            self.cursor.execute("REPLACE INTO locations "+
-                                "(name, type, complete_type_id, current_name, is_authoritative) "+
-                                "VALUES (%s, %s, %s, %s, %s)", (hname, short_type, type, name, 1))
-            self.nhistoric_names += 1
+            self.nhistoric_names += self.replace("INSERT INTO locations "+
+                                                 "(name, type, complete_type_id, current_name, is_authoritative) "+
+                                                 "VALUES (%s, %s, %s, %s, %s)", (hname, short_type, type, name, 1))
                     
 
-#        if self.nlocations % 1000 == 0:
-#            self.step()
-        
     def add_aliases(self, name, items):
         for alias in items:
-            self.cursor.execute("REPLACE INTO location_aliases "+
-                                "(location_name, alias, is_authoritative) "+
-                                "VALUES (%s, %s, %s)", (name, alias, 1))
-            self.naliases += 1
+            self.naliases += self.replace("INSERT INTO location_aliases "+
+                                          "(location_name, alias, is_authoritative) "+
+                                          "VALUES (%s, %s, %s)", (name, alias, 1))
+                
+
 
     def register_type(self, id):
         try:
             self.known_types.index(id)
         except ValueError:
-            self.cursor.execute("REPLACE INTO location_types (id, name) "+
+            self.cursor.execute("INSERT INTO location_types (id, name) "+
                                 "VALUES (%s, %s)", (id,""))
             self.known_types.append(id)