add initial import scripts and LEM data as exported on may 9th, 2007

git-svn-id: http://svn.parisson.org/svn/crem@1 3bf09e05-f825-4182-b9bc-eedd7160adf0

diff --git a/README b/README
new file mode 100644 (file)
index 0000000..7c6e6cc
--- /dev/null
+++ b/README
@@ -0,0 +1,26 @@
+LEM/CREM data and how to import it into Telemeta
+
+1 - Install Telemeta upon MySQL. Initialize the database using Telemeta's
+Django models. Ensure that everything is running fine before going any 
+further.
+
+2 - Run prepare on the source directory. The source directory must contain
+text files as they were exported from the 4D database. These files are 
+expected to be gzipped. 
+
+Example:
+$ scripts/prepare src/2007-05-09
+
+3 - Use the import.sql script to insert the data into your MySQL database.
+
+Example:
+
+$ mysql your_telemeta_database < scripts/import.sql
+
+That should be it. If you want to run Telemeta against SQLite instead of 
+MySQL, first follow the above instructions to import the data into MySQL.
+Then convert your data from MySQL to SQLite, this is a common task, google 
+about it.
+
+
+

diff --git a/scripts/import.sql b/scripts/import.sql
new file mode 100644 (file)
index 0000000..ceb680b
--- /dev/null
+++ b/scripts/import.sql
@@ -0,0 +1,29 @@
+DELETE FROM telemeta_collection;
+LOAD DATA INFILE '/tmp/crem_import/support.txt' INTO TABLE telemeta_collection
+FIELDS TERMINATED BY '\t'
+LINES TERMINATED BY '\r\n';
+
+UPDATE telemeta_collection SET id=REPLACE(id, '/', '--');
+UPDATE telemeta_item SET id=REPLACE(id, ":", "__");
+
+UPDATE telemeta_collection SET copied_from_id = NULL WHERE copied_from_id = '';
+
+DELETE FROM telemeta_item;
+LOAD DATA INFILE '/tmp/crem_import/phono.txt' INTO TABLE telemeta_item
+FIELDS TERMINATED BY '\t'
+LINES TERMINATED BY '\r\n';
+
+UPDATE telemeta_item SET id=REPLACE(id, '/', '--');
+UPDATE telemeta_item SET collection_id=REPLACE(collection_id, '/', '--');
+UPDATE telemeta_item SET title='' WHERE title='N';
+-- UPDATE telemeta_item SET file="items/test.wav";
+
+DELETE FROM telemeta_physicalformat;
+INSERT INTO telemeta_physicalformat (value)
+    SELECT DISTINCT physical_format FROM telemeta_collection
+    WHERE physical_format <> '' AND physical_format IS NOT NULL;
+
+DELETE FROM telemeta_publishingstatus;
+INSERT INTO telemeta_publishingstatus (value)
+    SELECT DISTINCT publishing_status FROM telemeta_collection
+    WHERE publishing_status <> '' AND publishing_status IS NOT NULL;

diff --git a/scripts/prepare b/scripts/prepare
new file mode 100755 (executable)
index 0000000..92a7618
--- /dev/null
+++ b/scripts/prepare
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+if [ "$1" == "" ]
+then 
+    echo "Please provide the source directory"
+    exit 1
+fi
+
+src=$1
+tmpdir=crem_import
+required="support.txt.gz phono.txt.gz"
+
+for f in $required
+do
+    if ! [ -f $src/$f ]
+    then 
+        echo "Can't find $f in $src"
+        exit 1
+    fi
+done    
+
+if [ -d /tmp/$tmpdir ]
+then
+    rm /tmp/$tmpdir/*
+else
+    mkdir /tmp/$tmpdir
+fi    
+
+echo -n "Converting charset and cleaning text files.. "
+
+zcat $src/support.txt.gz | tail -n +2 \
+    | sed 's/^ *//' | sed 's/ *\t */\t/g'| sed 's/ *$//' \
+    | iconv -f WINDOWS-1252 -c -t latin1 \
+    > /tmp/$tmpdir/support.txt
+
+zcat $src/phono.txt.gz | tail -n +2 \
+    | sed 's/^ *//' | sed 's/ *\t */\t/g'| sed 's/ *$//' \
+    | iconv -f WINDOWS-1252 -c -t latin1 | sed '/^\t/d' \
+    > /tmp/$tmpdir/phono.txt
+
+echo "Done"
+
+

diff --git a/src/2007-05-09/phono.txt.gz b/src/2007-05-09/phono.txt.gz
new file mode 100755 (executable)
index 0000000..fd0be4a
Binary files /dev/null and b/src/2007-05-09/phono.txt.gz differ

diff --git a/src/2007-05-09/support.txt.gz b/src/2007-05-09/support.txt.gz
new file mode 100755 (executable)
index 0000000..a7ec8c6
Binary files /dev/null and b/src/2007-05-09/support.txt.gz differ