From: olivier Date: Fri, 13 Jun 2008 14:37:18 +0000 (+0000) Subject: renamed 'alpha' import to 'prototype' X-Git-Url: https://git.parisson.com/?a=commitdiff_plain;h=f37b99b04b8222af802fa7be04ba799d71e4a35e;p=telemeta-data.git renamed 'alpha' import to 'prototype' git-svn-id: http://svn.parisson.org/svn/crem@25 3bf09e05-f825-4182-b9bc-eedd7160adf0 --- diff --git a/import/alpha/README b/import/alpha/README deleted file mode 100644 index b26fb7c..0000000 --- a/import/alpha/README +++ /dev/null @@ -1,33 +0,0 @@ -== CREM data and how to import it into Telemeta 0.3.x == - -Warning: the following instructions allow you to import the CREM's data -into telemeta. However, it is currently a rather data destructive process, -only meant for demonstration and testing purpose. Do not use this in a -production environment. - -1 - Install Telemeta upon MySQL. Initialize the database using Telemeta's -Django models. Ensure that everything is running fine before going any -further. - -2 - Run prepare on the source directory. The source directory must contain -text files as they were exported from the 4D database. These files are -expected to be gzipped. - -Example: -$ scripts/prepare src/2007-05-09 - -3 - Use the import.sql script to insert the data into your MySQL database. - -Example: -$ mysql your_telemeta_database < scripts/import.sql - -4 - Copy the WAV test file of your choice, into /items/test.wav -This single file is associated with all media items, for testing purpose. - -That should be it. If you want to run Telemeta against SQLite instead of -MySQL, first follow the above instructions to import the data into MySQL. -Then convert your data from MySQL to SQLite, this is a common task, google -about it. - - - diff --git a/import/alpha/scripts/import.sql b/import/alpha/scripts/import.sql deleted file mode 100644 index 0bb05be..0000000 --- a/import/alpha/scripts/import.sql +++ /dev/null @@ -1,31 +0,0 @@ -DELETE FROM telemeta_collection; -LOAD DATA INFILE '/tmp/crem_import/support.txt' INTO TABLE telemeta_collection -FIELDS TERMINATED BY '\t' -LINES TERMINATED BY '\r\n'; - -UPDATE telemeta_collection SET id=REPLACE(id, '/', '--'); -UPDATE telemeta_collection SET copied_from_id = NULL WHERE copied_from_id = ''; - -DELETE FROM telemeta_item; -LOAD DATA INFILE '/tmp/crem_import/phono.txt' INTO TABLE telemeta_item -FIELDS TERMINATED BY '\t' -LINES TERMINATED BY '\r\n'; - -UPDATE telemeta_item SET id=REPLACE(id, ":", "__"); -UPDATE telemeta_item SET id=REPLACE(id, '/', '--'); -UPDATE telemeta_item SET id=REPLACE(id, '?', 'i'); -UPDATE telemeta_item SET collection_id=REPLACE(collection_id, '/', '--'); -UPDATE telemeta_item SET title='' WHERE title='N'; -UPDATE telemeta_item SET etat=REPLACE(etat, ')', '_'); -UPDATE telemeta_item SET etat=REPLACE(etat, '(', '_'); -UPDATE telemeta_item SET file="items/test.wav"; - -DELETE FROM telemeta_physicalformat; -INSERT INTO telemeta_physicalformat (value) - SELECT DISTINCT physical_format FROM telemeta_collection - WHERE physical_format <> '' AND physical_format IS NOT NULL; - -DELETE FROM telemeta_publishingstatus; -INSERT INTO telemeta_publishingstatus (value) - SELECT DISTINCT publishing_status FROM telemeta_collection - WHERE publishing_status <> '' AND publishing_status IS NOT NULL; diff --git a/import/alpha/scripts/prepare b/import/alpha/scripts/prepare deleted file mode 100755 index 92a7618..0000000 --- a/import/alpha/scripts/prepare +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -if [ "$1" == "" ] -then - echo "Please provide the source directory" - exit 1 -fi - -src=$1 -tmpdir=crem_import -required="support.txt.gz phono.txt.gz" - -for f in $required -do - if ! [ -f $src/$f ] - then - echo "Can't find $f in $src" - exit 1 - fi -done - -if [ -d /tmp/$tmpdir ] -then - rm /tmp/$tmpdir/* -else - mkdir /tmp/$tmpdir -fi - -echo -n "Converting charset and cleaning text files.. " - -zcat $src/support.txt.gz | tail -n +2 \ - | sed 's/^ *//' | sed 's/ *\t */\t/g'| sed 's/ *$//' \ - | iconv -f WINDOWS-1252 -c -t latin1 \ - > /tmp/$tmpdir/support.txt - -zcat $src/phono.txt.gz | tail -n +2 \ - | sed 's/^ *//' | sed 's/ *\t */\t/g'| sed 's/ *$//' \ - | iconv -f WINDOWS-1252 -c -t latin1 | sed '/^\t/d' \ - > /tmp/$tmpdir/phono.txt - -echo "Done" - - diff --git a/import/alpha/src/2007-05-09/phono.txt.gz b/import/alpha/src/2007-05-09/phono.txt.gz deleted file mode 100755 index fd0be4a..0000000 Binary files a/import/alpha/src/2007-05-09/phono.txt.gz and /dev/null differ diff --git a/import/alpha/src/2007-05-09/support.txt.gz b/import/alpha/src/2007-05-09/support.txt.gz deleted file mode 100755 index a7ec8c6..0000000 Binary files a/import/alpha/src/2007-05-09/support.txt.gz and /dev/null differ diff --git a/import/prototype/README b/import/prototype/README new file mode 100644 index 0000000..b26fb7c --- /dev/null +++ b/import/prototype/README @@ -0,0 +1,33 @@ +== CREM data and how to import it into Telemeta 0.3.x == + +Warning: the following instructions allow you to import the CREM's data +into telemeta. However, it is currently a rather data destructive process, +only meant for demonstration and testing purpose. Do not use this in a +production environment. + +1 - Install Telemeta upon MySQL. Initialize the database using Telemeta's +Django models. Ensure that everything is running fine before going any +further. + +2 - Run prepare on the source directory. The source directory must contain +text files as they were exported from the 4D database. These files are +expected to be gzipped. + +Example: +$ scripts/prepare src/2007-05-09 + +3 - Use the import.sql script to insert the data into your MySQL database. + +Example: +$ mysql your_telemeta_database < scripts/import.sql + +4 - Copy the WAV test file of your choice, into /items/test.wav +This single file is associated with all media items, for testing purpose. + +That should be it. If you want to run Telemeta against SQLite instead of +MySQL, first follow the above instructions to import the data into MySQL. +Then convert your data from MySQL to SQLite, this is a common task, google +about it. + + + diff --git a/import/prototype/scripts/import.sql b/import/prototype/scripts/import.sql new file mode 100644 index 0000000..0bb05be --- /dev/null +++ b/import/prototype/scripts/import.sql @@ -0,0 +1,31 @@ +DELETE FROM telemeta_collection; +LOAD DATA INFILE '/tmp/crem_import/support.txt' INTO TABLE telemeta_collection +FIELDS TERMINATED BY '\t' +LINES TERMINATED BY '\r\n'; + +UPDATE telemeta_collection SET id=REPLACE(id, '/', '--'); +UPDATE telemeta_collection SET copied_from_id = NULL WHERE copied_from_id = ''; + +DELETE FROM telemeta_item; +LOAD DATA INFILE '/tmp/crem_import/phono.txt' INTO TABLE telemeta_item +FIELDS TERMINATED BY '\t' +LINES TERMINATED BY '\r\n'; + +UPDATE telemeta_item SET id=REPLACE(id, ":", "__"); +UPDATE telemeta_item SET id=REPLACE(id, '/', '--'); +UPDATE telemeta_item SET id=REPLACE(id, '?', 'i'); +UPDATE telemeta_item SET collection_id=REPLACE(collection_id, '/', '--'); +UPDATE telemeta_item SET title='' WHERE title='N'; +UPDATE telemeta_item SET etat=REPLACE(etat, ')', '_'); +UPDATE telemeta_item SET etat=REPLACE(etat, '(', '_'); +UPDATE telemeta_item SET file="items/test.wav"; + +DELETE FROM telemeta_physicalformat; +INSERT INTO telemeta_physicalformat (value) + SELECT DISTINCT physical_format FROM telemeta_collection + WHERE physical_format <> '' AND physical_format IS NOT NULL; + +DELETE FROM telemeta_publishingstatus; +INSERT INTO telemeta_publishingstatus (value) + SELECT DISTINCT publishing_status FROM telemeta_collection + WHERE publishing_status <> '' AND publishing_status IS NOT NULL; diff --git a/import/prototype/scripts/prepare b/import/prototype/scripts/prepare new file mode 100755 index 0000000..92a7618 --- /dev/null +++ b/import/prototype/scripts/prepare @@ -0,0 +1,43 @@ +#!/bin/bash + +if [ "$1" == "" ] +then + echo "Please provide the source directory" + exit 1 +fi + +src=$1 +tmpdir=crem_import +required="support.txt.gz phono.txt.gz" + +for f in $required +do + if ! [ -f $src/$f ] + then + echo "Can't find $f in $src" + exit 1 + fi +done + +if [ -d /tmp/$tmpdir ] +then + rm /tmp/$tmpdir/* +else + mkdir /tmp/$tmpdir +fi + +echo -n "Converting charset and cleaning text files.. " + +zcat $src/support.txt.gz | tail -n +2 \ + | sed 's/^ *//' | sed 's/ *\t */\t/g'| sed 's/ *$//' \ + | iconv -f WINDOWS-1252 -c -t latin1 \ + > /tmp/$tmpdir/support.txt + +zcat $src/phono.txt.gz | tail -n +2 \ + | sed 's/^ *//' | sed 's/ *\t */\t/g'| sed 's/ *$//' \ + | iconv -f WINDOWS-1252 -c -t latin1 | sed '/^\t/d' \ + > /tmp/$tmpdir/phono.txt + +echo "Done" + + diff --git a/import/prototype/src/2007-05-09/phono.txt.gz b/import/prototype/src/2007-05-09/phono.txt.gz new file mode 100755 index 0000000..fd0be4a Binary files /dev/null and b/import/prototype/src/2007-05-09/phono.txt.gz differ diff --git a/import/prototype/src/2007-05-09/support.txt.gz b/import/prototype/src/2007-05-09/support.txt.gz new file mode 100755 index 0000000..a7ec8c6 Binary files /dev/null and b/import/prototype/src/2007-05-09/support.txt.gz differ