From: Thomas Fillon Date: Tue, 19 Jul 2016 11:12:13 +0000 (+0200) Subject: Initial commit. Partial models covering Notice Documents X-Git-Url: https://git.parisson.com/?a=commitdiff_plain;h=7a06153749e435f8369fb5b7188a2b82dc11cce1;p=telemeta_mcm.git Initial commit. Partial models covering Notice Documents --- 7a06153749e435f8369fb5b7188a2b82dc11cce1 diff --git a/mcm/__init__.py b/mcm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mcm/admin.py b/mcm/admin.py new file mode 100644 index 0000000..c5a268f --- /dev/null +++ b/mcm/admin.py @@ -0,0 +1,58 @@ +from django.contrib import admin + +# Register your models here. +from .models import Author, Keyword, Reference +from .models import Event, EventType, EventVenue +from .models import Document +from .models import Notice + + +class AuthorAdmin(admin.ModelAdmin): + list_filter = ('name',) + search_fields = ['name'] + ordering = ['name'] + +class KeywordAdmin(admin.ModelAdmin): + list_filter = ('name',) + search_fields = ['name'] + ordering = ['name'] + +class ReferenceAdmin(admin.ModelAdmin): + list_filter = ('name',) + search_fields = ['name'] + ordering = ['name'] + +class EventAdmin(admin.ModelAdmin): + list_filter = ('name',) + search_fields = ['name'] + ordering = ['name'] + +class EventTypeAdmin(admin.ModelAdmin): + list_filter = ('name',) + search_fields = ['name'] + ordering = ['name'] + +class EventVenueAdmin(admin.ModelAdmin): + list_filter = ('name',) + search_fields = ['name'] + ordering = ['name'] + +class NoticeAdmin(admin.ModelAdmin): + list_display = ('code', 'title') + list_filter = ('code', 'title') + search_fields = ['title', 'code'] + filter_horizontal = ('authors', 'keywords',) + + + +admin.site.register(Author, AuthorAdmin) +admin.site.register(Keyword, KeywordAdmin) +admin.site.register(Reference, ReferenceAdmin) + +admin.site.register(Event, EventAdmin) +admin.site.register(EventType, EventTypeAdmin) +admin.site.register(EventVenue, EventVenueAdmin) + + +admin.site.register(Document) +admin.site.register(Notice, NoticeAdmin) diff --git a/mcm/apps.py b/mcm/apps.py new file mode 100644 index 0000000..80f5887 --- /dev/null +++ b/mcm/apps.py @@ -0,0 +1,7 @@ +from __future__ import unicode_literals + +from django.apps import AppConfig + + +class McmConfig(AppConfig): + name = 'mcm' diff --git a/mcm/management/__init__.py b/mcm/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mcm/management/commands/__init__.py b/mcm/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mcm/management/commands/find_duplicate.py b/mcm/management/commands/find_duplicate.py new file mode 100644 index 0000000..3ccc6c0 --- /dev/null +++ b/mcm/management/commands/find_duplicate.py @@ -0,0 +1,23 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import Count + +from ...models import Document +from ...models import Author +from ...models import Notice + + +class Command(BaseCommand): + help = 'Import items from XML' + + def handle(self, *args, **options): + + duplicate = Notice.objects.values('code').annotate(Count('id')).order_by().filter(id__count__gt=1) + + for item in duplicate: + code = item['code'] + docs = Notice.objects.filter(code=code) + print ('-----------') + print('code %s' % code) + for doc in docs: + print('\t Doc old id : %s' % doc.old_id) + print('\t Doc title : %s' % doc.title) diff --git a/mcm/management/commands/import_xml.py b/mcm/management/commands/import_xml.py new file mode 100644 index 0000000..8454b15 --- /dev/null +++ b/mcm/management/commands/import_xml.py @@ -0,0 +1,156 @@ +# -*- coding: utf-8 -* + +from django.core.management.base import BaseCommand, CommandError + +from ...models import Document +from ...models import Notice +from ...models import Author, Keyword, Reference +from ...models import Event, EventType, EventVenue + +#import lxml.etree.ElementTree as ET +import xml.etree.ElementTree as ET +import os + +replacements = {'auteur_affiche_(dessin)>': 'auteur_affiche_dessin>', + '[record_no]>': 'record_no>', + '<': '???<', + '&<': '?<', + '\00': '', + '\373': 'û', + 'sa lta nata': 'saltanata', + 'Gugak FM': 'Gugak FM', + '\xf5': 'ı', + '\x1e': '', + '\x02': '', + '\xf9': '?', + '\xb7': '?'} + +def cleanup_xml(xml_file): + root, ext = os.path.splitext(xml_file) + clean_xml_file = ''.join([root,'_clean', ext]) + if os.path.exists(clean_xml_file): + return clean_xml_file + line_err = None + with open(xml_file, 'U') as infile, open(clean_xml_file, 'w') as outfile: + l = 1 + for line in infile: + if l == line_err: + print repr(line) + for src, target in replacements.iteritems(): + line = line.replace(src, target) + #if line.startswith('Chant et flûte taegu'): + # print repr(line) + outfile.write(line) + l += 1 + + return clean_xml_file + + +class Command(BaseCommand): + help = 'Import items from XML' + + def add_arguments(self, parser): + parser.add_argument('xml_file', type=str) + + def handle(self, *args, **options): + xml_file = options['xml_file'] + clean_xml_file = cleanup_xml(xml_file) + parser = ET.XMLParser(encoding="utf-8") + tree = ET.parse(clean_xml_file, parser=parser) + root = tree.getroot() + + # Remove all object in Database + Document.objects.all().delete() + + document_traite = 0 + document_non_traite = 0 + erreur_date_parution = 0 + erreur_date_indexation = 0 + + for document in root.iter('Document'): + #print '------------' + doc_type = document.find('Type').text + #print doc_type + if doc_type == 'a-Notice spectacle': + document_traite +=1 + record_no = document.find('record_no').text + code = document.find('Cote').text + + + event_type = document.find('Type_Manifestation').text + event_type_obj, c = EventType.objects.get_or_create(name=event_type) + + try: + event_venue = document.find('Lieu_Manifestation').text + event_venue_obj, c = EventVenue.objects.get_or_create(name=event_venue) + except AttributeError: + if document.find('Lieu_Manifestation') is None: + event_venue_obj = None + try: + event = document.find('Festival_et_Manifestation').text + event_obj, c = Event.objects.get_or_create(name=event) + except AttributeError: + if document.find('Festival_et_Manifestation') is None: + event_obj = None + + notice, c = Notice.objects.get_or_create(old_id=record_no, + code=code, + event=event_obj, + event_type=event_type_obj, + event_venue=event_venue_obj) + + notice.title = document.find('Titre').text + try: + notice.text = document.find('Texte').text + except: + pass + + + import datetime + try: + release_date = datetime.datetime.strptime(document.find('Date_de_parution').text,'%d/%m/%y').date() + except ValueError: + #if document.find('Date_de_parution').text == '2015/09/08': + # release_date = datetime.datetime.strptime('08/09/2015','%d/%m/%y').date() + release_date = None + erreur_date_parution += 1 + try: + indexation_date = datetime.datetime.strptime(document.find('Date_d_indexation').text,'%d/%m/%y').date() + except ValueError: + indexation_date = None + erreur_date_indexation +=1 + + ## print '---------' + ## print record_no + ## print code + ## print title + ## print release_date + ## print indexation_date + + # Authors + for author in document.findall('auteurs'): + author_obj, auth_c = Author.objects.get_or_create(name=author.text) + notice.authors.add(author_obj) + # Keywords + for keyword in document.findall('Mots-cles'): + keyword_obj, keyword_c = Keyword.objects.get_or_create(name=keyword.text) + notice.keywords.add(keyword_obj) + # Referencess + for ref in document.findall('Reference'): + ref_obj, ref_c = Reference.objects.get_or_create(name=ref.text) + notice.references.add(ref_obj) + + + notice.save() + else: + document_non_traite += 1 + + print '-*-*--*-*-*-*-*-*-*-*' + print 'document_traité : %d' % document_traite + print 'document_non_traité : %d' % document_non_traite + print 'erreur_date_parution : %d' % erreur_date_parution + print 'erreur_date_indexation : %d' % erreur_date_indexation + + + + diff --git a/mcm/models.py b/mcm/models.py new file mode 100644 index 0000000..d889249 --- /dev/null +++ b/mcm/models.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +from django.db import models +from django.utils.translation import ugettext_lazy as _ + + +# Create your models here. + +class BaseMany(models.Model): + name = models.CharField(max_length=100, blank=False, unique=True) + + def __unicode__(self): + return self.name + + class Meta: + abstract = True + +class Author(BaseMany): + pass + +class Keyword(BaseMany): + pass + +class Reference(BaseMany): + pass + +class EventType(BaseMany): + pass + +class EventVenue(BaseMany): + pass + +class Event(BaseMany): + pass + + + +DOCUMENT_TYPES = ( + ('a', 'Notice spectacle'), + ('b', 'Disque'), + ('c', 'Vidéo DVD&VHS'), + ('d', 'Vidéo en ligne'), + ('e', 'Site Internet'), + ('f', 'Ouvrage & Thèse'), + ('g', 'Revue'), + ('h', 'Article'), + ('i', 'Photo'), + ('j', 'Affiche - Brochure'), + ('k', 'Pédagogique'), + ('l', 'Objet') + ) + +class Document(models.Model): + code = models.CharField(_('code'), unique=False, blank=True, + max_length=200) # Cote + authors = models.ManyToManyField(Author) # Auteur + title = models.CharField(_('title'), max_length=200) # Titre + #doc_type = models.CharField(max_length=1, + # choices=DOCUMENT_TYPES) # Type + old_id = models.IntegerField(unique=True, blank=False) # Record No + references = models.ManyToManyField(Reference) #Le Chant du Monde + release_date = models.DateField(_('release date'), null=True, blank=True)#1995 + #Musique d'Indonésie + keywords = models.ManyToManyField(Keyword) #Toraja + text = models.TextField(default='') + #Indonésie + event_type = models.ForeignKey(EventType)#Danse + event_venue = models.ForeignKey(EventVenue, blank=True, null=True)#Le Rond Point, Théâtre Renaud-Barrault, Paris + event = models.ForeignKey(Event, blank=True, null=True)#Japon 93 + #Compact Disc Digital Audio + #69'14 + #Le chant du monde + #CNR 2741004 + indexation_date = models.DateField(_('indexation date'), null=True, blank=True) #21/02/16 + #5604 + #b-Disque + #<[record_no]>2300703 + #Le Chant du Monde ; Editions du CNRS, 1995 + def __unicode__(self): + return self.title + +class Notice(Document): + pass diff --git a/mcm/tests.py b/mcm/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/mcm/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/mcm/views.py b/mcm/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/mcm/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/telemeta_mcm/__init__.py b/telemeta_mcm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/telemeta_mcm/settings.py b/telemeta_mcm/settings.py new file mode 100644 index 0000000..390d710 --- /dev/null +++ b/telemeta_mcm/settings.py @@ -0,0 +1,122 @@ +""" +Django settings for telemeta_mcm project. + +Generated by 'django-admin startproject' using Django 1.9.2. + +For more information on this file, see +https://docs.djangoproject.com/en/1.9/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.9/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = '6-4a&jq=l-@27upu*zcp153l(y51v66(93v+w!o17^lc*l@7z)' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'mcm.apps.McmConfig', + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', +] + +MIDDLEWARE_CLASSES = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'telemeta_mcm.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'telemeta_mcm.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.9/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/1.9/topics/i18n/ + +LANGUAGE_CODE = 'fr-FR' + +TIME_ZONE = 'Europe/Paris' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.9/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/telemeta_mcm/urls.py b/telemeta_mcm/urls.py new file mode 100644 index 0000000..a975812 --- /dev/null +++ b/telemeta_mcm/urls.py @@ -0,0 +1,21 @@ +"""telemeta_mcm URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/1.9/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.conf.urls import url, include + 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) +""" +from django.conf.urls import url +from django.contrib import admin + +urlpatterns = [ + url(r'^admin/', admin.site.urls), +] diff --git a/telemeta_mcm/wsgi.py b/telemeta_mcm/wsgi.py new file mode 100644 index 0000000..df28cda --- /dev/null +++ b/telemeta_mcm/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for telemeta_mcm project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "telemeta_mcm.settings") + +application = get_wsgi_application()