From 811878e2167cee535acc0e39f05610de3fe7f3ce Mon Sep 17 00:00:00 2001 From: Guillaume Pellerin Date: Mon, 22 Nov 2021 11:14:10 +0100 Subject: [PATCH] add recovery env, allow null student.user (temporary), update student copy command --- app/settings.py | 19 +++- env/db-recovery.yml | 17 +++ env/recovery.env | 14 +++ .../commands/teleforma-copy-students.py | 106 ++++++++++++++++++ .../migrations/0013_auto_20211122_0057.py | 25 +++++ 5 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 env/db-recovery.yml create mode 100644 env/recovery.env create mode 100644 teleforma/management/commands/teleforma-copy-students.py create mode 100644 teleforma/migrations/0013_auto_20211122_0057.py diff --git a/app/settings.py b/app/settings.py index adb49921..bff06868 100644 --- a/app/settings.py +++ b/app/settings.py @@ -14,6 +14,8 @@ DEBUG_ENV = os.environ.get('DEBUG') == 'True' DEBUG = DEBUG_ENV TEMPLATE_DEBUG = DEBUG +RECOVERY = True + # disable to debug websocket and improve performance DEBUG_TOOLBAR = False @@ -66,9 +68,24 @@ DATABASES = { 'HOST': os.environ.get('DB_HOST'), # Set to empty string for default. Not used with sqlite3. 'PORT': os.environ.get('DB_PORT'), - } + }, } +if RECOVERY: + DATABASES['recovery'] = { + # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + # Or path to database file if using sqlite3. + 'NAME': os.environ.get('POSTGRES_DATABASE'), + # Not used with sqlite3. + 'USER': os.environ.get('POSTGRES_USER'), + # Not used with sqlite3. + 'PASSWORD': os.environ.get('POSTGRES_PASSWORD_RECOVERY'), + # Set to empty string for localhost. Not used with sqlite3. + 'HOST': os.environ.get('DB_HOST_RECOVERY'), + # Set to empty string for default. Not used with sqlite3. + 'PORT': os.environ.get('DB_PORT'), + } # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name diff --git a/env/db-recovery.yml b/env/db-recovery.yml new file mode 100644 index 00000000..b05d9425 --- /dev/null +++ b/env/db-recovery.yml @@ -0,0 +1,17 @@ +version: '3' + +services: + db-recovery: + image: postgres:13 + env_file: + - env/recovery.env + environment: + PGDATA: /var/lib/postgresql/data/pgdata + volumes: + - ./var/lib/postgresql-recovery:/var/lib/postgresql/data:rw + - ./bin:/srv/bin + - ./var/backup/:/srv/backup + - /etc/localtime:/etc/localtime:ro + +volumes: + db-recovery: diff --git a/env/recovery.env b/env/recovery.env new file mode 100644 index 00000000..18b569ea --- /dev/null +++ b/env/recovery.env @@ -0,0 +1,14 @@ +DEBUG=True + +DB_PORT=5432 +DB_HOST=db +POSTGRES_HOST=db +POSTGRES_PASSWORD=mysecretpassword +POSTGRES_DATABASE=teleforma +POSTGRES_DB=teleforma +POSTGRES_USER=teleforma +POSTGRES_HOST_AUTH_METHOD=trust + +DB_HOST_RECOVERY=db-recovery +POSTGRES_HOST_RECOVERY=db-recovery +POSTGRES_PASSWORD_RECOVERY=EtOdlerlOt8ob diff --git a/teleforma/management/commands/teleforma-copy-students.py b/teleforma/management/commands/teleforma-copy-students.py new file mode 100644 index 00000000..f0e5c835 --- /dev/null +++ b/teleforma/management/commands/teleforma-copy-students.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +from optparse import make_option +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.contrib.auth.models import User +from django.template.defaultfilters import slugify +from teleforma.models import * +import logging +import os +from copy import deepcopy +from teleforma.models.crfpa import Profile +from teleforma.forms import get_unique_username + + +class Logger: + """A logging object""" + + def __init__(self, file): + self.logger = logging.getLogger('myapp') + self.hdlr = logging.FileHandler(file) + self.formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + self.hdlr.setFormatter(self.formatter) + self.logger.addHandler(self.hdlr) + self.logger.setLevel(logging.INFO) + + +class Command(BaseCommand): + help = "Copy students from one DB to another" + period_name = 'Annuelle' + db_from = 'recovery' + db_to = 'default' + logger = Logger('/var/log/app/sudent_import_recovery.log') + + def handle(self, *args, **options): + period = Period.objects.get(name=self.period_name) + + students_from = Student.objects.using(self.db_from).filter(period=period) + students_to = Student.objects.using(self.db_to).filter(period=period) + + user_tmp, c = User.objects.using(self.db_to).get_or_create(username='tmp') + + self.logger.logger.info('Number of student in from ' + str(students_from.count())) + self.logger.logger.info('Number of student in to' + str(students_to.count())) + + students_to_email = [student.user.email for student in students_to if (hasattr(student, 'user') and hasattr(student.user, 'email'))] + + new_students = [] + + for student in students_from: + # print(student) + if student.trainings.all(): + if hasattr(student, 'user'): + if not student.user.email in students_to_email: + new_students.append(student) + + self.logger.logger.info('Number of new students to copy' + str(len(new_students)) + '\n') + + for student in new_students[100:]: + user = deepcopy(student.user) + payments = deepcopy(student.payments.all()) + discounts = deepcopy(student.discounts.all()) + optional_fees = deepcopy(student.optional_fees.all()) + paybacks = deepcopy(student.paybacks.all()) + trainings = student.trainings.all() + + user.pk = None + user.username = get_unique_username(user.first_name, user.last_name) + user.save(using=self.db_to) + + student.pk = None + student.user = None + student.save(using=self.db_to) + student.user = user + student.save(using=self.db_to) + + for training in trainings: + training_to = Training.objects.using(self.db_to).get(name=training.name, period=period) + student.trainings.add(training_to) + + for payment in payments: + payment.pk = None + payment.save(using=self.db_to) + payment.student = student + payment.save(using=self.db_to) + + for discount in discounts: + discount.pk = None + discount.save(using=self.db_to) + discount.student = student + discount.save(using=self.db_to) + + for optional_fee in optional_fees: + optional_fee.pk = None + optional_fee.save(using=self.db_to) + optional_fee.student = student + optional_fee.save(using=self.db_to) + + for payback in paybacks: + payback.pk = None + payback.save(using=self.db_to) + payback.student = student + payback.save(using=self.db_to) + + self.logger.logger.info(str(student) + ' ' + student.user.username) + diff --git a/teleforma/migrations/0013_auto_20211122_0057.py b/teleforma/migrations/0013_auto_20211122_0057.py new file mode 100644 index 00000000..ec54c4ca --- /dev/null +++ b/teleforma/migrations/0013_auto_20211122_0057.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.3 on 2021-11-22 00:57 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('teleforma', '0012_merge_0011_coursetype_order_0011_merge_20210929_1024'), + ] + + operations = [ + migrations.AlterModelOptions( + name='coursetype', + options={'ordering': ['order'], 'verbose_name': 'course type'}, + ), + migrations.AlterField( + model_name='student', + name='user', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='student', to=settings.AUTH_USER_MODEL, unique=True, verbose_name='user'), + ), + ] -- 2.39.5