From 7894bebf1edb10a69246e99c26b9da2ca0234b9a Mon Sep 17 00:00:00 2001 From: Aly Badr Date: Thu, 4 Feb 2021 17:14:44 +0100 Subject: [PATCH] operation logs: migrate legacy operation logs * Adds a CLI, used only in the RERO ILS instance, to migrate Virtua 'create' and 'update' operation logs from the RERO legacy production database (Virtua). Co-Authored-by: Aly Badr --- data/operation_logs.json | 1 + rero_ils/modules/cli.py | 2 + rero_ils/modules/operation_logs/api.py | 19 +++++- rero_ils/modules/operation_logs/cli.py | 82 ++++++++++++++++++++++++++ scripts/setup | 3 + 5 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 data/operation_logs.json create mode 100644 rero_ils/modules/operation_logs/cli.py diff --git a/data/operation_logs.json b/data/operation_logs.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/data/operation_logs.json @@ -0,0 +1 @@ +[] diff --git a/rero_ils/modules/cli.py b/rero_ils/modules/cli.py index 38d812418e..a275bf5668 100644 --- a/rero_ils/modules/cli.py +++ b/rero_ils/modules/cli.py @@ -73,6 +73,7 @@ from .ill_requests.cli import create_ill_requests from .items.cli import create_items, reindex_items from .loans.cli import create_loans +from .operation_logs.cli import migrate_virtua_operation_logs from .patrons.cli import import_users from .tasks import process_bulk_queue from .utils import get_record_class_from_schema_or_pid_type, read_json_record @@ -101,6 +102,7 @@ def fixtures(): fixtures.add_command(create_patterns) fixtures.add_command(create_ill_requests) fixtures.add_command(create_collections) +fixtures.add_command(migrate_virtua_operation_logs) @users.command('confirm') diff --git a/rero_ils/modules/operation_logs/api.py b/rero_ils/modules/operation_logs/api.py index b12bdd1220..47a7de9b95 100644 --- a/rero_ils/modules/operation_logs/api.py +++ b/rero_ils/modules/operation_logs/api.py @@ -19,7 +19,8 @@ from functools import partial -from .models import OperationLogIdentifier, OperationLogMetadata +from .models import OperationLogIdentifier, OperationLogMetadata, \ + OperationLogOperation from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..minters import id_minter @@ -59,6 +60,22 @@ class OperationLog(IlsRecord): provider = OperationLogProvider model_cls = OperationLogMetadata + @classmethod + def get_create_operation_log_by_resource_pid(cls, pid_type, record_pid): + """Return a create operation log for a given resource and pid. + + :param pid_type: resource pid type. + :param record_pid: record pid. + """ + search = OperationLogsSearch() + search = search.filter('term', record__pid=record_pid)\ + .filter('term', record__type=pid_type)\ + .filter('term', operation=OperationLogOperation.CREATE) + oplgs = search.source(['pid']).scan() + try: + return OperationLog.get_record_by_pid(next(oplgs).pid) + except StopIteration: + return None class OperationLogsIndexer(IlsRecordsIndexer): """Operation log indexing class.""" diff --git a/rero_ils/modules/operation_logs/cli.py b/rero_ils/modules/operation_logs/cli.py new file mode 100644 index 0000000000..032880ed65 --- /dev/null +++ b/rero_ils/modules/operation_logs/cli.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2021 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Click command-line interface for operation_log record management.""" + +from __future__ import absolute_import, print_function + +import json + +import click +from flask import current_app +from flask.cli import with_appcontext + +from rero_ils.modules.operation_logs.api import OperationLog +from rero_ils.modules.operation_logs.models import OperationLogOperation +from rero_ils.modules.utils import extracted_data_from_ref + +from ..utils import read_json_record + + +@click.command('migrate_virtua_operation_logs') +@click.option('-v', '--verbose', 'verbose', is_flag=True, default=False) +@click.option('-d', '--debug', 'debug', is_flag=True, default=False) +@click.option('-l', '--lazy', 'lazy', is_flag=True, default=False) +@click.argument('infile', type=click.File('r')) +@with_appcontext +def migrate_virtua_operation_logs(infile, verbose, debug, lazy): + """Migrate Virtua operation log records in reroils. + + :param infile: Json operation log file. + :param lazy: lazy reads file + """ + enabled_logs = current_app.config.get('RERO_ILS_ENABLE_OPERATION_LOG') + click.secho('Migrate Virtua operation log records:', fg='green') + if lazy: + # try to lazy read json file (slower, better memory management) + data = read_json_record(infile) + else: + # load everything in memory (faster, bad memory management) + data = json.load(infile) + index_count = 0 + with click.progressbar(data) as bar: + for oplg in bar: + try: + operation = oplg.get('operation') + resource = extracted_data_from_ref( + oplg.get('record').get('$ref'), data='resource') + pid_type = enabled_logs.get(resource) + if pid_type and operation == OperationLogOperation.CREATE: + # The virtua create operation log overrides the reroils create + # operation log, the method to use is UPDATE + record_pid = extracted_data_from_ref( + oplg.get('record').get('$ref'), data='pid') + + create_rec = \ + OperationLog.get_create_operation_log_by_resource_pid( + pid_type, record_pid) + if create_rec: + create_rec.update(oplg, dbcommit=True, reindex=True) + elif pid_type and operation == OperationLogOperation.UPDATE: + # The virtua update operation log is a new entry in the reroils + # operation log, the method to use is CREATE + OperationLog.create(data=oplg, dbcommit=True, reindex=True) + except: + pass + index_count += len(data) + click.echo('created {index_count} operation logs.'.format( + index_count=index_count)) diff --git a/scripts/setup b/scripts/setup index 530591b40f..ed3d57d095 100755 --- a/scripts/setup +++ b/scripts/setup @@ -483,6 +483,9 @@ then diff --suppress-common-lines --side-by-side tmp/mapping_before_setup.txt tmp/mapping_after_setup.txt fi +info_msg "- Load Virtua operation logs: ${DATA_PATH}/operation_logs.json" +eval ${PREFIX} invenio fixtures migrate_virtua_operation_logs ${DATA_PATH}/operation_logs.json ${CREATE_LAZY} ${DONT_STOP} + date success_msg "Perfect ${PROGRAM}! See you soon…" exit 0