From 03591bf1e3029816fe52681cb15b62e91da49935 Mon Sep 17 00:00:00 2001 From: Peter Weber Date: Wed, 17 Feb 2021 17:07:45 +0100 Subject: [PATCH] metadata: import, export, create csv data files * Implements import and export of resources in csv format. * Implements creation of csv files from json files. * Changes from sqlite to postgres as test DB. Co-Authored-by: Peter Weber --- ...dencies.json => pid_dependencies_big.json} | 0 data/pid_dependencies_small.json | 255 ++++++++++++ rero_ils/modules/api.py | 11 +- rero_ils/modules/cli.py | 368 +++++++++++++++++- rero_ils/modules/errors.py | 4 - rero_ils/modules/patrons/api.py | 12 +- rero_ils/modules/providers.py | 20 +- rero_ils/modules/utils.py | 335 +++++++++++++++- scripts/create_csvs | 174 +++++++++ scripts/setup | 23 +- scripts/setup_csv | 358 +++++++++++++++++ tests/api/holdings/test_patterns.py | 6 +- tests/api/items/test_items_rest.py | 18 +- tests/api/locations/test_locations_rest.py | 17 +- tests/conftest.py | 2 + tests/data/001.pids | 1 + tests/data/documents.xml | 4 +- .../circ_policies/test_circ_policies_api.py | 4 +- tests/ui/holdings/test_holdings_api.py | 16 +- tests/ui/holdings/test_holdings_patterns.py | 6 +- tests/ui/item_types/test_item_types_api.py | 12 +- tests/ui/items/test_items_api.py | 4 +- tests/ui/locations/test_locations_api.py | 12 +- .../organisations/test_organisations_api.py | 2 +- .../test_patron_transaction_events_api.py | 8 +- .../test_patron_transactions_api.py | 8 +- tests/ui/patrons/test_patrons_api.py | 9 +- tests/ui/templates/test_templates_api.py | 8 +- tests/ui/test_api.py | 54 +-- tests/unit/test_circ_policies_jsonschema.py | 11 +- tests/unit/test_cli.py | 43 +- tests/unit/test_csv.py | 105 +++++ tests/unit/test_ill_requests_jsonschema.py | 5 +- tests/unit/test_items_jsonschema.py | 5 +- tests/unit/test_patron_types_jsonschema.py | 7 +- 35 files changed, 1775 insertions(+), 152 deletions(-) rename data/{pid_dependencies.json => pid_dependencies_big.json} (100%) create mode 100644 data/pid_dependencies_small.json create mode 100755 scripts/create_csvs create mode 100755 scripts/setup_csv create mode 100644 tests/data/001.pids create mode 100644 tests/unit/test_csv.py diff --git a/data/pid_dependencies.json b/data/pid_dependencies_big.json similarity index 100% rename from data/pid_dependencies.json rename to data/pid_dependencies_big.json diff --git a/data/pid_dependencies_small.json b/data/pid_dependencies_small.json new file mode 100644 index 0000000000..4ddc8cba14 --- /dev/null +++ b/data/pid_dependencies_small.json @@ -0,0 +1,255 @@ +[ + { + "name": "organisation", + "filename": "organisations.json" + }, + { + "name": "library", + "filename": "libraries.json", + "dependencies": [ + { + "name": "organisation" + } + ] + }, + { + "name": "location", + "filename": "locations.json", + "dependencies": [ + { + "name": "library" + } + ] + }, + { + "name": "pickup_location", + "filename": "locations.json" + }, + { + "name": "item_type", + "filename": "item_types.json", + "dependencies": [ + { + "name": "organisation" + } + ] + }, + { + "name": "patron_type", + "filename": "patron_types.json", + "dependencies": [ + { + "name": "organisation" + }, + { + "name": "library_exceptions", + "optional": "True", + "sublist": [ + { + "name": "library" + } + ] + } + ] + }, + { + "name": "circulation_policie", + "filename": "circulation_policies.json", + "dependencies": [ + { + "name": "organisation" + }, + { + "name": "settings", + "optional": "True", + "sublist": [ + { + "name": "patron_type" + }, + { + "name": "item_type" + } + ] + }, + { + "name": "library", + "optional": "True" + } + ] + }, + { + "name": "vendor", + "filename": "vendors.json", + "dependencies": [ + { + "name": "organisation" + } + ] + }, + { + "name": "patron", + "filename": "users.json", + "dependencies": [ + { + "name": "library", + "optional": "True" + }, + { + "name": "patron", + "optional": "True", + "sublist": [ + { + "name": "patron_type", + "optional": "True" + } + ] + } + ] + }, + { + "name": "document", + "filename": "documents_small.json" + }, + { + "name": "holding", + "filename": "holdings_small.json", + "dependencies": [ + { + "name": "location" + }, + { + "name": "circulation_category", + "ref": "item_type" + }, + { + "name": "document" + } + ] + }, + { + "name": "item", + "filename": "items_small.json", + "dependencies": [ + { + "name": "location" + }, + { + "name": "document" + }, + { + "name": "item_type" + }, + { + "name": "holding", + "optional": "True" + }, + { + "name": "organisation", + "optional": "True" + } + ] + }, + { + "name": "patterns", + "filename": "patterns.json" + }, + { + "name": "budget", + "filename": "budgets.json", + "dependencies": [ + { + "name": "organisation", + "optional": "True" + } + ] + }, + { + "name": "collection", + "filename": "collections.json", + "dependencies": [ + { + "name": "librarie", + "ref": "libraries", + "optional": "True" + }, + { + "name": "item", + "ref": "items", + "optional": "True" + }, + { + "name": "organisation" + } + ] + }, + { + "name": "acq_account", + "filename": "acq_accounts.json", + "dependencies": [ + { + "name": "budget" + }, + { + "name": "library" + }, + { + "name": "organisation", + "optional": "True" + } + ] + }, + { + "name": "template", + "filename": "templates.json", + "dependencies": [ + { + "name": "organisation" + }, + { + "name": "creator", + "ref": "patron" + } + ] + }, + { + "name": "loan", + "filename": "loans.json", + "dependencies": [ + { + "name": "item", + "optional": "True" + }, + { + "name": "patron", + "optional": "True" + }, + { + "name": "document", + "optional": "True" + }, + { + "name": "organisation", + "optional": "True" + } + ] + }, + { + "name": "local_field", + "filename": "local_fields.json", + "dependencies": [ + { + "name": "organisation", + "optional": "True" + }, + { + "name": "parent", + "optional": "True", + "refs": { + "document": "documents", + "item": "items", + "holding": "holdings" + } + } + ] + } +] diff --git a/rero_ils/modules/api.py b/rero_ils/modules/api.py index 3d4ea05872..d816febb7a 100644 --- a/rero_ils/modules/api.py +++ b/rero_ils/modules/api.py @@ -40,11 +40,11 @@ from invenio_records_rest.utils import obj_or_import_string from invenio_search import current_search from invenio_search.api import RecordsSearch +from jsonschema.exceptions import ValidationError from kombu.compat import Consumer from sqlalchemy import text from sqlalchemy.orm.exc import NoResultFound -from .errors import RecordValidationError from .utils import extracted_data_from_ref @@ -139,7 +139,7 @@ def _validate(self, **kwargs): not_required=self.pids_exist_check.get('not_required', {}) ) or True if validation_message is not True: - raise RecordValidationError(validation_message) + raise ValidationError(validation_message) return json def extended_validation(self, **kwargs): @@ -433,6 +433,13 @@ def organisation_pid(self): """Get organisation pid for circulation policy.""" return extracted_data_from_ref(self.get('organisation')) + @classmethod + def get_metadata_identifier_names(cls): + """Get metadata and identifier table names.""" + metadata = cls.model_cls.__tablename__ + identifier = cls.provider.identifier + return metadata, identifier + class IlsRecordsIndexer(RecordIndexer): """Indexing class for ils.""" diff --git a/rero_ils/modules/cli.py b/rero_ils/modules/cli.py index 7409bc6265..3ac7f57cfa 100644 --- a/rero_ils/modules/cli.py +++ b/rero_ils/modules/cli.py @@ -30,9 +30,11 @@ import sys import traceback from collections import OrderedDict +from datetime import datetime from glob import glob from pprint import pprint from time import sleep +from uuid import uuid4 import click import polib @@ -64,6 +66,7 @@ from .api import IlsRecordsIndexer from .collections.cli import create_collections +from .contributions.api import Contribution from .contributions.tasks import create_mef_record_online from .documents.api import Document, DocumentsSearch from .documents.dojson.contrib.marc21tojson import marc21 @@ -72,10 +75,14 @@ from .ill_requests.cli import create_ill_requests from .items.cli import create_items, reindex_items from .loans.cli import create_loans, load_virtua_transactions +from .monitoring import Monitoring from .operation_logs.cli import migrate_virtua_operation_logs from .patrons.cli import import_users, users_validate from .tasks import process_bulk_queue -from .utils import get_record_class_from_schema_or_pid_type, \ +from .utils import bulk_load_metadata, bulk_load_pids, bulk_load_pidstore, \ + bulk_save_metadata, bulk_save_pids, bulk_save_pidstore, \ + csv_metadata_line, csv_pidstore_line, \ + get_record_class_from_schema_or_pid_type, number_records_in_file, \ read_json_record, read_xml_record from ..modules.providers import append_fixtures_new_identifiers from ..modules.utils import get_schema_for_resource @@ -247,7 +254,7 @@ def check_json(paths, replace, indent, sort_keys, verbose): except ValueError as error: click.echo(fname + ': ', nl=False) click.secho('Invalid JSON', fg='red', nl=False) - click.echo(' -- ' + error.msg) + click.echo(f' -- {error}') error_cnt = 1 tot_error_cnt += error_cnt @@ -453,13 +460,19 @@ def count_cli(infile, lazy): help='verbose') @click.option('-w', '--wait', 'wait', is_flag=True, default=False, help="wait for enqueued tasks to finish") +@click.option('-o', '--out_file', 'outfile', type=click.File('w'), + default=None) @with_appcontext -def get_all_mef_records(infile, lazy, verbose, enqueue, wait): +def get_all_mef_records(infile, lazy, verbose, enqueue, wait, outfile): """Get all contributions for given document file.""" click.secho( f'Get all contributions for {infile.name}.', fg='green' ) + if outfile: + outfile.write('[') + contribution_schema = get_schema_for_resource('cont') + click.secho('Write to {file_name}.'.format(file_name=outfile.name)) if lazy: # try to lazy read json file (slower, better memory management) records = read_json_record(infile) @@ -474,15 +487,37 @@ def get_all_mef_records(infile, lazy, verbose, enqueue, wait): if ref and not refs.get(ref): count += 1 refs[ref] = 1 - if enqueue: - msg = create_mef_record_online.delay(ref) + if outfile: + try: + ref_split = ref.split('/') + ref_type = ref_split[-2] + ref_pid = ref_split[-1] + data = Contribution._get_mef_data_by_type( + pid=ref_pid, + pid_type=ref_type + ) + metadata = data['metadata'] + metadata['$schema'] = contribution_schema + if count > 1: + outfile.write(',') + for line in json.dumps(metadata, indent=2).split('\n'): + outfile.write('\n ' + line) + msg = 'ok' + except Exception as err: + msg = err else: - pid, online = create_mef_record_online(ref) - msg = f'contribution pid: {pid} {online}' + if enqueue: + msg = create_mef_record_online.delay(ref) + else: + pid, online = create_mef_record_online(ref) + msg = f'contribution pid: {pid} {online}' if verbose: - click.echo(f'{count:<10}ref: {ref}\t{msg}') - if enqueue and wait: - wait_empty_tasks(delay=3, verbose=True) + click.echo(f"{count:<10}ref: {ref}\t{msg}") + if outfile: + outfile.write('\n]\n') + else: + if enqueue and wait: + wait_empty_tasks(delay=3, verbose=True) click.echo(f'Count refs: {count}') @@ -1141,17 +1176,16 @@ def run(delayed, concurrency, with_stats, version_type=None, queue=None, @click.option('--yes-i-know', is_flag=True, callback=abort_if_false, expose_value=False, prompt='Do you really want to reindex all records?') -@click.option('-t', '--pid-type', multiple=True, required=True) +@click.option('-t', '--pid-types', multiple=True, required=True) @click.option('-n', '--no-info', 'no_info', is_flag=True, default=True) @with_appcontext -def reindex(pid_type, no_info): +def reindex(pid_types, no_info): """Reindex all records. :param pid_type: Pid type. """ - for type in pid_type: + for type in pid_types: click.secho(f'Sending {type} to indexing queue ...', fg='green') - query = ( x[0] for x in PersistentIdentifier.query. filter_by(object_type='rec', status=PIDStatus.REGISTERED). @@ -1165,6 +1199,49 @@ def reindex(pid_type, no_info): ) +@utils.command('reindex_missing') +@click.option('-t', '--pid-types', multiple=True, required=True) +@click.option('-v', '--verbose', 'verbose', is_flag=True, default=False) +@with_appcontext +def reindex_missing(pid_types, verbose): + """Index all missing records. + + :param pid_type: Pid type. + """ + for p_type in pid_types: + click.secho( + 'Indexing missing {pid_type}: '.format(pid_type=p_type), + fg='green', + nl=False + ) + record_class = get_record_class_from_schema_or_pid_type( + pid_type=p_type + ) + if not record_class: + click.secho( + 'ERROR pid type does not exist!', + fg='red', + ) + continue + pids_es, pids_db, pids_es_double, index = \ + Monitoring.get_es_db_missing_pids(p_type) + click.secho( + '{count}'.format(count=len(pids_db)), + fg='green', + ) + for idx, pid in enumerate(pids_db, 1): + record = record_class.get_record_by_pid(pid) + res = record.reindex() + if verbose: + click.secho( + '{count}\t{pid_type}\t{pid}'.format( + count=idx, + pid_type=p_type, + pid=pid + ) + ) + + def get_loc_languages(verbose=False): """Get languages from LOC.""" languages = {} @@ -1346,7 +1423,7 @@ def change_po(po, values): @utils.command('check_pid_dependencies') @click.option('-i', '--dependency_file', 'dependency_file', - type=click.File('r'), default='./data/pid_dependencies.json') + type=click.File('r'), default='./data/pid_dependencies_big.json') @click.option('-d', '--directory', 'directory', default='./data') @click.option('-v', '--verbose', 'verbose', is_flag=True, default=False) def check_pid_dependencies(dependency_file, directory, verbose): @@ -1715,3 +1792,264 @@ def add_cover_urls(verbose): url = get_cover_art(record=record, save_cover_url=True) if verbose: click.echo(f'{idx}:\tdocument: {pid}\t{url}') + + +@fixtures.command('create_csv') +@click.argument('record_type') +@click.argument('json_file') +@click.argument('output_directory') +@click.option('-l', '--lazy', 'lazy', is_flag=True, default=False) +@click.option('-v', '--verbose', 'verbose', is_flag=True, default=False) +@click.option('-p', '--create_pid', 'create_pid', is_flag=True, default=False) +@with_appcontext +def create_csv(record_type, json_file, output_directory, lazy, verbose, + create_pid): + """Create csv files from json. + + :param verbose: Verbose. + """ + click.secho( + "Create CSV files for: {record_type} from: {file_name}".format( + record_type=record_type, + file_name=json_file, + ), + fg='green' + ) + + path = current_jsonschemas.url_to_path( + get_schema_for_resource(record_type) + ) + add_schema = get_schema_for_resource(record_type) + schema = current_jsonschemas.get_schema(path=path) + schema = _records_state.replace_refs(schema) + count = 0 + errors_count = 0 + with open(json_file) as infile: + if lazy: + # try to lazy read json file (slower, better memory management) + records = read_json_record(infile) + else: + # load everything in memory (faster, bad memory management) + records = json.load(infile) + + file_name_pidstore = os.path.join( + output_directory, '{record_type}_pidstore.csv'.format( + record_type=record_type + ) + ) + click.secho('\t{name}'.format(name=file_name_pidstore), fg='green') + file_pidstore = open(file_name_pidstore, 'w') + file_name_metadata = os.path.join( + output_directory, '{record_type}_metadata.csv'.format( + record_type=record_type + ) + ) + click.secho('\t{name}'.format(name=file_name_metadata), fg='green') + file_metadata = open(file_name_metadata, 'w') + file_name_pids = os.path.join( + output_directory, '{record_type}_pids.csv'.format( + record_type=record_type + ) + ) + click.secho('\t{name}'.format(name=file_name_pids), fg='green') + file_pids = open(file_name_pids, 'w') + file_name_errors = os.path.join( + output_directory, '{record_type}_errors.json'.format( + record_type=record_type + ) + ) + file_errors = open(file_name_errors, 'w') + file_errors.write('[') + + for count, record in enumerate(records, 1): + pid = record.get('pid') + if create_pid: + pid = str(count) + record['pid'] = pid + uuid = str(uuid4()) + if verbose: + click.secho( + '{count}\t{record_type}\t{pid}:{uuid}'.format( + count=count, + record_type=record_type, + pid=pid, + uuid=uuid + ) + ) + date = str(datetime.utcnow()) + record['$schema'] = add_schema + try: + validate(record, schema) + file_metadata.write( + csv_metadata_line(record, uuid, date) + ) + file_pidstore.write( + csv_pidstore_line(record_type, pid, uuid, date) + ) + file_pids.write(pid + '\n') + except Exception as err: + click.secho( + '{count}\t{record_type}: {msg}'.format( + count=count, + record_type=record_type, + msg='Error validate in record: ' + ), + fg='red') + click.secho(str(err)) + if errors_count > 0: + file_errors.write(',') + errors_count += 1 + file_errors.write('\n') + for line in json.dumps(record, indent=2).split('\n'): + file_errors.write(' ' + line + '\n') + + file_pidstore.close() + file_metadata.close() + file_pids.close() + file_errors.write('\n]') + file_errors.close() + if errors_count == 0: + os.remove(file_name_errors) + click.secho( + 'Created: {count} Errors: {error_count}'.format( + count=count-errors_count, + error_count=errors_count + ), + fg='yellow' + ) + + +@fixtures.command('bulk_load') +@click.argument('record_type') +@click.argument('csv_metadata_file') +@click.option('-c', '--bulk_count', 'bulkcount', default=0, type=int, + help='Set the bulk load chunk size.') +@click.option('-r', '--reindex', 'reindex', help='add record to reindex.', + is_flag=True, default=False) +@click.option('-v', '--verbose', 'verbose', is_flag=True, default=False) +@with_appcontext +def bulk_load(record_type, csv_metadata_file, bulkcount, reindex, verbose): + """Agency record management. + + :param csv_metadata_file: metadata: CSV file. + :param bulk_count: Set the bulk load chunk size. + :param reindex: add record to reindex. + :param verbose: Verbose. + """ + if bulkcount > 0: + bulk_count = bulkcount + else: + bulk_count = current_app.config.get('BULK_CHUNK_COUNT', 100000) + + message = 'Load {record_type} CSV files into database.'.format( + record_type=record_type + ) + click.secho(message, fg='green') + file_name_metadata = csv_metadata_file + file_name_pidstore = file_name_metadata.replace('metadata', 'pidstore') + file_name_pids = file_name_metadata.replace('metadata', 'pids') + + record_counts = number_records_in_file(file_name_pidstore, 'csv') + message = ' Number of records to load: {count}'.format( + count=record_counts + ) + click.secho(message, fg='green') + + click.secho(' Load pids: {file_name}'.format( + file_name=file_name_pids + )) + bulk_load_pids(pid_type=record_type, ids=file_name_pids, + bulk_count=bulk_count, verbose=verbose) + click.secho(' Load pidstore: {file_name}'.format( + file_name=file_name_pidstore + )) + bulk_load_pidstore(pid_type=record_type, pidstore=file_name_pidstore, + bulk_count=bulk_count, verbose=verbose) + click.secho(' Load metatada: {file_name}'.format( + file_name=file_name_metadata + )) + bulk_load_metadata(pid_type=record_type, metadata=file_name_metadata, + bulk_count=bulk_count, verbose=verbose, reindex=reindex) + + +@fixtures.command('bulk_save') +@click.argument('output_directory') +@click.option('-t', '--pid_types', multiple=True, default=['all']) +@click.option('-d', '--deployment', 'deployment', is_flag=True, default=False) +@click.option('-v', '--verbose', 'verbose', is_flag=True, default=False) +@with_appcontext +def bulk_save(pid_types, output_directory, deployment, verbose): + """Record dump. + + :param pid_type: Records to export. + default=//all// + :param verbose: Verbose. + """ + file_name_tmp_pidstore = os.path.join( + output_directory, + 'tmp_pidstore.csv' + ) + try: + os.remove(file_name_tmp_pidstore) + except OSError: + pass + + all_pid_types = [] + endpoints = current_app.config.get('RECORDS_REST_ENDPOINTS') + for endpoint in endpoints: + all_pid_types.append(endpoint) + if pid_types[0] == 'all': + pid_types = all_pid_types + + for p_type in pid_types: + if p_type not in all_pid_types: + click.secho( + 'Error {pid_type} does not exist!'.format(pid_type=p_type), + fg='red' + ) + continue + # TODO: do we have to save loanid and how we can save it? + if p_type == 'loanid': + continue + click.secho( + 'Save {pid_type} CSV files to directory: {path}'.format( + pid_type=p_type, + path=output_directory, + ), + fg='green' + ) + file_prefix = endpoints[p_type].get('search_index') + if p_type in ['doc', 'hold', 'item', 'count']: + if deployment: + file_prefix += '_big' + else: + file_prefix += '_small' + file_name_metadata = os.path.join( + output_directory, + '{prefix}_metadata.csv'.format(prefix=file_prefix) + ) + bulk_save_metadata(pid_type=p_type, file_name=file_name_metadata, + verbose=verbose) + file_name_pidstore = os.path.join( + output_directory, + '{prefix}_pidstore.csv'.format(prefix=file_prefix) + ) + count = bulk_save_pidstore(pid_type=p_type, + file_name=file_name_pidstore, + file_name_tmp=file_name_tmp_pidstore, + verbose=verbose) + + file_name_pids = os.path.join( + output_directory, + '{prefix}_pids.csv'.format(prefix=file_prefix) + ) + bulk_save_pids(pid_type=p_type, file_name=file_name_pids, + verbose=verbose) + click.secho( + 'Saved records: {count}'.format(count=count), + fg='yellow' + ) + try: + os.remove(file_name_tmp_pidstore) + except OSError: + pass diff --git a/rero_ils/modules/errors.py b/rero_ils/modules/errors.py index 2d3d9d9c7c..d76c2e60cd 100644 --- a/rero_ils/modules/errors.py +++ b/rero_ils/modules/errors.py @@ -38,10 +38,6 @@ class MissingRequiredParameterError(RecordsError): """Exception raised when required parameter is missing.""" -class RecordValidationError(RecordsError): - """Exception raised when record is not validated.""" - - class RegularReceiveNotAllowed(Exception): """Holdings of type serials and irregular frequency.""" diff --git a/rero_ils/modules/patrons/api.py b/rero_ils/modules/patrons/api.py index ce137c375d..c0b0f295aa 100644 --- a/rero_ils/modules/patrons/api.py +++ b/rero_ils/modules/patrons/api.py @@ -25,11 +25,11 @@ from flask_babelex import gettext as _ from flask_login import current_user from invenio_circulation.proxies import current_circulation +from jsonschema.exceptions import ValidationError from werkzeug.local import LocalProxy from .models import PatronIdentifier, PatronMetadata from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch -from ..errors import RecordValidationError from ..fetchers import id_fetcher from ..libraries.api import Library from ..minters import id_minter @@ -161,7 +161,7 @@ def _validate(self, **kwargs): break self._validate_emails() if validation_message is not True: - raise RecordValidationError(validation_message) + raise ValidationError(validation_message) return json def _validate_emails(self): @@ -173,10 +173,10 @@ def _validate_emails(self): patron = self.get('patron') user = self._get_user_by_user_id(self.get('user_id')) if patron and patron.get('communication_channel') == 'email'\ - and user.email is None\ - and patron.get('additional_communication_email') is None: - raise RecordValidationError('At least one email should be defined ' - 'for an email communication channel.') + and user.email is None\ + and patron.get('additional_communication_email') is None: + raise ValidationError('At least one email should be defined ' + 'for an email communication channel.') @classmethod def create(cls, data, id_=None, delete_pid=False, diff --git a/rero_ils/modules/providers.py b/rero_ils/modules/providers.py index 9da8308633..2c3584fe2e 100644 --- a/rero_ils/modules/providers.py +++ b/rero_ils/modules/providers.py @@ -20,33 +20,27 @@ from __future__ import absolute_import, print_function import click -import sqlalchemy from invenio_db import db from invenio_pidstore.errors import PIDDoesNotExistError -from invenio_pidstore.models import PersistentIdentifier, PIDStatus +from invenio_pidstore.models import PIDStatus from invenio_pidstore.providers.base import BaseProvider -def append_fixtures_new_identifiers(identifier, pids, pid_type): +def append_fixtures_new_identifiers(identifier, pids, pid_type, limit=100000): """Insert pids into the indentifier table and update its sequence.""" idx = 0 for idx, pid in enumerate(pids, 1): db.session.add(identifier(recid=pid)) - if idx > 0 and idx % 100000 == 0: + if idx % limit == 0: click.echo(f'DB commit append: {idx}') db.session.commit() - max_pid = PersistentIdentifier.query.filter_by( - pid_type=pid_type - ).order_by(sqlalchemy.desc( - sqlalchemy.cast(PersistentIdentifier.pid_value, sqlalchemy.Integer) - )).first().pid_value - identifier._set_sequence(max_pid) - click.echo(f'DB commit append: {idx}') db.session.commit() + identifier._set_sequence(identifier.max()) + click.echo(f'DB commit append: {idx}') class Provider(BaseProvider): - """CircPolicy identifier provider. + """Identifier provider. 'identifier' and 'pid_type' must be set as following example OrganisationProvider = type( @@ -64,7 +58,7 @@ class Provider(BaseProvider): pid_provider = None """Provider name. The provider name is not recorded in the PID since the provider does not - provide any additional features besides creation of CircPolicy ids. + provide any additional features besides creation of ids. """ @classmethod diff --git a/rero_ils/modules/utils.py b/rero_ils/modules/utils.py index 8ff96736d3..2c5d6e9011 100644 --- a/rero_ils/modules/utils.py +++ b/rero_ils/modules/utils.py @@ -18,14 +18,19 @@ """Utilities for rero-ils editor.""" import cProfile +import os import pstats +import unicodedata from datetime import date, datetime, time from functools import wraps -from json import JSONDecodeError, JSONDecoder +from io import StringIO +from json import JSONDecodeError, JSONDecoder, dumps from time import sleep import click +import psycopg2 import pytz +import sqlalchemy from dateutil import parser from flask import current_app from invenio_cache.proxies import current_cache @@ -33,6 +38,7 @@ from invenio_records_rest.utils import obj_or_import_string from lazyreader import lazyread from lxml import etree +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT def cached(timeout=50, key_prefix='default', query_string=False): @@ -629,3 +635,330 @@ def get_timestamp(name): if not time_stamps: return None return time_stamps.get(name) + + +def csv_metadata_line(record, uuid, date): + """Build CSV metadata table line.""" + created_date = updated_date = date + sep = '\t' + data = unicodedata.normalize('NFC', dumps(record, ensure_ascii=False)) + metadata = ( + created_date, + updated_date, + uuid, + data, + '1', + ) + metadata_line = sep.join(metadata) + return metadata_line + '\n' + + +def csv_pidstore_line(pid_type, pid, uuid, date): + """Build CSV pidstore table line.""" + created_date = updated_date = date + sep = '\t' + pidstore_data = [ + created_date, + updated_date, + pid_type, + pid, + 'R', + 'rec', + uuid, + ] + pidstore_line = sep.join(pidstore_data) + return pidstore_line + '\n' + + +def raw_connection(): + """Return a raw connection to the database.""" + with current_app.app_context(): + URI = current_app.config.get('SQLALCHEMY_DATABASE_URI') + engine = sqlalchemy.create_engine(URI) + connection = engine.raw_connection() + connection.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + return connection + + +def db_copy_from(buffer, table, columns, raise_exception=True): + """Copy data from file to db.""" + connection = raw_connection() + cursor = connection.cursor() + try: + cursor.copy_from( + file=buffer, + table=table, + columns=columns, + sep='\t' + ) + connection.commit() + except psycopg2.DataError as error: + if raise_exception: + raise psycopg2.DataError(error) + else: + current_app.logger.error('data load error: {0}'.format(error)) + connection.close() + + +def db_copy_to(filehandle, table, columns, raise_exception=True): + """Copy data from db to file.""" + connection = raw_connection() + cursor = connection.cursor() + try: + cursor.copy_to( + file=filehandle, + table=table, + columns=columns, + sep='\t' + ) + cursor.connection.commit() + except psycopg2.DataError as error: + if raise_exception: + raise psycopg2.DataError(error) + else: + current_app.logger.error('data load error: {0}'.format(error)) + cursor.execute('VACUUM ANALYSE {table}'.format(table=table)) + cursor.close() + connection.close() + + +def bulk_load(pid_type, data, table, columns, bulk_count=0, verbose=False, + reindex=False): + """Bulk load pid_type data to table.""" + if bulk_count <= 0: + bulk_count = current_app.config.get('BULK_CHUNK_COUNT', 100000) + count = 0 + buffer = StringIO() + buffer_uuid = [] + index = -1 + if 'id' in columns: + index = columns.index('id') + start_time = datetime.now() + with open(data, 'r', encoding='utf-8', buffering=1) as input_file: + for line in input_file: + count += 1 + buffer.write(line.replace('\\', '\\\\')) + if index >= 0 and reindex: + buffer_uuid.append(line.split('\t')[index]) + if count % bulk_count == 0: + buffer.flush() + buffer.seek(0) + if verbose: + end_time = datetime.now() + diff_time = end_time - start_time + start_time = end_time + click.echo( + '{pid_type} copy from file: {count} {time}s'.format( + pid_type=pid_type, + count=count, + time=diff_time.seconds + ), + nl=False + ) + db_copy_from(buffer=buffer, table=table, columns=columns) + buffer.close() + + if index >= 0 and reindex: + do_bulk_index(uuids=buffer_uuid, doc_type=pid_type, + verbose=verbose) + buffer_uuid.clear() + else: + if verbose: + click.echo() + buffer = StringIO() + + if verbose: + end_time = datetime.now() + diff_time = end_time - start_time + click.echo( + '{pid_type} copy from file: {count} {time}s'.format( + pid_type=pid_type, + count=count, + time=diff_time.seconds + ), + nl=False + ) + buffer.flush() + buffer.seek(0) + db_copy_from(buffer=buffer, table=table, columns=columns) + buffer.close() + if index >= 0 and reindex: + do_bulk_index(uuids=buffer_uuid, doc_type=pid_type, + verbose=verbose) + buffer_uuid.clear() + else: + if verbose: + click.echo() + + +def bulk_load_metadata(pid_type, metadata, bulk_count=0, verbose=True, + reindex=False): + """Bulk load pid_type data to metadata table.""" + record_class = get_record_class_from_schema_or_pid_type(pid_type=pid_type) + table, identifier = record_class.get_metadata_identifier_names() + columns = ( + 'created', + 'updated', + 'id', + 'json', + 'version_id' + ) + bulk_load( + pid_type=pid_type, + data=metadata, + table=table, + columns=columns, + bulk_count=bulk_count, + verbose=verbose, + reindex=reindex + ) + + +def bulk_load_pidstore(pid_type, pidstore, bulk_count=0, verbose=True, + reindex=False): + """Bulk load pid_type data to metadata table.""" + table = 'pidstore_pid' + columns = ( + 'created', + 'updated', + 'pid_type', + 'pid_value', + 'status', + 'object_type', + 'object_uuid', + ) + bulk_load( + pid_type=pid_type, + data=pidstore, + table=table, + columns=columns, + bulk_count=bulk_count, + verbose=verbose, + reindex=reindex + ) + + +def bulk_load_pids(pid_type, ids, bulk_count=0, verbose=True, reindex=False): + """Bulk load pid_type data to id table.""" + record_class = get_record_class_from_schema_or_pid_type(pid_type=pid_type) + metadata, identifier = record_class.get_metadata_identifier_names() + columns = ('recid', ) + bulk_load( + pid_type=pid_type, + data=ids, + table=identifier.__tablename__, + columns=columns, + bulk_count=bulk_count, + verbose=verbose, + reindex=reindex + ) + max_pid = 0 + with open(ids) as file: + for line in file: + pid = int(line) + if pid > max_pid: + max_pid = pid + identifier._set_sequence(max_pid) + + +def bulk_save(pid_type, file_name, table, columns, verbose=False): + """Bulk save pid_type data to file.""" + with open(file_name, 'w', encoding='utf-8') as output_file: + db_copy_to( + filehandle=output_file, + table=table, + columns=columns + ) + + +def bulk_save_metadata(pid_type, file_name, verbose=False): + """Bulk save pid_type data from metadata table.""" + if verbose: + click.echo('Save {pid_type} metadata to file: {filename}'.format( + pid_type=pid_type, + filename=file_name + )) + record_class = get_record_class_from_schema_or_pid_type(pid_type=pid_type) + metadata, identifier = record_class.get_metadata_identifier_names() + columns = ( + 'created', + 'updated', + 'id', + 'json', + 'version_id' + ) + bulk_save( + pid_type=pid_type, + file_name=file_name, + table=metadata, + columns=columns, + verbose=verbose + ) + + +def bulk_save_pidstore(pid_type, file_name, file_name_tmp, verbose=False): + """Bulk save pid_type data from pids table.""" + if verbose: + click.echo('Save {pid_type} pidstore to file: {filename}'.format( + pid_type=pid_type, + filename=file_name + )) + if not os.path.isfile(file_name_tmp): + table = 'pidstore_pid' + columns = ( + 'created', + 'updated', + 'pid_type', + 'pid_value', + 'status', + 'object_type', + 'object_uuid', + ) + bulk_save( + pid_type=pid_type, + file_name=file_name_tmp, + table=table, + columns=columns, + verbose=verbose + ) + # clean pid file + with open(file_name_tmp, 'r') as file_in: + with open(file_name, "w") as file_out: + count = 0 + for line in file_in: + if pid_type in line: + count += 1 + file_out.write(line) + return count + + +def bulk_save_pids(pid_type, file_name, verbose=False): + """Bulk save pid_type data from id table.""" + if verbose: + click.echo('Save {pid_type} ids to file: {filename}'.format( + pid_type=pid_type, + filename=file_name + )) + record_class = get_record_class_from_schema_or_pid_type(pid_type=pid_type) + metadata, identifier = record_class.get_metadata_identifier_names() + columns = ('recid', ) + bulk_save( + pid_type=pid_type, + file_name=file_name, + table=identifier.__tablename__, + columns=columns, + verbose=verbose + ) + + +def number_records_in_file(json_file, type): + """Get number of records per file.""" + count = 0 + with open(json_file, 'r', buffering=1) as file: + for line in file: + if type == 'json': + if '"pid"' in line: + count += 1 + elif type == 'csv': + count += 1 + return count diff --git a/scripts/create_csvs b/scripts/create_csvs new file mode 100755 index 0000000000..777306d9a1 --- /dev/null +++ b/scripts/create_csvs @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# COLORS for messages +NC='\033[0m' # Default color +INFO_COLOR='\033[1;97;44m' # Bold + white + blue background +SUCCESS_COLOR='\033[1;97;42m' # Bold + white + green background +ERROR_COLOR='\033[1;97;41m' # Bold + white + red background + +PROGRAM=`basename $0` + +# MESSAGES +msg() { + echo -e "${1}" 1>&2 +} +# Display a colored message +# More info: https://misc.flogisoft.com/bash/tip_colors_and_formatting +# $1: choosen color +# $2: title +# $3: the message +colored_msg() { + msg "${1}[${2}]: ${3}${NC}" +} + +info_msg() { + colored_msg "${INFO_COLOR}" "INFO" "${1}" +} + +error_msg() { + colored_msg "${ERROR_COLOR}" "ERROR" "${1}" +} + +error_msg+exit() { + error_msg "${1}" && exit 1 +} + +success_msg() { + colored_msg "${SUCCESS_COLOR}" "SUCCESS" "${1}" +} + +DEPLOYMENT=false +CREATE_CONTRIBUTION=false +CREATE_LAZY="" +VERBOSE="" +INVERS_VERBOSE="--verbose" +PREFIX="" +IN_DATA_PATH=./data +OUT_DATA_PATH=./data +PIDDEPENDENCIES=false + +# Displays program name +msg "PROGRAM: ${PROGRAM}" + +# POETRY is a mandatory condition to launch this program! +if [[ -z "${VIRTUAL_ENV}" ]]; then + error_msg+exit "Error - Launch this script via poetry command:\n\tpoetry run ${PROGRAM}" +fi + +# options may be followed by one colon to indicate they have a required argument +if ! options=$(getopt -o dcplvstio: -l deployment,create_contribution,pid_dependencies,lazy,verbose,schema,time,in_data_path:out_data_path: -- "$@") +then + # something went wrong, getopt will put out an error message for us + exit 1 +fi + +while [ $# -gt 0 ] +do + case $1 in + -d|--deployment) + DEPLOYMENT=true + ;; + -c|--create_contribution) + CREATE_CONTRIBUTION=true + ;; + -p|--pid_dependencies) + PIDDEPENDENCIES=true + ;; + -l|--lazy) + CREATE_LAZY="--lazy" + ;; + -v|--verbose) + VERBOSE="--verbose" + INVERS_VERBOSE="" + ;; + -t|--time) + PREFIX="time" + ;; + -i|--in_data_path) + IN_DATA_PATH=$2 + shift + ;; + -o|--out_data_path) + OUT_DATA_PATH=$2 + shift + ;; + (--) shift; break;; + (-*) error_msg+exit "$0: Unrecognized option $1";; + (*) break;; + esac + shift +done + +if [[ ! -d "${IN_DATA_PATH}" ]]; then + error_msg+exit "Error - in data path does not exist: ${IN_DATA_PATH}" +fi +if [[ ! -d "${OUT_DATA_PATH}" ]]; then + error_msg+exit "Error - out data path does not exist: ${OUT_DATA_PATH}" +fi + +set -e +if ${PIDDEPENDENCIES} +then + if ${DEPLOYMENT} + then + info_msg "Check pid dependencies: ${IN_DATA_PATH}/pid_dependencies_big.json" + eval ${PREFIX} invenio utils check_pid_dependencies --directory ${IN_DATA_PATH} --dependency_file ${IN_DATA_PATH}/pid_dependencies_big.json ${VERBOSE} + else + info_msg "Check pid dependencies: ${IN_DATA_PATH}/pid_dependencies_small.json" + eval ${PREFIX} invenio utils check_pid_dependencies --directory ${IN_DATA_PATH} --dependency_file ${IN_DATA_PATH}/pid_dependencies_small.json ${VERBOSE} + fi +fi + +eval ${PREFIX} invenio fixtures create_csv org ${IN_DATA_PATH}/organisations.json ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv lib ${IN_DATA_PATH}/libraries.json ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv loc ${IN_DATA_PATH}/locations.json ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv itty ${IN_DATA_PATH}/item_types.json ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv ptty ${IN_DATA_PATH}/patron_types.json ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv cipo ${IN_DATA_PATH}/circulation_policies.json ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv vndr ${IN_DATA_PATH}/vendors.json ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} + +if ${DEPLOYMENT} +then + DOCUMENT=${IN_DATA_PATH}/documents_big.json + HOLDING=${IN_DATA_PATH}/holdings_big.json + ITEMS=${IN_DATA_PATH}/items_big.json + CONTRIBUTIONS=${IN_DATA_PATH}/contributions_big.json +else + DOCUMENT=${IN_DATA_PATH}/documents_small.json + HOLDING=${IN_DATA_PATH}/holdings_small.json + ITEMS=${IN_DATA_PATH}/items_small.json + CONTRIBUTIONS=${IN_DATA_PATH}/contributions_small.json +fi + +if ${CREATE_CONTRIBUTION} +then + eval ${PREFIX} invenio fixtures get_all_mef_records ${DOCUMENT} ${CREATE_LAZY} --out_file ${CONTRIBUTIONS} ${INVERS_VERBOSE} +fi + +eval ${PREFIX} invenio fixtures create_csv doc ${DOCUMENT} ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv hold ${HOLDING} ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv item ${ITEMS} ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv cont ${CONTRIBUTIONS} ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} + +# eval ${PREFIX} invenio fixtures create_csv illr ${IN_DATA_PATH}/ill_requests.json ${OUT_DATA_PATH} ${VALIDATESCHEMA} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv lofi ${IN_DATA_PATH}/local_fields.json ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv coll ${IN_DATA_PATH}/collections.json ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} --create_pid +eval ${PREFIX} invenio fixtures create_csv budg ${IN_DATA_PATH}/budgets.json ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv acac ${IN_DATA_PATH}/acq_accounts.json ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} +eval ${PREFIX} invenio fixtures create_csv tmpl ${IN_DATA_PATH}/templates.json ${OUT_DATA_PATH} ${CREATE_LAZY} ${VERBOSE} diff --git a/scripts/setup b/scripts/setup index 9163fa6f19..3862384ed2 100755 --- a/scripts/setup +++ b/scripts/setup @@ -299,6 +299,9 @@ if [ ${INDEX_PARALLEL} -gt 0 ]; then eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error fi eval ${PREFIX} invenio utils runindex --raise-on-error +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t org -t lib -t loc -t itty -t ptty -t cipo -t vndr -v +fi info_msg "- Users: ${DATA_PATH}/users.json" eval ${PREFIX} invenio fixtures import_users ${DATA_PATH}/users.json -v --append ${CREATE_LAZY} ${DONT_STOP} @@ -330,7 +333,7 @@ fi if ${LOADCONTRIBUTIONS} then - info_msg "- CONTRIBUTIONS: ${DOCUMENTS} ${CREATE_LAZY} ${DONT_STOP}" + info_msg "- CONTRIBUTIONS: ${CONTRIBUTIONS} ${CREATE_LAZY} ${DONT_STOP}" eval ${PREFIX} invenio fixtures create --pid_type cont --schema 'http://ils.rero.ch/schemas/contributions/contribution-v0.0.1.json' ${CONTRIBUTIONS} --append ${CREATE_LAZY} ${DONT_STOP} info_msg "Indexing Contributions:" eval ${PREFIX} invenio utils reindex -t cont --yes-i-know --no-info @@ -338,6 +341,9 @@ then eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error fi eval ${PREFIX} invenio utils runindex --raise-on-error + if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t cont -v + fi else info_msg "- Contributions from MEF: ${DOCUMENTS} ${CREATE_LAZY} ${ENQUEUE}" eval ${PREFIX} invenio fixtures get_all_mef_records ${DOCUMENTS} ${CREATE_LAZY} ${ENQUEUE} @@ -377,6 +383,9 @@ if [ ${INDEX_PARALLEL} -gt 0 ]; then eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error fi eval ${PREFIX} invenio utils runindex --raise-on-error +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t hold -v +fi info_msg "- Items: ${ITEMS} ${CREATE_LAZY} ${DONT_STOP}" eval ${PREFIX} invenio fixtures create --pid_type item --schema 'http://ils.rero.ch/schemas/items/item-v0.0.1.json' ${ITEMS} --append ${CREATE_LAZY} ${DONT_STOP} @@ -387,6 +396,9 @@ if [ ${INDEX_PARALLEL} -gt 0 ]; then eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error fi eval ${PREFIX} invenio utils runindex --raise-on-error +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t item -v +fi # index documents eval ${PREFIX} invenio utils reindex -t doc --yes-i-know --no-info @@ -394,6 +406,9 @@ if [ ${INDEX_PARALLEL} -gt 0 ]; then eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error fi eval ${PREFIX} invenio utils runindex --raise-on-error +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t doc -v +fi # index contributions # We have to reindex contributions to get the oraganisations pids indexed correctly. @@ -402,6 +417,9 @@ if [ ${INDEX_PARALLEL} -gt 0 ]; then eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error fi eval ${PREFIX} invenio utils runindex --raise-on-error +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t cont -v +fi info_msg "- Local fields ${DATA_PATH}/local_fields.json ${CREATE_LAZY} ${DONT_STOP}" eval ${PREFIX} invenio fixtures create --pid_type lofi ${DATA_PATH}/local_fields.json --append ${CREATE_LAZY} ${DONT_STOP} @@ -435,6 +453,9 @@ if [ ${INDEX_PARALLEL} -gt 0 ]; then eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error fi eval ${PREFIX} invenio utils runindex --raise-on-error +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t lofi -t coll -t budg -t acac -t tmpl -v +fi # create circulation transactions info_msg "Circulation transactions: ${DATA_PATH}/loans.json ${CREATE_LAZY} ${DONT_STOP}" diff --git a/scripts/setup_csv b/scripts/setup_csv new file mode 100755 index 0000000000..d56b8ad2b5 --- /dev/null +++ b/scripts/setup_csv @@ -0,0 +1,358 @@ +#!/usr/bin/env bash +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# COLORS for messages +NC='\033[0m' # Default color +INFO_COLOR='\033[1;97;44m' # Bold + white + blue background +SUCCESS_COLOR='\033[1;97;42m' # Bold + white + green background +ERROR_COLOR='\033[1;97;41m' # Bold + white + red background + +PROGRAM=`basename $0` + +# MESSAGES +msg() { + echo -e "${1}" 1>&2 +} +# Display a colored message +# More info: https://misc.flogisoft.com/bash/tip_colors_and_formatting +# $1: choosen color +# $2: title +# $3: the message +colored_msg() { + msg "${1}[${2}]: ${3}${NC}" +} + +info_msg() { + colored_msg "${INFO_COLOR}" "INFO" "${1}" +} + +error_msg() { + colored_msg "${ERROR_COLOR}" "ERROR" "${1}" +} + +error_msg+exit() { + error_msg "${1}" && exit 1 +} + +success_msg() { + colored_msg "${SUCCESS_COLOR}" "SUCCESS" "${1}" +} + +invert_warning_option() { + if ${ENABLE_WARNINGS} + then + ENABLE_WARNINGS=false + else + ENABLE_WARNINGS=true + fi +} + + +DATA_PATH=./data +CSV_PATH=./data +DEPLOYMENT=false +CREATE_LAZY="" +PREFIX="" +INDEX_PARALLEL=0 + +# Displays program name +msg "PROGRAM: ${PROGRAM}" + +# POETRY is a mandatory condition to launch this program! +if [[ -z "${VIRTUAL_ENV}" ]]; then + error_msg+exit "Error - Launch this script via poetry command:\n\tpoetry run ${PROGRAM}" +fi + +# options may be followed by one colon to indicate they have a required argument +if ! options=$(getopt -o dltD:C:i: -l deployment,lazy,time,data_path:,csv_path:,index_parallel: -- "$@") +then + # something went wrong, getopt will put out an error message for us + exit 1 +fi + +while [ $# -gt 0 ] +do + case $1 in + -d|--deployment) + DEPLOYMENT=true + ;; + -l|--lazy) + CREATE_LAZY="--lazy" + ;; + -t|--time) + PREFIX="time" + ;; # Get time for all commands + -D|--data_path) + DATA_PATH=$2 + shift + ;; + -C|--csv_path) + CSV_PATH=$2 + shift + ;; + -i|--index_parallel) + INDEX_PARALLEL=$2 + shift + ;; + (--) shift; break;; + (-*) error_msg+exit "$0: Unrecognized option $1";; + (*) break;; + esac + shift +done + +if [[ ! -d "${DATA_PATH}" ]]; then + error_msg+exit "Error - data path does not exist: ${DATA_PATH}" +fi + +if [[ ! -d "${CSV_PATH}" ]]; then + error_msg+exit "Error - csv path does not exist: ${CSV_PATH}" +fi + +# Purge celery +info_msg "Purge celery" +celery --app rero_ils.celery purge -f +# Clean redis +info_msg "Clean redis" +eval "${PREFIX} invenio shell --no-term-title -c \"import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')\"" +eval ${PREFIX} invenio scheduler init -r + +eval ${PREFIX} invenio db destroy --yes-i-know +eval ${PREFIX} invenio db init create +eval ${PREFIX} invenio index queue purge delete +set -e +eval ${PREFIX} invenio index destroy --force --yes-i-know +# Override index init to load templates before mapping +# TODO: check if invenio index init --force works (to delete utils init --force) +info_msg "Override index init to load templates before mapping" +eval ${PREFIX} invenio utils init_index --force +# eval ${PREFIX} invenio index init --force +eval ${PREFIX} invenio index queue init +# Delete invenio_circulations index +info_msg "Delete invenio_circulations index" +eval ${PREFIX} invenio index delete loans-loan-v1.0.0 --force --yes-i-know + +# create roles +info_msg "Create roles" +eval ${PREFIX} "invenio roles create -d 'Admins Group' admin" +eval ${PREFIX} "invenio roles create -d 'Super Users Group' superuser" +eval ${PREFIX} "invenio roles create -d 'Monitoring Group' monitoring" +eval ${PREFIX} "invenio roles create -d 'Patron' patron" +eval ${PREFIX} "invenio roles create -d 'Librarian' librarian" +eval ${PREFIX} "invenio roles create -d 'System Librarian' system_librarian" +eval ${PREFIX} "invenio roles create -d 'Documentation Editor' editor" +eval ${PREFIX} "invenio roles create -d 'Documment Importing' document_importer" + +# create users +info_msg "Create users" +eval ${PREFIX} invenio users create -a admin@rero.ch --password administrator +eval ${PREFIX} invenio users create -a editor@rero.ch --password editor +eval ${PREFIX} invenio users create -a monitoring@rero.ch --password monitor +eval ${PREFIX} invenio users create -a gipi@ngscan.com --password ngscan + +# confirm users +info_msg "Confirm users" +eval ${PREFIX} invenio users confirm admin@rero.ch +eval ${PREFIX} invenio users confirm editor@rero.ch +eval ${PREFIX} invenio users confirm monitoring@rero.ch +eval ${PREFIX} invenio users confirm gipi@ngscan.com + +# grant accesses to action roles +info_msg "Grant access to action roles" +eval ${PREFIX} invenio access allow superuser-access role admin +eval ${PREFIX} invenio access allow superuser-access role superuser +eval ${PREFIX} invenio access allow admin-access role admin + +# grant roles to users +info_msg "Grant roles to users" +eval ${PREFIX} invenio roles add admin@rero.ch admin +eval ${PREFIX} invenio roles add admin@rero.ch superuser +eval ${PREFIX} invenio roles add admin@rero.ch monitoring +eval ${PREFIX} invenio roles add editor@rero.ch editor +eval ${PREFIX} invenio roles add monitoring@rero.ch monitoring +eval ${PREFIX} invenio roles add gipi@ngscan.com document_importer + +# Generate fixtures +info_msg "Generate fixtures:" + +info_msg "- Organisations ${CSV_PATH}/org_metadata.csv" +eval ${PREFIX} invenio fixtures bulk_load org ${CSV_PATH}/org_metadata.csv --reindex --verbose + +info_msg "- Libraries: ${CSV_PATH}/lib_metadata.csv" +eval ${PREFIX} invenio fixtures bulk_load lib ${CSV_PATH}/lib_metadata.csv --reindex --verbose + +info_msg "- Locations: ${CSV_PATH}/loc_metadata.csv" +eval ${PREFIX} invenio fixtures bulk_load loc ${CSV_PATH}/loc_metadata.csv --reindex --verbose + +info_msg "- Item types: ${CSV_PATH}/itty_metadata.csv" +eval ${PREFIX} invenio fixtures bulk_load itty ${CSV_PATH}/itty_metadata.csv --reindex --verbose + +info_msg "- Patron types: ${CSV_PATH}/ptty_metadata.csv" +eval ${PREFIX} invenio fixtures bulk_load ptty ${CSV_PATH}/ptty_metadata.csv --reindex --verbose + +info_msg "- Circulation policies: ${CSV_PATH}/cipo_metadata.csv ${CREATE_LAZY} ${DONT_STOP}" +eval ${PREFIX} invenio fixtures bulk_load cipo ${CSV_PATH}/cipo_metadata.csv --reindex --verbose + +info_msg "Acquisition vendors: ${CSV_PATH}/vndr_metadata.csv" +eval ${PREFIX} invenio fixtures bulk_load vndr ${CSV_PATH}/vndr_metadata.csv --reindex --verbose + +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error +fi +eval ${PREFIX} invenio utils runindex --raise-on-error +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t org -t lib -t loc -t itty -t ptty -t cipo -t vndr -v +fi + +info_msg "- Users: ${DATA_PATH}/users.json" +eval ${PREFIX} invenio fixtures import_users ${DATA_PATH}/users.json -v --append + +info_msg "- ILL requests: ${DATA_PATH}/ill_request.json" +eval ${PREFIX} invenio fixtures create_ill_requests -f ${DATA_PATH}/ill_requests.json +eval ${PREFIX} invenio utils reindex -t illr --yes-i-know --no-info + +info_msg "- CONTRIBUTIONS: ${CONTRIBUTIONS} ${CREATE_LAZY} ${DONT_STOP}" +eval ${PREFIX} invenio fixtures bulk_load cont ${CSV_PATH}/cont_metadata.csv --verbose + +info_msg "- Documents: ${CSV_PATH}/cont_metadata.csv" +eval ${PREFIX} invenio fixtures bulk_load doc ${CSV_PATH}/doc_metadata.csv --verbose + +info_msg "- Holdings: ${CSV_PATH}/hold_metadata.csv" +eval ${PREFIX} invenio fixtures bulk_load hold ${CSV_PATH}/hold_metadata.csv --reindex --verbose +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error +fi +eval ${PREFIX} invenio utils runindex --raise-on-error +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t hold -v +fi + +info_msg "- Items: ${CSV_PATH}/item_metadata.csv" +eval ${PREFIX} invenio fixtures bulk_load item ${CSV_PATH}/item_metadata.csv --reindex --verbose +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error +fi +eval ${PREFIX} invenio utils runindex --raise-on-error +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t item -v +fi + +# index documents +eval ${PREFIX} invenio utils reindex -t doc --yes-i-know --no-info +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error +fi +eval ${PREFIX} invenio utils runindex --raise-on-error +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t doc -v +fi + +# index contributions +# We have to reindex contributions to get the oraganisations pids indexed correctly. +eval ${PREFIX} invenio utils reindex -t cont --yes-i-know --no-info +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error +fi +eval ${PREFIX} invenio utils runindex --raise-on-error +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t cont -v +fi + +info_msg "- Local fields ${DATA_PATH}/local_fields.json ${CREATE_LAZY} ${DONT_STOP}" +eval ${PREFIX} invenio fixtures bulk_load lofi ${CSV_PATH}/lofi_metadata.csv --reindex --verbose + +# create serials patterns +info_msg "Serials patterns: ${DATA_PATH}/patterns.json" +eval ${PREFIX} invenio fixtures create_patterns ${DATA_PATH}/patterns.json + +info_msg "- Collections: ${DATA_PATH}/collections.json" +eval ${PREFIX} invenio fixtures create_collections -f ${DATA_PATH}/collections.json +eval ${PREFIX} invenio utils reindex -t coll --yes-i-know --no-info + +# # ACQUISITION +# create library budgets +info_msg "Library budgets: ${DATA_PATH}/budgets.json ${CREATE_LAZY} ${DONT_STOP}" +eval ${PREFIX} invenio fixtures bulk_load budg ${CSV_PATH}/budg_metadata.csv --reindex --verbose + +# create acquisition accounts +info_msg "Acquisition accounts: ${DATA_PATH}/acq_accounts.json ${CREATE_LAZY} ${DONT_STOP}" +eval ${PREFIX} invenio fixtures bulk_load acac ${CSV_PATH}/acac_metadata.csv --reindex --verbose + +# create resource templates +info_msg "Resource templates:" +eval ${PREFIX} invenio fixtures bulk_load tmpl ${CSV_PATH}/tmpl_metadata.csv --reindex --verbose + +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils runindex -d -c ${INDEX_PARALLEL} --raise-on-error +fi +eval ${PREFIX} invenio utils runindex --raise-on-error +if [ ${INDEX_PARALLEL} -gt 0 ]; then + eval ${PREFIX} invenio utils reindex_missing -t lofi -t coll -t budg -t acac -t tmpl -v +fi + +# create circulation transactions +info_msg "Circulation transactions: ${DATA_PATH}/loans.json ${CREATE_LAZY} ${DONT_STOP}" +eval ${PREFIX} invenio fixtures create_loans ${DATA_PATH}/loans.json + +# process notifications +eval ${PREFIX} invenio notifications process + +# create token access for monitoring +# if the environement variable INVENIO_RERO_ACCESS_TOKEN_MONITORING is not set +# a new token will be generated +info_msg "Create token for: monitoring@rero.ch" +if [ -z ${INVENIO_RERO_ACCESS_TOKEN_MONITORING} ] +then + eval ${PREFIX} invenio utils tokens_create -n monitoring -u monitoring@rero.ch +else + eval ${PREFIX} invenio utils tokens_create -n monitoring -u monitoring@rero.ch -t ${INVENIO_RERO_ACCESS_TOKEN_MONITORING} +fi + +# create token access for ngscan (ezpump) +# if the environement variable INVENIO_RERO_ACCESS_TOKEN_NGSCAN is not set a new +# token will be generated +info_msg "Create token for: gipi@ngscan.com" +if [ -z ${INVENIO_RERO_ACCESS_TOKEN_NGSCAN} ] +then + eval ${PREFIX} invenio utils tokens_create -n ezpump -u gipi@ngscan.com +else + eval ${PREFIX} invenio utils tokens_create -n ezpump -u gipi@ngscan.com -t ${INVENIO_RERO_ACCESS_TOKEN_NGSCAN} +fi + +# # OAI configuration +info_msg "OAI configuration: ${DATA_PATH}/oaisources.yml" +eval ${PREFIX} invenio oaiharvester initconfig ${DATA_PATH}/oaisources.yml +if ${DEPLOYMENT} +then + eval ${PREFIX} invenio scheduler enable_tasks -a -v + # start oai harvesting asynchrone: beats must be running + info_msg "Start OAI harvesting asynchrone" + eval ${PREFIX} invenio oaiharvester harvest -n ebooks -q -k +else + eval ${PREFIX} invenio scheduler enable_tasks -n scheduler-timestamp -n bulk-indexer -n anonymize-loans -n claims-creation -n notification-creation -n accounts -n clear_and_renew_subscriptions -v + info_msg "For ebooks harvesting run:" + msg "\tinvenio oaiharvester harvest -n ebooks -a max=100 -q" +fi + + +info_msg "- Load Virtua operation logs: ${DATA_PATH}/operation_logs.json" +eval ${PREFIX} invenio fixtures migrate_virtua_operation_logs ${DATA_PATH}/operation_logs.json ${CREATE_LAZY} ${DONT_STOP} + +date +success_msg "Perfect ${PROGRAM}! See you soon…" +exit 0 diff --git a/tests/api/holdings/test_patterns.py b/tests/api/holdings/test_patterns.py index e6fdc69114..3421db49d4 100644 --- a/tests/api/holdings/test_patterns.py +++ b/tests/api/holdings/test_patterns.py @@ -26,10 +26,10 @@ import pytest from flask import url_for from invenio_accounts.testutils import login_user_via_session +from jsonschema.exceptions import ValidationError from utils import get_json, postdata from rero_ils.modules.documents.api import Document -from rero_ils.modules.errors import RecordValidationError from rero_ils.modules.holdings.api import Holding from rero_ils.modules.items.api import Item from rero_ils.modules.items.models import ItemIssueStatus, ItemStatus @@ -310,7 +310,7 @@ def test_pattern_validate_next_expected_date( del holding['patterns']['next_expected_date'] # test will fail when the serial holding has no field # next_expected_date for the regular frequency - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): Holding.create( data=holding, delete_pid=False, @@ -386,7 +386,7 @@ def test_irregular_issue_creation_update_delete_api( created_item.update(data=item, dbcommit=True, reindex=True) # Validation error if you try to update an issue with no holdings links item.pop('holding') - # with pytest.raises(RecordValidationError): + # with pytest.raises(ValidationError): created_item.update(data=item, dbcommit=True, reindex=True) # no errors when deleting an irregular issue pid = created_item.pid diff --git a/tests/api/items/test_items_rest.py b/tests/api/items/test_items_rest.py index 57786abc9d..e784a11b46 100644 --- a/tests/api/items/test_items_rest.py +++ b/tests/api/items/test_items_rest.py @@ -23,13 +23,11 @@ import ciso8601 import mock -import pytest from flask import url_for from invenio_accounts.testutils import login_user_via_session from utils import VerifyRecordPermissionPatch, flush_index, get_json, postdata from rero_ils.modules.circ_policies.api import CircPoliciesSearch -from rero_ils.modules.errors import RecordValidationError from rero_ils.modules.items.api import Item from rero_ils.modules.items.models import ItemNoteTypes, ItemStatus from rero_ils.modules.loans.api import Loan, LoanAction @@ -837,12 +835,16 @@ def test_items_notes(client, librarian_martigny, item_lib_martigny, item['notes'].append( {'type': ItemNoteTypes.GENERAL, 'content': 'Second public note'} ) - with pytest.raises(RecordValidationError): - client.put( - url_for('invenio_records_rest.item_item', pid_value=item.pid), - data=json.dumps(item), - headers=json_header - ) + res = client.put( + url_for('invenio_records_rest.item_item', pid_value=item.pid), + data=json.dumps(item), + headers=json_header + ) + assert get_json(res) == { + 'status': 400, + 'message': + 'Validation error: Can not have multiple notes of the same type..' + } item['notes'] = item.notes[:-1] # get a specific type of notes diff --git a/tests/api/locations/test_locations_rest.py b/tests/api/locations/test_locations_rest.py index 91966f469d..907a49b47e 100644 --- a/tests/api/locations/test_locations_rest.py +++ b/tests/api/locations/test_locations_rest.py @@ -21,14 +21,12 @@ from copy import deepcopy import mock -import pytest from flask import url_for from invenio_accounts.testutils import login_user_via_session from utils import VerifyRecordPermissionPatch, flush_index, get_json, \ postdata, to_relative_url from rero_ils.modules.documents.views import item_library_pickup_locations -from rero_ils.modules.errors import RecordValidationError from rero_ils.modules.locations.api import Location, LocationsSearch @@ -314,12 +312,15 @@ def test_location_secure_api_create(client, lib_fully, lib_martigny, if 'pickup_name' in fake_location_data: del fake_location_data['pickup_name'] fake_location_data['is_pickup'] = True - with pytest.raises(RecordValidationError): - res, _ = postdata( - client, - post_entrypoint, - fake_location_data - ) + res, _ = postdata( + client, + post_entrypoint, + fake_location_data + ) + assert get_json(res) == { + 'status': 400, + 'message': 'Validation error: Pickup location name field is required..' + } # Martigny del loc_public_martigny_data['pid'] diff --git a/tests/conftest.py b/tests/conftest.py index f0c41771df..8dddd2be2e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -155,6 +155,8 @@ def app_config(app_config): app_config['RATELIMIT_STORAGE_URL'] = 'memory://' app_config['CACHE_TYPE'] = 'simple' app_config['SEARCH_ELASTIC_HOSTS'] = None + app_config['SQLALCHEMY_DATABASE_URI'] = \ + 'postgresql+psycopg2://rero-ils:rero-ils@localhost/rero-ils' app_config['DB_VERSIONING'] = True app_config['CELERY_CACHE_BACKEND'] = "memory" app_config['CELERY_RESULT_BACKEND'] = "cache" diff --git a/tests/data/001.pids b/tests/data/001.pids new file mode 100644 index 0000000000..057d07d181 --- /dev/null +++ b/tests/data/001.pids @@ -0,0 +1 @@ +270072860 diff --git a/tests/data/documents.xml b/tests/data/documents.xml index b860b67fe2..2ce07f7acc 100644 --- a/tests/data/documents.xml +++ b/tests/data/documents.xml @@ -1,6 +1,7 @@ 00975nam a2200277 a 4500 + 270072860 20170518120100.0 000918s1999 gw ||| | ||||00| |ger d @@ -83,13 +84,14 @@ 00975nam a2200277 a 4500 + 270072861 20170518120100.0 000918s1999 gw ||| | ||||00| |ger d 9783503057221 - R270072860 + R270072861 4606 diff --git a/tests/ui/circ_policies/test_circ_policies_api.py b/tests/ui/circ_policies/test_circ_policies_api.py index 5a0836cf85..01bff56275 100644 --- a/tests/ui/circ_policies/test_circ_policies_api.py +++ b/tests/ui/circ_policies/test_circ_policies_api.py @@ -23,10 +23,10 @@ import mock import pytest +from jsonschema.exceptions import ValidationError from rero_ils.modules.circ_policies.api import CircPolicy, \ circ_policy_id_fetcher -from rero_ils.modules.errors import RecordValidationError def test_no_default_policy(app): @@ -89,7 +89,7 @@ def test_circ_policy_create(circ_policy_martigny_data_tmp, } }] } - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): cipo = CircPolicy.create(cipo_data, delete_pid=False) diff --git a/tests/ui/holdings/test_holdings_api.py b/tests/ui/holdings/test_holdings_api.py index 4a4a83a2b4..3e864a4afd 100644 --- a/tests/ui/holdings/test_holdings_api.py +++ b/tests/ui/holdings/test_holdings_api.py @@ -22,9 +22,9 @@ from __future__ import absolute_import, print_function import pytest +from jsonschema.exceptions import ValidationError from utils import flush_index, get_mapping -from rero_ils.modules.errors import RecordValidationError from rero_ils.modules.holdings.api import Holding, HoldingsSearch from rero_ils.modules.holdings.api import holding_id_fetcher as fetcher @@ -48,17 +48,19 @@ def test_holding_create(db, es_clear, document, org_martigny, loc_public_martigny, item_type_standard_martigny, holding_lib_martigny_data): """Test holding creation.""" + next_pid = Holding.provider.identifier.next() holding = Holding.create(holding_lib_martigny_data, dbcommit=True, reindex=True, delete_pid=True) + next_pid += 1 flush_index(HoldingsSearch.Meta.index) assert holding == holding_lib_martigny_data - assert holding.get('pid') == '1' + assert holding.get('pid') == str(next_pid) - holding = Holding.get_record_by_pid('1') + holding = Holding.get_record_by_pid(str(next_pid)) assert holding == holding_lib_martigny_data fetched_pid = fetcher(holding.id, holding) - assert fetched_pid.pid_value == '1' + assert fetched_pid.pid_value == str(next_pid) assert fetched_pid.pid_type == 'hold' search = HoldingsSearch() @@ -86,7 +88,7 @@ def test_holding_extended_validation(client, # 1.1. test next expected date for regular frequencies expected_date = holding_tmp['patterns']['next_expected_date'] del(holding_tmp['patterns']['next_expected_date']) - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): holding_tmp.validate() # reset data with original value @@ -97,14 +99,14 @@ def test_holding_extended_validation(client, 'type': 'general_note', 'content': 'other general_note...' }) - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): holding_tmp.validate() # 2. holding type electronic # 2.1. test holding type electronic attached to wrong document type holding_tmp['holdings_type'] = 'electronic' - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): holding_tmp.validate() # 2.2 test electronic holding diff --git a/tests/ui/holdings/test_holdings_patterns.py b/tests/ui/holdings/test_holdings_patterns.py index c935965d7f..50b9c50b7d 100644 --- a/tests/ui/holdings/test_holdings_patterns.py +++ b/tests/ui/holdings/test_holdings_patterns.py @@ -27,9 +27,9 @@ import jinja2 import pytest from invenio_accounts.testutils import login_user_via_session +from jsonschema.exceptions import ValidationError from rero_ils.modules.api import IlsRecordError -from rero_ils.modules.errors import RecordValidationError from rero_ils.modules.holdings.api import Holding from rero_ils.modules.holdings.models import HoldingNoteTypes from rero_ils.modules.items.api import Item @@ -338,7 +338,7 @@ def test_holding_validate_next_expected_date( del holding['patterns']['next_expected_date'] # test will fail when the serial holding has no field # next_expected_date for the regular frequency - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): Holding.create( data=holding, delete_pid=False, @@ -598,7 +598,7 @@ def test_holding_notes(client, librarian_martigny, holding['notes'].append( {'type': HoldingNoteTypes.CLAIM, 'content': 'new cliam note'} ) - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): holding.update(holding, dbcommit=True, reindex=True) holding['notes'] = holding.notes[:-1] diff --git a/tests/ui/item_types/test_item_types_api.py b/tests/ui/item_types/test_item_types_api.py index 4d92d5a52c..1232b8da11 100644 --- a/tests/ui/item_types/test_item_types_api.py +++ b/tests/ui/item_types/test_item_types_api.py @@ -20,8 +20,8 @@ from __future__ import absolute_import, print_function import pytest +from jsonschema.exceptions import ValidationError -from rero_ils.modules.errors import RecordValidationError from rero_ils.modules.item_types.api import ItemType, item_type_id_fetcher @@ -29,22 +29,24 @@ def test_item_type_create(db, item_type_data_tmp, org_martigny, item_type_online_martigny): """Test item type record creation.""" item_type_data_tmp['type'] = 'online' - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): itty = ItemType.create(item_type_data_tmp, delete_pid=True) db.session.rollback() + next_pid = ItemType.provider.identifier.next() item_type_data_tmp['type'] = 'standard' itty = ItemType.create(item_type_data_tmp, delete_pid=True) + next_pid += 1 assert itty == item_type_data_tmp - assert itty.get('pid') == '1' + assert itty.get('pid') == str(next_pid) - itty = ItemType.get_record_by_pid('1') + itty = ItemType.get_record_by_pid(str(next_pid)) assert itty == item_type_data_tmp fetched_pid = item_type_id_fetcher(itty.id, itty) - assert fetched_pid.pid_value == '1' + assert fetched_pid.pid_value == str(next_pid) assert fetched_pid.pid_type == 'itty' assert not ItemType.get_pid_by_name('no exists') diff --git a/tests/ui/items/test_items_api.py b/tests/ui/items/test_items_api.py index 124742be37..1c44fb3667 100644 --- a/tests/ui/items/test_items_api.py +++ b/tests/ui/items/test_items_api.py @@ -162,12 +162,12 @@ def test_item_extended_validation(client, holding_lib_martigny_w_patterns): # TODO: check why system is not raising validation error here # # can not have a standard item with issues on a serial holdings # data['type'] = 'standard' - # with pytest.raises(RecordValidationError): + # with pytest.raises(ValidationError): # Item.create(data, dbcommit=True, reindex=True, delete_pid=True) # data['type'] == 'issue' # data.pop('issue') # # can not create an issue item without issues on a serial holdings - # with pytest.raises(RecordValidationError): + # with pytest.raises(ValidationError): # Item.create(data, dbcommit=True, reindex=True, delete_pid=True) diff --git a/tests/ui/locations/test_locations_api.py b/tests/ui/locations/test_locations_api.py index 51b9f9c1e2..cb60346186 100644 --- a/tests/ui/locations/test_locations_api.py +++ b/tests/ui/locations/test_locations_api.py @@ -20,9 +20,9 @@ from __future__ import absolute_import, print_function import pytest +from jsonschema.exceptions import ValidationError from utils import get_mapping -from rero_ils.modules.errors import RecordValidationError from rero_ils.modules.locations.api import Location, LocationsSearch from rero_ils.modules.locations.api import location_id_fetcher as fetcher @@ -46,21 +46,23 @@ def test_location_create(db, es_clear, loc_public_martigny_data, lib_martigny, loc_online_martigny): """Test location creation.""" loc_public_martigny_data['is_online'] = True - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): loc = Location.create(loc_public_martigny_data, delete_pid=True) db.session.rollback() + next_pid = Location.provider.identifier.next() del loc_public_martigny_data['is_online'] loc = Location.create(loc_public_martigny_data, delete_pid=True) + next_pid += 1 assert loc == loc_public_martigny_data - assert loc.get('pid') == '1' + assert loc.get('pid') == str(next_pid) - loc = Location.get_record_by_pid('1') + loc = Location.get_record_by_pid(str(next_pid)) assert loc == loc_public_martigny_data fetched_pid = fetcher(loc.id, loc) - assert fetched_pid.pid_value == '1' + assert fetched_pid.pid_value == str(next_pid) assert fetched_pid.pid_type == 'loc' diff --git a/tests/ui/organisations/test_organisations_api.py b/tests/ui/organisations/test_organisations_api.py index 87f7691d0f..1aa76ca685 100644 --- a/tests/ui/organisations/test_organisations_api.py +++ b/tests/ui/organisations/test_organisations_api.py @@ -63,5 +63,5 @@ def test_organisation_create(app, db, org_martigny_data, org_sion_data): assert org.get('pid') == '2' identifier = Organisation.provider.identifier - append_fixtures_new_identifiers(identifier, ['1', '2'], 'org') + append_fixtures_new_identifiers(identifier, ['1', '2'], 'org', limit=1) assert identifier.next() == identifier.max() == 3 diff --git a/tests/ui/patron_transaction_events/test_patron_transaction_events_api.py b/tests/ui/patron_transaction_events/test_patron_transaction_events_api.py index 73d8c9e634..6e490409be 100644 --- a/tests/ui/patron_transaction_events/test_patron_transaction_events_api.py +++ b/tests/ui/patron_transaction_events/test_patron_transaction_events_api.py @@ -59,16 +59,18 @@ def test_patron_transaction_event_create( db.session.rollback() + next_pid = PatronTransactionEvent.provider.identifier.next() patron_event['type'] = 'fee' record = PatronTransactionEvent.create(patron_event, delete_pid=True) + next_pid += 1 assert record == patron_event - assert record.get('pid') == '2' + assert record.get('pid') == str(next_pid) - pttr = PatronTransactionEvent.get_record_by_pid('2') + pttr = PatronTransactionEvent.get_record_by_pid(str(next_pid)) assert pttr == patron_event fetched_pid = fetcher(pttr.id, pttr) - assert fetched_pid.pid_value == '2' + assert fetched_pid.pid_value == str(next_pid) assert fetched_pid.pid_type == 'pttr' diff --git a/tests/ui/patron_transactions/test_patron_transactions_api.py b/tests/ui/patron_transactions/test_patron_transactions_api.py index d8c639ef5f..c27347e823 100644 --- a/tests/ui/patron_transactions/test_patron_transactions_api.py +++ b/tests/ui/patron_transactions/test_patron_transactions_api.py @@ -41,16 +41,18 @@ def test_patron_transaction_create( db.session.rollback() + next_pid = PatronTransaction.provider.identifier.next() patron_transaction['status'] = 'open' record = PatronTransaction.create(patron_transaction, delete_pid=True) + next_pid += 1 assert record == patron_transaction - assert record.get('pid') == '2' + assert record.get('pid') == str(next_pid) - pttr = PatronTransaction.get_record_by_pid('2') + pttr = PatronTransaction.get_record_by_pid(str(next_pid)) assert pttr == patron_transaction fetched_pid = fetcher(pttr.id, pttr) - assert fetched_pid.pid_value == '2' + assert fetched_pid.pid_value == str(next_pid) assert fetched_pid.pid_type == 'pttr' diff --git a/tests/ui/patrons/test_patrons_api.py b/tests/ui/patrons/test_patrons_api.py index fe4e77ed9c..d3eee16c4a 100644 --- a/tests/ui/patrons/test_patrons_api.py +++ b/tests/ui/patrons/test_patrons_api.py @@ -25,8 +25,8 @@ import pytest from invenio_accounts.models import User from invenio_userprofiles import UserProfile +from jsonschema.exceptions import ValidationError -from rero_ils.modules.errors import RecordValidationError from rero_ils.modules.patrons.api import Patron, PatronsSearch, \ patron_id_fetcher from rero_ils.utils import create_user_from_data @@ -56,7 +56,7 @@ def test_patron_create(app, roles, lib_martigny, librarian_martigny_data_tmp, wrong_librarian_martigny_data_tmp = deepcopy(librarian_martigny_data_tmp) wrong_librarian_martigny_data_tmp.pop('libraries') - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): ptrn = Patron.create( wrong_librarian_martigny_data_tmp, dbcommit=True, @@ -81,7 +81,7 @@ def test_patron_create(app, roles, lib_martigny, librarian_martigny_data_tmp, '$ref': 'https://ils.rero.ch/api/patron_transactions/xxx' }, }] - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): ptrn = Patron.create( wrong_librarian_martigny_data_tmp, dbcommit=True, @@ -99,6 +99,7 @@ def test_patron_create(app, roles, lib_martigny, librarian_martigny_data_tmp, delete_pid=False ) user = User.query.filter_by(id=ptrn.get('user_id')).first() + user_id = ptrn.get('user_id') assert user assert user.active for field in [ @@ -190,7 +191,7 @@ def test_patron_create_without_email(app, roles, patron_type_children_martigny, # comminication channel require at least one email patron_martigny_data_tmp['patron']['communication_channel'] = 'email' - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): ptrn = Patron.create( patron_martigny_data_tmp, dbcommit=True, diff --git a/tests/ui/templates/test_templates_api.py b/tests/ui/templates/test_templates_api.py index c93bf77b53..bd25676b7e 100644 --- a/tests/ui/templates/test_templates_api.py +++ b/tests/ui/templates/test_templates_api.py @@ -52,16 +52,18 @@ def test_template_create(db, es_clear, templ_doc_public_martigny_data, db.session.rollback() + next_pid = Template.provider.identifier.next() del templ_doc_public_martigny_data['toto'] temp = Template.create(templ_doc_public_martigny_data, delete_pid=True) + next_pid += 1 assert temp == templ_doc_public_martigny_data - assert temp.get('pid') == '1' + assert temp.get('pid') == str(next_pid) - temp = Template.get_record_by_pid('1') + temp = Template.get_record_by_pid(str(next_pid)) assert temp == templ_doc_public_martigny_data fetched_pid = fetcher(temp.id, temp) - assert fetched_pid.pid_value == '1' + assert fetched_pid.pid_value == str(next_pid) assert fetched_pid.pid_type == 'tmpl' diff --git a/tests/ui/test_api.py b/tests/ui/test_api.py index 29329b2482..9b9f62804d 100644 --- a/tests/ui/test_api.py +++ b/tests/ui/test_api.py @@ -117,6 +117,8 @@ def test_ilsrecord(app, es_default_index, ils_record, ils_record_2): reindex=True ) assert record_1.pid == 'ilsrecord_pid' + assert record_1.id == RecordTest.get_id_by_pid(record_1.pid) + record_2 = RecordTest.create( data=ils_record_2, dbcommit=True, @@ -169,6 +171,10 @@ def test_ilsrecord(app, es_default_index, ils_record, ils_record_2): pid = RecordTest.get_pid_by_id(record.id) assert pid == record.pid + """Test IlsRecord record pid exist.""" + assert RecordTest.record_pid_exists('ilsrecord_pid') + assert not RecordTest.record_pid_exists('unknown') + """Test IlsRecord revert.""" record = RecordTest.get_record_by_pid('ilsrecord_pid') record = record.revert(record.revision_id - 1) @@ -261,50 +267,28 @@ def test_ilsrecord_failed_pid(app, es_default_index, ils_record, ils_record_2): db.session.rollback() assert FailedIlsRecord.count() == 0 + next_pid = FailedIlsRecord.provider.identifier.next() record1 = FailedIlsRecord.create(data=ils_record, delete_pid=True) + next_pid += 1 assert FailedIlsRecord.count() == 1 - assert record1.pid == '1' + assert record1.pid == str(next_pid) - # Add another record to test that it's 2. + # Add another record record2 = FailedIlsRecord.create(data=ils_record_2, delete_pid=True) - assert record2.pid == '2' + next_pid += 1 + assert record2.pid == str(next_pid) - # Add Manually 3 and 4. Add another one and check result ils_record_3 = { 'pid': '3', 'name': 'IlsRecord Name 3', } - ils_record_4 = { - 'pid': '4', - 'name': 'IlsRecord Name 4', - } - record3 = FailedIlsRecord.create(data=ils_record_3, delete_pid=False) - FailedIlsRecord.create(data=ils_record_4, delete_pid=False) - # without PID, this record should take 3 as next number. Not 5. - record5 = FailedIlsRecord.create(data=ils_record, delete_pid=True) - assert record5.pid == '3' - assert record3.pid == '3' - - assert FailedIlsRecord.count() == 4 - + with pytest.raises(IlsRecordError.PidAlreadyUsed): + record3 = FailedIlsRecord.create(data=ils_record_3, delete_pid=False) + next_pid += 1 + assert FailedIlsRecord.count() == 2 db.session.commit() - # New error to break PID 4, then add new one to check its PID: 4! - record6 = None - with pytest.raises(ValidationError): - record6 = FailedIlsRecord.create( - data={ - '$schema': schema, - 'name': 'Bad IlsRecord', - }, - delete_pid=False, - ) - db.session.rollback() - assert record6 is None - - # We should have 3 PID. - assert FailedIlsRecord.count() == 4 - - record7 = FailedIlsRecord.create(data=ils_record, delete_pid=True) + record4 = FailedIlsRecord.create(data=ils_record, delete_pid=True) + next_pid += 1 db.session.commit() - assert record7.pid == '4' + assert record4.pid == str(next_pid) diff --git a/tests/unit/test_circ_policies_jsonschema.py b/tests/unit/test_circ_policies_jsonschema.py index b1b73bba38..f4db16d503 100644 --- a/tests/unit/test_circ_policies_jsonschema.py +++ b/tests/unit/test_circ_policies_jsonschema.py @@ -27,7 +27,6 @@ from rero_ils.modules.circ_policies.api import DUE_SOON_REMINDER_TYPE, \ OVERDUE_REMINDER_TYPE -from rero_ils.modules.errors import RecordValidationError def test_required(circ_policy_schema, circ_policy_martigny_data_tmp): @@ -149,7 +148,7 @@ def test_circ_policy_reminders(circ_policy_schema, cipo['reminders'].append(due_soon_reminder) validate(cipo, circ_policy_schema) # Tow "DUE_SOON" reminder is disallow - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): due_soon_reminder_2 = deepcopy(due_soon_reminder) due_soon_reminder_2['days_delay'] = 5 cipo['reminders'].append(due_soon_reminder_2) @@ -164,7 +163,7 @@ def test_circ_policy_reminders(circ_policy_schema, 'communication_channel': 'mail', 'template': 'email/overdue' } - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): overdue_reminder1 = deepcopy(overdue_reminder) overdue_reminder2 = deepcopy(overdue_reminder) overdue_reminder2['template'] = 'email/overdue2' @@ -193,21 +192,21 @@ def test_circ_policy_overdue_fees(circ_policy_schema, cipo.validate() # two intervals with no upper limit - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): invalid_overdue_data = deepcopy(overdue_data) del invalid_overdue_data['intervals'][2]['to'] cipo['overdue_fees'] = invalid_overdue_data cipo.validate() # two intervals with conflict on lower interval limit - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): invalid_overdue_data = deepcopy(overdue_data) invalid_overdue_data['intervals'][2]['from'] = 4 cipo['overdue_fees'] = invalid_overdue_data cipo.validate() # two intervals with conflict on upper interval limit - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): invalid_overdue_data = deepcopy(overdue_data) invalid_overdue_data['intervals'][0]['to'] = 7 cipo['overdue_fees'] = invalid_overdue_data diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 268174b6ab..1f983eb903 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -21,7 +21,9 @@ from click.testing import CliRunner -from rero_ils.modules.cli import check_validate, tokens_create +from rero_ils.modules.cli import check_validate, extract_from_xml, \ + reindex_missing, tokens_create +from rero_ils.modules.organisations.api import Organisation def test_cli_validate(app, script_info): @@ -51,3 +53,42 @@ def test_cli_access_token(app, script_info, patron_martigny): obj=script_info ) assert res.output.strip().split('\n') == ['my_token'] + + +def test_cli_reindex_missing(app, script_info, org_sion_data): + """Test reindex missing cli.""" + org = Organisation.create( + data=org_sion_data, + delete_pid=False, + dbcommit=True, + ) + print(org) + + runner = CliRunner() + res = runner.invoke( + reindex_missing, + ['-t', 'xxx', '-t', 'org', '-v'], + obj=script_info + ) + assert res.output.strip().split('\n') == [ + 'Indexing missing xxx: ERROR pid type does not exist!', + 'Indexing missing org: 1', + '1\torg\torg2' + ] + + +def test_cli_extract_from_xml(app, tmpdir, document_marcxml, script_info): + """Test extract from xml cli.""" + pids_file_name = join(dirname(__file__), '..', 'data', '001.pids') + xml_file_name = join(dirname(__file__), '..', 'data', 'documents.xml') + temp_file_name = join(tmpdir, 'temp.xml') + runner = CliRunner() + result = runner.invoke( + extract_from_xml, + [pids_file_name, xml_file_name, temp_file_name, '-v'], + obj=script_info + ) + assert result.exit_code == 0 + results_output = result.output.split('\n') + assert results_output[0] == 'Extract pids from xml: ' + assert results_output[4] == 'Search pids count: 1' diff --git a/tests/unit/test_csv.py b/tests/unit/test_csv.py new file mode 100644 index 0000000000..b01746298a --- /dev/null +++ b/tests/unit/test_csv.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Test csv creation, import et export.""" + + +import json +from os.path import dirname, join + +from click.testing import CliRunner + +from rero_ils.modules.cli import bulk_load, bulk_save, create_csv + + +def test_create_csv(app, tmpdir, script_info): + """Test create_csv cli.""" + tmp_dir_name = tmpdir.dirname + json_file_name = join(dirname(__file__), '../data/documents.json') + runner = CliRunner() + result = runner.invoke( + create_csv, + ['doc', json_file_name, tmp_dir_name, '-l', '-v'], + obj=script_info + ) + assert result.exit_code == 0 + file_name_pidstore = join(tmp_dir_name, 'doc_pidstore.csv') + file_name_metadata = join(tmp_dir_name, 'doc_metadata.csv') + file_name_pids = join(tmp_dir_name, 'doc_pids.csv') + output = result.output.split('\n') + assert output[0] == 'Create CSV files for: doc from: {file_name}'.format( + file_name=json_file_name + ) + assert output[1] == '\t{file_name}'.format(file_name=file_name_pidstore) + assert output[2] == '\t{file_name}'.format(file_name=file_name_metadata) + assert output[3] == '\t{file_name}'.format(file_name=file_name_pids) + assert output[4].split(':')[0] == '1\tdoc\t1' + assert output[5].split(':')[0] == '2\tdoc\t2' + + result = runner.invoke( + bulk_load, + ['doc', file_name_metadata, '-r'], + obj=script_info + ) + assert result.exit_code == 0 + assert result.output.split('\n') == [ + 'Load doc CSV files into database.', + ' Number of records to load: 2', + ' Load pids: {file_name}'.format(file_name=file_name_pids), + ' Load pidstore: {file_name}'.format(file_name=file_name_pidstore), + ' Load metatada: {file_name}'.format(file_name=file_name_metadata), + '' + ] + result = runner.invoke( + bulk_save, + [tmp_dir_name, '-t', 'xxx', '-t', 'doc'], + obj=script_info + ) + assert result.exit_code == 0 + assert result.output.split('\n') == [ + 'Error xxx does not exist!', + 'Save doc CSV files to directory: {dir}'.format(dir=tmp_dir_name), + 'Saved records: 2', + '' + ] + + saved_name_meta = join(tmp_dir_name, 'documents_small_metadata.csv') + with open(file_name_metadata) as meta, open(saved_name_meta) as saved_meta: + for line1, line2 in zip(meta, saved_meta): + line1 = line1.strip().split('\t')[2:] + json1 = json.loads(line1[1]) + del line1[1] + line2 = line2.strip().split('\t')[2:] + json2 = json.loads(line2[1]) + del line2[1] + assert line1 == line2 + assert json1 == json2 + + saved_name_pidstore = join(tmp_dir_name, 'documents_small_pidstore.csv') + with open(file_name_pidstore) as pids, \ + open(saved_name_pidstore) as saved_pidstore: + for line1, line2 in zip(pids, saved_pidstore): + line1 = line1.strip().split('\t')[2:] + line2 = line2.strip().split('\t')[2:] + assert line1 == line2 + + saved_name_pids = join(tmp_dir_name, 'documents_small_pids.csv') + with open(file_name_pids) as pids, open(saved_name_pids) as saved_pids: + for line1, line2 in zip(pids, saved_pids): + line1 = line1.strip() + line2 = line2.strip() + assert line1 == line2 diff --git a/tests/unit/test_ill_requests_jsonschema.py b/tests/unit/test_ill_requests_jsonschema.py index b874d93b2b..269bc8f514 100644 --- a/tests/unit/test_ill_requests_jsonschema.py +++ b/tests/unit/test_ill_requests_jsonschema.py @@ -25,7 +25,6 @@ from jsonschema import validate from jsonschema.exceptions import ValidationError -from rero_ils.modules.errors import RecordValidationError from rero_ils.modules.ill_requests.api import ILLRequest from rero_ils.modules.ill_requests.models import ILLRequestNoteStatus @@ -69,7 +68,7 @@ def test_extended_validation(app, ill_request_martigny_data_tmp): data['copy'] = True if 'pages' in data: del data['pages'] - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): ILLRequest.validate(ILLRequest(data)) # test on 'notes' field :: have 2 note of the same type is disallowed @@ -79,7 +78,7 @@ def test_extended_validation(app, ill_request_martigny_data_tmp): 'content': 'dummy content' }] ILLRequest.validate(ILLRequest(data)) - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): data['notes'].append({ 'type': ILLRequestNoteStatus.PUBLIC_NOTE, 'content': 'second dummy note' diff --git a/tests/unit/test_items_jsonschema.py b/tests/unit/test_items_jsonschema.py index 9092b4acfd..2bad5e1211 100644 --- a/tests/unit/test_items_jsonschema.py +++ b/tests/unit/test_items_jsonschema.py @@ -25,7 +25,6 @@ from jsonschema import validate from jsonschema.exceptions import ValidationError -from rero_ils.modules.errors import RecordValidationError from rero_ils.modules.items.models import ItemNoteTypes from rero_ils.modules.utils import get_ref_for_pid @@ -113,7 +112,7 @@ def test_temporary_item_type(item_schema, item_lib_martigny): data = item_lib_martigny # tmp_itty cannot be the same than main_itty - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): data['temporary_item_type'] = { '$ref': data['item_type']['$ref'] } @@ -121,7 +120,7 @@ def test_temporary_item_type(item_schema, item_lib_martigny): data.validate() # check extented_validation # tmp_itty_enddate must be older than current date - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): current_date = datetime.datetime.now().strftime('%Y-%m-%d') data['temporary_item_type'] = { '$ref': get_ref_for_pid('itty', 'sample'), diff --git a/tests/unit/test_patron_types_jsonschema.py b/tests/unit/test_patron_types_jsonschema.py index 8d491d5fad..0a57c2151a 100644 --- a/tests/unit/test_patron_types_jsonschema.py +++ b/tests/unit/test_patron_types_jsonschema.py @@ -23,7 +23,6 @@ from jsonschema import validate from jsonschema.exceptions import ValidationError -from rero_ils.modules.errors import RecordValidationError from rero_ils.modules.utils import get_ref_for_pid @@ -100,13 +99,13 @@ def test_limits(patron_type_schema, patron_type_tmp): } } validate(data, patron_type_schema) - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): data['limits']['checkout_limits']['library_limit'] = 40 validate(data, patron_type_schema) # valid for JSON schema data.validate() # invalid against extented_validation rules data['limits']['checkout_limits']['library_limit'] = 15 - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): lib_ref = get_ref_for_pid('lib', 'dummy') data['limits']['checkout_limits']['library_exceptions'] = [ {'library': {'$ref': lib_ref}, 'value': 15} @@ -114,7 +113,7 @@ def test_limits(patron_type_schema, patron_type_tmp): validate(data, patron_type_schema) # valid for JSON schema data.validate() # invalid against extented_validation rules - with pytest.raises(RecordValidationError): + with pytest.raises(ValidationError): data['limits']['checkout_limits']['library_exceptions'] = [ {'library': {'$ref': lib_ref}, 'value': 5}, {'library': {'$ref': lib_ref}, 'value': 7}