diff --git a/.github/workflows/e2e-workflow-server.yml b/.github/workflows/e2e-workflow-server.yml index 09c8e48264..4da7eb18df 100644 --- a/.github/workflows/e2e-workflow-server.yml +++ b/.github/workflows/e2e-workflow-server.yml @@ -2,7 +2,7 @@ name: E2E - Server changes with python package installations on: pull_request: - branches: [master, ant, faser] + branches: [master, ant, next] paths: - "setup.py" - "requirements.txt" diff --git a/.github/workflows/e2e-workflow-ui.yml b/.github/workflows/e2e-workflow-ui.yml index 429b837d56..c50f31be2a 100644 --- a/.github/workflows/e2e-workflow-ui.yml +++ b/.github/workflows/e2e-workflow-ui.yml @@ -3,7 +3,7 @@ on: push: branches: uitest pull_request: - branches: [master, ant, faser] + branches: [master, next] paths: - "ui/**" - ".github/workflows/ui-test.yml" diff --git a/.github/workflows/e2e-workflow.yml b/.github/workflows/e2e-workflow.yml index 866baa5daf..6408868246 100644 --- a/.github/workflows/e2e-workflow.yml +++ b/.github/workflows/e2e-workflow.yml @@ -5,11 +5,12 @@ on: - master - qa - production + - next pull_request: branches: - qa - production - - faser + - next jobs: Cypress: name: Cypress @@ -25,7 +26,7 @@ jobs: run: docker-compose -f docker-compose.e2e.yml run web-api sh scripts/clean-and-init.sh - name: Update save folder - run: docker-compose -f docker-compose.e2e.yml run web-api cap files location local var/data --default + run: docker-compose -f docker-compose.e2e.yml run web-api cap files location local var/data --default - name: Sleep for 180 seconds run: sleep 180s diff --git a/.github/workflows/lint-commit.yml b/.github/workflows/lint-commit.yml index b73d95cf53..72c5827da4 100644 --- a/.github/workflows/lint-commit.yml +++ b/.github/workflows/lint-commit.yml @@ -2,7 +2,7 @@ name: Lint Commit on: pull_request: - branches: [ master ] + branches: [ master, next ] jobs: lint-shellcheck: @@ -28,7 +28,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.6.9 + python-version: '3.10' - name: Checking commit quality (messages, signatures, spelling checks, etc) env: @@ -44,7 +44,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.6.9 + python-version: '3.10' - name: Check Python manifest completeness run: | @@ -59,7 +59,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.6.9 + python-version: '3.10' - name: Check compliance with Python docstring conventions run: | diff --git a/.github/workflows/lint-push.yml b/.github/workflows/lint-push.yml index 640f42f5ce..c527cf5ea1 100644 --- a/.github/workflows/lint-push.yml +++ b/.github/workflows/lint-push.yml @@ -2,7 +2,7 @@ name: Lint Source on: push: - branches: [ master ] + branches: [ master, next ] jobs: lint-shellcheck: @@ -23,7 +23,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.6.9 + python-version: '3.10' - name: Check compliance with Python docstring conventions run: | @@ -38,7 +38,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.6.9 + python-version: '3.10' - name: Check compliance with pep8, pyflakes and circular complexity run: | @@ -55,7 +55,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.6.9 + python-version: '3.10' - name: Run black linter for python run: | @@ -70,7 +70,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.6.9 + python-version: '3.10' - name: isort your imports, so you don't have to. run: | @@ -100,7 +100,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.6.9 + python-version: '3.10' - name: Check Python manifest completeness run: | @@ -120,7 +120,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.6.9 + python-version: '3.10' - name: Checking commit quality (messages, signatures, spelling checks, etc) env: diff --git a/.github/workflows/ui-test.yml b/.github/workflows/ui-test.yml index adb065856a..d6c1b88bd5 100644 --- a/.github/workflows/ui-test.yml +++ b/.github/workflows/ui-test.yml @@ -1,12 +1,12 @@ name: UI test on: push: - branches: [master, ant] + branches: [master, next] paths: - "ui/**" - ".github/workflows/ui-test.yml" pull_request: - branches: [master, ant, faser] + branches: [master, next] paths: - "ui/**" - ".github/workflows/ui-test.yml" diff --git a/.github/workflows/web-api.yml b/.github/workflows/web-api.yml index 96b369757f..fb1eb49ed4 100644 --- a/.github/workflows/web-api.yml +++ b/.github/workflows/web-api.yml @@ -2,11 +2,11 @@ name: Backend tests on: push: - branches: [ master, ant ] + branches: [ master, next ] paths-ignore: - 'ui/**' pull_request: - branches: [ master, ant, faser ] + branches: [ master, next ] paths-ignore: - 'ui/**' @@ -19,10 +19,11 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.6.9 + python-version: '3.10' - name: Run darker run: | + pip install flake8 pip install darker[isort] darker . --check -i -L flake8 @@ -45,15 +46,22 @@ jobs: image: rabbitmq:3-management ports: - 5672:5672 - elasticsearchd: - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.4 + search: + image: opensearchproject/opensearch:2.2.0 env: + node.name: search bootstrap.memory_lock: true - ES_JAVA_OPTS: -Xms1024m -Xmx1024m + OPENSEARCH_JAVA_OPTS: -Xms1024m -Xmx1024m discovery.type: single-node - xpack.security.enabled: false + discovery.seed_hosts: search + DISABLE_SECURITY_PLUGIN: true ports: - 9200:9200 + options: >- + --health-cmd "curl http://localhost:9200/_cluster/health" + --health-interval 10s + --health-timeout 5s + --health-retries 10 steps: - uses: actions/checkout@v2 @@ -61,19 +69,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.6.9 + python-version: '3.10' - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install libsasl2-dev python-dev libldap2-dev libssl-dev libkrb5-dev + sudo apt-get install libsasl2-dev python3-dev libldap2-dev libssl-dev libkrb5-dev - name: Install dependencies run: | - pip install --upgrade pip==20.2.4 - pip install -r requirements.txt - pip install -e .[all,docs,tests,ldap] - pip install -r requirements-local-forks.txt + pip install . - name: Run integration and unit tests run: ./run-tests.sh --check-pytest diff --git a/Dockerfile b/Dockerfile index 52e58b1b1a..478f84ebc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,48 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2018 CERN. +# Copyright (C) 2023 CERN. # -# CERN Analysis Preservation is free software; you can redistribute it and/or modify it -# under the terms of the MIT License; see LICENSE file for more details. - -FROM gitlab-registry.cern.ch/analysispreservation/base:python3 +# Base cap image to install on with base python 3.10 + +FROM python:3.10.10 + +# Certficates configuartion +ENV PYTHONBUFFERED=0 \ + SSL_CERT_FILE="/etc/ssl/certs/ca-certificates.crt" \ + REQUESTS_CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt" \ + PATH="/root/.local/bin:${PATH}" \ + POETRY_VIRTUALENVS_CREATE=false +COPY docker/base/CERN_Root_Certification_Authority_2.pem /usr/local/share/ca-certificates/CERN_Root_Certification_Authority_2.crt + +# Install system dependencies +RUN update-ca-certificates && pip config set global.cert "${REQUESTS_CA_BUNDLE}" +RUN curl -s -L http://cern.ch/linux/docs/krb5.conf -o /etc/krb5.conf +RUN curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg + +RUN apt-get update && apt-get install --no-install-recommends -y \ + gcc libffi-dev locales libxslt-dev libxml2-dev libssl-dev build-essential python3-dev libldap2-dev libsasl2-dev ldap-utils krb5-user git redis \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Set the locale +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 # Install Invenio ENV WORKING_DIR=/opt/cap ENV INVENIO_INSTANCE_PATH=${WORKING_DIR}/var/instance +ENV CAP_FILES_DIR=${WORKING_DIR}/src/var # Debug off by default ARG DEBUG=False ENV DEBUG=${DEBUG} +# ENABLE_E2E False by default +ARG ENABLE_E2E=False +ENV ENABLE_E2E=${ENABLE_E2E} + # copy everything inside /src RUN mkdir -p ${WORKING_DIR}/src COPY ./ ${WORKING_DIR}/src @@ -28,17 +56,13 @@ RUN python -m site --user-site # Install/create static files RUN mkdir -p ${INVENIO_INSTANCE_PATH} +RUN mkdir -p ${CAP_FILES_DIR} +RUN pip install -e . -RUN pip install --upgrade wheel uwsgi uwsgitop uwsgi-tools - -# RUN if [ "$DEBUG" = "True" ]; then pip install -r requirements-devel.txt; fi; -RUN pip install -r requirements.txt -RUN pip install -e .[all,xrootd] -RUN pip install -r requirements-local-forks.txt - +RUN cat ./docker/base/CERN_Root_Certification_Authority_2.pem >> /usr/local/lib/python3.10/site-packages/certifi/cacert.pem -# copy uwsgi config files +# Copy uwsgi config files COPY ./docker/uwsgi/ ${INVENIO_INSTANCE_PATH} ARG APP_GITHUB_OAUTH_ACCESS_TOKEN @@ -49,9 +73,10 @@ ENV APP_GITLAB_OAUTH_ACCESS_TOKEN=${APP_GITLAB_OAUTH_ACCESS_TOKEN} RUN pip install gunicorn # Set folder permissions -RUN chgrp -R 0 ${WORKING_DIR} && \ - chmod -R g=u ${WORKING_DIR} +RUN chgrp -R 0 ${WORKING_DIR} ${CAP_FILES_DIR} && \ + chmod -R g=u ${WORKING_DIR} ${CAP_FILES_DIR} RUN useradd invenio --uid 1000 --gid 0 && \ - chown -R invenio:root ${WORKING_DIR} + chown -R invenio:root ${WORKING_DIR} ${CAP_FILES_DIR} && \ + chown -R invenio:root /usr/local/lib/python3.10/site-packages/certifi USER 1000 diff --git a/MANIFEST.in b/MANIFEST.in index be9fd928b1..13fffbe413 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -47,6 +47,7 @@ include .githooks/pre-push include .githooks/prepare-commit-msg include docker/harbour/backend/Dockerfile include docker/harbour/ui/Dockerfile +include docker/base/Dockerfile include docker/haproxy/Dockerfile include docker/nginx/Dockerfile include docker/postgres/Dockerfile diff --git a/cap.sample.env b/cap.sample.env index 2cbeaa09da..ef9d900a51 100644 --- a/cap.sample.env +++ b/cap.sample.env @@ -10,7 +10,7 @@ APP_BROKER_URL=redis://redis:6379/0 # APP_CELERY_BROKER_URL=amqp://guest:guest@rabbitmq:5672/ APP_ACCESS_SESSION_REDIS_HOST=redis APP_CELERY_RESULT_BACKEND=redis://redis:6379/2 -APP_SEARCH_ELASTIC_HOSTS=elasticsearch +APP_SEARCH_HOSTS=opensearch APP_JSONSCHEMAS_HOST=localhost:5000 # CERN oAuth client credentials diff --git a/cap/config.py b/cap/config.py index 683dd5c3bd..b288c7e214 100644 --- a/cap/config.py +++ b/cap/config.py @@ -328,7 +328,7 @@ def _(x): 'facet_cms_working_group': { 'terms': { 'size': 30, - 'script': 'doc.containsKey("basic_info.cadi_id") ? doc["basic_info.cadi_id"].value?.substring(0,3) : null', # noqa + 'script': 'doc.containsKey("basic_info.cadi_id") && doc["basic_info.cadi_id"].size() != 0 ? doc["basic_info.cadi_id"].value?.substring(0,3) : null' # noqa }, 'meta': {'title': 'CMS Working Group'}, }, @@ -576,25 +576,25 @@ def _(x): #: Default API endpoint for search UI. SEARCH_UI_SEARCH_API = '/api/deposits' -#: Default ElasticSearch hosts -es_user = os.environ.get('ELASTICSEARCH_USER') -es_password = os.environ.get('ELASTICSEARCH_PASSWORD') -if es_user and es_password: - es_params = dict( - http_auth=(es_user, es_password), - use_ssl=str(os.environ.get('ELASTICSEARCH_USE_SSL')).lower() == 'true', - verify_certs=str(os.environ.get('ELASTICSEARCH_VERIFY_CERTS')).lower() +#: Default Search hosts +search_user = os.environ.get('OPENSEARCH_USER') +search_password = os.environ.get('OPENSEARCH_PASSWORD') +if search_user and search_password: + search_params = dict( + http_auth=(search_user, search_password), + use_ssl=str(os.environ.get('OPENSEARCH_USE_SSL')).lower() == 'true', + verify_certs=str(os.environ.get('OPENSEARCH_VERIFY_CERTS')).lower() == 'true', - url_prefix=os.environ.get('ELASTICSEARCH_URL_PREFIX', ''), + url_prefix=os.environ.get('OPENSEARCH_URL_PREFIX', ''), ) else: - es_params = {} + search_params = {} -SEARCH_ELASTIC_HOSTS = [ +SEARCH_HOSTS = [ dict( - host=os.environ.get('ELASTICSEARCH_HOST', 'localhost'), - port=int(os.environ.get('ELASTICSEARCH_PORT', '9200')), - **es_params, + host=os.environ.get('OPENSEARCH_HOST', 'localhost'), + port=int(os.environ.get('OPENSEARCH_PORT', '9200')), + **search_params, ) ] @@ -956,3 +956,7 @@ def get_cms_stats_questionnaire_contacts(): #: Enable Prometheus flask exporter PROMETHEUS_ENABLE_EXPORTER_FLASK = False + +RECORDS_FILES_REST_ENDPOINTS = {} + +APP_HEALTH_BLUEPRINT_ENABLED=False diff --git a/cap/modules/cache/ext.py b/cap/modules/cache/ext.py index 012671549c..acb8b6742f 100644 --- a/cap/modules/cache/ext.py +++ b/cap/modules/cache/ext.py @@ -27,7 +27,7 @@ from __future__ import absolute_import, print_function -from flask_cache import Cache +from flask_caching import Cache class CAPCache(object): diff --git a/cap/modules/deposit/__init__.py b/cap/modules/deposit/__init__.py index 0223f9c704..6e1019de16 100644 --- a/cap/modules/deposit/__init__.py +++ b/cap/modules/deposit/__init__.py @@ -27,3 +27,5 @@ from __future__ import absolute_import, print_function from .cli import create_deposit # noqa +from .serializers import * +from .validators import * diff --git a/cap/modules/deposit/api.py b/cap/modules/deposit/api.py index 44d065d7f5..2f75e1ccd7 100644 --- a/cap/modules/deposit/api.py +++ b/cap/modules/deposit/api.py @@ -731,7 +731,7 @@ def create(cls, data, id_=None, owner=current_user): * create RecordMetadata instance * create bucket for storing deposit files * set owner permissions in the db - * index deposit in elasticsearch + * index deposit in opensearch """ if current_user and current_user.is_authenticated: owner = current_user diff --git a/cap/modules/deposit/cli.py b/cap/modules/deposit/cli.py index b1c9eda543..dbe7bae69c 100644 --- a/cap/modules/deposit/cli.py +++ b/cap/modules/deposit/cli.py @@ -32,7 +32,7 @@ from datetime import datetime import click -from flask_cli import with_appcontext +from flask.cli import with_appcontext from invenio_db import db from invenio_jsonschemas.errors import JSONSchemaNotFound from invenio_pidstore.errors import PIDDoesNotExistError diff --git a/cap/modules/deposit/search.py b/cap/modules/deposit/search.py index 0822c3b73e..4bc681d1fd 100644 --- a/cap/modules/deposit/search.py +++ b/cap/modules/deposit/search.py @@ -24,7 +24,7 @@ """Configuration for deposit search.""" -from elasticsearch_dsl import Q, TermsFacet +from opensearch_dsl import Q, TermsFacet from flask import g from flask_login import current_user from flask_principal import RoleNeed @@ -93,7 +93,6 @@ class Meta: """Configuration for deposit search.""" index = 'deposits' - doc_types = None fields = ('*',) facets = { 'status': TermsFacet(field='_deposit.status'), diff --git a/cap/modules/deposit/serializers/__init__.py b/cap/modules/deposit/serializers/__init__.py index 1723d9ea18..15e5895249 100644 --- a/cap/modules/deposit/serializers/__init__.py +++ b/cap/modules/deposit/serializers/__init__.py @@ -23,13 +23,10 @@ # as an Intergovernmental Organization or submit itself to any jurisdiction. """Record serialization.""" -from __future__ import absolute_import, print_function - -from invenio_deposit.serializers import json_file_response from invenio_records_rest.serializers.response import (record_responsify, search_responsify) -from .json import DepositSerializer +from .json import DepositSerializer, json_file_response from .schemas.json import DepositSchema, DepositFormSchema, DepositSearchSchema # Serializers diff --git a/cap/modules/deposit/serializers/json.py b/cap/modules/deposit/serializers/json.py index 1e027395cf..4f4b92b47c 100644 --- a/cap/modules/deposit/serializers/json.py +++ b/cap/modules/deposit/serializers/json.py @@ -23,6 +23,10 @@ # as an Intergovernmental Organization or submit itself to any jurisdiction. """CAP Basic Serializers.""" +from flask import jsonify, make_response + +from invenio_deposit.serializers import json_file_serializer, file_serializer + from cap.modules.deposit.api import CAPDeposit from cap.modules.deposit.links import links_factory as deposit_links_factory from cap.modules.records.serializers.json import CAPJSONSerializer @@ -49,3 +53,36 @@ def preprocess_record(self, pid, record, links_factory=None, **kwargs): if deposit.files: result['bucket'] = deposit.files.bucket return result + + +def json_files_serializer(objs, status=None): + """JSON Files Serializer. + + :parma objs: A list of:class:`invenio_files_rest.models.ObjectVersion` + instances. + :param status: A HTTP Status. (Default: ``None``) + :returns: A Flask response with JSON data. + :rtype: :py:class:`flask.Response`. + """ + files = [file_serializer(obj) for obj in objs] + response = make_response(jsonify(files), status) + return response + + +def json_file_response(obj=None, pid=None, record=None, status=None): + """JSON Files/File serializer. + + :param obj: A :class:`invenio_files_rest.models.ObjectVersion` instance or + a :class:`invenio_records_files.api.FilesIterator` if it's a list of + files. + :param pid: PID value. (not used) + :param record: The record metadata. (not used) + :param status: The HTTP status code. + :returns: A Flask response with JSON data. + :rtype: :py:class:`flask.Response`. + """ + from invenio_records_files.api import FilesIterator + if isinstance(obj, FilesIterator): + return json_files_serializer(obj, status=status) + else: + return json_file_serializer(obj, status=status) diff --git a/cap/modules/deposit/utils.py b/cap/modules/deposit/utils.py index 3e60c472cc..c98b7dbdce 100644 --- a/cap/modules/deposit/utils.py +++ b/cap/modules/deposit/utils.py @@ -111,7 +111,6 @@ def prepare_record( json=None, record=None, index=None, - doc_type=None, arguments=None, **kwargs ): @@ -122,8 +121,7 @@ def prepare_record( version = schema.version fullname = schema.fullname or "" except JSONSchemaNotFound: - name, version = doc_type.rsplit("-v", 1) - fullname = name + abort(404) collection = {"name": name, "version": version, "fullname": fullname} json["_collection"] = collection diff --git a/cap/modules/experiments/cli.py b/cap/modules/experiments/cli.py index fcd08493a8..4b9951b0b2 100644 --- a/cap/modules/experiments/cli.py +++ b/cap/modules/experiments/cli.py @@ -25,7 +25,7 @@ import json import click -from flask_cli import with_appcontext +from flask.cli import with_appcontext from jsonschema.exceptions import ValidationError from invenio_db import db @@ -69,7 +69,7 @@ def index_datasets(file): source = (dict(name=line.strip()) for line in fp) cache_das_datasets_in_es_from_file(source) - click.secho("Datasets indexed in Elasticsearch.", fg='green') + click.secho("Datasets indexed in Opensearch.", fg='green') @cms.command('index-triggers') @@ -81,7 +81,7 @@ def index_triggers(file): source = (x for x in json.load(fp)) cache_cms_triggers_in_es_from_file(source) - click.secho("Triggers indexed in Elasticsearch.", fg='green') + click.secho("Triggers indexed in Opensearch.", fg='green') def _questionnaire_data(data, title=None): diff --git a/cap/modules/experiments/search/cms_triggers.py b/cap/modules/experiments/search/cms_triggers.py index 70fbdffcc4..2c007cfecf 100644 --- a/cap/modules/experiments/search/cms_triggers.py +++ b/cap/modules/experiments/search/cms_triggers.py @@ -25,30 +25,28 @@ import re -from elasticsearch_dsl import Q, Search +from opensearch_dsl import Q, Search from invenio_search.proxies import current_search_client as es CMS_TRIGGERS_ES_CONFIG = { 'alias': 'cms-triggers', 'mappings': { - "doc": { - "properties": { - "trigger": { - "type": "keyword", - "normalizer": "my_normalizer", - "fields": { - "keyword": { - "type": "keyword" - } + "properties": { + "trigger": { + "type": "keyword", + "normalizer": "my_normalizer", + "fields": { + "keyword": { + "type": "keyword" } - }, - "year": { - "type": "keyword", - }, - "dataset": { - "type": "keyword", - "normalizer": "my_normalizer" } + }, + "year": { + "type": "keyword", + }, + "dataset": { + "type": "keyword", + "normalizer": "my_normalizer" } } }, diff --git a/cap/modules/experiments/search/das.py b/cap/modules/experiments/search/das.py index 76fb608a3d..12ee7b7f02 100644 --- a/cap/modules/experiments/search/das.py +++ b/cap/modules/experiments/search/das.py @@ -23,18 +23,16 @@ # as an Intergovernmental Organization or submit itself to any jurisdiction. """Search classes and methods for DAS querying.""" -from elasticsearch_dsl import Search +from opensearch_dsl import Search from invenio_search.proxies import current_search_client as es DAS_DATASETS_ES_CONFIG = { 'alias': 'das-datasets', "mappings": { - "doc": { - "properties": { - "name": { - "type": "keyword", - "normalizer": "lowercase_normalizer" - } + "properties": { + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer" } } }, diff --git a/cap/modules/experiments/utils/cadi.py b/cap/modules/experiments/utils/cadi.py index 9690577ba3..aab05f5536 100644 --- a/cap/modules/experiments/utils/cadi.py +++ b/cap/modules/experiments/utils/cadi.py @@ -26,10 +26,10 @@ from itertools import islice import requests -from elasticsearch_dsl import Q from flask import current_app, abort from invenio_db import db from invenio_search import RecordsSearch +from opensearch_dsl import Q from cap.modules.deposit.api import CAPDeposit from cap.modules.deposit.errors import (DepositDoesNotExist, @@ -261,7 +261,7 @@ def get_uuids_with_same_cadi_id(cadi_id): raise DepositDoesNotExist else: uuids = [ - r.get("_source", {}).get("_deposit", {}).get("id") + r._source._deposit.id for r in res ] diff --git a/cap/modules/experiments/utils/common.py b/cap/modules/experiments/utils/common.py index de6e43789a..d8b5a78397 100644 --- a/cap/modules/experiments/utils/common.py +++ b/cap/modules/experiments/utils/common.py @@ -31,7 +31,7 @@ import cern_sso as cern_sso_old import cap.modules.experiments.utils.cern_sso as cern_sso_current from cachetools.func import ttl_cache -from elasticsearch import helpers +from opensearchpy import helpers from flask import current_app from invenio_search.proxies import current_search_client as es @@ -155,7 +155,7 @@ def _batches(iterable, chunk_size=10): '_id': id_getter(obj) if id_getter else None, '_source': obj } for obj in batch] - helpers.bulk(es, actions, index=new_index, doc_type='doc') + helpers.bulk(es, actions, index=new_index) print('.', end='', flush=True) except Exception as e: es.indices.delete( diff --git a/cap/modules/fixtures/schemas/alice-analysis.json b/cap/modules/fixtures/schemas/alice-analysis.json index 15e5197c66..4eaa9ad3b2 100644 --- a/cap/modules/fixtures/schemas/alice-analysis.json +++ b/cap/modules/fixtures/schemas/alice-analysis.json @@ -75,7 +75,7 @@ "settings": { "analysis": { "analyzer": { - "lowercase_whitespace_analyzer": { + "default": { "type": "custom", "tokenizer": "whitespace", "filter": ["lowercase"] @@ -84,66 +84,60 @@ } }, "mappings": { - "alice-analysis-v0.0.1": { - "_all": { - "enabled": true, - "analyzer": "lowercase_whitespace_analyzer" + "properties": { + "_deposit": { + "type": "object", + "properties": { + "created_by": { + "type": "integer", + "copy_to": "created_by" + }, + "status": { + "type": "text", + "fields": { + "keywords": { + "type": "keyword" + } + }, + "copy_to": "status" + } + } }, - "properties": { - "_deposit": { - "type": "object", - "properties": { - "created_by": { - "type": "integer", - "copy_to": "created_by" + "_collection": { + "type": "object", + "properties": { + "fullname": { + "type": "keyword" + }, + "name": { + "type": "keyword" }, - "status": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - }, - "copy_to": "status" + "version": { + "type": "keyword" } - } - }, - "_collection": { - "type": "object", - "properties": { - "fullname": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" - } - } - }, - "_experiment": { - "type": "text" - }, - "_fetched_from": { - "type": "text", - "copy_to": "fetched_from" - }, - "_user_edited": { - "type": "boolean", - "copy_to": "user_edited" - }, - "_created": { - "type": "date", - "copy_to": "created" - }, - "_updated": { - "type": "date", - "copy_to": "updated" - }, - "created": { - "type": "date" } + }, + "_experiment": { + "type": "text" + }, + "_fetched_from": { + "type": "text", + "copy_to": "fetched_from" + }, + "_user_edited": { + "type": "boolean", + "copy_to": "user_edited" + }, + "_created": { + "type": "date", + "copy_to": "created" + }, + "_updated": { + "type": "date", + "copy_to": "updated" + }, + "created": { + "type": "date" } } } diff --git a/cap/modules/fixtures/schemas/atlas.json b/cap/modules/fixtures/schemas/atlas.json index cb989fa7ac..6d97bab00f 100644 --- a/cap/modules/fixtures/schemas/atlas.json +++ b/cap/modules/fixtures/schemas/atlas.json @@ -138,7 +138,7 @@ "settings": { "analysis": { "analyzer": { - "lowercase_whitespace_analyzer": { + "default": { "type": "custom", "tokenizer": "whitespace", "filter": ["lowercase"] @@ -147,70 +147,64 @@ } }, "mappings": { - "atlas-analysis-v0.0.1": { - "_all": { - "enabled": true, - "analyzer": "lowercase_whitespace_analyzer" + "properties": { + "_deposit": { + "type": "object", + "properties": { + "created_by": { + "type": "integer", + "copy_to": "created_by" + }, + "status": { + "type": "text", + "fields": { + "keywords": { + "type": "keyword" + } + }, + "copy_to": "status" + } + } }, - "properties": { - "_deposit": { - "type": "object", - "properties": { - "created_by": { - "type": "integer", - "copy_to": "created_by" + "_collection": { + "type": "object", + "properties": { + "fullname": { + "type": "keyword" }, - "status": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - }, - "copy_to": "status" + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" } - } - }, - "_collection": { - "type": "object", - "properties": { - "fullname": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" - } - } - }, - "_experiment": { - "type": "text" - }, - "_fetched_from": { - "type": "text", - "copy_to": "fetched_from" - }, - "_user_edited": { - "type": "boolean", - "copy_to": "user_edited" - }, - "_created": { - "type": "date", - "copy_to": "created" - }, - "_updated": { - "type": "date", - "copy_to": "updated" - }, - "created": { - "type": "date" - }, - "workflows": { - "type": "object", - "enabled": "false" } + }, + "_experiment": { + "type": "text" + }, + "_fetched_from": { + "type": "text", + "copy_to": "fetched_from" + }, + "_user_edited": { + "type": "boolean", + "copy_to": "user_edited" + }, + "_created": { + "type": "date", + "copy_to": "created" + }, + "_updated": { + "type": "date", + "copy_to": "updated" + }, + "created": { + "type": "date" + }, + "workflows": { + "type": "object", + "enabled": "false" } } } diff --git a/cap/modules/fixtures/schemas/cms-questionnaire.json b/cap/modules/fixtures/schemas/cms-questionnaire.json index 1c32c9c501..41c9874730 100644 --- a/cap/modules/fixtures/schemas/cms-questionnaire.json +++ b/cap/modules/fixtures/schemas/cms-questionnaire.json @@ -1626,7 +1626,7 @@ "settings":{ "analysis":{ "analyzer":{ - "lowercase_whitespace_analyzer":{ + "default":{ "type":"custom", "tokenizer":"whitespace", "filter":[ @@ -1637,75 +1637,69 @@ } }, "mappings":{ - "cms-stats-questionnaire-v0.0.1":{ - "_all":{ - "enabled":true, - "analyzer":"lowercase_whitespace_analyzer" + "properties":{ + "next_deadline_date": { + "type": "date" }, - "properties":{ - "next_deadline_date": { - "type": "date" - }, - "analysis_context": { - "type": "object", - "properties": { - "next_deadline_date": { - "type": "date", - "format": "yyyy-MM-dd", - "copy_to": "next_deadline_date" - } + "analysis_context": { + "type": "object", + "properties": { + "next_deadline_date": { + "type": "date", + "format": "yyyy-MM-dd", + "copy_to": "next_deadline_date" } - }, - "_deposit":{ - "type":"object", - "properties":{ - "created_by":{ - "type":"integer", - "copy_to":"created_by" - }, - "status":{ - "type":"text", - "fields":{ - "keyword":{ - "type":"keyword" - } - }, - "copy_to":"status" - } - } - }, - "_collection": { - "type": "object", - "properties": { - "fullname": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" + } + }, + "_deposit":{ + "type":"object", + "properties":{ + "created_by":{ + "type":"integer", + "copy_to":"created_by" + }, + "status":{ + "type":"text", + "fields":{ + "keywords":{ + "type":"keyword" } + }, + "copy_to":"status" } - }, - "_experiment":{ - "type":"text" - }, - "_fetched_from": { - "type": "text", - "copy_to": "fetched_from" - }, - "_created":{ - "type":"date", - "copy_to":"created" - }, - "_updated":{ - "type":"date", - "copy_to":"updated" - }, - "created":{ - "type":"date" } + }, + "_collection": { + "type": "object", + "properties": { + "fullname": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "_experiment":{ + "type":"text" + }, + "_fetched_from": { + "type": "text", + "copy_to": "fetched_from" + }, + "_created":{ + "type":"date", + "copy_to":"created" + }, + "_updated":{ + "type":"date", + "copy_to":"updated" + }, + "created":{ + "type":"date" } } } diff --git a/cap/modules/fixtures/schemas/cms-questionnaire_v2.json b/cap/modules/fixtures/schemas/cms-questionnaire_v2.json index 29c7ce6a49..975d6880b6 100644 --- a/cap/modules/fixtures/schemas/cms-questionnaire_v2.json +++ b/cap/modules/fixtures/schemas/cms-questionnaire_v2.json @@ -2239,7 +2239,7 @@ "settings":{ "analysis":{ "analyzer":{ - "lowercase_whitespace_analyzer":{ + "default":{ "type":"custom", "tokenizer":"whitespace", "filter":[ @@ -2250,44 +2250,38 @@ } }, "mappings":{ - "cms-stats-questionnaire-v0.0.1":{ - "_all":{ - "enabled":true, - "analyzer":"lowercase_whitespace_analyzer" + "properties":{ + "next_deadline_date": { + "type": "date" }, - "properties":{ - "next_deadline_date": { - "type": "date" - }, - "analysis_context": { - "type": "object", - "properties": { - "next_deadline_date": { - "type": "date", - "format": "yyyy-MM-dd", - "copy_to": "next_deadline_date" - } + "analysis_context": { + "type": "object", + "properties": { + "next_deadline_date": { + "type": "date", + "format": "yyyy-MM-dd", + "copy_to": "next_deadline_date" } - }, - "ml_app_use": { - "properties": { - "development": { - "properties": { - "dev": { - "properties": { - "cross_validation": { - "properties": { - "comments_no": { - "properties": { - "training": { - "type": "string" - }, - "testing": { - "type": "string" - }, - "validation": { - "type": "string" - } + } + }, + "ml_app_use": { + "properties": { + "development": { + "properties": { + "dev": { + "properties": { + "cross_validation": { + "properties": { + "comments_no": { + "properties": { + "training": { + "type": "text" + }, + "testing": { + "type": "text" + }, + "validation": { + "type": "text" } } } @@ -2297,57 +2291,57 @@ } } } - }, - "_collection": { - "type": "object", - "properties": { - "fullname": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" - } - } - }, - "_deposit":{ - "type":"object", - "properties":{ - "created_by":{ - "type":"integer", - "copy_to":"created_by" + } + }, + "_collection": { + "type": "object", + "properties": { + "fullname": { + "type": "keyword" }, - "status":{ - "type":"text", - "fields":{ - "keyword":{ - "type":"keyword" - } - }, - "copy_to":"status" + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" } + } + }, + "_deposit":{ + "type":"object", + "properties":{ + "created_by":{ + "type":"integer", + "copy_to":"created_by" + }, + "status":{ + "type":"text", + "fields":{ + "keywords":{ + "type":"keyword" + } + }, + "copy_to":"status" } - }, - "_experiment":{ - "type":"text" - }, - "_fetched_from": { - "type": "text", - "copy_to": "fetched_from" - }, - "_created":{ - "type":"date", - "copy_to":"created" - }, - "_updated":{ - "type":"date", - "copy_to":"updated" - }, - "created":{ - "type":"date" } + }, + "_experiment":{ + "type":"text" + }, + "_fetched_from": { + "type": "text", + "copy_to": "fetched_from" + }, + "_created":{ + "type":"date", + "copy_to":"created" + }, + "_updated":{ + "type":"date", + "copy_to":"updated" + }, + "created":{ + "type":"date" } } } diff --git a/cap/modules/fixtures/schemas/cms.json b/cap/modules/fixtures/schemas/cms.json index c9863655d2..b44846bb20 100644 --- a/cap/modules/fixtures/schemas/cms.json +++ b/cap/modules/fixtures/schemas/cms.json @@ -607,429 +607,423 @@ } }, "mappings": { - "cms-analysis-v0.0.1": { - "_all": { - "enabled": true, - "analyzer": "lowercase_whitespace_analyzer" - }, - "properties": { - "_deposit": { - "type": "object", - "properties": { - "created_by": { - "type": "integer", - "copy_to": "created_by" - }, - "status": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - }, - "copy_to": "status" - } - } - }, - "_collection": { - "type": "object", - "properties": { - "fullname": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" + "properties": { + "_deposit": { + "type": "object", + "properties": { + "created_by": { + "type": "integer", + "copy_to": "created_by" + }, + "status": { + "type": "text", + "fields": { + "keywords": { + "type": "keyword" } + }, + "copy_to": "status" } - }, - "_experiment": { - "type": "text" - }, - "_fetched_from": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer", - "copy_to": "fetched_from" - }, - "_user_edited": { - "type": "boolean", - "copy_to": "user_edited" - }, - "_created": { - "type": "date", - "copy_to": "created" - }, - "created": { - "type": "date" - }, - "_updated": { - "type": "date", - "copy_to": "updated" - }, - "cadi_id": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer" - }, - "dataset": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer" - }, - "trigger": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer" - }, - "ananote": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer" - }, - "basic_info": { - "type": "object", - "properties": { - "cadi_id": { - "type": "keyword", - "copy_to": "cadi_id" + } + }, + "_collection": { + "type": "object", + "properties": { + "fullname": { + "type": "keyword" }, - "analysis_keywords": { - "type": "object", - "properties": { - "collision_system": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer", - "fields": { - "keyword": { - "type": "keyword" - } + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "_experiment": { + "type": "text" + }, + "_fetched_from": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer", + "copy_to": "fetched_from" + }, + "_user_edited": { + "type": "boolean", + "copy_to": "user_edited" + }, + "_created": { + "type": "date", + "copy_to": "created" + }, + "created": { + "type": "date" + }, + "_updated": { + "type": "date", + "copy_to": "updated" + }, + "cadi_id": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer" + }, + "dataset": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer" + }, + "trigger": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer" + }, + "ananote": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer" + }, + "basic_info": { + "type": "object", + "properties": { + "cadi_id": { + "type": "keyword", + "copy_to": "cadi_id" + }, + "analysis_keywords": { + "type": "object", + "properties": { + "collision_system": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer", + "fields": { + "keywords": { + "type": "keyword" } - }, - "accelerator_parameters": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer", - "fields": { - "keyword": { - "type": "keyword" - } + } + }, + "accelerator_parameters": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer", + "fields": { + "keywords": { + "type": "keyword" } - }, - "physics_theme": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer", - "fields": { - "keyword": { - "type": "keyword" - } + } + }, + "physics_theme": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer", + "fields": { + "keywords": { + "type": "keyword" } - }, - "final_states": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer", - "fields": { - "keyword": { - "type": "keyword" - } + } + }, + "final_states": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer", + "fields": { + "keywords": { + "type": "keyword" } - }, - "sm_analysis_characteristics": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer", - "fields": { - "keyword": { - "type": "keyword" - } + } + }, + "sm_analysis_characteristics": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer", + "fields": { + "keywords": { + "type": "keyword" } - }, - "interpretation": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer", - "fields": { - "keyword": { - "type": "keyword" - } + } + }, + "interpretation": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer", + "fields": { + "keywords": { + "type": "keyword" } - }, - "further_search_categorisation": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer", - "fields": { - "keyword": { - "type": "keyword" - } + } + }, + "further_search_categorisation": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer", + "fields": { + "keywords": { + "type": "keyword" } - }, - "further_search_categorisation_heavy_ion": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer", - "fields": { - "keyword": { - "type": "keyword" - } + } + }, + "further_search_categorisation_heavy_ion": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer", + "fields": { + "keywords": { + "type": "keyword" } } } - }, - "ana_notes": { - "type": "text", - "copy_to": "ananote", - "analyzer": "lowercase_whitespace_analyzer" - }, - "created": { - "type": "date" } + }, + "ana_notes": { + "type": "text", + "copy_to": "ananote", + "analyzer": "lowercase_whitespace_analyzer" + }, + "created": { + "type": "date" } - }, - "input_data": { - "type": "object", - "properties": { - "primary_datasets": { - "type": "object", - "properties": { - "path": { - "type": "text", - "copy_to": "dataset", - "analyzer": "lowercase_whitespace_analyzer" - }, - "year": { - "type": "keyword" - }, - "triggers": { - "type": "object", - "properties": { - "trigger": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer", - "copy_to": "trigger" - }, - "prescale": { - "type": "half_float" - } + } + }, + "input_data": { + "type": "object", + "properties": { + "primary_datasets": { + "type": "object", + "properties": { + "path": { + "type": "text", + "copy_to": "dataset", + "analyzer": "lowercase_whitespace_analyzer" + }, + "year": { + "type": "keyword" + }, + "triggers": { + "type": "object", + "properties": { + "trigger": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer", + "copy_to": "trigger" + }, + "prescale": { + "type": "half_float" } - }, - "prescale": { - "type": "half_float" } + }, + "prescale": { + "type": "half_float" } - }, - "mc_sig_datasets": { - "type": "text", - "copy_to": "dataset", - "analyzer": "lowercase_whitespace_analyzer" - }, - "mc_bg_datasets": { - "type": "text", - "copy_to": "dataset", - "analyzer": "lowercase_whitespace_analyzer" - }, - "json_files": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer" - }, - "custom_datasets": { - "type": "object", - "properties": { - "path": { - "type": "text", - "copy_to": "dataset", - "analyzer": "lowercase_whitespace_analyzer" - } + } + }, + "mc_sig_datasets": { + "type": "text", + "copy_to": "dataset", + "analyzer": "lowercase_whitespace_analyzer" + }, + "mc_bg_datasets": { + "type": "text", + "copy_to": "dataset", + "analyzer": "lowercase_whitespace_analyzer" + }, + "json_files": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer" + }, + "custom_datasets": { + "type": "object", + "properties": { + "path": { + "type": "text", + "copy_to": "dataset", + "analyzer": "lowercase_whitespace_analyzer" } } } - }, - "cadi_status": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer" - }, - "cadi_info": { - "type": "object", - "properties": { - "status": { - "type": "keyword", - "copy_to": "cadi_status" - }, - "contact_person": { - "type": "text", - "copy_to": "researcher" - }, - "publication_status": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer" - }, - "created": { - "type": "date", - "format": "dd/MM/YYYY||dd-MM-YYYY||YYYY-MM-dd" - }, - "updated": { - "type": "date", - "format": "dd/MM/YYYY||dd-MM-YYYY||YYYY-MM-dd" - } + } + }, + "cadi_status": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer" + }, + "cadi_info": { + "type": "object", + "properties": { + "status": { + "type": "keyword", + "copy_to": "cadi_status" + }, + "contact_person": { + "type": "text", + "copy_to": "researcher" + }, + "publication_status": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer" + }, + "created": { + "type": "date", + "format": "dd/MM/YYYY||dd-MM-YYYY||YYYY-MM-dd" + }, + "updated": { + "type": "date", + "format": "dd/MM/YYYY||dd-MM-YYYY||YYYY-MM-dd" } - }, - "additional_resources": { - "type": "object", - "properties": { - "documentations": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer", - "copy_to": "keyword" - }, - "keywords": { - "type": "text", - "analyzer": "lowercase_whitespace_analyzer", - "copy_to": "keyword" - } + } + }, + "additional_resources": { + "type": "object", + "properties": { + "documentations": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer", + "copy_to": "keyword" + }, + "keywords": { + "type": "text", + "analyzer": "lowercase_whitespace_analyzer", + "copy_to": "keyword" } - }, - "main_measurements": { - "type": "nested", - "properties": { - "signal_event_selection": { - "type": "object", - "properties": { - "physics_objects": { - "type": "nested", - "properties": { - "object": { - "type": "keyword", - "copy_to": "object" - }, - "object_type": { - "type": "keyword", - "copy_to": "object_type" - }, - "pt_cut": { - "type": "keyword", - "copy_to": "pt" - }, - "pt_cut_min": { - "type": "keyword", - "copy_to": "pt_min" - }, - "pt_cut_max": { - "type": "keyword", - "copy_to": "pt_max" - }, - "eta_cut": { - "type": "keyword", - "copy_to": "eta" - }, - "eta_cut_min": { - "type": "keyword", - "copy_to": "eta_min" - }, - "eta_cut_max": { - "type": "keyword", - "copy_to": "eta_max" - }, - "electron_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" - }, - "muon_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" - }, - "tau_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" - }, - "jet_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" - }, - "photon_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" - }, - "met_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" - }, - "track_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" - }, - "number": { - "type": "object", - "properties": { - "sign": { - "type": "keyword" - }, - "number": { - "type": "keyword" - } + } + }, + "main_measurements": { + "type": "nested", + "properties": { + "signal_event_selection": { + "type": "object", + "properties": { + "physics_objects": { + "type": "nested", + "properties": { + "object": { + "type": "keyword", + "copy_to": "object" + }, + "object_type": { + "type": "keyword", + "copy_to": "object_type" + }, + "pt_cut": { + "type": "keyword", + "copy_to": "pt" + }, + "pt_cut_min": { + "type": "keyword", + "copy_to": "pt_min" + }, + "pt_cut_max": { + "type": "keyword", + "copy_to": "pt_max" + }, + "eta_cut": { + "type": "keyword", + "copy_to": "eta" + }, + "eta_cut_min": { + "type": "keyword", + "copy_to": "eta_min" + }, + "eta_cut_max": { + "type": "keyword", + "copy_to": "eta_max" + }, + "electron_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" + }, + "muon_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" + }, + "tau_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" + }, + "jet_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" + }, + "photon_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" + }, + "met_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" + }, + "track_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.physics_objects.object_type" + }, + "number": { + "type": "object", + "properties": { + "sign": { + "type": "keyword" + }, + "number": { + "type": "keyword" } } } - }, - "veto": { - "type": "nested", - "properties": { - "object": { - "type": "keyword", - "copy_to": "veto" - }, - "pt_cut": { - "type": "keyword", - "copy_to": "veto_pt" - }, - "pt_cut_min": { - "type": "keyword", - "copy_to": "veto_pt_min" - }, - "pt_cut_max": { - "type": "keyword", - "copy_to": "veto_pt_max" - }, - "eta_cut": { - "type": "keyword", - "copy_to": "veto_eta" - }, - "eta_cut_min": { - "type": "keyword", - "copy_to": "veto_eta_min" - }, - "eta_cut_max": { - "type": "keyword", - "copy_to": "veto_eta_max" - }, - "electron_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.veto.object_type" - }, - "muon_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.veto.object_type" - }, - "tau_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.veto.object_type" - }, - "jet_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.veto.object_type" - }, - "photon_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.veto.object_type" - }, - "met_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.veto.object_type" - }, - "track_type": { - "type": "keyword", - "copy_to": "main_measurements.signal_event_selection.veto.object_type" - }, - "number": { - "type": "object", - "properties": { - "sign": { - "type": "keyword" - }, - "number": { - "type": "keyword" - } + } + }, + "veto": { + "type": "nested", + "properties": { + "object": { + "type": "keyword", + "copy_to": "veto" + }, + "pt_cut": { + "type": "keyword", + "copy_to": "veto_pt" + }, + "pt_cut_min": { + "type": "keyword", + "copy_to": "veto_pt_min" + }, + "pt_cut_max": { + "type": "keyword", + "copy_to": "veto_pt_max" + }, + "eta_cut": { + "type": "keyword", + "copy_to": "veto_eta" + }, + "eta_cut_min": { + "type": "keyword", + "copy_to": "veto_eta_min" + }, + "eta_cut_max": { + "type": "keyword", + "copy_to": "veto_eta_max" + }, + "electron_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.veto.object_type" + }, + "muon_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.veto.object_type" + }, + "tau_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.veto.object_type" + }, + "jet_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.veto.object_type" + }, + "photon_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.veto.object_type" + }, + "met_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.veto.object_type" + }, + "track_type": { + "type": "keyword", + "copy_to": "main_measurements.signal_event_selection.veto.object_type" + }, + "number": { + "type": "object", + "properties": { + "sign": { + "type": "keyword" + }, + "number": { + "type": "keyword" } } } diff --git a/cap/modules/fixtures/schemas/lhcb.json b/cap/modules/fixtures/schemas/lhcb.json index c599fed550..32f074c355 100644 --- a/cap/modules/fixtures/schemas/lhcb.json +++ b/cap/modules/fixtures/schemas/lhcb.json @@ -384,7 +384,7 @@ "settings": { "analysis": { "analyzer": { - "lowercase_whitespace_analyzer": { + "default": { "type": "custom", "tokenizer": "whitespace", "filter": ["lowercase"] @@ -393,152 +393,146 @@ } }, "mappings": { - "lhcb-v0.0.1": { - "_all": { - "enabled": true, - "analyzer": "lowercase_whitespace_analyzer" + "properties": { + "_deposit": { + "type": "object", + "properties": { + "created_by": { + "type": "integer", + "copy_to": "created_by" + }, + "status": { + "type": "text", + "fields": { + "keywords": { + "type": "keyword" + } + }, + "copy_to": "status" + } + } }, - "properties": { - "_deposit": { - "type": "object", - "properties": { - "created_by": { - "type": "integer", - "copy_to": "created_by" + "_collection": { + "type": "object", + "properties": { + "fullname": { + "type": "keyword" }, - "status": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - }, - "copy_to": "status" + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" } - } - }, - "_collection": { - "type": "object", - "properties": { - "fullname": { - "type": "keyword" - }, + } + }, + "_experiment": { + "type": "text" + }, + "_fetched_from": { + "type": "text", + "copy_to": "fetched_from" + }, + "_user_edited": { + "type": "boolean", + "copy_to": "user_edited" + }, + "_created": { + "type": "date", + "copy_to": "created" + }, + "created": { + "type": "date" + }, + "_updated": { + "type": "date", + "copy_to": "updated" + }, + "keywords": { + "type": "keyword" + }, + "basic_info": { + "type": "object", + "properties": { + "analysis_proponents": { + "type": "object", + "properties": { "name": { - "type": "keyword" + "type": "text", + "copy_to": "researcher" }, - "version": { - "type": "keyword" - } - } - }, - "_experiment": { - "type": "text" - }, - "_fetched_from": { - "type": "text", - "copy_to": "fetched_from" - }, - "_user_edited": { - "type": "boolean", - "copy_to": "user_edited" - }, - "_created": { - "type": "date", - "copy_to": "created" - }, - "created": { - "type": "date" - }, - "_updated": { - "type": "date", - "copy_to": "updated" - }, - "keyword": { - "type": "keyword" - }, - "basic_info": { - "type": "object", - "properties": { - "analysis_proponents": { - "type": "object", - "properties": { - "name": { - "type": "text", - "copy_to": "researcher" - }, - "orcid": { - "type": "keyword", - "copy_to": "researcher" - } + "orcid": { + "type": "keyword", + "copy_to": "researcher" } - }, - "reviewers": { - "type": "object", - "properties": { - "name": { - "type": "text", - "copy_to": "reviewer" - }, - "orcid": { - "type": "keyword", - "copy_to": "reviewer" - } + } + }, + "reviewers": { + "type": "object", + "properties": { + "name": { + "type": "text", + "copy_to": "reviewer" + }, + "orcid": { + "type": "keyword", + "copy_to": "reviewer" } - }, - "review_egroup": { - "type": "keyword", - "copy_to": "reviewer" - }, - "institutes": { - "type": "keyword", - "copy_to": "institute" - }, - "keywords": { - "type": "keyword", - "copy_to": "keyword" } + }, + "review_egroup": { + "type": "keyword", + "copy_to": "reviewer" + }, + "institutes": { + "type": "keyword", + "copy_to": "institute" + }, + "keywords": { + "type": "keyword", + "copy_to": "keyword" } - }, - "ntuple_userdst_production": { - "type": "object", - "properties": { - "input_dataset": { - "type": "keyword", - "copy_to": "dataset" - } + } + }, + "ntuple_userdst_production": { + "type": "object", + "properties": { + "input_dataset": { + "type": "keyword", + "copy_to": "dataset" } - }, - "stripping_turbo_selection": { - "type": "object", - "properties": { - "bookkeping_locations": { - "type": "keyword", - "copy_to": "bookkeeping" - }, - "stripping_turbo_line": { - "type": "keyword", - "copy_to": "stripping_line" - } + } + }, + "stripping_turbo_selection": { + "type": "object", + "properties": { + "bookkeping_locations": { + "type": "keyword", + "copy_to": "bookkeeping" + }, + "stripping_turbo_line": { + "type": "keyword", + "copy_to": "stripping_line" } - }, - "additional_resources": { - "type": "object", - "properties": { - "publications": { - "type": "object", - "properties": { - "analysis_number": { - "type": "keyword", - "copy_to": "ananote" - }, - "arxiv_id": { - "type": "keyword", - "copy_to": "arxiv" - }, - "twiki": { - "type": "keyword", - "copy_to": "twiki" - } + } + }, + "additional_resources": { + "type": "object", + "properties": { + "publications": { + "type": "object", + "properties": { + "analysis_number": { + "type": "keyword", + "copy_to": "ananote" + }, + "arxiv_id": { + "type": "keyword", + "copy_to": "arxiv" + }, + "twiki": { + "type": "keyword", + "copy_to": "twiki" } } } diff --git a/cap/modules/mail/cli.py b/cap/modules/mail/cli.py index d9f06ef659..0e95689a44 100644 --- a/cap/modules/mail/cli.py +++ b/cap/modules/mail/cli.py @@ -25,7 +25,8 @@ import click from flask import current_app -from flask_cli import with_appcontext +from flask.cli import with_appcontext + from flask_mail import Message from cap.cli import MutuallyExclusiveOption diff --git a/cap/modules/records/search.py b/cap/modules/records/search.py index 736c416ec8..2702677406 100644 --- a/cap/modules/records/search.py +++ b/cap/modules/records/search.py @@ -1,6 +1,6 @@ """Configuration for record search.""" -from elasticsearch_dsl import Q +from opensearch_dsl import Q from flask import g from flask_login import current_user from flask_principal import RoleNeed @@ -58,7 +58,6 @@ class Meta: """Configuration for records search.""" index = 'records' - doc_types = None fields = ('*',) facets = {} default_filter = DefaultFilter(records_filter) diff --git a/cap/modules/records/serializers/json.py b/cap/modules/records/serializers/json.py index c4ce367793..568a9ad7c0 100644 --- a/cap/modules/records/serializers/json.py +++ b/cap/modules/records/serializers/json.py @@ -62,7 +62,7 @@ def serialize_search( """Serialize a search result. :param pid_fetcher: Persistent identifier fetcher. - :param search_result: Elasticsearch search result. + :param search_result: OpenSearch search result. :param links: Dictionary of links to add to response. """ links = ( diff --git a/cap/modules/schemas/cli.py b/cap/modules/schemas/cli.py index a722c4a31a..15d3391205 100644 --- a/cap/modules/schemas/cli.py +++ b/cap/modules/schemas/cli.py @@ -33,7 +33,8 @@ import jsonpatch import requests from flask import current_app -from flask_cli import with_appcontext +from flask.cli import with_appcontext +from invenio_db import db from invenio_accounts.models import Role, User from invenio_db import db from invenio_jsonschemas.errors import JSONSchemaNotFound @@ -368,9 +369,23 @@ def validate( # get all the records for this specific schema/type combination records = current_search_client.search( - search_path, - q=f'_deposit.status: {status} AND ' - f'$schema: "{schema_name_to_url(schema.name, schema.version)}"', + index=f"{search_path}-{schema.name}-v{schema.version}", + body={ + "query": { + "bool": { + "must": [ + {"match": {"_deposit.status": status}}, + { + "match": { + "$schema": schema_name_to_url( + schema.name, schema.version + ) + } + }, + ] + } + } + }, size=5000, )['hits']['hits'] pids = [rec['_id'] for rec in records] diff --git a/cap/modules/schemas/imp.py b/cap/modules/schemas/imp.py index 11ae3513a0..4a622b5588 100644 --- a/cap/modules/schemas/imp.py +++ b/cap/modules/schemas/imp.py @@ -45,17 +45,17 @@ def _filter_only_latest(schemas_list): return [next(g) for k, g in groupby(schemas_list, lambda s: s.name)] -@current_cache.memoize() def get_mappings(): - """Implementation for mappings getter for invenio_search module.""" - mappings = {} - schemas = Schema.query.filter_by(is_indexed=True).all() - - for schema in schemas: - mappings[schema.deposit_index] = {} - mappings[schema.record_index] = {} - - return mappings + @current_cache.memoize() + def get_mappings_inner(): + """Implementation for mappings getter for invenio_search module.""" + mappings = {} + schemas = Schema.query.filter_by(is_indexed=True).all() + for schema in schemas: + mappings[schema.deposit_index] = {} + mappings[schema.record_index] = {} + return mappings + return get_mappings_inner() def _filter_by_deposit_read_access(schemas_list): @@ -99,48 +99,58 @@ def _filter_by_record_create_access(schemas_list): ] -@current_cache.memoize() def get_cached_indexed_schemas_for_user_create(latest=True, user_id=None): - """Return all indexed schemas current user has read access to.""" - schemas = get_indexed_schemas(latest=latest) - schemas = _filter_by_deposit_create_access(schemas) - return schemas + @current_cache.memoize() + def get_cached_indexed_schemas_for_user_create_inner(latest, user_id): + schemas = get_indexed_schemas(latest=latest) + schemas = _filter_by_deposit_create_access(schemas) + return schemas + + return get_cached_indexed_schemas_for_user_create_inner(latest, user_id) -@current_cache.memoize() def get_cached_indexed_schemas_for_user_admin(latest=True, user_id=None): - """Return all indexed schemas current user has read access to.""" - schemas = get_indexed_schemas(latest=latest) - schemas = _filter_by_admin_access(schemas) - return schemas + @current_cache.memoize() + def get_cached_indexed_schemas_for_user_admin_inner(latest, user_id): + """Return all indexed schemas current user has read access to.""" + schemas = get_indexed_schemas(latest=latest) + schemas = _filter_by_admin_access(schemas) + return schemas + + return get_cached_indexed_schemas_for_user_admin_inner(latest, user_id) -@current_cache.memoize() def get_cached_indexed_schemas_for_user_read(latest=True, user_id=None): - """Return all indexed schemas current user has read access to.""" - schemas = get_indexed_schemas(latest=latest) - schemas = _filter_by_deposit_read_access(schemas) + @current_cache.memoize() + def get_cached_indexed_schemas_for_user_read_inner(): + """Return all indexed schemas current user has read access to.""" + schemas = get_indexed_schemas(latest=latest) + schemas = _filter_by_deposit_read_access(schemas) + return schemas - return schemas + return get_cached_indexed_schemas_for_user_read_inner() -@current_cache.memoize() -def get_cached_indexed_record_schemas_for_user_create( - latest=True, user_id=None -): - """Return all indexed schemas current user has read access to.""" - schemas = get_indexed_schemas(latest=latest) - schemas = _filter_by_record_create_access(schemas) - return schemas +def get_cached_indexed_record_schemas_for_user_create(latest=True, user_id=None): + @current_cache.memoize() + def get_cached_indexed_record_schemas_for_user_create_inner(latest, user_id): + """Return all indexed schemas current user has read access to.""" + schemas = get_indexed_schemas(latest=latest) + schemas = _filter_by_record_create_access(schemas) + return schemas + return get_cached_indexed_record_schemas_for_user_create_inner(latest, user_id) -@current_cache.memoize() -def get_cached_indexed_record_schemas_for_user_read(latest=True, user_id=None): - """Return all indexed schemas current user has read access to.""" - schemas = get_indexed_schemas(latest=latest) - schemas = _filter_by_record_read_access(schemas) - return schemas +def get_cached_indexed_record_schemas_for_user_read(latest=True, user_id=None): + @current_cache.memoize() + def get_cached_indexed_record_schemas_for_user_read_inner(latest, user_id): + """Return all indexed schemas current user has read access to.""" + schemas = get_indexed_schemas(latest=latest) + schemas = _filter_by_record_read_access(schemas) + return schemas + + return get_cached_indexed_record_schemas_for_user_read_inner(latest, user_id) def get_indexed_schemas(latest=True): diff --git a/cap/modules/schemas/models.py b/cap/modules/schemas/models.py index 6d3c7e95a2..c4e88b921a 100644 --- a/cap/modules/schemas/models.py +++ b/cap/modules/schemas/models.py @@ -443,7 +443,7 @@ def name_to_es_name(name): def create_index(index_name, mapping_body, aliases): - """Create index in elasticsearch, add under given aliases.""" + """Create index in opensearch, add under given aliases.""" if not es.indices.exists(index_name): current_search.mappings[index_name] = {} # invenio search needs it diff --git a/cap/modules/schemas/utils.py b/cap/modules/schemas/utils.py index c5635e478e..2d378e8c63 100644 --- a/cap/modules/schemas/utils.py +++ b/cap/modules/schemas/utils.py @@ -125,9 +125,8 @@ def actions_from_type(_type, perms): def get_default_mapping(name, version): - mapping_name = f"{name}-v{version}" default_mapping = {"mappings": {}} - collectiion_mapping = { + collection_mapping = { "properties": { "_collection": { "type": "object", @@ -139,7 +138,7 @@ def get_default_mapping(name, version): } } } - default_mapping["mappings"][mapping_name] = collectiion_mapping + default_mapping["mappings"] = collection_mapping return default_mapping diff --git a/cap/modules/search/facets.py b/cap/modules/search/facets.py index 01c4105287..f48417867e 100644 --- a/cap/modules/search/facets.py +++ b/cap/modules/search/facets.py @@ -28,7 +28,7 @@ from flask import current_app from werkzeug.datastructures import MultiDict -from elasticsearch_dsl import Q +from opensearch_dsl import Q from invenio_records_rest.facets import (_create_filter_dsl, _post_filter, _query_filter) diff --git a/cap/modules/search/query.py b/cap/modules/search/query.py index 7bb2314ca9..e77ddb7843 100644 --- a/cap/modules/search/query.py +++ b/cap/modules/search/query.py @@ -25,7 +25,7 @@ from __future__ import absolute_import, print_function -from elasticsearch_dsl.query import Q +from opensearch_dsl.query import Q from flask import current_app, request from flask_login import current_user from invenio_records_rest.errors import InvalidQueryRESTError @@ -56,7 +56,7 @@ def cap_search_factory(self, search, query_parser=None): :returns: Tuple with search instance and URL arguments. """ def _default_parser(qstr=None, **kwargs): - """Use of the Q() from elasticsearch_dsl.""" + """Use of the Q() from opensearch_dsl.""" def _escape_qstr(qstr): return ''.join((ESCAPE_CHAR_MAP.get(char, char) for char in qstr)) diff --git a/cap/modules/search/sorter.py b/cap/modules/search/sorter.py index 3888e3b6a8..a494160f20 100644 --- a/cap/modules/search/sorter.py +++ b/cap/modules/search/sorter.py @@ -48,7 +48,9 @@ def inner(asc): "type": "number", "script": { "lang": "painless", - "source": f"params.get(doc['{field_name}'].value + ' - ' + doc['{sub_field_name}'].value)", # noqa + "source": "doc.containsKey('{0}') && doc.containsKey('{1}') && doc['{0}'].size() != 0 && doc['{1}'].size() != 0 ? params.get(doc['{0}'].value + ' - ' + doc['{1}'].value) : 0".format( # noqa + field_name, sub_field_name + ), "params": sort_params, }, "order": "asc" if asc else "desc", diff --git a/cap/modules/services/views/cds.py b/cap/modules/services/views/cds.py index 92796c102a..3ddc543463 100644 --- a/cap/modules/services/views/cds.py +++ b/cap/modules/services/views/cds.py @@ -26,7 +26,7 @@ """CAP CDS service views.""" import requests -from flask import jsonify, abort +from flask import jsonify, abort, make_response from . import blueprint from ..serializers.cds import CDSRecordSchema @@ -58,7 +58,12 @@ def check_if_deleted(resp): def check_if_authorized(resp): """Checks the CDS authorization.""" if resp.headers.get('Expires'): - abort(401, 'You are unauthorized to view this CDS record.') + abort( + make_response( + jsonify({'message': 'You are unauthorized to view this CDS record.'}), + 401, + ) + ) def check_if_404(resp): diff --git a/cap/modules/services/views/status_checks.py b/cap/modules/services/views/status_checks.py index 6e28a4114e..6e593fc177 100644 --- a/cap/modules/services/views/status_checks.py +++ b/cap/modules/services/views/status_checks.py @@ -128,7 +128,7 @@ 'args': {'service': 'search'}, 'should_return': {'message': 'OK'}, 'service': 'cap_es', - 'log': 'Checking ElasticSearch API...', + 'log': 'Checking OpenSearch API...', 'category': 'Internal Services', }, { diff --git a/cap/views.py b/cap/views.py index 5696c4cb81..784c367732 100644 --- a/cap/views.py +++ b/cap/views.py @@ -17,9 +17,9 @@ import json from os.path import join -from elasticsearch import ConnectionError -from elasticsearch.exceptions import NotFoundError -from flask import Blueprint, abort, jsonify +from opensearchpy import ConnectionError +from opensearchpy.exceptions import NotFoundError +from flask import Blueprint, jsonify, abort from invenio_files_rest.models import Location from invenio_jsonschemas.errors import JSONSchemaNotFound from invenio_search import current_search diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 0e286f0459..c76b69b60a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -47,9 +47,10 @@ services: - "FLASK_DEBUG=True" - "DEBUG_MODE=True" - "DEV_HOST=nginx" + - "FLASK_APP=cap/wsgi.py" links: - cache - - es + - search - mq - db stdin_open: true @@ -69,7 +70,7 @@ services: - ./cap:/opt/cap/src/cap links: - cache - - es + - search - mq - db stdin_open: true @@ -80,10 +81,10 @@ services: file: docker-services.yml service: app restart: "always" - command: "celery worker -A cap.celery --loglevel=INFO" + command: "celery --app=cap.celery worker --loglevel=INFO" links: - cache - - es + - search - mq - db # Monitoring @@ -106,10 +107,10 @@ services: extends: file: docker-services.yml service: mq - es: + search: extends: file: docker-services.yml - service: es + service: search # statping: # extends: # file: docker-services.yml diff --git a/docker-compose.e2e-override-server.yml b/docker-compose.e2e-override-server.yml index 2f765da486..814aced17b 100644 --- a/docker-compose.e2e-override-server.yml +++ b/docker-compose.e2e-override-server.yml @@ -16,7 +16,7 @@ services: - ./cap:/opt/invenio/src/cap links: - cache - - es + - search - mq - db diff --git a/docker-compose.e2e-override-ui.yml b/docker-compose.e2e-override-ui.yml index 3cf7d749fa..f12a05a127 100644 --- a/docker-compose.e2e-override-ui.yml +++ b/docker-compose.e2e-override-ui.yml @@ -59,7 +59,7 @@ services: - "DEV_HOST=nginx" links: - cache - - es + - search - mq - db stdin_open: true @@ -91,7 +91,7 @@ services: - ./cap:/opt/cap/src/cap links: - cache - - es + - search - mq - db stdin_open: true @@ -102,10 +102,10 @@ services: file: docker-services.yml service: app restart: "always" - command: "celery worker -A cap.celery --loglevel=INFO" + command: "celery --app=cap.celery worker --loglevel=INFO" links: - cache - - es + - search - mq - db # Base services @@ -121,10 +121,10 @@ services: extends: file: docker-services.yml service: mq - es: + search: extends: file: docker-services.yml - service: es + service: search volumes: static_data: uploaded_data: diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml index 364c7501d2..2c7e146c5d 100644 --- a/docker-compose.e2e.yml +++ b/docker-compose.e2e.yml @@ -63,9 +63,10 @@ services: - "FLASK_DEBUG=True" - "DEBUG_MODE=True" - "DEV_HOST=nginx" + - "FLASK_APP=cap/wsgi.py" links: - cache - - es + - search - mq - db stdin_open: true @@ -102,7 +103,7 @@ services: - test_data:/test_data links: - cache - - es + - search - mq - db stdin_open: true @@ -113,10 +114,10 @@ services: file: docker-services.yml service: app restart: "always" - command: "celery worker -A cap.celery --loglevel=INFO" + command: "celery --app=cap.celery worker --loglevel=INFO" links: - cache - - es + - search - mq - db environment: @@ -143,10 +144,10 @@ services: extends: file: docker-services.yml service: mq - es: + search: extends: file: docker-services.yml - service: es + service: search volumes: static_data: uploaded_data: diff --git a/docker-compose.full.yml b/docker-compose.full.yml index 305941ab90..680e5f7d55 100644 --- a/docker-compose.full.yml +++ b/docker-compose.full.yml @@ -14,6 +14,12 @@ services: service: lb links: - frontend + cypress: + image: "cypress/included:6.3.0" + # entrypoint: cypress run --headless --browser chrome + environment: + - CYPRESS_BASE_URL=https://nginx + - CYPRESS_baseUrl=https://nginx # Frontend frontend: extends: @@ -38,7 +44,7 @@ services: - ./var/data:/var/data links: - cache - - es + - search - mq - db # API Files Application @@ -55,7 +61,7 @@ services: - ./var/data:/var/data links: - cache - - es + - search - mq - db # Worker @@ -64,11 +70,11 @@ services: file: docker-services.yml service: app restart: "always" - command: "celery worker -A cap.celery --loglevel=INFO" + command: "celery --app=cap.celery worker --loglevel=INFO" image: cap-worker links: - cache - - es + - search - mq - db # Monitoring @@ -91,9 +97,9 @@ services: extends: file: docker-services.yml service: mq - es: + search: extends: file: docker-services.yml - service: es + service: search volumes: static_data: diff --git a/docker-compose.yml b/docker-compose.yml index 82387e7e5d..69635901ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,14 +19,14 @@ services: extends: file: docker-services.yml service: mq - es: + search: extends: file: docker-services.yml - service: es - kibana: + service: search + search-dashboards: extends: file: docker-services.yml - service: kibana + service: search-dashboards # redis-commander: # extends: diff --git a/docker-services.yml b/docker-services.yml index c0986fd05b..58aeb38f55 100644 --- a/docker-services.yml +++ b/docker-services.yml @@ -27,7 +27,7 @@ services: - "INVENIO_CACHE_TYPE=redis" - "INVENIO_CELERY_BROKER_URL=amqp://guest:guest@mq:5672/" - "INVENIO_CELERY_RESULT_BACKEND=redis://cache:6379/2" - - "INVENIO_SEARCH_ELASTIC_HOSTS=['es:9200']" + - "INVENIO_SEARCH_HOSTS=['search:9200']" - "INVENIO_SECRET_KEY=CHANGE_ME" - "INVENIO_SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://cap:cap@db/cap" - "INVENIO_WSGI_PROXIES=2" @@ -78,26 +78,34 @@ services: ports: - "15672:15672" - "5672:5672" - es: - image: elasticsearch:5.6.4 - restart: "always" + search: + image: opensearchproject/opensearch:2.2.0 environment: - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - node.name=search + - discovery.seed_hosts=search + - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM + - discovery.type=single-node + - "DISABLE_SECURITY_PLUGIN=true" ulimits: memlock: soft: -1 hard: -1 - mem_limit: 1g + nofile: + soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems + hard: 65536 ports: - "9200:9200" - - "9300:9300" - kibana: - image: kibana:5.6.4 + - "9600:9600" + search-dashboards: + image: opensearchproject/opensearch-dashboards:2.2.0 + ports: + - 5601:5601 + expose: + - "5601" environment: - - "ELASTICSEARCH_URL=http://es:9200" - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - ports: ["5601:5601"] + - 'OPENSEARCH_HOSTS=["http://search:9200"]' + - "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true" flower: image: mher/flower ports: diff --git a/docker/base/CERN_Root_Certification_Authority_2.pem b/docker/base/CERN_Root_Certification_Authority_2.pem new file mode 100644 index 0000000000..646831be5f --- /dev/null +++ b/docker/base/CERN_Root_Certification_Authority_2.pem @@ -0,0 +1,38 @@ +-----BEGIN CERTIFICATE----- +MIIGqTCCBJGgAwIBAgIQAojDcLlcbrhBX0qrEka4mzANBgkqhkiG9w0BAQ0FADBK +MQswCQYDVQQGEwJjaDENMAsGA1UEChMEQ0VSTjEsMCoGA1UEAxMjQ0VSTiBSb290 +IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IDIwHhcNMTMwMzE5MTI1NTM2WhcNMzMw +MzE5MTMwNTM0WjBKMQswCQYDVQQGEwJjaDENMAsGA1UEChMEQ0VSTjEsMCoGA1UE +AxMjQ0VSTiBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IDIwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQDxqYPFW2qVVi3Rw1NKlEf7x70xF+6a8uE/ +Tu4ZVQF/K2RXI95QLkYfKItZvy9Az3ib/VlUho5f8fBaqy4n70uwC7+qd3Aq1/xQ +ysykPCbBBAsOSQQpTlhrMD2V5Ya9zrirphOhutddiqV96zBCyMM+Gz5uYv9u+cm4 +tg1EOmAMGh2UNxfTFNVmXKkk7eFTSC1+zgb28H6nd3xzV27sn9bfOfGh//ZPy5gm +Qx0Oh/tc6WMreWzRZBQm5SJiK0QOzPv09p5WmdY2WxZoqNTFBDACQO7ysFOktc74 +fPVFX/lmt4jFNSZRIOvvaACI/qlEaAJTR4FHIY9uSMsV8DrtzhI1Ucyv3kqlQpbF +jDouq44IryA/np4s/124bW+x8+n/v+at/AxPjvHBLiGhB+J38Z6KcJogoDnGzIXR +S+YUr/vGz34jOmkRuDN5STuuAXzyCKFXaoAm0AwjTziIv3E0jxC1taw6FpKevnd1 +CLsTLAEUiEjzStFkDhd/Hpipc57zmMFY8VYet2wVqSFjnt2REWOVbZlbCiMHmSeD +u5EuZLiU8xlkiaCfn4A5XZ6X0qprbgDviGJtwxzNvTg7Hn0ziW5/ELryfQXCwZJ+ +FVne8Zu8sbgy/sDkX+pyFuyB4XgiM0eMNkoexIXJaRdlMWDIL5ysiIXQKjhynAv5 +KLHbRjciVwIDAQABo4IBiTCCAYUwCwYDVR0PBAQDAgGGMA8GA1UdEwEB/wQFMAMB +Af8wHQYDVR0OBBYEFPp7+96bDaPyUrds7VsPC6KmpvgEMBAGCSsGAQQBgjcVAQQD +AgEAMIIBMgYDVR0gBIIBKTCCASUwggEhBgorBgEEAWAKBAEBMIIBETCBwgYIKwYB +BQUHAgIwgbUegbIAQwBFAFIATgAgAFIAbwBvAHQAIABDAGUAcgB0AGkAZgBpAGMA +YQB0AGkAbwBuACAAQQB1AHQAaABvAHIAaQB0AHkAIAAyACAAQwBlAHIAdABpAGYA +aQBjAGEAdABlACAAUABvAGwAaQBjAHkAIABhAG4AZAAgAEMAZQByAHQAaQBmAGkA +YwBhAHQAZQAgAFAAcgBhAGMAdABpAGMAZQAgAFMAdABhAHQAZQBtAGUAbgB0MEoG +CCsGAQUFBwIBFj5odHRwOi8vY2FmaWxlcy5jZXJuLmNoL2NhZmlsZXMvY3AtY3Bz +L2Nlcm4tcm9vdC1jYTItY3AtY3BzLnBkZjANBgkqhkiG9w0BAQ0FAAOCAgEAo0Px +l4CZ6C6bDH+b6jV5uUO0NIHtvLuVgQLMdKVHtQ2UaxeIrWwD+Kz1FyJCHTRXrCvE +OFOca9SEYK2XrbqZGvRKdDRsq+XYts6aCampXj5ahh6r4oQJ8U7aLVfziKTK13Gy +dYFoAUeUrlNklICt3v2wWBaa1tg2oSlU2g4iCg9kYpRnIW3VKSrVsdVk2lUa4EXs +nTEJ30OS7rqX3SdqZp8G+awtBEReh2XPhRgJ6w3xiScP/UdWYUam2LflCGX3RibB +/DZhgGHRRoE4/D0kQMP2XTz6cClbNklECTlp0qZIbiaf350HbcDEFzYRSSIi0emv +kRGcMgsi8yTTU87q8Cr4hETxAF3ZbSVNC0ZaTZ8RBbM9BXguhYzKkVBgG/cMpUjs +B6tY2HMZbAZ3TKQRb/bRyUigM9DniKWeXkeL/0Nsno+XbcpAqLjtVIRwCg6jTLUi +1NRsl3BP6C824dVaoI8Ry7m+o6O+mtocw4BMhHfTcoWCO8CWjT0ME67JzaAYa5eM ++OqoWtgbgweBlfO0/3GMnVGMAmI4FlhH2oWKWQgWdgr0Wgh9K05VcxSpJ87/zjhb +MQn/bEojWmp6eUppPaqNFcELvud41qoe6hLsOYQVUQ1sHi7n6ouhg4BAbwS2iyD2 +uiA6FHTCeLreFGUzs5osPKiz3GE5D6V9she9xIQ= +-----END CERTIFICATE----- diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile new file mode 100644 index 0000000000..2ef75f6305 --- /dev/null +++ b/docker/base/Dockerfile @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2023 CERN. +# +# + +FROM python:3.10.10 + +# Certficates configuartion +ENV PYTHONBUFFERED=0 \ + SSL_CERT_FILE="/etc/ssl/certs/ca-certificates.crt" \ + REQUESTS_CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt" \ + PATH="/root/.local/bin:${PATH}" \ + POETRY_VIRTUALENVS_CREATE=false +COPY CERN_Root_Certification_Authority_2.pem /usr/local/share/ca-certificates/CERN_Root_Certification_Authority_2.crt + +# Install system dependencies +RUN update-ca-certificates && pip config set global.cert "${REQUESTS_CA_BUNDLE}" +RUN curl -s -L http://cern.ch/linux/docs/krb5.conf -o /etc/krb5.conf +RUN curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg + +RUN apt-get update && apt-get install --no-install-recommends -y \ + gcc libffi-dev locales libxslt-dev libxml2-dev libssl-dev build-essential python3-dev libldap2-dev libsasl2-dev ldap-utils krb5-user git redis \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Set the locale +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index f1d25917f5..78f46e2db1 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -1,21 +1,19 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2018 CERN. +# Copyright (C) 2023 CERN. # # CERN Analysis Preservation is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. FROM nginx:1.24 - -RUN apt-get update -RUN apt-get install -y curl bash - +RUN apt-get update && apt-get install --no-install-recommends -y \ + curl dirmngr bash apt-transport-https lsb-release git gcc g++ make python3 ca-certificates build-essential \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - RUN apt-get install -y nodejs - -RUN apt-get update && apt-get -y install git python g++ make RUN npm install --global yarn COPY ./docker/nginx/nginx.conf /etc/nginx/nginx.conf @@ -59,7 +57,6 @@ WORKDIR $WORKING_DIR/ui/ RUN yarn config set cache ~/.my-yarn-cache-dir RUN yarn install -RUN yarn upgrade RUN yarn build RUN cp -rfp ./dist/* $NGINX_HTML_DIR @@ -95,4 +92,4 @@ RUN yarn RUN yarn build RUN mkdir -p $NGINX_HTML_DIR/docs/client -RUN cp -rfp ./_book/* $NGINX_HTML_DIR/docs/client \ No newline at end of file +RUN cp -rfp ./_book/* $NGINX_HTML_DIR/docs/client diff --git a/requirements-local-forks.txt b/requirements-local-forks.txt index 0454d7a56b..5142fe24b8 100644 --- a/requirements-local-forks.txt +++ b/requirements-local-forks.txt @@ -1,17 +1,13 @@ --e git+https://github.com/cernanalysispreservation/invenio-deposit.git#egg=invenio-deposit -# -e git+https://github.com/reanahub/reana-client.git@master#egg=reana-client -# -e git+https://github.com/reanahub/reana-commons.git@master#egg=reana-commons -cryptography<=3.3.2 -reana-client==0.6.0 -reana-commons==0.6.0 --e git+https://github.com/cernanalysispreservation/invenio-oauthclient.git@master#egg=invenio-oauthclient --e git+https://github.com/cernanalysispreservation/invenio-jsonschemas.git@master#egg=invenio-jsonschemas --e git+https://github.com/cernanalysispreservation/invenio-search.git@master#egg=invenio-search --e git+https://github.com/cernanalysispreservation/invenio-files-rest.git@fix-ensure-stream#egg=invenio-files-rest --e git+https://github.com/cernanalysispreservation/invenio-userprofiles.git#egg=invenio-userprofiles --e git+https://github.com/cernanalysispreservation/invenio-logging.git@v1.2.0-without-invenio-base-des#egg=invenio-logging[sentry,sentry-sdk] --e git+https://github.com/cernanalysispreservation/flask-celeryext.git@master#egg=flask-celeryext -# Install invenio-records from last commit before 1.1.0 version release --e git+https://github.com/inveniosoftware/invenio-records.git@a5d4efb1e1466aa571aeef013d699dca1452f7cd#egg=invenio-records --e git+https://github.com/celery/celery.git@6ccdc7b9f8e02d21275e923dccc7ccb9185e6153#egg=celery - +-e git+https://github.com/cernanalysispreservation/invenio-celery.git@mock/v1.1.1#egg=invenio-celery +-e git+https://github.com/inveniosoftware/invenio-logging@94bc56117593eae62ba975d576e8c7b991311c0d#egg=invenio-logging[sentry,sentry-sdk] +-e git+https://github.com/cernanalysispreservation/invenio-oauthclient.git@v1.1.3#egg=invenio-oauthclient +-e git+https://github.com/cernanalysispreservation/invenio-userprofiles.git@v1.0.1#egg=invenio-userprofiles +-e git+https://github.com/cernanalysispreservation/invenio-indexer.git@os-2-new#egg=invenio-indexer +-e git+https://github.com/cernanalysispreservation/invenio-jsonschemas.git@v1.0.1#egg=invenio-jsonschemas +-e git+https://github.com/cernanalysispreservation/invenio-oaiserver.git@os-2-new#egg=invenio-oaiserver +-e git+https://github.com/parths007/invenio-records-rest.git@os-2-new#egg=invenio-records-rest +-e git+https://github.com/parths007/invenio-search.git@v1.2.3#egg=invenio-search[opensearch2] +-e git+https://github.com/cernanalysispreservation/invenio-files-rest.git@mock/v1.0.5#egg=invenio-files-rest +-e git+https://github.com/cernanalysispreservation/invenio-records-files.git@new#egg=invenio-records-files +-e git+https://github.com/cernanalysispreservation/invenio-deposit.git@os-2#egg=invenio-deposit +-e git+https://github.com/cernanalysispreservation/invenio-query-parser.git@os#egg=invenio-query-parser diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 1d3018188b..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,183 +0,0 @@ - -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file requirements.txt setup.py -# -alembic==0.9.9 # via flask-alembic -angular-gettext-babel==0.3 # via invenio-search-ui -appnope==0.1.0 # via ipython -argparse==1.4.0 # via uwsgitop -arrow==0.12.1 # via invenio-formatter, invenio-records-rest, jinja2-time -asn1crypto==1.0.0 -Authlib==0.15.1 -babel==2.6.0 # via angular-gettext-babel, flask-babelex, invenio-assets -backports.functools-lru-cache==1.5 # via arrow -backports.shutil-get-terminal-size==1.0.0 # via ipython -binaryornot==0.4.4 # via cookiecutter -bleach==2.1.3 # via invenio-records-rest -blinker==1.4 # via flask-debugtoolbar, flask-mail, flask-principal, invenio-base, invenio-oauthclient, invenio-records -celery==4.4.0 -certifi==2018.4.16 # via requests, urllib3 -cffi==1.11.5 # via cryptography -chardet==3.0.4 # via binaryornot, requests -click==6.7 # via cookiecutter, dojson, flask, flask-cli, flask-shell-ipython -cookiecutter==1.6.0 # via invenio-base -coolname==1.1.0 -cryptography==3.3.2 # via invenio-accounts, pyopenssl, sqlalchemy-utils, urllib3 -decorator==4.3.0 # via ipython, traitlets, validators -dojson==1.4.0 # via invenio-oaiserver -elasticsearch-dsl==5.4.0 # via invenio-search -elasticsearch==5.5.2 # via elasticsearch-dsl, invenio-search -flask-admin==1.5.1 # via invenio-accounts, invenio-admin -flask-alembic==2.0.1 # via invenio-db -flask-assets==0.12 # via invenio-assets -flask-babelex==0.9.3 # via flask-security, invenio-access, invenio-accounts, invenio-formatter, invenio-i18n, invenio-oaiserver, invenio-oauth2server, invenio-oauthclient, invenio-pidstore, invenio-records-rest, invenio-records-ui, invenio-search-ui, invenio-theme, invenio-userprofiles -flask-breadcrumbs==0.4.0 # via invenio-accounts, invenio-oauth2server, invenio-oauthclient, invenio-theme, invenio-userprofiles -flask-cache==0.13.1 -flask-caching==1.4.0 # via invenio-cache -flask-celeryext==0.3.1 # via invenio-accounts, invenio-app, invenio-celery, invenio-indexer, invenio-records -flask-cli==0.4.0 -flask-collect==1.2.2 # via invenio-assets -flask-cors==3.0.6 # via invenio-rest -flask-debugtoolbar==0.10.1 -flask-kvsession==0.6.2 # via invenio-accounts -flask-limiter==1.0.1 # via invenio-app -flask-login==0.4.1 # via flask-security, invenio-accounts, invenio-admin, invenio-oauth2server -flask-mail==0.9.1 # via flask-security, invenio-accounts, invenio-mail, invenio-userprofiles -flask-menu==0.7.0 # via flask-breadcrumbs, invenio-accounts, invenio-admin, invenio-theme, invenio-userprofiles -flask-oauthlib==0.9.5 # via invenio-oauth2server, invenio-oauthclient -flask-principal==0.4.0 # via flask-security, invenio-admin -flask-security==3.0.0 # via invenio-accounts -flask-shell-ipython==0.3.1 # via invenio-app -flask-sqlalchemy==2.4 # via flask-alembic, invenio-db -flask-talisman==0.5.1 # via invenio-app -flask-wtf==0.14.2 # via flask-security, invenio-accounts, invenio-oauth2server, invenio-userprofiles -flask==0.12.4 -ftfy==4.4.3 # via invenio-records-rest -future==0.16.0 # via cookiecutter, invenio-accounts, invenio-oauth2server -fs==0.5.4 -html5lib==1.0.1 # via bleach, ftfy -idna==2.6 # via cryptography, requests, urllib3 -infinity==1.4 # via intervals -intervals==0.8.1 # via wtforms-components -invenio-access==1.0.1 # via invenio, invenio-accounts-rest -invenio-accounts-rest==1.0.0a4 -invenio-accounts==1.0.1 # via invenio, invenio-access, invenio-accounts-rest, invenio-oauth2server, invenio-oauthclient, invenio-userprofiles -invenio-admin==1.0.0 # via invenio -invenio-app==1.0.0 # via invenio -invenio-assets==1.0.0 # via invenio, invenio-search-ui, invenio-theme -invenio-base==1.0.1 # via invenio, invenio-app -invenio-cache==1.0.0 # via invenio-app -invenio-celery==1.1.0 # via invenio -invenio-config==1.0.0 # via invenio, invenio-app -git+https://github.com/cernanalysispreservation/invenio-deposit.git#egg=invenio-deposit -reana-client==0.6.0 -reana-commons==0.6.0 -invenio-db[postgresql,versioning]==1.0.1 # via invenio, invenio-accounts-rest, invenio-admin -invenio-formatter==1.0.1 # via invenio -invenio-i18n==1.0.0 # via invenio, invenio-accounts, invenio-theme -invenio-indexer==1.0.1 # via invenio, invenio-records-rest -invenio-jsonschemas==1.0.0 # via invenio -# 'invenio-logging' < v1.2.0 is needed because of 'invenio-base' version -# conflicts, USING 'cernanalysispreservation/invenio-logging' FORM now -# invenio-logging==1.0.0 -git+https://github.com/cernanalysispreservation/invenio-logging.git@v1.2.0-without-invenio-base-des#egg=invenio-logging[sentry,sentry-sdk] -invenio-mail==1.0.1 # via invenio, invenio-oauthclient -invenio-oaiserver==1.0.0 # via invenio -invenio-oauth2server==1.0.1 # via invenio, invenio-accounts-rest -git+https://github.com/cernanalysispreservation/invenio-oauthclient.git@master#egg=invenio-oauthclient -invenio-pidstore==1.0.0 # via invenio, invenio-indexer, invenio-oaiserver, invenio-records-rest, invenio-records-ui -invenio-query-parser==0.6.0 -invenio-records-rest==1.1.0 # via invenio -invenio-records-ui==1.0.1 # via invenio -invenio-records==1.0.0 # via invenio, invenio-indexer, invenio-oaiserver, invenio-records-rest, invenio-records-ui -invenio-rest==1.0.0 -invenio-files-rest==1.0.0 -# invenio-rest==1.0.0 # via invenio, invenio-accounts-rest, invenio-records-rest -invenio-search-ui==1.0.1 # via invenio -invenio-search[elasticsearch5]==1.0.0 # via invenio -invenio-records-files==1.0.0a11 -invenio-theme==1.0.0 # via invenio -invenio-userprofiles==1.0.1 -invenio[auth,base,elasticsearch5,metadata,postgresql]==3.0.0 -ipaddr==2.2.0 # via invenio-accounts -ipaddress==1.0.22 # via cryptography, maxminddb, uritools, urllib3 -ipython-genutils==0.2.0 # via traitlets -ipython==5.7.0 # via flask-shell-ipython -itsdangerous==0.24 # via flask, flask-debugtoolbar, flask-kvsession, flask-security -jinja2-time==0.2.0 # via cookiecutter -jinja2==2.10 # via cookiecutter, flask, flask-babelex, invenio-formatter, jinja2-time -jsmin==2.2.2 # via invenio-theme -jsonpatch==1.23 # via invenio-records -jsonpointer==2.0 # via jsonpatch -jsonref==0.1 # via invenio-jsonschemas, invenio-records -jsonresolver==0.2.1 # via invenio-records -jsonschema[format]==3.0.1 # via invenio-records -kombu==4.6.7 # TODO Temporary FIX -limits==1.3 # via flask-limiter -lxml==4.2.1 # via dojson, invenio-oaiserver -mako==1.0.7 # via alembic -markupsafe==1.1.1 # via jinja2, mako -marshmallow==2.15.3 # via invenio-oaiserver, invenio-records-rest, webargs -maxminddb-geolite2==2018.703 # via invenio-accounts -maxminddb==1.4.1 # via maxminddb-geolite2 -msgpack-python==0.5.6 # via invenio-celery -node-semver==0.1.1 # via invenio-assets -oauthlib==2.1.0 # via flask-oauthlib, invenio-oauth2server, requests-oauthlib -ordereddict==1.1 # via invenio-query-parser -pandas==1.0.1 -passlib==1.7.1 # via flask-security, invenio-accounts -pathlib2==2.3.2 # via ipython, pickleshare -pexpect==4.6.0 # via ipython -pickleshare==0.7.4 # via ipython -pluggy==0.7.1 # via jsonresolver -poyo==0.4.1 # via cookiecutter -prometheus-flask-exporter==0.20.3 -prompt-toolkit==1.0.15 # via ipython -psycopg2-binary==2.7.7 # via invenio-db -ptyprocess==0.5.2 # via pexpect -pyasn1-modules==0.2.1 # via python-ldap -pyasn1==0.4.3 # via pyasn1-modules, python-ldap -pycparser==2.18 # via cffi -pygithub==1.39 -pyngrok==1.4.2 -pygments==2.2.0 # via ipython -pyjwt==1.6.4 # via invenio-accounts, invenio-oauth2server, pygithub -pyopenssl==19.1.0 # via urllib3 -pypeg2==2.15.2 # via invenio-query-parser -python-dateutil==2.7.3 # via alembic, arrow, elasticsearch-dsl, invenio-records-rest -python-editor==1.0.3 # via alembic -python-gitlab==1.4.0 -python-ldap==3.1.0 -pytz==2018.4 # via babel, celery, invenio-indexer -redis==2.10.6 # via invenio-accounts, invenio-celery -requests-oauthlib==1.0.0 # via flask-oauthlib -requests==2.18.4 # via cookiecutter, invenio-search, python-gitlab, requests-oauthlib -scandir==1.7 # via pathlib2 -simplegeneric==0.8.1 # via ipython -simplejson==3.15.0 # via dojson, uwsgitop -simplekv==0.11.8 # via flask-kvsession, invenio-accounts -six==1.11.0 # via bleach, cryptography, elasticsearch-dsl, flask-breadcrumbs, flask-cors, flask-kvsession, flask-limiter, flask-menu, flask-talisman, html5lib, invenio-access, invenio-logging, invenio-oauth2server, invenio-oauthclient, invenio-query-parser, invenio-records-rest, jsonresolver, limits, pathlib2, prompt-toolkit, pyopenssl, python-dateutil, python-gitlab, sqlalchemy-utils, traitlets, validators, wtforms-alchemy, wtforms-components -speaklater==1.3 # via flask-babelex, invenio-assets -sqlalchemy-continuum==1.3.4 -sqlalchemy-utils[encrypted]==0.33.3 # via invenio-accounts, invenio-db, invenio-oauth2server, invenio-oauthclient, sqlalchemy-continuum, wtforms-alchemy -sqlalchemy==1.3.0 # via alembic, flask-alembic, flask-sqlalchemy, invenio-db, sqlalchemy-continuum, sqlalchemy-utils, wtforms-alchemy -traitlets==4.3.2 # via ipython -ua-parser==0.8.0 # via invenio-accounts -uritools==2.2.0 # via invenio-oauthclient -urllib3[secure]==1.22 -uwsgi-tools==1.1.1 -uwsgi==2.0.17.1 -uwsgitop==0.10 -validators==0.12.2 # via wtforms-components -wcwidth==0.1.7 # via ftfy, prompt-toolkit -webargs==3.0.1 # via invenio-oaiserver, invenio-records-rest -webassets==0.12.1 # via flask-assets, invenio-assets -webencodings==0.5.1 # via html5lib -werkzeug==0.14.1 # via flask, flask-debugtoolbar, flask-kvsession, jsonresolver -whichcraft==0.4.1 # via cookiecutter -wtforms-alchemy==0.16.7 # via invenio-oauth2server -wtforms-components==0.10.3 # via wtforms-alchemy -wtforms==2.2.1 # via flask-admin, flask-wtf, invenio-userprofiles, wtforms-alchemy, wtforms-components diff --git a/scripts/create-demo-users.sh b/scripts/create-demo-users.sh index 0cb32f4126..4782fd28ab 100644 --- a/scripts/create-demo-users.sh +++ b/scripts/create-demo-users.sh @@ -41,6 +41,7 @@ cap roles create analysis-preservation-support@cern.ch cap roles create faser-all@cern.ch cap roles add info@inveniosoftware.org analysis-preservation-support@cern.ch +cap access allow superuser-access role analysis-preservation-support@cern.ch cap roles add cms@inveniosoftware.org cms-members@cern.ch cap roles add alice@inveniosoftware.org alice-member@cern.ch diff --git a/setup.py b/setup.py index 47fc7cdde5..4714973a6f 100644 --- a/setup.py +++ b/setup.py @@ -13,23 +13,11 @@ readme = open('README.rst').read() -DATABASE = "postgresql" -ELASTICSEARCH = "elasticsearch5" -INVENIO_VERSION = '3.0.0' # "3.0.0rc2" - tests_require = [ 'check-manifest>=0.35', - 'coverage>=5.2.1', - 'isort>=4.3', - 'mock>=2.0.0', - 'pydocstyle>=2.0.0', - 'pytest-cov==2.5.1', - 'pytest-invenio>=1.0.5,<=1.3.4', - 'pytest-mock>=1.6.0', - 'pytest-pep8>=1.0.6', - 'pytest-random-order>=0.5.4', - 'pytest==5.3.5', - 'responses==0.10.6', + 'pytest-cov>=3', + 'pytest>=7', + 'responses>=0.22.0', ] extras_require = {'docs': ['Sphinx>=1.5.1'], 'tests': tests_require} @@ -49,54 +37,86 @@ setup_requires = ['Babel>=2.4.0', 'pytest-runner>=3.0.0,<5'] install_requires = [ - 'Flask==0.12.4', - 'Flask-Cli>=0.4.0', - 'Flask-Cache>=0.13.1', - 'Flask-Debugtoolbar>=0.10.1', + # CAP Base + 'Flask==1.1', + 'click==8.0.0', + 'jinja2==3.0.3', + 'itsdangerous==2.0.1', + 'werkzeug==1.0.1', + 'Flask-Caching==1.5.0', + 'Flask-Debugtoolbar', + 'flask-wtf==0.15.1', + 'flask-login==0.4.1', + # CAP specific libraries + 'jsonref>=1.0.0', + 'jsonresolver>=0.3.2', 'PyGithub>=1.35', 'python-gitlab>=1.0.2', + 'python-ldap==3.1.0', 'python-cern-sso-krb==1.3.3', - 'gssapi', - # FIX cryptography <=3.3.2 is needed for installation not to crash - # https://github.com/Azure/azure-cli/issues/16858 - 'cryptography<=3.3.2', - 'paramiko==2.7.1', - 'cachetools==3.1.0', - # Pinned libraries - 'urllib3[secure]==1.22', - 'sqlalchemy==1.3.0', - # temporary pinned since there are 'fs' conslicts between - # 'reana-commons' and 'invenio-files-rest' - 'fs==0.5.4', - 'invenio-accounts-rest>=1.0.0a4', - 'invenio-oauthclient>=1.0.0', - 'invenio-userprofiles>=1.0.0', - 'invenio-query-parser>=0.3.0', - 'invenio[{db},{es},base,auth,metadata]~={version}'.format( - db=DATABASE, es=ELASTICSEARCH, version=INVENIO_VERSION - ), - 'invenio-rest==1.0.0', - 'invenio-files-rest==1.0.0', - 'invenio-records-files==1.0.0a11', - 'jsonschema[format]==3.0.1', - 'coolname==1.1.0', + 'gssapi>=1.7.3', 'Authlib==0.15.1', - # 'invenio-logging' < v1.2.0 is needed because of 'invenio-base' version - # conflicts, USING 'cernanalysispreservation/invenio-logging' FORM now - # 'invenio-logging[sentry, sentry-sdk]<=1.2.0', - 'uWSGI==2.0.17.1', + 'uWSGI==2.0.21', 'uwsgi-tools==1.1.1', 'uwsgitop==0.10', - # needed version for future use of arguments - 'webargs==3.0.1', - 'pyOpenSSL==19.1.0', + 'webargs==5.5.0', 'gspread==3.7.0', - 'requests-gssapi', - 'beautifulsoup4', - # reana_client => bravado core dependency pin due to py3.6 drop + 'paramiko==2.7.1', + 'cachetools==3.1.0', + 'urllib3==1.26', + 'coolname==1.1.0', + 'requests-gssapi>=1.2.3', 'swagger-spec-validator==2.7.6', 'prometheus-flask-exporter==0.20.3', + 'wtforms==2.2.1', + 'beautifulsoup4>=4', + 'pandas>=1.5', + 'marshmallow==2.17.0', + 'reana-client==0.8.1', + 'reana-commons[yadage,snakemake]==0.8.4', + + # Invenio Base Deps + 'invenio-base==1.2.5', + 'invenio-admin==1.1.2', + 'invenio-assets==1.1.3', + 'invenio-formatter==1.0.2', + 'invenio-mail==1.0.2', + 'invenio-rest==1.1.2', + 'invenio-theme==1.1.4', + 'invenio-celery@git+https://github.com/cernanalysispreservation/invenio-celery.git@mock/v1.1.1#egg=invenio-celery', + 'invenio-logging[sentry,sentry-sdk] @ git+https://github.com/inveniosoftware/invenio-logging@94bc56117593eae62ba975d576e8c7b991311c0d', + + # Invenio Auth Deps + 'invenio-access==1.3.0', + 'invenio-accounts==1.1.1', + 'invenio-oauth2server==1.0.4', + 'invenio-oauthclient @ git+https://github.com/cernanalysispreservation/invenio-oauthclient.git@v1.1.3', + 'invenio-userprofiles @ git+https://github.com/cernanalysispreservation/invenio-userprofiles.git@v1.0.1', + + # Invenio Metadata Deps + 'invenio-indexer @ git+https://github.com/cernanalysispreservation/invenio-indexer.git@os-2-new', + 'invenio-jsonschemas @ git+https://github.com/cernanalysispreservation/invenio-jsonschemas.git@v1.0.1', + 'invenio-oaiserver @ git+https://github.com/cernanalysispreservation/invenio-oaiserver.git@os-2-new', + 'invenio-pidstore==1.1.0', + 'invenio-records-rest @ git+https://github.com/parths007/invenio-records-rest.git@os-2-new', + 'invenio-records-ui==1.0.1', + 'invenio-records==1.3.0', + 'invenio-search-ui==1.1.1', + 'invenio-search @ git+https://github.com/parths007/invenio-search.git@v1.2.3#egg=invenio-search', + + # Invenio Files deps + 'invenio-files-rest @ git+https://github.com/cernanalysispreservation/invenio-files-rest.git@mock/v1.0.5', + 'invenio-records-files @ git+https://github.com/cernanalysispreservation/invenio-records-files.git@new#egg=invenio-records-files', + + # Database deps + 'invenio-db[postgresql,versioning]==1.0.13', + + # Invenio required deps + 'invenio-deposit @ git+https://github.com/cernanalysispreservation/invenio-deposit.git@os-2', + 'invenio-accounts-rest==1.0.0a4', + 'invenio-query-parser @ git+https://github.com/cernanalysispreservation/invenio-query-parser.git@os', + 'invenio[auth,base,metadata,postgresql]==3.2.0', ] packages = find_packages() @@ -241,7 +261,5 @@ 'Programming Language :: Python', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Development Status :: 3 - Alpha', ], ) diff --git a/tests/conftest.py b/tests/conftest.py index 33d0302af0..4c7697e0ad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -77,7 +77,7 @@ @pytest.fixture() def cli_runner(app): - runner = CliRunner() + runner = CliRunner(env={'FLASK_APP': 'cap/wsgi.py'}) script_info = ScriptInfo(create_app=lambda info: app) def run(command): @@ -117,7 +117,7 @@ def default_config(): APP_DEFAULT_SECURE_HEADERS=APP_DEFAULT_SECURE_HEADERS, CELERY_ALWAYS_EAGER=True, CELERY_CACHE_BACKEND='memory', - CELERY_EAGER_PROPAGATES_EXCEPTIONS=True, + task_eager_propagates=True, CELERY_RESULT_BACKEND='cache', SQLALCHEMY_DATABASE_URI='postgresql://cap:cap@localhost/cap-test', JSONSCHEMAS_HOST='analysispreservation.cern.ch', @@ -152,8 +152,10 @@ def default_config(): CMS_HYPERNEWS_EMAIL_FORMAT='hn-cms-{}@cern0.ch', GITHUB_CAP_TOKEN="testtokengithub", GITLAB_CAP_TOKEN="testtokengitlab", - CMS_STATS_QUESTIONNAIRE_ADMIN_ROLES='cms-admins@cern0.ch') - + CMS_STATS_QUESTIONNAIRE_ADMIN_ROLES='cms-admins@cern0.ch', + RATELIMIT_APPLICATION='1000/minute', + RECORDS_FILES_REST_ENDPOINTS={}, + APP_HEALTH_BLUEPRINT_ENABLED=False) @pytest.fixture(scope='session') @@ -175,6 +177,43 @@ def base_app(create_app, default_config, request): yield app_ +@pytest.fixture() +def client(base_app): + """Test client for the base application fixture. + Scope: function + If you need the database and search indexes initialized, simply use the + Pytest-Flask fixture ``client`` instead. This fixture is mainly useful if + you need a test client without needing to initialize both the database and + search indexes. + """ + with base_app.test_client() as client: + yield client + + +@pytest.fixture(scope='module') +def appctx(base_app): + """Application context for the current base application. + Scope: module + This fixture pushes an application context on the stack, so that + ``current_app`` is defined and e.g ``url_for`` will also work. + """ + with base_app.app_context(): + yield base_app + + +@pytest.fixture(scope='module') +def script_info(base_app): + """Get ScriptInfo object for testing a CLI command. + Scope: module + .. code-block:: python + def test_cmd(script_info): + runner = CliRunner() + result = runner.invoke(mycmd, obj=script_info) + assert result.exit_code == 0 + """ + return ScriptInfo(create_app=lambda info: base_app) + + @pytest.fixture def db(base_app): """Setup database.""" @@ -264,7 +303,7 @@ def superuser(db, clear_caches): return superuser -@pytest.fixture('function') +@pytest.fixture() def clear_caches(): yield get_user_email_by_id.cache_clear() @@ -274,7 +313,7 @@ def clear_caches(): @pytest.fixture def es(base_app): - """Provide elasticsearch access.""" + """Provide Search access.""" list(current_search.delete(ignore=[400, 404])) current_search_client.indices.delete(index='*') list(current_search.create()) @@ -380,28 +419,27 @@ def _write_token(user): def get_default_mapping(name, version): - mapping_name = f"{name}-v{version}" - default_mapping = { "mappings": {} } - collectiion_mapping = { - "properties": { - "_collection": { - "type": "object", - "properties": { - "fullname": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" + mapping = { + "mappings": { + "properties": { + "_collection": { + "type": "object", + "properties": { + "fullname": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } } } } } } - default_mapping["mappings"][mapping_name] = collectiion_mapping - return default_mapping + return mapping @pytest.fixture diff --git a/tests/integration/cli/test_schema_records_validation.py b/tests/integration/cli/test_schema_records_validation.py index c557be48a5..f91c01896b 100644 --- a/tests/integration/cli/test_schema_records_validation.py +++ b/tests/integration/cli/test_schema_records_validation.py @@ -105,6 +105,19 @@ def test_validate_with_schema_url_correct_record( assert 'No errors found in record' in res.output +def test_validate_with_schema_url_correct_record_published( + app, db, es, cli_runner, users, create_schema, create_deposit): + create_schema('test', experiment='CMS') + create_deposit(users['cms_user'], 'test', publish=True) + + res = cli_runner( + 'fixtures validate -u https://analysispreservation.cern.ch/schemas/records/test-v1.0.0.json -s published') + + assert res.exit_code == 0 + assert '1 record(s) of test found.' in res.output + assert 'No errors found in record' in res.output + + def test_validate_with_ana_type_correct_record( app, db, es, cli_runner, script_info, users, create_schema, create_deposit): create_schema('test', experiment='CMS') diff --git a/tests/integration/deposits/test_files_for_deposit.py b/tests/integration/deposits/test_files_for_deposit.py index 1238c83683..abdf427615 100644 --- a/tests/integration/deposits/test_files_for_deposit.py +++ b/tests/integration/deposits/test_files_for_deposit.py @@ -27,7 +27,6 @@ import json from flask import current_app -from mock import patch from pytest import mark from six import BytesIO @@ -109,8 +108,7 @@ def test_deposit_files_when_no_files_returns_empty_list( resp = client.get('/deposits/{}/files'.format(pid), headers=auth_headers_for_user(owner)) - - assert resp.json == [] + assert json.loads(resp.data) == [] def test_deposit_files_returns_list_with_files_info(client, users, @@ -124,6 +122,7 @@ def test_deposit_files_returns_list_with_files_info(client, users, resp = client.get('/deposits/{}/files'.format(pid), headers=auth_headers_for_user(owner)) + files = resp.json assert files[0]["filename"] == 'file_1.txt' @@ -512,7 +511,7 @@ def test_put_header_tags(client, users, auth_headers_for_user, create_deposit): key = 'test.txt' headers = [ (current_app.config['FILES_REST_FILE_TAGS_HEADER'], - 'key1=val1;key2=val2;key3=val3'), + 'key1=val1&key2=val2&key3=val3'), ] owner = users['cms_user'] diff --git a/tests/integration/deposits/test_get_deposits.py b/tests/integration/deposits/test_get_deposits.py index ea010eeab6..eb235b7fdb 100644 --- a/tests/integration/deposits/test_get_deposits.py +++ b/tests/integration/deposits/test_get_deposits.py @@ -276,6 +276,7 @@ def test_get_deposit_with_default_serializer( 'key': file.key, 'size': file.file.size, 'version_id': str(file.version_id), + 'file_id': str(file.file.id), } ], 'is_owner': True, @@ -341,8 +342,8 @@ def test_get_deposits_with_correct_search_links( assert resp.status_code == 200 assert resp.json['links'] == { - 'self': 'http://analysispreservation.cern.ch/api/deposits/?page=1&size=10', - 'next': 'http://analysispreservation.cern.ch/api/deposits/?page=2&size=10', + 'self': 'http://analysispreservation.cern.ch/api/deposits/?size=10&page=1', + 'next': 'http://analysispreservation.cern.ch/api/deposits/?size=10&page=2', } @@ -580,90 +581,86 @@ def test_get_sorted_results_by_stage_strings( deposit_mapping_1 = { "mappings" : { - "test-analysis-v1.0.0": { - "properties": { - "_collection": { - "type": "object", - "properties": { - "fullname": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" - } + "properties": { + "_collection": { + "type": "object", + "properties": { + "fullname": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" } - }, - "analysis_stage": { - "type": "keyword", - "store": True - }, - "analysis_sub_stage": { - "type": "keyword", - "store": True - }, - "initial": { - "type": "object", - "properties": { - "status": { - "type": "object", - "properties": { - "main_status": { - "type": "keyword", - "copy_to": "analysis_stage" - }, - "sub_status": { - "type": "keyword", - "copy_to": "analysis_sub_stage" - } - } + } + }, + "analysis_stage": { + "type": "keyword", + "store": True + }, + "analysis_sub_stage": { + "type": "keyword", + "store": True + }, + "initial": { + "type": "object", + "properties": { + "status": { + "type": "object", + "properties": { + "main_status": { + "type": "keyword", + "copy_to": "analysis_stage" + }, + "sub_status": { + "type": "keyword", + "copy_to": "analysis_sub_stage" + } } } - }, - } + } + }, } } } deposit_mapping_2 = { "mappings" : { - "test-ana-v1.0.0": { - "properties": { - "_collection": { - "type": "object", - "properties": { - "fullname": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" - } + "properties": { + "_collection": { + "type": "object", + "properties": { + "fullname": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" } - }, - "analysis_stage": { - "type": "keyword", - "store": True - }, - "analysis_sub_stage": { - "type": "keyword", - "store": True - }, - "stat": { - "type": "object", - "properties": { - "main_status": { - "type": "keyword", - "copy_to": "analysis_stage" - }, - "sub_status": { - "type": "keyword", - "copy_to": "analysis_sub_stage" - } + } + }, + "analysis_stage": { + "type": "keyword", + "store": True + }, + "analysis_sub_stage": { + "type": "keyword", + "store": True + }, + "stat": { + "type": "object", + "properties": { + "main_status": { + "type": "keyword", + "copy_to": "analysis_stage" + }, + "sub_status": { + "type": "keyword", + "copy_to": "analysis_sub_stage" } } } @@ -843,7 +840,7 @@ def test_get_sorted_results_by_stage_strings( resp = client.get('/deposits?q=&sort=analysis_stage',headers=headers+json_headers) assert resp.status_code == 200 - assert resp.json['hits']['total'] == 4 + assert resp.json['hits']['total'] == 5 # Test with schema having no status deposit_mapping_6 = get_default_mapping('test-status', "1.0.0") @@ -864,7 +861,7 @@ def test_get_sorted_results_by_stage_strings( resp = client.get('/deposits?q=&sort=analysis_stage',headers=headers+json_headers) assert resp.status_code == 200 - assert resp.json['hits']['total'] == 4 + assert resp.json['hits']['total'] == 6 def test_get_deposits_with_facets_get_types_doesnt_confuse_naming( @@ -1017,28 +1014,26 @@ def test_get_deposits_with_range_query( } }, 'mappings': { - 'test-schema-v1.0.0': { - 'properties': { - 'next_deadline_date': {'type': 'date'}, - "_collection": { - "type": "object", - "properties": { - "fullname": {"type": "keyword"}, - "name": {"type": "keyword"}, - "version": {"type": "keyword"}, - }, + 'properties': { + 'next_deadline_date': {'type': 'date'}, + "_collection": { + "type": "object", + "properties": { + "fullname": {"type": "keyword"}, + "name": {"type": "keyword"}, + "version": {"type": "keyword"}, }, - "analysis_context": { - "type": "object", - "properties": { - 'next_deadline_date': { - 'type': 'date', - "format": "yyyy-MM-dd", - "copy_to": "next_deadline_date", - } - }, + }, + "analysis_context": { + "type": "object", + "properties": { + 'next_deadline_date': { + 'type': 'date', + "format": "yyyy-MM-dd", + "copy_to": "next_deadline_date", + } }, - } + }, } }, }, @@ -1546,8 +1541,8 @@ def test_get_deposit_with_form_json_serializer_x_cap_field( assert resp.status_code == 200 assert resp.json['x_cap_permission'] == [ - {"path": ["properties", "title"], "value": {"users": [example_user.email]}}, {"path": ["properties", "date"], "value": {"users": ["test_user@cern.ch"]}}, + {"path": ["properties", "title"], "value": {"users": [example_user.email]}}, ] assert resp.json['schemas']['schema']['properties']['date']['readOnly'] == True assert resp.json['schemas']['schema']['properties']['title'].get('readOnly') == None diff --git a/tests/integration/experiments/test_atlas_api.py b/tests/integration/experiments/test_atlas_api.py index 7ce0ff041e..6e82567521 100644 --- a/tests/integration/experiments/test_atlas_api.py +++ b/tests/integration/experiments/test_atlas_api.py @@ -26,7 +26,7 @@ import responses from flask import current_app -from mock import patch +from unittest.mock import patch @responses.activate diff --git a/tests/integration/experiments/test_cms_api.py b/tests/integration/experiments/test_cms_api.py index 524e176dbe..7cc6ac8d23 100644 --- a/tests/integration/experiments/test_cms_api.py +++ b/tests/integration/experiments/test_cms_api.py @@ -26,7 +26,8 @@ from __future__ import absolute_import, print_function -from mock import patch +from unittest.mock import patch +from pytest import mark from cap.modules.experiments.errors import ExternalAPIException @@ -233,7 +234,7 @@ def test_get_datasets_suggestions_when_no_query_passed_returns_empty_list( resp = client.get('/cms/primary-datasets?query=', headers=headers) assert resp.json == [] - +@mark.skip("@use_args decorator raises Validation Error if no args") def test_get_datasets_suggestions_when_no_query_arg_returns_400( client, users, auth_headers_for_user, das_datasets_index): headers = auth_headers_for_user(users['cms_user']) @@ -317,7 +318,7 @@ def test_get_main_datasets_suggestions_when_no_query_passed_returns_empty_list( assert resp.json == [] - +@mark.skip("@use_args decorator raises Validation Error if no args") def test_get_main_datasets_suggestions_when_no_query_arg_returns_400( client, users, auth_headers_for_user, das_datasets_index): headers = auth_headers_for_user(users['cms_user']) diff --git a/tests/integration/mail/test_post_action.py b/tests/integration/mail/test_post_action.py index 5fdffcc6be..a287cba972 100644 --- a/tests/integration/mail/test_post_action.py +++ b/tests/integration/mail/test_post_action.py @@ -25,7 +25,7 @@ import json from pytest import raises, mark -from mock import patch +from unittest.mock import patch from invenio_deposit.signals import post_action from cap.modules.schemas.helpers import ValidationError diff --git a/tests/integration/repos/test_upload_with_github.py b/tests/integration/repos/test_upload_with_github.py index d324980a47..388be56c6d 100644 --- a/tests/integration/repos/test_upload_with_github.py +++ b/tests/integration/repos/test_upload_with_github.py @@ -29,7 +29,7 @@ import tarfile import responses from pytest import mark -from mock import Mock, patch +from unittest.mock import Mock, patch from github import GithubException from invenio_files_rest.models import ObjectVersion @@ -217,7 +217,7 @@ def test_upload_when_user_gave_url_with_sha_and_tries_to_create_a_release_webhoo class MockProject(object): id = 123 def get_branch(self, name): - raise GithubException(404, data={'message': 'Branch not found'}) + raise GithubException(404, headers={}, data={'message': 'Branch not found'}) def get_commit(self, sha): mock = Mock() @@ -250,7 +250,7 @@ def test_upload_when_user_gave_url_with_sha_and_tries_to_create_a_release_webhoo class MockProject(object): id = 123 def get_branch(self, name): - raise GithubException(404, data={'message': 'Branch not found'}) + raise GithubException(404, headers={}, data={'message': 'Branch not found'}) def get_commit(self, sha): mock = Mock() @@ -284,7 +284,7 @@ def test_upload_when_user_gave_url_with_sha_and_tries_to_create_a_push_webhook_r class MockProject(object): id = 123 def get_branch(self, name): - raise GithubException(404, data={'message': 'Branch not found'}) + raise GithubException(404, headers={}, data={'message': 'Branch not found'}) def get_commit(self, sha): mock = Mock() @@ -319,7 +319,7 @@ def test_upload_when_user_gave_url_with_sha_and_tries_to_create_a_push_webhook_r class MockProject(object): id = 123 def get_branch(self, name): - raise GithubException(404, data={'message': 'Branch not found'}) + raise GithubException(404, headers={}, data={'message': 'Branch not found'}) def get_commit(self, sha): mock = Mock() @@ -807,8 +807,7 @@ def get_file_contents(self, filepath, ref): content_type='text/plain', headers={ 'Content-Length': '18', - 'Content-Encoding': 'gzip', - 'Content-Type': 'text/plain; charset=utf-8' + 'Content-Encoding': 'gzip' }, stream=True, status=200) @@ -860,8 +859,7 @@ def get_file_contents(self, filepath, ref): content_type='text/plain', headers={ 'Content-Length': '18', - 'Content-Encoding': 'gzip', - 'Content-Type': 'text/plain; charset=utf-8' + 'Content-Encoding': 'gzip' }, stream=True, status=200) diff --git a/tests/integration/repos/test_upload_with_gitlab.py b/tests/integration/repos/test_upload_with_gitlab.py index 08b18e9159..09d0c18cec 100644 --- a/tests/integration/repos/test_upload_with_gitlab.py +++ b/tests/integration/repos/test_upload_with_gitlab.py @@ -28,7 +28,7 @@ import json import tarfile import responses -from mock import Mock, patch +from unittest.mock import Mock, patch from pytest import mark from gitlab.exceptions import GitlabGetError diff --git a/tests/integration/repos/test_view_for_github_webhooks.py b/tests/integration/repos/test_view_for_github_webhooks.py index efe5f807ee..bb0d5cdcf3 100644 --- a/tests/integration/repos/test_view_for_github_webhooks.py +++ b/tests/integration/repos/test_view_for_github_webhooks.py @@ -29,7 +29,7 @@ from invenio_files_rest.models import ObjectVersion import responses -from mock import Mock, patch +from unittest.mock import Mock, patch from cap.modules.repos.github_api import Github from cap.modules.repos.models import GitSnapshot diff --git a/tests/integration/repos/test_view_for_gitlab_webhooks.py b/tests/integration/repos/test_view_for_gitlab_webhooks.py index a2a9d233f3..3040078d9c 100644 --- a/tests/integration/repos/test_view_for_gitlab_webhooks.py +++ b/tests/integration/repos/test_view_for_gitlab_webhooks.py @@ -28,7 +28,7 @@ import responses from invenio_files_rest.models import ObjectVersion -from mock import Mock, patch +from unittest.mock import Mock, patch from pytest import mark from cap.modules.repos.models import GitSnapshot diff --git a/tests/integration/schemas/test_schemas_views.py b/tests/integration/schemas/test_schemas_views.py index d5d97b5fdc..8df0bf6537 100644 --- a/tests/integration/schemas/test_schemas_views.py +++ b/tests/integration/schemas/test_schemas_views.py @@ -57,11 +57,9 @@ def test_get_when_user_outside_of_experiment_returns_403( record_options={'title': 'record_options'}, record_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'title': { - 'type': 'text' - } + 'properties': { + 'title': { + 'type': 'text' } } } @@ -69,11 +67,9 @@ def test_get_when_user_outside_of_experiment_returns_403( deposit_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } } @@ -134,11 +130,9 @@ def test_get(client, db, users, auth_headers_for_user): record_options={'title': 'record_options'}, record_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'title': { - 'type': 'text' - } + 'properties': { + 'title': { + 'type': 'text' } } } @@ -146,11 +140,9 @@ def test_get(client, db, users, auth_headers_for_user): deposit_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } } @@ -189,27 +181,24 @@ def test_get(client, db, users, auth_headers_for_user): { 'mappings': { - 'doc': { - 'properties': { - 'title': { - 'type': 'text' - } + + 'properties': { + 'title': { + 'type': 'text' } } + } }, 'deposit_mapping': { 'mappings': { - 'doc': + 'properties': { - 'properties': - { - 'keyword': { - 'type': 'keyword' - } - } + 'keyword': { + 'type': 'keyword' + } } } }, @@ -252,11 +241,9 @@ def test_get(client, db, users, auth_headers_for_user): 'record_mapping': { 'mappings': { - 'doc': { - 'properties': { - 'title': { - 'type': 'text' - } + 'properties': { + 'title': { + 'type': 'text' } } } @@ -265,11 +252,9 @@ def test_get(client, db, users, auth_headers_for_user): { 'mappings': { - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } } @@ -311,11 +296,9 @@ def test_get(client, db, users, auth_headers_for_user): 'record_mapping': { 'mappings': { - 'doc': { - 'properties': { - 'title': { - 'type': 'text' - } + 'properties': { + 'title': { + 'type': 'text' } } } @@ -324,11 +307,9 @@ def test_get(client, db, users, auth_headers_for_user): { 'mappings': { - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } } @@ -624,11 +605,9 @@ def test_post_by_no_suepruser(client, db, users, auth_headers_for_user, json_hea record_options={'title': 'record_options'}, record_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'title': { - 'type': 'text' - } + 'properties': { + 'title': { + 'type': 'text' } } } @@ -636,11 +615,9 @@ def test_post_by_no_suepruser(client, db, users, auth_headers_for_user, json_hea deposit_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } } @@ -670,11 +647,9 @@ def test_post(client, db, users, auth_headers_for_user, json_headers): record_options={'title': 'record_options'}, record_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'title': { - 'type': 'text' - } + 'properties': { + 'title': { + 'type': 'text' } } } @@ -682,11 +657,9 @@ def test_post(client, db, users, auth_headers_for_user, json_headers): deposit_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } } @@ -724,11 +697,9 @@ def test_post(client, db, users, auth_headers_for_user, json_headers): 'record_mapping': { 'mappings': { - 'doc': { - 'properties': { - 'title': { - 'type': 'text' - } + 'properties': { + 'title': { + 'type': 'text' } } } @@ -737,11 +708,9 @@ def test_post(client, db, users, auth_headers_for_user, json_headers): { 'mappings': { - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } } @@ -889,11 +858,9 @@ def test_post_with_valid_config_validation_github(client, db, users, auth_header record_options={'title': 'record_options'}, record_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'title': { - 'type': 'text' - } + 'properties': { + 'title': { + 'type': 'text' } } } @@ -901,11 +868,9 @@ def test_post_with_valid_config_validation_github(client, db, users, auth_header deposit_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } } @@ -955,11 +920,9 @@ def test_post_with_valid_config_validation_gitlab(client, db, users, auth_header record_options={'title': 'record_options'}, record_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'title': { - 'type': 'text' - } + 'properties': { + 'title': { + 'type': 'text' } } } @@ -967,11 +930,9 @@ def test_post_with_valid_config_validation_gitlab(client, db, users, auth_header deposit_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } } @@ -1020,11 +981,9 @@ def test_post_with_invalid_config_validation_gitlab(client, db, users, auth_head record_options={'title': 'record_options'}, record_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'title': { - 'type': 'text' - } + 'properties': { + 'title': { + 'type': 'text' } } } @@ -1032,11 +991,9 @@ def test_post_with_invalid_config_validation_gitlab(client, db, users, auth_head deposit_mapping={ 'mappings': { - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } } @@ -1243,19 +1200,15 @@ def test_put_when_no_superuser(client, db, auth_headers_for_user, users, json_he deposit_options={'title': 'deposit_options'}, record_schema={'title': 'record_schema'}, record_options={'title': 'record_options'}, - record_mapping={'doc': { - 'properties': { + record_mapping={'properties': { 'title': { 'type': 'text' } - } }}, deposit_mapping={ - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } }, @@ -1292,19 +1245,15 @@ def test_put(client, db, auth_headers_for_user, users, json_headers): deposit_options={'title': 'deposit_options'}, record_schema={'title': 'record_schema'}, record_options={'title': 'record_options'}, - record_mapping={'doc': { - 'properties': { + record_mapping={'properties': { 'title': { 'type': 'text' } - } }}, deposit_mapping={ - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } }, @@ -1346,21 +1295,17 @@ def test_put(client, db, auth_headers_for_user, users, json_headers): }, 'record_mapping': { # same as deposit_mapping because use_deposit_as_record == True - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } - } - } - }, - 'deposit_mapping': { - 'doc': { 'properties': { 'keyword': { 'type': 'keyword' } } + }, + 'deposit_mapping': { + 'properties': { + 'keyword': { + 'type': 'keyword' + } } }, 'links': { diff --git a/tests/integration/services/test_cds_api.py b/tests/integration/services/test_cds_api.py index 91e2003d68..926cb595a1 100644 --- a/tests/integration/services/test_cds_api.py +++ b/tests/integration/services/test_cds_api.py @@ -26,7 +26,7 @@ """Integration tests for CDS api.""" from __future__ import absolute_import, print_function -from mock import patch +from unittest.mock import patch import responses from cap.modules.experiments.errors import ExternalAPIException diff --git a/tests/integration/services/test_indico_api.py b/tests/integration/services/test_indico_api.py index fffe97ed2b..9ee5b3a4fb 100644 --- a/tests/integration/services/test_indico_api.py +++ b/tests/integration/services/test_indico_api.py @@ -27,7 +27,7 @@ from __future__ import absolute_import, print_function import responses -from mock import patch +from unittest.mock import patch from cap.modules.experiments.errors import ExternalAPIException diff --git a/tests/integration/services/test_oidc_api.py b/tests/integration/services/test_oidc_api.py index ba7eeec58e..883f9d0d06 100644 --- a/tests/integration/services/test_oidc_api.py +++ b/tests/integration/services/test_oidc_api.py @@ -26,7 +26,7 @@ """Integration tests for CERN OIDC api.""" import responses -from mock import patch +from unittest.mock import patch from cap.modules.auth.config import OIDC_API diff --git a/tests/integration/services/test_ror_api.py b/tests/integration/services/test_ror_api.py index 8c868760cd..bd3b5c9286 100644 --- a/tests/integration/services/test_ror_api.py +++ b/tests/integration/services/test_ror_api.py @@ -28,7 +28,7 @@ import json from invenio_rest.errors import RESTException -from mock import patch +from unittest.mock import patch from cap.modules.experiments.errors import ExternalAPIException diff --git a/tests/integration/test_collection_view.py b/tests/integration/test_collection_view.py index ba5c5ad04a..790111d10f 100644 --- a/tests/integration/test_collection_view.py +++ b/tests/integration/test_collection_view.py @@ -31,7 +31,7 @@ from pytest import mark -def test_collection_view_returns_user_drafts(client, users, create_deposit, +def test_collection_view_returns_user_drafts(location, client, users, create_deposit, auth_headers_for_user): my_deposit = create_deposit(users['lhcb_user'], 'lhcb', version="0.2.0") user_published_deposit = create_deposit(users['lhcb_user'], diff --git a/tests/integration/test_get_records.py b/tests/integration/test_get_records.py index 9339449df0..7b0a6b40f1 100644 --- a/tests/integration/test_get_records.py +++ b/tests/integration/test_get_records.py @@ -30,7 +30,6 @@ import json import responses from pytest import mark -from mock import Mock, patch from invenio_search import current_search from invenio_files_rest.models import ObjectVersion @@ -166,6 +165,7 @@ def test_get_records_default_serializer( 'key': file.key, 'size': file.file.size, 'version_id': str(file.version_id), + 'file_id': str(file.file.id), } ], 'is_owner': False, @@ -201,8 +201,8 @@ def test_get_records_with_correct_search_links( assert resp.status_code == 200 assert resp.json['links'] == { - 'self': 'http://analysispreservation.cern.ch/api/records/?page=1&sort=mostrecent&size=10', - 'next': 'http://analysispreservation.cern.ch/api/records/?page=2&sort=mostrecent&size=10', + 'self': 'http://analysispreservation.cern.ch/api/records/?sort=mostrecent&size=10&page=1', + 'next': 'http://analysispreservation.cern.ch/api/records/?sort=mostrecent&size=10&page=2', } diff --git a/tests/integration/test_permissions_api.py b/tests/integration/test_permissions_api.py index b521009db8..847d2bbd35 100644 --- a/tests/integration/test_permissions_api.py +++ b/tests/integration/test_permissions_api.py @@ -33,7 +33,7 @@ from conftest import add_role_to_user from invenio_accounts.models import Role, User -from mock import MagicMock, patch +from unittest.mock import MagicMock, patch from pytest import mark _datastore = LocalProxy(lambda: current_app.extensions['security'].datastore) diff --git a/tests/integration/test_reana_workflows.py b/tests/integration/test_reana_workflows.py index 51820d3bf2..8164676c41 100644 --- a/tests/integration/test_reana_workflows.py +++ b/tests/integration/test_reana_workflows.py @@ -27,7 +27,7 @@ from __future__ import absolute_import, print_function import json -from mock import patch +from unittest.mock import patch from pytest import mark from reana_client.errors import FileDeletionError diff --git a/tests/unit/deposit/test_cap_deposit.py b/tests/unit/deposit/test_cap_deposit.py index db2a8d2a7b..e9a09e31ba 100644 --- a/tests/unit/deposit/test_cap_deposit.py +++ b/tests/unit/deposit/test_cap_deposit.py @@ -28,7 +28,7 @@ from flask_security import login_user from invenio_access.models import ActionRoles, ActionUsers -from mock import patch +from unittest.mock import patch from pytest import mark, raises from sqlalchemy.exc import IntegrityError diff --git a/tests/unit/experiments/test_cadi_utils.py b/tests/unit/experiments/test_cadi_utils.py index a3b1b9c42e..8de2a97cdb 100644 --- a/tests/unit/experiments/test_cadi_utils.py +++ b/tests/unit/experiments/test_cadi_utils.py @@ -39,8 +39,7 @@ from invenio_accounts.testutils import create_test_user from invenio_search import current_search from ldap import LDAPError -from mock import patch -from mock.mock import MagicMock +from unittest.mock import patch, MagicMock from pytest import raises @@ -359,20 +358,18 @@ def test_get_deposit_by_cadi_id_returns_correct_deposit( } }, mapping={ - 'mappings': { - 'cms-analysis-v1.0.0': { - 'properties': { - "basic_info": { - "type": "object", - "properties": { - "cadi_id": { - "type": "keyword" - } - } - } - } - } - } + 'mappings': { + 'properties': { + "basic_info": { + "type": "object", + "properties": { + "cadi_id": { + "type": "keyword" + } + } + } + } + } }) create_deposit(superuser, 'cms-analysis', { '$ana_type': 'cms-analysis', @@ -395,32 +392,30 @@ def test_get_deposit_by_cadi_id_when_no_match_raises_DepositDoesNotExist( }, mapping={ 'mappings': { - 'cms-analysis-v1.0.0': { - 'properties': { - "basic_info": { - "type": "object", - "properties": { - "cadi_id": { - "type": "keyword" - } - } - }, - "_collection": { - "type": "object", - "properties": { - "fullname": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" - } + 'properties': { + "basic_info": { + "type": "object", + "properties": { + "cadi_id": { + "type": "keyword" } } - } - } + }, + "_collection": { + "type": "object", + "properties": { + "fullname": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + } + } } }) @@ -635,6 +630,7 @@ def test_synchronize_cadi_entries_when_LDAP_error_occured_during_permissions_ass # @TOFIX schemas module still uses mappings from files, that's why we use existing schemas # this should be patched in schemas PR +# To check the functionality of `user_edited` key in deposit. @patch('cap.modules.experiments.utils.cadi.get_all_from_cadi', MagicMock( return_value=[{ @@ -671,7 +667,7 @@ def test_synchronize_cadi_entries_when_LDAP_error_occured_during_permissions_ass u'name': '2HDM Higgs studies (H->ZZ and A->Zh)' }])) def test_synchronize_cadi_entries_when_entry_exist_updates_cadi_info( - appctx, db, es, superuser, create_deposit): + base_app, appctx, db, es, superuser, create_deposit): create_deposit( superuser, 'cms-analysis', { 'version': '0.0.1', @@ -680,15 +676,15 @@ def test_synchronize_cadi_entries_when_entry_exist_updates_cadi_info( 'cadi_id': 'EXO-00-001' } }) - # deposit with this cadi id already exists - deposit = get_deposit_by_cadi_id('EXO-00-000') + deposit = get_deposit_by_cadi_id('EXO-00-001') - synchronize_cadi_entries() + with base_app.test_request_context(): + synchronize_cadi_entries() - updated_deposit = get_deposit_by_cadi_id('EXO-00-000') + updated_deposit = get_deposit_by_cadi_id('EXO-00-000') - assert updated_deposit == { + assert updated_deposit == { 'version': '0.0.1', 'basic_info': { 'cadi_id': 'EXO-00-001' diff --git a/tests/unit/experiments/test_common_utils.py b/tests/unit/experiments/test_common_utils.py index 23c8012c0d..f7a8c79376 100644 --- a/tests/unit/experiments/test_common_utils.py +++ b/tests/unit/experiments/test_common_utils.py @@ -25,7 +25,7 @@ import subprocess -from mock import patch +from unittest.mock import patch from pytest import raises from cap.modules.experiments.utils.common import kinit diff --git a/tests/unit/mail/test_generate_attrs.py b/tests/unit/mail/test_generate_attrs.py index 3ffb799be4..4ee4fdff83 100644 --- a/tests/unit/mail/test_generate_attrs.py +++ b/tests/unit/mail/test_generate_attrs.py @@ -22,7 +22,7 @@ # waive the privileges and immunities granted to it by virtue of its status # as an Intergovernmental Organization or submit itself to any jurisdiction. -from mock import patch +from unittest.mock import patch from cap.modules.mail.attributes import generate_recipients, generate_body, \ generate_subject diff --git a/tests/unit/mail/test_users.py b/tests/unit/mail/test_users.py index 0ad26af4cd..ec421c007b 100644 --- a/tests/unit/mail/test_users.py +++ b/tests/unit/mail/test_users.py @@ -22,7 +22,7 @@ # waive the privileges and immunities granted to it by virtue of its status # as an Intergovernmental Organization or submit itself to any jurisdiction. """Tests for mail.""" -from mock import patch +from unittest.mock import patch from cap.modules.mail.users import get_all_users, get_users_by_record, \ get_users_by_experiment from cap.modules.mail.custom.recipients import get_owner, get_submitter diff --git a/tests/unit/repos/test_github_api.py b/tests/unit/repos/test_github_api.py index 07201a6776..f5ba91b290 100644 --- a/tests/unit/repos/test_github_api.py +++ b/tests/unit/repos/test_github_api.py @@ -1,5 +1,5 @@ from github import GithubException, UnknownObjectException -from mock import Mock, patch +from unittest.mock import Mock, patch from pytest import raises from cap.modules.repos.errors import (GitError, GitIntegrationError, @@ -11,7 +11,7 @@ @patch.object( Github, 'get_repo', Mock( - side_effect=UnknownObjectException(404, data={'message': 'Not Found'})) + side_effect=UnknownObjectException(404, headers={}, data={'message': 'Not Found'})) ) def test_github_api_when_project_doesnt_exist_or_no_access(): with raises(GitObjectNotFound): @@ -151,7 +151,7 @@ def get_branch(self, name): def test_github_api_with_sha(m_get_repo): class MockProject: def get_branch(self, name): - raise GithubException(404, data={'message': 'Branch not found'}) + raise GithubException(404, headers={}, data={'message': 'Branch not found'}) def get_commit(self, sha): mock = Mock() @@ -179,10 +179,10 @@ def test_github_api_when_commit_or_branch_with_sha_doesnt_exist_raises_GitObject m_get_repo): class MockProject: def get_branch(self, name): - raise GithubException(404, data={'message': 'Branch not found'}) + raise GithubException(404, headers={}, data={'message': 'Branch not found'}) def get_commit(self, sha): - raise GithubException(422, + raise GithubException(422, headers={}, data={'message': 'No commit found for SHA:'}) m_get_repo.return_value = MockProject() @@ -193,7 +193,7 @@ def get_commit(self, sha): @patch.object(Github, 'get_repo', Mock(side_effect=UnknownObjectException( - status=404, data={'message': 'Not Found'}))) + status=404, headers={}, data={'message': 'Not Found'}))) def test_github_api_when_repostiory_doesnt_exist_or_no_access_raises_GitObjectNotFound( ): with raises(GitObjectNotFound): @@ -210,10 +210,10 @@ def default_branch(self): def get_branch(self, name): assert name == 'def-branch' - raise GithubException(404, data={'message': 'Branch not found'}) + raise GithubException(404, headers={}, data={'message': 'Branch not found'}) def get_commit(self, sha): - raise GithubException(422, + raise GithubException(422, headers={}, data={'message': 'No commit found for SHA:'}) m_get_repo.return_value = MockProject() @@ -280,7 +280,7 @@ def get_branch(self, name): return Mock() def get_file_contents(self, filepath, ref): - raise UnknownObjectException(404, data={'message': 'Not Found'}) + raise UnknownObjectException(404, headers={}, data={'message': 'Not Found'}) m_get_repo.return_value = MockProject() @@ -351,6 +351,7 @@ def get_branch(self, name): def create_hook(self, name, config, events, active): raise GithubException( 422, + headers={}, data={ 'message': 'Validation Failed', 'errors': [{ @@ -383,7 +384,7 @@ def get_branch(self, name): return mock def create_hook(self, name, config, events, active): - raise UnknownObjectException(404, data={'message': 'Not Found'}) + raise UnknownObjectException(404, headers={}, data={'message': 'Not Found'}) m_get_repo.return_value = MockProject() @@ -421,7 +422,7 @@ def get_branch(self, name): def get_hook(self, hook_id): assert hook_id == 12345 - raise UnknownObjectException(404, data={'message': 'Not Found'}) + raise UnknownObjectException(404, headers={}, data={'message': 'Not Found'}) m_get_repo.return_value = MockProject() @@ -483,7 +484,7 @@ def get_branch(self, name): def get_hook(self, hook_id): assert hook_id == 123 - raise UnknownObjectException(404, data={'message': 'Not Found'}) + raise UnknownObjectException(404, headers={}, data={'message': 'Not Found'}) m_get_repo.return_value = MockProject() diff --git a/tests/unit/repos/test_gitlab_api.py b/tests/unit/repos/test_gitlab_api.py index 905d33a2da..72f5a58a1d 100644 --- a/tests/unit/repos/test_gitlab_api.py +++ b/tests/unit/repos/test_gitlab_api.py @@ -25,7 +25,7 @@ import responses from gitlab.exceptions import GitlabAuthenticationError, GitlabGetError -from mock import Mock, patch +from unittest.mock import Mock, patch from pytest import raises from cap.modules.repos.errors import (GitIntegrationError, GitObjectNotFound, diff --git a/tests/unit/repos/test_repos_tasks.py b/tests/unit/repos/test_repos_tasks.py index 5d111aba77..1aa6a10c37 100644 --- a/tests/unit/repos/test_repos_tasks.py +++ b/tests/unit/repos/test_repos_tasks.py @@ -2,7 +2,7 @@ import responses from invenio_files_rest.models import ObjectVersion -from mock import Mock, patch +from unittest.mock import Mock, patch from cap.modules.repos.errors import GitObjectNotFound from cap.modules.repos.models import GitSnapshot, GitWebhookSubscriber @@ -75,8 +75,7 @@ def test_download_repo_file(deposit, file_tar): content_type='text/plain', headers={ 'Content-Length': '18', - 'Content-Encoding': 'gzip', - 'Content-Type': 'text/plain; charset=utf-8' + 'Content-Encoding': 'gzip' }, stream=True, status=200) @@ -110,8 +109,7 @@ def test_download_repo_file_when_failed_creates_empty_file_object_with_failed_ta content_type='text/plain', headers={ 'Content-Length': '18', - 'Content-Encoding': 'gzip', - 'Content-Type': 'text/plain; charset=utf-8' + 'Content-Encoding': 'gzip' }, stream=True, status=400) diff --git a/tests/unit/schemas/test_model.py b/tests/unit/schemas/test_model.py index d39b95f32d..09d9636521 100644 --- a/tests/unit/schemas/test_model.py +++ b/tests/unit/schemas/test_model.py @@ -235,11 +235,9 @@ def test_on_save_mapping_is_created_and_index_name_added_to_mappings_map( is_indexed=True, record_mapping={ 'mappings': { - 'doc': { - 'properties': { - "title": { - "type": "text" - } + 'properties': { + "title": { + "type": "text" } } } @@ -247,11 +245,9 @@ def test_on_save_mapping_is_created_and_index_name_added_to_mappings_map( deposit_mapping={ 'mappings': { - 'doc': { - 'properties': { - "keyword": { - "type": "keyword" - } + 'properties': { + "keyword": { + "type": "keyword" } } } @@ -270,11 +266,9 @@ def test_on_save_mapping_is_created_and_index_name_added_to_mappings_map( 'records-cms-schema-v1.0.0': { 'mappings': { - 'doc': { - 'properties': { - 'title': { - 'type': 'text' - } + 'properties': { + 'title': { + 'type': 'text' } } } @@ -286,11 +280,9 @@ def test_on_save_mapping_is_created_and_index_name_added_to_mappings_map( { 'mappings': { - 'doc': { - 'properties': { - 'keyword': { - 'type': 'keyword' - } + 'properties': { + 'keyword': { + 'type': 'keyword' } } } diff --git a/tests/unit/schemas/test_utils.py b/tests/unit/schemas/test_utils.py index 1aa549d27b..940b4a8e61 100644 --- a/tests/unit/schemas/test_utils.py +++ b/tests/unit/schemas/test_utils.py @@ -45,22 +45,18 @@ def test_add_schema_from_fixture_when_schema_does_not_exist_create_new_one( record_options={'title': 'record_options'}, record_mapping={ 'mappings': { - 'doc': { - 'properties': { - "title": { - "type": "text" - } + 'properties': { + "title": { + "type": "text" } } } }, deposit_mapping={ 'mappings': { - 'doc': { - 'properties': { - "keyword": { - "type": "keyword" - } + 'properties': { + "keyword": { + "type": "keyword" } } } @@ -96,22 +92,18 @@ def test_add_schema_from_fixture_when_schema_already_exist_updates_json_for_sche record_options={'title': 'record_options'}, record_mapping={ 'mappings': { - 'doc': { - 'properties': { - "title": { - "type": "text" - } + 'properties': { + "title": { + "type": "text" } } } }, deposit_mapping={ 'mappings': { - 'doc': { - 'properties': { - "keyword": { - "type": "keyword" - } + 'properties': { + "keyword": { + "type": "keyword" } } } @@ -166,7 +158,8 @@ def test_get_indexed_schemas_for_user_when_latest(app, db, users): db.session.add(latest_schema2) db.session.commit() - login_user(users['cms_user']) + with app.test_request_context(): + login_user(users['cms_user']) from cap.modules.schemas.imp import get_indexed_schemas_for_user