From e281248df78bcaa3e5868b1eb89b469f97dab7dc Mon Sep 17 00:00:00 2001 From: Ilias Koutsakis Date: Tue, 17 Nov 2020 20:03:24 +0100 Subject: [PATCH] services: create zenodo deposit through CAP * updates the config, each user has access to their Zenod ccount/token * creates deposit, with metadata * uploads files to deposit * integration tests * closes #1938 * closes #1934 Signed-off-by: Ilias Koutsakis --- cap/config.py | 5 +- cap/modules/deposit/api.py | 127 +++++--- cap/modules/deposit/errors.py | 27 ++ cap/modules/deposit/tasks.py | 56 ++++ cap/modules/deposit/utils.py | 45 +++ cap/modules/services/views/zenodo.py | 27 -- docker-services.yml | 2 + tests/conftest.py | 19 +- tests/integration/test_zenodo_upload.py | 398 ++++++++++++++++++++++++ 9 files changed, 626 insertions(+), 80 deletions(-) create mode 100644 cap/modules/deposit/tasks.py create mode 100644 tests/integration/test_zenodo_upload.py diff --git a/cap/config.py b/cap/config.py index aa306492a1..f15b8090c0 100644 --- a/cap/config.py +++ b/cap/config.py @@ -720,10 +720,7 @@ def _(x): # Zenodo # ====== -ZENODO_SERVER_URL = os.environ.get('APP_ZENODO_SERVER_URL', - 'https://zenodo.org/api') - -ZENODO_ACCESS_TOKEN = os.environ.get('APP_ZENODO_ACCESS_TOKEN', 'CHANGE_ME') +ZENODO_SERVER_URL = os.environ.get('APP_ZENODO_SERVER_URL', 'https://zenodo.org/api') # noqa # Endpoints # ========= diff --git a/cap/modules/deposit/api.py b/cap/modules/deposit/api.py index 50ed9832dd..cfc1f5a40c 100644 --- a/cap/modules/deposit/api.py +++ b/cap/modules/deposit/api.py @@ -49,6 +49,7 @@ from sqlalchemy.orm.exc import NoResultFound from werkzeug.local import LocalProxy +from cap.modules.auth.ext import _fetch_token from cap.modules.deposit.errors import DisconnectWebhookError, FileUploadError from cap.modules.deposit.validators import NoRequiredValidator from cap.modules.experiments.permissions import exp_need_factory @@ -75,6 +76,8 @@ UpdateDepositPermission) from .review import Reviewable +from .tasks import upload_to_zenodo +from .utils import create_zenodo_deposit _datastore = LocalProxy(lambda: current_app.extensions['security'].datastore) @@ -254,53 +257,82 @@ def upload(self, pid, *args, **kwargs): _, rec = request.view_args.get('pid_value').data record_uuid = str(rec.id) data = request.get_json() - webhook = data.get('webhook', False) - event_type = data.get('event_type', 'release') - - try: - url = data['url'] - except KeyError: - raise FileUploadError('Missing url parameter.') - - try: - host, owner, repo, branch, filepath = parse_git_url(url) - api = create_git_api(host, owner, repo, branch, - current_user.id) - - if filepath: - if webhook: - raise FileUploadError( - 'You cannot create a webhook on a file') - - download_repo_file( - record_uuid, - f'repositories/{host}/{owner}/{repo}/{api.branch or api.sha}/{filepath}', # noqa - *api.get_file_download(filepath), - api.auth_headers, - ) - elif webhook: - if event_type == 'release': - if branch: - raise FileUploadError( - 'You cannot create a release webhook' - ' for a specific branch or sha.') + target = data.get('target') + + if target == 'zenodo': + # check for token + token = _fetch_token('zenodo') + if not token: + raise FileUploadError( + 'Please connect your Zenodo account ' + 'before creating a deposit.') + + files = data.get('files') + bucket = data.get('bucket') + zenodo_data = data.get('zenodo_data', {}) + + if files and bucket: + zenodo_deposit = create_zenodo_deposit(token, zenodo_data) # noqa + self.setdefault('_zenodo', []).append(zenodo_deposit) + self.commit() + + # upload files to zenodo deposit + upload_to_zenodo.delay( + files, bucket, token, + zenodo_deposit['id'], + zenodo_deposit['links']['bucket']) + else: + raise FileUploadError( + 'You cannot create an empty Zenodo deposit. ' + 'Please add some files.') + else: + webhook = data.get('webhook', False) + event_type = data.get('event_type', 'release') - if event_type == 'push' and \ - api.branch is None and api.sha: - raise FileUploadError( - 'You cannot create a push webhook' - ' for a specific sha.') + try: + url = data['url'] + except KeyError: + raise FileUploadError('Missing url parameter.') - create_webhook(record_uuid, api, event_type) - else: - download_repo.delay( - record_uuid, - f'repositories/{host}/{owner}/{repo}/{api.branch or api.sha}.tar.gz', # noqa - api.get_repo_download(), - api.auth_headers) + try: + host, owner, repo, branch, filepath = parse_git_url(url) # noqa + api = create_git_api(host, owner, repo, branch, + current_user.id) + + if filepath: + if webhook: + raise FileUploadError( + 'You cannot create a webhook on a file') + + download_repo_file( + record_uuid, + f'repositories/{host}/{owner}/{repo}/{api.branch or api.sha}/{filepath}', # noqa + *api.get_file_download(filepath), + api.auth_headers, + ) + elif webhook: + if event_type == 'release': + if branch: + raise FileUploadError( + 'You cannot create a release webhook' + ' for a specific branch or sha.') + + if event_type == 'push' and \ + api.branch is None and api.sha: + raise FileUploadError( + 'You cannot create a push webhook' + ' for a specific sha.') + + create_webhook(record_uuid, api, event_type) + else: + download_repo.delay( + record_uuid, + f'repositories/{host}/{owner}/{repo}/{api.branch or api.sha}.tar.gz', # noqa + api.get_repo_download(), + api.auth_headers) - except GitError as e: - raise FileUploadError(str(e)) + except GitError as e: + raise FileUploadError(str(e)) return self @@ -584,16 +616,15 @@ def validate(self, **kwargs): validator = NoRequiredValidator(schema, resolver=resolver) - result = {} - result['errors'] = [ + errors = [ FieldError( list(error.path)+error.validator_value, str(error.message)) for error in validator.iter_errors(self) ] - if result['errors']: - raise DepositValidationError(None, errors=result['errors']) + if errors: + raise DepositValidationError(None, errors=errors) except RefResolutionError: raise DepositValidationError('Schema {} not found.'.format( self['$schema'])) diff --git a/cap/modules/deposit/errors.py b/cap/modules/deposit/errors.py index bd48431dcc..b1bdcc2e22 100644 --- a/cap/modules/deposit/errors.py +++ b/cap/modules/deposit/errors.py @@ -87,6 +87,18 @@ def __init__(self, description, **kwargs): self.description = description or self.description +class AuthorizationError(RESTException): + """Exception during authorization.""" + + code = 401 + + def __init__(self, description, **kwargs): + """Initialize exception.""" + super(AuthorizationError, self).__init__(**kwargs) + + self.description = description or self.description + + class DisconnectWebhookError(RESTException): """Exception during disconnecting webhook for analysis.""" @@ -124,3 +136,18 @@ def __init__(self, description, errors=None, **kwargs): self.description = description or self.description self.errors = [FieldError(e[0], e[1]) for e in errors.items()] + + +class DataValidationError(RESTValidationError): + """Review validation error exception.""" + + code = 400 + + description = "Validation error. Try again with valid data" + + def __init__(self, description, errors=None, **kwargs): + """Initialize exception.""" + super(DataValidationError, self).__init__(**kwargs) + + self.description = description or self.description + self.errors = [FieldError(e['field'], e['message']) for e in errors] diff --git a/cap/modules/deposit/tasks.py b/cap/modules/deposit/tasks.py new file mode 100644 index 0000000000..8c3f1e5201 --- /dev/null +++ b/cap/modules/deposit/tasks.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2018 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. +"""Tasks.""" + +from __future__ import absolute_import, print_function + +import requests +from flask import current_app +from celery import shared_task +from invenio_db import db +from invenio_files_rest.models import FileInstance, ObjectVersion + + +@shared_task(autoretry_for=(Exception, ), + retry_kwargs={ + 'max_retries': 5, + 'countdown': 10 + }) +def upload_to_zenodo(files, bucket, token, zenodo_depid, zenodo_bucket_url): + """Upload to Zenodo the files the user selected.""" + for filename in files: + file_obj = ObjectVersion.get(bucket, filename) + file_ins = FileInstance.get(file_obj.file_id) + + with open(file_ins.uri, 'rb') as fp: + file = requests.put( + url=f'{zenodo_bucket_url}/{filename}', + data=fp, + params=dict(access_token=token), + ) + + if not file.ok: + current_app.logger.error( + f'Uploading file {filename} to deposit {zenodo_depid} ' + f'failed with {file.status_code}.') diff --git a/cap/modules/deposit/utils.py b/cap/modules/deposit/utils.py index 6e551cdb52..5e5aa88c49 100644 --- a/cap/modules/deposit/utils.py +++ b/cap/modules/deposit/utils.py @@ -25,9 +25,14 @@ from __future__ import absolute_import, print_function +import requests +from flask import current_app +from flask_login import current_user from invenio_access.models import Role from invenio_db import db +from cap.modules.deposit.errors import AuthorizationError, \ + DataValidationError, FileUploadError from cap.modules.records.utils import url_to_api_url @@ -75,3 +80,43 @@ def add_api_to_links(links): item['links'] = add_api_to_links(item.get('links')) return response + + +def create_zenodo_deposit(token, data): + """Create a Zenodo deposit using the logged in user's credentials.""" + zenodo_url = current_app.config.get("ZENODO_SERVER_URL") + deposit = requests.post( + url=f'{zenodo_url}/deposit/depositions', + params=dict(access_token=token), + json={'metadata': data}, + headers={'Content-Type': 'application/json'} + ) + + if not deposit.ok: + if deposit.status_code == 401: + raise AuthorizationError( + 'Authorization to Zenodo failed. Please reconnect.') + if deposit.status_code == 400: + data = deposit.json() + if data.get('message') == 'Validation error.': + raise DataValidationError( + 'Validation error on creating the Zenodo deposit.', + errors=data.get('errors')) + raise FileUploadError( + 'Something went wrong, Zenodo deposit not created.') + + # TODO: fix with serializers + data = deposit.json() + zenodo_deposit = { + 'id': data['id'], + 'title': data.get('metadata', {}).get('title'), + 'creator': current_user.id, + 'created': data['created'], + 'links': { + 'self': data['links']['self'], + 'bucket': data['links']['bucket'], + 'html': data['links']['html'], + 'publish': data['links']['publish'], + } + } + return zenodo_deposit diff --git a/cap/modules/services/views/zenodo.py b/cap/modules/services/views/zenodo.py index 99f8d78a24..7830cc957b 100644 --- a/cap/modules/services/views/zenodo.py +++ b/cap/modules/services/views/zenodo.py @@ -27,7 +27,6 @@ import requests from flask import current_app, jsonify -from invenio_files_rest.models import FileInstance, ObjectVersion from . import blueprint @@ -48,29 +47,3 @@ def get_zenodo_record(zenodo_id): """Get record from zenodo (route).""" resp, status = _get_zenodo_record(zenodo_id) return jsonify(resp), status - - -@blueprint.route('/zenodo//') -def upload_to_zenodo(bucket_id, filename): - """Upload code to zenodo.""" - zenodo_server_url = current_app.config.get('ZENODO_SERVER_URL') - params = {"access_token": current_app.config.get( - 'ZENODO_ACCESS_TOKEN')} - filename = filename + '.tar.gz' - - r = requests.post(zenodo_server_url, - params=params, json={}, - ) - - file_obj = ObjectVersion.get(bucket_id, filename) - file = FileInstance.get(file_obj.file_id) - - bucket_url = r.json()['links']['bucket'] - with open(file.uri, 'rb') as fp: - response = requests.put( - bucket_url + '/{}'.format(filename), - data=fp, - params=params, - ) - - return jsonify({"status": response.status_code}) diff --git a/docker-services.yml b/docker-services.yml index 670af14627..729dae948e 100644 --- a/docker-services.yml +++ b/docker-services.yml @@ -27,6 +27,8 @@ services: - "INVENIO_RATELIMIT_STORAGE_URL=redis://cache:6379/3" - "INVENIO_CERN_APP_CREDENTIALS_KEY=CHANGE_ME" - "INVENIO_CERN_APP_CREDENTIALS_SECRET=CHANGE_ME" + - "INVENIO_ZENODO_CLIENT_ID=CHANGE_ME" + - "INVENIO_ZENODO_CLIENT_SECRET=CHANGE_ME" - "DEV_HOST=CHANGE_ME" lb: build: ./docker/haproxy/ diff --git a/tests/conftest.py b/tests/conftest.py index eeff7d463d..e54282a475 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,7 @@ import tempfile from datetime import datetime, timedelta from uuid import uuid4 +from six import BytesIO import pytest from flask import current_app @@ -108,7 +109,8 @@ def default_config(): DEBUG=False, TESTING=True, APP_GITLAB_OAUTH_ACCESS_TOKEN='testtoken', - MAIL_DEFAULT_SENDER="analysis-preservation-support@cern.ch") + MAIL_DEFAULT_SENDER="analysis-preservation-support@cern.ch", + ZENODO_SERVER_URL='https://zenodo-test.org') @pytest.fixture(scope='session') @@ -401,6 +403,21 @@ def deposit(example_user, create_deposit): ) +@pytest.fixture +def deposit_with_file(example_user, create_schema, create_deposit): + """New deposit with files.""" + create_schema('test-schema', experiment='CMS') + return create_deposit( + example_user, + 'test-schema', + { + '$ana_type': 'test-schema', + 'title': 'test title' + }, + files={'test-file.txt': BytesIO(b'Hello world!')}, + experiment='CMS') + + @pytest.fixture def record(example_user, create_deposit): """Example record.""" diff --git a/tests/integration/test_zenodo_upload.py b/tests/integration/test_zenodo_upload.py new file mode 100644 index 0000000000..2591c38ca6 --- /dev/null +++ b/tests/integration/test_zenodo_upload.py @@ -0,0 +1,398 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2020 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. +# or submit itself to any jurisdiction. + +"""Integration tests for Zenodo Upload API.""" +import json +import re +from flask import current_app +from invenio_pidstore.resolver import Resolver + +import responses +from mock import patch + +from cap.modules.deposit.api import CAPDeposit + + +@patch('cap.modules.deposit.api._fetch_token', return_value='test-token') +@responses.activate +def test_create_and_upload_to_zenodo(mock_token, app, users, deposit_with_file, + auth_headers_for_user, json_headers): + user = users['cms_user'] + headers = auth_headers_for_user(user) + json_headers + zenodo_server_url = current_app.config.get('ZENODO_SERVER_URL') + pid = deposit_with_file['_deposit']['id'] + bucket = deposit_with_file.files.bucket + + # MOCK RESPONSES FROM ZENODO SERVER + # first the deposit creation + responses.add(responses.POST, + f'{zenodo_server_url}/deposit/depositions', + json={ + 'id': 111, + 'record_id': 111, + 'title': '', + 'links': { + 'bucket': 'http://zenodo-test.com/test-bucket', + 'html': 'https://sandbox.zenodo.org/deposit/111', + 'publish': 'https://sandbox.zenodo.org/api/deposit/depositions/111/actions/publish', + 'self': 'https://sandbox.zenodo.org/api/deposit/depositions/111' + }, + 'files': [], + 'created': '2020-11-20T11:49:39.147767+00:00' + }, + status=200) + + # then the file upload + responses.add(responses.PUT, + 'http://zenodo-test.com/test-bucket/test-file.txt', + json={ + 'mimetype': 'application/octet-stream', + 'links': { + 'self': 'https://sandbox.zenodo.org/api/files/test-bucket/test-file.txt', + 'uploads': 'https://sandbox.zenodo.org/api/files/test-bucket/test-file.txt?uploads' + }, + 'key': 'test-file.txt', + 'size': 100 + }, + status=200) + + # fix because responses makes request to ES, and deposit.commit() won't work without it + responses.add_callback( + responses.PUT, + re.compile(r'http://localhost:9200/deposits-records-test-schema-v1\.0\.0/' + r'test-schema-v1\.0\.0/(.*)?version=(.*)&version_type=external_gte'), + callback=lambda req: (200, {}, json.dumps({})), + content_type='application/json', + ) + + # create the zenodo deposit + with app.test_client() as client: + resp = client.post(f'/deposits/{pid}/actions/upload', + data=json.dumps(dict(target='zenodo', + bucket=str(bucket), + files=['test-file.txt'])), + headers=headers) + assert resp.status_code == 201 + + resolver = Resolver(pid_type='depid', + object_type='rec', + getter=lambda x: x) + _, uuid = resolver.resolve(pid) + record = CAPDeposit.get_record(uuid) + + assert len(record['_zenodo']) == 1 + assert record['_zenodo'][0]['id'] == 111 + assert record['_zenodo'][0]['title'] == None + assert record['_zenodo'][0]['created'] == '2020-11-20T11:49:39.147767+00:00' + + +@patch('cap.modules.deposit.api._fetch_token', return_value='test-token') +@responses.activate +def test_create_and_upload_to_zenodo_with_data(mock_token, app, users, deposit_with_file, + auth_headers_for_user, json_headers): + user = users['cms_user'] + headers = auth_headers_for_user(user) + json_headers + zenodo_server_url = current_app.config.get('ZENODO_SERVER_URL') + pid = deposit_with_file['_deposit']['id'] + bucket = deposit_with_file.files.bucket + + # MOCK RESPONSES FROM ZENODO SERVER + # first the deposit creation + responses.add(responses.POST, + f'{zenodo_server_url}/deposit/depositions', + json={ + 'id': 111, + 'record_id': 111, + 'submitted': False, + 'title': '', + 'links': { + 'bucket': 'http://zenodo-test.com/test-bucket', + 'html': 'https://sandbox.zenodo.org/deposit/111', + 'publish': 'https://sandbox.zenodo.org/api/deposit/depositions/111/actions/publish', + 'self': 'https://sandbox.zenodo.org/api/deposit/depositions/111' + }, + 'metadata': { + 'description': 'This is my first upload', + 'title': 'test-title' + }, + 'files': [], + 'created': '2020-11-20T11:49:39.147767+00:00' + }, + status=200) + + # then the file upload + responses.add(responses.PUT, + 'http://zenodo-test.com/test-bucket/test-file.txt', + json={ + 'mimetype': 'application/octet-stream', + 'links': { + 'self': 'https://sandbox.zenodo.org/api/files/test-bucket/test-file.txt', + 'uploads': 'https://sandbox.zenodo.org/api/files/test-bucket/test-file.txt?uploads' + }, + 'key': 'test-file.txt', + 'size': 100 + }, + status=200) + + # fix because responses makes request to ES, and deposit.commit() won't work without it + responses.add_callback( + responses.PUT, + re.compile(r'http://localhost:9200/deposits-records-test-schema-v1\.0\.0/' + r'test-schema-v1\.0\.0/(.*)?version=(.*)&version_type=external_gte'), + callback=lambda req: (200, {}, json.dumps({})), + content_type='application/json', + ) + + # create the zenodo deposit + with app.test_client() as client: + resp = client.post(f'/deposits/{pid}/actions/upload', + data=json.dumps(dict(target='zenodo', + bucket=str(bucket), + files=['test-file.txt'], + zenodo_data={ + 'title': 'test-title', + 'description': 'This is my first upload' + })), + headers=headers) + assert resp.status_code == 201 + + resolver = Resolver(pid_type='depid', + object_type='rec', + getter=lambda x: x) + _, uuid = resolver.resolve(pid) + record = CAPDeposit.get_record(uuid) + + assert len(record['_zenodo']) == 1 + assert record['_zenodo'][0]['id'] == 111 + assert record['_zenodo'][0]['title'] == 'test-title' + assert record['_zenodo'][0]['created'] == '2020-11-20T11:49:39.147767+00:00' + + +@patch('cap.modules.deposit.api._fetch_token', return_value='test-token') +@responses.activate +def test_create_deposit_with_wrong_data(mock_token, app, users, deposit_with_file, + auth_headers_for_user, json_headers): + user = users['cms_user'] + headers = auth_headers_for_user(user) + json_headers + zenodo_server_url = current_app.config.get('ZENODO_SERVER_URL') + pid = deposit_with_file['_deposit']['id'] + bucket = deposit_with_file.files.bucket + + responses.add(responses.POST, + f'{zenodo_server_url}/deposit/depositions', + json={ + 'status': 400, + 'message': 'Validation error.', + 'errors': [ + {'field': 'test', 'message': 'Unknown field name.'} + ]}, + status=400) + + with app.test_client() as client: + resp = client.post(f'/deposits/{pid}/actions/upload', + data=json.dumps(dict(target='zenodo', + bucket=str(bucket), + files=['test-file.txt'], + zenodo_data={'test': 'test'})), + headers=headers) + assert resp.status_code == 400 + assert resp.json['message'] == 'Validation error on creating the Zenodo deposit.' + assert resp.json['errors'] == [{'field': 'test', 'message': 'Unknown field name.'}] + + +@patch('cap.modules.deposit.api._fetch_token', return_value='test-token') +@responses.activate +def test_zenodo_upload_authorization_failure(mock_token, app, users, deposit_with_file, + auth_headers_for_user, json_headers): + user = users['cms_user'] + headers = auth_headers_for_user(user) + json_headers + zenodo_server_url = current_app.config.get('ZENODO_SERVER_URL') + pid = deposit_with_file['_deposit']['id'] + bucket = deposit_with_file.files.bucket + + responses.add(responses.POST, + f'{zenodo_server_url}/deposit/depositions', + json={}, + status=401) + + with app.test_client() as client: + resp = client.post(f'/deposits/{pid}/actions/upload', + data=json.dumps(dict(target='zenodo', + bucket=str(bucket), + files=['test-file.txt'])), + headers=headers) + assert resp.status_code == 401 + assert resp.json['message'] == 'Authorization to Zenodo failed. Please reconnect.' + + +@patch('cap.modules.deposit.api._fetch_token', return_value='test-token') +@responses.activate +def test_zenodo_upload_deposit_not_created_error(mock_token, app, users, deposit_with_file, + auth_headers_for_user, json_headers): + user = users['cms_user'] + headers = auth_headers_for_user(user) + json_headers + zenodo_server_url = current_app.config.get('ZENODO_SERVER_URL') + pid = deposit_with_file['_deposit']['id'] + bucket = deposit_with_file.files.bucket + + responses.add(responses.POST, + f'{zenodo_server_url}/deposit/depositions', + json={}, + status=500) + + with app.test_client() as client: + resp = client.post(f'/deposits/{pid}/actions/upload', + data=json.dumps(dict(target='zenodo', + bucket=str(bucket), + files=['test-file.txt'])), + headers=headers) + assert resp.status_code == 400 + assert resp.json['message'] == 'Something went wrong, Zenodo deposit not created.' + + +@patch('cap.modules.deposit.api._fetch_token', return_value='test-token') +@responses.activate +def test_zenodo_upload_file_not_uploaded_error(mock_token, app, users, deposit_with_file, + auth_headers_for_user, json_headers, capsys): + user = users['cms_user'] + headers = auth_headers_for_user(user) + json_headers + zenodo_server_url = current_app.config.get('ZENODO_SERVER_URL') + pid = deposit_with_file['_deposit']['id'] + bucket = deposit_with_file.files.bucket + + responses.add(responses.POST, + f'{zenodo_server_url}/deposit/depositions', + json={ + 'id': 111, + 'record_id': 111, + 'submitted': False, + 'title': '', + 'links': { + 'bucket': 'http://zenodo-test.com/test-bucket', + 'html': 'https://sandbox.zenodo.org/deposit/111', + 'publish': 'https://sandbox.zenodo.org/api/deposit/depositions/111/actions/publish', + 'self': 'https://sandbox.zenodo.org/api/deposit/depositions/111' + }, + 'files': [], + 'created': '2020-11-20T11:49:39.147767+00:00' + }, + status=200) + + responses.add(responses.PUT, + 'http://zenodo-test.com/test-bucket/test-file.txt', + json={}, + status=500) + + responses.add_callback( + responses.PUT, + re.compile(r'http://localhost:9200/deposits-records-test-schema-v1\.0\.0/' + r'test-schema-v1\.0\.0/(.*)?version=(.*)&version_type=external_gte'), + callback=lambda req: (200, {}, json.dumps({})), + content_type='application/json', + ) + + with app.test_client() as client: + resp = client.post(f'/deposits/{pid}/actions/upload', + data=json.dumps(dict(target='zenodo', + bucket=str(bucket), + files=['test-file.txt'])), + headers=headers) + assert resp.status_code == 201 + + captured = capsys.readouterr() + assert 'Uploading file test-file.txt to deposit 111 failed with 500' \ + in captured.err + + +@patch('cap.modules.deposit.api._fetch_token', return_value='test-token') +@responses.activate +def test_zenodo_upload_empty_files(mock_token, app, users, deposit_with_file, + auth_headers_for_user, json_headers): + user = users['cms_user'] + zenodo_server_url = current_app.config.get('ZENODO_SERVER_URL') + headers = auth_headers_for_user(user) + json_headers + pid = deposit_with_file['_deposit']['id'] + bucket = deposit_with_file.files.bucket + + responses.add(responses.POST, + f'{zenodo_server_url}/deposit/depositions', + json={ + 'id': 111, + 'record_id': 111, + 'submitted': False, + 'title': '', + 'links': { + 'bucket': 'http://zenodo-test.com/test-bucket', + 'html': 'https://sandbox.zenodo.org/deposit/111', + 'publish': 'https://sandbox.zenodo.org/api/deposit/depositions/111/actions/publish', + 'self': 'https://sandbox.zenodo.org/api/deposit/depositions/111' + }, + 'files': [] + }, + status=200) + + with app.test_client() as client: + resp = client.post(f'/deposits/{pid}/actions/upload', + data=json.dumps(dict(target='zenodo', + bucket=str(bucket), + files=[])), + headers=headers) + assert resp.status_code == 400 + assert resp.json['message'] == 'You cannot create an empty Zenodo deposit. Please add some files.' + + +@patch('cap.modules.deposit.api._fetch_token', return_value=None) +def test_zenodo_upload_no_token(mock_token, app, users, deposit_with_file, + auth_headers_for_user, json_headers): + user = users['cms_user'] + headers = auth_headers_for_user(user) + json_headers + pid = deposit_with_file['_deposit']['id'] + bucket = deposit_with_file.files.bucket + + with app.test_client() as client: + resp = client.post(f'/deposits/{pid}/actions/upload', + data=json.dumps(dict(target='zenodo', + bucket=str(bucket), + files=['test-file.txt'])), + headers=headers) + assert resp.status_code == 400 + assert resp.json['message'] == 'Please connect your Zenodo account before creating a deposit.' + + +@patch('cap.modules.deposit.api._fetch_token', return_value='test-token') +def test_zenodo_upload_no_access(mock_token, app, users, deposit_with_file, + auth_headers_for_user, json_headers): + user = users['lhcb_user'] + headers = auth_headers_for_user(user) + json_headers + pid = deposit_with_file['_deposit']['id'] + bucket = deposit_with_file.files.bucket + + with app.test_client() as client: + resp = client.post(f'/deposits/{pid}/actions/upload', + data=json.dumps(dict(target='zenodo', + bucket=str(bucket), + files=['test-file.txt'])), + headers=headers) + assert resp.status_code == 403