diff --git a/tests/functional/desy/fixtures/desy_ftp_records.json b/tests/functional/desy/fixtures/desy_ftp_records.json index 0ffb18d1..8f7302c4 100644 --- a/tests/functional/desy/fixtures/desy_ftp_records.json +++ b/tests/functional/desy/fixtures/desy_ftp_records.json @@ -34,7 +34,7 @@ } ], "self": { - "$ref": "http://inspirehep.net/api/literature/111111" + "$ref": "https://labs.inspirehep.net/api/literature/111111" }, "number_of_pages": 6, "titles": [ @@ -111,7 +111,7 @@ } ], "self": { - "$ref": "http://inspirehep.net/api/literature/222222" + "$ref": "https://labs.inspirehep.net/api/literature/222222" }, "number_of_pages": 6, "titles": [ @@ -188,7 +188,7 @@ } ], "self": { - "$ref": "http://inspirehep.net/api/literature/333333" + "$ref": "https://labs.inspirehep.net/api/literature/333333" }, "number_of_pages": 6, "titles": [ @@ -265,7 +265,7 @@ } ], "self": { - "$ref": "http://inspirehep.net/api/literature/444444" + "$ref": "https://labs.inspirehep.net/api/literature/444444" }, "number_of_pages": 6, "titles": [ diff --git a/tests/functional/desy/fixtures/desy_local_records.json b/tests/functional/desy/fixtures/desy_local_records.json index 4197a456..02f8fad2 100644 --- a/tests/functional/desy/fixtures/desy_local_records.json +++ b/tests/functional/desy/fixtures/desy_local_records.json @@ -34,7 +34,7 @@ } ], "self": { - "$ref": "http://inspirehep.net/api/literature/111111" + "$ref": "https://labs.inspirehep.net/api/literature/111111" }, "number_of_pages": 6, "titles": [ @@ -111,7 +111,7 @@ } ], "self": { - "$ref": "http://inspirehep.net/api/literature/222222" + "$ref": "https://labs.inspirehep.net/api/literature/222222" }, "number_of_pages": 6, "titles": [ @@ -188,7 +188,7 @@ } ], "self": { - "$ref": "http://inspirehep.net/api/literature/333333" + "$ref": "https://labs.inspirehep.net/api/literature/333333" }, "number_of_pages": 6, "titles": [ @@ -265,7 +265,7 @@ } ], "self": { - "$ref": "http://inspirehep.net/api/literature/444444" + "$ref": "https://labs.inspirehep.net/api/literature/444444" }, "number_of_pages": 6, "titles": [ diff --git a/tests/functional/desy/test_desy.py b/tests/functional/desy/test_desy.py index 5c3f4929..464907de 100644 --- a/tests/functional/desy/test_desy.py +++ b/tests/functional/desy/test_desy.py @@ -28,12 +28,14 @@ def override_generated_fields(record): record['acquisition_source']['datetime'] = u'2017-04-03T10:26:40.365216' - record['acquisition_source']['submission_number'] = u'5652c7f6190f11e79e8000224dabeaad' + record['acquisition_source']['submission_number'] = ( + u'5652c7f6190f11e79e8000224dabeaad' + ) return record -def compare_two_files_using_md5(file_1, file_2): +def assert_files_equal(file_1, file_2): """Compares two files calculating the md5 hash.""" def _generate_md5_hash(file_path): hasher = hashlib.md5() @@ -42,11 +44,13 @@ def _generate_md5_hash(file_path): hasher.update(buf) return hasher.hexdigest() - return _generate_md5_hash(file_1) == _generate_md5_hash(file_2) + file_1_hash = _generate_md5_hash(file_1) + file_2_hash = _generate_md5_hash(file_2) + assert file_1_hash == file_2_hash @pytest.fixture(scope="function") -def get_fft_1_path(): +def fft_1_path(): return get_test_suite_path( 'desy', 'fixtures', @@ -59,7 +63,7 @@ def get_fft_1_path(): @pytest.fixture(scope="function") -def get_fft_2_path(): +def fft_2_path(): return get_test_suite_path( 'desy', 'fixtures', @@ -72,7 +76,7 @@ def get_fft_2_path(): @pytest.fixture(scope="function") -def set_up_ftp_environment(): +def ftp_environment(): netrc_location = get_test_suite_path( 'desy', 'fixtures', @@ -81,7 +85,8 @@ def set_up_ftp_environment(): test_suite='functional', ) - # The test must wait until the docker environment is up (takes about 10 seconds). + # The test must wait until the docker environment is up (takes about 10 + # seconds). sleep(10) yield { @@ -98,7 +103,7 @@ def set_up_ftp_environment(): @pytest.fixture(scope="function") -def set_up_local_environment(): +def local_environment(): package_location = get_test_suite_path( 'desy', 'fixtures', @@ -133,12 +138,14 @@ def set_up_local_environment(): ] ) def test_desy_ftp( - set_up_ftp_environment, + ftp_environment, expected_results, - get_fft_1_path, - get_fft_2_path, + fft_1_path, + fft_2_path, ): - crawler = get_crawler_instance(set_up_ftp_environment.get('CRAWLER_HOST_URL')) + crawler = get_crawler_instance( + ftp_environment.get('CRAWLER_HOST_URL') + ) results = CeleryMonitor.do_crawl( app=celery_app, @@ -146,14 +153,17 @@ def test_desy_ftp( monitor_iter_limit=100, events_limit=2, crawler_instance=crawler, - project=set_up_ftp_environment.get('CRAWLER_PROJECT'), + project=ftp_environment.get('CRAWLER_PROJECT'), spider='desy', settings={}, - **set_up_ftp_environment.get('CRAWLER_ARGUMENTS') + **ftp_environment.get('CRAWLER_ARGUMENTS') ) gotten_results = [override_generated_fields(result) for result in results] - expected_results = [override_generated_fields(expected) for expected in expected_results] + expected_results = [ + override_generated_fields(expected) + for expected in expected_results + ] assert sorted(gotten_results) == expected_results @@ -161,8 +171,8 @@ def test_desy_ftp( for record in expected_results: fft_file_paths = sorted(record['_fft']) - assert compare_two_files_using_md5(fft_file_paths[0]['path'], get_fft_2_path) - assert compare_two_files_using_md5(fft_file_paths[1]['path'], get_fft_1_path) + assert_files_equal(fft_file_paths[0]['path'], fft_2_path) + assert_files_equal(fft_file_paths[1]['path'], fft_1_path) @pytest.mark.parametrize( @@ -179,12 +189,12 @@ def test_desy_ftp( ] ) def test_desy_local_package_path( - set_up_local_environment, + local_environment, expected_results, - get_fft_1_path, - get_fft_2_path, + fft_1_path, + fft_2_path, ): - crawler = get_crawler_instance(set_up_local_environment.get('CRAWLER_HOST_URL')) + crawler = get_crawler_instance(local_environment.get('CRAWLER_HOST_URL')) results = CeleryMonitor.do_crawl( app=celery_app, @@ -192,14 +202,17 @@ def test_desy_local_package_path( monitor_iter_limit=100, events_limit=2, crawler_instance=crawler, - project=set_up_local_environment.get('CRAWLER_PROJECT'), + project=local_environment.get('CRAWLER_PROJECT'), spider='desy', settings={}, - **set_up_local_environment.get('CRAWLER_ARGUMENTS') + **local_environment.get('CRAWLER_ARGUMENTS') ) gotten_results = [override_generated_fields(result) for result in results] - expected_results = [override_generated_fields(expected) for expected in expected_results] + expected_results = [ + override_generated_fields(expected) + for expected in expected_results + ] assert sorted(gotten_results) == expected_results @@ -207,5 +220,5 @@ def test_desy_local_package_path( for record in expected_results: fft_file_paths = sorted(record['_fft']) - assert compare_two_files_using_md5(fft_file_paths[0]['path'], get_fft_2_path) - assert compare_two_files_using_md5(fft_file_paths[1]['path'], get_fft_1_path) + assert_files_equal(fft_file_paths[0]['path'], fft_2_path) + assert_files_equal(fft_file_paths[1]['path'], fft_1_path) diff --git a/tests/unit/test_desy.py b/tests/unit/test_desy.py index 35ed24e2..2b567191 100644 --- a/tests/unit/test_desy.py +++ b/tests/unit/test_desy.py @@ -9,21 +9,30 @@ from __future__ import absolute_import, division, print_function -import pytest import os +import pytest from scrapy.crawler import Crawler from scrapy.http import TextResponse +from scrapy.settings import Settings +from hepcrawl import settings from hepcrawl.pipelines import InspireCeleryPushPipeline from hepcrawl.spiders import desy_spider - from hepcrawl.testlib.fixtures import fake_response_from_file def create_spider(): - crawler = Crawler(spidercls=desy_spider.DesySpider) - return desy_spider.DesySpider.from_crawler(crawler) + custom_settings = Settings() + custom_settings.setmodule(settings) + crawler = Crawler( + spidercls=desy_spider.DesySpider, + settings=custom_settings, + ) + return desy_spider.DesySpider.from_crawler( + crawler, + source_folder='idontexist_but_it_does_not_matter', + ) def get_records(response_file_name): @@ -60,7 +69,9 @@ def get_one_record(response_file_name): def override_generated_fields(record): record['acquisition_source']['datetime'] = '2017-05-04T17:49:07.975168' - record['acquisition_source']['submission_number'] = '5652c7f6190f11e79e8000224dabeaad' + record['acquisition_source']['submission_number'] = ( + '5652c7f6190f11e79e8000224dabeaad' + ) return record @@ -83,9 +94,12 @@ def test_pipeline_record(generated_record): '_fft': [ { 'creation_datetime': '2017-06-27T09:43:17', - 'description': '00013 Decomposition of the problematic rotation curves in our ' - 'sample according to the best-fit \\textsc{core}NFW models. ' - 'Colors and symbols are as in Figure \\ref{fig:dc14_fits}.', + 'description': ( + '00013 Decomposition of the problematic rotation curves ' + 'in our sample according to the best-fit ' + '\\textsc{core}NFW models. Colors and symbols are as in ' + 'Figure \\ref{fig:dc14_fits}.' + ), 'filename': 'cNFW_rogue_curves', 'format': '.txt', 'path': 'FFT/test_fft_1.txt', @@ -94,17 +108,21 @@ def test_pipeline_record(generated_record): }, { 'creation_datetime': '2017-06-27T09:43:16', - 'description': '00005 Comparison of the parameters of the best-fit DC14 models to ' - 'the cosmological halo mass-concentration relation from \\' - 'cite{dutton14} (left) and the stellar mass-halo mass relation ' - 'from \\cite{behroozi13} (right). The error bars correspond to the ' - 'extremal values of the multidimensional 68\\% confidence region ' - 'for each fit. The theoretical relations are shown as red lines ' - 'and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by ' - 'the dark and light grey bands, respectively. The ' - 'mass-concentration relation from \\cite{maccio08} and the stellar' - ' mass-halo mass relation from \\cite{behroozi13} are also shown ' - 'as the black dashed lines.', + 'description': ( + '00005 Comparison of the parameters of the best-fit DC14 ' + 'models to the cosmological halo mass-concentration ' + 'relation from \\cite{dutton14} (left) and the stellar ' + 'mass-halo mass relation from \\cite{behroozi13} (right). ' + 'The error bars correspond to the extremal values of the ' + 'multidimensional 68\\% confidence region for each fit. ' + 'The theoretical relations are shown as red lines and ' + 'their 1$\\sigma$ and 2$\\sigma$ scatter are represented ' + 'by the dark and light grey bands, respectively. The ' + 'mass-concentration relation from \\cite{maccio08} and ' + 'the stellar mass-halo mass relation from ' + '\\cite{behroozi13} are also shown as the black dashed ' + 'lines.' + ), 'filename': 'scalingRelations_DutBeh_DC14_all_Oh', 'format': '.txt', 'path': 'FFT/test_fft_2.txt', @@ -115,19 +133,24 @@ def test_pipeline_record(generated_record): 'abstracts': [ { 'source': 'Deutsches Elektronen-Synchrotron', - 'value': 'Dielectric laser acceleration of electrons has recently been\n' - ' demonstrated with significantly higher accelerating ' - 'gradients than other\n structure-based linear ' - 'accelerators. Towards the development of an integrated 1 MeV\n ' - ' electron accelerator based on dielectric laser accelerator ' - 'technologies,\n development in several relevant ' - 'technologies is needed. In this work, recent\n ' - 'developments on electron sources, bunching, accelerating, focussing, ' - 'deflecting and\n laser coupling structures are reported. ' - 'With an eye to the near future, components\n ' - 'required for a 1 MeV kinetic energy tabletop accelerator producing ' - 'sub-femtosecond\n electron bunches are outlined.\n ' - ' ' + 'value': ( + 'Dielectric laser acceleration of electrons has recently ' + 'been\n demonstrated with significantly ' + 'higher accelerating gradients than other\n ' + ' structure-based linear accelerators. Towards the ' + 'development of an integrated 1 MeV\n ' + 'electron accelerator based on dielectric laser ' + 'accelerator technologies,\n development ' + 'in several relevant technologies is needed. In this ' + 'work, recent\n developments on electron ' + 'sources, bunching, accelerating, focussing, deflecting ' + 'and\n laser coupling structures are ' + 'reported. With an eye to the near future, ' + 'components\n required for a 1 MeV kinetic ' + 'energy tabletop accelerator producing ' + 'sub-femtosecond\n electron bunches are ' + 'outlined.\n ' + ) } ], 'acquisition_source': { @@ -162,21 +185,26 @@ def test_pipeline_record(generated_record): } ], 'self': { - '$ref': 'http://inspirehep.net/api/literature/111111' + '$ref': 'https://labs.inspirehep.net/api/literature/111111' }, 'titles': [ { 'source': 'JACoW', - 'title': 'Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser\n ' - ' Acceleration (DLA) From the Source to Relativistic ' - 'Electrons\n ' + 'title': ( + 'Towards a Fully Integrated Accelerator on a Chip: ' + 'Dielectric Laser\n Acceleration (DLA) ' + 'From the Source to Relativistic Electrons\n ' + ) } ], 'urls': [ { 'description': 'Fulltext', - 'value': 'http://inspirehep.net/record/1608652/files/Towards a fully\n ' - ' integrated acc on a chip.pdf\n ' + 'value': ( + 'http://inspirehep.net/record/1608652/files/Towards ' + 'a fully\n integrated acc on a chip.pdf' + '\n ' + ) } ] } @@ -207,9 +235,12 @@ def test_pipeline_collection_records(generated_records): "_fft": [ { 'creation_datetime': '2017-06-27T09:43:17', - 'description': '00013 Decomposition of the problematic rotation curves in our ' - 'sample according to the best-fit \\textsc{core}NFW models. ' - 'Colors and symbols are as in Figure \\ref{fig:dc14_fits}.', + 'description': ( + '00013 Decomposition of the problematic rotation ' + 'curves in our sample according to the best-fit ' + '\\textsc{core}NFW models. Colors and symbols are as ' + 'in Figure \\ref{fig:dc14_fits}.' + ), 'filename': 'cNFW_rogue_curves', 'format': '.txt', 'path': 'FFT/test_fft_1.txt', @@ -218,17 +249,22 @@ def test_pipeline_collection_records(generated_records): }, { 'creation_datetime': '2017-06-27T09:43:16', - 'description': '00005 Comparison of the parameters of the best-fit DC14 models to ' - 'the cosmological halo mass-concentration relation from \\' - 'cite{dutton14} (left) and the stellar mass-halo mass relation ' - 'from \\cite{behroozi13} (right). The error bars correspond to the ' - 'extremal values of the multidimensional 68\\% confidence region ' - 'for each fit. The theoretical relations are shown as red lines ' - 'and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by ' - 'the dark and light grey bands, respectively. The ' - 'mass-concentration relation from \\cite{maccio08} and the stellar' - ' mass-halo mass relation from \\cite{behroozi13} are also shown ' - 'as the black dashed lines.', + 'description': ( + '00005 Comparison of the parameters of the best-fit ' + 'DC14 models to the cosmological halo ' + 'mass-concentration relation from \\cite{dutton14} ' + '(left) and the stellar mass-halo mass relation from ' + '\\cite{behroozi13} (right). The error bars correspond' + ' to the extremal values of the multidimensional 68\\%' + ' confidence region for each fit. The theoretical ' + 'relations are shown as red lines and their ' + '1$\\sigma$ and 2$\\sigma$ scatter are represented ' + 'by the dark and light grey bands, respectively. The ' + 'mass-concentration relation from \\cite{maccio08} ' + 'and the stellar mass-halo mass relation from ' + '\\cite{behroozi13} are also shown as the black ' + 'dashed lines.' + ), 'filename': 'scalingRelations_DutBeh_DC14_all_Oh', 'format': '.txt', 'path': 'FFT/test_fft_2.txt', @@ -243,19 +279,28 @@ def test_pipeline_collection_records(generated_records): } ], "self": { - "$ref": "http://inspirehep.net/api/literature/111111" + "$ref": "https://labs.inspirehep.net/api/literature/111111" }, "number_of_pages": 6, "titles": [ { "source": "JACoW", - "title": "Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser\n Acceleration (DLA) From the Source to Relativistic Electrons\n " + "title": ( + 'Towards a Fully Integrated Accelerator on a Chip: ' + 'Dielectric Laser\n Acceleration (DLA) ' + 'From the Source to Relativistic Electrons' + '\n ' + ) } ], "urls": [ { "description": "Fulltext", - "value": "http://inspirehep.net/record/1608652/files/Towards a fully\n integrated acc on a chip.pdf\n " + "value": ( + 'http://inspirehep.net/record/1608652/files/' + 'Towards a fully\n integrated acc on a ' + 'chip.pdf\n ' + ) } ], "dois": [ @@ -280,7 +325,25 @@ def test_pipeline_collection_records(generated_records): "abstracts": [ { "source": "Deutsches Elektronen-Synchrotron", - "value": "Dielectric laser acceleration of electrons has recently been\n demonstrated with significantly higher accelerating gradients than other\n structure-based linear accelerators. Towards the development of an integrated 1 MeV\n electron accelerator based on dielectric laser accelerator technologies,\n development in several relevant technologies is needed. In this work, recent\n developments on electron sources, bunching, accelerating, focussing, deflecting and\n laser coupling structures are reported. With an eye to the near future, components\n required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond\n electron bunches are outlined.\n " + "value": ( + "Dielectric laser acceleration of electrons has " + "recently been\n demonstrated with " + "significantly higher accelerating gradients than " + "other\n structure-based linear " + "accelerators. Towards the development of an " + "integrated 1 MeV\n electron " + "accelerator based on dielectric laser accelerator " + "technologies,\n development in " + "several relevant technologies is needed. In this work" + ", recent\n developments on electron " + "sources, bunching, accelerating, focussing, " + "deflecting and\n laser coupling " + "structures are reported. With an eye to the near " + "future, components\n required for a 1 " + "MeV kinetic energy tabletop accelerator producing sub" + "-femtosecond\n electron bunches are " + "outlined.\n " + ) } ] }, @@ -297,9 +360,12 @@ def test_pipeline_collection_records(generated_records): "_fft": [ { 'creation_datetime': '2017-06-27T09:43:17', - 'description': '00013 Decomposition of the problematic rotation curves in our ' - 'sample according to the best-fit \\textsc{core}NFW models. ' - 'Colors and symbols are as in Figure \\ref{fig:dc14_fits}.', + 'description': ( + "00013 Decomposition of the problematic rotation " + "curves in our sample according to the best-fit " + "\\textsc{core}NFW models. Colors and symbols are as " + "in Figure \\ref{fig:dc14_fits}." + ), 'filename': 'cNFW_rogue_curves', 'format': '.txt', 'path': 'FFT/test_fft_1.txt', @@ -308,17 +374,22 @@ def test_pipeline_collection_records(generated_records): }, { 'creation_datetime': '2017-06-27T09:43:16', - 'description': '00005 Comparison of the parameters of the best-fit DC14 models to ' - 'the cosmological halo mass-concentration relation from \\' - 'cite{dutton14} (left) and the stellar mass-halo mass relation ' - 'from \\cite{behroozi13} (right). The error bars correspond to the ' - 'extremal values of the multidimensional 68\\% confidence region ' - 'for each fit. The theoretical relations are shown as red lines ' - 'and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by ' - 'the dark and light grey bands, respectively. The ' - 'mass-concentration relation from \\cite{maccio08} and the stellar' - ' mass-halo mass relation from \\cite{behroozi13} are also shown ' - 'as the black dashed lines.', + 'description': ( + '00005 Comparison of the parameters of the best-fit ' + 'DC14 models to the cosmological halo ' + 'mass-concentration relation from \\cite{dutton14} ' + '(left) and the stellar mass-halo mass relation ' + 'from \\cite{behroozi13} (right). The error bars ' + 'correspond to the extremal values of the ' + 'multidimensional 68\\% confidence region for each ' + 'fit. The theoretical relations are shown as red ' + 'lines and their 1$\\sigma$ and 2$\\sigma$ scatter ' + 'are represented by the dark and light grey bands, ' + 'respectively. The mass-concentration relation ' + 'from \\cite{maccio08} and the stellar mass-halo ' + 'mass relation from \\cite{behroozi13} are also ' + 'shown as the black dashed lines.' + ), 'filename': 'scalingRelations_DutBeh_DC14_all_Oh', 'format': '.txt', 'path': 'FFT/test_fft_2.txt', @@ -333,19 +404,28 @@ def test_pipeline_collection_records(generated_records): } ], "self": { - "$ref": "http://inspirehep.net/api/literature/222222" + "$ref": "https://labs.inspirehep.net/api/literature/222222" }, "number_of_pages": 6, "titles": [ { "source": "JACoW", - "title": "Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser\n Acceleration (DLA) From the Source to Relativistic Electrons\n " + "title": ( + "Towards a Fully Integrated Accelerator on a Chip: " + "Dielectric Laser\n Acceleration " + "(DLA) From the Source to Relativistic Electrons" + "\n " + ) } ], "urls": [ { "description": "Fulltext", - "value": "http://inspirehep.net/record/1608652/files/Towards a fully\n integrated acc on a chip.pdf\n " + "value": ( + "http://inspirehep.net/record/1608652/files/" + "Towards a fully\n integrated acc on a " + "chip.pdf\n " + ) } ], "dois": [ @@ -370,12 +450,33 @@ def test_pipeline_collection_records(generated_records): "abstracts": [ { "source": "Deutsches Elektronen-Synchrotron", - "value": "Dielectric laser acceleration of electrons has recently been\n demonstrated with significantly higher accelerating gradients than other\n structure-based linear accelerators. Towards the development of an integrated 1 MeV\n electron accelerator based on dielectric laser accelerator technologies,\n development in several relevant technologies is needed. In this work, recent\n developments on electron sources, bunching, accelerating, focussing, deflecting and\n laser coupling structures are reported. With an eye to the near future, components\n required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond\n electron bunches are outlined.\n " + "value": ( + "Dielectric laser acceleration of electrons has " + "recently been\n demonstrated with " + "significantly higher accelerating gradients than " + "other\n structure-based linear " + "accelerators. Towards the development of an " + "integrated 1 MeV\n electron " + "accelerator based on dielectric laser accelerator " + "technologies,\n development in " + "several relevant technologies is needed. In this " + "work, recent\n developments on " + "electron sources, bunching, accelerating, " + "focussing, deflecting and\n laser " + "coupling structures are reported. With an eye to " + "the near future, components\n " + "required for a 1 MeV kinetic energy tabletop " + "accelerator producing sub-femtosecond" + "\n electron bunches are outlined." + "\n " + ) } ] } ] - generated_results = [override_generated_fields(rec) for rec in generated_records] + generated_results = [ + override_generated_fields(rec) for rec in generated_records + ] assert generated_results == expected