test.desy: fix urls and fixtures

Signed-off-by: David Caro <[email protected]>
inspirehep · Aug 24, 2017 · ec4bb9a · ec4bb9a
1 parent 82fe872
commit ec4bb9a
Show file tree

Hide file tree

Showing 4 changed files with 223 additions and 109 deletions.
diff --git a/tests/functional/desy/fixtures/desy_ftp_records.json b/tests/functional/desy/fixtures/desy_ftp_records.json
@@ -34,7 +34,7 @@
     }
   ],
   "self": {
-    "$ref": "http://inspirehep.net/api/literature/111111"
+    "$ref": "https://labs.inspirehep.net/api/literature/111111"
   },
   "number_of_pages": 6,
   "titles": [
@@ -111,7 +111,7 @@
     }
   ],
   "self": {
-    "$ref": "http://inspirehep.net/api/literature/222222"
+    "$ref": "https://labs.inspirehep.net/api/literature/222222"
   },
   "number_of_pages": 6,
   "titles": [
@@ -188,7 +188,7 @@
     }
   ],
   "self": {
-    "$ref": "http://inspirehep.net/api/literature/333333"
+    "$ref": "https://labs.inspirehep.net/api/literature/333333"
   },
   "number_of_pages": 6,
   "titles": [
@@ -265,7 +265,7 @@
     }
   ],
   "self": {
-    "$ref": "http://inspirehep.net/api/literature/444444"
+    "$ref": "https://labs.inspirehep.net/api/literature/444444"
   },
   "number_of_pages": 6,
   "titles": [

diff --git a/tests/functional/desy/fixtures/desy_local_records.json b/tests/functional/desy/fixtures/desy_local_records.json
@@ -34,7 +34,7 @@
     }
   ],
   "self": {
-    "$ref": "http://inspirehep.net/api/literature/111111"
+    "$ref": "https://labs.inspirehep.net/api/literature/111111"
   },
   "number_of_pages": 6,
   "titles": [
@@ -111,7 +111,7 @@
     }
   ],
   "self": {
-    "$ref": "http://inspirehep.net/api/literature/222222"
+    "$ref": "https://labs.inspirehep.net/api/literature/222222"
   },
   "number_of_pages": 6,
   "titles": [
@@ -188,7 +188,7 @@
     }
   ],
   "self": {
-    "$ref": "http://inspirehep.net/api/literature/333333"
+    "$ref": "https://labs.inspirehep.net/api/literature/333333"
   },
   "number_of_pages": 6,
   "titles": [
@@ -265,7 +265,7 @@
     }
   ],
   "self": {
-    "$ref": "http://inspirehep.net/api/literature/444444"
+    "$ref": "https://labs.inspirehep.net/api/literature/444444"
   },
   "number_of_pages": 6,
   "titles": [

diff --git a/tests/functional/desy/test_desy.py b/tests/functional/desy/test_desy.py
@@ -28,12 +28,14 @@
 
 def override_generated_fields(record):
     record['acquisition_source']['datetime'] = u'2017-04-03T10:26:40.365216'
-    record['acquisition_source']['submission_number'] = u'5652c7f6190f11e79e8000224dabeaad'
+    record['acquisition_source']['submission_number'] = (
+        u'5652c7f6190f11e79e8000224dabeaad'
+    )
 
     return record
 
 
-def compare_two_files_using_md5(file_1, file_2):
+def assert_files_equal(file_1, file_2):
     """Compares two files calculating the md5 hash."""
     def _generate_md5_hash(file_path):
         hasher = hashlib.md5()
@@ -42,11 +44,13 @@ def _generate_md5_hash(file_path):
             hasher.update(buf)
             return hasher.hexdigest()
 
-    return _generate_md5_hash(file_1) == _generate_md5_hash(file_2)
+    file_1_hash = _generate_md5_hash(file_1)
+    file_2_hash = _generate_md5_hash(file_2)
+    assert file_1_hash == file_2_hash
 
 
 @pytest.fixture(scope="function")
-def get_fft_1_path():
+def fft_1_path():
     return get_test_suite_path(
         'desy',
         'fixtures',
@@ -59,7 +63,7 @@ def get_fft_1_path():
 
 
 @pytest.fixture(scope="function")
-def get_fft_2_path():
+def fft_2_path():
     return get_test_suite_path(
         'desy',
         'fixtures',
@@ -72,7 +76,7 @@ def get_fft_2_path():
 
 
 @pytest.fixture(scope="function")
-def set_up_ftp_environment():
+def ftp_environment():
     netrc_location = get_test_suite_path(
         'desy',
         'fixtures',
@@ -81,7 +85,8 @@ def set_up_ftp_environment():
         test_suite='functional',
     )
 
-    # The test must wait until the docker environment is up (takes about 10 seconds).
+    # The test must wait until the docker environment is up (takes about 10
+    # seconds).
     sleep(10)
 
     yield {
@@ -98,7 +103,7 @@ def set_up_ftp_environment():
 
 
 @pytest.fixture(scope="function")
-def set_up_local_environment():
+def local_environment():
     package_location = get_test_suite_path(
         'desy',
         'fixtures',
@@ -133,36 +138,41 @@ def set_up_local_environment():
     ]
 )
 def test_desy_ftp(
-        set_up_ftp_environment,
+        ftp_environment,
         expected_results,
-        get_fft_1_path,
-        get_fft_2_path,
+        fft_1_path,
+        fft_2_path,
 ):
-    crawler = get_crawler_instance(set_up_ftp_environment.get('CRAWLER_HOST_URL'))
+    crawler = get_crawler_instance(
+        ftp_environment.get('CRAWLER_HOST_URL')
+    )
 
     results = CeleryMonitor.do_crawl(
         app=celery_app,
         monitor_timeout=5,
         monitor_iter_limit=100,
         events_limit=2,
         crawler_instance=crawler,
-        project=set_up_ftp_environment.get('CRAWLER_PROJECT'),
+        project=ftp_environment.get('CRAWLER_PROJECT'),
         spider='desy',
         settings={},
-        **set_up_ftp_environment.get('CRAWLER_ARGUMENTS')
+        **ftp_environment.get('CRAWLER_ARGUMENTS')
     )
 
     gotten_results = [override_generated_fields(result) for result in results]
-    expected_results = [override_generated_fields(expected) for expected in expected_results]
+    expected_results = [
+        override_generated_fields(expected)
+        for expected in expected_results
+    ]
 
     assert sorted(gotten_results) == expected_results
 
     # Check using MD5 Hash if downloaded files are there.
     for record in expected_results:
         fft_file_paths = sorted(record['_fft'])
 
-        assert compare_two_files_using_md5(fft_file_paths[0]['path'], get_fft_2_path)
-        assert compare_two_files_using_md5(fft_file_paths[1]['path'], get_fft_1_path)
+        assert_files_equal(fft_file_paths[0]['path'], fft_2_path)
+        assert_files_equal(fft_file_paths[1]['path'], fft_1_path)
 
 
 @pytest.mark.parametrize(
@@ -179,33 +189,36 @@ def test_desy_ftp(
     ]
 )
 def test_desy_local_package_path(
-        set_up_local_environment,
+        local_environment,
         expected_results,
-        get_fft_1_path,
-        get_fft_2_path,
+        fft_1_path,
+        fft_2_path,
 ):
-    crawler = get_crawler_instance(set_up_local_environment.get('CRAWLER_HOST_URL'))
+    crawler = get_crawler_instance(local_environment.get('CRAWLER_HOST_URL'))
 
     results = CeleryMonitor.do_crawl(
         app=celery_app,
         monitor_timeout=5,
         monitor_iter_limit=100,
         events_limit=2,
         crawler_instance=crawler,
-        project=set_up_local_environment.get('CRAWLER_PROJECT'),
+        project=local_environment.get('CRAWLER_PROJECT'),
         spider='desy',
         settings={},
-        **set_up_local_environment.get('CRAWLER_ARGUMENTS')
+        **local_environment.get('CRAWLER_ARGUMENTS')
     )
 
     gotten_results = [override_generated_fields(result) for result in results]
-    expected_results = [override_generated_fields(expected) for expected in expected_results]
+    expected_results = [
+        override_generated_fields(expected)
+        for expected in expected_results
+    ]
 
     assert sorted(gotten_results) == expected_results
 
     # Check using MD5 Hash if downloaded files are there.
     for record in expected_results:
         fft_file_paths = sorted(record['_fft'])
 
-        assert compare_two_files_using_md5(fft_file_paths[0]['path'], get_fft_2_path)
-        assert compare_two_files_using_md5(fft_file_paths[1]['path'], get_fft_1_path)
+        assert_files_equal(fft_file_paths[0]['path'], fft_2_path)
+        assert_files_equal(fft_file_paths[1]['path'], fft_1_path)