tests: add clean_dir to tests that using the DB

Addresses inspirehep#161

Signed-off-by: Spiros Delviniotis <[email protected]>

tests/unit/test_alpha.py

@@ -13,8 +13,10 @@
 
 from hepcrawl.spiders import alpha_spider
 
-from hepcrawl.testlib.fixtures import fake_response_from_file
-
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 @pytest.fixture
 def results():

tests/unit/test_aps.py

@@ -12,7 +12,10 @@
 import pytest
 
 from hepcrawl.spiders import aps_spider
-from hepcrawl.testlib.fixtures import fake_response_from_file
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 
 @pytest.fixture

tests/unit/test_arxiv_all.py

@@ -47,7 +47,9 @@ def many_results(spider):
     assert records
     pipeline = InspireCeleryPushPipeline()
     pipeline.open_spider(spider)
-    return [pipeline.process_item(record, spider) for record in records]
+    yield [pipeline.process_item(record, spider) for record in records]
+
+    clean_dir()
 
 
 def test_page_nr(many_results):

tests/unit/test_arxiv_single.py

@@ -17,7 +17,10 @@
 
 from hepcrawl.pipelines import InspireCeleryPushPipeline
 from hepcrawl.spiders import arxiv_spider
-from hepcrawl.testlib.fixtures import fake_response_from_file
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 
 @pytest.fixture
@@ -45,8 +48,9 @@ def results():
         validate(processed_record, 'hep')
         processed_records.append(processed_record)
 
-    return processed_records
+    yield processed_records
 
+    clean_dir()
 
 
 def test_abstracts(results):

tests/unit/test_pipelines.py

@@ -21,7 +21,10 @@
 from hepcrawl.spiders import arxiv_spider
 from hepcrawl.pipelines import InspireAPIPushPipeline
 
-from hepcrawl.testlib.fixtures import fake_response_from_file
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 
 @pytest.fixture
@@ -44,7 +47,9 @@ def json_spider_record(tmpdir):
     )
     parsed_record = items.next()
     assert parsed_record
-    return spider, parsed_record
+    yield spider, parsed_record
+
+    clean_dir()
 
 
 @pytest.fixture

tests/unit/test_pos.py

@@ -19,7 +19,10 @@
 from hepcrawl.pipelines import InspireCeleryPushPipeline
 from hepcrawl.spiders import pos_spider
 
-from hepcrawl.testlib.fixtures import fake_response_from_file
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 
 @pytest.fixture
@@ -52,7 +55,9 @@ def record(scrape_pos_page_body):
     pipeline = InspireCeleryPushPipeline()
     pipeline.open_spider(spider)
     record = request.callback(response)
-    return pipeline.process_item(record, spider)
+    yield pipeline.process_item(record, spider)
+
+    clean_dir()
 
 
 def test_titles(record):

tests/unit/test_world_scientific.py

@@ -18,7 +18,10 @@
 from hepcrawl.pipelines import InspireCeleryPushPipeline
 from hepcrawl.spiders import wsp_spider
 
-from hepcrawl.testlib.fixtures import fake_response_from_file
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 
 def create_spider():
@@ -44,11 +47,13 @@ def get_records(response_file_name):
     pipeline = InspireCeleryPushPipeline()
     pipeline.open_spider(spider)
 
-    return (pipeline.process_item(record, spider) for record in records)
+    yield (pipeline.process_item(record, spider) for record in records)
+
+    clean_dir()
 
 
 def get_one_record(response_file_name):
-    results = get_records(response_file_name)
+    results = get_records(response_file_name).next()
     return results.next()