tests: add clean_dir to tests that using the DB

Addresses inspirehep#161

Signed-off-by: Spiros Delviniotis <[email protected]>

tests/unit/test_alpha.py

@@ -13,8 +13,10 @@
 
 from hepcrawl.spiders import alpha_spider
 
-from hepcrawl.testlib.fixtures import fake_response_from_file
-
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 @pytest.fixture
 def results():

tests/unit/test_aps.py

@@ -12,7 +12,10 @@
 import pytest
 
 from hepcrawl.spiders import aps_spider
-from hepcrawl.testlib.fixtures import fake_response_from_file
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 
 @pytest.fixture

tests/unit/test_arxiv_all.py

@@ -16,7 +16,10 @@
 
 from hepcrawl.pipelines import InspireCeleryPushPipeline
 from hepcrawl.spiders import arxiv_spider
-from hepcrawl.testlib.fixtures import fake_response_from_file
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 
 @pytest.fixture
@@ -48,8 +51,9 @@ def _get_processed_record(item, spider):
     pipeline = InspireCeleryPushPipeline()
     pipeline.open_spider(spider)
 
-    return [_get_processed_record(parsed_item, spider) for parsed_item in parsed_items]
+    yield [_get_processed_record(parsed_item, spider) for parsed_item in parsed_items]
 
+    clean_dir()
 
 def test_page_nr(many_results):
     """Test extracting page_nr"""

tests/unit/test_arxiv_single.py

@@ -17,7 +17,10 @@
 
 from hepcrawl.pipelines import InspireCeleryPushPipeline
 from hepcrawl.spiders import arxiv_spider
-from hepcrawl.testlib.fixtures import fake_response_from_file
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 
 @pytest.fixture
@@ -44,8 +47,9 @@ def _get_processed_item(item, spider):
     pipeline = InspireCeleryPushPipeline()
     pipeline.open_spider(spider)
 
-    return [_get_processed_item(parsed_item, spider) for parsed_item in parsed_items]
+    yield [_get_processed_item(parsed_item, spider) for parsed_item in parsed_items]
 
+    clean_dir()
 
 
 def test_abstracts(results):

tests/unit/test_pipelines.py

@@ -21,7 +21,10 @@
 from hepcrawl.spiders import arxiv_spider
 from hepcrawl.pipelines import InspireAPIPushPipeline
 
-from hepcrawl.testlib.fixtures import fake_response_from_file
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 
 @pytest.fixture
@@ -44,7 +47,9 @@ def json_spider_record(tmpdir):
     )
     parsed_record = items.next()
     assert parsed_record
-    return spider, parsed_record
+    yield spider, parsed_record
+
+    clean_dir()
 
 
 @pytest.fixture

tests/unit/test_pos.py

@@ -19,7 +19,10 @@
 from hepcrawl.pipelines import InspireCeleryPushPipeline
 from hepcrawl.spiders import pos_spider
 
-from hepcrawl.testlib.fixtures import fake_response_from_file
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 
 @pytest.fixture
@@ -55,7 +58,9 @@ def record(scrape_pos_page_body):
     parsed_record = pipeline.process_item(parsed_item, spider)
     assert parsed_record
 
-    return parsed_record
+    yield parsed_record
+
+    clean_dir()
 
 
 def test_titles(record):

tests/unit/test_world_scientific.py

@@ -18,7 +18,10 @@
 from hepcrawl.pipelines import InspireCeleryPushPipeline
 from hepcrawl.spiders import wsp_spider
 
-from hepcrawl.testlib.fixtures import fake_response_from_file
+from hepcrawl.testlib.fixtures import (
+    fake_response_from_file,
+    clean_dir,
+)
 
 
 def create_spider():
@@ -44,7 +47,9 @@ def get_records(response_file_name):
     pipeline = InspireCeleryPushPipeline()
     pipeline.open_spider(spider)
 
-    return (pipeline.process_item(record, spider) for record in records)
+    yield (pipeline.process_item(record, spider) for record in records)
+
+    clean_dir()
 
 
 def get_one_record(response_file_name):