From 026bdfb985d1d5e29cf54612eefabc5a05d8ed00 Mon Sep 17 00:00:00 2001 From: Spiros Delviniotis Date: Fri, 18 Aug 2017 14:41:16 +0200 Subject: [PATCH] tests: add `clean_dir` to tests that using the DB Addresses #161 Signed-off-by: Spiros Delviniotis --- tests/unit/test_alpha.py | 6 ++++-- tests/unit/test_aps.py | 5 ++++- tests/unit/test_arxiv_all.py | 8 ++++++-- tests/unit/test_arxiv_single.py | 8 ++++++-- tests/unit/test_pipelines.py | 9 +++++++-- tests/unit/test_pos.py | 9 +++++++-- tests/unit/test_world_scientific.py | 9 +++++++-- 7 files changed, 41 insertions(+), 13 deletions(-) diff --git a/tests/unit/test_alpha.py b/tests/unit/test_alpha.py index 96bf9af1..d5ff6331 100644 --- a/tests/unit/test_alpha.py +++ b/tests/unit/test_alpha.py @@ -13,8 +13,10 @@ from hepcrawl.spiders import alpha_spider -from hepcrawl.testlib.fixtures import fake_response_from_file - +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) @pytest.fixture def results(): diff --git a/tests/unit/test_aps.py b/tests/unit/test_aps.py index 3bb3698c..e8e64962 100644 --- a/tests/unit/test_aps.py +++ b/tests/unit/test_aps.py @@ -12,7 +12,10 @@ import pytest from hepcrawl.spiders import aps_spider -from hepcrawl.testlib.fixtures import fake_response_from_file +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) @pytest.fixture diff --git a/tests/unit/test_arxiv_all.py b/tests/unit/test_arxiv_all.py index 45689def..63ce0822 100644 --- a/tests/unit/test_arxiv_all.py +++ b/tests/unit/test_arxiv_all.py @@ -16,7 +16,10 @@ from hepcrawl.pipelines import InspireCeleryPushPipeline from hepcrawl.spiders import arxiv_spider -from hepcrawl.testlib.fixtures import fake_response_from_file +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) @pytest.fixture @@ -48,8 +51,9 @@ def _get_processed_record(item, spider): pipeline = InspireCeleryPushPipeline() pipeline.open_spider(spider) - return [_get_processed_record(parsed_item, spider) for parsed_item in parsed_items] + yield [_get_processed_record(parsed_item, spider) for parsed_item in parsed_items] + clean_dir() def test_page_nr(many_results): """Test extracting page_nr""" diff --git a/tests/unit/test_arxiv_single.py b/tests/unit/test_arxiv_single.py index 329a2a49..709c6d9a 100644 --- a/tests/unit/test_arxiv_single.py +++ b/tests/unit/test_arxiv_single.py @@ -17,7 +17,10 @@ from hepcrawl.pipelines import InspireCeleryPushPipeline from hepcrawl.spiders import arxiv_spider -from hepcrawl.testlib.fixtures import fake_response_from_file +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) @pytest.fixture @@ -44,8 +47,9 @@ def _get_processed_item(item, spider): pipeline = InspireCeleryPushPipeline() pipeline.open_spider(spider) - return [_get_processed_item(parsed_item, spider) for parsed_item in parsed_items] + yield [_get_processed_item(parsed_item, spider) for parsed_item in parsed_items] + clean_dir() def test_abstracts(results): diff --git a/tests/unit/test_pipelines.py b/tests/unit/test_pipelines.py index 050df092..08b81319 100644 --- a/tests/unit/test_pipelines.py +++ b/tests/unit/test_pipelines.py @@ -21,7 +21,10 @@ from hepcrawl.spiders import arxiv_spider from hepcrawl.pipelines import InspireAPIPushPipeline -from hepcrawl.testlib.fixtures import fake_response_from_file +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) @pytest.fixture @@ -44,7 +47,9 @@ def json_spider_record(tmpdir): ) parsed_record = items.next() assert parsed_record - return spider, parsed_record + yield spider, parsed_record + + clean_dir() @pytest.fixture diff --git a/tests/unit/test_pos.py b/tests/unit/test_pos.py index bea29b34..0eccd4fc 100644 --- a/tests/unit/test_pos.py +++ b/tests/unit/test_pos.py @@ -19,7 +19,10 @@ from hepcrawl.pipelines import InspireCeleryPushPipeline from hepcrawl.spiders import pos_spider -from hepcrawl.testlib.fixtures import fake_response_from_file +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) @pytest.fixture @@ -55,7 +58,9 @@ def record(scrape_pos_page_body): parsed_record = pipeline.process_item(parsed_item, spider) assert parsed_record - return parsed_record + yield parsed_record + + clean_dir() def test_titles(record): diff --git a/tests/unit/test_world_scientific.py b/tests/unit/test_world_scientific.py index 291d00d0..38adc79b 100644 --- a/tests/unit/test_world_scientific.py +++ b/tests/unit/test_world_scientific.py @@ -18,7 +18,10 @@ from hepcrawl.pipelines import InspireCeleryPushPipeline from hepcrawl.spiders import wsp_spider -from hepcrawl.testlib.fixtures import fake_response_from_file +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) def create_spider(): @@ -44,7 +47,9 @@ def get_records(response_file_name): pipeline = InspireCeleryPushPipeline() pipeline.open_spider(spider) - return (pipeline.process_item(record, spider) for record in records) + yield (pipeline.process_item(record, spider) for record in records) + + clean_dir() def get_one_record(response_file_name):