From b0146900b2f5c28c73dfe4da7550510b6a7c0016 Mon Sep 17 00:00:00 2001 From: Spiros Delviniotis Date: Fri, 18 Aug 2017 14:41:16 +0200 Subject: [PATCH] tests: add `clean_dir` to tests that using the DB Addresses #161 Signed-off-by: Spiros Delviniotis --- tests/unit/test_alpha.py | 6 ++++-- tests/unit/test_aps.py | 5 ++++- tests/unit/test_arxiv_all.py | 4 +++- tests/unit/test_arxiv_single.py | 8 ++++++-- tests/unit/test_pipelines.py | 9 +++++++-- tests/unit/test_pos.py | 9 +++++++-- tests/unit/test_world_scientific.py | 11 ++++++++--- 7 files changed, 39 insertions(+), 13 deletions(-) diff --git a/tests/unit/test_alpha.py b/tests/unit/test_alpha.py index eef140b1..f4f21a7f 100644 --- a/tests/unit/test_alpha.py +++ b/tests/unit/test_alpha.py @@ -13,8 +13,10 @@ from hepcrawl.spiders import alpha_spider -from hepcrawl.testlib.fixtures import fake_response_from_file - +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) @pytest.fixture def results(): diff --git a/tests/unit/test_aps.py b/tests/unit/test_aps.py index eb53269d..ecb2ec48 100644 --- a/tests/unit/test_aps.py +++ b/tests/unit/test_aps.py @@ -12,7 +12,10 @@ import pytest from hepcrawl.spiders import aps_spider -from hepcrawl.testlib.fixtures import fake_response_from_file +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) @pytest.fixture diff --git a/tests/unit/test_arxiv_all.py b/tests/unit/test_arxiv_all.py index db8c1354..daed7f9c 100644 --- a/tests/unit/test_arxiv_all.py +++ b/tests/unit/test_arxiv_all.py @@ -47,7 +47,9 @@ def many_results(spider): assert records pipeline = InspireCeleryPushPipeline() pipeline.open_spider(spider) - return [pipeline.process_item(record, spider) for record in records] + yield [pipeline.process_item(record, spider) for record in records] + + clean_dir() def test_page_nr(many_results): diff --git a/tests/unit/test_arxiv_single.py b/tests/unit/test_arxiv_single.py index a6ed66d6..f1edfc5e 100644 --- a/tests/unit/test_arxiv_single.py +++ b/tests/unit/test_arxiv_single.py @@ -17,7 +17,10 @@ from hepcrawl.pipelines import InspireCeleryPushPipeline from hepcrawl.spiders import arxiv_spider -from hepcrawl.testlib.fixtures import fake_response_from_file +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) @pytest.fixture @@ -45,8 +48,9 @@ def results(): validate(processed_record, 'hep') processed_records.append(processed_record) - return processed_records + yield processed_records + clean_dir() def test_abstracts(results): diff --git a/tests/unit/test_pipelines.py b/tests/unit/test_pipelines.py index 050df092..08b81319 100644 --- a/tests/unit/test_pipelines.py +++ b/tests/unit/test_pipelines.py @@ -21,7 +21,10 @@ from hepcrawl.spiders import arxiv_spider from hepcrawl.pipelines import InspireAPIPushPipeline -from hepcrawl.testlib.fixtures import fake_response_from_file +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) @pytest.fixture @@ -44,7 +47,9 @@ def json_spider_record(tmpdir): ) parsed_record = items.next() assert parsed_record - return spider, parsed_record + yield spider, parsed_record + + clean_dir() @pytest.fixture diff --git a/tests/unit/test_pos.py b/tests/unit/test_pos.py index 20c872f4..fb39e143 100644 --- a/tests/unit/test_pos.py +++ b/tests/unit/test_pos.py @@ -19,7 +19,10 @@ from hepcrawl.pipelines import InspireCeleryPushPipeline from hepcrawl.spiders import pos_spider -from hepcrawl.testlib.fixtures import fake_response_from_file +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) @pytest.fixture @@ -52,7 +55,9 @@ def record(scrape_pos_page_body): pipeline = InspireCeleryPushPipeline() pipeline.open_spider(spider) record = request.callback(response) - return pipeline.process_item(record, spider) + yield pipeline.process_item(record, spider) + + clean_dir() def test_titles(record): diff --git a/tests/unit/test_world_scientific.py b/tests/unit/test_world_scientific.py index 36438ab4..ce231be4 100644 --- a/tests/unit/test_world_scientific.py +++ b/tests/unit/test_world_scientific.py @@ -18,7 +18,10 @@ from hepcrawl.pipelines import InspireCeleryPushPipeline from hepcrawl.spiders import wsp_spider -from hepcrawl.testlib.fixtures import fake_response_from_file +from hepcrawl.testlib.fixtures import ( + fake_response_from_file, + clean_dir, +) def create_spider(): @@ -44,11 +47,13 @@ def get_records(response_file_name): pipeline = InspireCeleryPushPipeline() pipeline.open_spider(spider) - return (pipeline.process_item(record, spider) for record in records) + yield (pipeline.process_item(record, spider) for record in records) + + clean_dir() def get_one_record(response_file_name): - results = get_records(response_file_name) + results = get_records(response_file_name).next() return results.next()