From 06e106fea6981650641ef20465c2df48394f8d45 Mon Sep 17 00:00:00 2001
From: Spiros Delviniotis <spirosdelviniotis@gmail.com>
Date: Thu, 17 Aug 2017 17:10:46 +0200
Subject: [PATCH] tests: support for `scrapy-crawl-once`

Addresses #161

Signed-off-by: Spiros Delviniotis <spirosdelviniotis@gmail.com>
---
 hepcrawl/testlib/fixtures.py         | 5 +++++
 tests/functional/arxiv/test_arxiv.py | 4 ++++
 tests/functional/wsp/test_wsp.py     | 8 +++-----
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/hepcrawl/testlib/fixtures.py b/hepcrawl/testlib/fixtures.py
index 513b0395..b8e9992b 100644
--- a/hepcrawl/testlib/fixtures.py
+++ b/hepcrawl/testlib/fixtures.py
@@ -11,6 +11,7 @@
 
 import os
 import json
+import shutil
 
 from scrapy.http import Request, TextResponse
 from scrapy.selector import Selector
@@ -131,3 +132,7 @@ def expected_json_results_from_file(*path_chunks, **kwargs):
         expected_data = json.load(fd)
 
     return expected_data
+
+
+def clean_dir(path='/tmp/WSP/'):
+    shutil.rmtree(path, ignore_errors=True)
diff --git a/tests/functional/arxiv/test_arxiv.py b/tests/functional/arxiv/test_arxiv.py
index a9677b89..0538980e 100644
--- a/tests/functional/arxiv/test_arxiv.py
+++ b/tests/functional/arxiv/test_arxiv.py
@@ -11,6 +11,7 @@
 
 from __future__ import absolute_import, division, print_function
 
+import os
 from time import sleep
 import pytest
 
@@ -20,6 +21,7 @@
 from hepcrawl.testlib.fixtures import (
     get_test_suite_path,
     expected_json_results_from_file,
+    clean_dir,
 )
 
 
@@ -51,6 +53,8 @@ def set_up_local_environment():
         }
     }
 
+    clean_dir(path=os.path.join(os.getcwd(), '.scrapy'))
+
 
 @pytest.mark.parametrize(
     'expected_results',
diff --git a/tests/functional/wsp/test_wsp.py b/tests/functional/wsp/test_wsp.py
index 70996466..541b51c2 100644
--- a/tests/functional/wsp/test_wsp.py
+++ b/tests/functional/wsp/test_wsp.py
@@ -13,7 +13,6 @@
 
 import pytest
 import os
-import shutil
 
 from time import sleep
 
@@ -21,6 +20,7 @@
 from hepcrawl.testlib.fixtures import (
     get_test_suite_path,
     expected_json_results_from_file,
+    clean_dir,
 )
 from hepcrawl.testlib.tasks import app as celery_app
 from hepcrawl.testlib.utils import get_crawler_instance
@@ -56,6 +56,7 @@ def set_up_ftp_environment():
     }
 
     clean_dir()
+    clean_dir(path=os.path.join(os.getcwd(), '.scrapy'))
 
 
 @pytest.fixture(scope="function")
@@ -81,6 +82,7 @@ def set_up_local_environment():
 
 def remove_generated_files(package_location):
     clean_dir()
+    clean_dir(path=os.path.join(os.getcwd(), '.scrapy'))
 
     _, dirs, files = next(os.walk(package_location))
     for dir_name in dirs:
@@ -90,10 +92,6 @@ def remove_generated_files(package_location):
             os.unlink(os.path.join(package_location, file_name))
 
 
-def clean_dir(path='/tmp/WSP/'):
-    shutil.rmtree(path, ignore_errors=True)
-
-
 @pytest.mark.parametrize(
     'expected_results',
     [