diff --git a/docker-compose.test.yml b/docker-compose.test.yml index d14d7c87..d9b69c75 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -18,6 +18,7 @@ services: - APP_CRAWLER_HOST_URL=http://scrapyd:6800 - APP_API_PIPELINE_TASK_ENDPOINT_DEFAULT=hepcrawl.testlib.tasks.submit_results - APP_FILES_STORE=/tmp/file_urls + - APP_CRAWL_ONCE_PATH=/var/lib/scrapy/crawl_once/ - COVERAGE_PROCESS_START=/code/.coveragerc - BASE_USER_UID=${BASE_USER_UID:-1000} - BASE_USER_GIT=${BASE_USER_GIT:-1000} @@ -28,6 +29,7 @@ services: - ${PWD}/tests/functional/scrapyd_coverage_runner.conf:/etc/scrapyd/scrapyd.conf - /tmp/WSP:/tmp/WSP - /tmp/file_urls:/tmp/file_urls + - ${PWD}/.scrapy/crawl_once:/var/lib/scrapy/crawl_once functional_wsp: <<: *service_base diff --git a/hepcrawl/settings.py b/hepcrawl/settings.py index 91f82829..216f7694 100644 --- a/hepcrawl/settings.py +++ b/hepcrawl/settings.py @@ -74,6 +74,10 @@ CRAWL_ONCE_ENABLED = True CRAWL_ONCE_DEFAULT = True +CRAWL_ONCE_PATH = os.environ.get( + 'APP_CRAWL_ONCE_PATH', + '/var/lib/scrapy/crawl_once/', +) # Enable or disable extensions # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html diff --git a/hepcrawl/testlib/fixtures.py b/hepcrawl/testlib/fixtures.py index c227d109..b78c1a31 100644 --- a/hepcrawl/testlib/fixtures.py +++ b/hepcrawl/testlib/fixtures.py @@ -15,6 +15,7 @@ from scrapy.http import Request, TextResponse from scrapy.selector import Selector +from hepcrawl.settings import CRAWL_ONCE_PATH def fake_response_from_file(file_name, test_suite='unit', url='http://www.example.com', response_type=TextResponse): @@ -133,7 +134,8 @@ def expected_json_results_from_file(*path_chunks, **kwargs): return expected_data -def clean_dir(path=os.path.join(os.getcwd(), '.scrapy')): + +def clean_dir(path=CRAWL_ONCE_PATH): """ Deletes all contained files of given target directory path. diff --git a/tests/Dockerfile.hepcrawl_base b/tests/Dockerfile.hepcrawl_base index eb91b69f..8db9f43e 100644 --- a/tests/Dockerfile.hepcrawl_base +++ b/tests/Dockerfile.hepcrawl_base @@ -26,10 +26,10 @@ RUN yum install -y epel-release && \ python-virtualenv && \ yum clean all -RUN mkdir /code /hepcrawl_venv +RUN mkdir /code /hepcrawl_venv /var/lib/scrapy RUN useradd test -RUN chown -R test:test /code /hepcrawl_venv +RUN chown -R test:test /code /hepcrawl_venv /var/lib/scrapy ADD ./docker_entrypoint.sh /docker_entrypoint.sh ADD ./fix_rights /fix_rights diff --git a/tests/docker_entrypoint.sh b/tests/docker_entrypoint.sh index 1762a421..4785699d 100755 --- a/tests/docker_entrypoint.sh +++ b/tests/docker_entrypoint.sh @@ -23,6 +23,8 @@ restore_venv_tmp_code_rights() { /fix_rights --codedir "$BASE_USER_UID:$BASE_USER_GID" echo "Restoring permissions of tmpdir to $BASE_USER_UID:$BASE_USER_GID" /fix_rights --tmpdir "$BASE_USER_UID:$BASE_USER_GID" + echo "Restoring permissions of vardir to $BASE_USER_UID:$BASE_USER_GID" + /fix_rights --vardir "$BASE_USER_UID:$BASE_USER_GID" else echo "No BASE_USER_UID env var defined, skipping venv, codedir, tmpdir permission" \ "restore." @@ -57,6 +59,7 @@ main() { /fix_rights --virtualenv 'test:test' /fix_rights --codedir 'test:test' /fix_rights --tmpdir 'test:test' + /fix_rights --vardir 'test:test' trap restore_venv_tmp_code_rights EXIT if ! [[ -f "$VENV_PATH/bin/activate" ]]; then diff --git a/tests/fix_rights b/tests/fix_rights index 98677b2c..ecf219b0 100755 Binary files a/tests/fix_rights and b/tests/fix_rights differ diff --git a/tests/fix_rights.c b/tests/fix_rights.c index 8eaa9911..2dead30c 100644 --- a/tests/fix_rights.c +++ b/tests/fix_rights.c @@ -11,6 +11,7 @@ char *VENV_PATH = "/hepcrawl_venv/"; char *CODE_PATH = "/code/"; char *TMP_PATH = "/tmp/"; +char *VAR_PATH = "/var/"; int main (int argc, char *argv[]) { @@ -30,7 +31,7 @@ int main (int argc, char *argv[]) { if (argc != 3) { fprintf( stderr, - "Usage: %s --virtualenv|--codedir|--tmpdir :\n", + "Usage: %s --virtualenv|--codedir|--tmpdir|--var :\n", argv[0] ); exit(EXIT_FAILURE); @@ -48,11 +49,14 @@ int main (int argc, char *argv[]) { } else if (strcmp(argv[1], "--tmpdir") == 0) { // tmp dir permissions chown_argv[3] = TMP_PATH; + } else if (strcmp(argv[1], "--vardir") == 0) { + // var dir permissions + chown_argv[3] = VAR_PATH; } else { fprintf(stderr, "Bad option %s.", argv[1]); fprintf( stderr, - "Usage: %s --virtualenv|--codedir|--tmpdir :\n", + "Usage: %s --virtualenv|--codedir|--tmpdir|--vardir :\n", argv[0] ); exit(EXIT_FAILURE);