global: add CRAWL_ONCE_PATH to settings

* Adds: variable in `settings.py` to specify where to store `scrapy-crawl-once` DB. Addresses inspirehep#161 Signed-off-by: Spiros Delviniotis <[email protected]>
spirosdelviniotis · Aug 21, 2017 · 9d28bfd · 9d28bfd
1 parent f19320d
commit 9d28bfd
Show file tree

Hide file tree

Showing 7 changed files with 19 additions and 5 deletions.
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -17,6 +17,7 @@ services:
       - APP_CELERY_RESULT_BACKEND=amqp://guest:guest@rabbitmq:5672//
       - APP_CRAWLER_HOST_URL=http://scrapyd:6800
       - APP_API_PIPELINE_TASK_ENDPOINT_DEFAULT=hepcrawl.testlib.tasks.submit_results
+      - APP_CRAWL_ONCE_PATH=/var/lib/scrapy/crawl_once/
       - COVERAGE_PROCESS_START=/code/.coveragerc
       - BASE_USER_UID=${BASE_USER_UID:-1000}
       - BASE_USER_GIT=${BASE_USER_GIT:-1000}
@@ -25,6 +26,7 @@ services:
       - ${DOCKER_DATA}/tmp/hepcrawl_venv:/hepcrawl_venv/
       - ${PWD}:/code/
       - ${PWD}/tests/functional/scrapyd_coverage_runner.conf:/etc/scrapyd/scrapyd.conf
+      - ${PWD}/.scrapy/crawl_once:/var/lib/scrapy/crawl_once
 
   functional_wsp:
     <<: *service_base

diff --git a/hepcrawl/settings.py b/hepcrawl/settings.py
@@ -74,6 +74,10 @@
 
 CRAWL_ONCE_ENABLED = True
 CRAWL_ONCE_DEFAULT = True
+CRAWL_ONCE_PATH = os.environ.get(
+    'APP_CRAWL_ONCE_PATH',
+    '/var/lib/scrapy/crawl_once/',
+)
 
 # Enable or disable extensions
 # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html

diff --git a/hepcrawl/testlib/fixtures.py b/hepcrawl/testlib/fixtures.py
@@ -15,6 +15,7 @@
 
 from scrapy.http import Request, TextResponse
 from scrapy.selector import Selector
+from hepcrawl.settings import CRAWL_ONCE_PATH
 
 
 def fake_response_from_file(file_name, test_suite='unit', url='http://www.example.com', response_type=TextResponse):
@@ -134,7 +135,7 @@ def expected_json_results_from_file(*path_chunks, **kwargs):
     return expected_data
 
 
-def clean_dir(path=os.path.join(os.getcwd(), '.scrapy')):
+def clean_dir(path=CRAWL_ONCE_PATH):
     """
     Args:
         path(str): path of directory to be deleted. Default path is the produced DB per spider that

diff --git a/tests/Dockerfile.hepcrawl_base b/tests/Dockerfile.hepcrawl_base
@@ -26,10 +26,10 @@ RUN yum install -y epel-release && \
         python-virtualenv && \
     yum clean all
 
-RUN mkdir /code /hepcrawl_venv
+RUN mkdir /code /hepcrawl_venv /var/lib/scrapy
 
 RUN useradd test
-RUN chown -R test:test /code /hepcrawl_venv
+RUN chown -R test:test /code /hepcrawl_venv /var/lib/scrapy
 
 ADD ./docker_entrypoint.sh /docker_entrypoint.sh
 ADD ./fix_rights /fix_rights

diff --git a/tests/docker_entrypoint.sh b/tests/docker_entrypoint.sh
@@ -23,6 +23,8 @@ restore_venv_tmp_code_rights() {
         /fix_rights --codedir "$BASE_USER_UID:$BASE_USER_GID"
         echo "Restoring permissions of tmpdir to $BASE_USER_UID:$BASE_USER_GID"
         /fix_rights --tmpdir "$BASE_USER_UID:$BASE_USER_GID"
+        echo "Restoring permissions of vardir to $BASE_USER_UID:$BASE_USER_GID"
+        /fix_rights --vardir "$BASE_USER_UID:$BASE_USER_GID"
     else
         echo "No BASE_USER_UID env var defined, skipping venv, codedir, tmpdir permission" \
             "restore."
@@ -57,6 +59,7 @@ main() {
     /fix_rights --virtualenv 'test:test'
     /fix_rights --codedir 'test:test'
     /fix_rights --tmpdir 'test:test'
+    /fix_rights --vardir 'test:test'
     trap restore_venv_tmp_code_rights EXIT
 
     if ! [[ -f "$VENV_PATH/bin/activate" ]]; then

diff --git a/tests/fix_rights b/tests/fix_rights
diff --git a/tests/fix_rights.c b/tests/fix_rights.c
@@ -11,6 +11,7 @@
 char *VENV_PATH = "/hepcrawl_venv/";
 char *CODE_PATH = "/code/";
 char *TMP_PATH = "/tmp/";
+char *VAR_PATH = "/var/";
 
 
 int main (int argc, char *argv[]) {
@@ -30,7 +31,7 @@ int main (int argc, char *argv[]) {
     if (argc != 3) {
         fprintf(
             stderr,
-            "Usage: %s --virtualenv|--codedir|--tmpdir <user>:<group>\n",
+            "Usage: %s --virtualenv|--codedir|--tmpdir|--var <user>:<group>\n",
             argv[0]
         );
         exit(EXIT_FAILURE);
@@ -48,11 +49,14 @@ int main (int argc, char *argv[]) {
     } else if (strcmp(argv[1], "--tmpdir") == 0) {
         // tmp dir permissions
         chown_argv[3] = TMP_PATH;
+    } else if (strcmp(argv[1], "--vardir") == 0) {
+        // var dir permissions
+        chown_argv[3] = VAR_PATH;
     } else {
         fprintf(stderr, "Bad option %s.", argv[1]);
         fprintf(
             stderr,
-            "Usage: %s --virtualenv|--codedir|--tmpdir <user>:<group>\n",
+            "Usage: %s --virtualenv|--codedir|--tmpdir|--vardir <user>:<group>\n",
             argv[0]
         );
         exit(EXIT_FAILURE);