tests: add WSP functional tests

* Adds: logger to `InspireCeleryPushLine.close_spider` in order to be able to inspect outgoing celery tasks to Inspire. * Adds: disable passive ftp mode for WSP spider. * Adds: dockerized environment for functional tests (Dockerfile for hepcrawl, docker-compose files). * Adds: docker FTPServer with needed fixtures for the WSP's functional tests. * Adds: mocked celery tasks to catch outgoing tasks to Inspire. * Adds: WSP functional test. * Adds: WSP functional test to travis. * Adds: dockerized execution on travis for unit tests and docs. Signed-off-by: Spiros Delviniotis <[email protected]>
spirosdelviniotis · May 2, 2017 · 6d06078 · 6d06078
1 parent 1e268aa
commit 6d06078
Show file tree

Hide file tree

Showing 24 changed files with 618 additions and 21 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of hepcrawl.
+# Copyright (C) 2017 CERN.
+#
+# hepcrawl is a free software; you can redistribute it and/or modify it
+# under the terms of the Revised BSD License; see LICENSE file for
+# more details.
+
+[run]
+parallel = True
+omit =
+    /hepcrawl_venv/lib/python2.7/site-packages/*
+
diff --git a/.travis.yml b/.travis.yml
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of hepcrawl.
-# Copyright (C) 2015, 2016 CERN.
+# Copyright (C) 2015, 2016, 2017 CERN.
 #
 # hepcrawl is a free software; you can redistribute it and/or modify it
 # under the terms of the Revised BSD License; see LICENSE file for
@@ -11,29 +11,42 @@ sudo: false
 
 language: python
 
-cache:
-  - pip
+services:
+  - docker
 
 python:
   - '2.7'
 
 env:
-  - EXTRAS=docs,tests
+  global:
+    - EXTRAS=docs,tests
+    - DOCKER_COMPOSE_VERSION=1.9.0
+    - DOCKER_DATA="$HOME/hepcrawl_docker_data"
+  matrix:
+    - SUITE=unit
+    - SUITE=functional_wsp
+
+matrix:
+  fast_finish: true
 
 before_install:
-  - "travis_retry pip install --upgrade pip setuptools py"
-  - "travis_retry pip install twine wheel coveralls check-manifest"
+  - travis_retry pip install twine wheel coveralls check-manifest
+  - sudo rm -f /usr/local/bin/docker-compose
+  - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+  - chmod +x docker-compose
+  - sudo mv docker-compose /usr/local/bin
 
 install:
-  - "travis_retry pip install -e .[${EXTRAS}]"
+  - travis_retry docker-compose -f docker-compose.deps.yml run --rm pip
 
 script:
-  - pip freeze
-  - sphinx-build -qnNW docs docs/_build/html
-  - python setup.py test
-  - python setup.py sdist && ls dist/*
+  - travis_retry docker-compose -f docker-compose.test.yml run --rm ${SUITE}
 
 after_success:
+  - docker-compose -f docker-compose.test.yml kill -s SIGTERM
+  - docker-compose -f docker-compose.test.yml rm -f
+  - coverage combine
+  - sed -i 's@\"/code/@'"\"$(pwd)/"'@g' .coverage
   - coveralls
 
 notifications:

diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of hepcrawl.
+# Copyright (C) 2015, 2016, 2017 CERN.
+#
+# hepcrawl is a free software; you can redistribute it and/or modify it
+# under the terms of the Revised BSD License; see LICENSE file for
+# more details.
+
+FROM centos
+
+RUN yum install -y epel-release && \
+    yum update -y && \
+    yum install -y \
+        file \
+        gcc \
+        libffi-devel \
+        libxml2-devel \
+        libxslt-devel \
+        libssl-devel \
+        make \
+        openssl-devel \
+        poppler-utils \
+        python-pip \
+        python-virtualenv && \
+    yum clean all
+
+RUN mkdir /code
+
+ADD /docker_entrypoint.sh /docker_entrypoint.sh
+ENTRYPOINT ["/docker_entrypoint.sh"]
+
+WORKDIR /code
+
+CMD true
diff --git a/docker-compose.deps.yml b/docker-compose.deps.yml
@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of hepcrawl.
+# Copyright (C) 2015, 2016, 2017 CERN.
+#
+# hepcrawl is a free software; you can redistribute it and/or modify it
+# under the terms of the Revised BSD License; see LICENSE file for
+# more details.
+
+version: '2'
+
+services:
+  pip:
+    build:
+      context: ${PWD}
+      dockerfile: Dockerfile
+    image: hepcrawl_base
+    command: bash -c "pip install -e .[all] && pip freeze"
+    volumes:
+      - ${DOCKER_DATA}/tmp/hepcrawl_venv:/hepcrawl_venv/
+      - ${PWD}:/code/
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of hepcrawl.
+# Copyright (C) 2017 CERN.
+#
+# hepcrawl is a free software; you can redistribute it and/or modify it
+# under the terms of the Revised BSD License; see LICENSE file for
+# more details.
+
+version: '2'
+
+services:
+  functional_wsp:
+    image: hepcrawl_base  # hepcrawl_base image is build at pip service of docker-compose.deps.yml
+    environment: &env_variables
+      - APP_BROKER_URL=amqp://guest:guest@rabbitmq:5672//
+      - APP_CELERY_RESULT_BACKEND=amqp://guest:guest@rabbitmq:5672//
+      - APP_CRAWLER_HOST_URL=http://scrapyd:6800
+      - APP_API_PIPELINE_TASK_ENDPOINT_DEFAULT=tests.functional.tasks.submit_results
+      - COVERAGE_PROCESS_START=/code/.coveragerc
+    command: py.test -vv tests/functional/WSP/test_wsp.py
+    volumes: &static_volume
+      - ${DOCKER_DATA}/tmp/hepcrawl_venv:/hepcrawl_venv/
+      - ${PWD}:/code/
+      - ${PWD}/tests/functional/scrapyd_coverage_runner.conf:/etc/scrapyd/scrapyd.conf
+    links:
+      - rabbitmq
+      - celery
+      - scrapyd
+      - ftp_server
+
+  unit:
+    image: hepcrawl_base
+    environment: *env_variables
+    command: bash -c "py.test tests/unit && sphinx-build -nNW docs docs/_build/html && python setup.py sdist && ls dist/*"
+    volumes: *static_volume
+
+  doc:
+    image: hepcrawl_base
+    environment: *env_variables
+    command: bash -c "sphinx-build -qnNW docs docs/_build/html && exec python setup.py sdist && exec ls dist/*"
+    volumes: *static_volume
+
+  celery:
+    image: hepcrawl_base
+    environment: *env_variables
+    command: celery worker --events --app tests.functional.tasks --loglevel=debug
+    volumes: *static_volume
+    links:
+      - rabbitmq
+      - ftp_server
+
+  scrapyd:
+    image: hepcrawl_base
+    environment: *env_variables
+    command: bash -c "rm -f twistd.pid && exec scrapyd"
+    volumes: *static_volume
+    links:
+      - celery
+      - ftp_server
+      - rabbitmq
+    depends_on:
+      - scrapyd_deploy
+
+  scrapyd_deploy:
+    image: hepcrawl_base
+    environment: *env_variables
+    command: bash -c "sleep 10 && scrapyd-deploy"  # make sure that the scrapyd is up
+    volumes: *static_volume
+
+  ftp_server:
+    image: stilliard/pure-ftpd:hardened
+    environment:
+      - PUBLICHOST=localhost
+    volumes:
+      - ${PWD}/tests/functional/WSP/fixtures/ftp_server/WSP:/home/ftpusers/bob/WSP
+      - ${PWD}/tests/functional/WSP/fixtures/ftp_server/pureftpd.passwd:/etc/pure-ftpd/passwd/pureftpd.passwd
+
+  rabbitmq:
+    image: rabbitmq
diff --git a/docker_entrypoint.sh b/docker_entrypoint.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# -*- coding: utf-8 -*-
+#
+# This file is part of hepcrawl.
+# Copyright (C) 2017 CERN.
+#
+# hepcrawl is a free software; you can redistribute it and/or modify it
+# under the terms of the Revised BSD License; see LICENSE file for
+# more details.
+
+set -e
+
+VENV_PATH=/hepcrawl_venv
+
+if [ ! -f "$VENV_PATH/bin/activate" ]; then
+    virtualenv "$VENV_PATH"
+    source "$VENV_PATH"/bin/activate
+    pip install --upgrade pip
+    pip install --upgrade setuptools wheel
+else
+    source "$VENV_PATH"/bin/activate
+fi
+
+find \( -name __pycache__ -o -name '*.pyc' \) -delete
+
+exec "$@"
diff --git a/hepcrawl/pipelines.py b/hepcrawl/pipelines.py
@@ -228,13 +228,16 @@ def open_spider(self, spider):
 
     def close_spider(self, spider):
         """Post results to BROKER API."""
+        from celery.utils.log import get_task_logger
+        logger = get_task_logger(__name__)
         if 'SCRAPY_JOB' in os.environ and self.count > 0:
             task_endpoint = spider.settings[
                 'API_PIPELINE_TASK_ENDPOINT_MAPPING'
             ].get(
                 spider.name,
                 spider.settings['API_PIPELINE_TASK_ENDPOINT_DEFAULT'],
             )
+            logger.info('Triggering celery task: %s.' % task_endpoint)
             self.celery.send_task(
                 task_endpoint,
                 kwargs=self._prepare_payload(spider),

diff --git a/hepcrawl/scrapy.cfg b/hepcrawl/scrapy.cfg
@@ -1,5 +1,5 @@
 # This file is part of hepcrawl.
-# Copyright (C) 2015, 2016 CERN.
+# Copyright (C) 2015, 2016, 2017 CERN.
 #
 # hepcrawl is a free software; you can redistribute it and/or modify it
 # under the terms of the Revised BSD License; see LICENSE file for

diff --git a/hepcrawl/settings.py b/hepcrawl/settings.py
@@ -100,17 +100,20 @@
 # INSPIRE Push Pipeline settings
 # ==============================
 API_PIPELINE_URL = "http://localhost:5555/api/task/async-apply"
-API_PIPELINE_TASK_ENDPOINT_DEFAULT = "inspire_crawler.tasks.submit_results"
+API_PIPELINE_TASK_ENDPOINT_DEFAULT = os.environ.get(
+    "APP_API_PIPELINE_TASK_ENDPOINT_DEFAULT",
+    "inspire_crawler.tasks.submit_results"
+)
 API_PIPELINE_TASK_ENDPOINT_MAPPING = {}   # e.g. {'my_spider': 'special.task'}
 
 # Celery
 # ======
 BROKER_URL = os.environ.get(
     "APP_BROKER_URL",
-    "amqp://guest:guest@localhost:5672//")
+    "amqp://guest:guest@rabbitmq:5672//")
 CELERY_RESULT_BACKEND = os.environ.get(
     "APP_CELERY_RESULT_BACKEND",
-    "amqp://guest:guest@localhost:5672//")
+    "amqp://guest:guest@rabbitmq:5672//")
 CELERY_ACCEPT_CONTENT = ['json', 'msgpack', 'yaml']
 CELERY_TIMEZONE = 'Europe/Amsterdam'
 CELERY_DISABLE_RATE_LIMITS = True

diff --git a/hepcrawl/spiders/wsp_spider.py b/hepcrawl/spiders/wsp_spider.py
@@ -91,13 +91,15 @@ def start_requests(self):
             yield Request(self.package_path, callback=self.handle_package_file)
         else:
             ftp_host, ftp_params = ftp_connection_info(self.ftp_host, self.ftp_netrc)
+
             dummy, new_files = ftp_list_files(
                 self.ftp_folder,
                 self.target_folder,
                 server=ftp_host,
                 user=ftp_params['ftp_user'],
                 password=ftp_params['ftp_password']
             )
+
             for remote_file in new_files:
                 # Cast to byte-string for scrapy compatibility
                 remote_file = str(remote_file)

diff --git a/hepcrawl/utils.py b/hepcrawl/utils.py
@@ -7,6 +7,8 @@
 # under the terms of the Revised BSD License; see LICENSE file for
 # more details.
 
+from __future__ import absolute_import, print_function
+
 import os
 import re
 from operator import itemgetter
@@ -17,6 +19,8 @@
 from urlparse import urlparse
 
 import ftputil
+import ftputil.session
+import ftplib
 import requests
 
 from scrapy import Selector
@@ -40,21 +44,28 @@ def unzip_xml_files(filename, target_folder):
     return xml_files
 
 
-def ftp_connection_info(ftp_host, netrc_file):
+def ftp_connection_info(ftp_host, netrc_file, passive_mode=False):
     """Return ftp connection info from netrc and optional host address."""
     if not ftp_host:
         ftp_host = netrc(netrc_file).hosts.keys()[0]
     logininfo = netrc(netrc_file).authenticators(ftp_host)
     connection_params = {
         "ftp_user": logininfo[0],
         "ftp_password": logininfo[2],
+        "ftp_passive": passive_mode,
     }
     return ftp_host, connection_params
 
 
-def ftp_list_files(server_folder, target_folder, server, user, password):
+def ftp_list_files(server_folder, target_folder, server, user, password, passive_mode=False):
     """List files from given FTP's server folder to target folder."""
-    with ftputil.FTPHost(server, user, password) as host:
+    session_factory = ftputil.session.session_factory(
+        base_class=ftplib.FTP,
+        port=21,
+        use_passive_mode=passive_mode,
+        encrypt_data_channel=True)
+
+    with ftputil.FTPHost(server, user, password, session_factory=session_factory) as host:
         files = host.listdir(host.curdir + '/' + server_folder)
         missing_files = []
         all_files = []

diff --git a/pytest.ini b/pytest.ini
@@ -1,14 +1,14 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of hepcrawl.
-# Copyright (C) 2015, 2016 CERN.
+# Copyright (C) 2015, 2016, 2017 CERN.
 #
 # hepcrawl is a free software; you can redistribute it and/or modify it
 # under the terms of the Revised BSD License; see LICENSE file for
 # more details.
 
 [pytest]
-addopts = --pep8 --ignore=docs --cov=hepcrawl --cov-report=term-missing
+addopts = --pep8 --ignore=docs --cov-config .coveragerc --cov=hepcrawl --cov-report=term-missing
 pep8ignore =
     tests/* ALL
     *.py E501

diff --git a/setup.py b/setup.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of hepcrawl.
-# Copyright (C) 2015, 2016 CERN.
+# Copyright (C) 2015, 2016, 2017 CERN.
 #
 # hepcrawl is a free software; you can redistribute it and/or modify it
 # under the terms of the Revised BSD License; see LICENSE file for
@@ -29,6 +29,7 @@
     'furl>=0.4.95',
     'ftputil>=3.3.1',
     'python-dateutil>=2.4.2',
+    'python-scrapyd-api>=2.0.1',
 ]
 
 tests_require = [