diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 355d6e4f..f9e8460e 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -16,7 +16,7 @@ services: - APP_BROKER_URL=amqp://guest:guest@rabbitmq:5672// - APP_CELERY_RESULT_BACKEND=amqp://guest:guest@rabbitmq:5672// - APP_CRAWLER_HOST_URL=http://scrapyd:6800 - - APP_API_PIPELINE_TASK_ENDPOINT_DEFAULT=tests.functional.tasks.submit_results + - APP_API_PIPELINE_TASK_ENDPOINT_DEFAULT=hepcrawl.testlib.tasks.submit_results - COVERAGE_PROCESS_START=/code/.coveragerc command: py.test -vv tests/functional/WSP/test_wsp.py volumes: &static_volume @@ -44,7 +44,7 @@ services: celery: image: hepcrawl_base environment: *env_variables - command: celery worker --events --app tests.functional.tasks --loglevel=debug + command: celery worker --events --app hepcrawl.testlib.tasks --loglevel=debug volumes: *static_volume links: - rabbitmq diff --git a/tests/__init__.py b/hepcrawl/testlib/__init__.py similarity index 68% rename from tests/__init__.py rename to hepcrawl/testlib/__init__.py index e8c02e63..9d0c62af 100644 --- a/tests/__init__.py +++ b/hepcrawl/testlib/__init__.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- # # This file is part of hepcrawl. -# Copyright (C) 2015, 2016, 2017 CERN. +# Copyright (C) 2017 CERN. # # hepcrawl is a free software; you can redistribute it and/or modify it # under the terms of the Revised BSD License; see LICENSE file for # more details. + +from __future__ import absolute_import, print_function, unicode_literals diff --git a/hepcrawl/testlib/celery_monitor.py b/hepcrawl/testlib/celery_monitor.py new file mode 100644 index 00000000..704965c2 --- /dev/null +++ b/hepcrawl/testlib/celery_monitor.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +# +# This file is part of hepcrawl. +# Copyright (C) 2017 CERN. +# +# hepcrawl is a free software; you can redistribute it and/or modify it +# under the terms of the Revised BSD License; see LICENSE file for +# more details. + +"""Celery monitor dealing with celery tasks for functional tests.""" + +from __future__ import absolute_import, print_function, unicode_literals + +from itertools import islice + + +class CeleryMonitor(object): + def __init__(self, app, monitor_timeout=3, monitor_iter_limit=100): + self.results = [] + self.recv = None + self.app = app + self.connection = None + self.monitor_timeout = monitor_timeout + self.monitor_iter_limit = monitor_iter_limit + + def __enter__(self): + state = self.app.events.State() + + def announce_succeeded_tasks(event): + state.event(event) + task = state.tasks.get(event['uuid']) + print('TASK SUCCEEDED: %s[%s] %s' % (task.name, task.uuid, task.info(),)) + tasks = self.app.AsyncResult(task.id) + for task in tasks.result: + self.results.append(task) + self.recv.should_stop = True + + def announce_failed_tasks(event): + state.event(event) + task = state.tasks.get(event['uuid']) + print('TASK FAILED: %s[%s] %s' % (task.name, task.uuid, task.info(),)) + self.results.append(task.info()) + self.recv.should_stop = True + + self.app.control.enable_events() + self.connection = self.app.connection() + self.recv = self.app.events.Receiver(self.connection, handlers={ + 'task-succeeded': announce_succeeded_tasks, + 'task-failed': announce_failed_tasks, + }) + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + events_iter = self.recv.itercapture(limit=None, timeout=self.monitor_timeout, wakeup=True) + self._wait_for_results(events_iter) + self.connection.__exit__() + + def _wait_for_results(self, events_iter): + any(islice(events_iter, self.monitor_iter_limit)) + + @classmethod + def do_crawl(cls, + app, + monitor_timeout, + monitor_iter_limit, + crawler_instance, + project='hepcrawl', + spider='WSP', + settings=None, + **crawler_arguments): + + if settings is None: + settings = {} + + with cls(app, monitor_timeout=monitor_timeout, monitor_iter_limit=monitor_iter_limit) as my_monitor: + crawler_instance.schedule( + project=project, + spider=spider, + settings=settings or {}, + **crawler_arguments + ) + + return my_monitor.results diff --git a/tests/functional/scrapyd_coverage_runner.py b/hepcrawl/testlib/scrapyd_coverage_runner.py similarity index 100% rename from tests/functional/scrapyd_coverage_runner.py rename to hepcrawl/testlib/scrapyd_coverage_runner.py diff --git a/tests/functional/tasks.py b/hepcrawl/testlib/tasks.py similarity index 97% rename from tests/functional/tasks.py rename to hepcrawl/testlib/tasks.py index bdef104b..206854c4 100644 --- a/tests/functional/tasks.py +++ b/hepcrawl/testlib/tasks.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # This file is part of hepcrawl. -# Copyright (C) 2015, 2016, 2017 CERN. +# Copyright (C) 2017 CERN. # # hepcrawl is a free software; you can redistribute it and/or modify it # under the terms of the Revised BSD License; see LICENSE file for @@ -11,12 +11,11 @@ from __future__ import absolute_import, print_function, unicode_literals -import json - from six.moves.urllib.parse import urlparse - from celery import Celery +import json + class Config(object): CELERY_RESULT_BACKEND = "amqp://guest:guest@rabbitmq:5672//" diff --git a/tests/functional/WSP/__init__.py b/tests/functional/WSP/__init__.py deleted file mode 100644 index e8c02e63..00000000 --- a/tests/functional/WSP/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of hepcrawl. -# Copyright (C) 2015, 2016, 2017 CERN. -# -# hepcrawl is a free software; you can redistribute it and/or modify it -# under the terms of the Revised BSD License; see LICENSE file for -# more details. diff --git a/tests/functional/WSP/test_wsp.py b/tests/functional/WSP/test_wsp.py index 3d26cdb0..f15bf661 100644 --- a/tests/functional/WSP/test_wsp.py +++ b/tests/functional/WSP/test_wsp.py @@ -15,81 +15,11 @@ import json import os -from itertools import islice from scrapyd_api import ScrapydAPI from time import sleep -from tests.functional.tasks import app - - -class CeleryMonitor(object): - def __init__(self, app, monitor_timeout=3, monitor_iter_limit=100): - self.results = [] - self.recv = None - self.app = app - self.connection = None - self.monitor_timeout = monitor_timeout - self.monitor_iter_limit = monitor_iter_limit - - def __enter__(self): - state = self.app.events.State() - - def announce_succeeded_tasks(event): - state.event(event) - task = state.tasks.get(event['uuid']) - print('TASK SUCCEEDED: %s[%s] %s' % (task.name, task.uuid, task.info(),)) - tasks = app.AsyncResult(task.id) - for task in tasks.result: - self.results.append(task) - self.recv.should_stop = True - - def announce_failed_tasks(event): - state.event(event) - task = state.tasks.get(event['uuid']) - print('TASK FAILED: %s[%s] %s' % (task.name, task.uuid, task.info(),)) - self.results.append(task.info()) - self.recv.should_stop = True - - self.app.control.enable_events() - self.connection = self.app.connection() - self.recv = self.app.events.Receiver(self.connection, handlers={ - 'task-succeeded': announce_succeeded_tasks, - 'task-failed': announce_failed_tasks, - }) - - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - events_iter = self.recv.itercapture(limit=None, timeout=self.monitor_timeout, wakeup=True) - self._wait_for_results(events_iter) - self.connection.__exit__() - - def _wait_for_results(self, events_iter): - any(islice(events_iter, self.monitor_iter_limit)) - - @classmethod - def do_crawl(cls, - app, - monitor_timeout, - monitor_iter_limit, - crawler_instance, - project='hepcrawl', - spider='WSP', - settings=None, - **crawler_arguments): - - if settings is None: - settings = {} - - with cls(app, monitor_timeout=monitor_timeout, monitor_iter_limit=monitor_iter_limit) as my_monitor: - crawler_instance.schedule( - project=project, - spider=spider, - settings=settings or {}, - **crawler_arguments - ) - - return my_monitor.results +from hepcrawl.testlib.tasks import app +from hepcrawl.testlib.celery_monitor import CeleryMonitor def get_crawler_instance(crawler_host, *args, **kwargs): diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py deleted file mode 100644 index e8c02e63..00000000 --- a/tests/functional/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of hepcrawl. -# Copyright (C) 2015, 2016, 2017 CERN. -# -# hepcrawl is a free software; you can redistribute it and/or modify it -# under the terms of the Revised BSD License; see LICENSE file for -# more details. diff --git a/tests/functional/scrapyd_coverage_runner.conf b/tests/functional/scrapyd_coverage_runner.conf index 3851724e..426f9e3c 100644 --- a/tests/functional/scrapyd_coverage_runner.conf +++ b/tests/functional/scrapyd_coverage_runner.conf @@ -9,4 +9,4 @@ [scrapyd] -runner = tests.functional.scrapyd_coverage_runner +runner = hepcrawl.testlib.scrapyd_coverage_runner diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index 8d1c8b69..9d0c62af 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1 +1,10 @@ - +# -*- coding: utf-8 -*- +# +# This file is part of hepcrawl. +# Copyright (C) 2017 CERN. +# +# hepcrawl is a free software; you can redistribute it and/or modify it +# under the terms of the Revised BSD License; see LICENSE file for +# more details. + +from __future__ import absolute_import, print_function, unicode_literals