diff --git a/.travis.yml b/.travis.yml index 39a9031ac99..ccbf8f21719 100644 --- a/.travis.yml +++ b/.travis.yml @@ -127,7 +127,7 @@ jobs: # test python, linting ---------------------------------------------------------------------- - stage: unit testing - name: Python linting + name: python linting language: python python: - "3.6" diff --git a/ops/travis/system-testing/tests/test_swarm_runs.py b/ops/travis/system-testing/tests/test_swarm_runs.py index 7eedacadbc4..3bcbca65016 100644 --- a/ops/travis/system-testing/tests/test_swarm_runs.py +++ b/ops/travis/system-testing/tests/test_swarm_runs.py @@ -25,7 +25,7 @@ def osparc_simcore_root_dir(here) -> Path: root_dir = here.parent.parent.parent.parent.resolve() assert root_dir.exists(), "Is this service within osparc-simcore repo?" assert any(root_dir.glob("services/web/server")), "%s not look like rootdir" % root_dir - return root_dir + return root_dir @pytest.fixture("session") def services_docker_compose(osparc_simcore_root_dir) -> Dict[str, str]: @@ -53,7 +53,9 @@ def docker_client(): client = docker.from_env() yield client -@tenacity.retry(stop=tenacity.stop_after_delay(240), wait=tenacity.wait_fixed(5), retry=tenacity.retry_if_exception_type(AssertionError)) +@tenacity.retry(stop=tenacity.stop_after_delay(240), + wait=tenacity.wait_fixed(5), + retry=tenacity.retry_if_exception_type(AssertionError)) def try_checking_task_state(running_service, service_name): tasks = running_service.tasks() assert tasks is not None @@ -72,8 +74,12 @@ def try_checking_task_state(running_service, service_name): difference = now - creation_time assert difference.total_seconds() > 5 -# the swarm should be up prior to testing... using make up-swarm + def test_services_running(docker_client, services_docker_compose, tools_docker_compose): + """ + the swarm should be up prior to testing... using make up-swarm + + """ running_services = docker_client.services.list() assert (len(services_docker_compose["services"]) + len(tools_docker_compose["services"])) == len(running_services) @@ -81,8 +87,8 @@ def test_services_running(docker_client, services_docker_compose, tools_docker_c # all the services shall be available here for service_name in services_docker_compose["services"].keys(): # find the service - running_service = [x for x in running_services if service_name in x.name] + running_service = [s for s in running_services if service_name in s.name] assert len(running_service) == 1 running_service = running_service[0] - # check health - try_checking_task_state(running_service, service_name) + # check health + try_checking_task_state(running_service, service_name) diff --git a/services/director/Dockerfile b/services/director/Dockerfile index b376fadef34..b36ee8ac706 100644 --- a/services/director/Dockerfile +++ b/services/director/Dockerfile @@ -12,7 +12,10 @@ RUN adduser -D -u 8004 scu &&\ addgroup -g $DOCKER_GID_ARG docker ENV HOME /home/scu -ENV PIP /home/scu/.venv/bin/pip3 +# TODO: tmp workaround for https://github.com/pypa/pip/issues/6197 +# - Added explicit --cache-dir that is removed at the end of the build +ENV PIP /home/scu/.venv/bin/pip3 --cache-dir=/pipcache +# ----- EXPOSE 8001 ENV REGISTRY_AUTH = '' @@ -47,14 +50,14 @@ RUN apk add --no-cache \ # create virtual environment RUN python3 -m venv $HOME/.venv &&\ - $PIP install --no-cache-dir --upgrade \ + $PIP install --upgrade \ pip \ wheel \ setuptools # this is done to accelerate installation of director later on COPY --chown=scu:scu services/director/requirements/base.txt requirements-base.txt -RUN $PIP install --no-cache-dir -r requirements-base.txt &&\ +RUN $PIP install -r requirements-base.txt &&\ rm requirements-base.txt COPY --chown=scu:scu services/director/docker docker @@ -83,16 +86,18 @@ FROM build as cache COPY --chown=scu:scu services/director $HOME/services/director COPY --chown=scu:scu packages/service-library $HOME/packages/service-library WORKDIR $HOME/services/director -RUN $PIP install --no-cache-dir -r requirements/prod.txt +RUN $PIP install -r requirements/prod.txt WORKDIR $HOME/packages/service-library -RUN $PIP install --no-cache-dir -r requirements/dev.txt +RUN $PIP install -r requirements/dev.txt # --------------------------Production stage ------------------- FROM cache as production WORKDIR $HOME ENV DEBUG 0 ENV RUN_DOCKER_ENGINE_ROOT=0 -RUN rm -rf $HOME/services \ - && rm -rf $HOME/packages +RUN rm -rf $HOME/services &&\ + rm -rf $HOME/packages &&\ + rm -rf /pipcache + ENTRYPOINT [ "/bin/sh", "docker/entrypoint.sh" ] CMD ["/bin/sh", "docker/boot.sh"] diff --git a/services/director/docker/boot.sh b/services/director/docker/boot.sh index 4224fbd12b6..b1dbe973326 100644 --- a/services/director/docker/boot.sh +++ b/services/director/docker/boot.sh @@ -7,10 +7,10 @@ then echo "INFO: Booting in development mode ..." echo "DEBUG: Director running as `id $(whoami)`" echo "DEBUG: Director running groups `groups`" - + echo "Installing director service ..." cd $HOME/services/director - $PIP install --no-cache-dir -r requirements/dev.txt + $PIP install -r requirements/dev.txt $PIP list cd $HOME simcore-service-director --loglevel=debug @@ -18,5 +18,3 @@ else echo "INFO: Booting in production mode ..." simcore-service-director --loglevel=info fi - - diff --git a/services/docker-compose.cache.yml b/services/docker-compose.cache.yml index 235f376e064..d274281bd09 100644 --- a/services/docker-compose.cache.yml +++ b/services/docker-compose.cache.yml @@ -11,10 +11,10 @@ services: target: cache storage: build: - target: cache + target: cache webclient: build: target: build webserver: build: - target: cache \ No newline at end of file + target: cache diff --git a/services/dy-2Dgraph/use-cases/cc/Dockerfile b/services/dy-2Dgraph/use-cases/cc/Dockerfile index 6e23c069dbe..b2bce9c830b 100644 --- a/services/dy-2Dgraph/use-cases/cc/Dockerfile +++ b/services/dy-2Dgraph/use-cases/cc/Dockerfile @@ -11,7 +11,9 @@ USER $NB_USER # install requirements -------------------------------------------------------- COPY --chown=jovyan:users cc/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +# TODO: tmp workaround for https://github.com/pypa/pip/issues/6197 +RUN pip --cache-dir=/pipcache install -r requirements.txt &&\ + rm -rf /pipcache # prepare for booting --------------------------------------------------------- COPY --chown=jovyan:users docker /docker # set of arguments to copy the right notebook --------------------------------- @@ -36,4 +38,4 @@ ARG NOTEBOOK_NAME ARG NOTEBOOK_FOLDER_NAME # copy the notebook in the image COPY --chown=jovyan:users cc/${NOTEBOOK_FOLDER_NAME}/${NOTEBOOK_NAME} ${NOTEBOOK_NAME} -ENTRYPOINT [ "/bin/bash", "/docker/boot.sh" ] \ No newline at end of file +ENTRYPOINT [ "/bin/bash", "/docker/boot.sh" ] diff --git a/services/dy-2Dgraph/use-cases/kember/Dockerfile b/services/dy-2Dgraph/use-cases/kember/Dockerfile index c3278912c7e..04148f95f81 100644 --- a/services/dy-2Dgraph/use-cases/kember/Dockerfile +++ b/services/dy-2Dgraph/use-cases/kember/Dockerfile @@ -4,7 +4,10 @@ LABEL maintainer="sanderegg" # install requirements -------------------------------------------------------- COPY --chown=jovyan:users kember/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +# TODO: tmp workaround for https://github.com/pypa/pip/issues/6197 +RUN pip --cache-dir=/pipcache install -r requirements.txt &&\ + rm -rf /pipcache + # prepare for booting --------------------------------------------------------- COPY --chown=jovyan:users docker /docker @@ -29,4 +32,4 @@ FROM common AS production ARG NOTEBOOK_NAME # copy the notebook in the image COPY --chown=jovyan:users kember/${NOTEBOOK_NAME} ${NOTEBOOK_NAME} -ENTRYPOINT [ "/bin/bash", "/docker/boot.sh" ] \ No newline at end of file +ENTRYPOINT [ "/bin/bash", "/docker/boot.sh" ] diff --git a/services/sidecar/Dockerfile b/services/sidecar/Dockerfile index fe003ad009a..a2deabd6579 100644 --- a/services/sidecar/Dockerfile +++ b/services/sidecar/Dockerfile @@ -12,7 +12,9 @@ RUN adduser -D -u 8004 scu &&\ addgroup -g $DOCKER_GID_ARG docker ENV HOME /home/scu -ENV PIP /home/scu/.venv/bin/pip3 +# TODO: tmp workaround for https://github.com/pypa/pip/issues/6197 +# - Added explicit --cache-dir that is removed at the end of the build +ENV PIP /home/scu/.venv/bin/pip3 --cache-dir=/pipcache EXPOSE 8000 VOLUME /home/scu/input @@ -38,14 +40,14 @@ RUN apk add --no-cache \ libc-dev RUN python3 -m venv $HOME/.venv &&\ - $PIP install --no-cache-dir --upgrade \ + $PIP install --upgrade \ pip \ wheel \ setuptools # TODO: check if scu:scu copy is necessary!? COPY --chown=scu:scu services/sidecar/requirements/base.txt requirements-base.txt -RUN $PIP install --no-cache-dir -r requirements-base.txt &&\ +RUN $PIP install -r requirements-base.txt &&\ rm requirements-base.txt COPY --chown=scu:scu services/sidecar/docker docker @@ -80,7 +82,7 @@ COPY --chown=scu:scu services/sidecar $HOME/services/sidecar COPY --chown=scu:scu services/storage/client-sdk $HOME/services/storage/client-sdk WORKDIR /home/scu/services/sidecar -RUN $PIP --no-cache-dir install -r requirements/prod.txt ;\ +RUN $PIP install -r requirements/prod.txt ;\ $PIP list @@ -101,7 +103,8 @@ FROM cache as production WORKDIR /home/scu/ RUN . $HOME/.venv/bin/activate; pip list &&\ - rm -rf $HOME/services + rm -rf $HOME/services &&\ + rm -rf /pipcache ENV DEBUG 0 ENV RUN_DOCKER_ENGINE_ROOT=0 diff --git a/services/sidecar/boot.sh b/services/sidecar/boot.sh index 90fe43cac75..5cda90b855a 100644 --- a/services/sidecar/boot.sh +++ b/services/sidecar/boot.sh @@ -9,8 +9,8 @@ then echo "DEBUG: Sidecar running groups `groups`" cd $HOME/services/sidecar - pip3 install --no-cache-dir -r requirements/dev.txt - pip3 list + $PIP install -r requirements/dev.txt + $PIP list cd $HOME celery worker --app sidecar.celery:app --concurrency 2 --loglevel=debug diff --git a/services/storage/Dockerfile b/services/storage/Dockerfile index 36fabb0a0f3..fdc99aebb2f 100644 --- a/services/storage/Dockerfile +++ b/services/storage/Dockerfile @@ -1,4 +1,3 @@ -# TODO: this is still not finished!! FROM python:3.6-alpine as base LABEL maintainer=mguidon @@ -20,8 +19,12 @@ ENV HOME /home/scu ENV VENV /home/scu/.venv/ ENV VENV3 $VENV ENV VENV2 /home/scu/.venv27/ -ENV PIP /home/scu/.venv/bin/pip3 -ENV PIP2 $VENV2/bin/pip + +# TODO: tmp workaround for https://github.com/pypa/pip/issues/6197 +# - Added explicit --cache-dir that is removed at the end of the build +ENV PIP /home/scu/.venv/bin/pip3 --cache-dir=/pipcache +ENV PIP2 $VENV2/bin/pip --cache-dir=/pipcache2 +# ----- EXPOSE 8080 @@ -48,16 +51,18 @@ RUN apk add --no-cache \ libc-dev \ libffi-dev \ python2 \ - python2-dev + python2-dev \ + linux-headers RUN python3 -m venv $HOME/.venv &&\ - $PIP install --no-cache-dir --upgrade \ + $PIP install --upgrade \ pip \ wheel \ - setuptools + setuptools &&\ + $PIP list RUN $PIP install virtualenv &&\ - $VENV3/bin/virtualenv --python=python2 $VENV2 + $VENV3/bin/virtualenv --python=python2 $VENV2 WORKDIR /home/scu @@ -66,11 +71,11 @@ WORKDIR /home/scu COPY --chown=scu:scu services/storage/requirements/py27.txt requirements-py27.txt COPY --chown=scu:scu services/storage/requirements/base.txt requirements-base.txt COPY --chown=scu:scu services/storage/docker docker -RUN $PIP install --no-cache-dir -r requirements-base.txt -RUN apk add --no-cache \ - linux-headers -RUN $PIP2 install --no-cache-dir -r requirements-py27.txt +RUN $PIP install -r requirements-base.txt +RUN $PIP2 install numpy==1.16.0 &&\ + $PIP2 install -r requirements-py27.txt + # --------------------------Development stage ------------------- FROM build as development @@ -78,15 +83,15 @@ ARG HOST_GID_ARG=1000 # install test 3rd party packages to accelerate runtime installs COPY --chown=scu:scu services/storage/tests/requirements.txt requirements-tests.txt -RUN $PIP install --no-cache-dir -r requirements-tests.txt +RUN $PIP install -r requirements-tests.txt # in dev mode we give access to `scu` to host's mapped volumes RUN addgroup -g $HOST_GID_ARG hgrp &&\ - addgroup scu hgrp && \ + addgroup scu hgrp &&\ chown -R scu:scu $HOME/.venv VOLUME /home/scu/packages -VOLUME /home/scu/services/storage/ +VOLUME /home/scu/services/storage ENV DEBUG 1 ENTRYPOINT [ "/bin/sh", "docker/entrypoint.sh" ] @@ -94,7 +99,6 @@ CMD ["/bin/sh", "docker/boot.sh"] # --------------------------Production multi-stage ------------------- -#FROM build as build-production FROM build as cache ENV SIMCORE_WEB_CONFIG production @@ -105,28 +109,22 @@ COPY --chown=scu:scu packages $HOME/packages COPY --chown=scu:scu services/storage $HOME/services/storage WORKDIR /home/scu/services/storage -RUN $PIP --no-cache-dir install -r requirements/prod.txt &&\ +RUN $PIP install -r requirements/prod.txt &&\ $PIP list -#------------------- -#FROM base as production -# TODO: PC some basic package missing - -#COPY --from=build-production --chown=scu:scu $HOME/services/server/boot.sh $HOME -#COPY --from=build-production --chown=scu:scu $HOME/.venv $HOME/.venv -#RUN . $HOME/.venv/bin/activate; pip list -# FIXME: temporary solution until found missing packages WORKDIR /home/scu - ENV DEBUG 0 FROM cache as production + # remove any unnecessary stuff RUN . $HOME/.venv/bin/activate; pip list &&\ rm -rf $HOME/packages &&\ - rm -rf $HOME/services/storage + rm -rf $HOME/services/storage &&\ + rm -rf /pipcache &&\ + rm -rf /pipcache2 + ENTRYPOINT [ "/bin/sh", "docker/entrypoint.sh" ] CMD ["/bin/sh", "docker/boot.sh"] - diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index 96744872326..974aa68916b 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -9,8 +9,8 @@ then echo "DEBUG: Workdir :`pwd`" cd $HOME/services/storage - pip install -r requirements/dev.txt - pip list + $PIP install -r requirements/dev.txt + $PIP list cd $HOME/ simcore-service-storage --config docker-dev-config.yaml diff --git a/services/storage/requirements/py27.in b/services/storage/requirements/py27.in new file mode 100644 index 00000000000..87de110c6dc --- /dev/null +++ b/services/storage/requirements/py27.in @@ -0,0 +1 @@ +blackfynn==2.5.0 diff --git a/services/storage/requirements/py27.txt b/services/storage/requirements/py27.txt index 87de110c6dc..e22d426554f 100644 --- a/services/storage/requirements/py27.txt +++ b/services/storage/requirements/py27.txt @@ -1 +1,27 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file services/storage/requirements/py27.txt services/storage/requirements/py27.in +# blackfynn==2.5.0 +boto3==1.9.86 # via blackfynn +botocore==1.12.86 # via boto3, s3transfer +certifi==2018.11.29 # via requests +chardet==3.0.4 # via requests +configparser==3.7.1 # via blackfynn +docopt==0.6.2 # via blackfynn +docutils==0.14 # via botocore +futures==3.2.0 # via s3transfer +idna==2.8 # via requests +jmespath==0.9.3 # via boto3, botocore +numpy==1.16.0 # via blackfynn, pandas +pandas==0.24.0 # via blackfynn +protobuf==3.2.0 # via blackfynn +psutil==5.5.0 # via blackfynn +python-dateutil==2.7.5 # via botocore, pandas +pytz==2018.4 # via blackfynn, pandas +requests==2.21.0 # via blackfynn +s3transfer==0.1.13 # via boto3 +six==1.12.0 # via protobuf, python-dateutil +urllib3==1.24.1 # via botocore, requests diff --git a/services/web/Dockerfile b/services/web/Dockerfile index eda472097c0..10ce7fec3ba 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -17,7 +17,9 @@ RUN apk add --no-cache \ su-exec ENV HOME /home/scu -ENV PIP /home/scu/.venv/bin/pip3 +# TODO: tmp workaround for https://github.com/pypa/pip/issues/6197 +# - Added explicit --cache-dir that is removed at the end of the build +ENV PIP /home/scu/.venv/bin/pip3 --cache-dir=/pipcache ENV SIMCORE_WEB_OUTDIR $HOME/services/web/client ENV IS_CONTAINER_CONTEXT Yes @@ -53,7 +55,7 @@ RUN apk add --no-cache \ libffi-dev RUN python3 -m venv $HOME/.venv &&\ - $PIP install --no-cache-dir --upgrade \ + $PIP install --upgrade \ pip \ wheel \ setuptools @@ -63,7 +65,7 @@ WORKDIR /home/scu # install base 3rd party packages to accelerate runtime installs COPY --chown=scu:scu services/web/server/requirements/base.txt requirements-base.txt COPY --chown=scu:scu services/web/server/docker docker -RUN $PIP install --no-cache-dir -r requirements-base.txt +RUN $PIP install -r requirements-base.txt # --------------------------Development stage ------------------- FROM build as development @@ -72,7 +74,7 @@ ARG HOST_GID_ARG=1000 # install test 3rd party packages to accelerate runtime installs COPY --chown=scu:scu services/web/server/tests/requirements.txt requirements-tests.txt -RUN $PIP install --no-cache-dir -r requirements-tests.txt +RUN $PIP install -r requirements-tests.txt # in dev mode we give access to `scu` to host's mapped volumes RUN addgroup -g $HOST_GID_ARG hgrp &&\ @@ -103,7 +105,7 @@ COPY --from=services_webclient:build --chown=scu:scu /home/scu/client/build-outp COPY --chown=scu:scu services/web/server $HOME/services/web/server WORKDIR /home/scu/services/web/server -RUN $PIP --no-cache-dir install -r requirements/prod.txt &&\ +RUN $PIP install -r requirements/prod.txt &&\ $PIP list FROM cache as production @@ -120,7 +122,8 @@ WORKDIR /home/scu RUN . $HOME/.venv/bin/activate; pip list &&\ rm -rf $HOME/packages &&\ - rm -rf $HOME/services/web/server + rm -rf $HOME/services/web/server &&\ + rm -rf /pipcache ENV DEBUG 0 ENTRYPOINT [ "/bin/sh", "docker/entrypoint.sh" ] diff --git a/services/web/server/docker/boot.sh b/services/web/server/docker/boot.sh index 3b445c49221..53d57054fd0 100755 --- a/services/web/server/docker/boot.sh +++ b/services/web/server/docker/boot.sh @@ -9,8 +9,8 @@ then echo "DEBUG: Workdir :`pwd`" cd $HOME/services/web/server - pip install -r requirements/dev.txt - pip list + $PIP install -r requirements/dev.txt + $PIP list cd $HOME/ simcore-service-webserver --config server-docker-dev.yaml