✨Run dynamic services via dynamic-sidecar (🏗️ OPS + CI action)

.github/workflows/ci-testing-deploy.yml

@@ -119,6 +119,44 @@ jobs:
           name: unit_api_server_coverage
           path: codeclimate.unit_api_server_coverage.json
 
+  unit-test-service-sidecar:
+    name: "[unit] service-sidecar"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: [3.8]
+        os: [ubuntu-20.04]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: setup docker
+        run: |
+          sudo ./ci/github/helpers/setup_docker_compose.bash
+          ./ci/github/helpers/setup_docker_experimental.bash
+          ./ci/github/helpers/setup_docker_buildx.bash
+          echo ::set-env name=DOCKER_BUILDX::1
+      - name: setup python environment
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python }}
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - uses: actions/cache@v2
+        name: getting cached data
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: setup requirements
+        run: make -C services/service-sidecar ci-install-requirements
+      - name: pylint via tox
+        run: make -C services/service-sidecar ci-tox-pylint
+      - name: mypy via tox
+        run: make -C services/service-sidecar ci-tox-mypy
+      - name: tests via tox
+        run: make -C services/service-sidecar ci-tox-tests
+
   unit-test-catalog:
     name: "[unit] catalog"
     runs-on: ${{ matrix.os }}

Makefile

@@ -367,6 +367,7 @@ nodenv: node_modules ## builds node_modules local environ (TODO)
 pylint: ## Runs python linter framework's wide
 	/bin/bash -c "pylint --jobs=0 --rcfile=.pylintrc $(strip $(shell find services packages -iname '*.py' \
 											-not -path "*egg*" \
+											-not -path "*service-sidecar*" \
 											-not -path "*migration*" \
 											-not -path "*datcore.py" \
 											-not -path "*sandbox*" \

services/service-sidecar/CHANGELOG.md

@@ -0,0 +1,35 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Planned]
+
+- migrate to setup.py file
+- add tests
+
+## [Unreleased]
+
+## [0.0.2] - 2020-07-15
+### Added
+- entry point to create containers without starting them
+- entry point to stop running containers
+
+### Modified
+- moved more configurations to environment variables
+- migrated to new project structure, after project was moved to osparc
+
+## [0.0.1] - 2020-07-14
+### Added
+- FastAPI based service
+- environs to manage configuration
+- basic in memory storage , which can be easily extended for future needs
+- async command execution of files
+- context manger for to cleanup files after usage
+- `docker-compose up` like entry point
+- `docker-compose down` like entry point
+- entry point to list docker-compose started container
+- entry point to recover a container's logs
+- entry point to inspect a container
+- all spawned services and networks are removed when receiving `SIGTERM`

services/service-sidecar/Dockerfile

@@ -0,0 +1,147 @@
+ARG PYTHON_VERSION="3.8.4"
+FROM python:${PYTHON_VERSION}-slim-buster as base
+#
+#  USAGE:
+#     cd sercices/service-sidecar
+#     docker build -f Dockerfile -t service-sidecar:prod --target production ../../
+#     docker run service-sidecar:prod
+#
+#  REQUIRED: context expected at ``osparc-simcore/`` folder because we need access to osparc-simcore/packages
+
+LABEL maintainer="Andrei Neagu <[email protected]>"
+
+RUN set -eux; \
+	apt-get update; \
+	apt-get install -y gosu; \
+	rm -rf /var/lib/apt/lists/*; \
+# verify that the binary works
+	gosu nobody true
+
+# simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu)
+ENV SC_USER_ID=8004 \
+      SC_USER_NAME=scu \
+      SC_BUILD_TARGET=base \
+      SC_BOOT_MODE=default
+
+RUN adduser \
+      --uid ${SC_USER_ID} \
+      --disabled-password \
+      --gecos "" \
+      --shell /bin/sh \
+      --home /home/${SC_USER_NAME} \
+      ${SC_USER_NAME}
+
+# Sets utf-8 encoding for Python et al
+ENV LANG=C.UTF-8
+# Turns off writing .pyc files; superfluous on an ephemeral container.
+ENV PYTHONDONTWRITEBYTECODE=1 \
+      VIRTUAL_ENV=/home/scu/.venv
+# Ensures that the python and pip executables used
+# in the image will be those from our virtualenv.
+ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
+
+
+# -------------------------- Build stage -------------------
+# Installs build/package management tools and third party dependencies
+#
+# + /build             WORKDIR
+#
+FROM base as build
+
+ENV SC_BUILD_TARGET=build
+
+RUN apt-get update &&\
+    apt-get install -y --no-install-recommends \
+    build-essential
+
+# NOTE: python virtualenv is used here such that installed
+# packages may be moved to production image easily by copying the venv
+RUN python -m venv ${VIRTUAL_ENV}
+
+RUN pip install --upgrade --no-cache-dir \
+    pip~=20.2.2  \
+    wheel \
+    setuptools
+
+WORKDIR /build
+
+# install base 3rd party dependencies
+# NOTE: copies to /build to avoid overwriting later which would invalidate this layer
+COPY --chown=scu:scu services/service-sidecar/requirements/_base.txt .
+RUN pip --no-cache-dir install -r _base.txt
+
+# --------------------------Cache stage -------------------
+# CI in master buils & pushes this target to speed-up image build
+#
+#  + /build
+#    + services/service-sidecar [scu:scu] WORKDIR
+#
+FROM build as cache
+
+ENV SC_BUILD_TARGET cache
+
+COPY --chown=scu:scu packages /build/packages
+COPY --chown=scu:scu services/service-sidecar /build/services/service-sidecar
+
+WORKDIR /build/services/service-sidecar
+
+RUN pip --no-cache-dir install -r requirements/prod.txt &&\
+    pip --no-cache-dir list -v
+
+
+# --------------------------Production stage -------------------
+# Final cleanup up to reduce image size and startup setup
+# Runs as scu (non-root user)
+#
+#  + /home/scu     $HOME = WORKDIR
+#    + services/service-sidecar [scu:scu]
+#
+FROM base as production
+
+ENV SC_BUILD_TARGET=production \
+    SC_BOOT_MODE=production
+
+ENV PYTHONOPTIMIZE=TRUE
+
+WORKDIR /home/scu
+
+# Starting from clean base image, copies pre-installed virtualenv from cache
+COPY --chown=scu:scu --from=cache  ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+
+# Copies booting scripts
+COPY --chown=scu:scu services/scheduler/docker services/scheduler/docker
+RUN chmod +x services/scheduler/docker/*.sh
+
+HEALTHCHECK --interval=30s \
+    --timeout=20s \
+    --start-period=30s \
+    --retries=3 \
+    CMD ["python3", "services/service-sidecar/docker/healthcheck.py", "http://localhost:8000/"]
+
+EXPOSE 8000
+
+ENTRYPOINT [ "/bin/sh", "services/service-sidecar/docker/entrypoint.sh" ]
+CMD ["/bin/sh", "services/service-sidecar/docker/boot.sh"]
+
+
+# --------------------------Development stage -------------------
+# Source code accessible in host but runs in container
+# Runs as myu with same gid/uid as host
+# Placed at the end to speed-up the build if images targeting production
+#
+#  + /devel         WORKDIR
+#    + services  (mounted volume)
+#
+FROM build as development
+
+ENV SC_BUILD_TARGET=development
+
+WORKDIR /devel
+
+RUN chown -R scu:scu ${VIRTUAL_ENV}
+
+EXPOSE 8000
+EXPOSE 3000
+
+ENTRYPOINT ["/bin/sh", "services/service-sidecar/docker/entrypoint.sh"]
+CMD ["/bin/sh", "services/service-sidecar/docker/boot.sh"]

services/service-sidecar/Makefile

@@ -0,0 +1,59 @@
+include ../../scripts/common.Makefile
+
+APP_NAME := $(notdir $(CURDIR))
+
+.DEFAULT_GOAL := help
+
+.PHONY: help
+help: ## display this message
+	@awk 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+.PHONY: _ensure-in-venv
+_ensure-in-venv:
+	@python3 -c "import os; os.environ['VIRTUAL_ENV']" || (echo "\n>>>> You are not in a virtualenv. Activate one <<<<\n"; exit 1)
+
+.PHONY: reqs
+reqs: _ensure-in-venv ## compiles pip requirements (.in -> .txt)
+	@$(MAKE_C) requirements reqs
+
+.PHONY: compile-dependencies
+compile-dependencies: _ensure-in-venv ## snapshot test & package dependencies
+	@pip-compile --build-isolation requirements/_base.in --output-file requirements/_base.txt
+	@pip-compile --build-isolation requirements/_test.in --output-file requirements/_test.txt
+
+.PHONY: install-dev-dependencies
+install-dev-dependencies: _ensure-in-venv ## install depenencies for development
+	@pip install -r requirements/_base.txt
+
+.PHONY: install-dev-dependencies
+install-dev-dependencies: _ensure-in-venv ## install depenencies for development
+	@pip install -r requirements/_base.txt
+
+.PHONY: dev-run
+dev-run: _ensure-in-venv ## starts the container on its own
+	@docker run -it --rm \
+			-p 8000:8000 \
+			-v $(CURDIR):/devel/services/service-sidecar \
+			local/service-sidecar:development
+
+.PHONY: ci-tox-pylint
+ci-tox-pylint: ## runs pylint in a new enviornment
+	@tox -r -e pylint
+
+.PHONY: ci-tox-mypy
+ci-tox-mypy: ## runs mypy in a new enviornment
+	@tox -r -e mypy
+
+.PHONY: ci-tox-tests
+ci-tox-tests: ## runs tests with coverage in a new enviornment
+	@tox -r -e py38,report
+
+.PHONY: ci-install-requirements
+ci-install-requirements: ## runs tests with coverage in a new enviornment
+	@pip install -r requirements/dev.txt
+
+.PHONY: run-github-action-locally
+run-github-action-locally:  ## runs the defined github action from the workflow locally
+	# Note: ⚡ act is required https://github.com/nektos/act
+	# Note: ⚠️  "act-environments-ubuntu:18.04" image is 18Gb takes a lot the first time
+	@act -C ../../. -P ubuntu-18.04=nektos/act-environments-ubuntu:18.04 -j unit-test-service-sidecar

services/service-sidecar/VERSION

@@ -0,0 +1 @@
+0.0.2

services/service-sidecar/docker/boot.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+set -o errexit
+set -o nounset
+
+IFS=$(printf '\n\t')
+
+INFO="INFO: [$(basename "$0")] "
+
+# BOOTING application ---------------------------------------------
+echo "$INFO" "Booting in ${SC_BOOT_MODE} mode ..."
+echo "$INFO" "User :$(id "$(whoami)")"
+echo "$INFO" "Workdir : $(pwd)"
+
+if [ "${SC_BUILD_TARGET}" = "development" ]; then
+  echo "$INFO" "Environment :"
+  printenv | sed 's/=/: /' | sed 's/^/    /' | sort
+  echo "$INFO" "Python :"
+  python --version | sed 's/^/    /'
+  command -v python | sed 's/^/    /'
+
+  cd services/service-sidecar || exit 1
+  pip --quiet --no-cache-dir install -r requirements/dev.txt
+  cd - || exit 1
+  echo "$INFO" "PIP :"
+  pip list | sed 's/^/    /'
+fi
+
+# RUNNING application ----------------------------------------
+if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ]
+then
+  # NOTE: ptvsd is programmatically enabled inside of the service
+  # this way we can have reload in place as well
+  exec uvicorn sidecar.app:app --reload --host 0.0.0.0
+else
+  exec service-sidecar-startup
+fi