Adds dynamic-sidecar osparc service (🏗️actions required) (#2271)

* moved service-sidecar contents

* added task to run unittests

* added unittests bash commands

* added command to run unittests locally

* added entry to build dynamic-sidecar

* coverage and deploy now await for dynamic-sidecar

* renamed service-sidecar to dynamic-sidecar

* renaming missing folder

* renaming missing fodler part 2

* updating makefile

* updating tox

* removing tox

* added ci codestyle check before tests run

* removed unsued targets

* fixed error message

* added missing test

* added codeoowner entry

* moved mocked fucntions to the same module

* renamed module

* updating log warning messages for mocked calls

* added some help

* clrifying command usage

* using verbatim arguments

* fixed typo

* renived tests

* renaming ServiceSidecarSettings to DynamicSidecarSettings

* updated name in changelog

* fixed ptsv entrypoint

* fixed healt endpoint to fail when status is not healthy

* fixed route and test

* mappign docs on the same route as other services

* added api prefix for interested routes

* codestyle

* container api routes refactored

* renamed storage to shared_store

* refacted shared_store to use Pydantic models

* missed a rename

* added makefile entry to generate openapi spec

* adding correct mypy file

* mypy suggestions

* added openapi.json for the service

* codestyle also updates openapi.json

* added common scripts to enforce codestyles in development and CI

* imported statuses from fastapi/starlette definitions

* updated docstring

* fixed imports, now importing from starlette

* moved mark.asyncio

* moved act into bash script

* added act entry to common.Makefile

* added entry for development of dunamic-sidecar

* added test to check the spec has was updated

* added missing fixture

* updated readme

* removed unused APIs and renamed existing

* added docker-compose config validation

* moved settings and shared state to dependencies

* minor improvments

* updated requirements

* added missing service to SERVICES_LIST

* Some suggestions (#3) from sanderegg

* naming

* consistency

* shellcheck

* proposed refactoring on dynamic sidecar (#4)

* removes responses and raise exceptions for errors

* Minor typo

* Should include standard entrypoint

* refactor health and added dependencies

* minor

* fixes

* fixed codestyle

* fixed typos

* fixed compose API entrypoints and tests

* refactored api structure and endpoints

* extended utils

* adapted new tests

* updated API spec

* moving routes under the same module

* moved tests and updated openapi.json

* updated script entryoint name

* renamed endpoint

* refactor

* refactor error raising and staus code for resource not found

* regenerated openapi.json spec

* codestyle :\

* merged /containers and /containers:inspect

* wrong default value

* test clenup

* updated openapi.json

* clarify comment

* removed unused API

* removed command timeout when posting to createing services via docker-compose

* removing unecessary timeouts

* forgot to implement it

* after the request is accepted the list of container names is returned

* removed flag which is no longer used

* fixed tests

* moved ApplicationHealth and codestyle fixes

* added test to check it can run twice the same compos spec with different project-name

* fixed codestyle

* setting default and documentation to argument

* updated oepnapi.json

* fixed test to properly work based on workspace

* refactoring application models structure

* rending applicaiton strucutre similar to other fastapi services

* replacing with response.json()

* applied codestyle

* comitting new updated dependencies

* refactoring error codes

* clened up exception throwing

* restored error emssage

Co-authored-by: Andrei Neagu <[email protected]>
Co-authored-by: Sylvain <[email protected]>
Co-authored-by: Pedro Crespo-Valero <[email protected]>

.github/CODEOWNERS

@@ -22,6 +22,7 @@ Makefile                                  @pcrespov, @sanderegg
 /scripts/template-projects/               @odeimaiz, @pcrespov
 /services/api-server/                     @pcrespov
 /services/director*/                      @sanderegg, @pcrespov
+/services/dynamic-sidecar/                @GitHK
 /services/catalog/                        @pcrespov, @sanderegg
 /services/migration/                      @pcrespov
 /services/sidecar/                        @pcrespov, @mguidon

.github/workflows/ci-testing-deploy.yml

@@ -313,6 +313,55 @@ jobs:
           name: codeclimate-${{ github.job }}-coverage
           path: codeclimate.${{ github.job }}_coverage.json
 
+  unit-test-dynamic-sidecar:
+    name: "[unit] dynamic-sidecar"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: [3.6]
+        os: [ubuntu-20.04]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: setup docker
+        run: |
+          sudo ./ci/github/helpers/setup_docker_compose.bash
+          ./ci/github/helpers/setup_docker_experimental.bash
+      - name: setup python environment
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - uses: actions/cache@v2
+        name: getting cached data
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-dynamic-sidecar-${{ hashFiles('services/dynamic-sidecar/requirements/ci.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-dynamic-sidecar-
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: install
+        run: ./ci/github/unit-testing/dynamic-sidecar.bash install
+      - name: codestyle
+        run: ./ci/github/unit-testing/dynamic-sidecar.bash codestyle
+      - name: test
+        run: ./ci/github/unit-testing/dynamic-sidecar.bash test
+      - uses: codecov/codecov-action@v1
+        with:
+          flags: unittests #optional
+      - name: prepare codeclimate coverage file
+        run: |
+          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
+          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
+          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
+      - name: upload codeclimate coverage
+        uses: actions/upload-artifact@v2
+        with:
+          name: codeclimate-${{ github.job }}-coverage
+          path: codeclimate.${{ github.job }}_coverage.json
+
   unit-test-frontend:
     name: "[unit] frontend"
     runs-on: ${{ matrix.os }}
@@ -1752,6 +1801,7 @@ jobs:
         unit-test-director,
         unit-test-director-v2,
         unit-test-sidecar,
+        unit-test-dynamic-sidecar,
         unit-test-service-integration,
         unit-test-service-library,
         unit-test-models-library,
@@ -1806,6 +1856,7 @@ jobs:
         unit-test-director,
         unit-test-director-v2,
         unit-test-sidecar,
+        unit-test-dynamic-sidecar,
         unit-test-frontend,
         unit-test-python-linting,
         unit-test-service-integration,

Makefile

@@ -32,6 +32,7 @@ SERVICES_LIST := \
 	catalog \
 	director \
 	director-v2 \
+	dynamic-sidecar \
 	migration \
 	sidecar \
 	storage \

ci/github/unit-testing/dynamic-sidecar.bash

@@ -0,0 +1,33 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
+IFS=$'\n\t'
+
+install() {
+    bash ci/helpers/ensure_python_pip.bash;
+    pushd services/dynamic-sidecar; pip3 install -r requirements/ci.txt -r requirements/_tools.txt; popd;
+    pip list -v
+}
+
+codestyle(){
+    scripts/codestyle.bash ci simcore_service_dynamic_sidecar services/dynamic-sidecar 
+}
+
+test() {
+    pytest --cov=simcore_service_dynamic_sidecar --durations=10 --cov-append \
+          --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+          -v -m "not travis" services/dynamic-sidecar/tests/unit
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" > /dev/null
+then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

scripts/act.bash

@@ -0,0 +1,33 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit
+set -o nounset
+set -o pipefail
+IFS=$'\n\t'
+
+# points to root soruce directory of this project, usually ../../
+ROOT_PROJECT_DIR=$1
+# name of the job, usually defined in the .github/workflows/ci-testing-deploy.yaml
+JOB_TO_RUN=$2
+
+DOCKER_IMAGE_NAME=dind-act-runner
+ACT_RUNNER=ubuntu-20.04=catthehacker/ubuntu:act-20.04
+ACT_VERSION_TAG=v0.2.20 # from https://github.com/nektos/act/releases
+
+docker build -t $DOCKER_IMAGE_NAME - <<EOF
+FROM docker:dind
+
+RUN apk add curl bash
+RUN curl -fsSL  https://raw.githubusercontent.com/nektos/act/master/install.sh | bash /dev/stdin -d $ACT_VERSION_TAG
+
+WORKDIR /project
+
+CMD /bin/sh -c "act -v -P $ACT_RUNNER -j $JOB_TO_RUN"
+EOF
+
+echo "$(pwd)/${ROOT_PROJECT_DIR}"
+docker run --rm -it \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v "$(pwd)"/"${ROOT_PROJECT_DIR}":/project \
+  -v "$(pwd)"/ci-logs:/logs \
+  $DOCKER_IMAGE_NAME

scripts/codestyle.bash

@@ -0,0 +1,38 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit
+set -o nounset
+set -o pipefail
+IFS=$'\n\t'
+
+SRC_DIRECTORY_NAME=${2}
+BASE_PATH_DIR=${3-MISSING_DIR}
+
+# used for development (fails on pylint and mypy)
+development() {
+  echo "enforcing codestyle to source_directory=$SRC_DIRECTORY_NAME"
+  echo "isort"
+  isort setup.py src/"$SRC_DIRECTORY_NAME" tests
+  echo "black"
+  black src/"$SRC_DIRECTORY_NAME" tests/
+  echo "pylint"
+  pylint --rcfile=../../.pylintrc src/"$SRC_DIRECTORY_NAME" tests/
+  echo "mypy"
+  mypy --ignore-missing-imports --config-file ../../mypy.ini src/"$SRC_DIRECTORY_NAME" tests/
+}
+
+# invoked by ci as test (also fails on isort and black)
+ci() {
+  echo "checking codestyle in service=$BASE_PATH_DIR with source_directory=$SRC_DIRECTORY_NAME"
+  echo "isort"
+  isort --check setup.py "$BASE_PATH_DIR"/src/"$SRC_DIRECTORY_NAME" "$BASE_PATH_DIR"/tests
+  echo "black"
+  black --check "$BASE_PATH_DIR"/src/"$SRC_DIRECTORY_NAME" "$BASE_PATH_DIR"/tests
+  echo "pylint"
+  pylint --rcfile=.pylintrc "$BASE_PATH_DIR"/src/"$SRC_DIRECTORY_NAME" "$BASE_PATH_DIR"/tests
+  echo "mypy"
+  mypy --config-file mypy.ini --ignore-missing-imports "$BASE_PATH_DIR"/src/"$SRC_DIRECTORY_NAME" "$BASE_PATH_DIR"/tests
+}
+
+# Allows to call a function based on arguments passed to the script
+"$@"

scripts/common.Makefile

@@ -131,6 +131,16 @@ code-analysis: $(REPO_BASE_DIR)/.codeclimate.yml ## runs code-climate analysis
 	@-rm $(CURDIR)/.codeclimate.yml
 
 
+.PHONY: codestyle
+codestyle: ## enforces codestyle (isort & black) finally runs pylint & mypy
+	@$(SCRIPTS_DIR)/codestyle.bash development $(shell basename "${SRC_DIR}")
+
+.PHONY: github-workflow-job
+github-workflow-job: ## runs a github workflow job using act locally, run using "make github-workflow-job job=JOB_NAME"
+	# running job "${job}"
+	$(SCRIPTS_DIR)/act.bash ../.. ${job}
+
+
 .PHONY: version-patch version-minor version-major
 version-patch: ## commits version with bug fixes not affecting the cookiecuter config
 	$(_bumpversion)

services/docker-compose-build.yml

@@ -124,6 +124,22 @@ services:
         org.label-schema.vcs-url: "${VCS_URL}"
         org.label-schema.vcs-ref: "${VCS_REF}"
 
+  dynamic-sidecar:
+    image: local/dynamic-sidecar:${BUILD_TARGET:?build_target_required}
+    build:
+      context: ../
+      dockerfile: services/dynamic-sidecar/Dockerfile
+      cache_from:
+        - local/dynamic-sidecar:${BUILD_TARGET:?build_target_required}
+        - ${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:cache
+        - ${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest}
+      target: ${BUILD_TARGET:?build_target_required}
+      labels:
+        org.label-schema.schema-version: "1.0"
+        org.label-schema.build-date: "${BUILD_DATE}"
+        org.label-schema.vcs-url: "${VCS_URL}"
+        org.label-schema.vcs-ref: "${VCS_REF}"
+
   storage:
     image: local/storage:${BUILD_TARGET:?build_target_required}
     build:

services/dynamic-sidecar/.env-devel

@@ -0,0 +1,12 @@
+# Environment used to configure storage services
+#
+# - To expose in cmd:  export $(grep -v '^#' .env-devel | xargs -0)
+#
+
+# environs in Dockerfile ----------------
+SC_BOOT_MODE=local-development
+
+
+# service specific required vars
+DYNAMIC_SIDECAR_compose_namespace=dev-namespace
+DYNAMIC_SIDECAR_docker_compose_down_timeout=15

services/dynamic-sidecar/CHANGELOG.md

@@ -0,0 +1,13 @@
+# Changelog
+All notable changes to this service will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.0.1] - 2021-04-14
+### Added
+- First working version of the service
+- FastAPI based service
+- all spawned services and networks are removed when receiving `SIGTERM`