task(sdet): Introduce gen3 qa test controller (#493)

* task(sdet): Introduce gen3-qa-controller to trigger automated tests in Kubernetes

* move Dockerfile to root folder, replace legacy/unused dockerfile

* introduce poetry config

* fix container cmd

* trying to make this work with Poetry

* applying poetry best practices and copying gen3-qa framework

* grant execute permissions to the poetry binary

* fix poetry lock

* re-organize poetry folders

* setting workdir to location with pyproject.toml

* moving controller scripts to the correct folder

* switching to sdet user earlier in the flow

* adopting quay copy of alpine

* moving everything to sdet_home

* remove chmod

* install everything as root and change owner recursively to sdet

* debugging

* debugging2

* alpine sux, adopting python:slim img

* Revert "alpine sux, adopting python:slim img"

This reverts commit b84431a.

* running with poetry run python

Co-authored-by: Marcelo Costa <[email protected]>

Dockerfile

@@ -1,29 +1,88 @@
-# To run: docker run -d --name=dataportal -p 80:80 quay.io/cdis/data-portal
-# To check running container: docker exec -it dataportal /bin/bash
+FROM quay.io/cdis/alpine:3.12.1
 
-FROM ubuntu:18.04
+USER root
+
+ENV SDET_HOME /var/sdet_home
+
+ARG user=sdet
+ARG group=sdet
+ARG uid=1000
+ARG gid=1000
+
+RUN addgroup -g ${gid} ${group} \
+    && adduser --home "$SDET_HOME" --uid ${uid} --ingroup ${group} --disabled-password --shell /bin/sh ${user}
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential \
-    ca-certificates \
-    curl \
-    libssl-dev \
-    libcurl4-openssl-dev \
-    git \
-    nginx \
+# Install python/pip
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    POETRY_VERSION=1.1.4 \
+    POETRY_HOME="${SDET_HOME}/poetry" \
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    POETRY_NO_INTERACTION=1 \
+    PYSETUP_PATH="${SDET_HOME}/pysetup" \
+    VENV_PATH="${SDET_HOME}/pysetup/.venv"
+
+# prepend poetry and venv to path
+ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+
+RUN apk add --update --no-cache python3 \
+    && ln -sf python3 /usr/bin/python \
+    && python3 -m ensurepip \
+    && pip3 install --no-cache --upgrade pip setuptools
+
+# install everything else
+RUN set -xe && apk add --no-cache --virtual .build-deps \
+    zip \
+    unzip \
+    less \
     vim \
-    && pip install pip==9.0.3 \
-    && pip install requests \
-    && curl -sL https://deb.nodesource.com/setup_8.x | bash - \
-    && apt-get install -y --no-install-recommends nodejs \
-    && ln -sf /dev/stdout /var/log/nginx/access.log \
-    && ln -sf /dev/stderr /var/log/nginx/error.log
+    gcc \
+    libc-dev \
+    libffi-dev \
+    make \
+    openssl-dev \
+    pcre-dev \
+    zlib-dev \
+    linux-headers \
+    curl \
+    wget \
+    jq \
+    nodejs \
+    npm
+
+# Copy the gen3-qa framework scripts (test suites + service and utils modules)
+COPY codecept.conf.js \
+     package.json \
+     package-lock.json \
+     test_setup.js \
+     .eslintrc.js \
+     helpers \
+     hooks \
+     services \
+     suites \
+     utils ${SDET_HOME}/
+
+# install poetry - respects $POETRY_VERSION & $POETRY_HOME
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+
+# Copy only requirements to cache them in docker layer
+RUN mkdir -p ${SDET_HOME}/controller/gen3qa-controller
+WORKDIR ${SDET_HOME}/controller
+COPY controller/poetry.lock controller/pyproject.toml ${SDET_HOME}/controller/
+
+# copy controller scripts
+COPY controller/gen3qa-controller ${SDET_HOME}/controller/gen3qa-controller/
 
-COPY . /gen3-qa
+# Project initialization:
+# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
+RUN poetry install --no-dev
 
-WORKDIR /gen3-qa
+RUN chown -R ${user}:${group} ${SDET_HOME}
+USER sdet
 
-ARG APP=dev
-ARG BASENAME
+CMD ["poetry", "run", "python", "gen3qa-controller/gen3qa-controller.py"]

controller/gen3qa-controller/gen3qa-controller.py

@@ -0,0 +1,11 @@
+import kubernetes
+import os
+
+kubernetes.config.load_incluster_config()
+v1 = kubernetes.client.CoreV1Api()
+
+w = kubernetes.watch.Watch()
+
+for event in w.stream(v1.list_namespaced_pod, os.environ["MY_POD_NAMESPACE"]):
+  print(event)
+  # TODO: Parse tests-config.json and trigger tests for target services