Merge branch 'main' into feature/models

# Conflicts:
#	poetry.lock
#	pyproject.toml

.github/workflows/build-retail-setup-push-tag-shared.yaml.yaml

@@ -0,0 +1,137 @@
+name: Build Retail Setup in Shared (Push Tag)
+
+on:
+  push:
+    tags:
+      - '*.*.*-develop'
+      - '*.*.*-staging'
+      - '*.*.*'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set variables
+        run: |
+            TAG="$( echo "${GITHUB_REF}" | cut -d'/' -f3 )"
+            if grep -qs -e '^.*.*-develop' <<< "${TAG}" ; then
+              echo "Found environment: DEVELOP - ${TAG}"
+              echo "MANIFESTS_ENVIRONMENT=develop" | tee "${GITHUB_ENV}"
+            elif grep -qs -e '^.*.*-staging' <<< "${TAG}" ; then
+              echo "Found environment: STAGING - ${TAG}"
+              echo "MANIFESTS_ENVIRONMENT=staging" | tee -a "${GITHUB_ENV}"
+            elif grep -qs -e '^.*.*' <<< "${TAG}" ; then
+              echo "No environment found, assuming: PRODUCTION - ${TAG}"
+              echo "MANIFESTS_ENVIRONMENT=production" | tee -a "${GITHUB_ENV}"
+            else
+              echo 'Not a valid tag. Skipping...'
+              exit 1
+            fi
+            echo "TAG=$TAG" | tee -a "${GITHUB_ENV}"
+            VERSION="${TAG}"
+            echo "VERSION=${VERSION}" | tee -a "${GITHUB_ENV}"
+            echo "COMMIT_SHA=$GITHUB_SHA" | tee -a "${GITHUB_ENV}"
+            echo "IMAGE_TAG=${{ secrets.ECR }}/retail:setup-${TAG}" | tee -a "${GITHUB_ENV}"
+            echo "IMAGE_SOURCE_URL=https://github.com/weni-ai/retail-setup" | tee -a "${GITHUB_ENV}"
+            echo "MANIFESTS_REPOSITORY=weni-ai/kubernetes-manifests-retail" | tee -a "${GITHUB_ENV}"
+            echo "MANIFESTS_APPLICATION=retail-setup" | tee -a "${GITHUB_ENV}"
+            echo "MANIFESTS_PATCH_TARGET=deployment.json" | tee -a "${GITHUB_ENV}"
+
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          ref: "${{env.GITHUB_SHA}}"
+          # On non dispatch build not need the repo
+          #repository: Ilhasoft/weni-engine
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to ECR
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ secrets.ECR }}
+          username: ${{ secrets.AWS_ACCESS_KEY_ID_SHARED }}       # Credentials used to push image to ECR repository
+          password: ${{ secrets.AWS_SECRET_ACCESS_KEY_SHARED }}
+
+      - name: Build and push - Retail Setup Image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          labels: |
+            tag=${{env.TAG}}
+            commit=${{env.COMMIT_SHA}}
+            repository=${{env.IMAGE_SOURCE_URL}}
+          file: docker/Dockerfile
+          push: true
+          tags: "${{env.IMAGE_TAG}}"
+          no-cache: true
+
+      - name: Check out Kubernetes Manifests
+        # Now, checkout in kubernetes manifests to update image in deployments patches
+        uses: actions/checkout@master
+        with:
+          ref: main
+          repository: "${{ env.MANIFESTS_REPOSITORY }}"
+          token: "${{ secrets.DEVOPS_GITHUB_PERMANENT_TOKEN }}"
+          path: ./kubernetes-manifests/
+          # Its uses other path to not mix with source code repository used to build
+
+      - name: Update image on deployment
+        run: |
+            which jq > /dev/null 2>&1 || ( sudo apt update ; sudo apt install -y jq )
+            # Dep: coreutils
+            verlte() {
+              [ "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
+            }
+            verlt(){
+              [ "$1" = "$2" ] && return 1 || verlte $1 $2
+            }
+            export PROJECT_DIR="${{ env.MANIFESTS_APPLICATION }}"
+            ENV_DIR="kubernetes-manifests/${{ env.MANIFESTS_APPLICATION }}/${MANIFESTS_ENVIRONMENT}"
+            for e in ${ENV_DIR}; do
+              echo "Update ${e}:"
+              if [ ! -d "${e}" ] ; then
+                echo "${e}: Does not exist, skipping"
+              elif [ ! -r "${e}/kustomization.yaml" ] ; then
+                echo "${e}/kustomization.yaml: Does not readable, skipping"
+              elif [ ! -r "${e}/${{ env.MANIFESTS_PATCH_TARGET }}" ] ; then
+                echo "${e}/${{ env.MANIFESTS_PATCH_TARGET }}: Does not readable, skipping"
+              else
+                OLD_IMAGE=$(
+                  cat "${e}/${{ env.MANIFESTS_PATCH_TARGET }}" \
+                    | jq '.[] | select(.path == "/spec/template/spec/containers/0/image") | .value'
+                )
+                echo "Old image to replace: ${OLD_IMAGE}"
+                OLD_VERSION=$(
+                  echo "${OLD_IMAGE}" \
+                    | sed s'/^.*[v:-]\([0-9]*\.[0-9]*\.[0-9]*\).*$/\1/'g \
+                    | head -n1
+                )
+                echo "Old image version to compare: ${OLD_VERSION}<=${{env.VERSION}}"
+                if verlte "${OLD_VERSION}" "${VERSION}" || [[ ! "${OLD_VERSION}" =~ [0-9]+\.[0-9]+\.[0-9]+ ]] ; then
+                  echo 'New configurations:'
+                  new_configuration=$(
+                    cat "${e}/${{ env.MANIFESTS_PATCH_TARGET }}" \
+                      | jq '(..|select(.path == "/spec/template/spec/containers/0/image")?) += {value: "'"${{env.IMAGE_TAG}}"'"}'
+                  )
+                  echo "${new_configuration}"
+                  echo "${new_configuration}" > "${e}/${{ env.MANIFESTS_PATCH_TARGET }}"
+                else
+                  echo "Version in file is greater than build, skipping update yaml"
+                fi
+              fi
+            done
+
+      - name: Commit & Push changes
+        # Once made a change, commit new configuration
+        uses: actions-js/push@master
+        with:
+          github_token: "${{ secrets.DEVOPS_GITHUB_PERMANENT_TOKEN }}"
+          repository: "${{ env.MANIFESTS_REPOSITORY }}"
+          directory: ./kubernetes-manifests/
+          branch: main
+          message: "From Retail Setup Build (Push Tag ${{ env.MANIFESTS_ENVIRONMENT }})"

.gitignore

@@ -60,6 +60,7 @@ coverage.xml
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
+static/
 
 # Flask stuff:
 instance/

docker/Dockerfile

@@ -0,0 +1,100 @@
+#syntax = docker/dockerfile:1
+
+ARG PYTHON_VERSION="3.10"
+ARG POETRY_VERSION="1.8.1"
+
+ARG BUILD_DEPS="\
+  gcc bzip2 git curl libpq-dev gettext \
+  libgdal-dev python3-cffi python3-gdal \
+  python3-dev default-libmysqlclient-dev build-essential \
+  build-essential \
+  cmake \
+  autoconf pkg-config autoconf libtool automake \
+  libmariadb-dev"
+
+ARG RUNTIME_DEPS="\
+  git \
+  tzdata \
+  postgresql-client \
+  netcat-traditional \
+  curl \
+  gosu \
+  gdal-bin"
+
+FROM python:${PYTHON_VERSION}-slim as base
+
+ARG POETRY_VERSION
+
+ENV PYTHONUNBUFFERED=1 \
+  PYTHONDONTWRITEBYTECODE=1 \
+  DEBIAN_FRONTEND=noninteractive \
+  PROJECT=Retail-engine \
+  PROJECT_PATH=/home/app \
+  APP_USER=app_user \
+  APP_GROUP=app_group \
+  USER_ID=1999 \
+  GROUP_ID=1999 \
+  PIP_DISABLE_PIP_VERSION_CHECK=1 \
+  PATH="/install/bin:${PATH}" \
+  APP_PORT=${APP_PORT} \
+  APPLICATION_NAME="Retail-engine" \
+  RUNTIME_DEPS=${RUNTIME_DEPS} \
+  BUILD_DEPS=${BUILD_DEPS} \
+  PYTHONIOENCODING=UTF-8 \
+  LIBRARY_PATH=/lib:/usr/lib
+
+ARG COMPRESS_ENABLED
+ARG BRANDING_ENABLED
+
+RUN addgroup --gid "${GROUP_ID}" "${APP_GROUP}" \
+  && useradd --system -m -d "${PROJECT_PATH}" -u "${USER_ID}" -g "${GROUP_ID}" "${APP_USER}"
+
+WORKDIR "${PROJECT_PATH}"
+
+RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
+
+FROM base as build-poetry
+
+ARG POETRY_VERSION
+
+COPY pyproject.toml poetry.lock ./
+
+RUN --mount=type=cache,mode=0755,target=/pip_cache,id=pip pip install --cache-dir /pip_cache -U poetry=="${POETRY_VERSION}" \
+  && poetry cache clear -n --all pypi \
+  && poetry export --without-hashes --output requirements.txt
+#  && poetry add -n --lock $(cat pip-requires.txt) \
+
+FROM base as build
+
+ARG BUILD_DEPS
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  apt-get update \
+  && apt-get install --no-install-recommends --no-install-suggests -y ${BUILD_DEPS}
+
+COPY --from=build-poetry "${PROJECT_PATH}/requirements.txt" /tmp/dep/
+RUN --mount=type=cache,mode=0755,target=/pip_cache,id=pip pip install --cache-dir /pip_cache --prefix=/install -r /tmp/dep/requirements.txt
+
+FROM base
+
+ARG BUILD_DEPS
+ARG RUNTIME_DEPS
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  apt-get update \
+  && SUDO_FORCE_REMOVE=yes apt-get remove --purge -y ${BUILD_DEPS} \
+  && apt-get autoremove -y \
+  && apt-get install -y --no-install-recommends ${RUNTIME_DEPS} \
+  && rm -rf /usr/share/man /usr/share/doc
+
+COPY --from=build /install /usr/local
+COPY --chown=${APP_USER}:${APP_GROUP} . ${PROJECT_PATH}
+
+RUN ls
+
+USER "${APP_USER}:${APP_GROUP}"
+EXPOSE 8000
+ENTRYPOINT ["bash", "./docker/entrypoint.sh"]
+CMD ["start"]

docker/docker-compose.yml

@@ -17,3 +17,17 @@ services:
     environment:
       - RABBITMQ_DEFAULT_USER=guest
       - RABBITMQ_DEFAULT_PASS=guest
+
+  retail-setup:
+    build: 
+      context: ..
+      dockerfile: ./docker/Dockerfile
+    ports:
+      - "8000:8000"
+    environment:
+      - DEBUG=${DEBUG:-true}
+      - ALLOWED_HOSTS=${ALLOWED_HOSTS:-*}
+      - SECRET_KEY=${SECRET_KEY:-123}
+      - DATABASE_URL=${DATABASE_URL:-postgresql://retail:retail@database:5432/retail}
+    depends_on:
+      - database

docker/entrypoint.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+export GUNICORN_APP=${GUNICORN_APP:-"retail.wsgi"}
+export GUNICORN_CONF=${GUNICORN_CONF:-"${PROJECT_PATH}/docker/gunicorn.conf.py"}
+export LOG_LEVEL=${LOG_LEVEL:-"INFO"}
+
+do_gosu(){
+    user="$1"
+    shift 1
+
+    is_exec="false"
+    if [ "$1" = "exec" ]; then
+        is_exec="true"
+        shift 1
+    fi
+
+    if [ "$(id -u)" = "0" ]; then
+        if [ "${is_exec}" = "true" ]; then
+            exec gosu "${user}" "$@"
+        else
+            gosu "${user}" "$@"
+            return "$?"
+        fi
+    else
+        if [ "${is_exec}" = "true" ]; then
+            exec "$@"
+        else
+            eval '"$@"'
+            return "$?"
+        fi
+    fi
+}
+
+if [[ "start" == "$1" ]]; then
+    echo "Running collectstatic"
+    do_gosu "${APP_USER}:${APP_GROUP}" python manage.py collectstatic --noinput
+    echo "Starting server"
+    do_gosu "${APP_USER}:${APP_GROUP}" exec gunicorn "${GUNICORN_APP}" -c "${GUNICORN_CONF}"
+elif [[ "edaconsume" == "$1" ]]; then
+    echo "Running eda-consumer"
+    do_gosu "${APP_USER}:${APP_GROUP}" python manage.py edaconsume
+else
+    exec "$@"
+fi

docker/gunicorn.conf.py

@@ -0,0 +1,11 @@
+import os
+import multiprocessing
+
+workers = os.environ.get("GUNICORN_WORKERS", multiprocessing.cpu_count() * 2 + 1)
+proc_name = "retail"
+default_proc_name = proc_name
+accesslog = "-"
+errorlog = "-"
+capture_output = True
+timeout = 120
+bind = "0.0.0.0"