Editable dev container for connector proxy demo (#2097)

For easier local development and training purposes, this adds an editable dev container for the connector proxy demo as well as a local example connector that can be changed in the live environment without needing to poetry install a connector for another location.

Makefile

@@ -15,9 +15,19 @@ BACKEND_DEV_OVERLAY ?= spiffworkflow-backend/dev.docker-compose.yml
 FRONTEND_CONTAINER ?= spiffworkflow-frontend
 FRONTEND_DEV_OVERLAY ?= spiffworkflow-frontend/dev.docker-compose.yml
 
+CONNECTOR_PROXY_CONTAINER ?= spiffworkflow-connector
+CONNECTOR_PROXY_DEV_OVERLAY ?= connector-proxy-demo/dev.docker-compose.yml
+
+YML_FILES := -f docker-compose.yml \
+	-f $(BACKEND_DEV_OVERLAY) \
+	-f $(FRONTEND_DEV_OVERLAY) \
+	-f $(CONNECTOR_PROXY_DEV_OVERLAY) \
+	-f $(ARENA_DEV_OVERLAY)
+
 DOCKER_COMPOSE ?= RUN_AS=$(ME) docker compose $(YML_FILES)
 IN_ARENA ?= $(DOCKER_COMPOSE) run --rm $(ARENA_CONTAINER)
 IN_BACKEND ?= $(DOCKER_COMPOSE) run --rm $(BACKEND_CONTAINER)
+IN_CONNECTOR_PROXY ?= $(DOCKER_COMPOSE) run --rm $(CONNECTOR_PROXY_CONTAINER)
 IN_FRONTEND ?= $(DOCKER_COMPOSE) run --rm $(FRONTEND_CONTAINER)
 
 SPIFFWORKFLOW_BACKEND_ENV ?= local_development
@@ -27,11 +37,6 @@ NODE_MODULES_DIR ?= spiffworkflow-frontend/node_modules
 JUST ?=
 ARGS ?=
 
-YML_FILES := -f docker-compose.yml \
-	-f $(BACKEND_DEV_OVERLAY) \
-	-f $(FRONTEND_DEV_OVERLAY) \
-	-f $(ARENA_DEV_OVERLAY)
-
 all: dev-env start-dev run-pyl
 	@true
 
@@ -43,7 +48,7 @@ build-images:
 		--build-arg GROUP_NAME=$(GROUP_NAME) \
 		$(JUST)
 
-dev-env: stop-dev build-images poetry-i be-poetry-i be-db-clean fe-npm-i
+dev-env: stop-dev build-images poetry-i cp-poetry-i be-poetry-i be-db-clean fe-npm-i
 	@true
 
 start-dev: stop-dev
@@ -97,6 +102,18 @@ be-tests: be-clear-log-file
 be-tests-par: be-clear-log-file
 	$(IN_BACKEND) poetry run pytest -n auto -x --random-order $(ARGS) tests/spiffworkflow_backend/$(JUST)
 
+cp-sh:
+	$(IN_CONNECTOR_PROXY) /bin/bash
+
+cp-poetry-i:
+	$(IN_CONNECTOR_PROXY) poetry install
+
+cp-poetry-lock:
+	$(IN_CONNECTOR_PROXY) poetry lock --no-update
+
+cp-logs:
+	docker logs -f $(CONNECTOR_PROXY_CONTAINER)
+
 fe-lint-fix:
 	$(IN_FRONTEND) npm run lint:fix
 
@@ -155,6 +172,7 @@ include event-stream/demo.mk
 	start-dev stop-dev \
 	be-clear-log-file be-logs be-mypy be-poetry-i be-poetry-lock be-poetry-rm \
 	be-db-clean be-db-migrate be-sh be-sqlite be-tests be-tests-par \
+	cp-logs cp-poetry-i cp-poetry-lock \
 	fe-lint-fix fe-logs fe-npm-clean fe-npm-i fe-npm-rm fe-sh fe-unimported  \
 	git-debranch git-debranch-offline \
 	poetry-i poetry-rm pre-commit ruff run-pyl \

connector-proxy-demo/bin/run_server_locally

@@ -12,4 +12,4 @@ if [[ -z "${FLASK_DEBUG:-}" ]]; then
 fi
 
 export FLASK_SESSION_SECRET_KEY=super_secret_key
-poetry run flask run -p 7004
+poetry run flask run -p ${CONNECTOR_PROXY_PORT:-7004} --host=0.0.0.0

connector-proxy-demo/connector-example/.tool-versions

@@ -0,0 +1 @@
+python 3.11.0

connector-proxy-demo/connector-example/README.md

@@ -0,0 +1,9 @@
+# connector-example
+Example Connector
+
+
+## Happy Hacking
+
+When running the dev containers, you can edit this connector and see
+changes reflected next time you run a process model with a Service Task
+that calls this connector.

connector-proxy-demo/connector-example/pyproject.toml

@@ -0,0 +1,67 @@
+[tool.poetry]
+name = "connector-example"
+version = "1.0.0"
+description = "Example Connector for SpiffWorkflow Service Tasks"
+authors = ["Jon Herron <[email protected]>"]
+readme = "README.md"
+packages = [{include = "connector_example", from = "src" }]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+spiffworkflow-connector-command = {git = "https://github.com/sartography/spiffworkflow-connector-command.git", rev = "main"}
+
+[tool.poetry.group.dev.dependencies]
+mypy = "^1.6.0"
+ruff = "^0.0.292"
+pytest = "^7.4.2"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.ruff]
+select = [
+  "B",   # flake8-bugbear
+  "C",   # mccabe
+  "E",   # pycodestyle error
+  # "ERA",  # eradicate
+  "F",   # pyflakes
+  "N",   # pep8-naming
+  "PL",  # pylint
+  "S",   # flake8-bandit
+  "UP",  # pyupgrade
+  "W",   # pycodestyle warning
+  "I001" # isort
+]
+
+ignore = [
+  "C901",  # "complexity" category
+  "PLR",  # "refactoring" category has "too many lines in method" type stuff
+  "PLE1205"  # saw this Too many arguments for `logging` format string give a false positive once
+]
+
+line-length = 130
+
+# target python 3.10
+target-version = "py310"
+
+exclude = [
+  "migrations"
+]
+
+[tool.ruff.per-file-ignores]
+"migrations/versions/*.py" = ["E501"]
+"tests/**/*.py" = ["PLR2004", "S101"]  # PLR2004 is about magic vars, S101 allows assert
+
+[tool.ruff.isort]
+force-single-line = true
+
+[tool.mypy]
+strict = true
+disallow_any_generics = false
+warn_unreachable = true
+pretty = true
+show_column_numbers = true
+show_error_codes = true
+show_error_context = true

connector-proxy-demo/connector-example/src/connector_example/commands/example.py

@@ -0,0 +1,32 @@
+from typing import Any
+
+from spiffworkflow_connector_command.command_interface import CommandErrorDict
+from spiffworkflow_connector_command.command_interface import CommandResponseDict
+from spiffworkflow_connector_command.command_interface import ConnectorCommand
+from spiffworkflow_connector_command.command_interface import ConnectorProxyResponseDict
+
+
+class Example(ConnectorCommand):
+    def __init__(self,
+        message: str,
+    ):
+        self.message = message
+
+    def execute(self, _config: Any, _task_data: Any) -> ConnectorProxyResponseDict:
+        error: CommandErrorDict | None = None
+
+        return_response: CommandResponseDict = {
+            "body": {
+                "connector_response": f"You passed the example connector: '{self.message}'. Have a good day!",
+            },
+            "mimetype": "application/json",
+        }
+
+        result: ConnectorProxyResponseDict = {
+            "command_response": return_response,
+            "error": error,
+            "command_response_version": 2,
+            "spiff__logs": [],
+        }
+
+        return result

connector-proxy-demo/dev.Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.11.6-slim-bookworm AS base
+
+WORKDIR /app
+
+RUN pip install --upgrade pip
+RUN pip install poetry==1.8.1 pytest-xdist==3.5.0
+
+CMD ["./bin/run_server_locally"]

connector-proxy-demo/dev.docker-compose.yml

@@ -0,0 +1,16 @@
+
+services:
+  spiffworkflow-connector:
+    build:
+      context: connector-proxy-demo
+      dockerfile: dev.Dockerfile
+    user: "${RUN_AS}"
+    environment:
+      FLASK_DEBUG: "1"
+      POETRY_VIRTUALENVS_IN_PROJECT: "true"
+      XDG_CACHE_HOME: "/app/.cache"
+    env_file:
+      - path: .env
+        required: false
+    volumes:
+      - ./connector-proxy-demo:/app

connector-proxy-demo/pyproject.toml

@@ -17,6 +17,8 @@ connector-aws = { git = "https://github.com/sartography/connector-aws.git"}
 connector-http = {git = "https://github.com/sartography/connector-http.git"}
 connector-slack = {git = "https://github.com/sartography/connector-slack.git"}
 connector-smtp = {git = "https://github.com/sartography/connector-smtp.git"}
+connector-example = {develop = true, path = "./connector-example" }
+
 gunicorn = "^20.1.0"
 
 [build-system]

docker-compose.yml

@@ -27,7 +27,7 @@ services:
       SPIFFWORKFLOW_BACKEND_URL: "http://localhost:${SPIFF_BACKEND_PORT:-8000}"
 
       SPIFFWORKFLOW_BACKEND_BPMN_SPEC_ABSOLUTE_DIR: "/app/process_models"
-      SPIFFWORKFLOW_BACKEND_CONNECTOR_PROXY_URL: "http://spiffworkflow-connector:8004"
+      SPIFFWORKFLOW_BACKEND_CONNECTOR_PROXY_URL: "${SPIFFWORKFLOW_BACKEND_CONNECTOR_PROXY_URL:-http://spiffworkflow-connector:8004}"
       SPIFFWORKFLOW_BACKEND_DATABASE_TYPE: "sqlite"
       SPIFFWORKFLOW_BACKEND_DATABASE_URI: "sqlite:////app/db_volume/db.sqlite3"
       SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA: "false"