🔨 adding helper release monitoring script

scripts/release/monitor/README.md

@@ -0,0 +1,7 @@
+# Helper for monitoring of release
+`python monitor_release/cli.py --help`
+
+Check current status of containers
+`python monitor_release/cli.py master containers`
+Check running sidecars:
+`python monitor_release/cli.py master sidecars`

scripts/release/monitor/monitor_release/cli.py

@@ -0,0 +1,29 @@
+from enum import Enum
+
+import typer
+from models import Deployment
+from portainer import check_containers_deploys, check_running_sidecars
+from rich.console import Console
+from settings import get_settings
+
+console = Console()
+
+
+class Action(str, Enum):
+    containers = "containers"
+    sidecars = "sidecars"
+
+
+def main(deployment: Deployment, action: Action):
+    settings = get_settings(deployment)
+    console.print(f"Deployment: {deployment}")
+    console.print(f"Action: {action}")
+
+    if action == Action.containers:
+        check_containers_deploys(settings, deployment)
+    if action == Action.sidecars:
+        check_running_sidecars(settings)
+
+
+if __name__ == "__main__":
+    typer.run(main)

scripts/release/monitor/monitor_release/models.py

@@ -0,0 +1,10 @@
+from enum import Enum
+
+
+class Deployment(str, Enum):
+    master = "master"
+    aws_staging = "aws-staging"
+    dalco_staging = "dalco-staging"
+    aws_production = "aws-production"
+    dalco_production = "dalco-production"
+    tip_production = "tip-production"

scripts/release/monitor/monitor_release/portainer.py

@@ -0,0 +1,65 @@
+from portainer_utils import (
+    check_simcore_deployed_services,
+    check_simcore_running_sidecars,
+    get_bearer_token,
+    get_containers,
+    get_services,
+    get_tasks,
+)
+from rich.console import Console
+from rich.table import Table
+
+console = Console()
+
+
+def check_containers_deploys(settings, deployment):
+    token = get_bearer_token(settings)
+    services = get_services(settings, token)
+    tasks = get_tasks(settings, token)
+    containers = get_containers(settings, token)
+
+    output = check_simcore_deployed_services(settings, services, tasks, containers)
+
+    table = Table(
+        "Service",
+        "Status",
+        "Last Updated",
+        "Git SHA",
+        title=f"[bold yellow]{deployment.upper()}[/bold yellow]",
+    )
+    for item in output.values():
+        service_name = item["service_name"]
+        container_status = "[bold red]Not running[/bold red]"
+        container_timestamp = None
+        container_git_sha = None
+        for task in item["tasks"]:
+            oldest_running_task_timestamp = None
+            if task["status"] == "running":
+                if (
+                    oldest_running_task_timestamp is None
+                    or oldest_running_task_timestamp > task["timestamp"]
+                ):
+                    container_status = f"[green]{task['status']}[/green]"
+                    container_timestamp = f"{task['timestamp']}"
+                    container_git_sha = task["git_sha"]
+
+                    oldest_running_task_timestamp = task["timestamp"]
+            if task["status"] == "starting":
+                container_status = f"[blue]{task['status']}[/blue]"
+                container_timestamp = f"{task['timestamp']}"
+                container_git_sha = task["git_sha"]
+                break
+
+        table.add_row(
+            service_name, container_status, container_timestamp, container_git_sha
+        )
+
+    console.print(table)
+
+
+def check_running_sidecars(settings):
+    token = get_bearer_token(settings)
+    services = get_services(settings, token)
+
+    output = check_simcore_running_sidecars(settings, services)
+    console.print(output)

scripts/release/monitor/monitor_release/portainer_utils.py

@@ -0,0 +1,110 @@
+import json
+
+import arrow
+import requests
+from settings import Settings
+
+
+def get_bearer_token(settings: Settings):
+    headers = {"accept": "application/json", "Content-Type": "application/json"}
+    payload = json.dumps(
+        {
+            "Username": settings.portainer_username,
+            "Password": settings.portainer_password,
+        }
+    )
+    response = requests.post(
+        f"{settings.portainer_url}/portainer/api/auth",
+        headers=headers,
+        data=payload,
+    )
+    bearer_token = response.json()["jwt"]
+    return bearer_token
+
+
+def get_services(settings: Settings, bearer_token):
+    services_url = f"{settings.portainer_url}/portainer/api/endpoints/{settings.portainer_endpoint_version}/docker/services"
+    response = requests.get(
+        services_url,
+        headers={
+            "Authorization": "Bearer " + bearer_token,
+            "Content-Type": "application/json",
+        },
+    )
+    services = response.json()
+    return services
+
+
+def get_tasks(settings: Settings, bearer_token):
+    tasks_url = f"{settings.portainer_url}/portainer/api/endpoints/{settings.portainer_endpoint_version}/docker/tasks"
+    response = requests.get(
+        tasks_url,
+        headers={
+            "Authorization": "Bearer " + bearer_token,
+            "Content-Type": "application/json",
+        },
+    )
+    tasks = response.json()
+    return tasks
+
+
+def get_containers(settings: Settings, bearer_token):
+    bearer_token = get_bearer_token(settings)
+
+    containers_url = f"{settings.portainer_url}/portainer/api/endpoints/{settings.portainer_endpoint_version}/docker/containers/json?all=true"
+    response = requests.get(
+        containers_url,
+        headers={
+            "Authorization": "Bearer " + bearer_token,
+            "Content-Type": "application/json",
+        },
+    )
+    containers = response.json()
+    return containers
+
+
+def check_simcore_running_sidecars(settings: Settings, services):
+    running_sidecars = []
+    for service in services:
+        if (
+            service["Spec"]["Name"].startswith("dy-sidecar")
+            and service["Spec"]["Labels"]["io.simcore.runtime.swarm-stack-name"]
+            == settings.swarm_stack_name
+        ):
+            running_sidecars.append(service["Spec"]["Name"])
+    return running_sidecars
+
+
+def _generate_containers_map(containers):
+    container_map = {}
+    for container in containers:
+        container_map[container["Id"]] = {
+            "git_sha": container.get("Labels").get("org.label-schema.vcs-ref")
+        }
+    return container_map
+
+
+def check_simcore_deployed_services(settings: Settings, services, tasks, containers):
+    container_map = _generate_containers_map(containers)
+    service_task_map = {}
+    for service in services:
+        if service["Spec"]["Name"].startswith(settings.starts_with):
+            service_task_map[service["ID"]] = {
+                "service_name": service["Spec"]["Name"],
+                "tasks": [],
+            }
+
+    for task in tasks:
+        if task["ServiceID"] in service_task_map:
+            container_id = task["Status"]["ContainerStatus"]["ContainerID"]
+
+            service_task_map[task["ServiceID"]]["tasks"].append(
+                {
+                    "created_at": arrow.get(task["CreatedAt"]).datetime,
+                    "status": task["Status"]["State"],
+                    "timestamp": arrow.get(task["Status"]["Timestamp"]).datetime,
+                    "git_sha": container_map.get(container_id, {}).get("git_sha"),
+                }
+            )
+
+    return service_task_map

scripts/release/monitor/monitor_release/postgres.py

@@ -0,0 +1 @@
+# placeholder

scripts/release/monitor/monitor_release/settings.py

@@ -0,0 +1,98 @@
+import os
+
+from dotenv import load_dotenv
+from pydantic import BaseModel
+
+
+class Settings(BaseModel):
+    portainer_url: str
+    portainer_username: str
+    portainer_password: str
+    starts_with: str
+    swarm_stack_name: str
+    portainer_endpoint_version: int
+
+
+def get_settings(deployment):
+    load_dotenv(".env")
+
+    if deployment == "master":
+        portainer_url = os.getenv("MASTER_PORTAINER_URL")
+        portainer_username = os.getenv("MASTER_PORTAINER_USERNAME")
+        portainer_password = os.getenv("MASTER_PORTAINER_PASSWORD")
+
+        return Settings(
+            portainer_url=portainer_url,
+            portainer_username=portainer_username,
+            portainer_password=portainer_password,
+            starts_with="master-simcore_master",
+            swarm_stack_name="master-simcore",
+            portainer_endpoint_version=1,
+        )
+    if deployment == "dalco-staging":
+        portainer_url = os.getenv("DALCO_STAGING_PORTAINER_URL")
+        portainer_username = os.getenv("DALCO_STAGING_PORTAINER_USERNAME")
+        portainer_password = os.getenv("DALCO_STAGING_PORTAINER_PASSWORD")
+
+        return Settings(
+            portainer_url=portainer_url,
+            portainer_username=portainer_username,
+            portainer_password=portainer_password,
+            starts_with="staging-simcore_staging",
+            swarm_stack_name="staging-simcore",
+            portainer_endpoint_version=1,
+        )
+    if deployment == "dalco-production":
+        portainer_url = os.getenv("DALCO_PRODUCTION_PORTAINER_URL")
+        portainer_username = os.getenv("DALCO_PRODUCTION_PORTAINER_USERNAME")
+        portainer_password = os.getenv("DALCO_PRODUCTION_PORTAINER_PASSWORD")
+
+        return Settings(
+            portainer_url=portainer_url,
+            portainer_username=portainer_username,
+            portainer_password=portainer_password,
+            starts_with="production-simcore_production",
+            swarm_stack_name="production-simcore",
+            portainer_endpoint_version=1,
+        )
+    if deployment == "tip-production":
+        portainer_url = os.getenv("TIP_PRODUCTION_PORTAINER_URL")
+        portainer_username = os.getenv("TIP_PRODUCTION_PORTAINER_USERNAME")
+        portainer_password = os.getenv("TIP_PRODUCTION_PORTAINER_PASSWORD")
+
+        return Settings(
+            portainer_url=portainer_url,
+            portainer_username=portainer_username,
+            portainer_password=portainer_password,
+            starts_with="production-simcore_production",
+            swarm_stack_name="production-simcore",
+            portainer_endpoint_version=1,
+        )
+    if deployment == "aws-staging":
+        portainer_url = os.getenv("AWS_STAGING_PORTAINER_URL")
+        portainer_username = os.getenv("AWS_STAGING_PORTAINER_USERNAME")
+        portainer_password = os.getenv("AWS_STAGING_PORTAINER_PASSWORD")
+
+        return Settings(
+            portainer_url=portainer_url,
+            portainer_username=portainer_username,
+            portainer_password=portainer_password,
+            starts_with="staging-simcore_staging",
+            swarm_stack_name="staging-simcore",
+            portainer_endpoint_version=2,
+        )
+    if deployment == "aws-production":
+        portainer_url = os.getenv("AWS_PRODUCTION_PORTAINER_URL")
+        portainer_username = os.getenv("AWS_PRODUCTION_PORTAINER_USERNAME")
+        portainer_password = os.getenv("AWS_PRODUCTION_PORTAINER_PASSWORD")
+
+        return Settings(
+            portainer_url=portainer_url,
+            portainer_username=portainer_username,
+            portainer_password=portainer_password,
+            starts_with="production-simcore_production",
+            swarm_stack_name="production-simcore",
+            portainer_endpoint_version=2,
+        )
+    else:
+        raise ValueError("Invalid environment type provided.")

scripts/release/monitor/requirements.in

@@ -0,0 +1,5 @@
+python-dotenv
+pydantic
+typer[all]
+rich
+requests

scripts/release/monitor/requirements.txt

@@ -0,0 +1,47 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile
+#
+annotated-types==0.5.0
+    # via pydantic
+certifi==2023.5.7
+    # via requests
+charset-normalizer==3.2.0
+    # via requests
+click==8.1.5
+    # via typer
+colorama==0.4.6
+    # via typer
+idna==3.4
+    # via requests
+markdown-it-py==3.0.0
+    # via rich
+mdurl==0.1.2
+    # via markdown-it-py
+pydantic==2.0.3
+    # via -r requirements.in
+pydantic-core==2.3.0
+    # via pydantic
+pygments==2.15.1
+    # via rich
+python-dotenv==1.0.0
+    # via -r requirements.in
+requests==2.31.0
+    # via -r requirements.in
+rich==13.4.2
+    # via
+    #   -r requirements.in
+    #   typer
+shellingham==1.5.0.post1
+    # via typer
+typer[all]==0.9.0
+    # via -r requirements.in
+typing-extensions==4.7.1
+    # via
+    #   pydantic
+    #   pydantic-core
+    #   typer
+urllib3==2.0.3
+    # via requests