Add basic deployment DAG

env.template → .env.template

@@ -33,6 +33,13 @@ AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=sqlite:////opt/airflow/db/airflow.db
 # Replace "access_key" and "secret+key" with the real values. Secret key must be URL-encoded
 AIRFLOW_CONN_AWS_DEFAULT=aws://test_key:test_secret@?region_name=us-east-1&endpoint_url=http://s3:5000
 
+# SSH connections
+AIRFLOW_CONN_SSH_MASTODON_CONN_ID=ssh://user@service
+AIRFLOW_CONN_SSH_MONOLITH_CONN_ID=ssh://user@service
+# HTTP connections
+AIRFLOW_CONN_MATRIX_WEBHOOK=https://matrix-webhook
+AIRFLOW_VAR_MATRIX_WEBHOOK_API_KEY=api_key
+
 S3_LOCAL_ENDPOINT=http://s3:5000
 AWS_CONN_ID=aws_default
 

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @OrcaCollective/team

docker-compose.dev.yml

@@ -34,7 +34,7 @@ services:
       MINIO_ROOT_USER: ${AWS_ACCESS_KEY}
       MINIO_ROOT_PASSWORD: ${AWS_SECRET_KEY}
       # Comma separated list of buckets to create on startup
-      BUCKETS_TO_CREATE: spd-lookup,airflow
+      BUCKETS_TO_CREATE: spd-lookup,airflow,techbloc-airflow-logs
     # Create empty buckets on every container startup
     # Note: $0 is included in the exec because "/bin/bash -c" swallows the first
     # argument, so it must be re-added at the beginning of the exec call

justfile

@@ -40,7 +40,7 @@ install: check-py-version
 
 # Create the .env file from the template
 dotenv:
-    @([ ! -f .env ] && cp env.template .env) || true
+    @([ ! -f .env ] && cp .env.template .env) || true
 
 # Run docker compose with the specified command
 _dc *args:

techbloc_airflow/dags/constants.py

@@ -0,0 +1,5 @@
+SSH_MASTODON_CONN_ID = "ssh_mastodon"
+SSH_MONOLITH_CONN_ID = "ssh_monolith"
+
+MATRIX_WEBHOOK_CONN_ID = "matrix_webhook"
+MATRIX_WEBHOOK_API_KEY = "matrix_webhook_api_key"

techbloc_airflow/dags/deployments/deployment_dags.py

@@ -0,0 +1,57 @@
+import json
+from datetime import datetime
+
+import constants
+from airflow.decorators import dag
+from airflow.providers.http.operators.http import SimpleHttpOperator
+from airflow.providers.ssh.operators.ssh import SSHOperator
+
+
+SERVICES = [
+    "1-312-hows-my-driving",
+    "spd-data-watch",
+    "OpenOversight",
+    "spd-lookup",
+]
+
+
+for service in SERVICES:
+    service_name = service.replace("-", "_")
+    dag_id = f"deploy_{service_name}"
+
+    @dag(
+        dag_id=dag_id,
+        start_date=datetime(2022, 11, 24),
+        catchup=False,
+        schedule=None,
+        tags=["deployment"],
+    )
+    def deployment_dag():
+
+        ssh_deploy = SSHOperator(
+            task_id=f"deploy_{service_name}",
+            ssh_conn_id=constants.SSH_MONOLITH_CONN_ID,
+            command="cd {{ params.service }} && just deploy",
+            # Note that AcceptEnv has to be set for IS_PROD on the host
+            # or this will fail silently!!
+            # https://airflow.apache.org/docs/apache-airflow-providers-ssh/stable/_api/airflow/providers/ssh/operators/ssh/index.html#airflow.providers.ssh.operators.ssh.SSHOperator  # noqa
+            environment={"IS_PROD": "true"},
+            params={
+                "service": service,
+            },
+        )
+
+        matrix_alert = SimpleHttpOperator(
+            task_id=f"notify_{service_name}_deploy",
+            http_conn_id=constants.MATRIX_WEBHOOK_CONN_ID,
+            data=json.dumps(
+                {
+                    "key": "{{ var.value.matrix_webhook_api_key }}",
+                    "body": f"Deployment complete for `{service}`",
+                }
+            ),
+        )
+
+        ssh_deploy >> matrix_alert
+
+    deployment_dag()