Indykite customizations

bq-workers/argocd-parser/Dockerfile

@@ -0,0 +1,39 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Use the official Python image.
+# https://hub.docker.com/_/python
+FROM python:3.7
+
+# Allow statements and log messages to immediately appear in the Cloud Run logs
+ENV PYTHONUNBUFFERED True
+
+# Copy application dependency manifests to the container image.
+# Copying this separately prevents re-running pip install on every code change.
+COPY requirements.txt .
+
+# Install production dependencies.
+RUN pip install -r requirements.txt
+
+# Copy local code to the container image.
+ENV APP_HOME /app
+WORKDIR $APP_HOME
+COPY . .
+
+# Run the web service on container startup.
+# Use gunicorn webserver with one worker process and 8 threads.
+# For environments with multiple CPU cores, increase the number of workers
+# to be equal to the cores available.
+CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app

bq-workers/argocd-parser/cloudbuild.yaml

@@ -0,0 +1,41 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+- # Build argocd-parser image
+  name: gcr.io/cloud-builders/docker:latest
+  args: ['build', 
+         '--tag=gcr.io/$PROJECT_ID/argocd-parser:${_TAG}', '.']
+  dir: 'bq-workers/argocd-parser'
+  id: build
+
+- # Push the container image to Artifact Registry
+  name: gcr.io/cloud-builders/docker
+  args: ['push', 'gcr.io/$PROJECT_ID/argocd-parser:${_TAG}']
+  waitFor: build
+  id: push
+
+- # Deploy to Cloud Run
+  name: google/cloud-sdk
+  args: ['gcloud', 'run', 'deploy', 'argocd-parser',
+         '--image', 'gcr.io/$PROJECT_ID/argocd-parser:${_TAG}',
+         '--region', '${_REGION}',
+         '--platform', 'managed'
+  ]
+  id: deploy
+  waitFor: push
+
+images: [
+  'gcr.io/$PROJECT_ID/argocd-parser:${_TAG}'
+]

bq-workers/argocd-parser/main.py

@@ -0,0 +1,92 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import os
+import json
+
+import shared
+
+from flask import Flask, request
+
+app = Flask(__name__)
+
+
+@app.route("/", methods=["POST"])
+def index():
+    """
+    Receives messages from a push subscription from Pub/Sub.
+    Parses the message, and inserts it into BigQuery.
+    """
+    event = None
+    if not request.is_json:
+        raise Exception("Expecting JSON payload")
+    envelope = request.get_json()
+    print(f"envelope recieved: {envelope}")
+
+    # Check that data has been posted
+    if not envelope:
+        raise Exception("Expecting JSON payload")
+    # Check that message is a valid pub/sub message
+    if "message" not in envelope:
+        raise Exception("Not a valid Pub/Sub Message")
+    msg = envelope["message"]
+
+    if "attributes" not in msg:
+        raise Exception("Missing pubsub attributes")
+
+    try:
+        event = process_argocd_event(msg)
+
+        # [Do not edit below]
+        shared.insert_row_into_bigquery(event)
+
+    except Exception as e:
+        entry = {
+                "severity": "WARNING",
+                "msg": "Data not saved to BigQuery",
+                "errors": str(e),
+                "json_payload": envelope
+            }
+        print(json.dumps(entry))
+
+    return "", 204
+
+
+def process_argocd_event(msg):
+    metadata = json.loads(base64.b64decode(msg["data"]).decode("utf-8").strip())
+
+    # Unique hash for the event
+    signature = shared.create_unique_id(msg)
+
+    argocd_event = {
+        "event_type": "deployment",  # Event type, eg "push", "pull_reqest", etc
+        "id": metadata["id"],  # Object ID, eg pull request ID
+        "metadata": json.dumps(metadata),  # The body of the msg
+        "time_created": metadata["time"],  # The timestamp of with the event
+        "signature": signature,  # The unique event signature
+        "msg_id": msg["message_id"],  # The pubsub message id
+        "source": "argocd",  # The name of the source, eg "github"
+    }
+
+    print(argocd_event)
+    return argocd_event
+
+
+if __name__ == "__main__":
+    PORT = int(os.getenv("PORT")) if os.getenv("PORT") else 8080
+
+    # This is used when running locally. Gunicorn is used to run the
+    # application on Cloud Run. See entrypoint in Dockerfile.
+    app.run(host="127.0.0.1", port=PORT, debug=True)

bq-workers/argocd-parser/main_test.py

@@ -0,0 +1,89 @@
+# Copyright 2020 Google, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import json
+
+import main
+import shared
+
+import mock
+import pytest
+
+
+@pytest.fixture
+def client():
+    main.app.testing = True
+    return main.app.test_client()
+
+
+def test_not_json(client):
+    with pytest.raises(Exception) as e:
+        client.post("/", data="foo")
+
+    assert "Expecting JSON payload" in str(e.value)
+
+
+def test_not_pubsub_message(client):
+    with pytest.raises(Exception) as e:
+        client.post(
+            "/",
+            data=json.dumps({"foo": "bar"}),
+            headers={"Content-Type": "application/json"},
+        )
+
+    assert "Not a valid Pub/Sub Message" in str(e.value)
+
+
+def test_missing_msg_attributes(client):
+    with pytest.raises(Exception) as e:
+        client.post(
+            "/",
+            data=json.dumps({"message": "bar"}),
+            headers={"Content-Type": "application/json"},
+        )
+
+    assert "Missing pubsub attributes" in str(e.value)
+
+
+def test_argocd_event_processed(client):
+    data = json.dumps({"foo": "bar", "id": "foo", "time": 0}).encode("utf-8")
+    pubsub_msg = {
+        "message": {
+            "data": base64.b64encode(data).decode("utf-8"),
+            "attributes": {"foo": "bar"},
+            "message_id": "foobar",
+        },
+    }
+
+    event = {
+        "event_type": "deployment",
+        "id": "foo",
+        "metadata": '{"foo": "bar", "id": "foo", "time": 0}',
+        "time_created": 0,
+        "signature": "a424b5326ac45bde4c42c9b74dc878e56623d84f",
+        "msg_id": "foobar",
+        "source": "argocd",
+    }
+
+    shared.insert_row_into_bigquery = mock.MagicMock()
+
+    r = client.post(
+        "/",
+        data=json.dumps(pubsub_msg),
+        headers={"Content-Type": "application/json"},
+    )
+
+    shared.insert_row_into_bigquery.assert_called_with(event)
+    assert r.status_code == 204

bq-workers/argocd-parser/requirements-test.txt

@@ -0,0 +1,2 @@
+-r requirements.txt
+pytest~=6.0.0

bq-workers/argocd-parser/requirements.txt

@@ -0,0 +1,4 @@
+Flask==2.0.3
+gunicorn==19.9.0
+google-cloud-bigquery==1.23.1
+git+https://github.com/GoogleCloudPlatform/fourkeys.git#egg=shared&subdirectory=shared

dashboard/Dockerfile

@@ -21,7 +21,7 @@ ENV GF_AUTH_ANONYMOUS_ORG_ROLE "Admin"
 
 # during setup, this variable will be configured onto the Cloud Run service
 # (default here to "US" as a fallback)
-ENV BQ_REGION "US"
+ENV BQ_REGION "EU"
 
 # Setting grafana config
 COPY grafana.ini /etc/grafana
@@ -32,4 +32,4 @@ COPY dashboards.yaml /etc/grafana/provisioning/dashboards
 COPY datasource.yaml /etc/grafana/provisioning/datasources
 
 # Installing the BigQuery Plugin
-RUN grafana-cli plugins install doitintl-bigquery-datasource
+RUN grafana-cli plugins install doitintl-bigquery-datasource

dashboard/cloudbuild.yaml

@@ -38,4 +38,4 @@ steps:
 # https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values
 substitutions:
   _TAG: latest
-  _REGION: us-central1
+  _REGION: europe-west3

data-generator/generate_data.py

@@ -209,7 +209,7 @@ def make_github_issue(root_cause):
             "updated_at": datetime.datetime.now(),
             "closed_at": datetime.datetime.now(),
             "number": random.randrange(0, 1000),
-            "labels": [{"name": "Incident"}],
+            "labels": [{"name": "incident"}],
             "body": "root cause: %s" % root_cause["id"],
         },
         "repository": {"name": "foobar"},

data-generator/generate_data_test.py

@@ -137,7 +137,7 @@ def valid_issue(vcs):
                 "updated_at": datetime.datetime(2021, 2, 2, 21, 20, 58, 77232),
                 "closed_at": datetime.datetime(2021, 2, 2, 21, 20, 58, 77235),
                 "number": 440,
-                "labels": [{"name": "Incident"}],
+                "labels": [{"name": "incident"}],
                 "body": "root cause: 2b04b6d3939608f19776193697e0e30c04d9c6b8",
             },
             "repository": {"name": "foobar"},

event-handler/sources.py

@@ -60,7 +60,7 @@ def circleci_verification(signature, body):
         # Get secret from Cloud Secret Manager
         secret = get_secret(PROJECT_NAME, "event-handler", "latest")
         # Compute the hashed signature
-        hashed = hmac.new(secret, body, 'sha256')
+        hashed = hmac.new(secret, body, "sha256")
         expected_signature += hashed.hexdigest()
 
     except Exception as e:
@@ -117,9 +117,7 @@ def get_secret(project_name, secret_name, version_num):
     """
     try:
         client = secretmanager.SecretManagerServiceClient()
-        name = client.secret_version_path(
-            project_name, secret_name, version_num
-        )
+        name = client.secret_version_path(project_name, secret_name, version_num)
         secret = client.access_secret_version(name)
         return secret.payload.data
     except Exception as e:
@@ -145,23 +143,17 @@ def get_source(headers):
     if "X-Pagerduty-Signature" in headers:
         return "pagerduty"
 
+    if "Argo-CD" in headers.get("User-Agent", ""):
+        return "argocd"
+
     return headers.get("User-Agent")
 
 
 AUTHORIZED_SOURCES = {
-    "github": EventSource(
-        "X-Hub-Signature", github_verification
-        ),
-    "gitlab": EventSource(
-        "X-Gitlab-Token", simple_token_verification
-        ),
-    "tekton": EventSource(
-        "tekton-secret", simple_token_verification
-        ),
-    "circleci": EventSource(
-        "Circleci-Signature", circleci_verification
-        ),
-    "pagerduty": EventSource(
-        "X-Pagerduty-Signature", pagerduty_verification
-        ),
+    "github": EventSource("X-Hub-Signature", github_verification),
+    "gitlab": EventSource("X-Gitlab-Token", simple_token_verification),
+    "tekton": EventSource("tekton-secret", simple_token_verification),
+    "circleci": EventSource("Circleci-Signature", circleci_verification),
+    "pagerduty": EventSource("X-Pagerduty-Signature", pagerduty_verification),
+    "argocd": EventSource("Argo-Signature", simple_token_verification),
 }

queries/changes.sql

@@ -2,9 +2,10 @@
 SELECT 
 source,
 event_type,
+repo_name,
 JSON_EXTRACT_SCALAR(commit, '$.id') change_id,
-TIMESTAMP_TRUNC(TIMESTAMP(JSON_EXTRACT_SCALAR(commit, '$.timestamp')),second) as time_created,
-FROM four_keys.events_raw e,
-UNNEST(JSON_EXTRACT_ARRAY(e.metadata, '$.commits')) as commit
+TIMESTAMP_TRUNC(TIMESTAMP(JSON_EXTRACT_SCALAR(commit, '$.timestamp')),second) AS time_created,
+FROM four_keys.events e,
+UNNEST(JSON_EXTRACT_ARRAY(e.metadata, '$.commits')) AS commit
 WHERE event_type = "push"
-GROUP BY 1,2,3,4
+GROUP BY 1,2,3,4,5