From ea51e4c51f5720b3c2ca13db239839bb6cc6535a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Dluho=C5=A1?= Date: Fri, 9 Sep 2022 10:52:08 +0200 Subject: [PATCH 1/3] Add ArgoCD support --- bq-workers/argocd-parser/Dockerfile | 39 ++++++++ bq-workers/argocd-parser/cloudbuild.yaml | 41 +++++++++ bq-workers/argocd-parser/main.py | 92 +++++++++++++++++++ bq-workers/argocd-parser/main_test.py | 89 ++++++++++++++++++ .../argocd-parser/requirements-test.txt | 2 + bq-workers/argocd-parser/requirements.txt | 4 + event-handler/sources.py | 30 +++--- queries/deployments.sql | 5 +- setup/setup.sh | 6 +- .../modules/fourkeys-argocd-parser/README.md | 39 ++++++++ .../modules/fourkeys-argocd-parser/main.tf | 77 ++++++++++++++++ .../fourkeys-argocd-parser/variables.tf | 26 ++++++ .../files/bq-workers/argocd-parser/Dockerfile | 39 ++++++++ .../bq-workers/argocd-parser/cloudbuild.yaml | 41 +++++++++ .../files/bq-workers/argocd-parser/main.py | 92 +++++++++++++++++++ .../bq-workers/argocd-parser/main_test.py | 89 ++++++++++++++++++ .../argocd-parser/requirements-test.txt | 2 + .../bq-workers/argocd-parser/requirements.txt | 4 + 18 files changed, 695 insertions(+), 22 deletions(-) create mode 100644 bq-workers/argocd-parser/Dockerfile create mode 100644 bq-workers/argocd-parser/cloudbuild.yaml create mode 100644 bq-workers/argocd-parser/main.py create mode 100644 bq-workers/argocd-parser/main_test.py create mode 100644 bq-workers/argocd-parser/requirements-test.txt create mode 100644 bq-workers/argocd-parser/requirements.txt create mode 100644 terraform/modules/fourkeys-argocd-parser/README.md create mode 100644 terraform/modules/fourkeys-argocd-parser/main.tf create mode 100644 terraform/modules/fourkeys-argocd-parser/variables.tf create mode 100644 terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/Dockerfile create mode 100644 terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/cloudbuild.yaml create mode 100644 terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/main.py create mode 100644 terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/main_test.py create mode 100644 terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/requirements-test.txt create mode 100644 terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/requirements.txt diff --git a/bq-workers/argocd-parser/Dockerfile b/bq-workers/argocd-parser/Dockerfile new file mode 100644 index 00000000..47453ea1 --- /dev/null +++ b/bq-workers/argocd-parser/Dockerfile @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Use the official Python image. +# https://hub.docker.com/_/python +FROM python:3.7 + +# Allow statements and log messages to immediately appear in the Cloud Run logs +ENV PYTHONUNBUFFERED True + +# Copy application dependency manifests to the container image. +# Copying this separately prevents re-running pip install on every code change. +COPY requirements.txt . + +# Install production dependencies. +RUN pip install -r requirements.txt + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . . + +# Run the web service on container startup. +# Use gunicorn webserver with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app diff --git a/bq-workers/argocd-parser/cloudbuild.yaml b/bq-workers/argocd-parser/cloudbuild.yaml new file mode 100644 index 00000000..9293dd60 --- /dev/null +++ b/bq-workers/argocd-parser/cloudbuild.yaml @@ -0,0 +1,41 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: +- # Build argocd-parser image + name: gcr.io/cloud-builders/docker:latest + args: ['build', + '--tag=gcr.io/$PROJECT_ID/argocd-parser:${_TAG}', '.'] + dir: 'bq-workers/argocd-parser' + id: build + +- # Push the container image to Artifact Registry + name: gcr.io/cloud-builders/docker + args: ['push', 'gcr.io/$PROJECT_ID/argocd-parser:${_TAG}'] + waitFor: build + id: push + +- # Deploy to Cloud Run + name: google/cloud-sdk + args: ['gcloud', 'run', 'deploy', 'argocd-parser', + '--image', 'gcr.io/$PROJECT_ID/argocd-parser:${_TAG}', + '--region', '${_REGION}', + '--platform', 'managed' + ] + id: deploy + waitFor: push + +images: [ + 'gcr.io/$PROJECT_ID/argocd-parser:${_TAG}' +] diff --git a/bq-workers/argocd-parser/main.py b/bq-workers/argocd-parser/main.py new file mode 100644 index 00000000..982617f9 --- /dev/null +++ b/bq-workers/argocd-parser/main.py @@ -0,0 +1,92 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +import json + +import shared + +from flask import Flask, request + +app = Flask(__name__) + + +@app.route("/", methods=["POST"]) +def index(): + """ + Receives messages from a push subscription from Pub/Sub. + Parses the message, and inserts it into BigQuery. + """ + event = None + if not request.is_json: + raise Exception("Expecting JSON payload") + envelope = request.get_json() + print(f"envelope recieved: {envelope}") + + # Check that data has been posted + if not envelope: + raise Exception("Expecting JSON payload") + # Check that message is a valid pub/sub message + if "message" not in envelope: + raise Exception("Not a valid Pub/Sub Message") + msg = envelope["message"] + + if "attributes" not in msg: + raise Exception("Missing pubsub attributes") + + try: + event = process_argocd_event(msg) + + # [Do not edit below] + shared.insert_row_into_bigquery(event) + + except Exception as e: + entry = { + "severity": "WARNING", + "msg": "Data not saved to BigQuery", + "errors": str(e), + "json_payload": envelope + } + print(json.dumps(entry)) + + return "", 204 + + +def process_argocd_event(msg): + metadata = json.loads(base64.b64decode(msg["data"]).decode("utf-8").strip()) + + # Unique hash for the event + signature = shared.create_unique_id(msg) + + argocd_event = { + "event_type": "deployment", # Event type, eg "push", "pull_reqest", etc + "id": metadata["id"], # Object ID, eg pull request ID + "metadata": json.dumps(metadata), # The body of the msg + "time_created": metadata["time"], # The timestamp of with the event + "signature": signature, # The unique event signature + "msg_id": msg["message_id"], # The pubsub message id + "source": "argocd", # The name of the source, eg "github" + } + + print(argocd_event) + return argocd_event + + +if __name__ == "__main__": + PORT = int(os.getenv("PORT")) if os.getenv("PORT") else 8080 + + # This is used when running locally. Gunicorn is used to run the + # application on Cloud Run. See entrypoint in Dockerfile. + app.run(host="127.0.0.1", port=PORT, debug=True) diff --git a/bq-workers/argocd-parser/main_test.py b/bq-workers/argocd-parser/main_test.py new file mode 100644 index 00000000..9aaa687f --- /dev/null +++ b/bq-workers/argocd-parser/main_test.py @@ -0,0 +1,89 @@ +# Copyright 2020 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json + +import main +import shared + +import mock +import pytest + + +@pytest.fixture +def client(): + main.app.testing = True + return main.app.test_client() + + +def test_not_json(client): + with pytest.raises(Exception) as e: + client.post("/", data="foo") + + assert "Expecting JSON payload" in str(e.value) + + +def test_not_pubsub_message(client): + with pytest.raises(Exception) as e: + client.post( + "/", + data=json.dumps({"foo": "bar"}), + headers={"Content-Type": "application/json"}, + ) + + assert "Not a valid Pub/Sub Message" in str(e.value) + + +def test_missing_msg_attributes(client): + with pytest.raises(Exception) as e: + client.post( + "/", + data=json.dumps({"message": "bar"}), + headers={"Content-Type": "application/json"}, + ) + + assert "Missing pubsub attributes" in str(e.value) + + +def test_argocd_event_processed(client): + data = json.dumps({"foo": "bar", "id": "foo", "time": 0}).encode("utf-8") + pubsub_msg = { + "message": { + "data": base64.b64encode(data).decode("utf-8"), + "attributes": {"foo": "bar"}, + "message_id": "foobar", + }, + } + + event = { + "event_type": "deployment", + "id": "foo", + "metadata": '{"foo": "bar", "id": "foo", "time": 0}', + "time_created": 0, + "signature": "a424b5326ac45bde4c42c9b74dc878e56623d84f", + "msg_id": "foobar", + "source": "argocd", + } + + shared.insert_row_into_bigquery = mock.MagicMock() + + r = client.post( + "/", + data=json.dumps(pubsub_msg), + headers={"Content-Type": "application/json"}, + ) + + shared.insert_row_into_bigquery.assert_called_with(event) + assert r.status_code == 204 diff --git a/bq-workers/argocd-parser/requirements-test.txt b/bq-workers/argocd-parser/requirements-test.txt new file mode 100644 index 00000000..8f9614a3 --- /dev/null +++ b/bq-workers/argocd-parser/requirements-test.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest~=6.0.0 \ No newline at end of file diff --git a/bq-workers/argocd-parser/requirements.txt b/bq-workers/argocd-parser/requirements.txt new file mode 100644 index 00000000..6f0798bb --- /dev/null +++ b/bq-workers/argocd-parser/requirements.txt @@ -0,0 +1,4 @@ +Flask==2.0.3 +gunicorn==19.9.0 +google-cloud-bigquery==1.23.1 +git+https://github.com/GoogleCloudPlatform/fourkeys.git#egg=shared&subdirectory=shared \ No newline at end of file diff --git a/event-handler/sources.py b/event-handler/sources.py index 0f4f34ce..7b369692 100644 --- a/event-handler/sources.py +++ b/event-handler/sources.py @@ -60,7 +60,7 @@ def circleci_verification(signature, body): # Get secret from Cloud Secret Manager secret = get_secret(PROJECT_NAME, "event-handler", "latest") # Compute the hashed signature - hashed = hmac.new(secret, body, 'sha256') + hashed = hmac.new(secret, body, "sha256") expected_signature += hashed.hexdigest() except Exception as e: @@ -117,9 +117,7 @@ def get_secret(project_name, secret_name, version_num): """ try: client = secretmanager.SecretManagerServiceClient() - name = client.secret_version_path( - project_name, secret_name, version_num - ) + name = client.secret_version_path(project_name, secret_name, version_num) secret = client.access_secret_version(name) return secret.payload.data except Exception as e: @@ -145,23 +143,17 @@ def get_source(headers): if "X-Pagerduty-Signature" in headers: return "pagerduty" + if "Argo-CD" in headers.get("User-Agent", ""): + return "argocd" + return headers.get("User-Agent") AUTHORIZED_SOURCES = { - "github": EventSource( - "X-Hub-Signature", github_verification - ), - "gitlab": EventSource( - "X-Gitlab-Token", simple_token_verification - ), - "tekton": EventSource( - "tekton-secret", simple_token_verification - ), - "circleci": EventSource( - "Circleci-Signature", circleci_verification - ), - "pagerduty": EventSource( - "X-Pagerduty-Signature", pagerduty_verification - ), + "github": EventSource("X-Hub-Signature", github_verification), + "gitlab": EventSource("X-Gitlab-Token", simple_token_verification), + "tekton": EventSource("tekton-secret", simple_token_verification), + "circleci": EventSource("Circleci-Signature", circleci_verification), + "pagerduty": EventSource("X-Pagerduty-Signature", pagerduty_verification), + "argocd": EventSource("Argo-Signature", simple_token_verification), } diff --git a/queries/deployments.sql b/queries/deployments.sql index 68a73ca7..3fabdb04 100644 --- a/queries/deployments.sql +++ b/queries/deployments.sql @@ -14,7 +14,8 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline # REGEX to get the commit sha from the URL REGEXP_EXTRACT( JSON_EXTRACT_SCALAR(metadata, '$.commit_url'), r".*commit\/(.*)") - ) end as main_commit, + ) + WHEN source = "argocd" then JSON_EXTRACT_SCALAR(metadata, '$.commit_sha') end as main_commit, CASE WHEN source LIKE "github%" THEN ARRAY( SELECT JSON_EXTRACT_SCALAR(string_element, '$') FROM UNNEST(JSON_EXTRACT_ARRAY(metadata, '$.deployment.additional_sha')) AS string_element) @@ -29,6 +30,8 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline OR (source LIKE "gitlab%" AND event_type = "pipeline" AND JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.status') = "success") # GitLab Deployments OR (source LIKE "gitlab%" AND event_type = "deployment" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "success") + # ArgoCD Deployments + OR (source = "argocd" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") ) ), deploys_tekton AS (# Tekton Pipelines diff --git a/setup/setup.sh b/setup/setup.sh index cb2451ab..97a3dc19 100755 --- a/setup/setup.sh +++ b/setup/setup.sh @@ -60,9 +60,10 @@ else (2) Tekton (3) GitLab (4) CircleCI - (5) Other + (5) ArgoCD + (6) Other - Enter a selection (1 - 5): " cicd_system_id + Enter a selection (1 - 6): " cicd_system_id read -p " Which incident management system(s) are you using? @@ -101,6 +102,7 @@ case $cicd_system_id in 2) CICD_SYSTEM="tekton" ;; 3) CICD_SYSTEM="gitlab" ;; 4) CICD_SYSTEM="circleci" ;; + 5) CICD_SYSTEM="argocd" ;; *) echo "Please see the documentation to learn how to extend to CI/CD sources other than Cloud Build, Tekton, GitLab, CircleCI or GitHub." esac diff --git a/terraform/modules/fourkeys-argocd-parser/README.md b/terraform/modules/fourkeys-argocd-parser/README.md new file mode 100644 index 00000000..dd1ba5ac --- /dev/null +++ b/terraform/modules/fourkeys-argocd-parser/README.md @@ -0,0 +1,39 @@ +## Requirements + +No requirements. + +## Providers + +| Name | Version | +| --------------------------------------------------------- | ------- | +| [google](#provider_google) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | +| [google_cloud_run_service.argocd_parser](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service) | resource | +| [google_project_iam_member.pubsub_service_account_token_creator](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | +| [google_project_service.data_source_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | +| [google_pubsub_subscription.argocd](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | +| [google_pubsub_topic.argocd](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | +| [google_pubsub_topic_iam_member.service_account_editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic_iam_member) | resource | +| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +| --------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------- | -------- | --------------- | :------: | +| [enable_apis](#input_enable_apis) | Toggle to include required APIs. | `bool` | `false` | no | +| [fourkeys_service_account_email](#input_fourkeys_service_account_email) | Service account for fourkeys. | `string` | n/a | yes | +| [parser_container_url](#input_parser_container_url) | URL of image to use in Cloud Run service configuration. | `string` | n/a | yes | +| [project_id](#input_project_id) | Project ID of the target project. | `string` | n/a | yes | +| [region](#input_region) | Region to deploy resources. | `string` | `"us-central1"` | no | + +## Outputs + +No outputs. diff --git a/terraform/modules/fourkeys-argocd-parser/main.tf b/terraform/modules/fourkeys-argocd-parser/main.tf new file mode 100644 index 00000000..7459ae05 --- /dev/null +++ b/terraform/modules/fourkeys-argocd-parser/main.tf @@ -0,0 +1,77 @@ +data "google_project" "project" { + project_id = var.project_id +} + +locals { + services = var.enable_apis ? [ + "run.googleapis.com" + ] : [] +} + +resource "google_project_service" "data_source_services" { + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false +} + +resource "google_cloud_run_service" "argocd_parser" { + project = var.project_id + name = "fourkeys-argocd-parser" + location = var.region + + template { + spec { + containers { + image = var.parser_container_url + env { + name = "PROJECT_NAME" + value = var.project_id + } + } + service_account_name = var.fourkeys_service_account_email + } + } + + traffic { + percent = 100 + latest_revision = true + } + + autogenerate_revision_name = true + depends_on = [ + google_project_service.data_source_services + ] +} + +resource "google_pubsub_topic" "argocd" { + project = var.project_id + name = "argocd" +} + +resource "google_pubsub_topic_iam_member" "service_account_editor" { + project = var.project_id + topic = google_pubsub_topic.argocd.id + role = "roles/editor" + member = "serviceAccount:${var.fourkeys_service_account_email}" +} + +resource "google_pubsub_subscription" "argocd" { + project = var.project_id + name = "argocd" + topic = google_pubsub_topic.argocd.id + + push_config { + push_endpoint = google_cloud_run_service.argocd_parser.status[0]["url"] + + oidc_token { + service_account_email = var.fourkeys_service_account_email + } + } +} + +resource "google_project_iam_member" "pubsub_service_account_token_creator" { + project = var.project_id + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" + role = "roles/iam.serviceAccountTokenCreator" +} diff --git a/terraform/modules/fourkeys-argocd-parser/variables.tf b/terraform/modules/fourkeys-argocd-parser/variables.tf new file mode 100644 index 00000000..34ebfe1e --- /dev/null +++ b/terraform/modules/fourkeys-argocd-parser/variables.tf @@ -0,0 +1,26 @@ +variable "project_id" { + type = string + description = "Project ID of the target project." +} + +variable "region" { + type = string + description = "Region to deploy resources." + default = "us-central1" +} + +variable "fourkeys_service_account_email" { + type = string + description = "Service account for fourkeys." +} + +variable "enable_apis" { + type = bool + description = "Toggle to include required APIs." + default = false +} + +variable "parser_container_url" { + type = string + description = "URL of image to use in Cloud Run service configuration." +} \ No newline at end of file diff --git a/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/Dockerfile b/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/Dockerfile new file mode 100644 index 00000000..47453ea1 --- /dev/null +++ b/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/Dockerfile @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Use the official Python image. +# https://hub.docker.com/_/python +FROM python:3.7 + +# Allow statements and log messages to immediately appear in the Cloud Run logs +ENV PYTHONUNBUFFERED True + +# Copy application dependency manifests to the container image. +# Copying this separately prevents re-running pip install on every code change. +COPY requirements.txt . + +# Install production dependencies. +RUN pip install -r requirements.txt + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . . + +# Run the web service on container startup. +# Use gunicorn webserver with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app diff --git a/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/cloudbuild.yaml b/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/cloudbuild.yaml new file mode 100644 index 00000000..9293dd60 --- /dev/null +++ b/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/cloudbuild.yaml @@ -0,0 +1,41 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: +- # Build argocd-parser image + name: gcr.io/cloud-builders/docker:latest + args: ['build', + '--tag=gcr.io/$PROJECT_ID/argocd-parser:${_TAG}', '.'] + dir: 'bq-workers/argocd-parser' + id: build + +- # Push the container image to Artifact Registry + name: gcr.io/cloud-builders/docker + args: ['push', 'gcr.io/$PROJECT_ID/argocd-parser:${_TAG}'] + waitFor: build + id: push + +- # Deploy to Cloud Run + name: google/cloud-sdk + args: ['gcloud', 'run', 'deploy', 'argocd-parser', + '--image', 'gcr.io/$PROJECT_ID/argocd-parser:${_TAG}', + '--region', '${_REGION}', + '--platform', 'managed' + ] + id: deploy + waitFor: push + +images: [ + 'gcr.io/$PROJECT_ID/argocd-parser:${_TAG}' +] diff --git a/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/main.py b/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/main.py new file mode 100644 index 00000000..982617f9 --- /dev/null +++ b/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/main.py @@ -0,0 +1,92 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +import json + +import shared + +from flask import Flask, request + +app = Flask(__name__) + + +@app.route("/", methods=["POST"]) +def index(): + """ + Receives messages from a push subscription from Pub/Sub. + Parses the message, and inserts it into BigQuery. + """ + event = None + if not request.is_json: + raise Exception("Expecting JSON payload") + envelope = request.get_json() + print(f"envelope recieved: {envelope}") + + # Check that data has been posted + if not envelope: + raise Exception("Expecting JSON payload") + # Check that message is a valid pub/sub message + if "message" not in envelope: + raise Exception("Not a valid Pub/Sub Message") + msg = envelope["message"] + + if "attributes" not in msg: + raise Exception("Missing pubsub attributes") + + try: + event = process_argocd_event(msg) + + # [Do not edit below] + shared.insert_row_into_bigquery(event) + + except Exception as e: + entry = { + "severity": "WARNING", + "msg": "Data not saved to BigQuery", + "errors": str(e), + "json_payload": envelope + } + print(json.dumps(entry)) + + return "", 204 + + +def process_argocd_event(msg): + metadata = json.loads(base64.b64decode(msg["data"]).decode("utf-8").strip()) + + # Unique hash for the event + signature = shared.create_unique_id(msg) + + argocd_event = { + "event_type": "deployment", # Event type, eg "push", "pull_reqest", etc + "id": metadata["id"], # Object ID, eg pull request ID + "metadata": json.dumps(metadata), # The body of the msg + "time_created": metadata["time"], # The timestamp of with the event + "signature": signature, # The unique event signature + "msg_id": msg["message_id"], # The pubsub message id + "source": "argocd", # The name of the source, eg "github" + } + + print(argocd_event) + return argocd_event + + +if __name__ == "__main__": + PORT = int(os.getenv("PORT")) if os.getenv("PORT") else 8080 + + # This is used when running locally. Gunicorn is used to run the + # application on Cloud Run. See entrypoint in Dockerfile. + app.run(host="127.0.0.1", port=PORT, debug=True) diff --git a/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/main_test.py b/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/main_test.py new file mode 100644 index 00000000..9aaa687f --- /dev/null +++ b/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/main_test.py @@ -0,0 +1,89 @@ +# Copyright 2020 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json + +import main +import shared + +import mock +import pytest + + +@pytest.fixture +def client(): + main.app.testing = True + return main.app.test_client() + + +def test_not_json(client): + with pytest.raises(Exception) as e: + client.post("/", data="foo") + + assert "Expecting JSON payload" in str(e.value) + + +def test_not_pubsub_message(client): + with pytest.raises(Exception) as e: + client.post( + "/", + data=json.dumps({"foo": "bar"}), + headers={"Content-Type": "application/json"}, + ) + + assert "Not a valid Pub/Sub Message" in str(e.value) + + +def test_missing_msg_attributes(client): + with pytest.raises(Exception) as e: + client.post( + "/", + data=json.dumps({"message": "bar"}), + headers={"Content-Type": "application/json"}, + ) + + assert "Missing pubsub attributes" in str(e.value) + + +def test_argocd_event_processed(client): + data = json.dumps({"foo": "bar", "id": "foo", "time": 0}).encode("utf-8") + pubsub_msg = { + "message": { + "data": base64.b64encode(data).decode("utf-8"), + "attributes": {"foo": "bar"}, + "message_id": "foobar", + }, + } + + event = { + "event_type": "deployment", + "id": "foo", + "metadata": '{"foo": "bar", "id": "foo", "time": 0}', + "time_created": 0, + "signature": "a424b5326ac45bde4c42c9b74dc878e56623d84f", + "msg_id": "foobar", + "source": "argocd", + } + + shared.insert_row_into_bigquery = mock.MagicMock() + + r = client.post( + "/", + data=json.dumps(pubsub_msg), + headers={"Content-Type": "application/json"}, + ) + + shared.insert_row_into_bigquery.assert_called_with(event) + assert r.status_code == 204 diff --git a/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/requirements-test.txt b/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/requirements-test.txt new file mode 100644 index 00000000..8f9614a3 --- /dev/null +++ b/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/requirements-test.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest~=6.0.0 \ No newline at end of file diff --git a/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/requirements.txt b/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/requirements.txt new file mode 100644 index 00000000..6f0798bb --- /dev/null +++ b/terraform/modules/fourkeys-images/files/bq-workers/argocd-parser/requirements.txt @@ -0,0 +1,4 @@ +Flask==2.0.3 +gunicorn==19.9.0 +google-cloud-bigquery==1.23.1 +git+https://github.com/GoogleCloudPlatform/fourkeys.git#egg=shared&subdirectory=shared \ No newline at end of file From ab8bb77ff27a4150fa13738eed3403c719fb318e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Dluho=C5=A1?= Date: Fri, 9 Sep 2022 15:12:37 +0200 Subject: [PATCH 2/3] fix: configure dashboard region --- dashboard/Dockerfile | 4 ++-- dashboard/cloudbuild.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dashboard/Dockerfile b/dashboard/Dockerfile index cc1c97e9..0f265814 100644 --- a/dashboard/Dockerfile +++ b/dashboard/Dockerfile @@ -21,7 +21,7 @@ ENV GF_AUTH_ANONYMOUS_ORG_ROLE "Admin" # during setup, this variable will be configured onto the Cloud Run service # (default here to "US" as a fallback) -ENV BQ_REGION "US" +ENV BQ_REGION "EU" # Setting grafana config COPY grafana.ini /etc/grafana @@ -32,4 +32,4 @@ COPY dashboards.yaml /etc/grafana/provisioning/dashboards COPY datasource.yaml /etc/grafana/provisioning/datasources # Installing the BigQuery Plugin -RUN grafana-cli plugins install doitintl-bigquery-datasource \ No newline at end of file +RUN grafana-cli plugins install doitintl-bigquery-datasource diff --git a/dashboard/cloudbuild.yaml b/dashboard/cloudbuild.yaml index 0affe2f3..a9a72423 100644 --- a/dashboard/cloudbuild.yaml +++ b/dashboard/cloudbuild.yaml @@ -38,4 +38,4 @@ steps: # https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values substitutions: _TAG: latest - _REGION: us-central1 + _REGION: europe-west3 From 482d7e81c361742f801db2d34cfe0fabe55fd63d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Dluho=C5=A1?= Date: Mon, 26 Sep 2022 11:53:28 +0200 Subject: [PATCH 3/3] feat: add repo_name to issue metadata --- data-generator/generate_data.py | 2 +- data-generator/generate_data_test.py | 2 +- queries/changes.sql | 9 +++++---- queries/deployments.sql | 24 +++++++++++++++--------- queries/events.sql | 10 ++++++++-- queries/incidents.sql | 9 ++++++--- setup/README.md | 2 +- 7 files changed, 37 insertions(+), 21 deletions(-) diff --git a/data-generator/generate_data.py b/data-generator/generate_data.py index 6f28ca96..d918860f 100755 --- a/data-generator/generate_data.py +++ b/data-generator/generate_data.py @@ -209,7 +209,7 @@ def make_github_issue(root_cause): "updated_at": datetime.datetime.now(), "closed_at": datetime.datetime.now(), "number": random.randrange(0, 1000), - "labels": [{"name": "Incident"}], + "labels": [{"name": "incident"}], "body": "root cause: %s" % root_cause["id"], }, "repository": {"name": "foobar"}, diff --git a/data-generator/generate_data_test.py b/data-generator/generate_data_test.py index 3356cb00..01444f5f 100644 --- a/data-generator/generate_data_test.py +++ b/data-generator/generate_data_test.py @@ -137,7 +137,7 @@ def valid_issue(vcs): "updated_at": datetime.datetime(2021, 2, 2, 21, 20, 58, 77232), "closed_at": datetime.datetime(2021, 2, 2, 21, 20, 58, 77235), "number": 440, - "labels": [{"name": "Incident"}], + "labels": [{"name": "incident"}], "body": "root cause: 2b04b6d3939608f19776193697e0e30c04d9c6b8", }, "repository": {"name": "foobar"}, diff --git a/queries/changes.sql b/queries/changes.sql index 711e7328..fdb73ef0 100644 --- a/queries/changes.sql +++ b/queries/changes.sql @@ -2,9 +2,10 @@ SELECT source, event_type, +repo_name, JSON_EXTRACT_SCALAR(commit, '$.id') change_id, -TIMESTAMP_TRUNC(TIMESTAMP(JSON_EXTRACT_SCALAR(commit, '$.timestamp')),second) as time_created, -FROM four_keys.events_raw e, -UNNEST(JSON_EXTRACT_ARRAY(e.metadata, '$.commits')) as commit +TIMESTAMP_TRUNC(TIMESTAMP(JSON_EXTRACT_SCALAR(commit, '$.timestamp')),second) AS time_created, +FROM four_keys.events e, +UNNEST(JSON_EXTRACT_ARRAY(e.metadata, '$.commits')) AS commit WHERE event_type = "push" -GROUP BY 1,2,3,4 \ No newline at end of file +GROUP BY 1,2,3,4,5 diff --git a/queries/deployments.sql b/queries/deployments.sql index 3fabdb04..85c19764 100644 --- a/queries/deployments.sql +++ b/queries/deployments.sql @@ -5,6 +5,7 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline source, id as deploy_id, time_created, + repo_name, CASE WHEN source = "cloud_build" then JSON_EXTRACT_SCALAR(metadata, '$.substitutions.COMMIT_SHA') WHEN source like "github%" then JSON_EXTRACT_SCALAR(metadata, '$.deployment.sha') WHEN source like "gitlab%" then COALESCE( @@ -20,7 +21,7 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline SELECT JSON_EXTRACT_SCALAR(string_element, '$') FROM UNNEST(JSON_EXTRACT_ARRAY(metadata, '$.deployment.additional_sha')) AS string_element) ELSE ARRAY[] end as additional_commits - FROM four_keys.events_raw + FROM four_keys.events WHERE ( # Cloud Build Deployments (source = "cloud_build" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") @@ -39,6 +40,7 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline source, id as deploy_id, time_created, + repo_name, IF(JSON_EXTRACT_SCALAR(param, '$.name') = "gitrevision", JSON_EXTRACT_SCALAR(param, '$.value'), Null) as main_commit, ARRAY[] AS additional_commits FROM ( @@ -46,8 +48,9 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline id, TIMESTAMP_TRUNC(time_created, second) as time_created, source, + repo_name, four_keys.json2array(JSON_EXTRACT(metadata, '$.data.pipelineRun.spec.params')) params - FROM four_keys.events_raw + FROM four_keys.events WHERE event_type = "dev.tekton.event.pipelinerun.successful.v1" AND metadata like "%gitrevision%") e, e.params as param ), @@ -56,9 +59,10 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline source, id AS deploy_id, time_created, + repo_name, JSON_EXTRACT_SCALAR(metadata, '$.pipeline.vcs.revision') AS main_commit, ARRAY[] AS additional_commits - FROM four_keys.events_raw + FROM four_keys.events WHERE (source = "circleci" AND event_type = "workflow-completed" AND JSON_EXTRACT_SCALAR(metadata, '$.workflow.name') LIKE "%deploy%" AND JSON_EXTRACT_SCALAR(metadata, '$.workflow.status') = "success") ), deploys AS ( @@ -73,21 +77,22 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline SELECT id, metadata as change_metadata - FROM four_keys.events_raw + FROM four_keys.events ), - deployment_changes as ( + deployment_changes AS ( SELECT source, deploy_id, deploys.time_created time_created, + repo_name, change_metadata, - four_keys.json2array(JSON_EXTRACT(change_metadata, '$.commits')) as array_commits, + four_keys.json2array(JSON_EXTRACT(change_metadata, '$.commits')) AS array_commits, main_commit FROM deploys JOIN - changes_raw on ( + changes_raw ON ( changes_raw.id = deploys.main_commit - or changes_raw.id in unnest(deploys.additional_commits) + OR changes_raw.id IN unnest(deploys.additional_commits) ) ) @@ -95,8 +100,9 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline source, deploy_id, time_created, + repo_name, main_commit, ARRAY_AGG(DISTINCT JSON_EXTRACT_SCALAR(array_commits, '$.id')) changes, FROM deployment_changes CROSS JOIN deployment_changes.array_commits - GROUP BY 1,2,3,4; + GROUP BY 1,2,3,4,5; diff --git a/queries/events.sql b/queries/events.sql index 9a9a90f0..d3c1f598 100644 --- a/queries/events.sql +++ b/queries/events.sql @@ -1,5 +1,11 @@ -# events table +# Events table + SELECT raw.id, + CASE + WHEN source LIKE 'github%' OR (source LIKE 'gitlab%' AND REGEXP_CONTAINS(LOWER(metadata), r'repository')) + THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.name') + WHEN source LIKE 'gitlab%' THEN JSON_EXTRACT_SCALAR(metadata, '$.project.name') + END AS repo_name, raw.event_type, raw.time_created, raw.metadata, @@ -8,5 +14,5 @@ SELECT raw.id, raw.msg_id, raw.source FROM four_keys.events_raw raw -JOIN four_keys.events_enriched enr +LEFT JOIN four_keys.events_enriched enr ON raw.signature = enr.events_raw_signature diff --git a/queries/incidents.sql b/queries/incidents.sql index bd80619e..f576286f 100644 --- a/queries/incidents.sql +++ b/queries/incidents.sql @@ -2,6 +2,7 @@ SELECT source, incident_id, +repo_name, MIN(IF(root.time_created < issue.time_created, root.time_created, issue.time_created)) as time_created, MAX(time_resolved) as time_resolved, ARRAY_AGG(root_cause IGNORE NULLS) changes, @@ -9,6 +10,7 @@ FROM ( SELECT source, +repo_name, CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.issue.number') WHEN source LIKE "gitlab%" AND event_type = "note" THEN JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.noteable_id') WHEN source LIKE "gitlab%" AND event_type = "issue" THEN JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.id') @@ -23,14 +25,15 @@ CASE WHEN source LIKE "github%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$. WHEN source LIKE "pagerduty%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.event.occurred_at')) END AS time_resolved, REGEXP_EXTRACT(metadata, r"root cause: ([[:alnum:]]*)") as root_cause, -CASE WHEN source LIKE "github%" THEN REGEXP_CONTAINS(JSON_EXTRACT(metadata, '$.issue.labels'), '"name":"Incident"') +CASE WHEN source LIKE "github%" THEN REGEXP_CONTAINS(JSON_EXTRACT(metadata, '$.issue.labels'), '"name":"incident"') WHEN source LIKE "gitlab%" THEN REGEXP_CONTAINS(JSON_EXTRACT(metadata, '$.object_attributes.labels'), '"title":"Incident"') WHEN source LIKE "pagerduty%" THEN TRUE # All Pager Duty events are incident-related END AS bug, -FROM four_keys.events_raw +FROM four_keys.events WHERE event_type LIKE "issue%" OR event_type LIKE "incident%" OR (event_type = "note" and JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.noteable_type') = 'Issue') ) issue LEFT JOIN (SELECT time_created, changes FROM four_keys.deployments d, d.changes) root on root.changes = root_cause -GROUP BY 1,2 +WHERE issue.time_resolved is not NULL +GROUP BY 1,2,3 HAVING max(bug) is True ; diff --git a/setup/README.md b/setup/README.md index 095ad964..e1d0a575 100644 --- a/setup/README.md +++ b/setup/README.md @@ -187,7 +187,7 @@ Four Keys uses GitLab and/or GitHub issues to track incidents. #### Creating an incident 1. Open an issue. -1. Add the tag `Incident`. +1. Add the tag `incident`. 1. In the body of the issue, input `root cause: {SHA of the commit}`. When the incident is resolved, close the issue. Four Keys will measure the incident from the time of the deployment to when the issue is closed.