apply refactorization of coda k8s/helm infra constructs

buildkite/helm/README.md

@@ -0,0 +1,37 @@
+## Introduction
+
+This chart bootstraps a Buildkite GraphQL API metrics exporter providing job and agent statistics to a Prometheus compatible endpoint.
+
+## Add Coda Helm chart repository:
+
+ ```console
+ helm repo add coda <insert-repository-url>
+ helm repo update
+ ```
+
+## Configuration
+
+The following table lists the configurable parameters of the `buildkite-exporter` chart and its default values.
+
+### Required Settings
+
+Parameter | Description
+--- | ---
+`exporter.buildkiteApiKey` | Buildkite GraphQL API access key
+
+### Optional Settings
+
+Parameter | Description | Default
+--- | --- | ---
+`exporter.pipeline` | Buildkite pipeline to scrape and export | `coda`
+`exporter.image` | Buildkite exporter container image to deploy | `codaprotocol/buildkite-exporter:0.1.0`
+`exporter.listeningPort` | port on which to listen for data collection requests | `8000`
+`exporter.optionalEnv` | optional environment variable configuration settings | `[]`
+
+## buildkite-exporter launch examples
+
+```console
+helm install buildkite-exporter \
+    --set exporter.buildkiteApiKey=<api-key> \
+    --set exporter.optionalEnv=[{'name': 'BUILDKITE_BRANCH', 'value': 'release'}]
+```

buildkite/helm/buildkite-exporter/Chart.yaml

@@ -0,0 +1,14 @@
+apiVersion: v2
+name: buildkite-exporter
+description: A Helm chart for O(1) Lab's Buildkite graphql exporter
+type: application
+version: 0.1.0
+appVersion: 0.1.0
+icon: https://pbs.twimg.com/profile_images/968998928241733632/Rhxmu_9O_400x400.jpg
+keywords:
+- buildkite
+- prometheus exporter
+- graphql
+home: https://o1labs.org/
+sources:
+- https://github.com/CodaProtocol

buildkite/helm/buildkite-exporter/templates/buildkite-exporter.yaml

@@ -0,0 +1,41 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{tpl $.Values.exporter.name .}}
+  labels:
+    app: {{tpl $.Values.exporter.name .}}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{tpl $.Values.exporter.name .}}
+  template:
+    metadata:
+      labels:
+        app: {{tpl $.Values.exporter.name .}}
+        pipeline: {{ $.Values.exporter.pipeline }}
+      annotations:
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: {{ $.Values.exporter.listeningPort | quote }}
+        prometheus.io/path: '/metrics'
+    spec:
+      containers:
+      - name: exporter
+        resources:
+          limits:
+            memory: 2.0Gi
+          requests:
+            memory: 2.0Gi
+            cpu: 1000m
+        image: {{ $.Values.exporter.image }}
+        env:
+          - name: BUILDKITE_API_KEY
+            value: {{ .Values.exporter.buildkiteApiKey }}
+{{- if .Values.exporter.optionalEnv }}
+{{ toYaml .Values.exporter.optionalEnv | nindent 10 }}
+{{- end }}
+        ports:
+        - containerPort: {{ .Values.exporter.listeningPort }}
+          hostPort: {{ .Values.exporter.listeningPort }}
+          protocol: TCP
+        imagePullPolicy: Always

buildkite/helm/buildkite-exporter/templates/exporter-service.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tpl $.Values.exporter.name . }}
+spec:
+  type: NodePort
+  selector:
+    app: {{ tpl $.Values.exporter.name . }}
+  ports:
+  - name: metric-collection
+    protocol: TCP
+    port: {{ $.Values.exporter.listeningPort }}
+    targetPort: {{ $.Values.exporter.listeningPort }}

buildkite/helm/buildkite-exporter/values.yaml

@@ -0,0 +1,10 @@
+exporter:
+  pipeline: coda
+  name: "buildkite-{{ .Values.exporter.pipeline }}-exporter"
+  image: codaprotocol/buildkite-exporter:0.1.0
+  listeningPort: 8000
+  optionalEnv:
+#  - name: BUILDKITE_EXPORTER_SCAN_INTERVAL
+#    value: 1800
+#  - name: BUILDKITE_BRANCH
+#    value: develop

buildkite/scripts/generate-diff.sh

@@ -11,7 +11,6 @@ COMMIT=$(diff -u <(git rev-list --first-parent HEAD) \
 
 if [[ $COMMIT != "" ]]; then
   # Get the files that have changed since that shared commit
-  echo "--- Generating diff based on shared commit: ${COMMIT}"
   git diff $COMMIT --name-only
 else
   if [ -n "${BUILDKITE_INCREMENTAL+x}" ]; then
@@ -22,12 +21,10 @@ else
       | jq '.data.pipeline.builds.edges[0].node.commit' | tr -d '"'
     )
 
-    echo "--- Generating incremental diff against: ${ci_recent_pass_commit}"
     git diff "${ci_recent_pass_commit}" --name-only
   else
     # TODO: Dump commits as artifacts when build succeeds so we can diff against
     # that on develop instead of always running all the tests
     git ls-files
   fi
 fi
-

buildkite/scripts/helm-ci.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -eou pipefail
+
+echo "--- Generating change DIFF"
+diff=$(
+  ./buildkite/scripts/generate-diff.sh
+)
+
+# Identifying modifications to helm charts (based on existence of Chart.yaml at change root)
+charts=$(
+  for val in $diff; do
+    find $(dirname $val) -name 'Chart.yaml';
+  done
+)
+
+# filter duplicates
+charts=$(echo $charts | xargs -n1 | sort -u | xargs)
+dirs=$(dirname $charts | xargs -n1 | sort -u | xargs)
+
+if [ -n "${HELM_LINT+x}" ]; then
+  for dir in $dirs; do
+    echo "--- Linting: ${dir}"
+    helm lint $dir
+
+    echo "--- Executing dry-run: ${dir}"
+    helm install test $dir --dry-run --namespace default
+  done
+fi
+
+if [ -n "${HELM_RELEASE+x}" ]; then
+  syncDir="sync_dir"
+  stageDir="updates"
+  mkdir -p $stageDir $syncDir
+
+  echo "--- Syncing with remote GCS Helm chart repository"
+  gsutil -m rsync ${CODA_CHART_REPO:-"gs://coda-charts/"} $syncDir
+
+  echo "--- Preparing the following charts for Release: ${charts}"
+  for dir in $dirs; do
+    git diff develop... "${dir}/Chart.yaml" | grep version
+
+    helm package $dir --destination $stageDir
+
+    if [ -n "${HELM_EXPERIMENTAL_OCI+x}" ]; then
+      echo "--- Helm experimental OCI activated - deploying to GCR registry"
+      helm chart save $(basename $dir)
+
+      gcloud auth configure-docker
+      docker login "gcr.io/o1labs-192920/coda-charts/$(basename ${dir})"
+
+      helm chart push "gcr.io/o1labs-192920/coda-charts/$(basename ${dir})"
+    fi
+  done
+
+  cp --force --recursive "${stageDir}" "${syncDir}"
+
+  helm repo index $syncDir
+
+  if [ -n "${AUTO_DEPLOY+x}" ]; then
+      echo "--- Deploying/Syncing with remote repository"
+      gsutil -m rsync $syncDir ${CODA_CHART_REPO:-"gs://coda-charts/"}
+  fi
+fi

buildkite/src/Jobs/HelmRelease.dhall

@@ -0,0 +1,35 @@
+let Prelude = ../External/Prelude.dhall
+
+let S = ../Lib/SelectFiles.dhall
+let Cmd = ../Lib/Cmds.dhall
+
+let Pipeline = ../Pipeline/Dsl.dhall
+let JobSpec = ../Pipeline/JobSpec.dhall
+
+let Command = ../Command/Base.dhall
+let Docker = ../Command/Docker/Type.dhall
+let Size = ../Command/Size.dhall
+
+let jobDocker = Cmd.Docker::{image = (../Constants/ContainerImages.dhall).codaToolchain}
+
+in
+
+Pipeline.build
+  Pipeline.Config::{
+    spec = JobSpec::{
+      dirtyWhen = [ S.contains "helm", S.strictly (S.contains "Chart.yaml") ],
+      name = "HelmRelease"
+    },
+    steps = [
+      Command.build
+        Command.Config::{
+          commands = [ Cmd.run "HELM_RELEASE=true buildkite/scripts/helm-ci.sh" ]
+          , label = "Helm chart release"
+          , key = "release-helm-chart"
+          , target = Size.Medium
+          , docker = None Docker.Type
+          , artifact_paths = [ S.contains "updates/*" ]
+          , depends_on = [ { name = "HelmChart", key = "lint-helm-chart" } ]
+        }
+    ]
+  }

buildkite/src/Jobs/Lint/HelmChart.dhall

@@ -0,0 +1,34 @@
+let Prelude = ../../External/Prelude.dhall
+
+let S = ../../Lib/SelectFiles.dhall
+let Cmd = ../../Lib/Cmds.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+let Command = ../../Command/Base.dhall
+let Docker = ../../Command/Docker/Type.dhall
+let Size = ../../Command/Size.dhall
+
+let jobDocker = Cmd.Docker::{image = (../../Constants/ContainerImages.dhall).codaToolchain}
+
+in
+
+Pipeline.build
+  Pipeline.Config::{
+    spec = JobSpec::{
+      dirtyWhen = [ S.contains "helm" ],
+      path = "Lint",
+      name = "HelmChart"
+    },
+    steps = [
+      Command.build
+        Command.Config::{
+          commands = [ Cmd.run "HELM_LINT=true buildkite/scripts/helm-ci.sh" ]
+          , label = "Helm chart lint steps"
+          , key = "lint-helm-chart"
+          , target = Size.Small
+          , docker = None Docker.Type
+        }
+    ]
+  }

dockerfiles/Dockerfile-toolchain

@@ -61,6 +61,11 @@ RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
     curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - && \
     sudo apt-get update -y && sudo apt-get install google-cloud-sdk kubectl -y
 
+# Helm tools
+RUN curl https://baltocdn.com/helm/signing.asc | apt-key add - && \
+    echo "deb https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list && \
+    sudo apt-get update -y && sudo apt-get install helm -y
+
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o /tmp/rust_install.sh
 RUN chmod +x /tmp/rust_install.sh
 RUN /tmp/rust_install.sh -y

frontend/bot/helm/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

frontend/bot/helm/Chart.yaml

@@ -0,0 +1,15 @@
+apiVersion: v2
+name: o1-bot
+description: A Helm chart for Coda Protocol's O(1) role-based testnet service bots
+type: application
+version: 0.0.1
+appVersion: 0.0.1
+icon: https://github.com/CodaProtocol/coda/blob/develop/frontend/website/public/static/img/[email protected]
+keywords:
+- bot
+- faucet
+- echo
+- community
+home: https://codaprotocol.com/
+sources:
+- https://github.com/CodaProtocol

frontend/bot/helm/README.md

@@ -0,0 +1,40 @@
+## Introduction
+
+This chart bootstraps O(1) Lab's Coda protocol bot for providing various services to deployed testnets.
+
+## Add Coda Helm chart repository:
+
+ ```console
+ helm repo add coda <insert-repository-url>
+ helm repo update
+ ```
+
+## Configuration
+
+The following table lists the configurable parameters of the `o1-bot` chart and its default values.
+
+### Required Settings
+
+Parameter | Description
+--- | ---
+`bot.name` | name of bot to deploy
+`bot.image` | container image representing bot function/role
+`bot.role` | role/function of bot to deploy
+`bot.testnet` | coda protocol testnet to deploy to
+
+### Optional Settings
+
+Parameter | Description | Default
+--- | --- | ---
+`bot.command` | override command to run on bot startup | `[]`
+`bot.args` | arguments to pass to startup override command | `[]`
+`bot.resources` | bot resource specification/mapping | `see default Values.yaml`
+`bot.optionalEnv` | optional environment variable configuration settings | `[]`
+`bot.volumes` | k8s volume objects to launch bot with | `[]`
+`bot.volumeMounts` | bot pod volume mount path settings | `[]`
+
+## buildkite-exporter launch examples
+
+```console
+helm install faucet ...
+```