ci: Add chaos test pipeline (#349)

* devops: Add pipeline of chaos test * update image * create crd * fix kube path * update * helm * test * deploy chaos * update * update * update * update * update * update * update * update * update * update * update * update * updaye * update * update * update * update * update * update * update
nervosnetwork · Jul 15, 2020 · 5907e13 · 5907e13
1 parent 9bf7da7
commit 5907e13
Show file tree

Hide file tree

Showing 8 changed files with 331 additions and 3 deletions.
diff --git a/charts/deploy-chaos/.helmignore b/charts/deploy-chaos/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/charts/deploy-chaos/Chart.yaml b/charts/deploy-chaos/Chart.yaml
@@ -0,0 +1,21 @@
+apiVersion: v2
+name: deploy-chaos
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application.
+appVersion: 1.16.0
diff --git a/charts/deploy-chaos/templates/_helpers.tpl b/charts/deploy-chaos/templates/_helpers.tpl
@@ -0,0 +1,63 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "deploy-chaos.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "deploy-chaos.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "deploy-chaos.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "deploy-chaos.labels" -}}
+helm.sh/chart: {{ include "deploy-chaos.chart" . }}
+{{ include "deploy-chaos.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "deploy-chaos.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "deploy-chaos.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "deploy-chaos.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "deploy-chaos.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
diff --git a/charts/deploy-chaos/templates/muta-chaos-crd.yaml b/charts/deploy-chaos/templates/muta-chaos-crd.yaml
@@ -0,0 +1,68 @@
+apiVersion: nervos.org/v1alpha1
+kind: Muta
+metadata:
+  name: chaos-{{ .Values.repo_name }}-{{ .Values.version }}
+  namespace: {{ .Values.namespace }} # Only supports deployment to the mutadev namespace
+spec:
+  image: mutadev/{{ .Values.repo_name }}:{{ .Values.version }} # docker image
+  resources:
+    limits:
+      cpu: {{ .Values.resources.cpu }}
+      memory: {{ .Values.resources.memory }}
+      ephemeral-storage: {{ .Values.resources.storage }}
+    requests:
+      cpu: {{ .Values.resources.cpu }}
+      memory: {{ .Values.resources.memory }}
+      ephemeral-storage: {{ .Values.resources.storage }}
+  chaos:  # all / stable-network-corrupt / stable-network-delay / stable-network-duplicate / stable-network-loss / stable-network-partition / stable-node-failure / stable-node-kill
+  {{- range .Values.chaos }}
+    - {{ . }}
+  {{- end }}
+  size: {{ .Values.size }} # Node numbers
+  persistent: {{ .Values.resources.persistent }} # Persistent data
+  config: # see https://github.com/nervosnetwork/muta/blob/master/devtools/chain/config.toml
+    data_path: "/app/data"
+    graphql:
+      listening_address: "0.0.0.0:8000"
+      graphql_uri: "/graphql"
+      graphiql_uri: "/"
+      workers: 0 # if 0, uses number of available logical cpu as threads count.
+      maxconn: 25000
+    network:
+      listening_address: "0.0.0.0:1337"
+      rpc_timeout: {{ .Values.chain_config.network.rpc_timeout }}
+    mempool:
+      pool_size: {{ .Values.chain_config.mempool.pool_size }}
+      broadcast_txs_size: {{ .Values.chain_config.mempool.broadcast_txs_size }}
+      broadcast_txs_interval: {{ .Values.chain_config.mempool.broadcast_txs_interval }}
+    executor:
+      light: false
+    logger:
+      filter: "info"
+      log_to_console: true
+      console_show_file_and_line: false
+      log_path: "logs/"
+      log_to_file: true
+      metrics: true
+      modules_level:
+        # "overlord::state::process": "debug"
+        # "core_consensus": "error"
+  genesis: # https://github.com/nervosnetwork/muta/blob/master/devtools/chain/genesis.toml
+    prevhash: {{ .Values.chain_genesis.prevhash }}
+    metadata:
+      chain_id: {{ .Values.chain_genesis.metadata.chain_id }}
+      timeout_gap: {{ .Values.chain_genesis.metadata.timeout_gap }}
+      cycles_limit: {{ .Values.chain_genesis.metadata.cycles_limit }}
+      cycles_price: {{ .Values.chain_genesis.metadata.cycles_price }}
+      interval: {{ .Values.chain_genesis.metadata.interval }}
+      propose_ratio: {{ .Values.chain_genesis.metadata.propose_ratio }}
+      prevote_ratio: {{ .Values.chain_genesis.metadata.prevote_ratio }}
+      precommit_ratio: {{ .Values.chain_genesis.metadata.precommit_ratio }}
+      brake_ratio: {{ .Values.chain_genesis.metadata.brake_ratio }}
+      tx_num_limit: {{ .Values.chain_genesis.metadata.tx_num_limit }}
+      max_tx_size: {{ .Values.chain_genesis.metadata.max_tx_size }}
+    services:
+    {{- range $service := .Values.chain_genesis.services }}
+      - name: {{ $service.name }}
+        payload: {{ $service.payload | toJson | quote }}
+    {{- end }}
diff --git a/charts/deploy-chaos/values.yaml b/charts/deploy-chaos/values.yaml
@@ -0,0 +1,46 @@
+# Default values for deploy-chaos.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+namespace: mutadev
+
+repo_name: muta
+version: latest
+
+resources:
+  cpu: 1100m
+  memory: 3Gi
+  storage: 5Gi
+  persistent: false
+
+chaos: 
+  - all
+
+size: 4
+
+chain_config:
+  network:
+    rpc_timeout: 10
+  mempool:
+    pool_size: 20000
+    broadcast_txs_size: 200
+    broadcast_txs_interval: 200
+
+chain_genesis:
+  prevhash: 0x44915be5b6c20b0678cf05fcddbbaa832e25d7e6ac538784cd5c24de00d47472
+  metadata:
+    chain_id: 0xb6a4d7da21443f5e816e8700eea87610e6d769657d6b8ec73028457bf2ca4036
+    timeout_gap: 20
+    cycles_limit: 99999999
+    cycles_price: 1
+    interval: 3000
+    propose_ratio: 15
+    prevote_ratio: 15
+    precommit_ratio: 10
+    brake_ratio: 3
+    tx_num_limit: 10000
+    max_tx_size: 1073741824
+  services:
+    - name: asset
+      payload: { "id": "0xf56924db538e77bb5951eb5ff0d02b88983c49c45eea30e8ae3e7234b311436c", "name": "MutaToken", "symbol": "MT", "supply": 320000011, "issuer": "0xf8389d774afdad8755ef8e629e5a154fddc6325a" }
+
diff --git a/devtools/kube/deploy-chaos-crd-template.yml b/devtools/kube/deploy-chaos-crd-template.yml
@@ -0,0 +1,64 @@
+apiVersion: nervos.org/v1alpha1
+kind: Muta
+metadata:
+  name: chaos-${REPO_NAME}-${VERSION}
+  namespace: mutadev # Only supports deployment to the mutadev namespace
+spec:
+  image: mutadev/muta:latest # docker image
+  resources:
+    limits:
+      cpu: 1100m
+      memory: 3Gi
+      ephemeral-storage: 5Gi
+    requests:
+      cpu: 1100m
+      memory: 3Gi
+      ephemeral-storage: 5Gi
+  chaos: # all / stable-network-corrupt / stable-network-delay / stable-network-duplicate / stable-network-loss / stable-network-partition / stable-node-failure / stable-node-kill
+    - all
+  size: 4 # Node numbers
+  persistent: false # Persistent data
+  config: # see https://github.com/nervosnetwork/muta/blob/master/devtools/chain/config.toml
+    data_path: "/app/data"
+    graphql:
+      listening_address: "0.0.0.0:8000"
+      graphql_uri: "/graphql"
+      graphiql_uri: "/"
+      workers: 0 # if 0, uses number of available logical cpu as threads count.
+      maxconn: 25000
+    network:
+      listening_address: "0.0.0.0:1337"
+      rpc_timeout: 10
+    mempool:
+      pool_size: 20000
+      broadcast_txs_size: 200
+      broadcast_txs_interval: 200
+    executor:
+      light: false
+    logger:
+      filter: "info"
+      log_to_console: true
+      console_show_file_and_line: false
+      log_path: "logs/"
+      log_to_file: true
+      metrics: true
+      modules_level:
+        # "overlord::state::process": "debug"
+        # "core_consensus": "error"
+  genesis: # https://github.com/nervosnetwork/muta/blob/master/devtools/chain/genesis.toml
+    prevhash: "0x44915be5b6c20b0678cf05fcddbbaa832e25d7e6ac538784cd5c24de00d47472"
+    metadata:
+      chain_id: "0xb6a4d7da21443f5e816e8700eea87610e6d769657d6b8ec73028457bf2ca4036"
+      timeout_gap: 20
+      cycles_limit: 99999999
+      cycles_price: 1
+      interval: 3000
+      propose_ratio: 15
+      prevote_ratio: 15
+      precommit_ratio: 10
+      brake_ratio: 3
+      tx_num_limit: 20000
+      max_tx_size: 1073741824
+    services:
+      - name: asset
+        payload: '{ "id": "0xf56924db538e77bb5951eb5ff0d02b88983c49c45eea30e8ae3e7234b311436c", "name": "MutaToken", "symbol": "MT", "supply": 320000011, "issuer": "0xf8389d774afdad8755ef8e629e5a154fddc6325a" }'
diff --git a/jenkins-x-chaos.yml b/jenkins-x-chaos.yml
@@ -14,6 +14,10 @@ pipelineConfig:
             environment:
               - name: BASE_WORKSPACE
                 value: /workspace/source
+              - name: NODE_SIZE
+                value: "4"
+              - name: CHAIN_GENESIS_TIMEOUT_GAP
+                value: "9999"
             options:
               containerOptions:
                 volumeMounts:
@@ -50,11 +54,51 @@ pipelineConfig:
                   - --release
                   - --example 
                   - muta-chain
-
               - name: push-image
                 image: gcr.io/kaniko-project/executor:9912ccbf8d22bbafbf971124600fbb0b13b9cbd6
                 command: /kaniko/executor
                 args:
                   - --dockerfile=/workspace/source/devtools/docker-build/Dockerfile
-                  - --destination=mutadev/${REPO_NAME}:${VERSION}
+                  - --destination=mutadev/${REPO_NAME}:pr-${PULL_NUMBER}-${BUILD_NUMBER}
                   - --context=/workspace/source
+
+              - name: create-chaos-crd
+                image: alpine/helm:3.2.4
+                command: helm
+                args:
+                  - install
+                  - chaos-${REPO_NAME}-pr-${PULL_NUMBER}-${BUILD_NUMBER}
+                  - charts/deploy-chaos
+                  - --namespace
+                  - mutadev
+                  - --set
+                  - size=${NODE_SIZE},repo_name=${REPO_NAME},version=pr-${PULL_NUMBER}-${BUILD_NUMBER},resources.cpu=1100m,resources.memory=8Gi,chain_genesis.metadata.timeout_gap=${CHAIN_GENESIS_TIMEOUT_GAP}
+
+              - name: watchdog
+                image: mutadev/muta-watchdog:v0.1.6
+                env:
+                  - name: WATCH_DURATION
+                    value: 4H
+                  - name: APP_NAMESPACE
+                    value: mutadev
+                  - name: APP_PORT
+                    value: "8000"
+                  - name: APP_GRAPHQL_URL
+                    value: graphql
+                  - name: JOB_BENCHMARK_DURATION
+                    value: "300"
+                  - name: JOB_BENCHMARK_TIMEOUT_GAP
+                    value: "9999"
+                  - name: JOB_BENCHMARK_CPU
+                    value: "3"
+                command: APP_NAME=chaos-${REPO_NAME}-pr-${PULL_NUMBER}-${BUILD_NUMBER} node /watchdog/index.js
+
+              - name: delete-chaos-crd
+                image: alpine/helm:3.2.4
+                command: helm
+                args:
+                  - uninstall
+                  - chaos-${REPO_NAME}-pr-${PULL_NUMBER}-${BUILD_NUMBER}
+                  - --namespace
+                  - mutadev
+
diff --git a/jenkins-x.yml b/jenkins-x.yml
@@ -56,5 +56,5 @@ pipelineConfig:
                 command: /kaniko/executor
                 args:
                   - --dockerfile=/workspace/source/devtools/docker-build/Dockerfile
-                  - --destination=mutadev/${REPO_NAME}:${VERSION}
+                  - --destination=mutadev/${REPO_NAME}:${BRANCH_NAME}
                   - --context=/workspace/source