feat(devnet): continuous integration gssmr devnet on AWS ECS (#2096)

* squashed changes

* update Dockerfile

* update local devnet

* update README

* fix lint

* update flag

* update authorities to include bob and charlie

* add scale down step

* update working directory

* use serviceScaler type

* fix imports

* unit tests for serviceScaler

* use go 1.17 on gha

* fix scale-down workflow

* update mocks

* add licenses

* Update devnet/README.md

Co-authored-by: Quentin McGaw <[email protected]>

* cr feedback

* use submodule

* cr feedback

* cr feedback

* fix test?

* fix update step in dockerfiles

* Update devnet/README.md

Co-authored-by: Quentin McGaw <[email protected]>

* cr feedback

* update workflow

Co-authored-by: Quentin McGaw <[email protected]>

.github/workflows/devnet.yml

@@ -0,0 +1,85 @@
+name: devnet
+on:
+  push:
+    branches:
+      - devnet
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17.0'
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Build, tag, and push alice image to Amazon ECR
+        id: build-alice
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: gssmr-devnet-alice
+          IMAGE_TAG: ${{ github.sha }}
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+        run: |
+          docker build --progress=plain \
+          -t=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t=$ECR_REGISTRY/$ECR_REPOSITORY:latest \
+          --build-arg DD_API_KEY=$DD_API_KEY --build-arg METRICS_NAMESPACE=gossamer.ecs.devnet \
+          -f=devnet/alice.Dockerfile . 
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest
+
+      - name: Build, tag, and push bob image to Amazon ECR
+        id: build-bob
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: gssmr-devnet-bob
+          IMAGE_TAG: ${{ github.sha }}
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+        run: |
+          docker build --progress=plain \
+          -t=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t=$ECR_REGISTRY/$ECR_REPOSITORY:latest \
+          --build-arg key=bob --build-arg DD_API_KEY=$DD_API_KEY --build-arg METRICS_NAMESPACE=gossamer.ecs.devnet \
+          -f=devnet/bob.Dockerfile . 
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest
+
+      - name: Build, tag, and push charlie image to Amazon ECR
+        id: build-charlie
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: gssmr-devnet-charlie
+          IMAGE_TAG: ${{ github.sha }}
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+        run: |
+          docker build --progress=plain \
+          -t=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t=$ECR_REGISTRY/$ECR_REPOSITORY:latest \
+          --build-arg key=charlie --build-arg DD_API_KEY=$DD_API_KEY --build-arg METRICS_NAMESPACE=gossamer.ecs.devnet \
+          -f=devnet/bob.Dockerfile . 
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest
+
+      - name: Scale down existing Bob and Charlie
+        id: scale-down
+        working-directory: ./devnet/cmd/scale-down-ecs-service
+        run: |
+          go run . -c gssmr-ecs -s="gssmr-ecs-(Charlie|Bob)Service-.+$"
+      
+      - name: docker compose up
+        id: docker-compose-up
+        working-directory: ./devnet/gssmr-ecs
+        run: |
+          curl -L https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh
+          docker context create ecs gssmr-ecs --from-env
+          docker context use gssmr-ecs
+          docker compose up

.gitignore

@@ -15,6 +15,7 @@
 *.key
 !test-key.key
 !test-key-password.key
+!devnet/alice.node.key
 *.wasm
 
 test_data

cmd/gossamer/config.go

@@ -680,6 +680,11 @@ func setDotNetworkConfig(ctx *cli.Context, tomlCfg ctoml.NetworkConfig, cfg *dot
 		cfg.PublicIP = pubip
 	}
 
+	// check --pubdns flag and update node configuration
+	if pubdns := ctx.GlobalString(PublicDNSFlag.Name); pubdns != "" {
+		cfg.PublicDNS = pubdns
+	}
+
 	if len(cfg.PersistentPeers) == 0 {
 		cfg.PersistentPeers = []string(nil)
 	}

cmd/gossamer/config_test.go

@@ -537,6 +537,22 @@ func TestNetworkConfigFromFlags(t *testing.T) {
 				PublicIP:          "10.0.5.2",
 			},
 		},
+		{
+			"Test gossamer --pubdns",
+			[]string{"config", "pubdns"},
+			[]interface{}{testCfgFile.Name(), "alice"},
+			dot.NetworkConfig{
+				Port:              testCfg.Network.Port,
+				Bootnodes:         testCfg.Network.Bootnodes,
+				ProtocolID:        testCfg.Network.ProtocolID,
+				NoBootstrap:       testCfg.Network.NoBootstrap,
+				NoMDNS:            false,
+				DiscoveryInterval: time.Second * 10,
+				MinPeers:          testCfg.Network.MinPeers,
+				MaxPeers:          testCfg.Network.MaxPeers,
+				PublicDNS:         "alice",
+			},
+		},
 	}
 
 	for _, c := range testcases {

cmd/gossamer/flags.go

@@ -223,6 +223,11 @@ var (
 		Name:  "pubip",
 		Usage: "Overrides public IP address used for peer to peer networking",
 	}
+	// PublicDNSFlag uses the supplied DNS for broadcasting
+	PublicDNSFlag = cli.StringFlag{
+		Name:  "pubdns",
+		Usage: "Overrides public DNS used for peer to peer networking",
+	}
 )
 
 // RPC service configuration flags
@@ -414,6 +419,7 @@ var (
 		NoBootstrapFlag,
 		NoMDNSFlag,
 		PublicIPFlag,
+		PublicDNSFlag,
 
 		// rpc flags
 		RPCEnabledFlag,

devnet/README.md

@@ -51,4 +51,26 @@ This key is injected in `alice.Dockerfile` so it uses the same public key for th
 
 ### docker-compose.yml
 
-The Docker Compose file.  Specifies the IP addresses of all the nodes.  
+The Docker Compose file.  Specifies the IP addresses of all the nodes. 
+
+
+## ECS Fargate Deployment
+
+The [`docker-compose.yml`](gssmr-ecs/docker-compose.yml) file within `devnet/gssmr-ecs` folder uses Docker Compose ECS plugin to deploy and update an existing AWS ECS Cluster using the Fargate launch type running a Gossamer devnet with 3 services corresponding to the 3 keys used `alice`, `bob`, and `charlie`.  
+
+### Deployment
+
+Currently deployment is handled via a github workflow.  Pushing the `devnet` branch will initiate the deploy process.  Steps are outlined in `/.github/workflows/devnet.yml`. 
+
+At a high level, images for the `alice`, `bob` and `charlie` correspond to ECS services under the same name.  The docker images are built based on the latest commit on the `devnet` branch.  These images are pushed to ECR.  A specific type of Docker context is required to use the ECS plugin.  Deploying and updating is as simple as:
+
+```
+docker context create ecs gssmr-ecs --from-env
+docker context use gssmr-ecs
+docker compose up
+```
+
+### Prometheus to Datadog
+
+Prometheus metrics are automatically piped to Datadog.  All metrics from the ECS devnet are prefixed with `gossamer.ecs.devnet`.  
+

devnet/alice.Dockerfile

@@ -4,7 +4,8 @@
 FROM golang:1.17
 
 ARG DD_API_KEY=somekey
-RUN DD_API_KEY=${DD_API_KEY}  DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
+ENV DD_API_KEY=${DD_API_KEY}
+RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
 
 WORKDIR /gossamer
 
@@ -23,7 +24,12 @@ RUN gossamer --key=alice init
 # use a hardcoded key for alice, so we can determine what the peerID is for subsequent nodes
 RUN cp devnet/alice.node.key ~/.gossamer/gssmr/node.key
 
-RUN go run devnet/cmd/update-dd-agent-confd/main.go -n=gossamer.local.devnet -t=key:alice > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml
+ARG METRICS_NAMESPACE=gossamer.local.devnet
+
+WORKDIR /gossamer/devnet
+
+RUN go run cmd/update-dd-agent-confd/main.go -n=${METRICS_NAMESPACE} -t=key:alice > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml
+
+ENTRYPOINT service datadog-agent start && gossamer --key=alice --babe-lead --publish-metrics --rpc --rpc-external=true --pubdns=alice
 
-ENTRYPOINT service datadog-agent start && gossamer --key=alice --babe-lead --publish-metrics --rpc --rpc-external=true --pubip=10.5.0.2
 EXPOSE 7001/tcp 8545/tcp 8546/tcp 8540/tcp 9876/tcp

devnet/alice.node.key

@@ -0,0 +1 @@
+93ce444331ced4d2f7bfb8296267544e20c2591dbf310c7ea3af672f2879cf8fa999ca3052b725123c2939085e3158dba26624fcfd05c361b1adcc8b27785e35

devnet/bob.Dockerfile

@@ -4,7 +4,8 @@
 FROM golang:1.17
 
 ARG DD_API_KEY=somekey
-RUN DD_API_KEY=${DD_API_KEY} DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
+ENV DD_API_KEY=${DD_API_KEY}
+RUN DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
 
 WORKDIR /gossamer
 
@@ -20,16 +21,16 @@ RUN cp -f devnet/chain/gssmr/genesis-spec.json chain/gssmr/genesis-spec.json
 
 ARG key
 RUN test -n "$key"
-ARG pubip
-RUN test -n "$pubip"
-
 ENV key=${key}
-ENV pubip=${pubip}
 
 RUN gossamer --key=${key} init
 
-RUN go run devnet/cmd/update-dd-agent-confd/main.go -n=gossamer.local.devnet -t=key:${key} > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml
+ARG METRICS_NAMESPACE=gossamer.local.devnet
+
+WORKDIR /gossamer/devnet
+
+RUN go run cmd/update-dd-agent-confd/main.go -n=${METRICS_NAMESPACE} -t=key:${key} > /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml
 
-ENTRYPOINT service datadog-agent start && gossamer --key=${key} --bootnodes=/ip4/10.5.0.2/tcp/7001/p2p/12D3KooWMER5iow67nScpWeVqEiRRx59PJ3xMMAYPTACYPRQbbWU --publish-metrics --rpc --pubip=${pubip}
+ENTRYPOINT service datadog-agent start && gossamer --key=${key} --bootnodes=/dns/alice/tcp/7001/p2p/12D3KooWMER5iow67nScpWeVqEiRRx59PJ3xMMAYPTACYPRQbbWU --publish-metrics --rpc --pubdns=${key}
 
 EXPOSE 7001/tcp 8545/tcp 8546/tcp 8540/tcp 9876/tcp

devnet/chain/gssmr/genesis-spec.json

@@ -28,6 +28,14 @@
 					[
 						"5DFNv4Txc4b88qHqQ6GG4D646QcT4fN3jjS2G3r1PyZkfDut",
 						1
+					],
+					[
+						"5FRbF5E5xpBxaCdjuVkCAtoLgvvoL6wJoE7kAVTUdZBwpgi7",
+						1
+					],
+					[
+						"5DYo8CvjQcBQFdehVhansDiZCPebpgqvNC8PQPi6K9cL9giT",
+						1
 					]
 				]
 			},

devnet/cmd/scale-down-ecs-service/main.go

@@ -0,0 +1,58 @@
+// Copyright 2021 ChainSafe Systems (ON)
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/jessevdk/go-flags"
+)
+
+type options struct {
+	ServicesRegex   string        `short:"s" long:"services" description:"regex query used to match against AWS service names" required:"true"`          //nolint:lll
+	Cluster         string        `short:"c" long:"cluster" description:"ECS cluster name, must be exact match" required:"true"`                         //nolint:lll
+	RequestInterval time.Duration `short:"i" long:"interval" description:"Interval between AWS requests when waiting for service to scale" default:"5s"` //nolint:lll
+}
+
+func main() {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+
+	var opts options
+	_, err := flags.Parse(&opts)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	sess := session.Must(session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	}))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error)
+	go func() {
+		ss := newServiceScaler(opts.RequestInterval, opts.Cluster, ecs.New(sess))
+		done <- ss.scaleServices(ctx, opts.ServicesRegex)
+	}()
+
+	for {
+		select {
+		case err := <-done:
+			if err != nil {
+				log.Fatal(err)
+			}
+			// happy path
+			return
+		case <-sigs:
+			cancel()
+		}
+	}
+}