Testnet and mainnet rpc cicd

.github/workflows/ci.yml

@@ -97,7 +97,7 @@ jobs:
     needs: devnet_build_push
     environment: devnet
     env:
-      AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     steps:
       - uses: actions/checkout@v4
@@ -122,6 +122,48 @@ jobs:
           done
           aws ecs update-service --region ap-southeast-1 --cluster devnet-xdcnode-cluster --service ecs-service-rpc1 --force-new-deployment --no-cli-pager | head -n 10;
 
+  rpcnode_terraform_apply:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/dev-upgrade' && !startsWith(github.ref, 'refs/tags/')
+    needs: devnet_build_push
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Terraform Apply
+        run: |
+          cd cicd/terraform
+          terraform init ${{ env.tf_init_cli_options }}
+          terraform apply ${{ env.tf_apply_cli_options }}
+
+  testnet_dev-upgrade_node:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/dev-upgrade' && !startsWith(github.ref, 'refs/tags/')
+    needs: rpcnode_terraform_apply
+    environment: testnet
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: ECS Update
+        run: |
+          aws ecs update-service --region ap-southeast-1 --cluster testnet-xdcnode-cluster --service ecs-service-testnet-rpc1 --force-new-deployment --no-cli-pager | head -n 10;
+
+  mainnet_dev-upgrade_node:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/dev-upgrade' && !startsWith(github.ref, 'refs/tags/')
+    needs: rpcnode_terraform_apply
+    environment: mainnet
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: ECS Update
+        run: |
+          aws ecs update-service --region ap-southeast-1 --cluster mainnet-xdcnode-cluster --service ecs-service-mainnet-rpc1 --force-new-deployment --no-cli-pager | head -n 10;
 
   devnet_send_notification:
     runs-on: ubuntu-latest

.gitignore

@@ -50,7 +50,6 @@ profile.cov
 **/yarn-error.log
 coverage.txt
 go.sum
-cicd/devnet/terraform/.terraform*
+**/.terraform*
 cicd/devnet/tmp
-.env
 cicd/devnet/terraform/node-config.json

cicd/README.md

@@ -6,5 +6,5 @@ docker build  -f cicd/Dockerfile .
 ```
 ## Docker Run
 ```
-docker run -it -e NETWORK=devnet -e PRIVATE_KEYS=$KEY $IMAGE
+docker run -it -e NETWORK=devnet -e PRIVATE_KEY=$KEY $IMAGE
 ``

cicd/devnet/terraform/module/region/container-definition.tpl

@@ -3,7 +3,7 @@
     "name": "tfXdcNode",
     "image": "xinfinorg/${image_environment}:${image_tag}",
     "environment": [
-      {"name": "PRIVATE_KEYS", "value": "${private_keys}"},
+      {"name": "PRIVATE_KEY", "value": "${private_key}"},
       {"name": "LOG_LEVEL", "value": "${log_level}"},
       {"name": "NODE_NAME", "value": "${node_name}"},
       {"name": "NETWORK", "value": "${chain_network}"}

cicd/devnet/terraform/module/region/ecs.tf

@@ -6,7 +6,7 @@ data template_file devnet_container_definition {
     image_environment = "${lookup(each.value, "imageEnvironment", "devnet")}"
     image_tag = "${lookup(each.value, "imageTag", "latest")}"
     node_name = "${each.key}"
-    private_keys = "${each.value.pk}"
+    private_key = "${each.value.pk}"
     cloudwatch_group = "tf-${each.key}"
     cloudwatch_region = "${var.region}"
     log_level = "${lookup(each.value, "logLevel", "${var.logLevel}")}"

cicd/mainnet/start.sh

@@ -76,7 +76,7 @@ XDC --ethstats ${netstats} --gcmode archive \
 --datadir /work/xdcchain --networkid 50 \
 -port $port --rpc --rpccorsdomain "*" --rpcaddr 0.0.0.0 \
 --rpcport $rpc_port \
---rpcapi admin,db,eth,debug,miner,net,shh,txpool,personal,web3,XDPoS \
+--rpcapi admin,db,eth,debug,net,shh,txpool,personal,web3,XDPoS \
 --rpcvhosts "*" --unlock "${wallet}" --password /work/.pwd --mine \
 --gasprice "1" --targetgaslimit "420000000" --verbosity ${log_level} \
 --debugdatadir /work/xdcchain \

cicd/terraform/.env

@@ -0,0 +1,13 @@
+log_level=3
+
+# Ohio
+us_east_2_start=0
+us_east_2_end=36
+
+# Ireland
+eu_west_1_start=37
+eu_west_1_end=72
+
+# Sydney
+ap_southeast_2_start=73
+ap_southeast_2_end=108

cicd/terraform/iam.tf

@@ -0,0 +1,28 @@
+# IAM policies
+data "aws_iam_policy_document" "xdc_ecs_tasks_execution_role" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["ecs-tasks.amazonaws.com"]
+    }
+  }
+}
+
+# Create the role
+resource "aws_iam_role" "xdc_ecs_tasks_execution_role" {
+  name               = "xdc-ecs-task-execution-role"
+  assume_role_policy = "${data.aws_iam_policy_document.xdc_ecs_tasks_execution_role.json}"
+}
+
+# Attached the AWS managed policies to the new role
+resource "aws_iam_role_policy_attachment" "xdc_ecs_tasks_execution_role" {
+  for_each = toset([
+    "arn:aws:iam::aws:policy/AmazonElasticFileSystemClientFullAccess", 
+    "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy",
+    "arn:aws:iam::aws:policy/AmazonElasticFileSystemsUtils"
+  ])
+  role       = aws_iam_role.xdc_ecs_tasks_execution_role.name
+  policy_arn = each.value
+}

cicd/terraform/main.tf

@@ -0,0 +1,59 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.13.1"
+    }
+  }
+}
+
+# Default
+provider "aws" {
+  region  = "us-east-1"
+}
+
+# WARNING: APSE-1 will only be used to host rpc node
+# Workaround to avoid conflicts with existing ecs cluster in existing regions
+provider "aws" {
+  alias = "ap-southeast-1"
+  region  = "ap-southeast-1"
+}
+
+module "testnet-rpc" {
+  source = "./module/region"
+  region = "ap-southeast-1"
+  nodeKeys = local.rpcTestnetNodeKeys
+  enableFixedIp = true
+  logLevel = local.logLevel
+  xdc_ecs_tasks_execution_role_arn = aws_iam_role.xdc_ecs_tasks_execution_role.arn
+
+  cpu = 1024
+  memory = 4096
+
+  network = "testnet"
+  vpc_cidr = "10.1.0.0/16"
+  subnet_cidr = "10.1.0.0/20"
+  providers = {
+    aws = aws.ap-southeast-1
+  }
+}
+
+module "mainnet-rpc" {
+  source = "./module/region"
+  region = "ap-southeast-1"
+  nodeKeys = local.rpcMainnetNodeKeys
+  enableFixedIp = true
+  logLevel = local.logLevel
+  xdc_ecs_tasks_execution_role_arn = aws_iam_role.xdc_ecs_tasks_execution_role.arn
+
+  cpu = 1024
+  memory = 4096
+
+  network = "mainnet"
+  vpc_cidr = "10.2.0.0/16"
+  subnet_cidr = "10.2.0.0/20"
+  providers = {
+    aws = aws.ap-southeast-1
+  }
+}
+

cicd/terraform/module/region/container-definition.tpl

@@ -0,0 +1,44 @@
+[
+  {
+    "name": "tfXdcNode",
+    "image": "xinfinorg/${image_environment}:${image_tag}",
+    "environment": [
+      {"name": "PRIVATE_KEY", "value": "${private_key}"},
+      {"name": "LOG_LEVEL", "value": "${log_level}"},
+      {"name": "NODE_NAME", "value": "${node_name}"},
+      {"name": "NETWORK", "value": "${chain_network}"}
+    ],
+    "essential": true,
+    "logConfiguration": {
+      "logDriver": "awslogs",
+      "options": {
+        "awslogs-group": "${cloudwatch_group}",
+        "awslogs-region": "${cloudwatch_region}",
+        "awslogs-stream-prefix": "ecs"
+      }
+    },
+    "portMappings": [
+      {
+        "hostPort": 8555,
+        "protocol": "tcp",
+        "containerPort": 8555
+      },
+      {
+        "hostPort": 8545,
+        "protocol": "tcp",
+        "containerPort": 8545
+      },
+      {
+        "hostPort": 30303,
+        "protocol": "tcp",
+        "containerPort": 30303
+      }
+    ],
+    "mountPoints": [
+      {
+        "containerPath": "/work/xdcchain",
+        "sourceVolume": "efs"
+      }
+    ]
+  }
+]

cicd/terraform/module/region/ecs.tf

@@ -0,0 +1,96 @@
+data template_file container_definition {
+  for_each = var.nodeKeys
+  template = "${file("${path.module}/container-definition.tpl")}"
+
+  vars = {
+    image_environment = "${lookup(each.value, "imageEnvironment", "devnet")}"
+    image_tag = "${lookup(each.value, "imageTag", "latest")}"
+    node_name = "${each.key}"
+    private_key = "${each.value.pk}"
+    cloudwatch_group = "tf-${each.key}"
+    cloudwatch_region = "${var.region}"
+    log_level = "${lookup(each.value, "logLevel", "${var.logLevel}")}"
+    chain_network = var.network
+  }
+}
+
+resource "aws_ecs_task_definition" "task_definition_group" {
+  for_each = var.nodeKeys
+
+  family = "${var.network}-${each.key}"
+  requires_compatibilities = ["FARGATE"]
+  network_mode = "awsvpc"
+  container_definitions = data.template_file.container_definition[each.key].rendered
+  execution_role_arn = var.xdc_ecs_tasks_execution_role_arn
+  task_role_arn = var.xdc_ecs_tasks_execution_role_arn
+
+  # New nodes will consume a lot more CPU usage than existing nodes. 
+  # This is due to sync is resource heavy. Recommending set to below if doing sync:
+  # CPU = 2048, Memory = 4096
+  # Please set it back to cpu 256 and memory of 2048 after sync is done to save the cost
+  # cpu = 256
+  # memory = 2048
+  cpu = var.cpu
+  memory = var.memory
+  volume {
+    name = "efs"
+
+    efs_volume_configuration {
+      file_system_id          = aws_efs_file_system.efs[each.key].id
+      root_directory          = "/"
+      transit_encryption      = "ENABLED"
+      authorization_config {
+        access_point_id = aws_efs_access_point.efs_access_point[each.key].id
+        iam             = "DISABLED"
+      }
+    }
+  }
+
+  tags = {
+       Name = "Tf${var.network}Ecs-${each.key}"
+   }
+}
+
+data "aws_ecs_task_definition" "ecs_task_definition" {
+  for_each = var.nodeKeys
+  task_definition = aws_ecs_task_definition.task_definition_group[each.key].family
+}
+
+# ECS cluster
+resource "aws_ecs_cluster" "ecs_cluster" {
+  name    = "${var.network}-xdcnode-cluster"
+  tags    = {
+    Name        = "Tf${var.network}EcsCluster"
+  }
+}
+
+
+resource "aws_ecs_service" "ecs_service" {
+  for_each             = var.enableFixedIp ? {} : var.nodeKeys
+  name                 = "ecs-service-${each.key}"
+  cluster              = aws_ecs_cluster.ecs_cluster.id
+  task_definition      = "${aws_ecs_task_definition.task_definition_group[each.key].family}:${max(aws_ecs_task_definition.task_definition_group[each.key].revision, data.aws_ecs_task_definition.ecs_task_definition[each.key].revision)}"
+  launch_type          = "FARGATE"
+  scheduling_strategy  = "REPLICA"
+  desired_count        = 1
+  force_new_deployment = true
+  deployment_minimum_healthy_percent = 0
+  deployment_maximum_percent = 100
+
+  network_configuration {
+    subnets          = [aws_subnet.subnet.id]
+    assign_public_ip = true
+    security_groups = [
+      aws_default_security_group.xdcnode_security_group.id
+    ]
+  }
+
+  deployment_circuit_breaker {
+    enable = true
+    rollback = false
+  }
+
+  tags = {
+    Name        = "Tf${var.network}EcsService-${each.key}"
+  }
+}