[Python 3.9 Upgrade] [NodeJS Upgrade] Add AL2023 agent and switch perftest and benchmarktest from AL2 to AL2023 with Python 3.9

lib/ci-stack.ts

@@ -154,7 +154,7 @@ export class CIStack extends Stack {
         + 'If you do not copy the AMI in required region and update the code then the jenkins agents will not spin up.');
 
       this.agentNodes = [
-        agentNode.AL2_X64, agentNode.AL2_X64_DOCKER_HOST, agentNode.AL2_X64_DOCKER_HOST_PERF_TEST, agentNode.AL2_X64_DOCKER_HOST_BENCHMARK_TEST,
+        agentNode.AL2_X64, agentNode.AL2_X64_DOCKER_HOST, agentNode.AL2023_X64_DOCKER_HOST_PERF_TEST, agentNode.AL2023_X64_DOCKER_HOST_BENCHMARK_TEST,
         agentNode.AL2_ARM64, agentNode.AL2_ARM64_DOCKER_HOST, agentNode.UBUNTU2004_X64_GRADLE_CHECK, agentNode.UBUNTU2004_X64_DOCKER_BUILDER,
         agentNode.MACOS12_X64_MULTI_HOST, agentNode.WINDOWS2019_X64, agentNode.WINDOWS2019_X64_GRADLE_CHECK,
       ];

lib/compute/agent-nodes.ts

@@ -16,9 +16,9 @@ export class AgentNodes {
 
   readonly AL2_X64_DOCKER_HOST: AgentNodeProps;
 
-  readonly AL2_X64_DOCKER_HOST_PERF_TEST: AgentNodeProps;
+  readonly AL2023_X64_DOCKER_HOST_PERF_TEST: AgentNodeProps;
 
-  readonly AL2_X64_DOCKER_HOST_BENCHMARK_TEST: AgentNodeProps;
+  readonly AL2023_X64_DOCKER_HOST_BENCHMARK_TEST: AgentNodeProps;
 
   readonly AL2_ARM64: AgentNodeProps;
 
@@ -65,30 +65,30 @@ export class AgentNodes {
       + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* --exclude=gh* -y && docker ps',
       remoteFs: '/var/jenkins',
     };
-    this.AL2_X64_DOCKER_HOST_PERF_TEST = {
+    this.AL2023_X64_DOCKER_HOST_PERF_TEST = {
       agentType: 'unix',
-      workerLabelString: 'Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Perf-Test',
+      workerLabelString: 'Jenkins-Agent-AL2023-X64-M52xlarge-Docker-Host-Perf-Test',
       instanceType: 'M52xlarge',
       remoteUser: 'ec2-user',
       maxTotalUses: -1,
       minimumNumberOfSpareInstances: 1,
       numExecutors: 8,
       amiId: 'ami-0174f9a449737d559',
-      initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum /var/lib/yum/history && sudo yum repolist &&'
-      + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* --exclude=gh* -y && docker ps',
+      initScript: 'sudo dnf clean all && sudo rm -rf /var/cache/dnf && sudo dnf repolist &&'
+          + ' sudo dnf update --skip-broken --exclude=openssh* --exclude=docker* --exclude=gh* -y && docker ps',
       remoteFs: '/var/jenkins',
     };
-    this.AL2_X64_DOCKER_HOST_BENCHMARK_TEST = {
+    this.AL2023_X64_DOCKER_HOST_BENCHMARK_TEST = {
       agentType: 'unix',
-      workerLabelString: 'Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Benchmark-Test',
+      workerLabelString: 'Jenkins-Agent-AL2023-X64-M52xlarge-Docker-Host-Benchmark-Test',
       instanceType: 'M52xlarge',
       remoteUser: 'ec2-user',
       maxTotalUses: -1,
       minimumNumberOfSpareInstances: 1,
       numExecutors: 2,
       amiId: 'ami-0174f9a449737d559',
-      initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum /var/lib/yum/history && sudo yum repolist &&'
-          + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* --exclude=gh* -y && docker ps',
+      initScript: 'sudo dnf clean all && sudo rm -rf /var/cache/dnf && sudo dnf repolist &&'
+          + ' sudo dnf update --skip-broken --exclude=openssh* --exclude=docker* --exclude=gh* -y && docker ps',
       remoteFs: '/var/jenkins',
     };
     this.AL2_ARM64 = {

packer/jenkins-agent-al2023-arm64.json

@@ -0,0 +1,92 @@
+{
+  "variables": {
+    "name-base":"Jenkins-Agent-AL2023-ARM64",
+    "os-version": "AL2023",
+    "build-region":"us-east-1",
+    "build-vpc":"vpc-<>",
+    "build-subnet":"subnet-<>",
+    "build-secgrp":"sg-<>",
+    "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}",
+    "aws_ami_region":"us-east-1"
+  },
+  "builders": [
+    {
+      "name":"{{user `name-base`}}-{{user `build-time`}}" ,
+      "ami_description":"{{user `name-base`}}",
+      "type":"amazon-ebs",
+      "encrypt_boot":"false",
+      "region":"{{user `build-region`}}",
+      "ami_regions":"{{user `aws_ami_region`}}",
+      "instance_type":"c6g.4xlarge",
+      "ami_name":"{{user `name-base`}}-{{user `build-time`}}",
+      "vpc_id":"{{user `build-vpc`}}",
+      "subnet_id":"{{user `build-subnet`}}",
+      "security_group_ids":"{{user `build-secgrp`}}",
+      "source_ami_filter": {
+        "filters": {
+          "virtualization-type":"hvm",
+          "architecture":"arm64",
+          "name": "amzn2-ami-kernel-5.10-hvm-*",
+          "block-device-mapping.volume-type":"gp2",
+          "root-device-type":"ebs"
+        },
+        "owners": [
+          "amazon"
+        ],
+        "most_recent":true
+      },
+      "associate_public_ip_address":false,
+      "ssh_username":  "ec2-user",
+      "ssh_timeout": "3h",
+      "tags": {
+        "Name": "{{user `name-base`}}-{{user `build-time`}}",
+        "OS_Version":"{{user `os-version`}}",
+        "User":"Packer",
+        "Encrypted_AMI":"False",
+        "Created":"{{user `build-time`}}"
+      },
+      "launch_block_device_mappings": [
+        {
+          "device_name":"/dev/xvda",
+          "volume_size":100,
+          "delete_on_termination":true,
+          "volume_type":"gp3"
+        }
+      ]
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "file",
+      "source": "files/common/vm-max-map.sh",
+      "destination":
+         "/tmp/vm_max_map.sh"
+    },
+    {
+      "type": "file",
+      "source": "files/common/docker-daemon.json",
+      "destination":
+         "/tmp/daemon.json"
+    },
+    {
+      "type":"shell",
+      "scripts": [
+        "scripts/al2023/al2023-agent-setups.sh"
+      ]
+    },
+    {
+      "type": "shell",
+      "inline": [
+        "sudo mv -v /tmp/vm_max_map.sh /etc/profile.d/vm_max_map.sh",
+        "sudo mv -v /tmp/daemon.json /etc/docker/daemon.json"
+      ]
+    },
+    {
+      "type": "shell",
+      "inline": [
+        "sudo reboot"
+      ],
+      "expect_disconnect": true
+    }
+  ]
+}

packer/jenkins-agent-al2023-x64.json

@@ -0,0 +1,92 @@
+{
+  "variables": {
+    "name-base":"Jenkins-Agent-AL2023-X64",
+    "os-version": "AL2023",
+    "build-region":"us-east-1",
+    "build-vpc":"vpc-<>",
+    "build-subnet":"subnet-<>",
+    "build-secgrp":"sg-<>",
+    "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}",
+    "aws_ami_region":"us-east-1"
+  },
+  "builders": [
+    {
+      "name":"{{user `name-base`}}-{{user `build-time`}}" ,
+      "ami_description":"{{user `name-base`}}",
+      "type":"amazon-ebs",
+      "encrypt_boot":"false",
+      "region":"{{user `build-region`}}",
+      "ami_regions":"{{user `aws_ami_region`}}",
+      "instance_type":"c5.4xlarge",
+      "ami_name":"{{user `name-base`}}-{{user `build-time`}}",
+      "vpc_id":"{{user `build-vpc`}}",
+      "subnet_id":"{{user `build-subnet`}}",
+      "security_group_ids":"{{user `build-secgrp`}}",
+      "source_ami_filter": {
+        "filters": {
+          "virtualization-type":"hvm",
+          "architecture":"x86_64",
+          "name": "amzn2-ami-kernel-5.10-hvm-*",
+          "block-device-mapping.volume-type":"gp2",
+          "root-device-type":"ebs"
+        },
+        "owners": [
+          "amazon"
+        ],
+        "most_recent":true
+      },
+      "associate_public_ip_address":false,
+      "ssh_username":  "ec2-user",
+      "ssh_timeout": "3h",
+      "tags": {
+        "Name": "{{user `name-base`}}-{{user `build-time`}}",
+        "OS_Version":"{{user `os-version`}}",
+        "User":"Packer",
+        "Encrypted_AMI":"False",
+        "Created":"{{user `build-time`}}"
+      },
+      "launch_block_device_mappings": [
+        {
+          "device_name":"/dev/xvda",
+          "volume_size":100,
+          "delete_on_termination":true,
+          "volume_type":"gp3"
+        }
+      ]
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "file",
+      "source": "files/common/vm-max-map.sh",
+      "destination":
+         "/tmp/vm_max_map.sh"
+    },
+    {
+      "type": "file",
+      "source": "files/common/docker-daemon.json",
+      "destination":
+         "/tmp/daemon.json"
+    },
+    {
+      "type":"shell",
+      "scripts": [
+        "scripts/al2023/al2023-agent-setups.sh"
+      ]
+    },
+    {
+      "type": "shell",
+      "inline": [
+        "sudo mv -v /tmp/vm_max_map.sh /etc/profile.d/vm_max_map.sh",
+        "sudo mv -v /tmp/daemon.json /etc/docker/daemon.json"
+      ]
+    },
+    {
+      "type": "shell",
+      "inline": [
+        "sudo reboot"
+      ],
+      "expect_disconnect": true
+    }
+  ]
+}

packer/scripts/al2023/al2023-agent-setups.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+set -ex
+
+CURR_USER="ec2-user"
+if [ `whoami` != "$CURR_USER" ]; then
+    echo "You must run this script on AL2 AMI with 'ec2-user' as user, exit 1"
+    exit 1
+fi
+
+sudo dnf clean all
+sudo rm -rf /var/cache/dnf
+sudo dnf repolist
+sudo dnf update --skip-broken --exclude=openssh* --exclude=docker* -y
+
+sudo dnf install -y java-11-amazon-corretto java-11-amazon-corretto-devel
+sudo dnf install -y which git tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq
+sudo dnf install -y docker
+sudo dnf groupinstall -y "Development Tools"
+
+if ! command -v "python3.9" > /dev/null; then
+    echo "Python 3.9 is not found on AL2023, means the default 'python3' might not be on version 3.9 anymore, exit 1"
+    exit 1
+fi
+
+curl -o- https://bootstrap.pypa.io/get-pip.py | python3
+echo "export PATH=$PATH:$HOME/.local/bin" >> $HOME/.bashrc
+export PATH=$PATH:$HOME/.local/bin
+pip install pipenv awscli docker-compose
+pipenv --version && aws --version && docker-compose --version
+
+sudo sed -i 's/OPTIONS/# OPTIONS/g' /etc/sysconfig/docker
+cat /etc/sysconfig/docker
+sudo systemctl restart docker && sudo systemctl enable docker && sudo systemctl --no-pager status docker
+sudo usermod -a -G docker $CURR_USER
+
+sudo dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
+sudo dnf install -y gh
+
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash
+. ~/.nvm/nvm.sh && nvm install 18.16.0
+
+# Node and node-packages are required globally to execute aws cdk command to setup opensearch-cluster.
+npm install -g fs-extra [email protected] @aws-cdk/cloudformation-diff aws-cdk [email protected]
+
+sudo dnf clean all
+
+sudo mkdir -p /var/jenkins && sudo chown -R $CURR_USER:$CURR_USER /var/jenkins