Support passing additional args to worker bootstrap script. Update NG…

… CFN for special regions. (aws#188)

* *: support passing additional args to worker bootstrap script

* (ng): update cfn init to support special regions

eks/ng/nodes.go

@@ -36,13 +36,6 @@ e.g.
 aws ssm get-parameters --names /aws/service/eks/optimized-ami/1.18/amazon-linux-2/recommended/image_id
 aws ssm get-parameters --names /aws/service/bottlerocket/aws-k8s-1.18/x86_64/latest/image_id
 
-TODO
-
-  BootstrapArguments:
-    Type: String
-    Description: Arguments to pass to the bootstrap script. See files/bootstrap.sh in https://github.com/awslabs/amazon-eks-ami
-
-
 NOTE for new regions
 "AWS::SSM::Parameter" may not be onboarded yet, so we need templatize CFN template
 so that we do not pass invalid "AWS::SSM::Parameter" at all in those regions
@@ -308,7 +301,7 @@ const metadataAL2InstallSSM = `    Metadata:
             01InstallAWSCLI:
               # AL2 doesn't have aws cli installed
               command: |
-                curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
+                curl "https://s3.${AWS::Region}.${AWS::URLSuffix}/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
                 unzip awscli-bundle.zip
                 sudo ./awscli-bundle/install -i /usr/local/aws -b /usr/bin/aws
                 which aws
@@ -357,11 +350,14 @@ const userDataAL2InstallSSM = `        UserData:
               EOF
 
               # https://docs.aws.amazon.com/inspector/latest/userguide/inspector_installing-uninstalling-agents.html
-              curl -O https://inspector-agent.amazonaws.com/linux/latest/install
-              chmod +x install
-              sudo ./install -u false
-              rm install
-
+              if [[ "${AWS::Partition}" == "aws-iso-b" ]] || [[ "${AWS::Partition}" == "aws-iso" ]]; then
+                echo "skipping inspector installation"
+              else
+                curl -O https://inspector-agent.amazonaws.com/linux/latest/install
+                chmod +x install
+                sudo ./install -u false
+                rm install
+              fi
               sudo yum install -y yum-utils device-mapper-persistent-data lvm2
               sudo amazon-linux-extras install docker -y
 
@@ -472,8 +468,14 @@ func (ts *tester) createASGs() error {
 				tg.UserData += fmt.Sprintf(` %s`, cur.KubeletExtraArgs)
 			}
 			tg.UserData += "'"
+			if cur.BootstrapArgs != "" {
+				ts.cfg.Logger.Info("adding further additional bootstrap arguments to user data",
+					zap.String("bootstrap-args", cur.BootstrapArgs),
+				)
+				tg.UserData += fmt.Sprintf(` %s`, cur.BootstrapArgs)
+			}
 			tg.UserData += "\n"
-			tg.UserData += `              /opt/aws/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource ASG --region ${AWS::Region}`
+			tg.UserData += `              /opt/aws/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource ASG --region ${AWS::Region} --url='https://cloudformation.${AWS::Region}.${AWS::URLSuffix}' --role='${RoleName}'`
 		}
 		tg.ASGTagData = ""
 		if cur.ClusterAutoscaler != nil && cur.ClusterAutoscaler.Enable {

eksconfig/add-on-node-groups.go

@@ -86,6 +86,10 @@ type ASG struct {
 	// ref. https://github.com/kubernetes/kubernetes/issues/64659
 	KubeletExtraArgs string `json:"kubelet-extra-args"`
 
+	// BootstrapArgs additional bootstrap arguments.
+	// e.g. '--pause-container-account 012345678901 --pause-container-version 3.3'
+	BootstrapArgs string `json:"bootstrap-args"`
+
 	// ClusterAutoscaler is enabled to run cluster auto-scaler per node group.
 	// ref. https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler
 	ClusterAutoscaler *NGClusterAutoscaler `json:"cluster-autoscaler,omitempty"`
@@ -132,6 +136,7 @@ func getDefaultAddOnNodeGroups(name string) *AddOnNodeGroups {
 					ASGDesiredCapacity:                 1,
 				},
 				KubeletExtraArgs:  "",
+				BootstrapArgs:     "",
 				ClusterAutoscaler: &NGClusterAutoscaler{Enable: false},
 			},
 		},

eksconfig/env_test.go

@@ -154,7 +154,7 @@ spec:
 	defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_SERVICE_PRINCIPALS")
 	os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_MANAGED_POLICY_ARNS", "a,b,c")
 	defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_MANAGED_POLICY_ARNS")
-	os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS", `{"ng-test-name-cpu":{"name":"ng-test-name-cpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64","image-id-ssm-parameter":"/aws/service/eks/optimized-ami/1.30/amazon-linux-2/recommended/image_id","asg-min-size":17,"kubelet-extra-args":"bbb qq", "cluster-autoscaler" : {"enable" : false}, "asg-max-size":99,"asg-desired-capacity":77,"instance-types":["type-cpu-2"],"volume-size":40},"ng-test-name-gpu":{"name":"ng-test-name-gpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64_GPU","asg-min-size":30,"asg-max-size":35,"asg-desired-capacity":34,"instance-types":["type-gpu-2"],"image-id":"my-gpu-ami","volume-size":500, "cluster-autoscaler": {"enable":false},"kubelet-extra-args":"aaa aa"}}`)
+	os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS", `{"ng-test-name-cpu":{"name":"ng-test-name-cpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64","image-id-ssm-parameter":"/aws/service/eks/optimized-ami/1.30/amazon-linux-2/recommended/image_id","asg-min-size":17,"kubelet-extra-args":"bbb qq","bootstrap-args":"--pause-container-account 012345678901", "cluster-autoscaler" : {"enable" : false}, "asg-max-size":99,"asg-desired-capacity":77,"instance-types":["type-cpu-2"],"volume-size":40},"ng-test-name-gpu":{"name":"ng-test-name-gpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64_GPU","asg-min-size":30,"asg-max-size":35,"asg-desired-capacity":34,"instance-types":["type-gpu-2"],"image-id":"my-gpu-ami","volume-size":500, "cluster-autoscaler": {"enable":false},"kubelet-extra-args":"aaa aa"}}`)
 	defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS")
 	os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_LOGS_DIR", "a")
 	defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_LOGS_DIR")
@@ -805,6 +805,7 @@ spec:
 				InstanceTypes:        []string{"type-cpu-2"},
 				VolumeSize:           40,
 			},
+			BootstrapArgs:     "--pause-container-account 012345678901",
 			KubeletExtraArgs:  "bbb qq",
 			ClusterAutoscaler: &NGClusterAutoscaler{Enable: false},
 		},