From b83a7df61d763ba8c942293e613913ac3faeddb6 Mon Sep 17 00:00:00 2001 From: cmdallas Date: Fri, 20 Nov 2020 10:30:35 -0800 Subject: [PATCH] Support passing additional args to worker bootstrap script. Update NG CFN for special regions. (#188) * *: support passing additional args to worker bootstrap script * (ng): update cfn init to support special regions --- eks/ng/nodes.go | 30 ++++++++++++++++-------------- eksconfig/add-on-node-groups.go | 5 +++++ eksconfig/env_test.go | 3 ++- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/eks/ng/nodes.go b/eks/ng/nodes.go index bae41167a..7c54ab096 100644 --- a/eks/ng/nodes.go +++ b/eks/ng/nodes.go @@ -36,13 +36,6 @@ e.g. aws ssm get-parameters --names /aws/service/eks/optimized-ami/1.18/amazon-linux-2/recommended/image_id aws ssm get-parameters --names /aws/service/bottlerocket/aws-k8s-1.18/x86_64/latest/image_id -TODO - - BootstrapArguments: - Type: String - Description: Arguments to pass to the bootstrap script. See files/bootstrap.sh in https://github.com/awslabs/amazon-eks-ami - - NOTE for new regions "AWS::SSM::Parameter" may not be onboarded yet, so we need templatize CFN template so that we do not pass invalid "AWS::SSM::Parameter" at all in those regions @@ -308,7 +301,7 @@ const metadataAL2InstallSSM = ` Metadata: 01InstallAWSCLI: # AL2 doesn't have aws cli installed command: | - curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip" + curl "https://s3.${AWS::Region}.${AWS::URLSuffix}/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip" unzip awscli-bundle.zip sudo ./awscli-bundle/install -i /usr/local/aws -b /usr/bin/aws which aws @@ -357,11 +350,14 @@ const userDataAL2InstallSSM = ` UserData: EOF # https://docs.aws.amazon.com/inspector/latest/userguide/inspector_installing-uninstalling-agents.html - curl -O https://inspector-agent.amazonaws.com/linux/latest/install - chmod +x install - sudo ./install -u false - rm install - + if [[ "${AWS::Partition}" == "aws-iso-b" ]] || [[ "${AWS::Partition}" == "aws-iso" ]]; then + echo "skipping inspector installation" + else + curl -O https://inspector-agent.amazonaws.com/linux/latest/install + chmod +x install + sudo ./install -u false + rm install + fi sudo yum install -y yum-utils device-mapper-persistent-data lvm2 sudo amazon-linux-extras install docker -y @@ -472,8 +468,14 @@ func (ts *tester) createASGs() error { tg.UserData += fmt.Sprintf(` %s`, cur.KubeletExtraArgs) } tg.UserData += "'" + if cur.BootstrapArgs != "" { + ts.cfg.Logger.Info("adding further additional bootstrap arguments to user data", + zap.String("bootstrap-args", cur.BootstrapArgs), + ) + tg.UserData += fmt.Sprintf(` %s`, cur.BootstrapArgs) + } tg.UserData += "\n" - tg.UserData += ` /opt/aws/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource ASG --region ${AWS::Region}` + tg.UserData += ` /opt/aws/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource ASG --region ${AWS::Region} --url='https://cloudformation.${AWS::Region}.${AWS::URLSuffix}' --role='${RoleName}'` } tg.ASGTagData = "" if cur.ClusterAutoscaler != nil && cur.ClusterAutoscaler.Enable { diff --git a/eksconfig/add-on-node-groups.go b/eksconfig/add-on-node-groups.go index 8432a30b5..f69875abf 100644 --- a/eksconfig/add-on-node-groups.go +++ b/eksconfig/add-on-node-groups.go @@ -86,6 +86,10 @@ type ASG struct { // ref. https://github.com/kubernetes/kubernetes/issues/64659 KubeletExtraArgs string `json:"kubelet-extra-args"` + // BootstrapArgs additional bootstrap arguments. + // e.g. '--pause-container-account 012345678901 --pause-container-version 3.3' + BootstrapArgs string `json:"bootstrap-args"` + // ClusterAutoscaler is enabled to run cluster auto-scaler per node group. // ref. https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler ClusterAutoscaler *NGClusterAutoscaler `json:"cluster-autoscaler,omitempty"` @@ -132,6 +136,7 @@ func getDefaultAddOnNodeGroups(name string) *AddOnNodeGroups { ASGDesiredCapacity: 1, }, KubeletExtraArgs: "", + BootstrapArgs: "", ClusterAutoscaler: &NGClusterAutoscaler{Enable: false}, }, }, diff --git a/eksconfig/env_test.go b/eksconfig/env_test.go index d162f819c..6b1948e8d 100644 --- a/eksconfig/env_test.go +++ b/eksconfig/env_test.go @@ -154,7 +154,7 @@ spec: defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_SERVICE_PRINCIPALS") os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_MANAGED_POLICY_ARNS", "a,b,c") defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_MANAGED_POLICY_ARNS") - os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS", `{"ng-test-name-cpu":{"name":"ng-test-name-cpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64","image-id-ssm-parameter":"/aws/service/eks/optimized-ami/1.30/amazon-linux-2/recommended/image_id","asg-min-size":17,"kubelet-extra-args":"bbb qq", "cluster-autoscaler" : {"enable" : false}, "asg-max-size":99,"asg-desired-capacity":77,"instance-types":["type-cpu-2"],"volume-size":40},"ng-test-name-gpu":{"name":"ng-test-name-gpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64_GPU","asg-min-size":30,"asg-max-size":35,"asg-desired-capacity":34,"instance-types":["type-gpu-2"],"image-id":"my-gpu-ami","volume-size":500, "cluster-autoscaler": {"enable":false},"kubelet-extra-args":"aaa aa"}}`) + os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS", `{"ng-test-name-cpu":{"name":"ng-test-name-cpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64","image-id-ssm-parameter":"/aws/service/eks/optimized-ami/1.30/amazon-linux-2/recommended/image_id","asg-min-size":17,"kubelet-extra-args":"bbb qq","bootstrap-args":"--pause-container-account 012345678901", "cluster-autoscaler" : {"enable" : false}, "asg-max-size":99,"asg-desired-capacity":77,"instance-types":["type-cpu-2"],"volume-size":40},"ng-test-name-gpu":{"name":"ng-test-name-gpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64_GPU","asg-min-size":30,"asg-max-size":35,"asg-desired-capacity":34,"instance-types":["type-gpu-2"],"image-id":"my-gpu-ami","volume-size":500, "cluster-autoscaler": {"enable":false},"kubelet-extra-args":"aaa aa"}}`) defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS") os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_LOGS_DIR", "a") defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_LOGS_DIR") @@ -805,6 +805,7 @@ spec: InstanceTypes: []string{"type-cpu-2"}, VolumeSize: 40, }, + BootstrapArgs: "--pause-container-account 012345678901", KubeletExtraArgs: "bbb qq", ClusterAutoscaler: &NGClusterAutoscaler{Enable: false}, },