Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support passing additional args to worker bootstrap script. Update NG CFN for special regions. #188

Merged
merged 2 commits into from
Nov 20, 2020
Merged

Support passing additional args to worker bootstrap script. Update NG CFN for special regions. #188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ba6b4e9
*: support passing additional args to worker bootstrap script
cmdallas Nov 19, 2020
86a9a6b
(ng): update cfn init to support special regions
cmdallas Nov 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 16 additions & 14 deletions eks/ng/nodes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,6 @@ e.g.
aws ssm get-parameters --names /aws/service/eks/optimized-ami/1.18/amazon-linux-2/recommended/image_id
aws ssm get-parameters --names /aws/service/bottlerocket/aws-k8s-1.18/x86_64/latest/image_id

TODO

BootstrapArguments:
Type: String
Description: Arguments to pass to the bootstrap script. See files/bootstrap.sh in https://github.com/awslabs/amazon-eks-ami


NOTE for new regions
"AWS::SSM::Parameter" may not be onboarded yet, so we need templatize CFN template
so that we do not pass invalid "AWS::SSM::Parameter" at all in those regions
Expand Down Expand Up @@ -308,7 +301,7 @@ const metadataAL2InstallSSM = ` Metadata:
01InstallAWSCLI:
# AL2 doesn't have aws cli installed
command: |
curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
curl "https://s3.${AWS::Region}.${AWS::URLSuffix}/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
unzip awscli-bundle.zip
sudo ./awscli-bundle/install -i /usr/local/aws -b /usr/bin/aws
which aws
Expand Down Expand Up @@ -357,11 +350,14 @@ const userDataAL2InstallSSM = ` UserData:
EOF

# https://docs.aws.amazon.com/inspector/latest/userguide/inspector_installing-uninstalling-agents.html
curl -O https://inspector-agent.amazonaws.com/linux/latest/install
chmod +x install
sudo ./install -u false
rm install

if [[ "${AWS::Partition}" == "aws-iso-b" ]] || [[ "${AWS::Partition}" == "aws-iso" ]]; then
echo "skipping inspector installation"
else
curl -O https://inspector-agent.amazonaws.com/linux/latest/install
chmod +x install
sudo ./install -u false
rm install
fi
sudo yum install -y yum-utils device-mapper-persistent-data lvm2
sudo amazon-linux-extras install docker -y

Expand Down Expand Up @@ -472,8 +468,14 @@ func (ts *tester) createASGs() error {
tg.UserData += fmt.Sprintf(` %s`, cur.KubeletExtraArgs)
}
tg.UserData += "'"
if cur.BootstrapArgs != "" {
ts.cfg.Logger.Info("adding further additional bootstrap arguments to user data",
zap.String("bootstrap-args", cur.BootstrapArgs),
)
tg.UserData += fmt.Sprintf(` %s`, cur.BootstrapArgs)
}
tg.UserData += "\n"
tg.UserData += ` /opt/aws/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource ASG --region ${AWS::Region}`
tg.UserData += ` /opt/aws/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource ASG --region ${AWS::Region} --url='https://cloudformation.${AWS::Region}.${AWS::URLSuffix}' --role='${RoleName}'`
}
tg.ASGTagData = ""
if cur.ClusterAutoscaler != nil && cur.ClusterAutoscaler.Enable {
Expand Down
5 changes: 5 additions & 0 deletions eksconfig/add-on-node-groups.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ type ASG struct {
// ref. https://github.com/kubernetes/kubernetes/issues/64659
KubeletExtraArgs string `json:"kubelet-extra-args"`

// BootstrapArgs additional bootstrap arguments.
// e.g. '--pause-container-account 012345678901 --pause-container-version 3.3'
BootstrapArgs string `json:"bootstrap-args"`

// ClusterAutoscaler is enabled to run cluster auto-scaler per node group.
// ref. https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler
ClusterAutoscaler *NGClusterAutoscaler `json:"cluster-autoscaler,omitempty"`
Expand Down Expand Up @@ -132,6 +136,7 @@ func getDefaultAddOnNodeGroups(name string) *AddOnNodeGroups {
ASGDesiredCapacity: 1,
},
KubeletExtraArgs: "",
BootstrapArgs: "",
ClusterAutoscaler: &NGClusterAutoscaler{Enable: false},
},
},
Expand Down
3 changes: 2 additions & 1 deletion eksconfig/env_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ spec:
defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_SERVICE_PRINCIPALS")
os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_MANAGED_POLICY_ARNS", "a,b,c")
defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ROLE_MANAGED_POLICY_ARNS")
os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS", `{"ng-test-name-cpu":{"name":"ng-test-name-cpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64","image-id-ssm-parameter":"/aws/service/eks/optimized-ami/1.30/amazon-linux-2/recommended/image_id","asg-min-size":17,"kubelet-extra-args":"bbb qq", "cluster-autoscaler" : {"enable" : false}, "asg-max-size":99,"asg-desired-capacity":77,"instance-types":["type-cpu-2"],"volume-size":40},"ng-test-name-gpu":{"name":"ng-test-name-gpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64_GPU","asg-min-size":30,"asg-max-size":35,"asg-desired-capacity":34,"instance-types":["type-gpu-2"],"image-id":"my-gpu-ami","volume-size":500, "cluster-autoscaler": {"enable":false},"kubelet-extra-args":"aaa aa"}}`)
os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS", `{"ng-test-name-cpu":{"name":"ng-test-name-cpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64","image-id-ssm-parameter":"/aws/service/eks/optimized-ami/1.30/amazon-linux-2/recommended/image_id","asg-min-size":17,"kubelet-extra-args":"bbb qq","bootstrap-args":"--pause-container-account 012345678901", "cluster-autoscaler" : {"enable" : false}, "asg-max-size":99,"asg-desired-capacity":77,"instance-types":["type-cpu-2"],"volume-size":40},"ng-test-name-gpu":{"name":"ng-test-name-gpu","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64_GPU","asg-min-size":30,"asg-max-size":35,"asg-desired-capacity":34,"instance-types":["type-gpu-2"],"image-id":"my-gpu-ami","volume-size":500, "cluster-autoscaler": {"enable":false},"kubelet-extra-args":"aaa aa"}}`)
defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS")
os.Setenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_LOGS_DIR", "a")
defer os.Unsetenv("AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_LOGS_DIR")
Expand Down Expand Up @@ -805,6 +805,7 @@ spec:
InstanceTypes: []string{"type-cpu-2"},
VolumeSize: 40,
},
BootstrapArgs: "--pause-container-account 012345678901",
KubeletExtraArgs: "bbb qq",
ClusterAutoscaler: &NGClusterAutoscaler{Enable: false},
},
Expand Down