Skip to content

Commit

Permalink
Merge pull request #14983 from johngmyers/boot-gossip
Browse files Browse the repository at this point in the history
Use state store for nodeup.Config in Gossip clusters
  • Loading branch information
k8s-ci-robot authored Jan 12, 2023
2 parents 7a9abba + 0a41995 commit 8127590
Show file tree
Hide file tree
Showing 17 changed files with 685 additions and 68 deletions.
8 changes: 8 additions & 0 deletions pkg/apis/kops/model/features.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ func UseKopsControllerForNodeBootstrap(cluster *kops.Cluster) bool {
}
}

// UseKopsControllerForNodeConfig checks if nodeup should use kops-controller to get nodeup.Config.
func UseKopsControllerForNodeConfig(cluster *kops.Cluster) bool {
if cluster.IsGossip() {
return false
}
return UseKopsControllerForNodeBootstrap(cluster)
}

// UseCiliumEtcd is true if we are using the Cilium etcd cluster.
func UseCiliumEtcd(cluster *kops.Cluster) bool {
if cluster.Spec.Networking.Cilium == nil {
Expand Down
2 changes: 1 addition & 1 deletion pkg/model/bootstrapscript.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ func (b *BootstrapScript) buildEnvironmentVariables(cluster *kops.Cluster) (map[
env["GOSSIP_DNS_CONN_LIMIT"] = os.Getenv("GOSSIP_DNS_CONN_LIMIT")
}

if os.Getenv("S3_ENDPOINT") != "" && (!model.UseKopsControllerForNodeBootstrap(cluster) || b.ig.HasAPIServer()) {
if os.Getenv("S3_ENDPOINT") != "" && (!model.UseKopsControllerForNodeConfig(cluster) || b.ig.HasAPIServer()) {
env["S3_ENDPOINT"] = os.Getenv("S3_ENDPOINT")
env["S3_REGION"] = os.Getenv("S3_REGION")
env["S3_ACCESS_KEY_ID"] = os.Getenv("S3_ACCESS_KEY_ID")
Expand Down
6 changes: 5 additions & 1 deletion pkg/model/iam/iam_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -698,10 +698,14 @@ func ReadableStatePaths(cluster *kops.Cluster, role Subject) ([]string, error) {

case *NodeRoleNode:
// Give access to keys for client certificates as needed.
if !model.UseKopsControllerForNodeBootstrap(cluster) {
if !model.UseKopsControllerForNodeConfig(cluster) {
paths = append(paths,
"/cluster-completed.spec",
"/igconfig/node/*",
)
}
if !model.UseKopsControllerForNodeBootstrap(cluster) {
paths = append(paths,
"/secrets/dockerconfig",
"/pki/private/kube-proxy/*",
)
Expand Down
43 changes: 42 additions & 1 deletion pkg/model/iam/iam_builder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ func TestRoundTrip(t *testing.T) {

func TestPolicyGeneration(t *testing.T) {
grid := []struct {
Gossip bool
Role Subject
AllowContainerRegistry bool
Policy string
Expand All @@ -111,6 +112,18 @@ func TestPolicyGeneration(t *testing.T) {
AllowContainerRegistry: true,
Policy: "tests/iam_builder_master_strict_ecr.json",
},
{
Gossip: true,
Role: &NodeRoleMaster{},
AllowContainerRegistry: false,
Policy: "tests/iam_builder_master_gossip.json",
},
{
Gossip: true,
Role: &NodeRoleMaster{},
AllowContainerRegistry: true,
Policy: "tests/iam_builder_master_gossip_ecr.json",
},
{
Role: &NodeRoleNode{},
AllowContainerRegistry: false,
Expand All @@ -121,6 +134,18 @@ func TestPolicyGeneration(t *testing.T) {
AllowContainerRegistry: true,
Policy: "tests/iam_builder_node_strict_ecr.json",
},
{
Gossip: true,
Role: &NodeRoleNode{},
AllowContainerRegistry: false,
Policy: "tests/iam_builder_node_gossip.json",
},
{
Gossip: true,
Role: &NodeRoleNode{},
AllowContainerRegistry: true,
Policy: "tests/iam_builder_node_gossip_ecr.json",
},
{
Role: &NodeRoleBastion{},
AllowContainerRegistry: false,
Expand All @@ -131,6 +156,18 @@ func TestPolicyGeneration(t *testing.T) {
AllowContainerRegistry: true,
Policy: "tests/iam_builder_bastion.json",
},
{
Gossip: true,
Role: &NodeRoleBastion{},
AllowContainerRegistry: false,
Policy: "tests/iam_builder_bastion.json",
},
{
Gossip: true,
Role: &NodeRoleBastion{},
AllowContainerRegistry: true,
Policy: "tests/iam_builder_bastion.json",
},
}

for i, x := range grid {
Expand Down Expand Up @@ -178,7 +215,11 @@ func TestPolicyGeneration(t *testing.T) {
Role: x.Role,
Partition: "aws-test",
}
b.Cluster.SetName("iam-builder-test.k8s.local")
if x.Gossip {
b.Cluster.SetName("iam-builder-test.k8s.local")
} else {
b.Cluster.SetName("iam-builder-test.nonexistant")
}

p, err := b.BuildAWSPolicy()
if err != nil {
Expand Down
226 changes: 226 additions & 0 deletions pkg/model/iam/tests/iam_builder_master_gossip.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,226 @@
{
"Statement": [
{
"Action": "ec2:AttachVolume",
"Condition": {
"StringEquals": {
"aws:ResourceTag/KubernetesCluster": "iam-builder-test.k8s.local",
"aws:ResourceTag/k8s.io/role/master": "1"
}
},
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Action": [
"s3:Get*"
],
"Effect": "Allow",
"Resource": "arn:aws-test:s3:::kops-tests/iam-builder-test.k8s.local/*"
},
{
"Action": [
"s3:GetBucketLocation",
"s3:GetEncryptionConfiguration",
"s3:ListBucket",
"s3:ListBucketVersions"
],
"Effect": "Allow",
"Resource": [
"arn:aws-test:s3:::kops-tests"
]
},
{
"Action": "ec2:CreateTags",
"Condition": {
"StringEquals": {
"aws:RequestTag/KubernetesCluster": "iam-builder-test.k8s.local",
"ec2:CreateAction": [
"CreateSecurityGroup"
]
}
},
"Effect": "Allow",
"Resource": [
"arn:aws-test:ec2:*:*:security-group/*"
]
},
{
"Action": [
"ec2:CreateTags",
"ec2:DeleteTags"
],
"Condition": {
"Null": {
"aws:RequestTag/KubernetesCluster": "true"
},
"StringEquals": {
"aws:ResourceTag/KubernetesCluster": "iam-builder-test.k8s.local"
}
},
"Effect": "Allow",
"Resource": [
"arn:aws-test:ec2:*:*:security-group/*"
]
},
{
"Action": "ec2:CreateTags",
"Condition": {
"StringEquals": {
"aws:RequestTag/KubernetesCluster": "iam-builder-test.k8s.local",
"ec2:CreateAction": [
"CreateVolume",
"CreateSnapshot"
]
}
},
"Effect": "Allow",
"Resource": [
"arn:aws-test:ec2:*:*:volume/*",
"arn:aws-test:ec2:*:*:snapshot/*"
]
},
{
"Action": [
"ec2:CreateTags",
"ec2:DeleteTags"
],
"Condition": {
"Null": {
"aws:RequestTag/KubernetesCluster": "true"
},
"StringEquals": {
"aws:ResourceTag/KubernetesCluster": "iam-builder-test.k8s.local"
}
},
"Effect": "Allow",
"Resource": [
"arn:aws-test:ec2:*:*:volume/*",
"arn:aws-test:ec2:*:*:snapshot/*"
]
},
{
"Action": [
"autoscaling:DescribeAutoScalingGroups",
"autoscaling:DescribeAutoScalingInstances",
"autoscaling:DescribeLaunchConfigurations",
"autoscaling:DescribeScalingActivities",
"autoscaling:DescribeTags",
"ec2:AttachVolume",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CreateSecurityGroup",
"ec2:CreateTags",
"ec2:DeleteRoute",
"ec2:DeleteSecurityGroup",
"ec2:DeleteVolume",
"ec2:DescribeAccountAttributes",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeInstanceTypes",
"ec2:DescribeInstances",
"ec2:DescribeLaunchTemplateVersions",
"ec2:DescribeRegions",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSubnets",
"ec2:DescribeTags",
"ec2:DescribeVolumes",
"ec2:DescribeVolumesModifications",
"ec2:DescribeVpcs",
"ec2:DetachVolume",
"ec2:ModifyInstanceAttribute",
"ec2:ModifyVolume",
"elasticloadbalancing:AddTags",
"elasticloadbalancing:CreateListener",
"elasticloadbalancing:CreateTargetGroup",
"elasticloadbalancing:DeleteListener",
"elasticloadbalancing:DescribeListeners",
"elasticloadbalancing:DescribeLoadBalancerAttributes",
"elasticloadbalancing:DescribeLoadBalancerPolicies",
"elasticloadbalancing:DescribeLoadBalancers",
"elasticloadbalancing:DescribeTargetGroups",
"elasticloadbalancing:DescribeTargetHealth",
"elasticloadbalancing:ModifyListener",
"elasticloadbalancing:RegisterTargets",
"iam:GetServerCertificate",
"iam:ListServerCertificates",
"kms:CreateGrant",
"kms:Decrypt",
"kms:DescribeKey",
"kms:Encrypt",
"kms:GenerateDataKey*",
"kms:GenerateRandom",
"kms:ReEncrypt*"
],
"Effect": "Allow",
"Resource": "*"
},
{
"Action": [
"autoscaling:SetDesiredCapacity",
"autoscaling:TerminateInstanceInAutoScalingGroup",
"ec2:AttachVolume",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CreateRoute",
"ec2:DeleteRoute",
"ec2:DeleteSecurityGroup",
"ec2:DeleteVolume",
"ec2:DetachVolume",
"ec2:ModifyInstanceAttribute",
"ec2:ModifyVolume",
"ec2:RevokeSecurityGroupIngress",
"elasticloadbalancing:AddTags",
"elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
"elasticloadbalancing:AttachLoadBalancerToSubnets",
"elasticloadbalancing:ConfigureHealthCheck",
"elasticloadbalancing:CreateLoadBalancerListeners",
"elasticloadbalancing:CreateLoadBalancerPolicy",
"elasticloadbalancing:DeleteListener",
"elasticloadbalancing:DeleteLoadBalancer",
"elasticloadbalancing:DeleteLoadBalancerListeners",
"elasticloadbalancing:DeleteTargetGroup",
"elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
"elasticloadbalancing:DeregisterTargets",
"elasticloadbalancing:DetachLoadBalancerFromSubnets",
"elasticloadbalancing:ModifyListener",
"elasticloadbalancing:ModifyLoadBalancerAttributes",
"elasticloadbalancing:ModifyTargetGroup",
"elasticloadbalancing:RegisterInstancesWithLoadBalancer",
"elasticloadbalancing:RegisterTargets",
"elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer",
"elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
],
"Condition": {
"StringEquals": {
"aws:ResourceTag/KubernetesCluster": "iam-builder-test.k8s.local"
}
},
"Effect": "Allow",
"Resource": "*"
},
{
"Action": [
"ec2:CreateSecurityGroup",
"ec2:CreateSnapshot",
"ec2:CreateVolume",
"elasticloadbalancing:CreateListener",
"elasticloadbalancing:CreateLoadBalancer",
"elasticloadbalancing:CreateTargetGroup"
],
"Condition": {
"StringEquals": {
"aws:RequestTag/KubernetesCluster": "iam-builder-test.k8s.local"
}
},
"Effect": "Allow",
"Resource": "*"
},
{
"Action": "ec2:CreateSecurityGroup",
"Effect": "Allow",
"Resource": "arn:aws-test:ec2:*:*:vpc/*"
}
],
"Version": "2012-10-17"
}
Loading

0 comments on commit 8127590

Please sign in to comment.