diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 39dedd6168117..19a1667b6df25 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -419,6 +419,14 @@ func TestNTHQueueProcessor(t *testing.T) { newIntegrationTest("nthsqsresources.example.com", "nth_sqs_resources").runTestCloudformation(t) } +// TestCustomIRSA runs a simple configuration, but with some additional IAM roles for ServiceAccounts +func TestCustomIRSA(t *testing.T) { + newIntegrationTest("minimal.example.com", "irsa"). + withServiceAccountRole("myserviceaccount.default", false). + withServiceAccountRole("myotherserviceaccount.myapp", true). + runTestTerraformAWS(t) +} + func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarness, expectedDataFilenames []string, tfFileName string, expectedTfFileName string, phase *cloudup.Phase) { ctx := context.Background() @@ -620,7 +628,6 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) { }...) } } - expectedFilenames = append(expectedFilenames, i.expectServiceAccountRolePolicies...) i.runTest(t, h, expectedFilenames, tfFileName, tfFileName, nil) diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 0fdcecd6718b0..f42f78a89f183 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -1163,6 +1163,39 @@ spec: type: boolean permissionsBoundary: type: string + serviceAccountExternalPermissions: + description: ServiceAccountExternalPermissions defines the relatinship + between Kubernetes ServiceAccounts and permissions with external + resources. + items: + description: ServiceAccountExternalPermissions grants a ServiceAccount + permissions to external resources. + properties: + aws: + description: AWS grants permissions to AWS resources. + properties: + inlinePolicy: + description: InlinePolicy is an IAM Policy that will + be attached inline to the IAM Role. + type: string + policyARNs: + description: PolicyARNs is a list of existing IAM Policies. + items: + type: string + type: array + type: object + name: + description: Name is the name of the Kubernetes ServiceAccount. + type: string + namespace: + description: Namespace is the namespace of the Kubernetes + ServiceAccount. + type: string + required: + - name + - namespace + type: object + type: array required: - legacy type: object diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 68992f27dbb5e..602dd6044a8b7 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -218,6 +218,24 @@ type ServiceAccountIssuerDiscoveryConfig struct { EnableAWSOIDCProvider bool `json:"enableAWSOIDCProvider,omitempty"` } +// ServiceAccountExternalPermissions grants a ServiceAccount permissions to external resources. +type ServiceAccountExternalPermission struct { + // Name is the name of the Kubernetes ServiceAccount. + Name string `json:"name"` + // Namespace is the namespace of the Kubernetes ServiceAccount. + Namespace string `json:"namespace"` + // AWS grants permissions to AWS resources. + AWS *AWSPermission `json:"aws,omitempty"` +} + +// AWSPermission grants permissions to AWS resources. +type AWSPermission struct { + // PolicyARNs is a list of existing IAM Policies. + PolicyARNs []string `json:"policyARNs,omitempty"` + // InlinePolicy is an IAM Policy that will be attached inline to the IAM Role. + InlinePolicy string `json:"inlinePolicy,omitempty"` +} + // NodeAuthorizationSpec is used to node authorization type NodeAuthorizationSpec struct { // NodeAuthorizer defined the configuration for the node authorizer @@ -280,6 +298,8 @@ type IAMSpec struct { Legacy bool `json:"legacy"` AllowContainerRegistry bool `json:"allowContainerRegistry,omitempty"` PermissionsBoundary *string `json:"permissionsBoundary,omitempty"` + // ServiceAccountExternalPermissions defines the relatinship between Kubernetes ServiceAccounts and permissions with external resources. + ServiceAccountExternalPermissions []ServiceAccountExternalPermission `json:"serviceAccountExternalPermissions,omitempty"` } // HookSpec is a definition hook diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index dede788895203..7a62d236c88ea 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -217,6 +217,24 @@ type ServiceAccountIssuerDiscoveryConfig struct { EnableAWSOIDCProvider bool `json:"enableAWSOIDCProvider,omitempty"` } +// ServiceAccountExternalPermissions grants a ServiceAccount permissions to external resources. +type ServiceAccountExternalPermission struct { + // Name is the name of the Kubernetes ServiceAccount. + Name string `json:"name"` + // Namespace is the namespace of the Kubernetes ServiceAccount. + Namespace string `json:"namespace"` + // AWS grants permissions to AWS resources. + AWS *AWSPermission `json:"aws,omitempty"` +} + +// AWSPermission grants permissions to AWS resources. +type AWSPermission struct { + // PolicyARNs is a list of existing IAM Policies. + PolicyARNs []string `json:"policyARNs,omitempty"` + // InlinePolicy is an IAM Policy that will be attached inline to the IAM Role. + InlinePolicy string `json:"inlinePolicy,omitempty"` +} + // NodeAuthorizationSpec is used to node authorization type NodeAuthorizationSpec struct { // NodeAuthorizer defined the configuration for the node authorizer @@ -278,6 +296,8 @@ type IAMSpec struct { Legacy bool `json:"legacy"` AllowContainerRegistry bool `json:"allowContainerRegistry,omitempty"` PermissionsBoundary *string `json:"permissionsBoundary,omitempty"` + // ServiceAccountExternalPermissions defines the relatinship between Kubernetes ServiceAccounts and permissions with external resources. + ServiceAccountExternalPermissions []ServiceAccountExternalPermission `json:"serviceAccountExternalPermissions,omitempty"` } // HookSpec is a definition hook diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 94041adfd4878..c12fdb801299d 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -53,6 +53,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*AWSPermission)(nil), (*kops.AWSPermission)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_AWSPermission_To_kops_AWSPermission(a.(*AWSPermission), b.(*kops.AWSPermission), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.AWSPermission)(nil), (*AWSPermission)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_AWSPermission_To_v1alpha2_AWSPermission(a.(*kops.AWSPermission), b.(*AWSPermission), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*AccessSpec)(nil), (*kops.AccessSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_AccessSpec_To_kops_AccessSpec(a.(*AccessSpec), b.(*kops.AccessSpec), scope) }); err != nil { @@ -983,6 +993,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*ServiceAccountExternalPermission)(nil), (*kops.ServiceAccountExternalPermission)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_ServiceAccountExternalPermission_To_kops_ServiceAccountExternalPermission(a.(*ServiceAccountExternalPermission), b.(*kops.ServiceAccountExternalPermission), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.ServiceAccountExternalPermission)(nil), (*ServiceAccountExternalPermission)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_ServiceAccountExternalPermission_To_v1alpha2_ServiceAccountExternalPermission(a.(*kops.ServiceAccountExternalPermission), b.(*ServiceAccountExternalPermission), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*ServiceAccountIssuerDiscoveryConfig)(nil), (*kops.ServiceAccountIssuerDiscoveryConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_ServiceAccountIssuerDiscoveryConfig_To_kops_ServiceAccountIssuerDiscoveryConfig(a.(*ServiceAccountIssuerDiscoveryConfig), b.(*kops.ServiceAccountIssuerDiscoveryConfig), scope) }); err != nil { @@ -1122,6 +1142,28 @@ func Convert_kops_AWSLoadBalancerControllerConfig_To_v1alpha2_AWSLoadBalancerCon return autoConvert_kops_AWSLoadBalancerControllerConfig_To_v1alpha2_AWSLoadBalancerControllerConfig(in, out, s) } +func autoConvert_v1alpha2_AWSPermission_To_kops_AWSPermission(in *AWSPermission, out *kops.AWSPermission, s conversion.Scope) error { + out.PolicyARNs = in.PolicyARNs + out.InlinePolicy = in.InlinePolicy + return nil +} + +// Convert_v1alpha2_AWSPermission_To_kops_AWSPermission is an autogenerated conversion function. +func Convert_v1alpha2_AWSPermission_To_kops_AWSPermission(in *AWSPermission, out *kops.AWSPermission, s conversion.Scope) error { + return autoConvert_v1alpha2_AWSPermission_To_kops_AWSPermission(in, out, s) +} + +func autoConvert_kops_AWSPermission_To_v1alpha2_AWSPermission(in *kops.AWSPermission, out *AWSPermission, s conversion.Scope) error { + out.PolicyARNs = in.PolicyARNs + out.InlinePolicy = in.InlinePolicy + return nil +} + +// Convert_kops_AWSPermission_To_v1alpha2_AWSPermission is an autogenerated conversion function. +func Convert_kops_AWSPermission_To_v1alpha2_AWSPermission(in *kops.AWSPermission, out *AWSPermission, s conversion.Scope) error { + return autoConvert_kops_AWSPermission_To_v1alpha2_AWSPermission(in, out, s) +} + func autoConvert_v1alpha2_AccessSpec_To_kops_AccessSpec(in *AccessSpec, out *kops.AccessSpec, s conversion.Scope) error { if in.DNS != nil { in, out := &in.DNS, &out.DNS @@ -3822,6 +3864,17 @@ func autoConvert_v1alpha2_IAMSpec_To_kops_IAMSpec(in *IAMSpec, out *kops.IAMSpec out.Legacy = in.Legacy out.AllowContainerRegistry = in.AllowContainerRegistry out.PermissionsBoundary = in.PermissionsBoundary + if in.ServiceAccountExternalPermissions != nil { + in, out := &in.ServiceAccountExternalPermissions, &out.ServiceAccountExternalPermissions + *out = make([]kops.ServiceAccountExternalPermission, len(*in)) + for i := range *in { + if err := Convert_v1alpha2_ServiceAccountExternalPermission_To_kops_ServiceAccountExternalPermission(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.ServiceAccountExternalPermissions = nil + } return nil } @@ -3834,6 +3887,17 @@ func autoConvert_kops_IAMSpec_To_v1alpha2_IAMSpec(in *kops.IAMSpec, out *IAMSpec out.Legacy = in.Legacy out.AllowContainerRegistry = in.AllowContainerRegistry out.PermissionsBoundary = in.PermissionsBoundary + if in.ServiceAccountExternalPermissions != nil { + in, out := &in.ServiceAccountExternalPermissions, &out.ServiceAccountExternalPermissions + *out = make([]ServiceAccountExternalPermission, len(*in)) + for i := range *in { + if err := Convert_kops_ServiceAccountExternalPermission_To_v1alpha2_ServiceAccountExternalPermission(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.ServiceAccountExternalPermissions = nil + } return nil } @@ -6266,6 +6330,46 @@ func Convert_kops_SSHCredentialSpec_To_v1alpha2_SSHCredentialSpec(in *kops.SSHCr return autoConvert_kops_SSHCredentialSpec_To_v1alpha2_SSHCredentialSpec(in, out, s) } +func autoConvert_v1alpha2_ServiceAccountExternalPermission_To_kops_ServiceAccountExternalPermission(in *ServiceAccountExternalPermission, out *kops.ServiceAccountExternalPermission, s conversion.Scope) error { + out.Name = in.Name + out.Namespace = in.Namespace + if in.AWS != nil { + in, out := &in.AWS, &out.AWS + *out = new(kops.AWSPermission) + if err := Convert_v1alpha2_AWSPermission_To_kops_AWSPermission(*in, *out, s); err != nil { + return err + } + } else { + out.AWS = nil + } + return nil +} + +// Convert_v1alpha2_ServiceAccountExternalPermission_To_kops_ServiceAccountExternalPermission is an autogenerated conversion function. +func Convert_v1alpha2_ServiceAccountExternalPermission_To_kops_ServiceAccountExternalPermission(in *ServiceAccountExternalPermission, out *kops.ServiceAccountExternalPermission, s conversion.Scope) error { + return autoConvert_v1alpha2_ServiceAccountExternalPermission_To_kops_ServiceAccountExternalPermission(in, out, s) +} + +func autoConvert_kops_ServiceAccountExternalPermission_To_v1alpha2_ServiceAccountExternalPermission(in *kops.ServiceAccountExternalPermission, out *ServiceAccountExternalPermission, s conversion.Scope) error { + out.Name = in.Name + out.Namespace = in.Namespace + if in.AWS != nil { + in, out := &in.AWS, &out.AWS + *out = new(AWSPermission) + if err := Convert_kops_AWSPermission_To_v1alpha2_AWSPermission(*in, *out, s); err != nil { + return err + } + } else { + out.AWS = nil + } + return nil +} + +// Convert_kops_ServiceAccountExternalPermission_To_v1alpha2_ServiceAccountExternalPermission is an autogenerated conversion function. +func Convert_kops_ServiceAccountExternalPermission_To_v1alpha2_ServiceAccountExternalPermission(in *kops.ServiceAccountExternalPermission, out *ServiceAccountExternalPermission, s conversion.Scope) error { + return autoConvert_kops_ServiceAccountExternalPermission_To_v1alpha2_ServiceAccountExternalPermission(in, out, s) +} + func autoConvert_v1alpha2_ServiceAccountIssuerDiscoveryConfig_To_kops_ServiceAccountIssuerDiscoveryConfig(in *ServiceAccountIssuerDiscoveryConfig, out *kops.ServiceAccountIssuerDiscoveryConfig, s conversion.Scope) error { out.DiscoveryStore = in.DiscoveryStore out.EnableAWSOIDCProvider = in.EnableAWSOIDCProvider diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 6b2d0d3527a64..9e7eb1f0a8ef6 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -83,6 +83,27 @@ func (in *AWSLoadBalancerControllerConfig) DeepCopy() *AWSLoadBalancerController return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSPermission) DeepCopyInto(out *AWSPermission) { + *out = *in + if in.PolicyARNs != nil { + in, out := &in.PolicyARNs, &out.PolicyARNs + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSPermission. +func (in *AWSPermission) DeepCopy() *AWSPermission { + if in == nil { + return nil + } + out := new(AWSPermission) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AccessSpec) DeepCopyInto(out *AccessSpec) { *out = *in @@ -1944,6 +1965,13 @@ func (in *IAMSpec) DeepCopyInto(out *IAMSpec) { *out = new(string) **out = **in } + if in.ServiceAccountExternalPermissions != nil { + in, out := &in.ServiceAccountExternalPermissions, &out.ServiceAccountExternalPermissions + *out = make([]ServiceAccountExternalPermission, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -4375,6 +4403,27 @@ func (in *SSHCredentialSpec) DeepCopy() *SSHCredentialSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceAccountExternalPermission) DeepCopyInto(out *ServiceAccountExternalPermission) { + *out = *in + if in.AWS != nil { + in, out := &in.AWS, &out.AWS + *out = new(AWSPermission) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceAccountExternalPermission. +func (in *ServiceAccountExternalPermission) DeepCopy() *ServiceAccountExternalPermission { + if in == nil { + return nil + } + out := new(ServiceAccountExternalPermission) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceAccountIssuerDiscoveryConfig) DeepCopyInto(out *ServiceAccountIssuerDiscoveryConfig) { *out = *in diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index af75ec09cd63a..3c5b87933f515 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -254,6 +254,52 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie } } + if spec.IAM != nil { + if len(spec.IAM.ServiceAccountExternalPermissions) > 0 { + if spec.ServiceAccountIssuerDiscovery == nil || !spec.ServiceAccountIssuerDiscovery.EnableAWSOIDCProvider { + allErrs = append(allErrs, field.Forbidden(fieldPath.Child("iam", "serviceAccountExternalPermissions"), "serviceAccountExternalPermissions requires AWS OIDC Provider to be enabled")) + } + allErrs = append(allErrs, validateSAExternalPermissions(spec.IAM.ServiceAccountExternalPermissions, fieldPath.Child("iam", "serviceAccountExternalPermissions"))...) + } + } + + return allErrs +} + +func validateSAExternalPermissions(externalPermissions []kops.ServiceAccountExternalPermission, path *field.Path) (allErrs field.ErrorList) { + if len(externalPermissions) == 0 { + return allErrs + } + + sas := make(map[string]string) + for _, sa := range externalPermissions { + key := fmt.Sprintf("%s/%s", sa.Namespace, sa.Name) + p := path.Key(key) + if sa.Namespace == "" { + allErrs = append(allErrs, field.Required(p.Child("namespace"), "namespace cannot be empty")) + } + if sa.Name == "" { + allErrs = append(allErrs, field.Required(p.Child("name"), "name cannot be empty")) + } + _, duplicate := sas[key] + if duplicate { + allErrs = append(allErrs, field.Duplicate(p, key)) + } + sas[key] = "" + aws := sa.AWS + ap := p.Child("aws") + if aws == nil { + allErrs = append(allErrs, field.Required(ap, "AWS permissions must be set")) + continue + } + + if len(aws.PolicyARNs) == 0 && aws.InlinePolicy == "" { + allErrs = append(allErrs, field.Required(ap, "either inlinePolicy or policyARN must be set")) + } + if len(aws.PolicyARNs) > 0 && aws.InlinePolicy != "" { + allErrs = append(allErrs, field.Forbidden(ap, "cannot set both inlinePolicy and policyARN")) + } + } return allErrs } diff --git a/pkg/apis/kops/validation/validation_test.go b/pkg/apis/kops/validation/validation_test.go index 86a5157d95880..cbecb611edeed 100644 --- a/pkg/apis/kops/validation/validation_test.go +++ b/pkg/apis/kops/validation/validation_test.go @@ -1191,3 +1191,92 @@ func Test_Validate_CloudConfiguration(t *testing.T) { }) } } + +func TestValidateSAExternalPermissions(t *testing.T) { + grid := []struct { + Description string + Input []kops.ServiceAccountExternalPermission + ExpectedErrors []string + }{ + + { + Description: "Duplicate SA", + Input: []kops.ServiceAccountExternalPermission{ + { + Name: "MySA", + Namespace: "MyNS", + AWS: &kops.AWSPermission{ + PolicyARNs: []string{"-"}, + }, + }, + { + Name: "MySA", + Namespace: "MyNS", + AWS: &kops.AWSPermission{ + PolicyARNs: []string{"-"}, + }, + }, + }, + ExpectedErrors: []string{"Duplicate value::iam.serviceAccountExternalPermissions[MyNS/MySA]"}, + }, + { + Description: "Missing permissions", + Input: []kops.ServiceAccountExternalPermission{ + { + Name: "MySA", + Namespace: "MyNS", + }, + }, + ExpectedErrors: []string{"Required value::iam.serviceAccountExternalPermissions[MyNS/MySA].aws"}, + }, + { + Description: "Setting both arn and inline", + Input: []kops.ServiceAccountExternalPermission{ + { + Name: "MySA", + Namespace: "MyNS", + AWS: &kops.AWSPermission{ + PolicyARNs: []string{"-"}, + InlinePolicy: "-", + }, + }, + }, + ExpectedErrors: []string{"Forbidden::iam.serviceAccountExternalPermissions[MyNS/MySA].aws"}, + }, + { + Description: "Empty SA name", + Input: []kops.ServiceAccountExternalPermission{ + { + Namespace: "MyNS", + AWS: &kops.AWSPermission{ + PolicyARNs: []string{"-"}, + InlinePolicy: "-", + }, + }, + }, + ExpectedErrors: []string{"Required value::iam.serviceAccountExternalPermissions[MyNS/].name"}, + }, + { + Description: "Empty SA namespace", + Input: []kops.ServiceAccountExternalPermission{ + { + Name: "MySA", + AWS: &kops.AWSPermission{ + PolicyARNs: []string{"-"}, + InlinePolicy: "-", + }, + }, + }, + ExpectedErrors: []string{"Required value::iam.serviceAccountExternalPermissions[/MySA].namespace"}, + }, + } + + for _, g := range grid { + fldPath := field.NewPath("iam.serviceAccountExternalPermissions") + t.Run(g.Description, func(t *testing.T) { + errs := validateSAExternalPermissions(g.Input, fldPath) + testErrors(t, g.Input, errs, g.ExpectedErrors) + }) + } + +} diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 7fabb8de3c11b..5361c8280e27c 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -83,6 +83,27 @@ func (in *AWSLoadBalancerControllerConfig) DeepCopy() *AWSLoadBalancerController return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSPermission) DeepCopyInto(out *AWSPermission) { + *out = *in + if in.PolicyARNs != nil { + in, out := &in.PolicyARNs, &out.PolicyARNs + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSPermission. +func (in *AWSPermission) DeepCopy() *AWSPermission { + if in == nil { + return nil + } + out := new(AWSPermission) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AccessSpec) DeepCopyInto(out *AccessSpec) { *out = *in @@ -2110,6 +2131,13 @@ func (in *IAMSpec) DeepCopyInto(out *IAMSpec) { *out = new(string) **out = **in } + if in.ServiceAccountExternalPermissions != nil { + in, out := &in.ServiceAccountExternalPermissions, &out.ServiceAccountExternalPermissions + *out = make([]ServiceAccountExternalPermission, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -4589,6 +4617,27 @@ func (in *SSHCredentialSpec) DeepCopy() *SSHCredentialSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceAccountExternalPermission) DeepCopyInto(out *ServiceAccountExternalPermission) { + *out = *in + if in.AWS != nil { + in, out := &in.AWS, &out.AWS + *out = new(AWSPermission) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceAccountExternalPermission. +func (in *ServiceAccountExternalPermission) DeepCopy() *ServiceAccountExternalPermission { + if in == nil { + return nil + } + out := new(ServiceAccountExternalPermission) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceAccountIssuerDiscoveryConfig) DeepCopyInto(out *ServiceAccountIssuerDiscoveryConfig) { *out = *in diff --git a/pkg/featureflag/featureflag.go b/pkg/featureflag/featureflag.go index 34d10681197a4..9227f10f378e9 100644 --- a/pkg/featureflag/featureflag.go +++ b/pkg/featureflag/featureflag.go @@ -91,9 +91,9 @@ var ( LegacyIAM = New("LegacyIAM", Bool(false)) // ClusterAddons activates experimental cluster-addons support ClusterAddons = New("ClusterAddons", Bool(false)) - // UseServiceAccountIAM controls whether we use pod-level IAM permissions for our system pods. + // UseServiceAccountIAM controls whether we use pod-level IAM permissions for our system pods and kOps addons. UseServiceAccountIAM = New("UseServiceAccountIAM", Bool(false)) - // PublicJWKS enables public jwks access. This is generally not as secure as republishing. + // PublicJWKS enables public jwks access. PublicJWKS = New("PublicJWKS", Bool(false)) // Azure toggles the Azure support. Azure = New("Azure", Bool(false)) diff --git a/pkg/model/awsmodel/BUILD.bazel b/pkg/model/awsmodel/BUILD.bazel index 4e01bbf00e084..81c0a55cd89fd 100644 --- a/pkg/model/awsmodel/BUILD.bazel +++ b/pkg/model/awsmodel/BUILD.bazel @@ -35,6 +35,7 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/aws/endpoints:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", "//vendor/k8s.io/legacy-cloud-providers/aws:go_default_library", diff --git a/pkg/model/awsmodel/iam.go b/pkg/model/awsmodel/iam.go index c207481988959..572bd1f58ad44 100644 --- a/pkg/model/awsmodel/iam.go +++ b/pkg/model/awsmodel/iam.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/aws/aws-sdk-go/aws/endpoints" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/dns" @@ -124,26 +125,64 @@ func (b *IAMModelBuilder) Build(c *fi.ModelBuilderContext) error { } } + iamSpec := b.Cluster.Spec.IAM + if iamSpec != nil { + for _, sa := range iamSpec.ServiceAccountExternalPermissions { + var p *iam.Policy + aws := sa.AWS + if aws.InlinePolicy != "" { + bp, err := b.buildPolicy(aws.InlinePolicy) + p = bp + if err != nil { + return fmt.Errorf("error inline policy: %w", err) + } + } + serviceAccount := &iam.GenericServiceAccount{ + NamespacedName: types.NamespacedName{ + Name: sa.Name, + Namespace: sa.Namespace, + }, + Policy: p, + } + iamRole, err := b.BuildServiceAccountRoleTasks(serviceAccount, c) + if err != nil { + return fmt.Errorf("error building service account role tasks: %w", err) + } + if len(aws.PolicyARNs) > 0 { + name := "external-" + fi.StringValue(iamRole.Name) + externalPolicies := aws.PolicyARNs + c.AddTask(&awstasks.IAMRolePolicy{ + Name: fi.String(name), + ExternalPolicies: &externalPolicies, + Managed: true, + Role: iamRole, + Lifecycle: b.Lifecycle, + }) + } + } + + } + return nil } // BuildServiceAccountRoleTasks build tasks specifically for the ServiceAccount role. -func (b *IAMModelBuilder) BuildServiceAccountRoleTasks(role iam.Subject, c *fi.ModelBuilderContext) error { +func (b *IAMModelBuilder) BuildServiceAccountRoleTasks(role iam.Subject, c *fi.ModelBuilderContext) (*awstasks.IAMRole, error) { iamName, err := b.IAMNameForServiceAccountRole(role) if err != nil { - return err + return nil, err } iamRole, err := b.buildIAMRole(role, iamName, c) if err != nil { - return err + return nil, err } if err := b.buildIAMRolePolicy(role, iamName, iamRole, c); err != nil { - return err + return nil, err } - return nil + return iamRole, nil } func (b *IAMModelBuilder) buildIAMRole(role iam.Subject, iamName string, c *fi.ModelBuilderContext) (*awstasks.IAMRole, error) { @@ -312,20 +351,14 @@ func (b *IAMModelBuilder) buildIAMTasks(role iam.Subject, iamName string, c *fi. } if additionalPolicy != "" { - p := &iam.Policy{ - Version: iam.PolicyDefaultVersion, - } - - statements, err := iam.ParseStatements(additionalPolicy) + p, err := b.buildPolicy(additionalPolicy) if err != nil { return fmt.Errorf("additionalPolicy %q is invalid: %v", roleKey, err) } - p.Statement = append(p.Statement, statements...) - policy, err := p.AsJSON() if err != nil { - return fmt.Errorf("error building IAM policy: %v", err) + return fmt.Errorf("error building IAM policy: %w", err) } t.PolicyDocument = fi.NewStringResource(policy) @@ -340,6 +373,20 @@ func (b *IAMModelBuilder) buildIAMTasks(role iam.Subject, iamName string, c *fi. return nil } +func (b *IAMModelBuilder) buildPolicy(policyString string) (*iam.Policy, error) { + p := &iam.Policy{ + Version: iam.PolicyDefaultVersion, + } + + statements, err := iam.ParseStatements(policyString) + if err != nil { + return nil, err + } + + p.Statement = append(p.Statement, statements...) + return p, nil +} + // IAMServiceEC2 returns the name of the IAM service for EC2 in the current region. // It is ec2.amazonaws.com in the default aws partition, but different in other isolated/custom partitions func IAMServiceEC2(region string) string { diff --git a/pkg/model/iam/BUILD.bazel b/pkg/model/iam/BUILD.bazel index 9a8aec940e8c8..8715cdc2d49d9 100644 --- a/pkg/model/iam/BUILD.bazel +++ b/pkg/model/iam/BUILD.bazel @@ -33,8 +33,11 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/kops:go_default_library", + "//pkg/testutils:go_default_library", "//pkg/testutils/golden:go_default_library", "//pkg/util/stringorslice:go_default_library", + "//upup/pkg/fi:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", ], ) diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index 4b75f0d799ce5..cdcc42cd590a1 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -678,6 +678,9 @@ func (b *PolicyResource) Open() (io.Reader, error) { if err != nil { return nil, fmt.Errorf("error building IAM policy: %v", err) } + if policy == nil { + return bytes.NewReader([]byte{}), nil + } j, err := policy.AsJSON() if err != nil { return nil, fmt.Errorf("error building IAM policy: %v", err) diff --git a/pkg/model/iam/iam_builder_test.go b/pkg/model/iam/iam_builder_test.go index efb3d6e50c5bd..6c8d0fdd252ef 100644 --- a/pkg/model/iam/iam_builder_test.go +++ b/pkg/model/iam/iam_builder_test.go @@ -22,9 +22,12 @@ import ( "github.com/aws/aws-sdk-go/aws" + "k8s.io/apimachinery/pkg/types" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/testutils" "k8s.io/kops/pkg/testutils/golden" "k8s.io/kops/pkg/util/stringorslice" + "k8s.io/kops/upup/pkg/fi" ) func TestIAMPrefix(t *testing.T) { @@ -226,3 +229,34 @@ func TestPolicyGeneration(t *testing.T) { golden.AssertMatchesFile(t, actualPolicy, x.Policy) } } + +func TestEmptyPolicy(t *testing.T) { + + role := &GenericServiceAccount{ + NamespacedName: types.NamespacedName{ + Name: "myaccount", + Namespace: "default", + }, + Policy: nil, + } + + cluster := testutils.BuildMinimalCluster("irsa.example.com") + b := &PolicyBuilder{ + Cluster: cluster, + Role: role, + } + + pr := &PolicyResource{ + Builder: b, + } + + policy, err := fi.ResourceAsString(pr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if policy != "" { + t.Errorf("empty policy should result in empty string, but was %q", policy) + } + +} diff --git a/pkg/model/iam/subject.go b/pkg/model/iam/subject.go index 7e1d5acafbdd1..9245c99f8f9a7 100644 --- a/pkg/model/iam/subject.go +++ b/pkg/model/iam/subject.go @@ -77,6 +77,19 @@ func (_ *NodeRoleBastion) ServiceAccount() (types.NamespacedName, bool) { return types.NamespacedName{}, false } +type GenericServiceAccount struct { + NamespacedName types.NamespacedName + Policy *Policy +} + +func (g *GenericServiceAccount) ServiceAccount() (types.NamespacedName, bool) { + return g.NamespacedName, true +} + +func (g *GenericServiceAccount) BuildAWSPolicy(*PolicyBuilder) (*Policy, error) { + return g.Policy, nil +} + // BuildNodeRoleSubject returns a Subject implementation for the specified InstanceGroupRole. func BuildNodeRoleSubject(igRole kops.InstanceGroupRole, enableLifecycleHookPermissions bool) (Subject, error) { switch igRole { diff --git a/pkg/model/issuerdiscovery.go b/pkg/model/issuerdiscovery.go index db206a83c1da2..93194a23e80b7 100644 --- a/pkg/model/issuerdiscovery.go +++ b/pkg/model/issuerdiscovery.go @@ -28,7 +28,6 @@ import ( "gopkg.in/square/go-jose.v2" "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model/iam" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/fitasks" @@ -53,7 +52,8 @@ type oidcDiscovery struct { } func (b *IssuerDiscoveryModelBuilder) Build(c *fi.ModelBuilderContext) error { - if !featureflag.PublicJWKS.Enabled() { + said := b.Cluster.Spec.ServiceAccountIssuerDiscovery + if said == nil { return nil } diff --git a/tests/integration/update_cluster/irsa/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy b/tests/integration/update_cluster/irsa/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy new file mode 100644 index 0000000000000..67b82f7100e47 --- /dev/null +++ b/tests/integration/update_cluster/irsa/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy @@ -0,0 +1,17 @@ +{ + "Statement": [ + { + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "api.internal.minimal.example.com:sub": "system:serviceaccount:kube-system:dns-controller" + } + }, + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::123456789012:oidc-provider/api.internal.minimal.example.com" + } + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/irsa/data/aws_iam_role_masters.minimal.example.com_policy b/tests/integration/update_cluster/irsa/data/aws_iam_role_masters.minimal.example.com_policy new file mode 100644 index 0000000000000..66d5de1d5ae1e --- /dev/null +++ b/tests/integration/update_cluster/irsa/data/aws_iam_role_masters.minimal.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/irsa/data/aws_iam_role_myotherserviceaccount.myapp.sa.minimal.example.com_policy b/tests/integration/update_cluster/irsa/data/aws_iam_role_myotherserviceaccount.myapp.sa.minimal.example.com_policy new file mode 100644 index 0000000000000..9f648a493d912 --- /dev/null +++ b/tests/integration/update_cluster/irsa/data/aws_iam_role_myotherserviceaccount.myapp.sa.minimal.example.com_policy @@ -0,0 +1,17 @@ +{ + "Statement": [ + { + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "discovery.example.com/minimal.example.com/oidc:sub": "system:serviceaccount:myapp:myotherserviceaccount" + } + }, + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com/oidc" + } + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/irsa/data/aws_iam_role_myserviceaccount.default.sa.minimal.example.com_policy b/tests/integration/update_cluster/irsa/data/aws_iam_role_myserviceaccount.default.sa.minimal.example.com_policy new file mode 100644 index 0000000000000..d695610b015c8 --- /dev/null +++ b/tests/integration/update_cluster/irsa/data/aws_iam_role_myserviceaccount.default.sa.minimal.example.com_policy @@ -0,0 +1,17 @@ +{ + "Statement": [ + { + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "discovery.example.com/minimal.example.com/oidc:sub": "system:serviceaccount:default:myserviceaccount" + } + }, + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com/oidc" + } + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/irsa/data/aws_iam_role_nodes.minimal.example.com_policy b/tests/integration/update_cluster/irsa/data/aws_iam_role_nodes.minimal.example.com_policy new file mode 100644 index 0000000000000..66d5de1d5ae1e --- /dev/null +++ b/tests/integration/update_cluster/irsa/data/aws_iam_role_nodes.minimal.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/irsa/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy b/tests/integration/update_cluster/irsa/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy new file mode 100644 index 0000000000000..6e706aa3a7722 --- /dev/null +++ b/tests/integration/update_cluster/irsa/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy @@ -0,0 +1,34 @@ +{ + "Statement": [ + { + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Action": [ + "route53:GetChange" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:route53:::change/*" + ] + }, + { + "Action": [ + "route53:ListHostedZones" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/irsa/data/aws_iam_role_policy_masters.minimal.example.com_policy b/tests/integration/update_cluster/irsa/data/aws_iam_role_policy_masters.minimal.example.com_policy new file mode 100644 index 0000000000000..7a2e4b678f2bf --- /dev/null +++ b/tests/integration/update_cluster/irsa/data/aws_iam_role_policy_masters.minimal.example.com_policy @@ -0,0 +1,211 @@ +{ + "Statement": [ + { + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeInstances", + "ec2:DescribeInternetGateways", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DescribeVolumesModifications", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress" + ], + "Condition": { + "StringEquals": { + "ec2:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": "autoscaling:CompleteLifecycleAction", + "Condition": { + "StringEquals": { + "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": "autoscaling:DescribeLifecycleHooks", + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": "autoscaling:DescribeAutoScalingInstances", + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeLaunchTemplateVersions" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup" + ], + "Condition": { + "StringEquals": { + "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "autoscaling:CompleteLifecycleAction", + "autoscaling:DescribeAutoScalingInstances" + ], + "Condition": { + "StringEquals": { + "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "iam:ListServerCertificates", + "iam:GetServerCertificate" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Action": [ + "route53:GetChange" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:route53:::change/*" + ] + }, + { + "Action": [ + "route53:ListHostedZones" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/irsa/data/aws_iam_role_policy_myotherserviceaccount.myapp.sa.minimal.example.com_policy b/tests/integration/update_cluster/irsa/data/aws_iam_role_policy_myotherserviceaccount.myapp.sa.minimal.example.com_policy new file mode 100644 index 0000000000000..cd58ca06e93cc --- /dev/null +++ b/tests/integration/update_cluster/irsa/data/aws_iam_role_policy_myotherserviceaccount.myapp.sa.minimal.example.com_policy @@ -0,0 +1,23 @@ +{ + "Statement": [ + { + "Action": [ + "dynamodb:*" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "es:*" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/irsa/data/aws_iam_role_policy_nodes.minimal.example.com_policy b/tests/integration/update_cluster/irsa/data/aws_iam_role_policy_nodes.minimal.example.com_policy new file mode 100644 index 0000000000000..5ff94c80f4dfd --- /dev/null +++ b/tests/integration/update_cluster/irsa/data/aws_iam_role_policy_nodes.minimal.example.com_policy @@ -0,0 +1,22 @@ +{ + "Statement": [ + { + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": "autoscaling:DescribeAutoScalingInstances", + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/irsa/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key b/tests/integration/update_cluster/irsa/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key new file mode 100644 index 0000000000000..81cb0127830e7 --- /dev/null +++ b/tests/integration/update_cluster/irsa/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/irsa/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/irsa/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data new file mode 100644 index 0000000000000..0d5e4276a9adb --- /dev/null +++ b/tests/integration/update_cluster/irsa/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -0,0 +1,328 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: + manageStorageClasses: true +containerRuntime: containerd +containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.4 +docker: + skipInstall: true +encryptionConfig: null +etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.20.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://discovery.example.com/minimal.example.com/oidc + serviceAccountJWKSURI: https://discovery.example.com/minimal.example.com/oidc/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.20.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.20.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.20.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +{} + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: + amd64: + - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet + - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/protokube + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/channels + arm64: + - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet + - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/protokube + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/channels +ClusterName: minimal.example.com +ConfigBase: memfs://clusters.example.com/minimal.example.com +InstanceGroupName: master-us-test-1a +InstanceGroupRole: Master +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false +channels: +- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml +staticManifests: +- key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/irsa/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/irsa/data/aws_launch_template_nodes.minimal.example.com_user_data new file mode 100644 index 0000000000000..cd4e1c4fbe367 --- /dev/null +++ b/tests/integration/update_cluster/irsa/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -0,0 +1,232 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: + manageStorageClasses: true +containerRuntime: containerd +containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.4 +docker: + skipInstall: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.20.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +{} + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: + amd64: + - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet + - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz + arm64: + - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet + - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz +ClusterName: minimal.example.com +ConfigBase: memfs://clusters.example.com/minimal.example.com +InstanceGroupName: nodes +InstanceGroupRole: Node +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests +channels: +- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/irsa/id_rsa.pub b/tests/integration/update_cluster/irsa/id_rsa.pub new file mode 100755 index 0000000000000..81cb0127830e7 --- /dev/null +++ b/tests/integration/update_cluster/irsa/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/irsa/in-v1alpha2.yaml b/tests/integration/update_cluster/irsa/in-v1alpha2.yaml new file mode 100644 index 0000000000000..18b2c4551c803 --- /dev/null +++ b/tests/integration/update_cluster/irsa/in-v1alpha2.yaml @@ -0,0 +1,103 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-10T22:42:27Z" + name: minimal.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: events + iam: + serviceAccountExternalPermissions: + - name: myserviceaccount + namespace: default + aws: + policyARNs: + - arn:aws:iam::123456789012:policy/UsersManageOwnCredentials + - name: myotherserviceaccount + namespace: myapp + aws: + inlinePolicy: | + [ + { + "Effect": "Allow", + "Action": ["dynamodb:*"], + "Resource": ["*"] + }, + { + "Effect": "Allow", + "Action": ["es:*"], + "Resource": ["*"] + } + ] + kubelet: + anonymousAuth: false + kubernetesVersion: v1.20.0 + masterInternalName: api.internal.minimal.example.com + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + cni: {} + nonMasqueradeCIDR: 100.64.0.0/10 + serviceAccountIssuerDiscovery: + enableAWSOIDCProvider: true + discoveryStore: memfs://discovery.example.com/minimal.example.com + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: nodes + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: t2.medium + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: master-us-test-1a + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a diff --git a/tests/integration/update_cluster/irsa/kubernetes.tf b/tests/integration/update_cluster/irsa/kubernetes.tf new file mode 100644 index 0000000000000..5bf06bdf7cbdf --- /dev/null +++ b/tests/integration/update_cluster/irsa/kubernetes.tf @@ -0,0 +1,706 @@ +locals { + cluster_name = "minimal.example.com" + default-myserviceaccount_role_arn = aws_iam_role.myserviceaccount-default-sa-minimal-example-com.arn + default-myserviceaccount_role_name = aws_iam_role.myserviceaccount-default-sa-minimal-example-com.name + master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] + master_security_group_ids = [aws_security_group.masters-minimal-example-com.id] + masters_role_arn = aws_iam_role.masters-minimal-example-com.arn + masters_role_name = aws_iam_role.masters-minimal-example-com.name + myapp-myotherserviceaccount_role_arn = aws_iam_role.myotherserviceaccount-myapp-sa-minimal-example-com.arn + myapp-myotherserviceaccount_role_name = aws_iam_role.myotherserviceaccount-myapp-sa-minimal-example-com.name + node_autoscaling_group_ids = [aws_autoscaling_group.nodes-minimal-example-com.id] + node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id] + node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id] + nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn + nodes_role_name = aws_iam_role.nodes-minimal-example-com.name + region = "us-test-1" + route_table_public_id = aws_route_table.minimal-example-com.id + subnet_us-test-1a_id = aws_subnet.us-test-1a-minimal-example-com.id + vpc_cidr_block = aws_vpc.minimal-example-com.cidr_block + vpc_id = aws_vpc.minimal-example-com.id +} + +output "cluster_name" { + value = "minimal.example.com" +} + +output "default-myserviceaccount_role_arn" { + value = aws_iam_role.myserviceaccount-default-sa-minimal-example-com.arn +} + +output "default-myserviceaccount_role_name" { + value = aws_iam_role.myserviceaccount-default-sa-minimal-example-com.name +} + +output "master_autoscaling_group_ids" { + value = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] +} + +output "master_security_group_ids" { + value = [aws_security_group.masters-minimal-example-com.id] +} + +output "masters_role_arn" { + value = aws_iam_role.masters-minimal-example-com.arn +} + +output "masters_role_name" { + value = aws_iam_role.masters-minimal-example-com.name +} + +output "myapp-myotherserviceaccount_role_arn" { + value = aws_iam_role.myotherserviceaccount-myapp-sa-minimal-example-com.arn +} + +output "myapp-myotherserviceaccount_role_name" { + value = aws_iam_role.myotherserviceaccount-myapp-sa-minimal-example-com.name +} + +output "node_autoscaling_group_ids" { + value = [aws_autoscaling_group.nodes-minimal-example-com.id] +} + +output "node_security_group_ids" { + value = [aws_security_group.nodes-minimal-example-com.id] +} + +output "node_subnet_ids" { + value = [aws_subnet.us-test-1a-minimal-example-com.id] +} + +output "nodes_role_arn" { + value = aws_iam_role.nodes-minimal-example-com.arn +} + +output "nodes_role_name" { + value = aws_iam_role.nodes-minimal-example-com.name +} + +output "region" { + value = "us-test-1" +} + +output "route_table_public_id" { + value = aws_route_table.minimal-example-com.id +} + +output "subnet_us-test-1a_id" { + value = aws_subnet.us-test-1a-minimal-example-com.id +} + +output "vpc_cidr_block" { + value = aws_vpc.minimal-example-com.cidr_block +} + +output "vpc_id" { + value = aws_vpc.minimal-example-com.id +} + +provider "aws" { + region = "us-test-1" +} + +resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id + version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version + } + max_size = 1 + metrics_granularity = "1Minute" + min_size = 1 + name = "master-us-test-1a.masters.minimal.example.com" + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "minimal.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "master-us-test-1a.masters.minimal.example.com" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" + propagate_at_launch = true + value = "master" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/master" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "master-us-test-1a" + } + tag { + key = "kubernetes.io/cluster/minimal.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id] +} + +resource "aws_autoscaling_group" "nodes-minimal-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.nodes-minimal-example-com.id + version = aws_launch_template.nodes-minimal-example-com.latest_version + } + max_size = 2 + metrics_granularity = "1Minute" + min_size = 2 + name = "nodes.minimal.example.com" + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "minimal.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "nodes.minimal.example.com" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" + propagate_at_launch = true + value = "node" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/node" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "nodes" + } + tag { + key = "kubernetes.io/cluster/minimal.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id] +} + +resource "aws_ebs_volume" "us-test-1a-etcd-events-minimal-example-com" { + availability_zone = "us-test-1a" + encrypted = false + iops = 3000 + size = 20 + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.etcd-events.minimal.example.com" + "k8s.io/etcd/events" = "us-test-1a/us-test-1a" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + throughput = 125 + type = "gp3" +} + +resource "aws_ebs_volume" "us-test-1a-etcd-main-minimal-example-com" { + availability_zone = "us-test-1a" + encrypted = false + iops = 3000 + size = 20 + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.etcd-main.minimal.example.com" + "k8s.io/etcd/main" = "us-test-1a/us-test-1a" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + throughput = 125 + type = "gp3" +} + +resource "aws_iam_instance_profile" "masters-minimal-example-com" { + name = "masters.minimal.example.com" + role = aws_iam_role.masters-minimal-example-com.name + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "masters.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_instance_profile" "nodes-minimal-example-com" { + name = "nodes.minimal.example.com" + role = aws_iam_role.nodes-minimal-example-com.name + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_openid_connect_provider" "minimal-example-com" { + client_id_list = ["amazonaws.com"] + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + thumbprint_list = ["9e99a48a9960b14926bb7f3b02e22da2b0ab7280", "a9d53002e97e00e043244f3d170d6f4c414104fd"] + url = "https://discovery.example.com/minimal.example.com/oidc" +} + +resource "aws_iam_role_policy_attachment" "external-myserviceaccount-default-sa-minimal-example-com-3186075376" { + policy_arn = "arn:aws:iam::123456789012:policy/UsersManageOwnCredentials" + role = aws_iam_role.myserviceaccount-default-sa-minimal-example-com.name +} + +resource "aws_iam_role_policy" "masters-minimal-example-com" { + name = "masters.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_masters.minimal.example.com_policy") + role = aws_iam_role.masters-minimal-example-com.name +} + +resource "aws_iam_role_policy" "myotherserviceaccount-myapp-sa-minimal-example-com" { + name = "myotherserviceaccount.myapp.sa.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_myotherserviceaccount.myapp.sa.minimal.example.com_policy") + role = aws_iam_role.myotherserviceaccount-myapp-sa-minimal-example-com.name +} + +resource "aws_iam_role_policy" "nodes-minimal-example-com" { + name = "nodes.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_nodes.minimal.example.com_policy") + role = aws_iam_role.nodes-minimal-example-com.name +} + +resource "aws_iam_role" "masters-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_masters.minimal.example.com_policy") + name = "masters.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "masters.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_role" "myotherserviceaccount-myapp-sa-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_myotherserviceaccount.myapp.sa.minimal.example.com_policy") + name = "myotherserviceaccount.myapp.sa.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "myotherserviceaccount.myapp.sa.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_role" "myserviceaccount-default-sa-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_myserviceaccount.default.sa.minimal.example.com_policy") + name = "myserviceaccount.default.sa.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "myserviceaccount.default.sa.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_role" "nodes-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.minimal.example.com_policy") + name = "nodes.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_internet_gateway" "minimal-example-com" { + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_key_pair" "kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { + key_name = "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" + public_key = file("${path.module}/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 64 + volume_type = "gp3" + } + } + block_device_mappings { + device_name = "/dev/sdc" + virtual_name = "ephemeral0" + } + iam_instance_profile { + name = aws_iam_instance_profile.masters-minimal-example-com.id + } + image_id = "ami-12345678" + instance_type = "m3.medium" + key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_put_response_hop_limit = 1 + http_tokens = "optional" + } + name = "master-us-test-1a.masters.minimal.example.com" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + security_groups = [aws_security_group.masters-minimal-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "master-us-test-1a.masters.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "master-us-test-1a.masters.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "master-us-test-1a.masters.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data") +} + +resource "aws_launch_template" "nodes-minimal-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 128 + volume_type = "gp3" + } + } + iam_instance_profile { + name = aws_iam_instance_profile.nodes-minimal-example-com.id + } + image_id = "ami-12345678" + instance_type = "t2.medium" + key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_put_response_hop_limit = 1 + http_tokens = "optional" + } + name = "nodes.minimal.example.com" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + security_groups = [aws_security_group.nodes-minimal-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_nodes.minimal.example.com_user_data") +} + +resource "aws_route_table_association" "us-test-1a-minimal-example-com" { + route_table_id = aws_route_table.minimal-example-com.id + subnet_id = aws_subnet.us-test-1a-minimal-example-com.id +} + +resource "aws_route_table" "minimal-example-com" { + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/kops/role" = "public" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_route" "route-0-0-0-0--0" { + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.minimal-example-com.id + route_table_id = aws_route_table.minimal-example-com.id +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-masters-minimal-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-nodes-minimal-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-masters-minimal-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 443 + type = "ingress" +} + +resource "aws_security_group_rule" "from-masters-minimal-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-masters-minimal-example-com-ingress-all-0to0-masters-minimal-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-masters-minimal-example-com-ingress-all-0to0-nodes-minimal-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + source_security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-all-0to0-nodes-minimal-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-tcp-1to2379-masters-minimal-example-com" { + from_port = 1 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 2379 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-tcp-2382to4000-masters-minimal-example-com" { + from_port = 2382 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 4000 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-tcp-4003to65535-masters-minimal-example-com" { + from_port = 4003 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-udp-1to65535-masters-minimal-example-com" { + from_port = 1 + protocol = "udp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group" "masters-minimal-example-com" { + description = "Security group for masters" + name = "masters.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "masters.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_security_group" "nodes-minimal-example-com" { + description = "Security group for nodes" + name = "nodes.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_subnet" "us-test-1a-minimal-example-com" { + availability_zone = "us-test-1a" + cidr_block = "172.20.32.0/19" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_vpc_dhcp_options_association" "minimal-example-com" { + dhcp_options_id = aws_vpc_dhcp_options.minimal-example-com.id + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_vpc_dhcp_options" "minimal-example-com" { + domain_name = "us-test-1.compute.internal" + domain_name_servers = ["AmazonProvidedDNS"] + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_vpc" "minimal-example-com" { + cidr_block = "172.20.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +terraform { + required_version = ">= 0.12.26" + required_providers { + aws = { + "source" = "hashicorp/aws" + "version" = ">= 3.34.0" + } + } +} diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index 9583c8faf6e16..b242d15a3dae4 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -431,7 +431,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann for _, serviceAccountRole := range serviceAccountRoles { iamModelBuilder := &awsmodel.IAMModelBuilder{AWSModelContext: awsModelContext, Lifecycle: b.Lifecycle, Cluster: b.Cluster} - err := iamModelBuilder.BuildServiceAccountRoleTasks(serviceAccountRole, c) + _, err := iamModelBuilder.BuildServiceAccountRoleTasks(serviceAccountRole, c) if err != nil { return nil, err } @@ -593,7 +593,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann for _, serviceAccountRole := range serviceAccountRoles { iamModelBuilder := &awsmodel.IAMModelBuilder{AWSModelContext: awsModelContext, Lifecycle: b.Lifecycle, Cluster: b.Cluster} - err := iamModelBuilder.BuildServiceAccountRoleTasks(serviceAccountRole, c) + _, err := iamModelBuilder.BuildServiceAccountRoleTasks(serviceAccountRole, c) if err != nil { return nil, err }