Re-add integration tests for jwks

We removed them from kubernetes#10756, but they can be re-added.

cmd/kops/integration_test.go

@@ -79,6 +79,12 @@ func newIntegrationTest(clusterName, srcDir string) *integrationTest {
 	}
 }
 
+// withCAKey indicates that we should use a fixed ca.crt & ca.key from the source directory as our CA.
+// This is needed when the CA is exposed, for example when using AWS WebIdentity federation.
+func (i *integrationTest) withCAKey() *integrationTest {
+	i.caKey = true
+	return i
+}
 func (i *integrationTest) withVersion(version string) *integrationTest {
 	i.version = version
 	return i
@@ -114,6 +120,12 @@ func (i *integrationTest) withPrivate() *integrationTest {
 	return i
 }
 
+// withServiceAccountRoles indicates we expect to assign per-ServiceAccount IAM roles (instead of just using the node roles)
+func (i *integrationTest) withServiceAccountRoles() *integrationTest {
+	i.expectServiceAccountRoles = true
+	return i
+}
+
 func (i *integrationTest) withBastionUserData() *integrationTest {
 	i.bastionUserData = true
 	return i
@@ -262,6 +274,18 @@ func TestPrivateDns2(t *testing.T) {
 	newIntegrationTest("privatedns2.example.com", "privatedns2").withPrivate().runTestTerraformAWS(t)
 }
 
+// TestPublicJWKS runs a simple configuration, but with UseServiceAccountIAM and PublicJWKS enabled
+func TestPublicJWKS(t *testing.T) {
+	featureflag.ParseFlags("+UseServiceAccountIAM,+PublicJWKS")
+	unsetFeatureFlags := func() {
+		featureflag.ParseFlags("-UseServiceAccountIAM,-PublicJWKS")
+	}
+	defer unsetFeatureFlags()
+
+	// We have to use a fixed CA because the fingerprint is inserted into the AWS WebIdentity configuration.
+	newIntegrationTest("minimal.example.com", "public-jwks").withCAKey().withServiceAccountRoles().runTestTerraformAWS(t)
+}
+
 // TestSharedSubnet runs the test on a configuration with a shared subnet (and VPC)
 func TestSharedSubnet(t *testing.T) {
 	newIntegrationTest("sharedsubnet.example.com", "shared_subnet").runTestTerraformAWS(t)

pkg/apis/kops/validation/legacy.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/pkg/util/subnet"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
 )
 
 // legacy contains validation functions that don't match the apimachinery style
@@ -374,8 +375,23 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList {
 
 	publicDataStore := c.Spec.PublicDataStore
 	if publicDataStore != "" {
-		if !strings.HasPrefix(publicDataStore, "s3://") {
-			allErrs = append(allErrs, field.Invalid(fieldSpec.Child("publicDataStore"), publicDataStore, "S3 is the only supported VFS for publicStore"))
+		base, err := vfs.Context.BuildVfsPath(publicDataStore)
+		if err != nil {
+			allErrs = append(allErrs, field.Invalid(fieldSpec.Child("publicDataStore"), publicDataStore, "not a valid VFS path"))
+		} else {
+			switch base := base.(type) {
+			case *vfs.S3Path:
+				// OK
+			case *vfs.MemFSPath:
+				if !base.IsClusterReadable() {
+					// If this _is_ a test, we should call MarkClusterReadable
+					allErrs = append(allErrs, field.Invalid(fieldSpec.Child("publicDataStore"), publicDataStore, "S3 is the only supported VFS for publicStore"))
+				} else {
+					// memfs is OK for tests
+				}
+			default:
+				allErrs = append(allErrs, field.Invalid(fieldSpec.Child("publicDataStore"), publicDataStore, "S3 is the only supported VFS for publicStore"))
+			}
 		}
 	}
 

pkg/model/iam/subject.go

@@ -18,6 +18,7 @@ package iam
 
 import (
 	"fmt"
+	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -108,6 +109,12 @@ func ServiceAccountIssuer(clusterSpec *kops.ClusterSpec) (string, error) {
 				return "", err
 			}
 			return baseURL + "/oidc", nil
+		case *vfs.MemFSPath:
+			if !base.IsClusterReadable() {
+				// If this _is_ a test, we should call MarkClusterReadable
+				return "", fmt.Errorf("cluster.spec.publicDataStore=%q is only supported in tests", clusterSpec.PublicDataStore)
+			}
+			return strings.Replace(base.Path(), "memfs://", "https://", 1) + "/oidc", nil
 		default:
 			return "", fmt.Errorf("cluster.spec.publicDataStore=%q is of unexpected type %T", clusterSpec.PublicDataStore, base)
 		}

tests/integration/update_cluster/public-jwks/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy

@@ -4,12 +4,12 @@
       "Action": "sts:AssumeRoleWithWebIdentity",
       "Condition": {
         "StringEquals": {
-          "api.minimal.example.com:sub": "system:serviceaccount:kube-system:dns-controller"
+          "discovery.example.com/minimal.example.com/oidc:sub": "system:serviceaccount:kube-system:dns-controller"
         }
       },
       "Effect": "Allow",
       "Principal": {
-        "Federated": "arn:aws:iam::123456789012:oidc-provider/api.minimal.example.com"
+        "Federated": "arn:aws:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com/oidc"
       }
     }
   ],

tests/integration/update_cluster/public-jwks/data/aws_iam_role_policy_masters.minimal.example.com_policy

@@ -135,35 +135,6 @@
       "Resource": [
         "*"
       ]
-    },
-    {
-      "Action": [
-        "route53:ChangeResourceRecordSets",
-        "route53:ListResourceRecordSets",
-        "route53:GetHostedZone"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
-      ]
-    },
-    {
-      "Action": [
-        "route53:GetChange"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws:route53:::change/*"
-      ]
-    },
-    {
-      "Action": [
-        "route53:ListHostedZones"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "*"
-      ]
     }
   ],
   "Version": "2012-10-17"

tests/integration/update_cluster/public-jwks/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data

@@ -168,7 +168,7 @@ etcdClusters:
     version: 3.4.13
 kubeAPIServer:
   allowPrivileged: true
-  anonymousAuth: true
+  anonymousAuth: false
   apiAudiences:
   - kubernetes.svc.default
   apiServerCount: 1
@@ -190,6 +190,8 @@ kubeAPIServer:
   - http://127.0.0.1:4001
   etcdServersOverrides:
   - /events#http://127.0.0.1:4002
+  featureGates:
+    ServiceAccountIssuerDiscovery: "true"
   image: k8s.gcr.io/kube-apiserver:v1.20.0
   kubeletPreferredAddressTypes:
   - InternalIP
@@ -205,8 +207,8 @@ kubeAPIServer:
   requestheaderUsernameHeaders:
   - X-Remote-User
   securePort: 443
-  serviceAccountIssuer: https://api.minimal.example.com
-  serviceAccountJWKSURI: https://api.minimal.example.com/openid/v1/jwks
+  serviceAccountIssuer: https://discovery.example.com/minimal.example.com/oidc
+  serviceAccountJWKSURI: https://discovery.example.com/minimal.example.com/oidc/keys.json
   serviceClusterIPRange: 100.64.0.0/13
   storageBackend: etcd3
 kubeControllerManager:

tests/integration/update_cluster/public-jwks/in-v1alpha2.yaml

@@ -28,6 +28,7 @@ spec:
   networking:
     cni: {}
   nonMasqueradeCIDR: 100.64.0.0/10
+  publicDataStore: memfs://discovery.example.com/minimal.example.com
   sshAccess:
     - 0.0.0.0/0
   topology:

tests/integration/update_cluster/public-jwks/kubernetes.tf

@@ -245,8 +245,8 @@ resource "aws_iam_instance_profile" "nodes-minimal-example-com" {
 
 resource "aws_iam_openid_connect_provider" "minimal-example-com" {
   client_id_list  = ["amazonaws.com"]
-  thumbprint_list = ["a8de31f85544b9e73aeb26ded19330e0e996fb79"]
-  url             = "https://api.minimal.example.com"
+  thumbprint_list = ["9e99a48a9960b14926bb7f3b02e22da2b0ab7280", "a9d53002e97e00e043244f3d170d6f4c414104fd"]
+  url             = "https://discovery.example.com/minimal.example.com/oidc"
 }
 
 resource "aws_iam_role_policy" "dns-controller-kube-system-sa-minimal-example-com" {

upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go

@@ -53,6 +53,20 @@ func TestBootstrapChannelBuilder_BuildTasks(t *testing.T) {
 	runChannelBuilderTest(t, "awsiamauthenticator", []string{"authentication.aws-k8s-1.12"})
 }
 
+func TestBootstrapChannelBuilder_PublicJWKS(t *testing.T) {
+	h := testutils.NewIntegrationTestHarness(t)
+	defer h.Close()
+
+	h.SetupMockAWS()
+
+	featureflag.ParseFlags("+PublicJWKS,+UseServiceAccountIAM")
+	unsetFeatureFlag := func() {
+		featureflag.ParseFlags("-PublicJWKS,-UseServiceAccountIAM")
+	}
+	defer unsetFeatureFlag()
+	runChannelBuilderTest(t, "public-jwks", []string{"dns-controller.addons.k8s.io-k8s-1.12", "kops-controller.addons.k8s.io-k8s-1.16", "anonymous-issuer-discovery.addons.k8s.io-k8s-1.16"})
+}
+
 func TestBootstrapChannelBuilder_AWSCloudController(t *testing.T) {
 	h := testutils.NewIntegrationTestHarness(t)
 	defer h.Close()

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/cluster.yaml

@@ -30,6 +30,7 @@ spec:
   networking:
     kubenet: {}
   nonMasqueradeCIDR: 100.64.0.0/10
+  publicDataStore: memfs://discovery.example.com/minimal.example.com
   sshAccess:
     - 0.0.0.0/0
   topology:

upup/pkg/fi/fitasks/managedfile.go

@@ -105,11 +105,16 @@ func (_ *ManagedFile) Render(c *fi.Context, a, e, changes *ManagedFile) error {
 
 	var acl vfs.ACL
 	if fi.BoolValue(e.Public) {
-		switch p.(type) {
+		switch p := p.(type) {
 		case *vfs.S3Path:
 			acl = &vfs.S3Acl{
 				RequestACL: fi.String("public-read"),
 			}
+		case *vfs.MemFSPath:
+			if !p.IsClusterReadable() {
+				return fmt.Errorf("the %q path is intended for use in tests", p.Path())
+			}
+			acl = nil
 		default:
 			return fmt.Errorf("the %q path does not support public ACL", p.Path())
 		}