Skip to content

Commit

Permalink
Create golden image test for nodeup kube-apiserver
Browse files Browse the repository at this point in the history
The building of the manifests for nodeup tasks can be a little opaque;
create a test to verify and expose what is happening.
  • Loading branch information
justinsb committed May 18, 2020
1 parent 2e5d476 commit 35c6056
Show file tree
Hide file tree
Showing 11 changed files with 302 additions and 10 deletions.
3 changes: 3 additions & 0 deletions nodeup/pkg/model/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,14 @@ go_test(
"//nodeup/pkg/distros:go_default_library",
"//pkg/apis/kops:go_default_library",
"//pkg/apis/nodeup:go_default_library",
"//pkg/assets:go_default_library",
"//pkg/client/simple/vfsclientset:go_default_library",
"//pkg/configbuilder:go_default_library",
"//pkg/flagbuilder:go_default_library",
"//pkg/pki:go_default_library",
"//pkg/testutils:go_default_library",
"//upup/pkg/fi:go_default_library",
"//upup/pkg/fi/cloudup:go_default_library",
"//upup/pkg/fi/nodeup/nodetasks:go_default_library",
"//util/pkg/exec:go_default_library",
"//util/pkg/hashing:go_default_library",
Expand Down
3 changes: 2 additions & 1 deletion nodeup/pkg/model/containerd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package model
import (
"os"
"path"
"path/filepath"
"testing"

"k8s.io/kops/pkg/apis/kops"
Expand Down Expand Up @@ -169,5 +170,5 @@ func runContainerdBuilderTest(t *testing.T, key string) {
return
}

testutils.ValidateTasks(t, basedir, context)
testutils.ValidateTasks(t, filepath.Join(basedir, "tasks.yaml"), context)
}
2 changes: 1 addition & 1 deletion nodeup/pkg/model/docker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -262,5 +262,5 @@ func runDockerBuilderTest(t *testing.T, key string) {
return
}

testutils.ValidateTasks(t, basedir, context)
testutils.ValidateTasks(t, filepath.Join(basedir, "tasks.yaml"), context)
}
7 changes: 7 additions & 0 deletions nodeup/pkg/model/kube_apiserver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -238,3 +238,10 @@ func Test_KubeAPIServer_BuildFlags(t *testing.T) {
}
}
}

func TestKubeAPIServerBuilder(t *testing.T) {
RunGoldenTest(t, "tests/golden/minimal", "kube-apiserver", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error {
builder := KubeAPIServerBuilder{NodeupModelContext: nodeupModelContext}
return builder.Build(target)
})
}
57 changes: 56 additions & 1 deletion nodeup/pkg/model/kubelet_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,17 @@ package model

import (
"fmt"
"path/filepath"
"testing"

"k8s.io/kops/nodeup/pkg/distros"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/pkg/client/simple/vfsclientset"
"k8s.io/kops/pkg/testutils"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup"
"k8s.io/kops/util/pkg/vfs"
)

func Test_InstanceGroupKubeletMerge(t *testing.T) {
Expand Down Expand Up @@ -181,7 +186,7 @@ func Test_RunKubeletBuilder(t *testing.T) {
context.AddTask(task)
}

testutils.ValidateTasks(t, basedir, context)
testutils.ValidateTasks(t, filepath.Join(basedir, "tasks.yaml"), context)
}

func BuildNodeupModelContext(basedir string) (*NodeupModelContext, error) {
Expand Down Expand Up @@ -214,3 +219,53 @@ func BuildNodeupModelContext(basedir string) (*NodeupModelContext, error) {

return nodeUpModelContext, nil
}

func mockedPopulateClusterSpec(c *kops.Cluster) (*kops.Cluster, error) {
vfs.Context.ResetMemfsContext(true)

assetBuilder := assets.NewAssetBuilder(c, "")
basePath, err := vfs.Context.BuildVfsPath("memfs://tests")
if err != nil {
return nil, fmt.Errorf("error building vfspath: %v", err)
}
clientset := vfsclientset.NewVFSClientset(basePath, true)
return cloudup.PopulateClusterSpec(clientset, c, assetBuilder)
}

func RunGoldenTest(t *testing.T, basedir string, key string, builder func(*NodeupModelContext, *fi.ModelBuilderContext) error) {
h := testutils.NewIntegrationTestHarness(t)
defer h.Close()

h.MockKopsVersion("1.18.0")
h.SetupMockAWS()

context := &fi.ModelBuilderContext{
Tasks: make(map[string]fi.Task),
}
nodeupModelContext, err := BuildNodeupModelContext(basedir)
if err != nil {
t.Fatalf("error loading model %q: %v", basedir, err)
return
}
nodeupModelContext.KeyStore = &fakeKeyStore{T: t}

// Populate the cluster
{
err := cloudup.PerformAssignments(nodeupModelContext.Cluster)
if err != nil {
t.Fatalf("error from PerformAssignments: %v", err)
}

full, err := mockedPopulateClusterSpec(nodeupModelContext.Cluster)
if err != nil {
t.Fatalf("unexpected error from mockedPopulateClusterSpec: %v", err)
}
nodeupModelContext.Cluster = full
}

if err := builder(nodeupModelContext, context); err != nil {
t.Fatalf("error from Build: %v", err)
}

testutils.ValidateTasks(t, filepath.Join(basedir, "tasks-"+key+".yaml"), context)
}
3 changes: 2 additions & 1 deletion nodeup/pkg/model/protokube_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package model

import (
"path"
"path/filepath"
"testing"

"k8s.io/kops/pkg/apis/kops"
Expand Down Expand Up @@ -76,5 +77,5 @@ func runProtokubeBuilderTest(t *testing.T, key string) {
context.AddTask(task)
}

testutils.ValidateTasks(t, basedir, context)
testutils.ValidateTasks(t, filepath.Join(basedir, "tasks.yaml"), context)
}
64 changes: 64 additions & 0 deletions nodeup/pkg/model/tests/golden/minimal/cluster.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
apiVersion: kops.k8s.io/v1alpha2
kind: Cluster
metadata:
name: minimal.example.com
spec:
kubernetesApiAccess:
- 0.0.0.0/0
channel: stable
cloudProvider: aws
configBase: memfs://clusters.example.com/minimal.example.com
etcdClusters:
- cpuRequest: 200m
etcdMembers:
- instanceGroup: master-us-test-1a
name: us-test-1a
memoryRequest: 100Mi
name: main
provider: Manager
backups:
backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main
- cpuRequest: 100m
etcdMembers:
- instanceGroup: master-us-test-1a
name: us-test-1a
memoryRequest: 100Mi
name: events
provider: Manager
backups:
backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events
kubernetesVersion: v1.18.0
masterInternalName: api.internal.minimal.example.com
masterPublicName: api.minimal.example.com
networkCIDR: 172.20.0.0/16
networking:
kubenet: {}
nonMasqueradeCIDR: 100.64.0.0/10
sshAccess:
- 0.0.0.0/0
topology:
masters: public
nodes: public
subnets:
- cidr: 172.20.32.0/19
name: us-test-1a
type: Public
zone: us-test-1a

---

apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
name: master-us-test-1a
labels:
kops.k8s.io/cluster: minimal.example.com
spec:
associatePublicIp: true
image: ami-1234
machineType: m3.medium
maxSize: 1
minSize: 1
role: Master
subnets:
- us-test-1a
161 changes: 161 additions & 0 deletions nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
contents: |
apiVersion: v1
kind: Pod
metadata:
annotations:
dns.alpha.kubernetes.io/external: api.minimal.example.com
dns.alpha.kubernetes.io/internal: api.internal.minimal.example.com
scheduler.alpha.kubernetes.io/critical-pod: ""
creationTimestamp: null
labels:
k8s-app: kube-apiserver
name: kube-apiserver
namespace: kube-system
spec:
containers:
- args:
- --allow-privileged=true
- --anonymous-auth=false
- --apiserver-count=1
- --authorization-mode=AlwaysAllow
- --bind-address=0.0.0.0
- --client-ca-file=/srv/kubernetes/ca.crt
- --cloud-provider=aws
- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota
- --etcd-cafile=/etc/kubernetes/pki/kube-apiserver/etcd-ca.crt
- --etcd-certfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.crt
- --etcd-keyfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.key
- --etcd-servers-overrides=/events#https://127.0.0.1:4002
- --etcd-servers=https://127.0.0.1:4001
- --insecure-bind-address=127.0.0.1
- --insecure-port=0
- --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
- --proxy-client-cert-file=/srv/kubernetes/apiserver-aggregator.cert
- --proxy-client-key-file=/srv/kubernetes/apiserver-aggregator.key
- --requestheader-allowed-names=aggregator
- --requestheader-client-ca-file=/srv/kubernetes/apiserver-aggregator-ca.cert
- --requestheader-extra-headers-prefix=X-Remote-Extra-
- --requestheader-group-headers=X-Remote-Group
- --requestheader-username-headers=X-Remote-User
- --secure-port=443
- --service-cluster-ip-range=100.64.0.0/13
- --storage-backend=etcd3
- --tls-cert-file=/srv/kubernetes/server.cert
- --tls-private-key-file=/srv/kubernetes/server.key
- --token-auth-file=/srv/kubernetes/known_tokens.csv
- --v=2
- --logtostderr=false
- --alsologtostderr
- --log-file=/var/log/kube-apiserver.log
command:
- /usr/local/bin/kube-apiserver
image: k8s.gcr.io/kube-apiserver:v1.18.0
livenessProbe:
httpGet:
host: 127.0.0.1
path: /healthz
port: 443
scheme: HTTPS
initialDelaySeconds: 45
timeoutSeconds: 15
name: kube-apiserver
ports:
- containerPort: 443
hostPort: 443
name: https
resources:
requests:
cpu: 150m
volumeMounts:
- mountPath: /var/log/kube-apiserver.log
name: logfile
- mountPath: /etc/ssl
name: etcssl
readOnly: true
- mountPath: /etc/pki/tls
name: etcpkitls
readOnly: true
- mountPath: /etc/pki/ca-trust
name: etcpkica-trust
readOnly: true
- mountPath: /usr/share/ssl
name: usrsharessl
readOnly: true
- mountPath: /usr/ssl
name: usrssl
readOnly: true
- mountPath: /usr/lib/ssl
name: usrlibssl
readOnly: true
- mountPath: /usr/local/openssl
name: usrlocalopenssl
readOnly: true
- mountPath: /var/ssl
name: varssl
readOnly: true
- mountPath: /etc/openssl
name: etcopenssl
readOnly: true
- mountPath: /etc/kubernetes/pki/kube-apiserver
name: pki
- mountPath: /srv/kubernetes
name: srvkube
readOnly: true
- mountPath: /srv/sshproxy
name: srvsshproxy
readOnly: true
hostNetwork: true
priorityClassName: system-cluster-critical
tolerations:
- key: CriticalAddonsOnly
operator: Exists
volumes:
- hostPath:
path: /var/log/kube-apiserver.log
name: logfile
- hostPath:
path: /etc/ssl
name: etcssl
- hostPath:
path: /etc/pki/tls
name: etcpkitls
- hostPath:
path: /etc/pki/ca-trust
name: etcpkica-trust
- hostPath:
path: /usr/share/ssl
name: usrsharessl
- hostPath:
path: /usr/ssl
name: usrssl
- hostPath:
path: /usr/lib/ssl
name: usrlibssl
- hostPath:
path: /usr/local/openssl
name: usrlocalopenssl
- hostPath:
path: /var/ssl
name: varssl
- hostPath:
path: /etc/openssl
name: etcopenssl
- hostPath:
path: /etc/kubernetes/pki/kube-apiserver
type: DirectoryOrCreate
name: pki
- hostPath:
path: /srv/kubernetes
name: srvkube
- hostPath:
path: /srv/sshproxy
name: srvsshproxy
status: {}
path: /etc/kubernetes/manifests/kube-apiserver.manifest
type: file
---
contents: ""
ifNotExists: true
mode: "0400"
path: /var/log/kube-apiserver.log
type: file
3 changes: 2 additions & 1 deletion pkg/model/components/etcdmanager/model_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package etcdmanager

import (
"fmt"
"path/filepath"
"testing"

"k8s.io/kops/pkg/assets"
Expand Down Expand Up @@ -56,7 +57,7 @@ func Test_RunEtcdManagerBuilder(t *testing.T) {
return
}

testutils.ValidateTasks(t, basedir, context)
testutils.ValidateTasks(t, filepath.Join(basedir, "tasks.yaml"), context)
})
}
}
Expand Down
Loading

0 comments on commit 35c6056

Please sign in to comment.