diff --git a/cmd/kube-apiserver-healthcheck/BUILD.bazel b/cmd/kube-apiserver-healthcheck/BUILD.bazel index 9f1d5297f2de2..e80718cc070f3 100644 --- a/cmd/kube-apiserver-healthcheck/BUILD.bazel +++ b/cmd/kube-apiserver-healthcheck/BUILD.bazel @@ -5,7 +5,10 @@ go_library( srcs = ["main.go"], importpath = "k8s.io/kops/cmd/kube-apiserver-healthcheck", visibility = ["//visibility:private"], - deps = ["//vendor/k8s.io/klog:go_default_library"], + deps = [ + "//pkg/wellknownports:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], ) go_binary( diff --git a/cmd/kube-apiserver-healthcheck/README.md b/cmd/kube-apiserver-healthcheck/README.md index 6ab3d7de1e3e5..4f98f1504328d 100644 --- a/cmd/kube-apiserver-healthcheck/README.md +++ b/cmd/kube-apiserver-healthcheck/README.md @@ -4,7 +4,7 @@ This is a small sidecar container that allows for health-checking the kube-apiserver without enabling anonymous authentication and without enabling the unauthenticated port. -It listens on port 8080 (http), and proxies a few known-safe requests +It listens on port 3990 (http), and proxies a few known-safe requests to the real apiserver listening on 443. It uses a client certificate to authenticate itself to the apiserver. @@ -14,5 +14,5 @@ it also lets us have better load-balancer health-checks. Because it runs as a sidecar next to kube-apiserver, it is in the same network namespace, and thus it can reach apiserver on https://127.0.0.1 . The kube-apiserver-healthcheck process listens on -8080, but the health checks for the apiserver container are configured +3990, but the health checks for the apiserver container are configured for :8080 and actually go via the sidecar. diff --git a/cmd/kube-apiserver-healthcheck/main.go b/cmd/kube-apiserver-healthcheck/main.go index 365852a821f05..9ca0436702aa1 100644 --- a/cmd/kube-apiserver-healthcheck/main.go +++ b/cmd/kube-apiserver-healthcheck/main.go @@ -28,6 +28,7 @@ import ( "os" "k8s.io/klog" + "k8s.io/kops/pkg/wellknownports" ) // healthCheckServer is the http server @@ -129,7 +130,7 @@ func (s *healthCheckServer) proxyRequest(w http.ResponseWriter, forwardRequest * } func run() error { - listen := ":8080" + listen := fmt.Sprintf(":%d", wellknownports.KubeAPIServerHealthCheck) clientCert := "" clientKey := "" diff --git a/cmd/kube-apiserver-healthcheck/proxy_test.go b/cmd/kube-apiserver-healthcheck/proxy_test.go index 82d8680be1cbe..81c5e9260e2c9 100644 --- a/cmd/kube-apiserver-healthcheck/proxy_test.go +++ b/cmd/kube-apiserver-healthcheck/proxy_test.go @@ -27,19 +27,19 @@ func TestBuildProxyRequest(t *testing.T) { In string Out string }{ - {In: "http://127.0.0.1:8080/readyz", Out: "https://127.0.0.1/readyz"}, - {In: "http://127.0.0.1:8080/livez", Out: "https://127.0.0.1/livez"}, - {In: "http://127.0.0.1:8080/healthz", Out: "https://127.0.0.1/healthz"}, - {In: "http://127.0.0.1:8080/ready", Out: ""}, - {In: "http://127.0.0.1:8080/", Out: ""}, - {In: "http://127.0.0.1:8080/readyz/foo", Out: ""}, - {In: "http://127.0.0.1:8080/readyzfoo", Out: ""}, - {In: "http://127.0.0.1:8080/readyz?", Out: "https://127.0.0.1/readyz"}, - {In: "http://127.0.0.1:8080/readyz?foo=bar", Out: "https://127.0.0.1/readyz"}, - {In: "http://127.0.0.1:8080/readyz?exclude=1", Out: "https://127.0.0.1/readyz?exclude=1"}, - {In: "http://127.0.0.1:8080/readyz?exclude=1&exclude=2", Out: "https://127.0.0.1/readyz?exclude=1&exclude=2"}, - {In: "http://127.0.0.1:8080/readyz?exclude=1&verbose", Out: "https://127.0.0.1/readyz?exclude=1"}, - {In: "http://127.0.0.1:8080/readyz?exclude", Out: "https://127.0.0.1/readyz?exclude="}, + {In: "http://127.0.0.1:3990/readyz", Out: "https://127.0.0.1/readyz"}, + {In: "http://127.0.0.1:3990/livez", Out: "https://127.0.0.1/livez"}, + {In: "http://127.0.0.1:3990/healthz", Out: "https://127.0.0.1/healthz"}, + {In: "http://127.0.0.1:3990/ready", Out: ""}, + {In: "http://127.0.0.1:3990/", Out: ""}, + {In: "http://127.0.0.1:3990/readyz/foo", Out: ""}, + {In: "http://127.0.0.1:3990/readyzfoo", Out: ""}, + {In: "http://127.0.0.1:3990/readyz?", Out: "https://127.0.0.1/readyz"}, + {In: "http://127.0.0.1:3990/readyz?foo=bar", Out: "https://127.0.0.1/readyz"}, + {In: "http://127.0.0.1:3990/readyz?exclude=1", Out: "https://127.0.0.1/readyz?exclude=1"}, + {In: "http://127.0.0.1:3990/readyz?exclude=1&exclude=2", Out: "https://127.0.0.1/readyz?exclude=1&exclude=2"}, + {In: "http://127.0.0.1:3990/readyz?exclude=1&verbose", Out: "https://127.0.0.1/readyz?exclude=1"}, + {In: "http://127.0.0.1:3990/readyz?exclude", Out: "https://127.0.0.1/readyz?exclude="}, } for _, g := range grid { diff --git a/docs/boot-sequence.md b/docs/boot-sequence.md index db36e484720ea..5c543c14f2efa 100644 --- a/docs/boot-sequence.md +++ b/docs/boot-sequence.md @@ -67,11 +67,6 @@ that have hostNetwork will work - so all the "core" containers run with hostNetw ## api-server bringup -The api-server will listen on localhost:8080 on the master. This is an unsecured endpoint, -but is only reachable from the master, and only for pods running with hostNetwork=true. This -is how components like kube-scheduler and kube-controller-manager can reach the API without -requiring a token. - APIServer also listens on the HTTPS port (443) on all interfaces. This is a secured endpoint, and requires valid authentication/authorization to use it. This is the endpoint that node kubelets will reach, and also that end-users will reach. diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index f03cb375febcb..55025f73e406b 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -60,6 +60,7 @@ go_library( "//pkg/rbac:go_default_library", "//pkg/systemd:go_default_library", "//pkg/tokens:go_default_library", + "//pkg/wellknownports:go_default_library", "//pkg/wellknownusers:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awsup:go_default_library", diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index d91d457f1be1a..8074174eaf300 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -26,6 +26,7 @@ import ( "k8s.io/kops/pkg/k8scodecs" "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/pkg/wellknownusers" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" @@ -409,10 +410,10 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { probeAction := &v1.HTTPGetAction{ Host: "127.0.0.1", Path: "/healthz", - Port: intstr.FromInt(8080), + Port: intstr.FromInt(wellknownports.KubeAPIServerHealthCheck), } if useHealthcheckProxy { - // kube-apiserver-healthcheck sidecar container runs on port 8080 + // kube-apiserver-healthcheck sidecar container runs on port 3990 } else if kubeAPIServer.InsecurePort != 0 { probeAction.Port = intstr.FromInt(int(kubeAPIServer.InsecurePort)) } else if kubeAPIServer.SecurePort != 0 { diff --git a/pkg/model/components/apiserver.go b/pkg/model/components/apiserver.go index 60ae1de84dafd..2b28083f9c730 100644 --- a/pkg/model/components/apiserver.go +++ b/pkg/model/components/apiserver.go @@ -232,7 +232,7 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error { c.AnonymousAuth = fi.Bool(false) if b.IsKubernetesGTE("1.17") { - // We query via the kube-apiserver-healthcheck proxy, which listens on port 8080 + // We query via the kube-apiserver-healthcheck proxy, which listens on port 3990 c.InsecurePort = 0 } else { // Older versions of kubernetes continue to rely on the insecure port: kubernetes issue #43784 diff --git a/pkg/model/components/kubeapiserver/BUILD.bazel b/pkg/model/components/kubeapiserver/BUILD.bazel index 35bc8c3082e5d..1749540967b76 100644 --- a/pkg/model/components/kubeapiserver/BUILD.bazel +++ b/pkg/model/components/kubeapiserver/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "//pkg/assets:go_default_library", "//pkg/k8scodecs:go_default_library", "//pkg/model:go_default_library", + "//pkg/wellknownports:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/fitasks:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/model/components/kubeapiserver/model.go b/pkg/model/components/kubeapiserver/model.go index 03aaaa1df0887..2d210d651267c 100644 --- a/pkg/model/components/kubeapiserver/model.go +++ b/pkg/model/components/kubeapiserver/model.go @@ -24,6 +24,7 @@ import ( "k8s.io/kops/pkg/assets" "k8s.io/kops/pkg/k8scodecs" "k8s.io/kops/pkg/model" + "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/fitasks" ) @@ -100,7 +101,7 @@ spec: # The sidecar serves a healthcheck on the same port, # but with a .kube-apiserver-healthcheck prefix path: /.kube-apiserver-healthcheck/healthz - port: 8080 + port: %d host: 127.0.0.1 initialDelaySeconds: 5 timeoutSeconds: 5 @@ -126,7 +127,7 @@ spec: func (b *KubeApiserverBuilder) buildHealthcheckSidecar() (*corev1.Pod, error) { // TODO: pull from bundle bundle := "(embedded kube-apiserver-healthcheck manifest)" - manifest := []byte(defaultManifest) + manifest := []byte(fmt.Sprintf(defaultManifest, wellknownports.KubeAPIServerHealthCheck)) var pod *corev1.Pod var container *corev1.Container diff --git a/pkg/model/components/kubeapiserver/tests/minimal/tasks.yaml b/pkg/model/components/kubeapiserver/tests/minimal/tasks.yaml index 20a7fe787afd3..14c21c2477856 100644 --- a/pkg/model/components/kubeapiserver/tests/minimal/tasks.yaml +++ b/pkg/model/components/kubeapiserver/tests/minimal/tasks.yaml @@ -18,7 +18,7 @@ Contents: httpGet: host: 127.0.0.1 path: /.kube-apiserver-healthcheck/healthz - port: 8080 + port: 3990 initialDelaySeconds: 5 timeoutSeconds: 5 name: healthcheck diff --git a/pkg/wellknownports/wellknownports.go b/pkg/wellknownports/wellknownports.go index 2461ac03003ff..69892b4b62d3b 100644 --- a/pkg/wellknownports/wellknownports.go +++ b/pkg/wellknownports/wellknownports.go @@ -43,6 +43,14 @@ const ( // DNSControllerGossipMemberlist is the port where dns-controller listens for the memberlist-backed gossip DNSControllerGossipMemberlist = 3993 + + // 4001 is etcd main, 4002 is etcd events, 4003 is etcd cilium + + // KubeAPIServerHealthCheck is the port where kube-apiserver-healthcheck listens. + KubeAPIServerHealthCheck = 3990 + + // NodeLocalDNSHealthCheck is the port where the node-local-dns health check listens. + NodeLocalDNSHealthCheck = 3989 ) type PortRange struct { diff --git a/upup/models/bindata.go b/upup/models/bindata.go index 1898edcd4a780..2cb4d4f8ed01e 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -15216,7 +15216,7 @@ data: force_tcp } prometheus :9253 - health {{ KubeDNS.NodeLocalDNS.LocalIP }}:8080 + health {{ KubeDNS.NodeLocalDNS.LocalIP }}:{{ NodeLocalDNSHealthCheck }} } in-addr.arpa:53 { errors @@ -15315,7 +15315,7 @@ spec: httpGet: host: {{ .KubeDNS.NodeLocalDNS.LocalIP }} path: /health - port: 8080 + port: {{ NodeLocalDNSHealthCheck }} initialDelaySeconds: 60 timeoutSeconds: 5 volumeMounts: diff --git a/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template index e74e7ce084026..3c88f5b75a51e 100644 --- a/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template +++ b/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template @@ -55,7 +55,7 @@ data: force_tcp } prometheus :9253 - health {{ KubeDNS.NodeLocalDNS.LocalIP }}:8080 + health {{ KubeDNS.NodeLocalDNS.LocalIP }}:{{ NodeLocalDNSHealthCheck }} } in-addr.arpa:53 { errors @@ -154,7 +154,7 @@ spec: httpGet: host: {{ .KubeDNS.NodeLocalDNS.LocalIP }} path: /health - port: 8080 + port: {{ NodeLocalDNSHealthCheck }} initialDelaySeconds: 60 timeoutSeconds: 5 volumeMounts: diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 761faef836f10..df72f1fc31287 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -111,6 +111,9 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS return tf.cluster.Spec.KubeDNS.ServerIP } } + dest["NodeLocalDNSHealthCheck"] = func() string { + return fmt.Sprintf("%d", wellknownports.NodeLocalDNSHealthCheck) + } dest["KopsControllerArgv"] = tf.KopsControllerArgv dest["KopsControllerConfig"] = tf.KopsControllerConfig