diff --git a/dns-controller/cmd/dns-controller/BUILD.bazel b/dns-controller/cmd/dns-controller/BUILD.bazel index a7eaa486f0944..bc80a0a89171b 100644 --- a/dns-controller/cmd/dns-controller/BUILD.bazel +++ b/dns-controller/cmd/dns-controller/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//dnsprovider/pkg/dnsprovider/providers/coredns:go_default_library", "//dnsprovider/pkg/dnsprovider/providers/google/clouddns:go_default_library", "//pkg/resources/digitalocean/dns:go_default_library", + "//pkg/wellknownports:go_default_library", "//protokube/pkg/gossip:go_default_library", "//protokube/pkg/gossip/dns:go_default_library", "//protokube/pkg/gossip/dns/provider:go_default_library", diff --git a/dns-controller/cmd/dns-controller/main.go b/dns-controller/cmd/dns-controller/main.go index b36fee3fcb40c..b4af4a3d12a52 100644 --- a/dns-controller/cmd/dns-controller/main.go +++ b/dns-controller/cmd/dns-controller/main.go @@ -41,6 +41,7 @@ import ( k8scoredns "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/coredns" _ "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/google/clouddns" _ "k8s.io/kops/pkg/resources/digitalocean/dns" + "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/protokube/pkg/gossip" gossipdns "k8s.io/kops/protokube/pkg/gossip/dns" gossipdnsprovider "k8s.io/kops/protokube/pkg/gossip/dns/provider" @@ -70,10 +71,10 @@ func main() { flags.StringSliceVarP(&zones, "zone", "z", []string{}, "Configure permitted zones and their mappings") flags.StringVar(&dnsProviderID, "dns", "aws-route53", "DNS provider we should use (aws-route53, google-clouddns, digitalocean, coredns, gossip)") flag.StringVar(&gossipProtocol, "gossip-protocol", "mesh", "mesh/memberlist") - flags.StringVar(&gossipListen, "gossip-listen", "0.0.0.0:3998", "The address on which to listen if gossip is enabled") + flags.StringVar(&gossipListen, "gossip-listen", fmt.Sprintf("0.0.0.0:%d", wellknownports.DNSControllerGossipWeaveMesh), "The address on which to listen if gossip is enabled") flags.StringVar(&gossipSecret, "gossip-secret", gossipSecret, "Secret to use to secure gossip") flag.StringVar(&gossipProtocolSecondary, "gossip-protocol-secondary", "", "mesh/memberlist") - flag.StringVar(&gossipListenSecondary, "gossip-listen-secondary", "0.0.0.0:4000", "address:port on which to bind for gossip") + flag.StringVar(&gossipListenSecondary, "gossip-listen-secondary", fmt.Sprintf("0.0.0.0:%d", wellknownports.DNSControllerGossipMemberlist), "address:port on which to bind for gossip") flags.StringVar(&gossipSecretSecondary, "gossip-secret-secondary", gossipSecret, "Secret to use to secure gossip") flags.StringSliceVar(&gossipSeedsSecondary, "gossip-seed-secondary", gossipSeedsSecondary, "If set, will enable gossip zones and seed using the provided addresses") flags.StringVar(&watchNamespace, "watch-namespace", "", "Limits the functionality for pods, services and ingress to specific namespace, by default all") diff --git a/docs/development/ports.md b/docs/development/ports.md index e133f5d7b32e2..ae84e669edeff 100644 --- a/docs/development/ports.md +++ b/docs/development/ports.md @@ -3,15 +3,24 @@ This document includes the port used by system components, so we can avoid port collisions. -| Port | Description | -|------|----------------------------------------| -| 22 | SSH | -| 443 | Kubernetes API | -| 179 | Calico | -| 2380 | etcd main peering | -| 2381 | etcd events peering | -| 3998 | dns gossip - protokube | -| 3999 | dns gossip - dns-controller | -| 4001 | etcd main client | -| 4002 | etcd events client | -| 4789 | VXLAN | +See also pkg/wellknownports/wellknownports.go + + +| Port | Description | +|------|------------------------------------------| +| 22 | SSH | +| 443 | Kubernetes API | +| 179 | Calico | +| 2380 | etcd main peering | +| 2381 | etcd events peering | +| 3992 | dns gossip - protokube - memberlist | +| 3993 | dns gossip - dns-controller - memberlist | +| 3994 | etcd-manager - main - quarantined | +| 3995 | etcd-manager - events - quarantined | +| 3996 | etcd-manager - main - grpc | +| 3997 | etcd-manager - events - grpc | +| 3998 | dns gossip - protokube - weave mesh | +| 3999 | dns gossip - dns-controller - weave mesh | +| 4001 | etcd main client | +| 4002 | etcd events client | +| 4789 | VXLAN | diff --git a/hack/.packages b/hack/.packages index 5ae4c8e9da659..82741b6a00af6 100644 --- a/hack/.packages +++ b/hack/.packages @@ -140,6 +140,7 @@ k8s.io/kops/pkg/util/subnet k8s.io/kops/pkg/util/templater k8s.io/kops/pkg/validation k8s.io/kops/pkg/values +k8s.io/kops/pkg/wellknownports k8s.io/kops/protokube/cmd/protokube k8s.io/kops/protokube/pkg/etcd k8s.io/kops/protokube/pkg/gossip diff --git a/pkg/model/components/etcdmanager/BUILD.bazel b/pkg/model/components/etcdmanager/BUILD.bazel index 9d2e1467869bd..8772b8b99cdc2 100644 --- a/pkg/model/components/etcdmanager/BUILD.bazel +++ b/pkg/model/components/etcdmanager/BUILD.bazel @@ -19,6 +19,7 @@ go_library( "//pkg/model:go_default_library", "//pkg/model/components:go_default_library", "//pkg/urls:go_default_library", + "//pkg/wellknownports:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awsup:go_default_library", "//upup/pkg/fi/cloudup/do:go_default_library", diff --git a/pkg/model/components/etcdmanager/model.go b/pkg/model/components/etcdmanager/model.go index cdd43961ac4a5..3d439a3e367c6 100644 --- a/pkg/model/components/etcdmanager/model.go +++ b/pkg/model/components/etcdmanager/model.go @@ -37,6 +37,7 @@ import ( "k8s.io/kops/pkg/k8scodecs" "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/pkg/model" + "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/do" @@ -286,9 +287,9 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po pod.Labels["k8s-app"] = pod.Name // TODO: Use a socket file for the quarantine port - quarantinedClientPort := 3994 + quarantinedClientPort := wellknownports.EtcdMainQuarantinedClientPort - grpcPort := 3996 + grpcPort := wellknownports.EtcdMainGRPC // The dns suffix logic mirrors the existing logic, so we should be compatible with existing clusters // (etcd makes it difficult to change peer urls, treating it as a cluster event, for reasons unknown) @@ -310,8 +311,8 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po case "events": clientPort = 4002 peerPort = 2381 - grpcPort = 3997 - quarantinedClientPort = 3995 + grpcPort = wellknownports.EtcdEventsGRPC + quarantinedClientPort = wellknownports.EtcdEventsQuarantinedClientPort default: return nil, fmt.Errorf("unknown etcd cluster key %q", etcdCluster.Name) diff --git a/pkg/model/openstackmodel/BUILD.bazel b/pkg/model/openstackmodel/BUILD.bazel index 7e28e14a1e00a..ff81130d26872 100644 --- a/pkg/model/openstackmodel/BUILD.bazel +++ b/pkg/model/openstackmodel/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//pkg/apis/kops:go_default_library", "//pkg/dns:go_default_library", "//pkg/model:go_default_library", + "//pkg/wellknownports:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/openstack:go_default_library", "//upup/pkg/fi/cloudup/openstacktasks:go_default_library", diff --git a/pkg/model/openstackmodel/firewall.go b/pkg/model/openstackmodel/firewall.go index 47fd13e3b1bc1..71d23b83d7353 100644 --- a/pkg/model/openstackmodel/firewall.go +++ b/pkg/model/openstackmodel/firewall.go @@ -22,9 +22,9 @@ import ( "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks" - //TODO: Replace with klog "k8s.io/klog" "k8s.io/kops/pkg/dns" + "k8s.io/kops/pkg/wellknownports" ) const ( @@ -438,18 +438,20 @@ func (b *FirewallModelBuilder) addProtokubeRules(c *fi.ModelBuilderContext, sgMa nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode) masterSG := sgMap[masterName] nodeSG := sgMap[nodeName] - protokubeRule := &openstacktasks.SecurityGroupRule{ - Lifecycle: b.Lifecycle, - Direction: s(string(rules.DirIngress)), - Protocol: s(string(rules.ProtocolTCP)), - EtherType: s(string(rules.EtherType4)), - PortRangeMin: i(3994), - PortRangeMax: i(3999), + for _, portRange := range wellknownports.DNSGossipPortRanges() { + protokubeRule := &openstacktasks.SecurityGroupRule{ + Lifecycle: b.Lifecycle, + Direction: s(string(rules.DirIngress)), + Protocol: s(string(rules.ProtocolTCP)), + EtherType: s(string(rules.EtherType4)), + PortRangeMin: i(portRange.Min), + PortRangeMax: i(portRange.Max), + } + addDirectionalGroupRule(c, masterSG, nodeSG, protokubeRule) + addDirectionalGroupRule(c, nodeSG, masterSG, protokubeRule) + addDirectionalGroupRule(c, masterSG, masterSG, protokubeRule) + addDirectionalGroupRule(c, nodeSG, nodeSG, protokubeRule) } - addDirectionalGroupRule(c, masterSG, nodeSG, protokubeRule) - addDirectionalGroupRule(c, nodeSG, masterSG, protokubeRule) - addDirectionalGroupRule(c, masterSG, masterSG, protokubeRule) - addDirectionalGroupRule(c, nodeSG, nodeSG, protokubeRule) } return nil } diff --git a/pkg/wellknownports/BUILD.bazel b/pkg/wellknownports/BUILD.bazel new file mode 100644 index 0000000000000..ea4e5654a797a --- /dev/null +++ b/pkg/wellknownports/BUILD.bazel @@ -0,0 +1,8 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["wellknownports.go"], + importpath = "k8s.io/kops/pkg/wellknownports", + visibility = ["//visibility:public"], +) diff --git a/pkg/wellknownports/wellknownports.go b/pkg/wellknownports/wellknownports.go new file mode 100644 index 0000000000000..d1b38744d59ad --- /dev/null +++ b/pkg/wellknownports/wellknownports.go @@ -0,0 +1,56 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wellknownports + +const ( + // EtcdMainQuarantinedClientPort is the port used by etcd when quarantined, for the main etcd + EtcdMainQuarantinedClientPort = 3994 + + // EtcdEventsQuarantinedClientPort is the port used by etcd when quarantined, for the events etcd + EtcdEventsQuarantinedClientPort = 3995 + + // EtcdMainGRPC is the GRPC port used by etcd-manager, for the main etcd + EtcdMainGRPC = 3996 + + // EtcdEventsGRPC is the GRPC port used by etcd-manager, for the events etcd + EtcdEventsGRPC = 3997 + + // DNSControllerGossipWeaveMesh is the port where dns-controller listens for the weave-mesh backend gossip + DNSControllerGossipWeaveMesh = 3998 + + // ProtokubeGossipWeaveMesh is the port where protokube listens for the weave-mesh-backed gossip + ProtokubeGossipWeaveMesh = 3999 + + // ProtokubeGossipMemberlist is the port where protokube listens for the memberlist-backed gossip + ProtokubeGossipMemberlist = 4000 + + // DNSControllerGossipMemberlist is the port where dns-controller listens for the memberlist-backed gossip + DNSControllerGossipMemberlist = 3993 +) + +type PortRange struct { + Min int + Max int +} + +func DNSGossipPortRanges() []PortRange { + return []PortRange{ + // 3993 is used by dns-controller, which is less important, so we might be able to drop it + {Min: 3993, Max: 3993}, + {Min: 3998, Max: 4000}, + } +} diff --git a/protokube/cmd/protokube/BUILD.bazel b/protokube/cmd/protokube/BUILD.bazel index 925405cdf7cf9..02dce985946a9 100644 --- a/protokube/cmd/protokube/BUILD.bazel +++ b/protokube/cmd/protokube/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "//dnsprovider/pkg/dnsprovider/providers/aws/route53:go_default_library", "//dnsprovider/pkg/dnsprovider/providers/coredns:go_default_library", "//dnsprovider/pkg/dnsprovider/providers/google/clouddns:go_default_library", + "//pkg/wellknownports:go_default_library", "//protokube/pkg/gossip:go_default_library", "//protokube/pkg/gossip/dns:go_default_library", "//protokube/pkg/gossip/memberlist:go_default_library", diff --git a/protokube/cmd/protokube/main.go b/protokube/cmd/protokube/main.go index ef5f8aeca3288..483053b065be6 100644 --- a/protokube/cmd/protokube/main.go +++ b/protokube/cmd/protokube/main.go @@ -26,8 +26,11 @@ import ( "path" "strings" + "github.com/spf13/pflag" + "k8s.io/klog" "k8s.io/kops/dns-controller/pkg/dns" "k8s.io/kops/dnsprovider/pkg/dnsprovider" + "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/protokube/pkg/gossip" gossipdns "k8s.io/kops/protokube/pkg/gossip/dns" _ "k8s.io/kops/protokube/pkg/gossip/memberlist" @@ -35,8 +38,6 @@ import ( "k8s.io/kops/protokube/pkg/protokube" // Load DNS plugins - "github.com/spf13/pflag" - "k8s.io/klog" _ "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/aws/route53" k8scoredns "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/coredns" _ "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/google/clouddns" @@ -80,10 +81,10 @@ func run() error { flags.IntVar(&dnsUpdateInterval, "dns-update-interval", 5, "Configure interval at which to update DNS records.") flag.StringVar(&flagChannels, "channels", flagChannels, "channels to install") flag.StringVar(&gossipProtocol, "gossip-protocol", "mesh", "mesh/memberlist") - flag.StringVar(&gossipListen, "gossip-listen", "0.0.0.0:3999", "address:port on which to bind for gossip") + flag.StringVar(&gossipListen, "gossip-listen", fmt.Sprintf("0.0.0.0:%d", wellknownports.ProtokubeGossipWeaveMesh), "address:port on which to bind for gossip") flags.StringVar(&gossipSecret, "gossip-secret", gossipSecret, "Secret to use to secure gossip") flag.StringVar(&gossipProtocolSecondary, "gossip-protocol-secondary", "memberlist", "mesh/memberlist") - flag.StringVar(&gossipListenSecondary, "gossip-listen-secondary", "0.0.0.0:4000", "address:port on which to bind for gossip") + flag.StringVar(&gossipListenSecondary, "gossip-listen-secondary", fmt.Sprintf("0.0.0.0:%d", wellknownports.ProtokubeGossipMemberlist), "address:port on which to bind for gossip") flags.StringVar(&gossipSecretSecondary, "gossip-secret-secondary", gossipSecret, "Secret to use to secure gossip") flag.StringVar(&peerCA, "peer-ca", peerCA, "Path to a file containing the peer ca in PEM format") flag.StringVar(&peerCert, "peer-cert", peerCert, "Path to a file containing the peer certificate") diff --git a/upup/pkg/fi/cloudup/BUILD.bazel b/upup/pkg/fi/cloudup/BUILD.bazel index 817ef3528ad9d..1181061e02a4b 100644 --- a/upup/pkg/fi/cloudup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/BUILD.bazel @@ -56,6 +56,7 @@ go_library( "//pkg/resources/spotinst:go_default_library", "//pkg/templates:go_default_library", "//pkg/util/subnet:go_default_library", + "//pkg/wellknownports:go_default_library", "//upup/models:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/assettasks:go_default_library", diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 68fc88ce6afaa..035b801999ccc 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -44,6 +44,7 @@ import ( "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model" "k8s.io/kops/pkg/resources/spotinst" + "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/gce" "k8s.io/kops/util/pkg/env" @@ -229,7 +230,7 @@ func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) { if tf.cluster.Spec.DNSControllerGossipConfig.Seed != nil { argv = append(argv, "--gossip-seed="+*tf.cluster.Spec.DNSControllerGossipConfig.Seed) } else { - argv = append(argv, "--gossip-seed=127.0.0.1:3999") + argv = append(argv, fmt.Sprintf("--gossip-seed=127.0.0.1:%d", wellknownports.ProtokubeGossipWeaveMesh)) } if tf.cluster.Spec.DNSControllerGossipConfig.Secondary != nil { @@ -246,16 +247,16 @@ func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) { if tf.cluster.Spec.DNSControllerGossipConfig.Secondary.Seed != nil { argv = append(argv, "--gossip-seed-secondary="+*tf.cluster.Spec.DNSControllerGossipConfig.Secondary.Seed) } else { - argv = append(argv, "--gossip-seed-secondary=127.0.0.1:4000") + argv = append(argv, fmt.Sprintf("--gossip-seed-secondary=127.0.0.1:%d", wellknownports.ProtokubeGossipMemberlist)) } } } else { // Default to primary mesh and secondary memberlist - argv = append(argv, "--gossip-seed=127.0.0.1:3999") + argv = append(argv, fmt.Sprintf("--gossip-seed=127.0.0.1:%d", wellknownports.ProtokubeGossipWeaveMesh)) argv = append(argv, "--gossip-protocol-secondary=memberlist") - argv = append(argv, "--gossip-listen-secondary=0.0.0.0:3997") - argv = append(argv, "--gossip-seed-secondary=127.0.0.1:4000") + argv = append(argv, fmt.Sprintf("--gossip-listen-secondary=0.0.0.0:%d", wellknownports.DNSControllerGossipMemberlist)) + argv = append(argv, fmt.Sprintf("--gossip-seed-secondary=127.0.0.1:%d", wellknownports.ProtokubeGossipMemberlist)) } } else { switch kops.CloudProviderID(tf.cluster.Spec.CloudProvider) {