Skip to content

Commit

Permalink
Add support for multiple zones
Browse files Browse the repository at this point in the history
  • Loading branch information
@donovanmuller
donovanmuller committed Nov 5, 2024
1 parent f39865f commit a9589fb
Show file tree
Hide file tree
Showing 22 changed files with 401 additions and 326 deletions.
6 changes: 4 additions & 2 deletions chart/k8gb/templates/coredns-cm.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ metadata:
apiVersion: v1
data:
Corefile: |-
{{ .Values.k8gb.dnsZone }}:5353 {
{{- range .Values.k8gb.dnsZones }}
{{ .zone }}:5353 {
errors
health
{{- if .Values.k8gb.coredns.extra_plugins }}
Expand All @@ -20,10 +21,11 @@ data:
forward . /etc/resolv.conf
k8s_crd {
filter k8gb.absa.oss/dnstype=local
negttl {{ .Values.k8gb.dnsZoneNegTTL }}
negttl {{ .dnsZoneNegTTL | default 30 }}
loadbalance weight
}
}
{{- end }}
{{- with .Values.k8gb.coredns.extraServerBlocks -}}
{{- tpl . $ | nindent 4 }}
{{- end }}
Expand Down
41 changes: 28 additions & 13 deletions chart/k8gb/values.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,17 +266,11 @@
"deployRbac": {
"type": "boolean"
},
"dnsZone": {
"format": "idn-hostname",
"minLength": 1
},
"dnsZoneNegTTL": {
"type": "integer",
"minimum": 0
},
"edgeDNSZone": {
"format": "idn-hostname",
"minLength": 1
"dnsZones": {
"type": "array",
"items": {
"$ref": "#/definitions/k8gbDnsZone"
}
},
"edgeDNSServers": {
"type": "array",
Expand Down Expand Up @@ -330,9 +324,8 @@
"required": [
"clusterGeoTag",
"extGslbClustersGeoTags",
"dnsZone",
"edgeDNSServers",
"edgeDNSZone"
"dnsZones"
],
"title": "k8gb"
},
Expand Down Expand Up @@ -405,6 +398,28 @@
}
}
},
"k8gbDnsZone": {
"type": "object",
"properties": {
"edgeZone": {
"type": "string",
"format": "idn-hostname"
},
"zone": {
"type": "string",
"format": "idn-hostname"
},
"dnsZoneNegTTL": {
"type": "integer",
"minimum": 0
}
},
"required": [
"zone",
"dnsZoneNegTTL",
"edgeZone"
]
},
"Ns1": {
"type": "object",
"additionalProperties": false,
Expand Down
13 changes: 7 additions & 6 deletions chart/k8gb/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,13 @@ k8gb:
deployCrds: true
# -- whether it should also deploy the service account, cluster role and cluster role binding
deployRbac: true
# -- dnsZone controlled by gslb
dnsZone: "cloud.example.com"
# -- Negative TTL for SOA record
dnsZoneNegTTL: 300
# -- main zone which would contain gslb zone to delegate
edgeDNSZone: "example.com" # main zone which would contain gslb zone to delegate
dnsZones:
- # -- main zone which would contain gslb zone to delegate
edgeZone: "example.com"
# -- dnsZone controlled by gslb
zone: "cloud.example.com"
# -- Negative TTL for SOA record
dnsZoneNegTTL: 300
# -- host/ip[:port] format is supported here where port defaults to 53
edgeDNSServers:
# -- use this DNS server as a main resolver to enable cross k8gb DNS based communication
Expand Down
8 changes: 6 additions & 2 deletions controllers/depresolver/depresolver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,14 @@ type Config struct {
fallbackEdgeDNSServerName string `env:"EDGE_DNS_SERVER"`
// to avoid breaking changes is used as fallback server port for EdgeDNSServers
fallbackEdgeDNSServerPort int `env:"EDGE_DNS_SERVER_PORT, default=53"`
// DNSZones
DNSZones utils.DNSZoneList
// EdgeDNSZone main zone which would contain gslb zone to delegate; e.g. example.com
EdgeDNSZone string `env:"EDGE_DNS_ZONE"`
// to avoid breaking changes is used as fallback server for DNSZones
fallbackEdgeDNSZone string `env:"EDGE_DNS_ZONE"`
// DNSZone controlled by gslb; e.g. cloud.example.com
DNSZone string `env:"DNS_ZONE"`
// to avoid breaking changes is used as fallback server for DNSZones
fallbackDNSZone string `env:"DNS_ZONE"`
// K8gbNamespace k8gb namespace
K8gbNamespace string `env:"POD_NAMESPACE"`
// Infoblox configuration
Expand Down
95 changes: 66 additions & 29 deletions controllers/depresolver/depresolver_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@ const (
ExtClustersGeoTagsKey = "EXT_GSLB_CLUSTERS_GEO_TAGS"
ExtDNSEnabledKey = "EXTDNS_ENABLED"
EdgeDNSServersKey = "EDGE_DNS_SERVERS"
EdgeDNSZoneKey = "EDGE_DNS_ZONE"
DNSZoneKey = "DNS_ZONE"
DNSZonesKey = "DNS_ZONES"
InfobloxGridHostKey = "INFOBLOX_GRID_HOST"
InfobloxVersionKey = "INFOBLOX_WAPI_VERSION"
InfobloxPortKey = "INFOBLOX_WAPI_PORT"
Expand Down Expand Up @@ -67,6 +66,12 @@ const (

// Deprecated: Please use EDGE_DNS_SERVERS instead.
EdgeDNSServerPortKey = "EDGE_DNS_SERVER_PORT"

// Deprecated: Please use DNS_ZONES instead.
EdgeDNSZoneKey = "EDGE_DNS_ZONE"

// Deprecated: Please use DNS_ZONES instead.
DNSZoneKey = "DNS_ZONE"
)

// ResolveOperatorConfig executes once. It reads operator's configuration
Expand All @@ -87,6 +92,9 @@ func (dr *DependencyResolver) ResolveOperatorConfig() (*Config, error) {
fallbackDNS := fmt.Sprintf("%s:%v", dr.config.fallbackEdgeDNSServerName, dr.config.fallbackEdgeDNSServerPort)
edgeDNSServerList := env.GetEnvAsArrayOfStringsOrFallback(EdgeDNSServersKey, []string{fallbackDNS})
dr.config.EdgeDNSServers = parseEdgeDNSServers(edgeDNSServerList)
fallbackDNSZone := fmt.Sprintf("%s:%s", dr.config.fallbackEdgeDNSZone, dr.config.fallbackDNSZone)
DNSZoneList := env.GetEnvAsArrayOfStringsOrFallback(DNSZonesKey, []string{fallbackDNSZone})
dr.config.DNSZones = parseDNSZones(DNSZoneList)
dr.config.ExtClustersGeoTags = excludeGeoTag(dr.config.ExtClustersGeoTags, dr.config.ClusterGeoTag)
dr.config.Log.Level, _ = zerolog.ParseLevel(strings.ToLower(dr.config.Log.level))
dr.config.Log.Format = parseLogOutputFormat(strings.ToLower(dr.config.Log.format))
Expand Down Expand Up @@ -156,11 +164,7 @@ func (dr *DependencyResolver) validateConfig(config *Config, recognizedDNSTypes
return fmt.Errorf("error for port of edge dns server(%v): it must be a positive integer between 1 and 65535", s)
}
}
err = field(EdgeDNSZoneKey, config.EdgeDNSZone).isNotEmpty().matchRegexp(hostNameRegex).err
if err != nil {
return err
}
err = field(DNSZoneKey, config.DNSZone).isNotEmpty().matchRegexp(hostNameRegex).err
err = field(DNSZonesKey, config.DNSZones).isNotEmpty().matchRegexp(dnsZonesRegex).err
if err != nil {
return err
}
Expand All @@ -181,15 +185,17 @@ func (dr *DependencyResolver) validateConfig(config *Config, recognizedDNSTypes
return nil
}

serverNames := config.GetExternalClusterNSNames()
serverNames[config.ClusterGeoTag] = config.GetClusterNSName()
for geoTag, nsName := range serverNames {
if len(nsName) > dnsNameMax {
return fmt.Errorf("ns name '%s' exceeds %v charactes limit for [GeoTag: '%s', %s: '%s', %s: '%s']",
nsName, dnsLabelMax, geoTag, EdgeDNSZoneKey, config.EdgeDNSZone, DNSZoneKey, config.DNSZone)
}
if err := validateLabels(nsName); err != nil {
return fmt.Errorf("error for geo tag: %s. %s in ns name %s", geoTag, err, nsName)
for _, zone := range config.DNSZones {
serverNames := config.GetExternalClusterNSNames(zone)
serverNames[config.ClusterGeoTag] = config.GetClusterNSName(zone)
for geoTag, nsName := range serverNames {
if len(nsName) > dnsNameMax {
return fmt.Errorf("ns name '%s' exceeds %v charactes limit for [GeoTag: '%s', %s: '%s']",
nsName, dnsLabelMax, geoTag, DNSZonesKey, zone)
}
if err := validateLabels(nsName); err != nil {
return fmt.Errorf("error for geo tag: %s. %s in ns name %s", geoTag, err, nsName)
}
}
}

Expand Down Expand Up @@ -272,13 +278,22 @@ func (dr *DependencyResolver) GetDeprecations() (deprecations []string) {
Msg: "Port is an optional item in the comma-separated list of dns edge servers, in following form: dns1:53,dns2 (if not provided after the " +
"hostname and colon, it defaults to '53')",
},
EdgeDNSZoneKey: newVar{
Name: EdgeDNSZoneKey,
Msg: "Pass the DNS zone as comma-separated list in following form: edgezone1:zone1,edgezone2:zone2",
},
DNSZoneKey: newVar{
Name: DNSZoneKey,
Msg: "Pass the DNS zone as comma-separated list in following form: edgezone1:zone1,edgezone2:zone2",
},
}

for k, v := range deprecated {
if os.Getenv(k) != "" {
deprecations = append(deprecations, fmt.Sprintf("'%s' has been deprecated, use %s instead. Details: %s", k, v.Name, v.Msg))
}
}
//nolint:nakedret
return
}

Expand Down Expand Up @@ -327,6 +342,28 @@ func parseEdgeDNSServers(serverList []string) (r []utils.DNSServer) {
return r
}

func parseDNSZones(zones []string) (r []utils.DNSZone) {
r = []utils.DNSZone{}
var edgeZone, zone string
for _, chunk := range zones {
chunk = strings.TrimSpace(chunk)
switch strings.Count(chunk, ":") {
case 1:
chunks := strings.Split(chunk, ":")
edgeZone = chunks[0]
zone = chunks[1]
r = append(r, utils.DNSZone{
EdgeZone: edgeZone,
Zone: zone,
})
default:
// not supported
continue
}
}
return r
}

// excludeGeoTag excludes the clusterGeoTag from external geo tags
func excludeGeoTag(tags []string, tag string) (r []string) {
r = []string{}
Expand Down Expand Up @@ -366,28 +403,28 @@ func parseLogOutputFormat(value string) LogFormat {
return NoFormat
}

func (c *Config) GetExternalClusterNSNames() (m map[string]string) {
func (c *Config) GetExternalClusterNSNames(zone utils.DNSZone) (m map[string]string) {
m = make(map[string]string, len(c.ExtClustersGeoTags))
for _, tag := range c.ExtClustersGeoTags {
m[tag] = getNsName(tag, c.DNSZone, c.EdgeDNSZone, c.EdgeDNSServers[0].Host)
m[tag] = getNsName(tag, c.EdgeDNSServers[0].Host, zone)
}
return
}

func (c *Config) GetClusterNSName() string {
return getNsName(c.ClusterGeoTag, c.DNSZone, c.EdgeDNSZone, c.EdgeDNSServers[0].Host)
func (c *Config) GetClusterNSName(zone utils.DNSZone) string {
return getNsName(c.ClusterGeoTag, c.EdgeDNSServers[0].Host, zone)
}

func (c *Config) GetExternalClusterHeartbeatFQDNs(gslbName string) (m map[string]string) {
func (c *Config) GetExternalClusterHeartbeatFQDNs(gslbName string, zone utils.DNSZone) (m map[string]string) {
m = make(map[string]string, len(c.ExtClustersGeoTags))
for _, tag := range c.ExtClustersGeoTags {
m[tag] = getHeartbeatFQDN(gslbName, tag, c.EdgeDNSZone)
m[tag] = getHeartbeatFQDN(gslbName, tag, zone)
}
return
}

func (c *Config) GetClusterHeartbeatFQDN(gslbName string) string {
return getHeartbeatFQDN(gslbName, c.ClusterGeoTag, c.EdgeDNSZone)
func (c *Config) GetClusterHeartbeatFQDN(gslbName string, zone utils.DNSZone) string {
return getHeartbeatFQDN(gslbName, c.ClusterGeoTag, zone)
}

// getNsName returns NS for geo tag.
Expand All @@ -398,14 +435,14 @@ func (c *Config) GetClusterHeartbeatFQDN(gslbName string) string {
// will generate "gslb-ns-us-k8gb-test-gslb.cloud.example.com"
// If edgeDNSServer == localhost or 127.0.0.1 than edgeDNSServer is returned.
// The function is private and expects only valid inputs.
func getNsName(tag, dnsZone, edgeDNSZone, edgeDNSServer string) string {
func getNsName(tag, edgeDNSServer string, zone utils.DNSZone) string {
if edgeDNSServer == "127.0.0.1" || edgeDNSServer == "localhost" {
return edgeDNSServer
}
const prefix = "gslb-ns"
d := strings.TrimSuffix(dnsZone, "."+edgeDNSZone)
d := strings.TrimSuffix(zone.Zone, "."+zone.EdgeZone)
domainX := strings.ReplaceAll(d, ".", "-")
return fmt.Sprintf("%s-%s-%s.%s", prefix, tag, domainX, edgeDNSZone)
return fmt.Sprintf("%s-%s-%s.%s", prefix, tag, domainX, zone.EdgeZone)
}

// getHeartbeatFQDN returns heartbeat for geo tag.
Expand All @@ -415,6 +452,6 @@ func getNsName(tag, dnsZone, edgeDNSZone, edgeDNSServer string) string {
// gslb.Name: test-gslb-1
// will generate "test-gslb-1-heartbeat-us.cloud.example.com"
// The function is private and expects only valid inputs.
func getHeartbeatFQDN(name, geoTag, edgeDNSZone string) string {
return fmt.Sprintf("%s-heartbeat-%s.%s", name, geoTag, edgeDNSZone)
func getHeartbeatFQDN(name, geoTag string, zone utils.DNSZone) string {
return fmt.Sprintf("%s-heartbeat-%s.%s", name, geoTag, zone.EdgeZone)
}
Loading

0 comments on commit a9589fb

Please sign in to comment.