From ef0fc0958de169810487e1469e9684110e814e0c Mon Sep 17 00:00:00 2001 From: Manabu McCloskey Date: Thu, 20 Jun 2024 14:28:19 -0700 Subject: [PATCH] configure core dns for internal name resolution (#317) Signed-off-by: Manabu McCloskey --- pkg/build/build.go | 6 + pkg/build/coredns.go | 76 +++++++++++ .../templates/coredns/cm-coredns-custom.yaml | 8 ++ .../coredns/cm-coredns-default.yaml.tmpl | 13 ++ pkg/build/templates/coredns/cm-coredns.yaml | 30 +++++ .../templates/coredns/deployment-coredns.yaml | 126 ++++++++++++++++++ pkg/cmd/create/root.go | 10 +- 7 files changed, 264 insertions(+), 5 deletions(-) create mode 100644 pkg/build/coredns.go create mode 100644 pkg/build/templates/coredns/cm-coredns-custom.yaml create mode 100644 pkg/build/templates/coredns/cm-coredns-default.yaml.tmpl create mode 100644 pkg/build/templates/coredns/cm-coredns.yaml create mode 100644 pkg/build/templates/coredns/deployment-coredns.yaml diff --git a/pkg/build/build.go b/pkg/build/build.go index 4c1e8b6a..2bda6656 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -172,6 +172,12 @@ func (b *Build) Run(ctx context.Context, recreateCluster bool) error { defer os.RemoveAll(dir) setupLog.V(1).Info("Created temp directory for cloning repositories", "dir", dir) + setupLog.Info("Setting up CoreDNS") + err = setupCoreDNS(ctx, kubeClient, b.scheme, b.cfg) + if err != nil { + return err + } + setupLog.V(1).Info("Running controllers") if err := b.RunControllers(ctx, mgr, managerExit, dir); err != nil { setupLog.Error(err, "Error running controllers") diff --git a/pkg/build/coredns.go b/pkg/build/coredns.go new file mode 100644 index 00000000..c225c35d --- /dev/null +++ b/pkg/build/coredns.go @@ -0,0 +1,76 @@ +package build + +import ( + "context" + "embed" + "fmt" + + "github.com/cnoe-io/idpbuilder/pkg/k8s" + "github.com/cnoe-io/idpbuilder/pkg/util" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const ( + coreDNSTemplatePath = "templates/coredns" +) + +//go:embed templates +var templates embed.FS + +func setupCoreDNS(ctx context.Context, kubeClient client.Client, scheme *runtime.Scheme, templateData util.CorePackageTemplateConfig) error { + checkCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "coredns-conf-default", + Namespace: "kube-system", + }, + } + err := kubeClient.Get(ctx, client.ObjectKeyFromObject(checkCM), checkCM) + if err == nil { + return nil + } + + objs, err := k8s.BuildCustomizedObjects("", coreDNSTemplatePath, templates, scheme, templateData) + if err != nil { + return fmt.Errorf("rendering embedded coredns files: %w", err) + } + + for i := range objs { + obj := objs[i] + switch t := obj.(type) { + case *appsv1.Deployment: + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: t.Name, + Namespace: t.Namespace, + }, + } + _, err = controllerutil.CreateOrUpdate(ctx, kubeClient, dep, func() error { + dep.Spec = t.Spec + return nil + }) + if err != nil { + return fmt.Errorf("creating/updating deployment: %w", err) + } + case *corev1.ConfigMap: + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: t.Name, + Namespace: t.Namespace, + }, + } + _, err = controllerutil.CreateOrUpdate(ctx, kubeClient, cm, func() error { + cm.Data = t.Data + return nil + }) + if err != nil { + return fmt.Errorf("creating/updating configmap: %w", err) + } + } + } + return nil +} diff --git a/pkg/build/templates/coredns/cm-coredns-custom.yaml b/pkg/build/templates/coredns/cm-coredns-custom.yaml new file mode 100644 index 00000000..ddf0c599 --- /dev/null +++ b/pkg/build/templates/coredns/cm-coredns-custom.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-conf-custom + namespace: kube-system +data: + custom.conf: | + # insert custom rules here diff --git a/pkg/build/templates/coredns/cm-coredns-default.yaml.tmpl b/pkg/build/templates/coredns/cm-coredns-default.yaml.tmpl new file mode 100644 index 00000000..3b3f5a88 --- /dev/null +++ b/pkg/build/templates/coredns/cm-coredns-default.yaml.tmpl @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-conf-default + namespace: kube-system +data: + default.conf: | + # subdomain names resolves to ingress IP. e.g. gitea.cnoe.localtest.me becomes ingress-nginx-controller.ingress-nginx.svc.cluster.local + rewrite stop { + name regex (.*).{{ .Host }} ingress-nginx-controller.ingress-nginx.svc.cluster.local + } + # host name resolves to ingress IP + rewrite name exact {{ .Host }} ingress-nginx-controller.ingress-nginx.svc.cluster.local diff --git a/pkg/build/templates/coredns/cm-coredns.yaml b/pkg/build/templates/coredns/cm-coredns.yaml new file mode 100644 index 00000000..b54b955e --- /dev/null +++ b/pkg/build/templates/coredns/cm-coredns.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system +data: + Corefile: | + .:53 { + errors + health { + lameduck 5s + } + ready + + import ../coredns-configs/*.conf + + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 + loop + reload + loadbalance + } diff --git a/pkg/build/templates/coredns/deployment-coredns.yaml b/pkg/build/templates/coredns/deployment-coredns.yaml new file mode 100644 index 00000000..3bcb14d6 --- /dev/null +++ b/pkg/build/templates/coredns/deployment-coredns.yaml @@ -0,0 +1,126 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: kube-dns + name: coredns + namespace: kube-system +spec: + progressDeadlineSeconds: 600 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: kube-dns + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + k8s-app: kube-dns + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: + - kube-dns + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - -conf + - /etc/coredns/Corefile + image: registry.k8s.io/coredns/coredns:v1.11.1 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 8181 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/coredns + name: config-volume + readOnly: true + - mountPath: /etc/coredns-configs + name: custom-configs + readOnly: true + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: coredns + serviceAccountName: coredns + terminationGracePeriodSeconds: 30 + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - configMap: + defaultMode: 420 + items: + - key: Corefile + path: Corefile + name: coredns + name: config-volume + - name: custom-configs + projected: + sources: + - configMap: + name: coredns-conf-custom + - configMap: + name: coredns-conf-default diff --git a/pkg/cmd/create/root.go b/pkg/cmd/create/root.go index 8936b169..191daf81 100644 --- a/pkg/cmd/create/root.go +++ b/pkg/cmd/create/root.go @@ -19,11 +19,11 @@ import ( var ( // Flags - recreateCluster bool - buildName string - kubeVersion string - extraPortsMapping string - kindConfigPath string + recreateCluster bool + buildName string + kubeVersion string + extraPortsMapping string + kindConfigPath string // TODO: Remove extraPackagesDirs after 0.6.0 release extraPackagesDirs []string extraPackages []string