aws health checks

api/v1alpha1/dnsrecord_types.go

@@ -45,7 +45,15 @@ type HealthCheckSpec struct {
 }
 
 type HealthCheckStatus struct {
-	Conditions []metav1.Condition `json:"conditions,omitempty"`
+	Conditions []metav1.Condition       `json:"conditions,omitempty"`
+	Probes     []HealthCheckStatusProbe `json:"probes,omitempty"`
+}
+
+type HealthCheckStatusProbe struct {
+	ID        string `json:"id"`
+	IPAddress string `json:"ipAddress"`
+	Host      string `json:"host"`
+	Synced    bool   `json:"synced,omitempty"`
 }
 
 // DNSRecordSpec defines the desired state of DNSRecord

bundle/manifests/dns-operator.clusterserviceversion.yaml

@@ -56,7 +56,7 @@ metadata:
     capabilities: Basic Install
     categories: Integration & Delivery
     containerImage: quay.io/kuadrant/dns-operator:latest
-    createdAt: "2024-04-04T08:27:01Z"
+    createdAt: "2024-04-17T03:35:09Z"
     description: A Kubernetes Operator to manage the lifecycle of DNS resources
     operators.operatorframework.io/builder: operator-sdk-v1.33.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v4

bundle/manifests/kuadrant.io_dnsrecords.yaml

@@ -320,6 +320,23 @@ spec:
                       - type
                       type: object
                     type: array
+                  probes:
+                    items:
+                      properties:
+                        host:
+                          type: string
+                        id:
+                          type: string
+                        ipAddress:
+                          type: string
+                        synced:
+                          type: boolean
+                      required:
+                      - host
+                      - id
+                      - ipAddress
+                      type: object
+                    type: array
                 type: object
               observedGeneration:
                 description: observedGeneration is the most recently observed generation

config/crd/bases/kuadrant.io_dnsrecords.yaml

@@ -320,6 +320,23 @@ spec:
                       - type
                       type: object
                     type: array
+                  probes:
+                    items:
+                      properties:
+                        host:
+                          type: string
+                        id:
+                          type: string
+                        ipAddress:
+                          type: string
+                        synced:
+                          type: boolean
+                      required:
+                      - host
+                      - id
+                      - ipAddress
+                      type: object
+                    type: array
                 type: object
               observedGeneration:
                 description: observedGeneration is the most recently observed generation

go.mod

@@ -71,6 +71,7 @@ require (
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/rs/xid v1.5.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect

go.sum

@@ -680,6 +680,8 @@ github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
 github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rubenv/sql-migrate v0.0.0-20200212082348-64f95ea68aa3/go.mod h1:rtQlpHw+eR6UrqaS3kX1VYeaCxzCVdimDS7g5Ln4pPc=
 github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=

internal/controller/dnsrecord_controller.go

@@ -89,6 +89,9 @@ func (r *DNSRecordReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	dnsRecord := previous.DeepCopy()
 
 	if dnsRecord.DeletionTimestamp != nil && !dnsRecord.DeletionTimestamp.IsZero() {
+		if err := r.ReconcileHealthChecks(ctx, dnsRecord); err != nil {
+			return ctrl.Result{}, err
+		}
 		requeueTime, err := r.deleteRecord(ctx, dnsRecord)
 		if err != nil {
 			logger.Error(err, "Failed to delete DNSRecord")
@@ -100,6 +103,7 @@ func (r *DNSRecordReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		if requeueTime == validationRequeueTime {
 			return ctrl.Result{RequeueAfter: requeueTime}, nil
 		}
+
 		logger.Info("Removing Finalizer", "name", DNSRecordFinalizer)
 		controllerutil.RemoveFinalizer(dnsRecord, DNSRecordFinalizer)
 		if err = r.Update(ctx, dnsRecord); client.IgnoreNotFound(err) != nil {
@@ -142,6 +146,11 @@ func (r *DNSRecordReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	// success
 	dnsRecord.Status.ObservedGeneration = dnsRecord.Generation
 	dnsRecord.Status.Endpoints = dnsRecord.Spec.Endpoints
+
+	if err := r.ReconcileHealthChecks(ctx, dnsRecord); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	return r.updateStatus(ctx, previous, dnsRecord, requeueAfter)
 }
 
@@ -266,6 +275,27 @@ func setDNSRecordCondition(dnsRecord *v1alpha1.DNSRecord, conditionType string,
 	meta.SetStatusCondition(&dnsRecord.Status.Conditions, cond)
 }
 
+func (r *DNSRecordReconciler) getDNSProvider(ctx context.Context, dnsRecord *v1alpha1.DNSRecord) (provider.Provider, error) {
+	managedZone := &v1alpha1.ManagedZone{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      dnsRecord.Spec.ManagedZoneRef.Name,
+			Namespace: dnsRecord.Namespace,
+		},
+	}
+	err := r.Get(ctx, client.ObjectKeyFromObject(managedZone), managedZone, &client.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	providerConfig := provider.Config{
+		DomainFilter:   externaldnsendpoint.NewDomainFilter([]string{managedZone.Spec.DomainName}),
+		ZoneTypeFilter: externaldnsprovider.NewZoneTypeFilter(""),
+		ZoneIDFilter:   externaldnsprovider.NewZoneIDFilter([]string{managedZone.Status.ID}),
+	}
+
+	return r.ProviderFactory.ProviderFor(ctx, managedZone, providerConfig)
+}
+
 func (r *DNSRecordReconciler) applyChanges(ctx context.Context, dnsRecord *v1alpha1.DNSRecord, managedZone *v1alpha1.ManagedZone, isDelete bool) (time.Duration, error) {
 	logger := log.FromContext(ctx)
 	filterDomain, _ := strings.CutPrefix(managedZone.Spec.DomainName, v1alpha1.WildcardPrefix)
@@ -274,13 +304,7 @@ func (r *DNSRecordReconciler) applyChanges(ctx context.Context, dnsRecord *v1alp
 	}
 	rootDomainFilter := externaldnsendpoint.NewDomainFilter([]string{filterDomain})
 
-	providerConfig := provider.Config{
-		DomainFilter:   externaldnsendpoint.NewDomainFilter([]string{managedZone.Spec.DomainName}),
-		ZoneTypeFilter: externaldnsprovider.NewZoneTypeFilter(""),
-		ZoneIDFilter:   externaldnsprovider.NewZoneIDFilter([]string{managedZone.Status.ID}),
-	}
-	logger.V(3).Info("applyChanges", "zone", managedZone.Spec.DomainName, "rootDomainFilter", rootDomainFilter, "providerConfig", providerConfig)
-	dnsProvider, err := r.ProviderFactory.ProviderFor(ctx, managedZone, providerConfig)
+	dnsProvider, err := r.getDNSProvider(ctx, dnsRecord)
 	if err != nil {
 		return noRequeueDuration, err
 	}

internal/controller/dnsrecord_healthchecks.go

@@ -0,0 +1,183 @@
+package controller
+
+import (
+	"context"
+	"crypto/md5"
+	"fmt"
+	"io"
+	"reflect"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	externaldns "sigs.k8s.io/external-dns/endpoint"
+
+	"github.com/kuadrant/dns-operator/api/v1alpha1"
+	"github.com/kuadrant/dns-operator/internal/provider"
+)
+
+// healthChecksConfig represents the user configuration for the health checks
+type healthChecksConfig struct {
+	Endpoint         string
+	Port             *int64
+	FailureThreshold *int64
+	Protocol         *provider.HealthCheckProtocol
+}
+
+func (r *DNSRecordReconciler) ReconcileHealthChecks(ctx context.Context, dnsRecord *v1alpha1.DNSRecord) error {
+	var results []provider.HealthCheckResult
+	var err error
+
+	dnsProvider, err := r.getDNSProvider(ctx, dnsRecord)
+	if err != nil {
+		return err
+	}
+
+	healthCheckReconciler := dnsProvider.HealthCheckReconciler()
+
+	// Get the configuration for the health checks. If no configuration is
+	// set, ensure that the health checks are deleted
+	config := getHealthChecksConfig(dnsRecord)
+
+	for _, dnsEndpoint := range dnsRecord.Spec.Endpoints {
+		addresses := provider.GetExternalAddresses(dnsEndpoint, dnsRecord)
+		for _, address := range addresses {
+			probeStatus := r.getProbeStatus(address, dnsRecord)
+
+			// no config means delete the health checks
+			if config == nil {
+				result, err := healthCheckReconciler.Delete(ctx, dnsEndpoint, probeStatus)
+				if err != nil {
+					return err
+				}
+
+				results = append(results, result)
+				continue
+			}
+
+			// creating / updating health checks
+			endpointId, err := idForEndpoint(dnsRecord, dnsEndpoint, address)
+			if err != nil {
+				return err
+			}
+
+			spec := provider.HealthCheckSpec{
+				Id:               endpointId,
+				Name:             fmt.Sprintf("%s-%s-%s", *dnsRecord.Spec.RootHost, dnsEndpoint.DNSName, address),
+				Host:             dnsRecord.Spec.RootHost,
+				Path:             config.Endpoint,
+				Port:             config.Port,
+				Protocol:         config.Protocol,
+				FailureThreshold: config.FailureThreshold,
+			}
+
+			result, err := healthCheckReconciler.Reconcile(ctx, spec, dnsEndpoint, probeStatus, address)
+			if err != nil {
+				return err
+			}
+			results = append(results, result)
+		}
+	}
+
+	result := r.reconcileHealthCheckStatus(results, dnsRecord)
+	return result
+}
+
+func (r *DNSRecordReconciler) getProbeStatus(address string, dnsRecord *v1alpha1.DNSRecord) *v1alpha1.HealthCheckStatusProbe {
+	if dnsRecord.Status.HealthCheck == nil || dnsRecord.Status.HealthCheck.Probes == nil {
+		return nil
+	}
+	for _, probeStatus := range dnsRecord.Status.HealthCheck.Probes {
+		if probeStatus.IPAddress == address {
+			return &probeStatus
+		}
+	}
+
+	return nil
+}
+
+func (r *DNSRecordReconciler) reconcileHealthCheckStatus(results []provider.HealthCheckResult, dnsRecord *v1alpha1.DNSRecord) error {
+	var previousCondition *metav1.Condition
+	probesCondition := &metav1.Condition{
+		Reason: "AllProbesSynced",
+		Type:   "healthProbesSynced",
+	}
+
+	var allSynced = metav1.ConditionTrue
+
+	if dnsRecord.Status.HealthCheck == nil {
+		dnsRecord.Status.HealthCheck = &v1alpha1.HealthCheckStatus{
+			Conditions: []metav1.Condition{},
+			Probes:     []v1alpha1.HealthCheckStatusProbe{},
+		}
+	}
+
+	previousCondition = meta.FindStatusCondition(dnsRecord.Status.HealthCheck.Conditions, "HealthProbesSynced")
+	if previousCondition != nil {
+		probesCondition = previousCondition
+	}
+
+	dnsRecord.Status.HealthCheck.Probes = []v1alpha1.HealthCheckStatusProbe{}
+
+	for _, result := range results {
+		if result.ID == "" {
+			continue
+		}
+		status := true
+		if result.Result == provider.HealthCheckFailed {
+			status = false
+			allSynced = metav1.ConditionFalse
+		}
+
+		dnsRecord.Status.HealthCheck.Probes = append(dnsRecord.Status.HealthCheck.Probes, v1alpha1.HealthCheckStatusProbe{
+			ID:        result.ID,
+			IPAddress: result.IPAddress,
+			Host:      result.Host,
+			Synced:    status,
+		})
+	}
+
+	probesCondition.ObservedGeneration = dnsRecord.Generation
+	probesCondition.Status = allSynced
+
+	if allSynced == metav1.ConditionTrue {
+		probesCondition.Message = fmt.Sprintf("all %v probes synced successfully", len(dnsRecord.Status.HealthCheck.Probes))
+		probesCondition.Reason = "AllProbesSynced"
+	} else {
+		probesCondition.Reason = "UnsyncedProbes"
+		probesCondition.Message = "some probes have not yet successfully synced to the DNS Provider"
+	}
+
+	//probe condition changed? - update transition time
+	if !reflect.DeepEqual(previousCondition, probesCondition) {
+		probesCondition.LastTransitionTime = metav1.Now()
+	}
+
+	dnsRecord.Status.HealthCheck.Conditions = []metav1.Condition{*probesCondition}
+
+	return nil
+}
+
+func getHealthChecksConfig(dnsRecord *v1alpha1.DNSRecord) *healthChecksConfig {
+	if dnsRecord.Spec.HealthCheck == nil || dnsRecord.DeletionTimestamp != nil {
+		return nil
+	}
+
+	port := int64(*dnsRecord.Spec.HealthCheck.Port)
+	failureThreshold := int64(*dnsRecord.Spec.HealthCheck.FailureThreshold)
+
+	return &healthChecksConfig{
+		Endpoint:         dnsRecord.Spec.HealthCheck.Endpoint,
+		Port:             &port,
+		FailureThreshold: &failureThreshold,
+		Protocol:         (*provider.HealthCheckProtocol)(dnsRecord.Spec.HealthCheck.Protocol),
+	}
+}
+
+// idForEndpoint returns a unique identifier for an endpoint
+func idForEndpoint(dnsRecord *v1alpha1.DNSRecord, endpoint *externaldns.Endpoint, address string) (string, error) {
+	hash := md5.New()
+	if _, err := io.WriteString(hash, fmt.Sprintf("%s/%s@%s:%s", dnsRecord.Name, endpoint.SetIdentifier, endpoint.DNSName, address)); err != nil {
+		return "", fmt.Errorf("unexpected error creating ID for endpoint %s", endpoint.SetIdentifier)
+	}
+	return fmt.Sprintf("%x", hash.Sum(nil)), nil
+}