Skip to content

Commit

Permalink
Merge pull request #78 from philbrookes/gh-44
Browse files Browse the repository at this point in the history
aws health checks
  • Loading branch information
maleck13 authored Apr 17, 2024
2 parents 68b0577 + 84e2c1b commit bc470c4
Show file tree
Hide file tree
Showing 19 changed files with 767 additions and 13 deletions.
10 changes: 9 additions & 1 deletion api/v1alpha1/dnsrecord_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,15 @@ type HealthCheckSpec struct {
}

type HealthCheckStatus struct {
Conditions []metav1.Condition `json:"conditions,omitempty"`
Conditions []metav1.Condition `json:"conditions,omitempty"`
Probes []HealthCheckStatusProbe `json:"probes,omitempty"`
}

type HealthCheckStatusProbe struct {
ID string `json:"id"`
IPAddress string `json:"ipAddress"`
Host string `json:"host"`
Synced bool `json:"synced,omitempty"`
}

// DNSRecordSpec defines the desired state of DNSRecord
Expand Down
20 changes: 20 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion bundle/manifests/dns-operator.clusterserviceversion.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ metadata:
capabilities: Basic Install
categories: Integration & Delivery
containerImage: quay.io/kuadrant/dns-operator:latest
createdAt: "2024-04-04T08:27:01Z"
createdAt: "2024-04-17T03:35:09Z"
description: A Kubernetes Operator to manage the lifecycle of DNS resources
operators.operatorframework.io/builder: operator-sdk-v1.33.0
operators.operatorframework.io/project_layout: go.kubebuilder.io/v4
Expand Down
17 changes: 17 additions & 0 deletions bundle/manifests/kuadrant.io_dnsrecords.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,23 @@ spec:
- type
type: object
type: array
probes:
items:
properties:
host:
type: string
id:
type: string
ipAddress:
type: string
synced:
type: boolean
required:
- host
- id
- ipAddress
type: object
type: array
type: object
observedGeneration:
description: observedGeneration is the most recently observed generation
Expand Down
17 changes: 17 additions & 0 deletions config/crd/bases/kuadrant.io_dnsrecords.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,23 @@ spec:
- type
type: object
type: array
probes:
items:
properties:
host:
type: string
id:
type: string
ipAddress:
type: string
synced:
type: boolean
required:
- host
- id
- ipAddress
type: object
type: array
type: object
observedGeneration:
description: observedGeneration is the most recently observed generation
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ require (
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.opencensus.io v0.24.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -680,6 +680,8 @@ github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rubenv/sql-migrate v0.0.0-20200212082348-64f95ea68aa3/go.mod h1:rtQlpHw+eR6UrqaS3kX1VYeaCxzCVdimDS7g5Ln4pPc=
github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
Expand Down
38 changes: 31 additions & 7 deletions internal/controller/dnsrecord_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@ func (r *DNSRecordReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
dnsRecord := previous.DeepCopy()

if dnsRecord.DeletionTimestamp != nil && !dnsRecord.DeletionTimestamp.IsZero() {
if err := r.ReconcileHealthChecks(ctx, dnsRecord); err != nil {
return ctrl.Result{}, err
}
requeueTime, err := r.deleteRecord(ctx, dnsRecord)
if err != nil {
logger.Error(err, "Failed to delete DNSRecord")
Expand All @@ -100,6 +103,7 @@ func (r *DNSRecordReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
if requeueTime == validationRequeueTime {
return ctrl.Result{RequeueAfter: requeueTime}, nil
}

logger.Info("Removing Finalizer", "name", DNSRecordFinalizer)
controllerutil.RemoveFinalizer(dnsRecord, DNSRecordFinalizer)
if err = r.Update(ctx, dnsRecord); client.IgnoreNotFound(err) != nil {
Expand Down Expand Up @@ -142,6 +146,11 @@ func (r *DNSRecordReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
// success
dnsRecord.Status.ObservedGeneration = dnsRecord.Generation
dnsRecord.Status.Endpoints = dnsRecord.Spec.Endpoints

if err := r.ReconcileHealthChecks(ctx, dnsRecord); err != nil {
return ctrl.Result{}, err
}

return r.updateStatus(ctx, previous, dnsRecord, requeueAfter)
}

Expand Down Expand Up @@ -266,6 +275,27 @@ func setDNSRecordCondition(dnsRecord *v1alpha1.DNSRecord, conditionType string,
meta.SetStatusCondition(&dnsRecord.Status.Conditions, cond)
}

func (r *DNSRecordReconciler) getDNSProvider(ctx context.Context, dnsRecord *v1alpha1.DNSRecord) (provider.Provider, error) {
managedZone := &v1alpha1.ManagedZone{
ObjectMeta: metav1.ObjectMeta{
Name: dnsRecord.Spec.ManagedZoneRef.Name,
Namespace: dnsRecord.Namespace,
},
}
err := r.Get(ctx, client.ObjectKeyFromObject(managedZone), managedZone, &client.GetOptions{})
if err != nil {
return nil, err
}

providerConfig := provider.Config{
DomainFilter: externaldnsendpoint.NewDomainFilter([]string{managedZone.Spec.DomainName}),
ZoneTypeFilter: externaldnsprovider.NewZoneTypeFilter(""),
ZoneIDFilter: externaldnsprovider.NewZoneIDFilter([]string{managedZone.Status.ID}),
}

return r.ProviderFactory.ProviderFor(ctx, managedZone, providerConfig)
}

func (r *DNSRecordReconciler) applyChanges(ctx context.Context, dnsRecord *v1alpha1.DNSRecord, managedZone *v1alpha1.ManagedZone, isDelete bool) (time.Duration, error) {
logger := log.FromContext(ctx)
filterDomain, _ := strings.CutPrefix(managedZone.Spec.DomainName, v1alpha1.WildcardPrefix)
Expand All @@ -274,13 +304,7 @@ func (r *DNSRecordReconciler) applyChanges(ctx context.Context, dnsRecord *v1alp
}
rootDomainFilter := externaldnsendpoint.NewDomainFilter([]string{filterDomain})

providerConfig := provider.Config{
DomainFilter: externaldnsendpoint.NewDomainFilter([]string{managedZone.Spec.DomainName}),
ZoneTypeFilter: externaldnsprovider.NewZoneTypeFilter(""),
ZoneIDFilter: externaldnsprovider.NewZoneIDFilter([]string{managedZone.Status.ID}),
}
logger.V(3).Info("applyChanges", "zone", managedZone.Spec.DomainName, "rootDomainFilter", rootDomainFilter, "providerConfig", providerConfig)
dnsProvider, err := r.ProviderFactory.ProviderFor(ctx, managedZone, providerConfig)
dnsProvider, err := r.getDNSProvider(ctx, dnsRecord)
if err != nil {
return noRequeueDuration, err
}
Expand Down
183 changes: 183 additions & 0 deletions internal/controller/dnsrecord_healthchecks.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
package controller

import (
"context"
"crypto/md5"
"fmt"
"io"
"reflect"

"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
externaldns "sigs.k8s.io/external-dns/endpoint"

"github.com/kuadrant/dns-operator/api/v1alpha1"
"github.com/kuadrant/dns-operator/internal/provider"
)

// healthChecksConfig represents the user configuration for the health checks
type healthChecksConfig struct {
Endpoint string
Port *int64
FailureThreshold *int64
Protocol *provider.HealthCheckProtocol
}

func (r *DNSRecordReconciler) ReconcileHealthChecks(ctx context.Context, dnsRecord *v1alpha1.DNSRecord) error {
var results []provider.HealthCheckResult
var err error

dnsProvider, err := r.getDNSProvider(ctx, dnsRecord)
if err != nil {
return err
}

healthCheckReconciler := dnsProvider.HealthCheckReconciler()

// Get the configuration for the health checks. If no configuration is
// set, ensure that the health checks are deleted
config := getHealthChecksConfig(dnsRecord)

for _, dnsEndpoint := range dnsRecord.Spec.Endpoints {
addresses := provider.GetExternalAddresses(dnsEndpoint, dnsRecord)
for _, address := range addresses {
probeStatus := r.getProbeStatus(address, dnsRecord)

// no config means delete the health checks
if config == nil {
result, err := healthCheckReconciler.Delete(ctx, dnsEndpoint, probeStatus)
if err != nil {
return err
}

results = append(results, result)
continue
}

// creating / updating health checks
endpointId, err := idForEndpoint(dnsRecord, dnsEndpoint, address)
if err != nil {
return err
}

spec := provider.HealthCheckSpec{
Id: endpointId,
Name: fmt.Sprintf("%s-%s-%s", *dnsRecord.Spec.RootHost, dnsEndpoint.DNSName, address),
Host: dnsRecord.Spec.RootHost,
Path: config.Endpoint,
Port: config.Port,
Protocol: config.Protocol,
FailureThreshold: config.FailureThreshold,
}

result, err := healthCheckReconciler.Reconcile(ctx, spec, dnsEndpoint, probeStatus, address)
if err != nil {
return err
}
results = append(results, result)
}
}

result := r.reconcileHealthCheckStatus(results, dnsRecord)
return result
}

func (r *DNSRecordReconciler) getProbeStatus(address string, dnsRecord *v1alpha1.DNSRecord) *v1alpha1.HealthCheckStatusProbe {
if dnsRecord.Status.HealthCheck == nil || dnsRecord.Status.HealthCheck.Probes == nil {
return nil
}
for _, probeStatus := range dnsRecord.Status.HealthCheck.Probes {
if probeStatus.IPAddress == address {
return &probeStatus
}
}

return nil
}

func (r *DNSRecordReconciler) reconcileHealthCheckStatus(results []provider.HealthCheckResult, dnsRecord *v1alpha1.DNSRecord) error {
var previousCondition *metav1.Condition
probesCondition := &metav1.Condition{
Reason: "AllProbesSynced",
Type: "healthProbesSynced",
}

var allSynced = metav1.ConditionTrue

if dnsRecord.Status.HealthCheck == nil {
dnsRecord.Status.HealthCheck = &v1alpha1.HealthCheckStatus{
Conditions: []metav1.Condition{},
Probes: []v1alpha1.HealthCheckStatusProbe{},
}
}

previousCondition = meta.FindStatusCondition(dnsRecord.Status.HealthCheck.Conditions, "HealthProbesSynced")
if previousCondition != nil {
probesCondition = previousCondition
}

dnsRecord.Status.HealthCheck.Probes = []v1alpha1.HealthCheckStatusProbe{}

for _, result := range results {
if result.ID == "" {
continue
}
status := true
if result.Result == provider.HealthCheckFailed {
status = false
allSynced = metav1.ConditionFalse
}

dnsRecord.Status.HealthCheck.Probes = append(dnsRecord.Status.HealthCheck.Probes, v1alpha1.HealthCheckStatusProbe{
ID: result.ID,
IPAddress: result.IPAddress,
Host: result.Host,
Synced: status,
})
}

probesCondition.ObservedGeneration = dnsRecord.Generation
probesCondition.Status = allSynced

if allSynced == metav1.ConditionTrue {
probesCondition.Message = fmt.Sprintf("all %v probes synced successfully", len(dnsRecord.Status.HealthCheck.Probes))
probesCondition.Reason = "AllProbesSynced"
} else {
probesCondition.Reason = "UnsyncedProbes"
probesCondition.Message = "some probes have not yet successfully synced to the DNS Provider"
}

//probe condition changed? - update transition time
if !reflect.DeepEqual(previousCondition, probesCondition) {
probesCondition.LastTransitionTime = metav1.Now()
}

dnsRecord.Status.HealthCheck.Conditions = []metav1.Condition{*probesCondition}

return nil
}

func getHealthChecksConfig(dnsRecord *v1alpha1.DNSRecord) *healthChecksConfig {
if dnsRecord.Spec.HealthCheck == nil || dnsRecord.DeletionTimestamp != nil {
return nil
}

port := int64(*dnsRecord.Spec.HealthCheck.Port)
failureThreshold := int64(*dnsRecord.Spec.HealthCheck.FailureThreshold)

return &healthChecksConfig{
Endpoint: dnsRecord.Spec.HealthCheck.Endpoint,
Port: &port,
FailureThreshold: &failureThreshold,
Protocol: (*provider.HealthCheckProtocol)(dnsRecord.Spec.HealthCheck.Protocol),
}
}

// idForEndpoint returns a unique identifier for an endpoint
func idForEndpoint(dnsRecord *v1alpha1.DNSRecord, endpoint *externaldns.Endpoint, address string) (string, error) {
hash := md5.New()
if _, err := io.WriteString(hash, fmt.Sprintf("%s/%s@%s:%s", dnsRecord.Name, endpoint.SetIdentifier, endpoint.DNSName, address)); err != nil {
return "", fmt.Errorf("unexpected error creating ID for endpoint %s", endpoint.SetIdentifier)
}
return fmt.Sprintf("%x", hash.Sum(nil)), nil
}
Loading

0 comments on commit bc470c4

Please sign in to comment.