From 917612340d197c49c4c36af4ff76d55657f1c7cb Mon Sep 17 00:00:00 2001 From: nicklesimba Date: Mon, 30 May 2022 16:07:21 -0500 Subject: [PATCH] WIP added changes for alerting and calling reconciler through controlloop Signed-off-by: nicklesimba --- cmd/controlloop/controlloop.go | 57 +++++++++++++++++++++++++++++++++- pkg/controlloop/pod.go | 14 ++++----- pkg/reconciler/ip.go | 46 +++++++++++++-------------- 3 files changed, 85 insertions(+), 32 deletions(-) diff --git a/cmd/controlloop/controlloop.go b/cmd/controlloop/controlloop.go index d80b9066d..4cfc182b5 100644 --- a/cmd/controlloop/controlloop.go +++ b/cmd/controlloop/controlloop.go @@ -3,8 +3,11 @@ package main import ( "flag" "fmt" + "log" + "net/http" "os" "os/signal" + "time" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,8 +24,11 @@ import ( wbclient "github.com/k8snetworkplumbingwg/whereabouts/pkg/client/clientset/versioned" wbinformers "github.com/k8snetworkplumbingwg/whereabouts/pkg/client/informers/externalversions" + "github.com/k8snetworkplumbingwg/whereabouts/pkg/controlloop" "github.com/k8snetworkplumbingwg/whereabouts/pkg/logging" - "github.com/k8snetworkplumbingwg/whereabouts/pkg/reconciler/controlloop" + "github.com/k8snetworkplumbingwg/whereabouts/pkg/reconciler" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" ) const ( @@ -46,7 +52,9 @@ func main() { logging.SetLogStderr(true) stopChan := make(chan struct{}) + returnErr := make(chan error) defer close(stopChan) + defer close(returnErr) handleSignals(stopChan, os.Interrupt) networkController, err := newPodController(stopChan) @@ -56,6 +64,53 @@ func main() { } networkController.Start(stopChan) + + totalReconcilerSuccess := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "reconciler_success_total", + Help: "Increments upon successful run of IP reconciler", + }) + + prometheus.MustRegister(totalReconcilerSuccess) + + http.Handle("/metrics", promhttp.Handler()) + + // TODO: i want to generalize this - use random ip address instead of a specific one + go func() { + log.Fatal(http.ListenAndServe(":1984", nil)) // I might need to be using port 8443? Not sure + }() + + // here's where my for { select {} } loop should go - and use tickers + // https://gobyexample.com/tickers + // general logic - loop indefinitely, with the following conditions: + // a. stopChan sends a value: quit out of the loop and return function + // b. ticker ticks: start a goroutine to run ip-reconciler + // c. default: continue to spin + ticker := time.NewTicker(10 * time.Second) // temp value, will eventually be days/weeks duration + + i := 0 + for /*i := 0; i < 5; i++*/ { + i++ + logging.Verbosef("iteration #%d", i) + select { + case <-stopChan: + return + case t := <-ticker.C: + fmt.Println("Running ip-reconciler, tick at ", t) + go reconciler.InvokeIPReconciler(returnErr) + case err := <-returnErr: // why is this case only reached one time? + logging.Verbosef("reached counter decision") + if err == nil { + totalReconcilerSuccess.Inc() + logging.Verbosef("ip reconciler success!") + } else { + logging.Verbosef("ip reconciler failure: %s", err) + } + } + } + + logging.Verbosef("loop ended...you have one minute to curl for counter results.") + time.Sleep(1 * time.Minute) + return } func handleSignals(stopChannel chan struct{}, signals ...os.Signal) { diff --git a/pkg/controlloop/pod.go b/pkg/controlloop/pod.go index 4fcabbaf3..f86f7acef 100644 --- a/pkg/controlloop/pod.go +++ b/pkg/controlloop/pod.go @@ -33,11 +33,12 @@ import ( ) const ( - defaultMountPath = "/host" - ipReconcilerQueueName = "pod-updates" - syncPeriod = time.Second - whereaboutsConfigPath = "/etc/cni/net.d/whereabouts.d/whereabouts.conf" - maxRetries = 2 + defaultMountPath = "/host" + ipReconcilerQueueName = "pod-updates" + syncPeriod = time.Second + whereaboutsConfigPath = "/etc/cni/net.d/whereabouts.d/whereabouts.conf" + maxRetries = 2 + defaultReconcilerTimeout = 30 // added for reconciler invocation ) const ( @@ -116,9 +117,6 @@ func (pc *PodController) Start(stopChan <-chan struct{}) { } go wait.Until(pc.worker, syncPeriod, stopChan) - - <-stopChan - logging.Verbosef("shutting down network controller") } func (pc *PodController) worker() { diff --git a/pkg/reconciler/ip.go b/pkg/reconciler/ip.go index 96cfc0426..b54e1da5c 100644 --- a/pkg/reconciler/ip.go +++ b/pkg/reconciler/ip.go @@ -1,48 +1,48 @@ +// package main package reconciler import ( "context" - "flag" - "os" + "time" "github.com/k8snetworkplumbingwg/whereabouts/pkg/logging" - "github.com/k8snetworkplumbingwg/whereabouts/pkg/reconciler" ) -const defaultReconcilerTimeout = 30 - -func main() { - kubeConfigFile := flag.String("kubeconfig", "", "the path to the Kubernetes configuration file") - logLevel := flag.String("log-level", "error", "the logging level for the `ip-reconciler` app. Valid values are: \"debug\", \"verbose\", \"error\", and \"panic\".") - reconcilerTimeout := flag.Int("timeout", defaultReconcilerTimeout, "the value for a request timeout in seconds.") - flag.Parse() +const ( + defaultReconcilerTimeout = 30 +) - logging.SetLogLevel(*logLevel) +// TODO: get ip_test.go working with this - currently idk if it does... +func InvokeIPReconciler(returnErr chan error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(defaultReconcilerTimeout*time.Second)) + defer cancel() - var err error - var ipReconcileLoop *reconciler.ReconcileLooper - if kubeConfigFile == nil { - ipReconcileLoop, err = reconciler.NewReconcileLooper(context.Background(), *reconcilerTimeout) - } else { - ipReconcileLoop, err = reconciler.NewReconcileLooperWithKubeconfig(context.Background(), *kubeConfigFile, *reconcilerTimeout) - } + ipReconcileLoop, err := NewReconcileLooper(ctx, defaultReconcilerTimeout) if err != nil { _ = logging.Errorf("failed to create the reconcile looper: %v", err) - os.Exit(couldNotStartOrphanedIPMonitor) + returnErr <- err + return } - cleanedUpIps, err := ipReconcileLoop.ReconcileIPPools(context.Background()) + cleanedUpIps, err := ipReconcileLoop.ReconcileIPPools(ctx) if err != nil { _ = logging.Errorf("failed to clean up IP for allocations: %v", err) - os.Exit(failedToReconcileIPPools) + returnErr <- err + return } + if len(cleanedUpIps) > 0 { logging.Debugf("successfully cleanup IPs: %+v", cleanedUpIps) } else { logging.Debugf("no IP addresses to cleanup") } - if err := ipReconcileLoop.ReconcileOverlappingIPAddresses(context.Background()); err != nil { - os.Exit(failedToReconcileClusterWideIPs) + if err := ipReconcileLoop.ReconcileOverlappingIPAddresses(ctx); err != nil { + returnErr <- err + return } + + logging.Verbosef("no errors with ip reconciler...returning in a sec") + returnErr <- err + return }