WIP added changes for alerting and calling reconciler through control…

…loop

Signed-off-by: nicklesimba <[email protected]>

cmd/controlloop/controlloop.go

@@ -3,8 +3,11 @@ package main
 import (
 	"flag"
 	"fmt"
+	"log"
+	"net/http"
 	"os"
 	"os/signal"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -21,8 +24,11 @@ import (
 
 	wbclient "github.com/k8snetworkplumbingwg/whereabouts/pkg/client/clientset/versioned"
 	wbinformers "github.com/k8snetworkplumbingwg/whereabouts/pkg/client/informers/externalversions"
+	"github.com/k8snetworkplumbingwg/whereabouts/pkg/controlloop"
 	"github.com/k8snetworkplumbingwg/whereabouts/pkg/logging"
-	"github.com/k8snetworkplumbingwg/whereabouts/pkg/reconciler/controlloop"
+	"github.com/k8snetworkplumbingwg/whereabouts/pkg/reconciler"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 const (
@@ -46,7 +52,9 @@ func main() {
 	logging.SetLogStderr(true)
 
 	stopChan := make(chan struct{})
+	returnErr := make(chan error)
 	defer close(stopChan)
+	defer close(returnErr)
 	handleSignals(stopChan, os.Interrupt)
 
 	networkController, err := newPodController(stopChan)
@@ -56,6 +64,53 @@ func main() {
 	}
 
 	networkController.Start(stopChan)
+
+	totalReconcilerSuccess := prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "reconciler_success_total",
+		Help: "Increments upon successful run of IP reconciler",
+	})
+
+	prometheus.MustRegister(totalReconcilerSuccess)
+
+	http.Handle("/metrics", promhttp.Handler())
+
+	// TODO: i want to generalize this - use random ip address instead of a specific one
+	go func() {
+		log.Fatal(http.ListenAndServe(":1984", nil)) // I might need to be using port 8443? Not sure
+	}()
+
+	// here's where my for { select {} } loop should go - and use tickers
+	// https://gobyexample.com/tickers
+	// general logic - loop indefinitely, with the following conditions:
+	// a. stopChan sends a value: quit out of the loop and return function
+	// b. ticker ticks: start a goroutine to run ip-reconciler
+	// c. default: continue to spin
+	ticker := time.NewTicker(10 * time.Second) // temp value, will eventually be days/weeks duration
+
+	i := 0
+	for /*i := 0; i < 5; i++*/ {
+		i++
+		logging.Verbosef("iteration #%d", i)
+		select {
+		case <-stopChan:
+			return
+		case t := <-ticker.C:
+			fmt.Println("Running ip-reconciler, tick at ", t)
+			go reconciler.InvokeIPReconciler(returnErr)
+		case err := <-returnErr: // why is this case only reached one time?
+			logging.Verbosef("reached counter decision")
+			if err == nil {
+				totalReconcilerSuccess.Inc()
+				logging.Verbosef("ip reconciler success!")
+			} else {
+				logging.Verbosef("ip reconciler failure: %s", err)
+			}
+		}
+	}
+
+	logging.Verbosef("loop ended...you have one minute to curl for counter results.")
+	time.Sleep(1 * time.Minute)
+	return
 }
 
 func handleSignals(stopChannel chan struct{}, signals ...os.Signal) {

pkg/controlloop/pod.go

@@ -33,11 +33,12 @@ import (
 )
 
 const (
-	defaultMountPath      = "/host"
-	ipReconcilerQueueName = "pod-updates"
-	syncPeriod            = time.Second
-	whereaboutsConfigPath = "/etc/cni/net.d/whereabouts.d/whereabouts.conf"
-	maxRetries            = 2
+	defaultMountPath         = "/host"
+	ipReconcilerQueueName    = "pod-updates"
+	syncPeriod               = time.Second
+	whereaboutsConfigPath    = "/etc/cni/net.d/whereabouts.d/whereabouts.conf"
+	maxRetries               = 2
+	defaultReconcilerTimeout = 30 // added for reconciler invocation
 )
 
 const (
@@ -116,9 +117,6 @@ func (pc *PodController) Start(stopChan <-chan struct{}) {
 	}
 
 	go wait.Until(pc.worker, syncPeriod, stopChan)
-
-	<-stopChan
-	logging.Verbosef("shutting down network controller")
 }
 
 func (pc *PodController) worker() {

pkg/reconciler/ip.go

@@ -1,48 +1,48 @@
+// package main
 package reconciler
 
 import (
 	"context"
-	"flag"
-	"os"
+	"time"
 
 	"github.com/k8snetworkplumbingwg/whereabouts/pkg/logging"
-	"github.com/k8snetworkplumbingwg/whereabouts/pkg/reconciler"
 )
 
-const defaultReconcilerTimeout = 30
-
-func main() {
-	kubeConfigFile := flag.String("kubeconfig", "", "the path to the Kubernetes configuration file")
-	logLevel := flag.String("log-level", "error", "the logging level for the `ip-reconciler` app. Valid values are: \"debug\", \"verbose\", \"error\", and \"panic\".")
-	reconcilerTimeout := flag.Int("timeout", defaultReconcilerTimeout, "the value for a request timeout in seconds.")
-	flag.Parse()
+const (
+	defaultReconcilerTimeout = 30
+)
 
-	logging.SetLogLevel(*logLevel)
+// TODO: get ip_test.go working with this - currently idk if it does...
+func InvokeIPReconciler(returnErr chan error) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(defaultReconcilerTimeout*time.Second))
+	defer cancel()
 
-	var err error
-	var ipReconcileLoop *reconciler.ReconcileLooper
-	if kubeConfigFile == nil {
-		ipReconcileLoop, err = reconciler.NewReconcileLooper(context.Background(), *reconcilerTimeout)
-	} else {
-		ipReconcileLoop, err = reconciler.NewReconcileLooperWithKubeconfig(context.Background(), *kubeConfigFile, *reconcilerTimeout)
-	}
+	ipReconcileLoop, err := NewReconcileLooper(ctx, defaultReconcilerTimeout)
 	if err != nil {
 		_ = logging.Errorf("failed to create the reconcile looper: %v", err)
-		os.Exit(couldNotStartOrphanedIPMonitor)
+		returnErr <- err
+		return
 	}
 
-	cleanedUpIps, err := ipReconcileLoop.ReconcileIPPools(context.Background())
+	cleanedUpIps, err := ipReconcileLoop.ReconcileIPPools(ctx)
 	if err != nil {
 		_ = logging.Errorf("failed to clean up IP for allocations: %v", err)
-		os.Exit(failedToReconcileIPPools)
+		returnErr <- err
+		return
 	}
+
 	if len(cleanedUpIps) > 0 {
 		logging.Debugf("successfully cleanup IPs: %+v", cleanedUpIps)
 	} else {
 		logging.Debugf("no IP addresses to cleanup")
 	}
 
-	if err := ipReconcileLoop.ReconcileOverlappingIPAddresses(context.Background()); err != nil {
-		os.Exit(failedToReconcileClusterWideIPs)
+	if err := ipReconcileLoop.ReconcileOverlappingIPAddresses(ctx); err != nil {
+		returnErr <- err
+		return
 	}
+
+	logging.Verbosef("no errors with ip reconciler...returning in a sec")
+	returnErr <- err
+	return
 }