[DoNotMerge] Ip reconciler alerting

cmd/controlloop/main.go

@@ -3,8 +3,11 @@ package main
 import (
 	"flag"
 	"fmt"
+	"log"
+	"net/http"
 	"os"
 	"os/signal"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -19,10 +22,13 @@ import (
 	nadclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
 	nadinformers "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions"
 
+	"github.com/k8snetworkplumbingwg/whereabouts/cmd/reconciler"
 	wbclient "github.com/k8snetworkplumbingwg/whereabouts/pkg/client/clientset/versioned"
 	wbinformers "github.com/k8snetworkplumbingwg/whereabouts/pkg/client/informers/externalversions"
 	"github.com/k8snetworkplumbingwg/whereabouts/pkg/logging"
 	"github.com/k8snetworkplumbingwg/whereabouts/pkg/reconciler/controlloop"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 const (
@@ -39,14 +45,18 @@ const (
 )
 
 func main() {
+	logging.Verbosef("This is a test... :)")
+
 	logLevel := flag.String("log-level", defaultLogLevel, "Specify the pod controller application logging level")
 	if logLevel != nil && logging.GetLoggingLevel().String() != *logLevel {
 		logging.SetLogLevel(*logLevel)
 	}
 	logging.SetLogStderr(true)
 
 	stopChan := make(chan struct{})
+	returnErr := make(chan error)
 	defer close(stopChan)
+	defer close(returnErr)
 	handleSignals(stopChan, os.Interrupt)
 
 	networkController, err := newPodController(stopChan)
@@ -55,7 +65,47 @@ func main() {
 		os.Exit(couldNotCreateController)
 	}
 
-	networkController.Start(stopChan)
+	networkController.Start(stopChan) // code seems to spin here... further code not executed
+
+	logging.Verbosef("2nd test print, doubt this is gonna be reached...") // only gets reached if I ctrl+C during the code's runtime.
+
+	totalReconcilerSuccess := prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "reconciler_success_total",
+		Help: "Increments upon successful run of IP reconciler",
+	})
+
+	prometheus.MustRegister(totalReconcilerSuccess)
+
+	http.Handle("/metrics", promhttp.Handler())
+	log.Fatal(http.ListenAndServe(":8080", nil))
+
+	// here's where my for { select {} } loop should go - and use tickers
+	// https://gobyexample.com/tickers
+	// general logic - loop indefinitely, with the following conditions:
+	// a. stopChan sends a value: quit out of the loop and return function
+	// b. ticker ticks: start a goroutine to run ip-reconciler
+	// c. default: continue to spin
+	ticker := time.NewTicker(10 * time.Second) // temp value, will eventually be days/weeks duration
+
+	for i := 0; i < 5; i++ { // iterating 5 times just for the sake of terminating code and getting logfile output
+		logging.Verbosef("iteration #", i)
+		select {
+		case <-stopChan:
+			return
+		case t := <-ticker.C:
+			// fmt.Println("Running ip-reconciler, tick at ", t)
+			logging.Verbosef("time to run reconciler (dummy), tick at ", t)
+			go reconciler.InvokeIPReconciler(returnErr) // need to implement a timeout for this
+		case err := <-returnErr:
+			if err == nil {
+				totalReconcilerSuccess.Inc()
+				logging.Verbosef("ip reconciler success!")
+			} else {
+				logging.Verbosef("ip reconciler failure: ", err)
+			}
+		}
+	}
+	return
 }
 
 func handleSignals(stopChannel chan struct{}, signals ...os.Signal) {

cmd/reconciler/errors.go

@@ -1,4 +1,5 @@
-package main
+// package main
+package reconciler
 
 const (
 	kubeconfigNotFound = iota + 1

cmd/reconciler/ip.go

@@ -1,40 +1,46 @@
-package main
+// package main
+package reconciler
 
 import (
 	"context"
+	"errors"
 	"flag"
-	"os"
+	"time"
 
 	"github.com/k8snetworkplumbingwg/whereabouts/pkg/logging"
 	"github.com/k8snetworkplumbingwg/whereabouts/pkg/reconciler"
 )
 
 const defaultReconcilerTimeout = 30
 
-func main() {
-	kubeConfigFile := flag.String("kubeconfig", "", "the path to the Kubernetes configuration file")
-	logLevel := flag.String("log-level", "error", "the logging level for the `ip-reconciler` app. Valid values are: \"debug\", \"verbose\", \"error\", and \"panic\".")
-	reconcilerTimeout := flag.Int("timeout", defaultReconcilerTimeout, "the value for a request timeout in seconds.")
-	flag.Parse()
+func InvokeIPReconciler(returnErr chan error) {
+	go func() {
+		doInvokeIPReconciler(returnErr)
+	}()
+	select {
+	case <-time.After(5 * time.Second):
+		returnErr <- errors.New("timed out (this is the 5 sec test timeout, nbd)")
+		return
+	case <-returnErr:
+		return
+	}
+}
 
-	logging.SetLogLevel(*logLevel)
+func doInvokeIPReconciler(returnErr chan error) {
+	reconcilerTimeout := flag.Int("timeout", defaultReconcilerTimeout, "the value for a request timeout in seconds.")
+	ipReconcileLoop, err := reconciler.NewReconcileLooper(context.Background(), *reconcilerTimeout)
 
-	var err error
-	var ipReconcileLoop *reconciler.ReconcileLooper
-	if kubeConfigFile == nil {
-		ipReconcileLoop, err = reconciler.NewReconcileLooper(context.Background(), *reconcilerTimeout)
-	} else {
-		ipReconcileLoop, err = reconciler.NewReconcileLooperWithKubeconfig(context.Background(), *kubeConfigFile, *reconcilerTimeout)
-	}
 	if err != nil {
 		_ = logging.Errorf("failed to create the reconcile looper: %v", err)
-		os.Exit(couldNotStartOrphanedIPMonitor)
+		returnErr <- err
+		return //returns might not even be necessary thanks to timeout code in InvokeIPReconciler
 	}
 
 	cleanedUpIps, err := ipReconcileLoop.ReconcileIPPools(context.Background())
 	if err != nil {
 		_ = logging.Errorf("failed to clean up IP for allocations: %v", err)
-		os.Exit(failedToReconcileIPPools)
+		returnErr <- err
+		return //returns might not even be necessary thanks to timeout code in InvokeIPReconciler
 	}
 	if len(cleanedUpIps) > 0 {
 		logging.Debugf("successfully cleanup IPs: %+v", cleanedUpIps)
@@ -43,6 +49,47 @@ func main() {
 	}
 
 	if err := ipReconcileLoop.ReconcileOverlappingIPAddresses(context.Background()); err != nil {
-		os.Exit(failedToReconcileClusterWideIPs)
+		returnErr <- err
+		return //returns might not even be necessary thanks to timeout code in InvokeIPReconciler
 	}
+
+	// increment success metric here - no error occurred.
+	returnErr <- err
 }
+
+// i think this function will get deleted
+// func main() {
+// 	kubeConfigFile := flag.String("kubeconfig", "", "the path to the Kubernetes configuration file")
+// 	logLevel := flag.String("log-level", "error", "the logging level for the `ip-reconciler` app. Valid values are: \"debug\", \"verbose\", \"error\", and \"panic\".")
+// 	reconcilerTimeout := flag.Int("timeout", defaultReconcilerTimeout, "the value for a request timeout in seconds.")
+// 	flag.Parse()
+
+// 	logging.SetLogLevel(*logLevel)
+
+// 	var err error
+// 	var ipReconcileLoop *reconciler.ReconcileLooper
+// 	if kubeConfigFile == nil {
+// 		ipReconcileLoop, err = reconciler.NewReconcileLooper(context.Background(), *reconcilerTimeout)
+// 	} else {
+// 		ipReconcileLoop, err = reconciler.NewReconcileLooperWithKubeconfig(context.Background(), *kubeConfigFile, *reconcilerTimeout)
+// 	}
+// 	if err != nil {
+// 		_ = logging.Errorf("failed to create the reconcile looper: %v", err)
+// 		os.Exit(couldNotStartOrphanedIPMonitor)
+// 	}
+
+// 	cleanedUpIps, err := ipReconcileLoop.ReconcileIPPools(context.Background())
+// 	if err != nil {
+// 		_ = logging.Errorf("failed to clean up IP for allocations: %v", err)
+// 		os.Exit(failedToReconcileIPPools)
+// 	}
+// 	if len(cleanedUpIps) > 0 {
+// 		logging.Debugf("successfully cleanup IPs: %+v", cleanedUpIps)
+// 	} else {
+// 		logging.Debugf("no IP addresses to cleanup")
+// 	}
+
+// 	if err := ipReconcileLoop.ReconcileOverlappingIPAddresses(context.Background()); err != nil {
+// 		os.Exit(failedToReconcileClusterWideIPs)
+// 	}
+// }

cmd/reconciler/ip_test.go

@@ -1,4 +1,8 @@
-package main
+// package main
+package reconciler
+
+// does this code need to exist still? or does it need to be updated? ip.go's code body will just exist as a helper function elsewhere.
+// im just unsure of the architecture of all this. TODO
 
 import (
 	"context"

cmd/reconciler/suite_test.go

@@ -1,15 +1,17 @@
-package main
+// package main
+package reconciler
 
 import (
 	"encoding/base64"
 	"fmt"
 	"io/ioutil"
-	"k8s.io/client-go/kubernetes"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
+	"k8s.io/client-go/kubernetes"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 

go.mod

@@ -9,11 +9,11 @@ require (
 	github.com/coreos/etcd v3.3.13+incompatible
 	github.com/grpc-ecosystem/grpc-gateway v1.11.1 // indirect
 	github.com/imdario/mergo v0.3.10
-	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20210510153419-66a699ae3b05
 	github.com/onsi/ginkgo v1.14.1
 	github.com/onsi/gomega v1.10.2
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.12.1
 	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
 	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
 	golang.org/x/tools v0.1.9 // indirect
@@ -28,4 +28,7 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
+replace (
+	github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
+	google.golang.org/grpc => google.golang.org/grpc v1.29.0
+)