From 406d1682e2c757ecaf42c52a0a21e30b2a8370a3 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Fri, 29 Nov 2024 17:30:33 +0000 Subject: [PATCH 1/2] progres towards expiry check --- cmd/nginx-ingress/main.go | 2 +- .../license_reporting/license_reporting.go | 20 +++++++++++++++++-- .../license_reporting_test.go | 4 +++- internal/nginx/manager.go | 13 +++--------- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index d8c4d92f98..c82c37e1e4 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -124,7 +124,7 @@ func main() { var licenseReporter *license_reporting.LicenseReporter if *nginxPlus { - licenseReporter = license_reporting.NewLicenseReporter(kubeClient) + licenseReporter = license_reporting.NewLicenseReporter(kubeClient, eventRecorder, pod) } nginxManager, useFakeNginxManager := createNginxManager(ctx, managerCollector, licenseReporter) diff --git a/internal/license_reporting/license_reporting.go b/internal/license_reporting/license_reporting.go index 7ad8bd5b5c..665a461d4a 100644 --- a/internal/license_reporting/license_reporting.go +++ b/internal/license_reporting/license_reporting.go @@ -3,6 +3,7 @@ package licensereporting import ( "context" "encoding/json" + "fmt" "log/slog" "os" "path/filepath" @@ -11,9 +12,11 @@ import ( nl "github.com/nginxinc/kubernetes-ingress/internal/logger" clusterInfo "github.com/nginxinc/kubernetes-ingress/internal/common_cluster_info" + api_v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/record" ) var ( @@ -59,15 +62,19 @@ type LicenseReporterConfig struct { Period time.Duration K8sClientReader kubernetes.Interface PodNSName types.NamespacedName + EventLog record.EventRecorder + Pod *api_v1.Pod } // NewLicenseReporter creates a new LicenseReporter -func NewLicenseReporter(client kubernetes.Interface) *LicenseReporter { +func NewLicenseReporter(client kubernetes.Interface, eventLog record.EventRecorder, pod *api_v1.Pod) *LicenseReporter { return &LicenseReporter{ config: LicenseReporterConfig{ - Period: 24 * time.Hour, + EventLog: eventLog, + Period: time.Hour, K8sClientReader: client, PodNSName: types.NamespacedName{Namespace: os.Getenv("POD_NAMESPACE"), Name: os.Getenv("POD_NAME")}, + Pod: pod, }, } } @@ -93,4 +100,13 @@ func (lr *LicenseReporter) collectAndWrite(ctx context.Context) { } info := newLicenseInfo(clusterID, installationID, nodeCount) writeLicenseInfo(l, info) + lr.checkLicenseExpiry(ctx) +} + +func (lr *LicenseReporter) checkLicenseExpiry(ctx context.Context) { + l := nl.LoggerFromContext(ctx) + days := 5 + eventText := fmt.Sprintf("Expires in %d days", days) + nl.Debug(l, eventText) + lr.config.EventLog.Event(lr.config.Pod, api_v1.EventTypeNormal, "ExpiryCheck", eventText) } diff --git a/internal/license_reporting/license_reporting_test.go b/internal/license_reporting/license_reporting_test.go index 63b0bb0a32..e498f309de 100644 --- a/internal/license_reporting/license_reporting_test.go +++ b/internal/license_reporting/license_reporting_test.go @@ -11,7 +11,9 @@ import ( nic_glog "github.com/nginxinc/kubernetes-ingress/internal/logger/glog" "github.com/nginxinc/kubernetes-ingress/internal/logger/levels" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/record" ) func TestNewLicenseInfo(t *testing.T) { @@ -64,7 +66,7 @@ func TestWriteLicenseInfo(t *testing.T) { } func TestNewLicenseReporter(t *testing.T) { - reporter := NewLicenseReporter(fake.NewSimpleClientset()) + reporter := NewLicenseReporter(fake.NewSimpleClientset(), record.NewFakeRecorder(2048), &v1.Pod{}) if reporter == nil { t.Fatal("NewLicenseReporter() returned nil") } diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index c1b2cfa1a9..c6164745b8 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -303,16 +303,9 @@ func (lm *LocalManager) ClearAppProtectFolder(name string) { // Start starts NGINX. func (lm *LocalManager) Start(done chan error) { if lm.nginxPlus { - isR33OrGreater, versionErr := lm.Version().PlusGreaterThanOrEqualTo("nginx-plus-r33") - if versionErr != nil { - nl.Errorf(lm.logger, "Error determining whether nginx version is >= r33: %v", versionErr) - } - if isR33OrGreater { - ctx, cancel := context.WithCancel(context.Background()) - nl.ContextWithLogger(ctx, lm.logger) - go lm.licenseReporter.Start(ctx) - lm.licenseReporterCancel = cancel - } + ctx, cancel := context.WithCancel(context.Background()) + go lm.licenseReporter.Start(nl.ContextWithLogger(ctx, lm.logger)) + lm.licenseReporterCancel = cancel } nl.Debug(lm.logger, "Starting nginx") From 58dac50b5f2a32480e20440432ee8782183a0f8b Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Mon, 2 Dec 2024 13:19:36 +0000 Subject: [PATCH 2/2] consume license expiry endpoint --- cmd/nginx-ingress/main.go | 3 + .../license_reporting/license_reporting.go | 57 ++++++++-- .../license_reporting_test.go | 100 ++++++++++++++++++ 3 files changed, 149 insertions(+), 11 deletions(-) diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index c82c37e1e4..8b0c78153f 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -218,6 +218,9 @@ func main() { process := startChildProcesses(nginxManager, appProtectV5) plusClient := createPlusClient(ctx, *nginxPlus, useFakeNginxManager, nginxManager) + if *nginxPlus { + licenseReporter.Config.PlusClient = plusClient + } plusCollector, syslogListener, latencyCollector := createPlusAndLatencyCollectors(ctx, registry, constLabels, kubeClient, plusClient, staticCfgParams.NginxServiceMesh) cnf := configs.NewConfigurator(configs.ConfiguratorParams{ diff --git a/internal/license_reporting/license_reporting.go b/internal/license_reporting/license_reporting.go index 665a461d4a..2626272d80 100644 --- a/internal/license_reporting/license_reporting.go +++ b/internal/license_reporting/license_reporting.go @@ -10,6 +10,7 @@ import ( "time" nl "github.com/nginxinc/kubernetes-ingress/internal/logger" + "github.com/nginxinc/nginx-plus-go-client/v2/client" clusterInfo "github.com/nginxinc/kubernetes-ingress/internal/common_cluster_info" api_v1 "k8s.io/api/core/v1" @@ -19,6 +20,8 @@ import ( "k8s.io/client-go/tools/record" ) +const expiryThreshold = 30 * (time.Hour * 24) + var ( reportingDir = "/etc/nginx/reporting" reportingFile = "tracking.info" @@ -54,7 +57,7 @@ func writeLicenseInfo(l *slog.Logger, info *licenseInfo) { // LicenseReporter can start the license reporting process type LicenseReporter struct { - config LicenseReporterConfig + Config LicenseReporterConfig } // LicenseReporterConfig contains the information needed for license reporting @@ -64,12 +67,13 @@ type LicenseReporterConfig struct { PodNSName types.NamespacedName EventLog record.EventRecorder Pod *api_v1.Pod + PlusClient *client.NginxClient } // NewLicenseReporter creates a new LicenseReporter func NewLicenseReporter(client kubernetes.Interface, eventLog record.EventRecorder, pod *api_v1.Pod) *LicenseReporter { return &LicenseReporter{ - config: LicenseReporterConfig{ + Config: LicenseReporterConfig{ EventLog: eventLog, Period: time.Hour, K8sClientReader: client, @@ -81,32 +85,63 @@ func NewLicenseReporter(client kubernetes.Interface, eventLog record.EventRecord // Start begins the license report writer process for NIC func (lr *LicenseReporter) Start(ctx context.Context) { - wait.JitterUntilWithContext(ctx, lr.collectAndWrite, lr.config.Period, 0.1, true) + wait.JitterUntilWithContext(ctx, lr.collectAndWrite, lr.Config.Period, 0.1, true) } func (lr *LicenseReporter) collectAndWrite(ctx context.Context) { l := nl.LoggerFromContext(ctx) - clusterID, err := clusterInfo.GetClusterID(ctx, lr.config.K8sClientReader) + clusterID, err := clusterInfo.GetClusterID(ctx, lr.Config.K8sClientReader) if err != nil { nl.Errorf(l, "Error collecting ClusterIDS: %v", err) } - nodeCount, err := clusterInfo.GetNodeCount(ctx, lr.config.K8sClientReader) + nodeCount, err := clusterInfo.GetNodeCount(ctx, lr.Config.K8sClientReader) if err != nil { nl.Errorf(l, "Error collecting ClusterNodeCount: %v", err) } - installationID, err := clusterInfo.GetInstallationID(ctx, lr.config.K8sClientReader, lr.config.PodNSName) + installationID, err := clusterInfo.GetInstallationID(ctx, lr.Config.K8sClientReader, lr.Config.PodNSName) if err != nil { nl.Errorf(l, "Error collecting InstallationID: %v", err) } info := newLicenseInfo(clusterID, installationID, nodeCount) writeLicenseInfo(l, info) - lr.checkLicenseExpiry(ctx) + if lr.Config.PlusClient != nil { + lr.checkLicenseExpiry(ctx) + } } func (lr *LicenseReporter) checkLicenseExpiry(ctx context.Context) { l := nl.LoggerFromContext(ctx) - days := 5 - eventText := fmt.Sprintf("Expires in %d days", days) - nl.Debug(l, eventText) - lr.config.EventLog.Event(lr.config.Pod, api_v1.EventTypeNormal, "ExpiryCheck", eventText) + licenseData, err := lr.Config.PlusClient.GetNginxLicense(context.Background()) + if err != nil { + nl.Errorf(l, "could not get license data, %v", err) + return + } + var licenseEventText string + if expiring, days := licenseExpiring(licenseData); expiring { + licenseEventText = fmt.Sprintf("License expiring in %d day(s)", days) + nl.Warn(l, licenseEventText) + lr.Config.EventLog.Event(lr.Config.Pod, api_v1.EventTypeWarning, "LicenseExpiry", licenseEventText) + } + var usageGraceEventText string + if ending, days := usageGraceEnding(licenseData); ending { + usageGraceEventText = fmt.Sprintf("Usage reporting grace period ending in %d day(s)", days) + nl.Warn(l, usageGraceEventText) + lr.Config.EventLog.Event(lr.Config.Pod, api_v1.EventTypeWarning, "UsageGraceEnding", usageGraceEventText) + } +} + +func licenseExpiring(licenseData *client.NginxLicense) (bool, int64) { + expiry := time.Unix(int64(licenseData.ActiveTill), 0) //nolint:gosec + now := time.Now() + timeUntilLicenseExpiry := expiry.Sub(now) + daysUntilLicenseExpiry := int64(timeUntilLicenseExpiry.Hours() / 24) + expiryDays := int64(expiryThreshold.Hours() / 24) + return daysUntilLicenseExpiry < expiryDays, daysUntilLicenseExpiry +} + +func usageGraceEnding(licenseData *client.NginxLicense) (bool, int64) { + grace := time.Second * time.Duration(licenseData.Reporting.Grace) //nolint:gosec + daysUntilUsageGraceEnds := int64(grace.Hours() / 24) + expiryDays := int64(expiryThreshold.Hours() / 24) + return daysUntilUsageGraceEnds < expiryDays, daysUntilUsageGraceEnds } diff --git a/internal/license_reporting/license_reporting_test.go b/internal/license_reporting/license_reporting_test.go index e498f309de..90170fdb46 100644 --- a/internal/license_reporting/license_reporting_test.go +++ b/internal/license_reporting/license_reporting_test.go @@ -7,9 +7,11 @@ import ( "os" "path/filepath" "testing" + "time" nic_glog "github.com/nginxinc/kubernetes-ingress/internal/logger/glog" "github.com/nginxinc/kubernetes-ingress/internal/logger/levels" + "github.com/nginxinc/nginx-plus-go-client/v2/client" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/fake" @@ -71,3 +73,101 @@ func TestNewLicenseReporter(t *testing.T) { t.Fatal("NewLicenseReporter() returned nil") } } + +func TestLicenseExpiring(t *testing.T) { + t.Parallel() + + testCases := []struct { + licenseData client.NginxLicense + belowExpiringThreshold bool + days int64 + name string + }{ + { + licenseData: client.NginxLicense{ + ActiveTill: uint64(time.Now().Add(time.Hour).Unix()), //nolint:gosec + }, + belowExpiringThreshold: true, + days: 0, + name: "License expires in 1 hour", + }, + { + licenseData: client.NginxLicense{ + ActiveTill: uint64(time.Now().Add(-time.Hour).Unix()), //nolint:gosec + }, + belowExpiringThreshold: true, + days: 0, + name: "License expired 1 hour ago", + }, + { + licenseData: client.NginxLicense{ + ActiveTill: uint64(time.Now().Add(time.Hour * 24 * 31).Unix()), //nolint:gosec + }, + belowExpiringThreshold: false, + days: 30, // Rounds down + name: "License expires in 31 days", + }, + } + + for _, tc := range testCases { + actualExpiring, actualDays := licenseExpiring(&tc.licenseData) + if actualExpiring != tc.belowExpiringThreshold { + t.Fatalf("%s: Expected different value for expiring %t", tc.name, tc.belowExpiringThreshold) + } + if actualDays != tc.days { + t.Fatalf("%s: Expected different value for days %d != %d", tc.name, actualDays, tc.days) + } + } +} + +func TestUsageGraceEnding(t *testing.T) { + t.Parallel() + + testCases := []struct { + licenseData client.NginxLicense + belowExpiringThreshold bool + days int64 + name string + }{ + { + licenseData: client.NginxLicense{ + Reporting: client.LicenseReporting{ + Grace: 3600, // seconds + }, + }, + belowExpiringThreshold: true, + days: 0, + name: "Grace period ends in an hour", + }, + { + licenseData: client.NginxLicense{ + Reporting: client.LicenseReporting{ + Grace: 60 * 60 * 24 * 31, // 31 days + }, + }, + belowExpiringThreshold: false, + days: 31, + name: "Grace period ends 31 days", + }, + { + licenseData: client.NginxLicense{ + Reporting: client.LicenseReporting{ + Grace: 0, + }, + }, + belowExpiringThreshold: true, + days: 0, + name: "Grace period ended", + }, + } + + for _, tc := range testCases { + actualEnding, actualDays := usageGraceEnding(&tc.licenseData) + if actualEnding != tc.belowExpiringThreshold { + t.Fatalf("%s: Expected different value for expiring %t", tc.name, tc.belowExpiringThreshold) + } + if actualDays != tc.days { + t.Fatalf("%s: Expected different value for days %d != %d", tc.name, actualDays, tc.days) + } + } +}