From e18dd3fb68a63ede3cf7a3c6c7cef8764073492c Mon Sep 17 00:00:00 2001 From: Alex Fenlon Date: Wed, 27 Nov 2024 15:59:54 +0000 Subject: [PATCH 1/3] add events to special secrets --- cmd/nginx-ingress/main.go | 45 ++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 45697b747c..0907041e56 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -34,6 +34,7 @@ import ( "github.com/prometheus/client_golang/prometheus" api_v1 "k8s.io/api/core/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + pkg_runtime "k8s.io/apimachinery/pkg/runtime" util_version "k8s.io/apimachinery/pkg/util/version" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" @@ -75,6 +76,8 @@ const ( appProtectVersionPath = "/opt/app_protect/RELEASE" appProtectv4BundleFolder = "/etc/nginx/waf/bundles/" appProtectv5BundleFolder = "/etc/app_protect/bundles/" + fatalEventFlushTime = 200 * time.Millisecond + secretErrorReason = "SecretError" ) func main() { @@ -89,9 +92,14 @@ func main() { buildOS := os.Getenv("BUILD_OS") controllerNamespace := os.Getenv("POD_NAMESPACE") + podName := os.Getenv("POD_NAME") config, kubeClient := mustCreateConfigAndKubeClient(ctx) mustValidateKubernetesVersionInfo(ctx, kubeClient) + pod, err := kubeClient.CoreV1().Pods(controllerNamespace).Get(context.TODO(), podName, meta_v1.GetOptions{}) + if err != nil { + nl.Fatalf(l, "Failed to get pod: %v", err) + } eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(func(format string, args ...interface{}) { nl.Infof(l, format, args...) @@ -143,12 +151,17 @@ func main() { templateExecutor, templateExecutorV2 := createTemplateExecutors(ctx) - sslRejectHandshake := processDefaultServerSecret(ctx, kubeClient, nginxManager) - - isWildcardEnabled := processWildcardSecret(ctx, kubeClient, nginxManager) + sslRejectHandshake, err := processDefaultServerSecret(kubeClient, nginxManager) + if err != nil { + logEventAndExit(ctx, eventRecorder, pod, secretErrorReason, err) + } staticSSLPath := nginxManager.GetSecretsDir() + isWildcardEnabled, err := processWildcardSecret(kubeClient, nginxManager) + if err != nil { + logEventAndExit(ctx, eventRecorder, pod, secretErrorReason, err) + } globalConfigurationValidator := createGlobalConfigurationValidator() mustProcessGlobalConfiguration(ctx) @@ -562,14 +575,13 @@ func startChildProcesses(nginxManager nginx.Manager, appProtectV5 bool) childPro } } -func processDefaultServerSecret(ctx context.Context, kubeClient *kubernetes.Clientset, nginxManager nginx.Manager) bool { - l := nl.LoggerFromContext(ctx) +func processDefaultServerSecret(kubeClient *kubernetes.Clientset, nginxManager nginx.Manager) (bool, error) { var sslRejectHandshake bool if *defaultServerSecret != "" { secret, err := getAndValidateSecret(kubeClient, *defaultServerSecret) if err != nil { - nl.Fatalf(l, "Error trying to get the default server TLS secret %v: %v", *defaultServerSecret, err) + return sslRejectHandshake, fmt.Errorf("error trying to get the default server TLS secret %v: %w", *defaultServerSecret, err) } bytes := configs.GenerateCertAndKeyFileContent(secret) @@ -581,25 +593,25 @@ func processDefaultServerSecret(ctx context.Context, kubeClient *kubernetes.Clie // file doesn't exist - it is OK! we will reject TLS connections in the default server sslRejectHandshake = true } else { - nl.Fatalf(l, "Error checking the default server TLS cert and key in %s: %v", configs.DefaultServerSecretPath, err) + return sslRejectHandshake, fmt.Errorf("error checking the default server TLS cert and key in %s: %w", configs.DefaultServerSecretPath, err) } } } - return sslRejectHandshake + return sslRejectHandshake, nil } -func processWildcardSecret(ctx context.Context, kubeClient *kubernetes.Clientset, nginxManager nginx.Manager) bool { - l := nl.LoggerFromContext(ctx) - if *wildcardTLSSecret != "" { +func processWildcardSecret(kubeClient *kubernetes.Clientset, nginxManager nginx.Manager) (bool, error) { + isWildcardEnabled := *wildcardTLSSecret != "" + if isWildcardEnabled { secret, err := getAndValidateSecret(kubeClient, *wildcardTLSSecret) if err != nil { - nl.Fatalf(l, "Error trying to get the wildcard TLS secret %v: %v", *wildcardTLSSecret, err) + return false, fmt.Errorf("error trying to get the wildcard TLS secret %v: %w", *wildcardTLSSecret, err) } bytes := configs.GenerateCertAndKeyFileContent(secret) nginxManager.CreateSecret(configs.WildcardSecretFileName, bytes, nginx.ReadWriteOnlyFileMode) } - return *wildcardTLSSecret != "" + return isWildcardEnabled, nil } func createGlobalConfigurationValidator() *cr_validation.GlobalConfigurationValidator { @@ -942,6 +954,13 @@ func updateSelfWithVersionInfo(ctx context.Context, eventLog record.EventRecorde } } +func logEventAndExit(ctx context.Context, eventLog record.EventRecorder, obj pkg_runtime.Object, reason string, err error) { + l := nl.LoggerFromContext(ctx) + eventLog.Eventf(obj, api_v1.EventTypeWarning, reason, err.Error()) + time.Sleep(fatalEventFlushTime) // wait for the event to be flushed + nl.Fatal(l, err.Error()) +} + func initLogger(logFormat string, level slog.Level, out io.Writer) context.Context { programLevel := new(slog.LevelVar) // Info by default var h slog.Handler From 1e670f7bfaff5106a4dca44c52f7eccb917a470a Mon Sep 17 00:00:00 2001 From: Paul Abel <128620221+pdabelf5@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:48:22 +0000 Subject: [PATCH 2/3] add secret type for getAndValidateSecret() (#6877) --- cmd/nginx-ingress/main.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 0907041e56..1a45efe3c8 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -579,7 +579,7 @@ func processDefaultServerSecret(kubeClient *kubernetes.Clientset, nginxManager n var sslRejectHandshake bool if *defaultServerSecret != "" { - secret, err := getAndValidateSecret(kubeClient, *defaultServerSecret) + secret, err := getAndValidateSecret(kubeClient, *defaultServerSecret, api_v1.SecretTypeTLS) if err != nil { return sslRejectHandshake, fmt.Errorf("error trying to get the default server TLS secret %v: %w", *defaultServerSecret, err) } @@ -603,7 +603,7 @@ func processDefaultServerSecret(kubeClient *kubernetes.Clientset, nginxManager n func processWildcardSecret(kubeClient *kubernetes.Clientset, nginxManager nginx.Manager) (bool, error) { isWildcardEnabled := *wildcardTLSSecret != "" if isWildcardEnabled { - secret, err := getAndValidateSecret(kubeClient, *wildcardTLSSecret) + secret, err := getAndValidateSecret(kubeClient, *wildcardTLSSecret, api_v1.SecretTypeTLS) if err != nil { return false, fmt.Errorf("error trying to get the wildcard TLS secret %v: %w", *wildcardTLSSecret, err) } @@ -673,7 +673,8 @@ func getSocketClient(sockPath string) *http.Client { } // getAndValidateSecret gets and validates a secret. -func getAndValidateSecret(kubeClient *kubernetes.Clientset, secretNsName string) (secret *api_v1.Secret, err error) { +// nolint:unparam +func getAndValidateSecret(kubeClient *kubernetes.Clientset, secretNsName string, secretType api_v1.SecretType) (secret *api_v1.Secret, err error) { ns, name, err := k8s.ParseNamespaceName(secretNsName) if err != nil { return nil, fmt.Errorf("could not parse the %v argument: %w", secretNsName, err) @@ -682,9 +683,12 @@ func getAndValidateSecret(kubeClient *kubernetes.Clientset, secretNsName string) if err != nil { return nil, fmt.Errorf("could not get %v: %w", secretNsName, err) } - err = secrets.ValidateTLSSecret(secret) - if err != nil { - return nil, fmt.Errorf("%v is invalid: %w", secretNsName, err) + switch secretType { + case api_v1.SecretTypeTLS: + err = secrets.ValidateTLSSecret(secret) + if err != nil { + return nil, fmt.Errorf("%v is invalid: %w", secretNsName, err) + } } return secret, nil } @@ -791,7 +795,7 @@ func createPlusAndLatencyCollectors( syslogListener = metrics.NewSyslogFakeServer() if *prometheusTLSSecretName != "" { - prometheusSecret, err = getAndValidateSecret(kubeClient, *prometheusTLSSecretName) + prometheusSecret, err = getAndValidateSecret(kubeClient, *prometheusTLSSecretName, api_v1.SecretTypeTLS) if err != nil { nl.Fatalf(l, "Error trying to get the prometheus TLS secret %v: %v", *prometheusTLSSecretName, err) } @@ -843,7 +847,7 @@ func createHealthProbeEndpoint(kubeClient *kubernetes.Clientset, plusClient *cli var err error if *serviceInsightTLSSecretName != "" { - serviceInsightSecret, err = getAndValidateSecret(kubeClient, *serviceInsightTLSSecretName) + serviceInsightSecret, err = getAndValidateSecret(kubeClient, *serviceInsightTLSSecretName, api_v1.SecretTypeTLS) if err != nil { nl.Fatalf(l, "Error trying to get the service insight TLS secret %v: %v", *serviceInsightTLSSecretName, err) } From ea66430f1ee5a48e331256d88e049326eb449e16 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Thu, 28 Nov 2024 12:16:55 +0000 Subject: [PATCH 3/3] ensure the eventBroascaster gets shutdown cleanly --- cmd/nginx-ingress/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 1a45efe3c8..13917ec05b 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -109,6 +109,7 @@ func main() { }) eventRecorder := eventBroadcaster.NewRecorder(scheme.Scheme, api_v1.EventSource{Component: "nginx-ingress-controller"}) + defer eventBroadcaster.Shutdown() mustValidateIngressClass(ctx, kubeClient) checkNamespaces(ctx, kubeClient)