Add clientReader to get a secret that is not cached by manager

Signed-off-by: jooho lee <[email protected]>

controllers/inferenceservice_controller.go

@@ -42,20 +42,22 @@ import (
 // OpenshiftInferenceServiceReconciler holds the controller configuration.
 type OpenshiftInferenceServiceReconciler struct {
 	client                         client.Client
+	clientReader                   client.Reader
 	log                            logr.Logger
 	MeshDisabled                   bool
 	mmISVCReconciler               *reconcilers.ModelMeshInferenceServiceReconciler
 	kserveServerlessISVCReconciler *reconcilers.KserveServerlessInferenceServiceReconciler
 	kserveRawISVCReconciler        *reconcilers.KserveRawInferenceServiceReconciler
 }
 
-func NewOpenshiftInferenceServiceReconciler(client client.Client, log logr.Logger, meshDisabled bool) *OpenshiftInferenceServiceReconciler {
+func NewOpenshiftInferenceServiceReconciler(client client.Client, clientReader client.Reader, log logr.Logger, meshDisabled bool) *OpenshiftInferenceServiceReconciler {
 	return &OpenshiftInferenceServiceReconciler{
 		client:                         client,
+		clientReader:                   clientReader,
 		log:                            log,
 		MeshDisabled:                   meshDisabled,
 		mmISVCReconciler:               reconcilers.NewModelMeshInferenceServiceReconciler(client),
-		kserveServerlessISVCReconciler: reconcilers.NewKServeServerlessInferenceServiceReconciler(client),
+		kserveServerlessISVCReconciler: reconcilers.NewKServeServerlessInferenceServiceReconciler(client, clientReader),
 		kserveRawISVCReconciler:        reconcilers.NewKServeRawInferenceServiceReconciler(client),
 	}
 }
@@ -152,7 +154,7 @@ func (r *OpenshiftInferenceServiceReconciler) SetupWithManager(mgr ctrl.Manager)
 				}
 				return reconcileRequests
 			}))
-		
+
 	kserveWithMeshEnabled, kserveWithMeshEnabledErr := utils.VerifyIfComponentIsEnabled(context.Background(), mgr.GetClient(), utils.KServeWithServiceMeshComponent)
 	if kserveWithMeshEnabledErr != nil {
 		r.log.V(1).Error(kserveWithMeshEnabledErr, "could not determine if kserve have service mesh enabled")

controllers/kserve_inferenceservice_controller_test.go

@@ -275,7 +275,6 @@ var _ = Describe("The Openshift Kserve model controller", func() {
 			kserveLocalGateway := &istioclientv1beta1.Gateway{}
 			err := convertToStructuredResource(kserveLocalGatewayPath, kserveLocalGateway)
 			Expect(err).NotTo(HaveOccurred())
-
 			Expect(cli.Create(ctx, kserveLocalGateway)).Should(Succeed())
 
 			// Stub: Create a certificate Secret, which must be created by the openshift service-ca operator.
@@ -312,7 +311,15 @@ var _ = Describe("The Openshift Kserve model controller", func() {
 			// Verify that the certificate secret is created in the istio-system namespace.
 			Eventually(func() error {
 				secret := &corev1.Secret{}
-				err := cli.Get(ctx, client.ObjectKey{Namespace: constants.IstioNamespace, Name: fmt.Sprintf("%s-%s", inferenceService.Name, inferenceService.Namespace)}, secret)
+				err := cli.Get(ctx, types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace}, secret)
+				if err != nil {
+					return err
+				}
+				return nil
+			}, timeout, interval).Should(Succeed())
+
+			Eventually(func() error {
+				err = cli.Get(ctx, client.ObjectKey{Namespace: constants.IstioNamespace, Name: fmt.Sprintf("%s-%s", inferenceService.Name, inferenceService.Namespace)}, secret)
 				if err != nil {
 					return err
 				}

controllers/reconcilers/kserve_isvc_gateway_reconciler.go

@@ -32,42 +32,28 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	istiov1beta1 "istio.io/api/networking/v1beta1"
 	istioclientv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1"
-	"k8s.io/client-go/rest"
 )
 
 var _ SubResourceReconciler = (*KserveGatewayReconciler)(nil)
 var meshNamespace string
 
 type KserveGatewayReconciler struct {
 	client         client.Client
-	clientset      *kubernetes.Clientset
+	clientReader   client.Reader
 	secretHandler  resources.SecretHandler
 	gatewayHandler resources.GatewayHandler
 	deltaProcessor processors.DeltaProcessor
 }
-
-func NewKserveGatewayReconciler(client client.Client) *KserveGatewayReconciler {
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		config, err = rest.InClusterConfig()
-		if err != nil {
-			panic(err.Error())
-		}
-	}
-
-	clientset, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		panic(err.Error())
-	}
+// The clientReader uses the API server to retrieve Secrets that are not cached. By default, only Secrets with the specific label "opendatahub.io/managed: true" are cached.
+func NewKserveGatewayReconciler(client client.Client, clientReader client.Reader) *KserveGatewayReconciler {
 
 	return &KserveGatewayReconciler{
 		client:         client,
-		clientset:      clientset,
+		clientReader:   clientReader,
 		secretHandler:  resources.NewSecretHandler(client),
 		gatewayHandler: resources.NewGatewayHandler(client),
 		deltaProcessor: processors.NewDeltaProcessor(),
@@ -86,8 +72,8 @@ func (r *KserveGatewayReconciler) Reconcile(ctx context.Context, log logr.Logger
 	}
 
 	// return if serving cert secret in the source namespace is not created
-	// srcCertSecret, err := r.secretHandler.Get(ctx, types.NamespacedName{Name: isvc.Name, Namespace: isvc.Namespace})
-	srcCertSecret, err := r.clientset.CoreV1().Secrets(isvc.Namespace).Get(ctx, isvc.Name, metav1.GetOptions{})
+	srcCertSecret := &corev1.Secret{}
+	err := r.clientReader.Get(ctx, types.NamespacedName{Name: isvc.Name, Namespace: isvc.Namespace}, srcCertSecret)
 	if err != nil {
 		if errors.IsNotFound(err) {
 			log.V(1).Info(fmt.Sprintf("Waiting for the creation of the serving certificate Secret(%s) in %s namespace", isvc.Name, isvc.Namespace))
@@ -97,8 +83,9 @@ func (r *KserveGatewayReconciler) Reconcile(ctx context.Context, log logr.Logger
 	}
 
 	// Copy src secret to destination namespace when there is not the synced secret.
-	// copiedCertSecret, err := r.secretHandler.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-%s", isvc.Name, isvc.Namespace), Namespace: meshNamespace})
-	copiedCertSecret, err := r.clientset.CoreV1().Secrets(meshNamespace).Get(ctx, fmt.Sprintf("%s-%s", isvc.Name, isvc.Namespace), metav1.GetOptions{})
+	// This use clientReader because the secret that it looks for is not cached.
+	copiedCertSecret := &corev1.Secret{}
+	err = r.clientReader.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-%s", isvc.Name, isvc.Namespace), Namespace: meshNamespace}, copiedCertSecret)
 	if err != nil {
 		if errors.IsNotFound(err) {
 			if err := r.copyServingCertSecretFromIsvcNamespace(ctx, srcCertSecret, nil); err != nil {
@@ -141,8 +128,8 @@ func (r *KserveGatewayReconciler) Reconcile(ctx context.Context, log logr.Logger
 }
 
 func (r *KserveGatewayReconciler) getDesiredResource(isvc *kservev1beta1.InferenceService) (*istioclientv1beta1.Gateway, error) {
-	hostname,err := getURLWithoutScheme(isvc)
-	if err != nil{
+	hostname, err := getURLWithoutScheme(isvc)
+	if err != nil {
 		return nil, err
 	}
 

controllers/reconcilers/kserve_isvc_service_cert_reconciler.go

@@ -52,6 +52,7 @@ func (r *KserveIsvcServiceReconciler) Cleanup(_ context.Context, _ logr.Logger,
 	return nil
 }
 
+//To support KServe local gateway using HTTPS, each InferenceService (ISVC) needs a certificate. This reconciliation process helps add a serving certificate annotation to the ISVC service.
 func (r *KserveIsvcServiceReconciler) Reconcile(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) error {
 	log.V(1).Info("Reconciling InferenceService Service serving cert")
 

controllers/reconcilers/kserve_serverless_inferenceservice_reconciler.go

@@ -33,7 +33,8 @@ type KserveServerlessInferenceServiceReconciler struct {
 	subResourceReconcilers []SubResourceReconciler
 }
 
-func NewKServeServerlessInferenceServiceReconciler(client client.Client) *KserveServerlessInferenceServiceReconciler {
+func NewKServeServerlessInferenceServiceReconciler(client client.Client, clientReader client.Reader) *KserveServerlessInferenceServiceReconciler {
+
 	subResourceReconciler := []SubResourceReconciler{
 		NewKserveServiceMeshMemberReconciler(client),
 		NewKserveRouteReconciler(client),
@@ -47,7 +48,7 @@ func NewKServeServerlessInferenceServiceReconciler(client client.Client) *Kserve
 		NewKServeNetworkPolicyReconciler(client),
 		NewKserveAuthConfigReconciler(client),
 		NewKserveIsvcServiceReconciler(client),
-		NewKserveGatewayReconciler(client),
+		NewKserveGatewayReconciler(client, clientReader),
 		NewKserveMetricsDashboardReconciler(client),
 	}
 

controllers/suite_test.go

@@ -43,6 +43,8 @@ import (
 	routev1 "github.com/openshift/api/route/v1"
 	istioclientv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/fake"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -157,6 +159,7 @@ var _ = BeforeSuite(func() {
 
 	err = (NewOpenshiftInferenceServiceReconciler(
 		mgr.GetClient(),
+		mgr.GetAPIReader(),
 		ctrl.Log.WithName("controllers").WithName("InferenceService-controller"),
 		false)).
 		SetupWithManager(mgr)
@@ -284,3 +287,7 @@ func createTestNamespaceName() string {
 	}
 	return "test-ns-" + string(b)
 }
+
+func NewFakeClientsetWrapper(fakeClient *fake.Clientset) kubernetes.Interface {
+	return fakeClient
+}

go.mod

@@ -41,6 +41,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect