From 6a30bd6dbffdf0c9600f339a2e439bc18e4c0c0c Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 8 Jan 2025 10:32:29 +0800 Subject: [PATCH 1/2] Print K8s Resolver Event for Agent Grpc request debugging Signed-off-by: Future-Outlier --- flytestdlib/resolver/k8s_resolver.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/flytestdlib/resolver/k8s_resolver.go b/flytestdlib/resolver/k8s_resolver.go index e6c5f6f480..4b10efc437 100644 --- a/flytestdlib/resolver/k8s_resolver.go +++ b/flytestdlib/resolver/k8s_resolver.go @@ -10,6 +10,7 @@ import ( "google.golang.org/grpc/grpclog" "google.golang.org/grpc/resolver" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" @@ -157,6 +158,16 @@ func (k *kResolver) run() { logger.Infof(k.ctx, "Starting k8s resolver for target: %s", k.target) watcher, err := k.k8sClient.CoreV1().Endpoints(k.target.serviceNamespace).Watch(k.ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + k.target.serviceName}) if err != nil { + logger.Errorf( + k.ctx, + "k8s resolver: failed to create watcher for target [%s]: namespace [%s], service [%s], error [%v]", + k.target, k.target.serviceNamespace, k.target.serviceName, err, + ) + if statusErr, ok := err.(*errors.StatusError); ok { + logger.Errorf(k.ctx, "k8s resolver: status error details: %v", statusErr.ErrStatus) + } + + logger.Infof(k.ctx, "k8s resolver: failed to create watcher: [%v]", err) grpclog.Errorf("k8s resolver: failed to create watcher: %v", err) return } @@ -166,6 +177,11 @@ func (k *kResolver) run() { case <-k.ctx.Done(): return case event, ok := <-watcher.ResultChan(): + logger.Info(k.ctx, "k8s resolver watchet event response: [%v]", event) + logger.Info(k.ctx, "k8s resolver watchet event Object response: [%v]", event.Object) + if event.Object != nil { + logger.Info(k.ctx, "k8s resolver watchet event Object Kind: [%v]", event.Object.GetObjectKind()) + } if !ok { logger.Debugf(k.ctx, "k8s resolver: watcher closed") return From 4796e96da2b0153f01ad616defe60ec3ad017075 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Fri, 10 Jan 2025 09:19:58 +0800 Subject: [PATCH 2/2] update Signed-off-by: Future-Outlier --- flytestdlib/resolver/k8s_resolver.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/flytestdlib/resolver/k8s_resolver.go b/flytestdlib/resolver/k8s_resolver.go index 4b10efc437..07e1883b77 100644 --- a/flytestdlib/resolver/k8s_resolver.go +++ b/flytestdlib/resolver/k8s_resolver.go @@ -155,12 +155,14 @@ func (k *kResolver) resolve(e *v1.Endpoints) { func (k *kResolver) run() { k.wg.Add(1) defer k.wg.Done() - logger.Infof(k.ctx, "Starting k8s resolver for target: %s", k.target) + + logger.Infof(k.ctx, "Starting k8s resolver for target: [%s], service namespace: [%s], service name: [%s]", k.target, k.target.serviceNamespace, k.target.serviceName) + watcher, err := k.k8sClient.CoreV1().Endpoints(k.target.serviceNamespace).Watch(k.ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + k.target.serviceName}) if err != nil { logger.Errorf( k.ctx, - "k8s resolver: failed to create watcher for target [%s]: namespace [%s], service [%s], error [%v]", + "k8s resolver: failed to create watcher for target [%s]: service namespace: [%s], service name: [%s], "+"error [%v]", k.target, k.target.serviceNamespace, k.target.serviceName, err, ) if statusErr, ok := err.(*errors.StatusError); ok { @@ -177,11 +179,7 @@ func (k *kResolver) run() { case <-k.ctx.Done(): return case event, ok := <-watcher.ResultChan(): - logger.Info(k.ctx, "k8s resolver watchet event response: [%v]", event) - logger.Info(k.ctx, "k8s resolver watchet event Object response: [%v]", event.Object) - if event.Object != nil { - logger.Info(k.ctx, "k8s resolver watchet event Object Kind: [%v]", event.Object.GetObjectKind()) - } + logger.Info(k.ctx, "k8s resolver watcher event response: [%v]", event) if !ok { logger.Debugf(k.ctx, "k8s resolver: watcher closed") return