From 94923b840a54ac0b06ab11324a3c4c8383af6ceb Mon Sep 17 00:00:00 2001 From: Nico Schieder Date: Mon, 21 Nov 2016 10:40:19 +0100 Subject: [PATCH 1/2] Added retry for the TLS secret fetch --- nginx-controller/controller/controller.go | 19 +++++++++++++------ nginx-controller/controller/utils.go | 8 ++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/nginx-controller/controller/controller.go b/nginx-controller/controller/controller.go index d6b90b4409..dc79ebd765 100644 --- a/nginx-controller/controller/controller.go +++ b/nginx-controller/controller/controller.go @@ -285,7 +285,11 @@ func (lbc *LoadBalancerController) syncEndp(key string) { ings := lbc.getIngressForEndpoints(obj) for _, ing := range ings { - ingEx := lbc.createIngress(&ing) + ingEx, err := lbc.createIngress(&ing) + if err != nil { + lbc.ingQueue.requeueAfter(key, err, 5*time.Second) + return + } glog.V(3).Infof("Updating Endpoints for %v/%v", ing.Name, ing.Namespace) name := ing.Namespace + "-" + ing.Name lbc.cnf.UpdateEndpoints(name, &ingEx) @@ -376,7 +380,11 @@ func (lbc *LoadBalancerController) syncIng(key string) { glog.V(2).Infof("Adding or Updating Ingress: %v\n", key) ing := obj.(*extensions.Ingress) - ingEx := lbc.createIngress(ing) + ingEx, err := lbc.createIngress(ing) + if err != nil { + lbc.ingQueue.requeueAfter(key, err, 5*time.Second) + return + } lbc.cnf.AddOrUpdateIngress(name, &ingEx) } } @@ -413,7 +421,7 @@ func (lbc *LoadBalancerController) getIngressForEndpoints(obj interface{}) []ext return ings } -func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) nginx.IngressEx { +func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) (nginx.IngressEx, error) { ingEx := nginx.IngressEx{ Ingress: ing, } @@ -423,8 +431,7 @@ func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) nginx. secretName := tls.SecretName secret, err := lbc.client.Secrets(ing.Namespace).Get(secretName) if err != nil { - glog.Warningf("Error retrieving secret %v for Ingress %v: %v", secretName, ing.Name, err) - continue + return ingEx, fmt.Errorf("Error retrieving secret %v for Ingress %v: %v", secretName, ing.Name, err) } ingEx.Secrets[secretName] = secret } @@ -454,7 +461,7 @@ func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) nginx. } } - return ingEx + return ingEx, nil } func (lbc *LoadBalancerController) getEndpointsForIngressBackend(backend *extensions.IngressBackend, namespace string) ([]string, error) { diff --git a/nginx-controller/controller/utils.go b/nginx-controller/controller/utils.go index 2c57ca7f6d..ae83aae691 100644 --- a/nginx-controller/controller/utils.go +++ b/nginx-controller/controller/utils.go @@ -59,6 +59,14 @@ func (t *taskQueue) requeue(key string, err error) { t.queue.Add(key) } +func (t *taskQueue) requeueAfter(key string, err error, after time.Duration) { + glog.Errorf("Requeuing %v after %s, err %v", key, after.String(), err) + go func(key string, after time.Duration) { + time.Sleep(after) + t.queue.Add(key) + }(key, after) +} + // worker processes work in the queue through sync. func (t *taskQueue) worker() { for { From 46eff6c90ac5e79a35f8c67b24519db59497f924 Mon Sep 17 00:00:00 2001 From: Nico Schieder Date: Fri, 2 Dec 2016 15:00:31 +0100 Subject: [PATCH 2/2] Cleaning up --- nginx-controller/controller/controller.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nginx-controller/controller/controller.go b/nginx-controller/controller/controller.go index 7b99a7b4c4..037810138c 100644 --- a/nginx-controller/controller/controller.go +++ b/nginx-controller/controller/controller.go @@ -289,12 +289,12 @@ func (lbc *LoadBalancerController) syncEndp(key string) { } ingEx, err := lbc.createIngress(&ing) if err != nil { - lbc.ingQueue.requeueAfter(key, err, 5*time.Second) - return + glog.Warningf("Error updating endpoints for %v/%v: %v, skipping", ing.Namespace, ing.Name, err) + continue } glog.V(3).Infof("Updating Endpoints for %v/%v", ing.Name, ing.Namespace) name := ing.Namespace + "-" + ing.Name - lbc.cnf.UpdateEndpoints(name, &ingEx) + lbc.cnf.UpdateEndpoints(name, ingEx) } } @@ -423,7 +423,7 @@ func (lbc *LoadBalancerController) syncIng(key string) { lbc.ingQueue.requeueAfter(key, err, 5*time.Second) return } - lbc.cnf.AddOrUpdateIngress(name, &ingEx) + lbc.cnf.AddOrUpdateIngress(name, ingEx) } } @@ -462,8 +462,8 @@ func (lbc *LoadBalancerController) getIngressForEndpoints(obj interface{}) []ext return ings } -func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) (nginx.IngressEx, error) { - ingEx := nginx.IngressEx{ +func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) (*nginx.IngressEx, error) { + ingEx := &nginx.IngressEx{ Ingress: ing, } @@ -472,7 +472,7 @@ func (lbc *LoadBalancerController) createIngress(ing *extensions.Ingress) (nginx secretName := tls.SecretName secret, err := lbc.client.Secrets(ing.Namespace).Get(secretName) if err != nil { - return ingEx, fmt.Errorf("Error retrieving secret %v for Ingress %v: %v", secretName, ing.Name, err) + return nil, fmt.Errorf("Error retrieving secret %v for Ingress %v: %v", secretName, ing.Name, err) } ingEx.Secrets[secretName] = secret }