Fix resetting service type to default when not specified (#8165)

These changes ensure that when deleting the type of an http(s) external service declaration in an Elastic resource definition, the service type is properly reset to the default ClusterIP value.

This can be broken down into 3 changes:
- When the type of an external service is not set, it defaults to ClusterIP. Thats seems mandatory to me to be able to detect to reset ClusterIP when no service is configured while there is still the version of the service typed LoadBalancer on the server side. It's ok to do this change because the default value of the Type field is this value (v1/types.go). So, this will not change the current type for existing clusters that have not set the type.
- When an external service is reconciled, we requeue if we get an alreadyExists error. This is because the deletion of the service can take many seconds (e.g.: gcp lb deletion takes >30s), resulting in the creation while the resource is still being deleted.
- When the type of a service changes from something different to the default ClusterIP value, server side changes are not applied. We may lose some case where we could update instead of recreate but it will avoid some cases where certain fields have no sense with the new type.

pkg/controller/common/service_control.go

@@ -88,6 +88,11 @@ func needsUpdate(expected *corev1.Service, reconciled *corev1.Service) bool {
 
 // applyServerSideValues applies any default that may have been set from the reconciled version.
 func applyServerSideValues(expected, reconciled *corev1.Service) {
+	// skip if the service type changes from something different to the default ClusterIP value
+	if reconciled.Spec.Type != corev1.ServiceTypeClusterIP && expected.Spec.Type != reconciled.Spec.Type {
+		return
+	}
+
 	// Type may be defaulted by the api server
 	if expected.Spec.Type == "" {
 		expected.Spec.Type = reconciled.Spec.Type

pkg/controller/common/service_control_test.go

@@ -317,14 +317,12 @@ func Test_applyServerSideValues(t *testing.T) {
 			args: args{
 				expected: corev1.Service{Spec: corev1.ServiceSpec{}},
 				reconciled: corev1.Service{Spec: corev1.ServiceSpec{
-					Type:            corev1.ServiceTypeClusterIP,
 					ClusterIP:       "1.2.3.4",
 					ClusterIPs:      []string{"1.2.3.4"},
 					SessionAffinity: corev1.ServiceAffinityClientIP,
 				}},
 			},
 			want: corev1.Service{Spec: corev1.ServiceSpec{
-				Type:            corev1.ServiceTypeClusterIP,
 				ClusterIP:       "1.2.3.4",
 				ClusterIPs:      []string{"1.2.3.4"},
 				SessionAffinity: corev1.ServiceAffinityClientIP,
@@ -335,14 +333,12 @@ func Test_applyServerSideValues(t *testing.T) {
 			args: args{
 				expected: corev1.Service{Spec: corev1.ServiceSpec{}},
 				reconciled: corev1.Service{Spec: corev1.ServiceSpec{
-					Type:            corev1.ServiceTypeClusterIP,
 					ClusterIP:       "None",
 					ClusterIPs:      []string{"None"},
 					SessionAffinity: corev1.ServiceAffinityClientIP,
 				}},
 			},
 			want: corev1.Service{Spec: corev1.ServiceSpec{
-				Type:            corev1.ServiceTypeClusterIP,
 				ClusterIP:       "None",
 				ClusterIPs:      []string{"None"},
 				SessionAffinity: corev1.ServiceAffinityClientIP,
@@ -531,14 +527,12 @@ func Test_applyServerSideValues(t *testing.T) {
 			args: args{
 				expected: corev1.Service{Spec: corev1.ServiceSpec{}},
 				reconciled: corev1.Service{Spec: corev1.ServiceSpec{
-					Type:            corev1.ServiceTypeClusterIP,
 					ClusterIP:       "1.2.3.4",
 					SessionAffinity: corev1.ServiceAffinityClientIP,
 					IPFamilies:      []corev1.IPFamily{corev1.IPv6Protocol},
 				}},
 			},
 			want: corev1.Service{Spec: corev1.ServiceSpec{
-				Type:            corev1.ServiceTypeClusterIP,
 				ClusterIP:       "1.2.3.4",
 				SessionAffinity: corev1.ServiceAffinityClientIP,
 				IPFamilies:      []corev1.IPFamily{corev1.IPv6Protocol},
@@ -551,13 +545,11 @@ func Test_applyServerSideValues(t *testing.T) {
 					IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol},
 				}},
 				reconciled: corev1.Service{Spec: corev1.ServiceSpec{
-					Type:            corev1.ServiceTypeClusterIP,
 					ClusterIP:       "1.2.3.4",
 					SessionAffinity: corev1.ServiceAffinityClientIP,
 				}},
 			},
 			want: corev1.Service{Spec: corev1.ServiceSpec{
-				Type:            corev1.ServiceTypeClusterIP,
 				ClusterIP:       "1.2.3.4",
 				SessionAffinity: corev1.ServiceAffinityClientIP,
 				IPFamilies:      []corev1.IPFamily{corev1.IPv6Protocol},
@@ -602,7 +594,9 @@ func Test_applyServerSideValues(t *testing.T) {
 		{
 			name: "Reconciled LoadBalancerClass is used if the expected one is empty",
 			args: args{
-				expected: corev1.Service{Spec: corev1.ServiceSpec{}},
+				expected: corev1.Service{Spec: corev1.ServiceSpec{
+					Type: corev1.ServiceTypeLoadBalancer,
+				}},
 				reconciled: corev1.Service{Spec: corev1.ServiceSpec{
 					Type:              corev1.ServiceTypeLoadBalancer,
 					LoadBalancerClass: ptr.To("service.k8s.aws/nlb"),
@@ -643,6 +637,37 @@ func Test_applyServerSideValues(t *testing.T) {
 				Type: corev1.ServiceTypeClusterIP,
 			}},
 		},
+		{
+			name: "Do not apply server side values if Type changed to the default ClusterIP from another type",
+			args: args{
+				expected: corev1.Service{Spec: corev1.ServiceSpec{
+					Type: corev1.ServiceTypeClusterIP,
+				}},
+				reconciled: corev1.Service{Spec: corev1.ServiceSpec{
+					Type:                  corev1.ServiceTypeNodePort,
+					ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyCluster,
+				}},
+			},
+			want: corev1.Service{Spec: corev1.ServiceSpec{
+				Type: corev1.ServiceTypeClusterIP,
+			}},
+		},
+		{
+			name: "Apply server side values if Type changed from the default ClusterIP to another type",
+			args: args{
+				expected: corev1.Service{Spec: corev1.ServiceSpec{
+					Type: corev1.ServiceTypeNodePort,
+				}},
+				reconciled: corev1.Service{Spec: corev1.ServiceSpec{
+					Type:      corev1.ServiceTypeNodePort,
+					ClusterIP: "1.2.3.4",
+				}},
+			},
+			want: corev1.Service{Spec: corev1.ServiceSpec{
+				Type:      corev1.ServiceTypeNodePort,
+				ClusterIP: "1.2.3.4",
+			}},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/controller/elasticsearch/driver/driver.go

@@ -145,6 +145,9 @@ func (d *defaultDriver) Reconcile(ctx context.Context) *reconciler.Results {
 
 	externalService, err := common.ReconcileService(ctx, d.Client, services.NewExternalService(d.ES), &d.ES)
 	if err != nil {
+		if k8serrors.IsAlreadyExists(err) {
+			return results.WithReconciliationState(defaultRequeue.WithReason(fmt.Sprintf("Pending %s service recreation", services.ExternalServiceName(d.ES.Name))))
+		}
 		return results.WithError(err)
 	}
 

pkg/controller/elasticsearch/services/services.go

@@ -115,6 +115,10 @@ func NewExternalService(es esv1.Elasticsearch) *corev1.Service {
 	svc.ObjectMeta.Namespace = es.Namespace
 	svc.ObjectMeta.Name = ExternalServiceName(es.Name)
 
+	// defaults to ClusterIP if not set
+	if svc.Spec.Type == "" {
+		svc.Spec.Type = corev1.ServiceTypeClusterIP
+	}
 	labels := label.NewLabels(nsn)
 	ports := []corev1.ServicePort{
 		{

pkg/controller/elasticsearch/services/services_test.go

@@ -68,6 +68,10 @@ func TestNewExternalService(t *testing.T) {
 		httpConf commonv1.HTTPConfig
 		wantSvc  func() corev1.Service
 	}{
+		{
+			name:    "default clusterIP service",
+			wantSvc: mkHTTPSService,
+		},
 		{
 			name: "no TLS",
 			httpConf: commonv1.HTTPConfig{
@@ -92,11 +96,7 @@ func TestNewExternalService(t *testing.T) {
 					},
 				},
 			},
-			wantSvc: func() corev1.Service {
-				svc := mkHTTPService()
-				svc.Spec.Ports[0].Name = "https"
-				return svc
-			},
+			wantSvc: mkHTTPSService,
 		},
 		{
 			name: "user-provided certificate",
@@ -107,11 +107,7 @@ func TestNewExternalService(t *testing.T) {
 					},
 				},
 			},
-			wantSvc: func() corev1.Service {
-				svc := mkHTTPService()
-				svc.Spec.Ports[0].Name = "https"
-				return svc
-			},
+			wantSvc: mkHTTPSService,
 		},
 	}
 
@@ -171,6 +167,7 @@ func mkHTTPService() corev1.Service {
 			},
 		},
 		Spec: corev1.ServiceSpec{
+			Type: corev1.ServiceTypeClusterIP,
 			Ports: []corev1.ServicePort{
 				{
 					Name:     "http",
@@ -186,6 +183,12 @@ func mkHTTPService() corev1.Service {
 	}
 }
 
+func mkHTTPSService() corev1.Service {
+	svc := mkHTTPService()
+	svc.Spec.Ports[0].Name = "https"
+	return svc
+}
+
 func mkTransportService() corev1.Service {
 	return corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{