diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index cb60f0ca6e..cba3b24b99 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -65,9 +65,9 @@ func (in *Exporter) DeepCopy() *Exporter { func (in *Instrumentation) DeepCopyInto(out *Instrumentation) { *out = *in out.Status = in.Status - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) out.TypeMeta = in.TypeMeta + in.Spec.DeepCopyInto(&out.Spec) + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Instrumentation. @@ -275,6 +275,14 @@ func (in *OpenTelemetryCollectorList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenTelemetryCollectorSpec) DeepCopyInto(out *OpenTelemetryCollectorSpec) { *out = *in + in.Resources.DeepCopyInto(&out.Resources) + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Args != nil { in, out := &in.Args, &out.Args *out = make(map[string]string, len(*in)) @@ -297,7 +305,6 @@ func (in *OpenTelemetryCollectorSpec) DeepCopyInto(out *OpenTelemetryCollectorSp *out = new(int32) **out = **in } - out.TargetAllocator = in.TargetAllocator if in.SecurityContext != nil { in, out := &in.SecurityContext, &out.SecurityContext *out = new(v1.SecurityContext) @@ -308,13 +315,14 @@ func (in *OpenTelemetryCollectorSpec) DeepCopyInto(out *OpenTelemetryCollectorSp *out = new(v1.PodSecurityContext) (*in).DeepCopyInto(*out) } - if in.VolumeClaimTemplates != nil { - in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates - *out = make([]v1.PersistentVolumeClaim, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) + if in.PodAnnotations != nil { + in, out := &in.PodAnnotations, &out.PodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val } } + out.TargetAllocator = in.TargetAllocator if in.VolumeMounts != nil { in, out := &in.VolumeMounts, &out.VolumeMounts *out = make([]v1.VolumeMount, len(*in)) @@ -322,13 +330,6 @@ func (in *OpenTelemetryCollectorSpec) DeepCopyInto(out *OpenTelemetryCollectorSp (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Volumes != nil { - in, out := &in.Volumes, &out.Volumes - *out = make([]v1.Volume, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } if in.Ports != nil { in, out := &in.Ports, &out.Ports *out = make([]v1.ServicePort, len(*in)) @@ -350,7 +351,13 @@ func (in *OpenTelemetryCollectorSpec) DeepCopyInto(out *OpenTelemetryCollectorSp (*in)[i].DeepCopyInto(&(*out)[i]) } } - in.Resources.DeepCopyInto(&out.Resources) + if in.VolumeClaimTemplates != nil { + in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates + *out = make([]v1.PersistentVolumeClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]v1.Toleration, len(*in)) @@ -358,18 +365,11 @@ func (in *OpenTelemetryCollectorSpec) DeepCopyInto(out *OpenTelemetryCollectorSp (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.NodeSelector != nil { - in, out := &in.NodeSelector, &out.NodeSelector - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.PodAnnotations != nil { - in, out := &in.PodAnnotations, &out.PodAnnotations - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]v1.Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) } } } diff --git a/controllers/opentelemetrycollector_controller.go b/controllers/opentelemetrycollector_controller.go index 7854514df5..4250d677b7 100644 --- a/controllers/opentelemetrycollector_controller.go +++ b/controllers/opentelemetrycollector_controller.go @@ -21,7 +21,8 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" - autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -31,6 +32,7 @@ import ( "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" "github.com/open-telemetry/opentelemetry-operator/internal/config" + "github.com/open-telemetry/opentelemetry-operator/pkg/autodetect" "github.com/open-telemetry/opentelemetry-operator/pkg/collector/reconcile" ) @@ -173,14 +175,25 @@ func (r *OpenTelemetryCollectorReconciler) RunTasks(ctx context.Context, params // SetupWithManager tells the manager what our controller is interested in. func (r *OpenTelemetryCollectorReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + err := r.config.AutoDetect() // We need to call this so we can get the correct autodetect version + if err != nil { + return err + } + builder := ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.OpenTelemetryCollector{}). Owns(&corev1.ConfigMap{}). Owns(&corev1.ServiceAccount{}). Owns(&corev1.Service{}). Owns(&appsv1.Deployment{}). - Owns(&autoscalingv1.HorizontalPodAutoscaler{}). Owns(&appsv1.DaemonSet{}). - Owns(&appsv1.StatefulSet{}). - Complete(r) + Owns(&appsv1.StatefulSet{}) + + autoscalingVersion := r.config.AutoscalingVersion() + if autoscalingVersion == autodetect.AutoscalingVersionV2 { + builder = builder.Owns(&autoscalingv2.HorizontalPodAutoscaler{}) + } else { + builder = builder.Owns(&autoscalingv2beta2.HorizontalPodAutoscaler{}) + } + + return builder.Complete(r) } diff --git a/controllers/opentelemetrycollector_controller_test.go b/controllers/opentelemetrycollector_controller_test.go index 981449f7e5..b9324225b1 100644 --- a/controllers/opentelemetrycollector_controller_test.go +++ b/controllers/opentelemetrycollector_controller_test.go @@ -36,14 +36,21 @@ import ( "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" "github.com/open-telemetry/opentelemetry-operator/controllers" "github.com/open-telemetry/opentelemetry-operator/internal/config" + "github.com/open-telemetry/opentelemetry-operator/pkg/autodetect" "github.com/open-telemetry/opentelemetry-operator/pkg/collector/reconcile" + "github.com/open-telemetry/opentelemetry-operator/pkg/platform" ) var logger = logf.Log.WithName("unit-tests") +var mockAutoDetector = &mockAutoDetect{ + HPAVersionFunc: func() (autodetect.AutoscalingVersion, error) { + return autodetect.AutoscalingVersionV2Beta2, nil + }, +} func TestNewObjectsOnReconciliation(t *testing.T) { // prepare - cfg := config.New(config.WithCollectorImage("default-collector"), config.WithTargetAllocatorImage("default-ta-allocator")) + cfg := config.New(config.WithCollectorImage("default-collector"), config.WithTargetAllocatorImage("default-ta-allocator"), config.WithAutoDetect(mockAutoDetector)) nsn := types.NamespacedName{Name: "my-instance", Namespace: "default"} reconciler := controllers.NewReconciler(controllers.Params{ Client: k8sClient, @@ -129,7 +136,7 @@ func TestNewObjectsOnReconciliation(t *testing.T) { func TestNewStatefulSetObjectsOnReconciliation(t *testing.T) { // prepare - cfg := config.New() + cfg := config.New(config.WithAutoDetect(mockAutoDetector)) nsn := types.NamespacedName{Name: "my-instance", Namespace: "default"} reconciler := controllers.NewReconciler(controllers.Params{ Client: k8sClient, @@ -341,3 +348,21 @@ func TestRegisterWithManager(t *testing.T) { // verify assert.NoError(t, err) } + +var _ autodetect.AutoDetect = (*mockAutoDetect)(nil) + +type mockAutoDetect struct { + PlatformFunc func() (platform.Platform, error) + HPAVersionFunc func() (autodetect.AutoscalingVersion, error) +} + +func (m *mockAutoDetect) HPAVersion() (autodetect.AutoscalingVersion, error) { + return m.HPAVersionFunc() +} + +func (m *mockAutoDetect) Platform() (platform.Platform, error) { + if m.PlatformFunc != nil { + return m.PlatformFunc() + } + return platform.Unknown, nil +} diff --git a/internal/config/main.go b/internal/config/main.go index f20af554bb..e2eee51a22 100644 --- a/internal/config/main.go +++ b/internal/config/main.go @@ -49,6 +49,7 @@ type Config struct { labelsFilter []string platform platform.Platform autoDetectFrequency time.Duration + autoscalingVersion autodetect.AutoscalingVersion } // New constructs a new configuration based on the given options. @@ -61,6 +62,7 @@ func New(opts ...Option) Config { logger: logf.Log.WithName("config"), platform: platform.Unknown, version: version.Get(), + autoscalingVersion: autodetect.DefaultAutoscalingVersion, } for _, opt := range opts { opt(&o) @@ -81,6 +83,7 @@ func New(opts ...Option) Config { autoInstrumentationPythonImage: o.autoInstrumentationPythonImage, autoInstrumentationDotNetImage: o.autoInstrumentationDotNetImage, labelsFilter: o.labelsFilter, + autoscalingVersion: o.autoscalingVersion, } } @@ -132,6 +135,13 @@ func (c *Config) AutoDetect() error { } } + hpaVersion, err := c.autoDetect.HPAVersion() + if err != nil { + return err + } + c.autoscalingVersion = hpaVersion + c.logger.Info("In Autodetect, Set HPA version to [", c.autoscalingVersion, "] from [", hpaVersion, "]") + return nil } @@ -160,6 +170,11 @@ func (c *Config) Platform() platform.Platform { return c.platform } +// AutoscalingVersion represents the preferred version of autoscaling. +func (c *Config) AutoscalingVersion() autodetect.AutoscalingVersion { + return c.autoscalingVersion +} + // AutoInstrumentationJavaImage returns OpenTelemetry Java auto-instrumentation container image. func (c *Config) AutoInstrumentationJavaImage() string { return c.autoInstrumentationJavaImage diff --git a/internal/config/main_test.go b/internal/config/main_test.go index ba981ab2dd..fbdc6d8995 100644 --- a/internal/config/main_test.go +++ b/internal/config/main_test.go @@ -102,6 +102,10 @@ type mockAutoDetect struct { PlatformFunc func() (platform.Platform, error) } +func (m *mockAutoDetect) HPAVersion() (autodetect.AutoscalingVersion, error) { + return autodetect.DefaultAutoscalingVersion, nil +} + func (m *mockAutoDetect) Platform() (platform.Platform, error) { if m.PlatformFunc != nil { return m.PlatformFunc() diff --git a/internal/config/options.go b/internal/config/options.go index beef2b4b83..a3846ce156 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -45,6 +45,7 @@ type options struct { labelsFilter []string platform platform.Platform autoDetectFrequency time.Duration + autoscalingVersion autodetect.AutoscalingVersion } func WithAutoDetect(a autodetect.AutoDetect) Option { diff --git a/pkg/autodetect/main.go b/pkg/autodetect/main.go index 487d5b4a29..57d6748f6a 100644 --- a/pkg/autodetect/main.go +++ b/pkg/autodetect/main.go @@ -16,6 +16,9 @@ package autodetect import ( + "errors" + "sort" + "k8s.io/client-go/discovery" "k8s.io/client-go/rest" @@ -27,12 +30,23 @@ var _ AutoDetect = (*autoDetect)(nil) // AutoDetect provides an assortment of routines that auto-detect traits based on the runtime. type AutoDetect interface { Platform() (platform.Platform, error) + HPAVersion() (AutoscalingVersion, error) } type autoDetect struct { dcl discovery.DiscoveryInterface } +type AutoscalingVersion int + +const ( + AutoscalingVersionV2 AutoscalingVersion = iota + AutoscalingVersionV2Beta2 + AutoscalingVersionUnknown +) + +const DefaultAutoscalingVersion = AutoscalingVersionV2 + // New creates a new auto-detection worker, using the given client when talking to the current cluster. func New(restConfig *rest.Config) (AutoDetect, error) { dcl, err := discovery.NewDiscoveryClientForConfig(restConfig) @@ -64,3 +78,51 @@ func (a *autoDetect) Platform() (platform.Platform, error) { return platform.Kubernetes, nil } + +func (a *autoDetect) HPAVersion() (AutoscalingVersion, error) { + apiList, err := a.dcl.ServerGroups() + if err != nil { + return AutoscalingVersionUnknown, err + } + + for _, apiGroup := range apiList.Groups { + if apiGroup.Name == "autoscaling" { + // Sort this so we can make sure to get v2 before v2beta2 + versions := apiGroup.Versions + sort.Slice(versions, func(i, j int) bool { + return versions[i].Version < versions[j].Version + }) + + for _, version := range versions { + if version.Version == "v2" || version.Version == "v2beta2" { + return toAutoscalingVersion(version.Version), nil + } + } + return AutoscalingVersionUnknown, errors.New("Failed to find appropriate version of apiGroup autoscaling, only v2 and v2beta2 are supported") + } + } + + return AutoscalingVersionUnknown, errors.New("Failed to find apiGroup autoscaling") +} + +func (v AutoscalingVersion) String() string { + switch v { + case AutoscalingVersionV2: + return "v2" + case AutoscalingVersionV2Beta2: + return "v2beta2" + case AutoscalingVersionUnknown: + return "unknown" + } + return "unknown" +} + +func toAutoscalingVersion(version string) AutoscalingVersion { + switch version { + case "v2": + return AutoscalingVersionV2Beta2 + case "v2beta2": + return AutoscalingVersionV2Beta2 + } + return AutoscalingVersionUnknown +} diff --git a/pkg/autodetect/main_test.go b/pkg/autodetect/main_test.go index 6ffc1b9e79..42be4eba1a 100644 --- a/pkg/autodetect/main_test.go +++ b/pkg/autodetect/main_test.go @@ -88,3 +88,9 @@ func TestUnknownPlatformOnError(t *testing.T) { assert.Error(t, err) assert.Equal(t, platform.Unknown, plt) } + +func TestAutoscalingVersionToString(t *testing.T) { + assert.Equal(t, "v2", autodetect.AutoscalingVersionV2.String()) + assert.Equal(t, "v2beta2", autodetect.AutoscalingVersionV2Beta2.String()) + assert.Equal(t, "unknown", autodetect.AutoscalingVersionUnknown.String()) +} diff --git a/pkg/collector/horizontalpodautoscaler.go b/pkg/collector/horizontalpodautoscaler.go index 7c7003fb48..0cdd3dca20 100644 --- a/pkg/collector/horizontalpodautoscaler.go +++ b/pkg/collector/horizontalpodautoscaler.go @@ -16,40 +16,92 @@ package collector import ( "github.com/go-logr/logr" - autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" "github.com/open-telemetry/opentelemetry-operator/internal/config" + "github.com/open-telemetry/opentelemetry-operator/pkg/autodetect" "github.com/open-telemetry/opentelemetry-operator/pkg/naming" ) const defaultCPUTarget int32 = 90 -func HorizontalPodAutoscaler(cfg config.Config, logger logr.Logger, otelcol v1alpha1.OpenTelemetryCollector) autoscalingv1.HorizontalPodAutoscaler { +func HorizontalPodAutoscaler(cfg config.Config, logger logr.Logger, otelcol v1alpha1.OpenTelemetryCollector) client.Object { + autoscalingVersion := cfg.AutoscalingVersion() + labels := Labels(otelcol, cfg.LabelsFilter()) labels["app.kubernetes.io/name"] = naming.Collector(otelcol) annotations := Annotations(otelcol) cpuTarget := defaultCPUTarget + var result client.Object + + objectMeta := metav1.ObjectMeta{ + Name: naming.HorizontalPodAutoscaler(otelcol), + Namespace: otelcol.Namespace, + Labels: labels, + Annotations: annotations, + } + + if autoscalingVersion == autodetect.AutoscalingVersionV2Beta2 { + targetCPUUtilization := autoscalingv2beta2.MetricSpec{ + Type: autoscalingv2beta2.ResourceMetricSourceType, + Resource: &autoscalingv2beta2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.UtilizationMetricType, + AverageUtilization: &cpuTarget, + }, + }, + } + metrics := []autoscalingv2beta2.MetricSpec{targetCPUUtilization} - return autoscalingv1.HorizontalPodAutoscaler{ - ObjectMeta: metav1.ObjectMeta{ - Name: naming.HorizontalPodAutoscaler(otelcol), - Namespace: otelcol.Namespace, - Labels: labels, - Annotations: annotations, - }, - Spec: autoscalingv1.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscalingv1.CrossVersionObjectReference{ - APIVersion: v1alpha1.GroupVersion.String(), - Kind: "OpenTelemetryCollector", - Name: naming.OpenTelemetryCollector(otelcol), + autoscaler := autoscalingv2beta2.HorizontalPodAutoscaler{ + ObjectMeta: objectMeta, + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ + APIVersion: v1alpha1.GroupVersion.String(), + Kind: "OpenTelemetryCollector", + Name: naming.OpenTelemetryCollector(otelcol), + }, + MinReplicas: otelcol.Spec.Replicas, + MaxReplicas: *otelcol.Spec.MaxReplicas, + Metrics: metrics, + }, + } + result = &autoscaler + } else { + targetCPUUtilization := autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: &cpuTarget, + }, }, + } + metrics := []autoscalingv2.MetricSpec{targetCPUUtilization} - MinReplicas: otelcol.Spec.Replicas, - MaxReplicas: *otelcol.Spec.MaxReplicas, - TargetCPUUtilizationPercentage: &cpuTarget, - }, + autoscaler := autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: objectMeta, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: v1alpha1.GroupVersion.String(), + Kind: "OpenTelemetryCollector", + Name: naming.OpenTelemetryCollector(otelcol), + }, + MinReplicas: otelcol.Spec.Replicas, + MaxReplicas: *otelcol.Spec.MaxReplicas, + Metrics: metrics, + }, + } + result = &autoscaler } + + return result } diff --git a/pkg/collector/horizontalpodautoscaler_test.go b/pkg/collector/horizontalpodautoscaler_test.go index b168816549..6aaa71ca5c 100644 --- a/pkg/collector/horizontalpodautoscaler_test.go +++ b/pkg/collector/horizontalpodautoscaler_test.go @@ -18,15 +18,27 @@ import ( "testing" "github.com/stretchr/testify/assert" + autoscalingv2 "k8s.io/api/autoscaling/v2" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" "github.com/open-telemetry/opentelemetry-operator/internal/config" + "github.com/open-telemetry/opentelemetry-operator/pkg/autodetect" . "github.com/open-telemetry/opentelemetry-operator/pkg/collector" + "github.com/open-telemetry/opentelemetry-operator/pkg/platform" ) func TestHPA(t *testing.T) { - // prepare + type test struct { + name string + autoscalingVersion autodetect.AutoscalingVersion + } + v2Test := test{autodetect.AutoscalingVersionV2.String(), autodetect.AutoscalingVersionV2} + v2beta2Test := test{autodetect.AutoscalingVersionV2Beta2.String(), autodetect.AutoscalingVersionV2Beta2} + tests := []test{v2Test, v2beta2Test} + var minReplicas int32 = 3 var maxReplicas int32 = 5 @@ -40,13 +52,59 @@ func TestHPA(t *testing.T) { }, } - cfg := config.New() - hpa := HorizontalPodAutoscaler(cfg, logger, otelcol) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mockAutoDetector := &mockAutoDetect{ + HPAVersionFunc: func() (autodetect.AutoscalingVersion, error) { + return test.autoscalingVersion, nil + }, + } + configuration := config.New(config.WithAutoDetect(mockAutoDetector)) + err := configuration.AutoDetect() + assert.NoError(t, err) + raw := HorizontalPodAutoscaler(configuration, logger, otelcol) + + if configuration.AutoscalingVersion() == autodetect.AutoscalingVersionV2Beta2 { + hpa := raw.(*autoscalingv2beta2.HorizontalPodAutoscaler) + + // verify + assert.Equal(t, "my-instance-collector", hpa.Name) + assert.Equal(t, "my-instance-collector", hpa.Labels["app.kubernetes.io/name"]) + assert.Equal(t, int32(3), *hpa.Spec.MinReplicas) + assert.Equal(t, int32(5), hpa.Spec.MaxReplicas) + assert.Equal(t, 1, len(hpa.Spec.Metrics)) + assert.Equal(t, corev1.ResourceCPU, hpa.Spec.Metrics[0].Resource.Name) + assert.Equal(t, int32(90), *hpa.Spec.Metrics[0].Resource.Target.AverageUtilization) + } else { + hpa := raw.(*autoscalingv2.HorizontalPodAutoscaler) + + // verify + assert.Equal(t, "my-instance-collector", hpa.Name) + assert.Equal(t, "my-instance-collector", hpa.Labels["app.kubernetes.io/name"]) + assert.Equal(t, int32(3), *hpa.Spec.MinReplicas) + assert.Equal(t, int32(5), hpa.Spec.MaxReplicas) + assert.Equal(t, 1, len(hpa.Spec.Metrics)) + assert.Equal(t, corev1.ResourceCPU, hpa.Spec.Metrics[0].Resource.Name) + assert.Equal(t, int32(90), *hpa.Spec.Metrics[0].Resource.Target.AverageUtilization) + } + }) + } +} - // verify - assert.Equal(t, "my-instance-collector", hpa.Name) - assert.Equal(t, "my-instance-collector", hpa.Labels["app.kubernetes.io/name"]) - assert.Equal(t, int32(3), *hpa.Spec.MinReplicas) - assert.Equal(t, int32(5), hpa.Spec.MaxReplicas) - assert.Equal(t, int32(90), *hpa.Spec.TargetCPUUtilizationPercentage) +var _ autodetect.AutoDetect = (*mockAutoDetect)(nil) + +type mockAutoDetect struct { + PlatformFunc func() (platform.Platform, error) + HPAVersionFunc func() (autodetect.AutoscalingVersion, error) +} + +func (m *mockAutoDetect) HPAVersion() (autodetect.AutoscalingVersion, error) { + return m.HPAVersionFunc() +} + +func (m *mockAutoDetect) Platform() (platform.Platform, error) { + if m.PlatformFunc != nil { + return m.PlatformFunc() + } + return platform.Unknown, nil } diff --git a/pkg/collector/reconcile/horizontalpodautoscaler.go b/pkg/collector/reconcile/horizontalpodautoscaler.go index 606f4dffce..302f3c4a5a 100644 --- a/pkg/collector/reconcile/horizontalpodautoscaler.go +++ b/pkg/collector/reconcile/horizontalpodautoscaler.go @@ -18,12 +18,15 @@ import ( "context" "fmt" - autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/open-telemetry/opentelemetry-operator/pkg/autodetect" "github.com/open-telemetry/opentelemetry-operator/pkg/collector" ) @@ -31,7 +34,7 @@ import ( // HorizontalPodAutoscaler reconciles HorizontalPodAutoscalers if autoscale is true and replicas is nil. func HorizontalPodAutoscalers(ctx context.Context, params Params) error { - desired := []autoscalingv1.HorizontalPodAutoscaler{} + desired := []client.Object{} // check if autoscale mode is on, e.g MaxReplicas is not nil if params.Instance.Spec.MaxReplicas != nil { @@ -51,52 +54,48 @@ func HorizontalPodAutoscalers(ctx context.Context, params Params) error { return nil } -func expectedHorizontalPodAutoscalers(ctx context.Context, params Params, expected []autoscalingv1.HorizontalPodAutoscaler) error { - one := int32(1) +func expectedHorizontalPodAutoscalers(ctx context.Context, params Params, expected []client.Object) error { + autoscalingVersion := params.Config.AutoscalingVersion() + var existing client.Object + if autoscalingVersion == autodetect.AutoscalingVersionV2Beta2 { + existing = &autoscalingv2beta2.HorizontalPodAutoscaler{} + } else { + existing = &autoscalingv2.HorizontalPodAutoscaler{} + } + for _, obj := range expected { - desired := obj + desired, _ := meta.Accessor(obj) - if err := controllerutil.SetControllerReference(¶ms.Instance, &desired, params.Scheme); err != nil { + if err := controllerutil.SetControllerReference(¶ms.Instance, desired, params.Scheme); err != nil { return fmt.Errorf("failed to set controller reference: %w", err) } - existing := &autoscalingv1.HorizontalPodAutoscaler{} - nns := types.NamespacedName{Namespace: desired.Namespace, Name: desired.Name} + nns := types.NamespacedName{Namespace: desired.GetNamespace(), Name: desired.GetName()} err := params.Client.Get(ctx, nns, existing) if k8serrors.IsNotFound(err) { - if err := params.Client.Create(ctx, &desired); err != nil { + if err := params.Client.Create(ctx, obj); err != nil { return fmt.Errorf("failed to create: %w", err) } - params.Log.V(2).Info("created", "hpa.name", desired.Name, "hpa.namespace", desired.Namespace) + params.Log.V(2).Info("created", "hpa.name", desired.GetName(), "hpa.namespace", desired.GetNamespace()) continue } else if err != nil { return fmt.Errorf("failed to get %w", err) } - updated := existing.DeepCopy() - if updated.Annotations == nil { - updated.Annotations = map[string]string{} - } - if updated.Labels == nil { - updated.Labels = map[string]string{} - } + updated := existing.DeepCopyObject().(client.Object) + updated.SetOwnerReferences(desired.GetOwnerReferences()) + setAutoscalerSpec(params, autoscalingVersion, updated) - updated.OwnerReferences = desired.OwnerReferences - if params.Instance.Spec.MaxReplicas != nil { - updated.Spec.MaxReplicas = *params.Instance.Spec.MaxReplicas - if params.Instance.Spec.MinReplicas != nil { - updated.Spec.MinReplicas = params.Instance.Spec.MinReplicas - } else { - updated.Spec.MinReplicas = &one - } + annotations := updated.GetAnnotations() + for k, v := range desired.GetAnnotations() { + annotations[k] = v } - - for k, v := range desired.Annotations { - updated.Annotations[k] = v - } - for k, v := range desired.Labels { - updated.Labels[k] = v + updated.SetAnnotations(annotations) + labels := updated.GetLabels() + for k, v := range desired.GetLabels() { + labels[k] = v } + updated.SetLabels(labels) patch := client.MergeFrom(existing) @@ -104,13 +103,36 @@ func expectedHorizontalPodAutoscalers(ctx context.Context, params Params, expect return fmt.Errorf("failed to apply changes: %w", err) } - params.Log.V(2).Info("applied", "hpa.name", desired.Name, "hpa.namespace", desired.Namespace) + params.Log.V(2).Info("applied", "hpa.name", desired.GetName(), "hpa.namespace", desired.GetNamespace()) } return nil } -func deleteHorizontalPodAutoscalers(ctx context.Context, params Params, expected []autoscalingv1.HorizontalPodAutoscaler) error { +func setAutoscalerSpec(params Params, autoscalingVersion autodetect.AutoscalingVersion, updated client.Object) { + one := int32(1) + if params.Instance.Spec.MaxReplicas != nil { + if autoscalingVersion == autodetect.AutoscalingVersionV2Beta2 { + updated.(*autoscalingv2beta2.HorizontalPodAutoscaler).Spec.MaxReplicas = *params.Instance.Spec.MaxReplicas + if params.Instance.Spec.MinReplicas != nil { + updated.(*autoscalingv2beta2.HorizontalPodAutoscaler).Spec.MinReplicas = params.Instance.Spec.MinReplicas + } else { + updated.(*autoscalingv2beta2.HorizontalPodAutoscaler).Spec.MinReplicas = &one + } + } else { + updated.(*autoscalingv2.HorizontalPodAutoscaler).Spec.MaxReplicas = *params.Instance.Spec.MaxReplicas + if params.Instance.Spec.MinReplicas != nil { + updated.(*autoscalingv2.HorizontalPodAutoscaler).Spec.MinReplicas = params.Instance.Spec.MinReplicas + } else { + updated.(*autoscalingv2.HorizontalPodAutoscaler).Spec.MinReplicas = &one + } + } + } +} + +func deleteHorizontalPodAutoscalers(ctx context.Context, params Params, expected []client.Object) error { + autoscalingVersion := params.Config.AutoscalingVersion() + opts := []client.ListOption{ client.InNamespace(params.Instance.Namespace), client.MatchingLabels(map[string]string{ @@ -119,26 +141,53 @@ func deleteHorizontalPodAutoscalers(ctx context.Context, params Params, expected }), } - list := &autoscalingv1.HorizontalPodAutoscalerList{} - if err := params.Client.List(ctx, list, opts...); err != nil { - return fmt.Errorf("failed to list: %w", err) - } + if autoscalingVersion == autodetect.AutoscalingVersionV2Beta2 { + list := &autoscalingv2beta2.HorizontalPodAutoscalerList{} + if err := params.Client.List(ctx, list, opts...); err != nil { + return fmt.Errorf("failed to list: %w", err) + } + + for i := range list.Items { + existing := list.Items[i] + del := true + for _, k := range expected { + keep := k.(*autoscalingv2beta2.HorizontalPodAutoscaler) + if keep.Name == existing.Name && keep.Namespace == existing.Namespace { + del = false + break + } + } - for i := range list.Items { - existing := list.Items[i] - del := true - for _, keep := range expected { - if keep.Name == existing.Name && keep.Namespace == existing.Namespace { - del = false - break + if del { + if err := params.Client.Delete(ctx, &existing); err != nil { + return fmt.Errorf("failed to delete: %w", err) + } + params.Log.V(2).Info("deleted", "hpa.name", existing.Name, "hpa.namespace", existing.Namespace) } } + } else { + list := &autoscalingv2.HorizontalPodAutoscalerList{} + if err := params.Client.List(ctx, list, opts...); err != nil { + return fmt.Errorf("failed to list: %w", err) + } + + for i := range list.Items { + existing := list.Items[i] + del := true + for _, k := range expected { + keep := k.(*autoscalingv2.HorizontalPodAutoscaler) + if keep.Name == existing.Name && keep.Namespace == existing.Namespace { + del = false + break + } + } - if del { - if err := params.Client.Delete(ctx, &existing); err != nil { - return fmt.Errorf("failed to delete: %w", err) + if del { + if err := params.Client.Delete(ctx, &existing); err != nil { + return fmt.Errorf("failed to delete: %w", err) + } + params.Log.V(2).Info("deleted", "hpa.name", existing.Name, "hpa.namespace", existing.Namespace) } - params.Log.V(2).Info("deleted", "hpa.name", existing.Name, "hpa.namespace", existing.Namespace) } } diff --git a/pkg/collector/reconcile/horizontalpodautoscaler_test.go b/pkg/collector/reconcile/horizontalpodautoscaler_test.go index 62b901da37..af2737c518 100644 --- a/pkg/collector/reconcile/horizontalpodautoscaler_test.go +++ b/pkg/collector/reconcile/horizontalpodautoscaler_test.go @@ -22,27 +22,34 @@ import ( "github.com/stretchr/testify/assert" v1 "k8s.io/api/apps/v1" - autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" "github.com/open-telemetry/opentelemetry-operator/internal/config" + "github.com/open-telemetry/opentelemetry-operator/pkg/autodetect" "github.com/open-telemetry/opentelemetry-operator/pkg/collector" + "github.com/open-telemetry/opentelemetry-operator/pkg/platform" ) func TestExpectedHPA(t *testing.T) { - params := paramsWithHPA() - expectedHPA := collector.HorizontalPodAutoscaler(params.Config, logger, params.Instance) + params := paramsWithHPA(autodetect.AutoscalingVersionV2Beta2) + err := params.Config.AutoDetect() + assert.NoError(t, err) + autoscalingVersion := params.Config.AutoscalingVersion() + expectedHPA := collector.HorizontalPodAutoscaler(params.Config, logger, params.Instance) t.Run("should create HPA", func(t *testing.T) { - err := expectedHorizontalPodAutoscalers(context.Background(), params, []autoscalingv1.HorizontalPodAutoscaler{expectedHPA}) + err = expectedHorizontalPodAutoscalers(context.Background(), params, []client.Object{expectedHPA}) assert.NoError(t, err) - exists, err := populateObjectIfExists(t, &autoscalingv1.HorizontalPodAutoscaler{}, types.NamespacedName{Namespace: "default", Name: "test-collector"}) + exists, err := populateObjectIfExists(t, &autoscalingv2beta2.HorizontalPodAutoscaler{}, types.NamespacedName{Namespace: "default", Name: "test-collector"}) assert.NoError(t, err) assert.True(t, exists) }) @@ -50,26 +57,42 @@ func TestExpectedHPA(t *testing.T) { t.Run("should update HPA", func(t *testing.T) { minReplicas := int32(1) maxReplicas := int32(3) - updateParms := paramsWithHPA() + updateParms := paramsWithHPA(autodetect.AutoscalingVersionV2Beta2) updateParms.Instance.Spec.Replicas = &minReplicas updateParms.Instance.Spec.MaxReplicas = &maxReplicas updatedHPA := collector.HorizontalPodAutoscaler(updateParms.Config, logger, updateParms.Instance) - createObjectIfNotExists(t, "test-collector", &updatedHPA) - err := expectedHorizontalPodAutoscalers(context.Background(), updateParms, []autoscalingv1.HorizontalPodAutoscaler{updatedHPA}) - assert.NoError(t, err) - - actual := autoscalingv1.HorizontalPodAutoscaler{} - exists, err := populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"}) - - assert.NoError(t, err) - assert.True(t, exists) - assert.Equal(t, int32(1), *actual.Spec.MinReplicas) - assert.Equal(t, int32(3), actual.Spec.MaxReplicas) + if autoscalingVersion == autodetect.AutoscalingVersionV2Beta2 { + updatedAutoscaler := *updatedHPA.(*autoscalingv2beta2.HorizontalPodAutoscaler) + createObjectIfNotExists(t, "test-collector", &updatedAutoscaler) + err := expectedHorizontalPodAutoscalers(context.Background(), updateParms, []client.Object{updatedHPA}) + assert.NoError(t, err) + + actual := autoscalingv2beta2.HorizontalPodAutoscaler{} + exists, err := populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"}) + + assert.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, int32(1), *actual.Spec.MinReplicas) + assert.Equal(t, int32(3), actual.Spec.MaxReplicas) + } else { + updatedAutoscaler := *updatedHPA.(*autoscalingv2.HorizontalPodAutoscaler) + createObjectIfNotExists(t, "test-collector", &updatedAutoscaler) + err := expectedHorizontalPodAutoscalers(context.Background(), updateParms, []client.Object{updatedHPA}) + assert.NoError(t, err) + + actual := autoscalingv2.HorizontalPodAutoscaler{} + exists, err := populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"}) + + assert.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, int32(1), *actual.Spec.MinReplicas) + assert.Equal(t, int32(3), actual.Spec.MaxReplicas) + } }) t.Run("should delete HPA", func(t *testing.T) { - err := deleteHorizontalPodAutoscalers(context.Background(), params, []autoscalingv1.HorizontalPodAutoscaler{expectedHPA}) + err = deleteHorizontalPodAutoscalers(context.Background(), params, []client.Object{expectedHPA}) assert.NoError(t, err) actual := v1.Deployment{} @@ -78,7 +101,7 @@ func TestExpectedHPA(t *testing.T) { }) } -func paramsWithHPA() Params { +func paramsWithHPA(autoscalingVersion autodetect.AutoscalingVersion) Params { configYAML, err := ioutil.ReadFile("../testdata/test.yaml") if err != nil { fmt.Printf("Error getting yaml file: %v", err) @@ -87,8 +110,19 @@ func paramsWithHPA() Params { minReplicas := int32(3) maxReplicas := int32(5) + mockAutoDetector := &mockAutoDetect{ + HPAVersionFunc: func() (autodetect.AutoscalingVersion, error) { + return autoscalingVersion, nil + }, + } + configuration := config.New(config.WithAutoDetect(mockAutoDetector), config.WithCollectorImage(defaultCollectorImage), config.WithTargetAllocatorImage(defaultTaAllocationImage)) + err = configuration.AutoDetect() + if err != nil { + logger.Error(err, "configuration.autodetect failed") + } + return Params{ - Config: config.New(config.WithCollectorImage(defaultCollectorImage), config.WithTargetAllocatorImage(defaultTaAllocationImage)), + Config: configuration, Client: k8sClient, Instance: v1alpha1.OpenTelemetryCollector{ TypeMeta: metav1.TypeMeta{ @@ -120,3 +154,21 @@ func paramsWithHPA() Params { Recorder: record.NewFakeRecorder(10), } } + +var _ autodetect.AutoDetect = (*mockAutoDetect)(nil) + +type mockAutoDetect struct { + PlatformFunc func() (platform.Platform, error) + HPAVersionFunc func() (autodetect.AutoscalingVersion, error) +} + +func (m *mockAutoDetect) HPAVersion() (autodetect.AutoscalingVersion, error) { + return m.HPAVersionFunc() +} + +func (m *mockAutoDetect) Platform() (platform.Platform, error) { + if m.PlatformFunc != nil { + return m.PlatformFunc() + } + return platform.Unknown, nil +} diff --git a/pkg/collector/reconcile/suite_test.go b/pkg/collector/reconcile/suite_test.go index 6d8e011cbc..0ce59bc928 100644 --- a/pkg/collector/reconcile/suite_test.go +++ b/pkg/collector/reconcile/suite_test.go @@ -259,8 +259,7 @@ func createObjectIfNotExists(tb testing.TB, name string, object client.Object) { tb.Helper() err := k8sClient.Get(context.Background(), client.ObjectKey{Namespace: "default", Name: name}, object) if errors.IsNotFound(err) { - err := k8sClient.Create(context.Background(), - object) + err := k8sClient.Create(context.Background(), object) assert.NoError(tb, err) } }