Don't set environment variable AZURE_EMULATOR_ENABLED for etcd pods (…

…#958)

* Remove usage of environment variables for `emulatorEnabled` fields for storage emulators

* Use gardener/etcd-backup-restore@2d7dab8 image and dependency version

config/samples/etcd-secret-azurite.yaml

@@ -10,5 +10,5 @@ data:
   storageAccount: ZGV2c3RvcmVhY2NvdW50MQ==
   storageKey: RWJ5OHZkTTAyeE5PY3FGbHFVd0pQTGxtRXRsQ0RYSjFPVXpGVDUwdVNSWjZJRnN1RnEyVVZFckN6NEk2dHEvSzFTWkZQVE90ci9LQkhCZWtzb0dNR3c9PQ==
   emulatorEnabled: dHJ1ZQ== # true
-  # TODO: replace with `domain` once we support using [production-style URLs](https://github.com/Azure/Azurite?tab=readme-ov-file#production-style-url) for Azurite
-  storageAPIEndpoint: aHR0cDovL2F6dXJpdGUtc2VydmljZToxMDAwMA== # http://azurite-service:10000, emulatorEnabled has to be true
+  # TODO: going further, support [production-style URLs](https://github.com/Azure/Azurite?tab=readme-ov-file#production-style-url) for Azurite
+  domain: YXp1cml0ZS1zZXJ2aWNlLmRlZmF1bHQ6MTAwMDA= # azurite-service.default:10000 (assuming azurite service runs in `default` namespace), emulatorEnabled has to be true

go.mod

@@ -51,7 +51,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.3.0 // indirect
 	github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible // indirect
 	github.com/andybalholm/brotli v1.1.1 // indirect
-	github.com/aws/aws-sdk-go v1.54.20 // indirect
+	github.com/aws/aws-sdk-go v1.55.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -166,3 +166,6 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.3 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+// TODO: remove once etcd-backup-restore v0.33.0 is released
+replace github.com/gardener/etcd-backup-restore => github.com/gardener/etcd-backup-restore v0.32.1-0.20241230124652-2d7dab8e6dc1 // https://github.com/gardener/etcd-backup-restore/commit/2d7dab8e6dc1385b50d1f64d0e5b70467564c2a3

go.sum

@@ -71,8 +71,8 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7X
 github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
-github.com/aws/aws-sdk-go v1.54.20 h1:FZ2UcXya7bUkvkpf7TaPmiL7EubK0go1nlXGLRwEsoo=
-github.com/aws/aws-sdk-go v1.54.20/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
+github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -139,8 +139,8 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv
 github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
 github.com/gardener/cert-management v0.17.1 h1:vawZGN+rsCRMviacnnMSWELbuIJsNXHaqaLbZ4hYADw=
 github.com/gardener/cert-management v0.17.1/go.mod h1:cwSsyN935017HojKVuWqw2TBhiaxSisX132D9Tn+n9I=
-github.com/gardener/etcd-backup-restore v0.32.0 h1:CGvEfh30XLKvGbaM3BARDUg8X6zdxdKDERetdkDuHfY=
-github.com/gardener/etcd-backup-restore v0.32.0/go.mod h1:wAWmDUi8zIYfJAODggtElB5/R4Wh4J9dylfNKdtwPZc=
+github.com/gardener/etcd-backup-restore v0.32.1-0.20241230124652-2d7dab8e6dc1 h1:Y3AJKKkRlByYScqYi03bBTNOcuvxvONgG3vDgsxEsDI=
+github.com/gardener/etcd-backup-restore v0.32.1-0.20241230124652-2d7dab8e6dc1/go.mod h1:BUbIBNL2OvtIIwzlAnisBG46y5WvNJGRo/NwuKwajBo=
 github.com/gardener/gardener v1.110.1 h1:KJFiVMlzeuJR6aToUUTlXJN6v88ACH/cWOsvX/ftX5s=
 github.com/gardener/gardener v1.110.1/go.mod h1:Ge2wQMWm0NmQZP3L/WMejpfXsnGbfTFBEZud819P3vU=
 github.com/gardener/machine-controller-manager v0.55.1 h1:d6mTnuYko+jWeIi7tAFWgWnL1nR5hGcI6pRCDcH0TGY=

hack/ci-e2e-kind-azure.sh

@@ -10,9 +10,9 @@ set -o pipefail
 # Constants for Azurite credentials and configurations
 STORAGE_ACCOUNT="devstoreaccount1"
 STORAGE_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
-AZURITE_ENDPOINT="http://localhost:10000"
-AZURITE_HOST="azurite-service.default:10000"
-AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=${STORAGE_ACCOUNT};AccountKey=${STORAGE_KEY};BlobEndpoint=http://${AZURITE_HOST}/${STORAGE_ACCOUNT};"
+AZURITE_DOMAIN="azurite-service.default:10000"
+AZURITE_DOMAIN_LOCAL="127.0.0.1:10000"
+AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=${STORAGE_ACCOUNT};AccountKey=${STORAGE_KEY};BlobEndpoint=http://${AZURITE_DOMAIN}/${STORAGE_ACCOUNT};"
 
 make kind-up
 
@@ -25,16 +25,18 @@ kubectl wait --for=condition=ready node --all
 
 # Setup Azure application credentials
 export AZURE_APPLICATION_CREDENTIALS="/tmp/azuriteCredentials"
+rm -rf "${AZURE_APPLICATION_CREDENTIALS}"
 mkdir -p "${AZURE_APPLICATION_CREDENTIALS}"
 echo -n "${STORAGE_ACCOUNT}" > "${AZURE_APPLICATION_CREDENTIALS}/storageAccount"
 echo -n "${STORAGE_KEY}" > "${AZURE_APPLICATION_CREDENTIALS}/storageKey"
+echo -n "true" > "${AZURE_APPLICATION_CREDENTIALS}/emulatorEnabled"
+echo -n "${AZURITE_DOMAIN_LOCAL}" > "${AZURE_APPLICATION_CREDENTIALS}/domain"
 
 # Deploy Azurite and run end-to-end tests
 make deploy-azurite
 make STORAGE_ACCOUNT="${STORAGE_ACCOUNT}" \
   STORAGE_KEY="${STORAGE_KEY}" \
-  AZURE_EMULATOR_ENABLED="true" \
-  AZURITE_HOST="${AZURITE_HOST}" \
+  AZURITE_DOMAIN="${AZURITE_DOMAIN}" \
   AZURE_STORAGE_CONNECTION_STRING="${AZURE_STORAGE_CONNECTION_STRING}" \
   PROVIDERS="azure" \
   TEST_ID="$BUCKET_NAME" \

hack/kind/cluster.yaml

@@ -5,6 +5,7 @@ nodes:
   image: kindest/node:v1.29.2
   # port forward 80 on the host to 80 on this node
   extraPortMappings:
+    # localstack host
   - containerPort: 4566
     hostPort: 4566
     # optional: set the bind address on the host
@@ -24,3 +25,12 @@ nodes:
     # optional: set the protocol to one of TCP, UDP, SCTP.
     # TCP is the default
     protocol: TCP
+    # Azurite host
+  - containerPort: 10000
+    hostPort: 10000
+    # optional: set the bind address on the host
+    # 0.0.0.0 is the current default
+    listenAddress: "127.0.0.1"
+    # optional: set the protocol to one of TCP, UDP, SCTP.
+    # TCP is the default
+    protocol: TCP

internal/common/constants.go

@@ -65,8 +65,6 @@ const (
 	EnvAWSApplicationCredentials = "AWS_APPLICATION_CREDENTIALS"
 	// EnvAzureApplicationCredentials is the environment variable key for Azure application credentials.
 	EnvAzureApplicationCredentials = "AZURE_APPLICATION_CREDENTIALS" // #nosec G101 -- this is the name of an env var, and not the credential itself.
-	// EnvAzureEmulatorEnabled is the environment variable key that is checked to see if the Azure storage emulator Azurite is enabled.
-	EnvAzureEmulatorEnabled = "AZURE_EMULATOR_ENABLED"
 	// EnvGoogleApplicationCredentials is the environment variable key for Google application credentials.
 	EnvGoogleApplicationCredentials = "GOOGLE_APPLICATION_CREDENTIALS" // #nosec G101 -- this is the name of an env var, and not the credential itself.
 	// EnvOpenstackApplicationCredentials is the environment variable key for OpenStack application credentials.

internal/images/images.yaml

@@ -8,7 +8,8 @@ images:
     name: 'etcdbrctl'
   sourceRepository: github.com/gardener/etcd-backup-restore
   repository: europe-docker.pkg.dev/gardener-project/public/gardener/etcdbrctl
-  tag: "v0.32.0"
+  # TODO: switch to v0.33.0 after etcd-backup-restore:v0.33.0 is released
+  tag: "v0.33.0-dev-2d7dab8e6dc1385b50d1f64d0e5b70467564c2a3"
 - name: alpine
   repository: europe-docker.pkg.dev/gardener-project/public/3rd/alpine
   tag: "3.20.3"

internal/store/store.go

@@ -125,7 +125,6 @@ func GetProviderEnvVars(store *druidv1alpha1.StoreSpec) ([]corev1.EnvVar, error)
 
 	case ABS:
 		envVars = append(envVars, utils.GetEnvVarFromValue(common.EnvAzureApplicationCredentials, common.VolumeMountPathNonGCSProviderBackupSecret))
-		envVars = append(envVars, utils.GetEnvVarFromSecret(common.EnvAzureEmulatorEnabled, store.SecretRef.Name, "emulatorEnabled", true))
 
 	case GCS:
 		envVars = append(envVars, utils.GetEnvVarFromValue(common.EnvGoogleApplicationCredentials, fmt.Sprintf("%sserviceaccount.json", common.VolumeMountPathGCSBackupSecret)))

test/e2e/etcd_multi_node_test.go

@@ -265,7 +265,7 @@ func purgeSnapstoreIfNeeded(ctx context.Context, cl client.Client, provider Test
 		purgeLocalSnapstoreJob := purgeLocalSnapstore(ctx, cl, storageContainer, storePrefix)
 		defer cleanUpTestHelperJob(ctx, cl, purgeLocalSnapstoreJob.Name)
 	} else {
-		store, err := getSnapstore(string(snapstoreProvider), storageContainer, storePrefix)
+		store, err := getSnapstore(string(snapstoreProvider), storageContainer, storePrefix, isEmulatorEnabled(provider))
 		ExpectWithOffset(1, err).ShouldNot(HaveOccurred())
 		ExpectWithOffset(1, purgeSnapstore(store)).To(Succeed())
 	}

test/e2e/utils.go

@@ -361,11 +361,10 @@ func getProviders() ([]TestProvider, error) {
 						},
 					},
 				}
-				azuriteHost := getEnvOrFallback("AZURITE_HOST", "")
-				if azuriteHost != "" {
+				azuriteDomain := getEnvOrFallback("AZURITE_DOMAIN", "")
+				if azuriteDomain != "" {
 					provider.Storage.SecretData["emulatorEnabled"] = []byte("true")
-					// TODO: replace with `domain`
-					provider.Storage.SecretData["storageAPIEndpoint"] = []byte("http://" + azuriteHost)
+					provider.Storage.SecretData["domain"] = []byte(azuriteDomain)
 				}
 			}
 		case providerGCP:
@@ -403,6 +402,23 @@ func getProviders() ([]TestProvider, error) {
 	return providers, nil
 }
 
+func isEmulatorEnabled(provider TestProvider) bool {
+	switch provider.Name {
+	case "aws":
+		return provider.Storage.SecretData["endpoint"] != nil
+	case "az":
+		if val, ok := provider.Storage.SecretData["emulatorEnabled"]; ok {
+			return string(val) == "true"
+		}
+		return false
+	case "gcp":
+		return false
+	case "local":
+		return false
+	}
+	return false
+}
+
 func getKubeconfig(kubeconfigPath string) (*rest.Config, error) {
 	return clientcmd.BuildConfigFromFlags("", kubeconfigPath)
 }
@@ -590,11 +606,12 @@ func executeRemoteCommand(ctx context.Context, kubeconfigPath, namespace, podNam
 	return strings.TrimSpace(buf.String()), strings.TrimSpace(errBuf.String()), nil
 }
 
-func getSnapstore(storageProvider, storageContainer, storePrefix string) (brtypes.SnapStore, error) {
+func getSnapstore(storageProvider, storageContainer, storePrefix string, isEmulatorEnabled bool) (brtypes.SnapStore, error) {
 	snapstoreConfig := &brtypes.SnapstoreConfig{
-		Provider:  storageProvider,
-		Container: storageContainer,
-		Prefix:    path.Join(storePrefix, "v2"),
+		Provider:          storageProvider,
+		Container:         storageContainer,
+		Prefix:            path.Join(storePrefix, "v2"),
+		IsEmulatorEnabled: isEmulatorEnabled,
 	}
 	store, err := snapstore.GetSnapstore(snapstoreConfig)
 	if err != nil {

test/integration/controllers/compaction/reconciler_test.go

@@ -532,18 +532,6 @@ func validateStoreAzureForCompactionJob(instance *druidv1alpha1.Etcd, j *batchv1
 									"Name":  Equal(common.EnvAzureApplicationCredentials),
 									"Value": Equal(common.VolumeMountPathNonGCSProviderBackupSecret),
 								}),
-								common.EnvAzureEmulatorEnabled: MatchFields(IgnoreExtras, Fields{
-									"Name": Equal(common.EnvAzureEmulatorEnabled),
-									"ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{
-										"SecretKeyRef": PointTo(MatchFields(IgnoreExtras, Fields{
-											"LocalObjectReference": MatchFields(IgnoreExtras, Fields{
-												"Name": Equal(instance.Spec.Backup.Store.SecretRef.Name),
-											}),
-											"Key":      Equal("emulatorEnabled"),
-											"Optional": Equal(ptr.To(true)),
-										})),
-									})),
-								}),
 							}),
 						}),
 					}),