From 3f851ac3caf48e1a18e52fd51bf5f0d94b52b8ba Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 18 Oct 2021 12:29:49 -0400 Subject: [PATCH] Add support for S3Repositories (#345) --- api/v1beta1/solrbackup_types.go | 13 +- api/v1beta1/solrcloud_types.go | 52 ++++ api/v1beta1/zz_generated.deepcopy.go | 66 ++++- .../bases/solr.apache.org_solrbackups.yaml | 7 +- .../crd/bases/solr.apache.org_solrclouds.yaml | 86 ++++++ controllers/solrbackup_controller.go | 21 +- controllers/solrcloud_controller.go | 11 +- .../solrcloud_controller_backup_test.go | 276 ++++++++++++++++++ controllers/util/backup_util.go | 12 +- controllers/util/backup_util_test.go | 161 +++++++++- controllers/util/solr_backup_repo_util.go | 85 +++++- .../util/solr_backup_repo_util_test.go | 78 ++++- controllers/util/solr_util.go | 10 + controllers/util/solr_util_test.go | 17 ++ docs/solr-backup/README.md | 119 ++++++-- docs/upgrade-notes.md | 8 + example/test_solrcloud.yaml | 6 - example/test_solrcloud_backuprepos.yaml | 23 ++ helm/solr-operator/Chart.yaml | 34 ++- helm/solr-operator/crds/crds.yaml | 93 +++++- 20 files changed, 1108 insertions(+), 70 deletions(-) create mode 100644 controllers/solrcloud_controller_backup_test.go diff --git a/api/v1beta1/solrbackup_types.go b/api/v1beta1/solrbackup_types.go index 672f1d6a..895f9ee1 100644 --- a/api/v1beta1/solrbackup_types.go +++ b/api/v1beta1/solrbackup_types.go @@ -47,6 +47,10 @@ type SolrBackupSpec struct { // +optional Collections []string `json:"collections,omitempty"` + // The location to store the backup in the specified backup repository. + // +optional + Location string `json:"location,omitempty"` + // Persistence is the specification on how to persist the backup data. // +optional Persistence *PersistenceSource `json:"persistence,omitempty"` @@ -202,7 +206,8 @@ type SolrBackupStatus struct { CollectionBackupStatuses []CollectionBackupStatus `json:"collectionBackupStatuses,omitempty"` // Whether the backups are in progress of being persisted - PersistenceStatus BackupPersistenceStatus `json:"persistenceStatus"` + // +optional + PersistenceStatus *BackupPersistenceStatus `json:"persistenceStatus,omitempty"` // Version of the Solr being backed up // +optional @@ -221,6 +226,10 @@ type CollectionBackupStatus struct { // Solr Collection name Collection string `json:"collection"` + // BackupName of this collection's backup in Solr + // +optional + BackupName string `json:"backupName,omitempty"` + // Whether the collection is being backed up // +optional InProgress bool `json:"inProgress,omitempty"` @@ -284,7 +293,7 @@ func (sb *SolrBackup) SharedLabelsWith(labels map[string]string) map[string]stri return newLabels } -// HeadlessServiceName returns the name of the headless service for the cloud +// PersistenceJobName returns the name of the persistence job for the backup func (sb *SolrBackup) PersistenceJobName() string { return fmt.Sprintf("%s-solr-backup-persistence", sb.GetName()) } diff --git a/api/v1beta1/solrcloud_types.go b/api/v1beta1/solrcloud_types.go index def3a627..c7fcc6aa 100644 --- a/api/v1beta1/solrcloud_types.go +++ b/api/v1beta1/solrcloud_types.go @@ -391,6 +391,10 @@ type SolrBackupRepository struct { //+optional GCS *GcsRepository `json:"gcs,omitempty"` + // An S3Repository for Solr to use when backing up and restoring collections. + //+optional + S3 *S3Repository `json:"s3,omitempty"` + // Allows specification of a "repository" for Solr to use when backing up data "locally". // Repositories defined here are considered "managed" and can take advantage of special operator features, such as // post-backup compression. @@ -410,6 +414,54 @@ type GcsRepository struct { BaseLocation string `json:"baseLocation,omitempty"` } +type S3Repository struct { + // The S3 region to store the backup data in + Region string `json:"region"` + + // The name of the S3 bucket that all backup data will be stored in + Bucket string `json:"bucket"` + + // Options for specifying S3Credentials. This is optional in case you want to mount this information yourself. + // However, if you do not include these credentials, and you do not load them yourself via a mount or EnvVars, + // you will likely see errors when taking s3 backups. + // + // If running in EKS, you can create an IAMServiceAccount that uses a role permissioned for this S3 bucket. + // Then use that serviceAccountName for your SolrCloud, and the credentials should be auto-populated. + // + // +optional + Credentials *S3Credentials `json:"credentials,omitempty"` + + // An already-created chroot within the bucket to store data in. Defaults to the root path "/" if not specified. + // +optional + BaseLocation string `json:"baseLocation,omitempty"` + + // The full endpoint URL to use when connecting with S3 (or a supported S3 compatible interface) + // +optional + Endpoint string `json:"endpoint,omitempty"` + + // The full proxy URL to use when connecting with S3 + // +optional + ProxyUrl string `json:"proxyUrl,omitempty"` +} + +type S3Credentials struct { + // The name & key of a Kubernetes secret holding an AWS Access Key ID + // +optional + AccessKeyIdSecret *corev1.SecretKeySelector `json:"accessKeyIdSecret,omitempty"` + + // The name & key of a Kubernetes secret holding an AWS Secret Access Key + // +optional + SecretAccessKeySecret *corev1.SecretKeySelector `json:"secretAccessKeySecret,omitempty"` + + // The name & key of a Kubernetes secret holding an AWS Session Token + // +optional + SessionTokenSecret *corev1.SecretKeySelector `json:"sessionTokenSecret,omitempty"` + + // The name & key of a Kubernetes secret holding an AWS credentials file + // +optional + CredentialsFileSecret *corev1.SecretKeySelector `json:"credentialsFileSecret,omitempty"` +} + type ManagedRepository struct { // This is a volumeSource for a volume that will be mounted to all solrNodes to store backups and load restores. // The data within the volume will be namespaced for this instance, so feel free to use the same volume for multiple clouds. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index aacf29b8..9ce64260 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -537,6 +537,41 @@ func (in *PodOptions) DeepCopy() *PodOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *S3Credentials) DeepCopyInto(out *S3Credentials) { + *out = *in + if in.AccessKeyIdSecret != nil { + in, out := &in.AccessKeyIdSecret, &out.AccessKeyIdSecret + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } + if in.SecretAccessKeySecret != nil { + in, out := &in.SecretAccessKeySecret, &out.SecretAccessKeySecret + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } + if in.SessionTokenSecret != nil { + in, out := &in.SessionTokenSecret, &out.SessionTokenSecret + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } + if in.CredentialsFileSecret != nil { + in, out := &in.CredentialsFileSecret, &out.CredentialsFileSecret + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new S3Credentials. +func (in *S3Credentials) DeepCopy() *S3Credentials { + if in == nil { + return nil + } + out := new(S3Credentials) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *S3PersistenceSource) DeepCopyInto(out *S3PersistenceSource) { *out = *in @@ -559,6 +594,26 @@ func (in *S3PersistenceSource) DeepCopy() *S3PersistenceSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *S3Repository) DeepCopyInto(out *S3Repository) { + *out = *in + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(S3Credentials) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new S3Repository. +func (in *S3Repository) DeepCopy() *S3Repository { + if in == nil { + return nil + } + out := new(S3Repository) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *S3Secrets) DeepCopyInto(out *S3Secrets) { *out = *in @@ -690,6 +745,11 @@ func (in *SolrBackupRepository) DeepCopyInto(out *SolrBackupRepository) { *out = new(GcsRepository) (*in).DeepCopyInto(*out) } + if in.S3 != nil { + in, out := &in.S3, &out.S3 + *out = new(S3Repository) + (*in).DeepCopyInto(*out) + } if in.Managed != nil { in, out := &in.Managed, &out.Managed *out = new(ManagedRepository) @@ -758,7 +818,11 @@ func (in *SolrBackupStatus) DeepCopyInto(out *SolrBackupStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - in.PersistenceStatus.DeepCopyInto(&out.PersistenceStatus) + if in.PersistenceStatus != nil { + in, out := &in.PersistenceStatus, &out.PersistenceStatus + *out = new(BackupPersistenceStatus) + (*in).DeepCopyInto(*out) + } if in.FinishTime != nil { in, out := &in.FinishTime, &out.FinishTime *out = (*in).DeepCopy() diff --git a/config/crd/bases/solr.apache.org_solrbackups.yaml b/config/crd/bases/solr.apache.org_solrbackups.yaml index 7e19f815..53ef5705 100644 --- a/config/crd/bases/solr.apache.org_solrbackups.yaml +++ b/config/crd/bases/solr.apache.org_solrbackups.yaml @@ -67,6 +67,9 @@ spec: items: type: string type: array + location: + description: The location to store the backup in the specified backup repository. + type: string persistence: description: Persistence is the specification on how to persist the backup data. properties: @@ -1068,6 +1071,9 @@ spec: asyncBackupStatus: description: The status of the asynchronous backup call to solr type: string + backupName: + description: BackupName of this collection's backup in Solr + type: string collection: description: Solr Collection name type: string @@ -1127,7 +1133,6 @@ spec: description: Whether the backup was successful type: boolean required: - - persistenceStatus - solrVersion type: object type: object diff --git a/config/crd/bases/solr.apache.org_solrclouds.yaml b/config/crd/bases/solr.apache.org_solrclouds.yaml index edfb8f3e..73d911af 100644 --- a/config/crd/bases/solr.apache.org_solrclouds.yaml +++ b/config/crd/bases/solr.apache.org_solrclouds.yaml @@ -1022,6 +1022,92 @@ spec: name: description: 'A name used to identify this local storage profile. Values should follow RFC-1123. (See here for more details: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names)' type: string + s3: + description: An S3Repository for Solr to use when backing up and restoring collections. + properties: + baseLocation: + description: An already-created chroot within the bucket to store data in. Defaults to the root path "/" if not specified. + type: string + bucket: + description: The name of the S3 bucket that all backup data will be stored in + type: string + credentials: + description: "Options for specifying S3Credentials. This is optional in case you want to mount this information yourself. However, if you do not include these credentials, and you do not load them yourself via a mount or EnvVars, you will likely see errors when taking s3 backups. \n If running in EKS, you can create an IAMServiceAccount that uses a role permissioned for this S3 bucket. Then use that serviceAccountName for your SolrCloud, and the credentials should be auto-populated." + properties: + accessKeyIdSecret: + description: The name & key of a Kubernetes secret holding an AWS Access Key ID + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + credentialsFileSecret: + description: The name & key of a Kubernetes secret holding an AWS credentials file + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + secretAccessKeySecret: + description: The name & key of a Kubernetes secret holding an AWS Secret Access Key + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + sessionTokenSecret: + description: The name & key of a Kubernetes secret holding an AWS Session Token + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + type: object + endpoint: + description: The full endpoint URL to use when connecting with S3 (or a supported S3 compatible interface) + type: string + proxyUrl: + description: The full proxy URL to use when connecting with S3 + type: string + region: + description: The S3 region to store the backup data in + type: string + required: + - bucket + - region + type: object required: - name type: object diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go index bef5a134..228036b0 100644 --- a/controllers/solrbackup_controller.go +++ b/controllers/solrbackup_controller.go @@ -131,15 +131,17 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) if backup.Status.Finished && backup.Status.FinishTime == nil { now := metav1.Now() backup.Status.FinishTime = &now - backup.Status.Successful = backup.Status.PersistenceStatus.Successful + if backup.Spec.Persistence != nil { + backup.Status.Successful = backup.Status.PersistenceStatus.Successful + } } - if !reflect.DeepEqual(oldStatus, backup.Status) { + if !reflect.DeepEqual(oldStatus, &backup.Status) { logger.Info("Updating status for solr-backup") err = r.Status().Update(ctx, backup) } - if backup.Status.Finished { + if err != nil && backup.Status.Finished { requeueOrNot = reconcile.Result{} } @@ -187,12 +189,13 @@ func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, bac // This should only occur before the backup processes have been started if backup.Status.SolrVersion == "" { // Prep the backup directory in the persistentVolume - err := util.EnsureDirectoryForBackup(solrCloud, backupRepository, backup.Name, r.config) + err = util.EnsureDirectoryForBackup(solrCloud, backupRepository, backup, r.config) if err != nil { return solrCloud, collectionBackupsFinished, actionTaken, err } // Make sure that all solr nodes are active and have the backupRestore shared volume mounted + // TODO: we do not need all replicas to be healthy. We should just check that leaders exist for all shards. (or just let Solr do that) cloudReady := solrCloud.Status.BackupRestoreReady && (solrCloud.Status.Replicas == solrCloud.Status.ReadyReplicas) if !cloudReady { logger.Info("Cloud not ready for backup backup", "solrCloud", solrCloud.Name) @@ -238,11 +241,12 @@ func reconcileSolrCollectionBackup(backup *solrv1beta1.SolrBackup, solrCloud *so if started && collectionBackupStatus.StartTime == nil { collectionBackupStatus.StartTime = &now } + collectionBackupStatus.BackupName = util.FullCollectionBackupName(collection, backup.Name) } else if collectionBackupStatus.InProgress { // Check the state of the backup, when it is in progress, and update the state accordingly - finished, successful, asyncStatus, error := util.CheckBackupForCollection(solrCloud, collection, backup.Name, httpHeaders, logger) - if error != nil { - return false, error + finished, successful, asyncStatus, err := util.CheckBackupForCollection(solrCloud, collection, backup.Name, httpHeaders, logger) + if err != nil { + return false, err } collectionBackupStatus.Finished = finished if finished { @@ -271,6 +275,9 @@ func reconcileSolrCollectionBackup(backup *solrv1beta1.SolrBackup, solrCloud *so } func (r *SolrBackupReconciler) persistSolrCloudBackups(ctx context.Context, backup *solrv1beta1.SolrBackup, solrCloud *solrv1beta1.SolrCloud, logger logr.Logger) (err error) { + if backup.Status.PersistenceStatus == nil { + backup.Status.PersistenceStatus = &solrv1beta1.BackupPersistenceStatus{} + } if backup.Status.PersistenceStatus.Finished { return nil } diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go index 19df029e..fa8f4934 100644 --- a/controllers/solrcloud_controller.go +++ b/controllers/solrcloud_controller.go @@ -581,6 +581,8 @@ func (r *SolrCloudReconciler) reconcileCloudStatus(ctx context.Context, solrClou } if allPodsBackupReady && len(foundPods.Items) > 0 { newStatus.BackupRestoreReady = true + } else { + newStatus.BackupRestoreReady = false } // If there are multiple versions of solr running, use the first otherVersion as the current running solr version of the cloud @@ -607,12 +609,9 @@ func isPodReadyForBackup(pod *corev1.Pod, solrCloud *solrv1beta1.SolrCloud) bool return false } - for _, repo := range solrCloud.Spec.BackupRepositories { - if !util.IsBackupVolumePresent(&repo, pod) { - return false - } - } - + // TODO: There is no way to possibly do this with the new S3 option. + // This is wrong, but not the end of the world. + // Replace with new functionality in https://github.com/apache/solr-operator/issues/326 return true } diff --git a/controllers/solrcloud_controller_backup_test.go b/controllers/solrcloud_controller_backup_test.go new file mode 100644 index 00000000..55eef651 --- /dev/null +++ b/controllers/solrcloud_controller_backup_test.go @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controllers + +import ( + "context" + "crypto/md5" + "fmt" + solrv1beta1 "github.com/apache/solr-operator/api/v1beta1" + "github.com/apache/solr-operator/controllers/util" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = FDescribe("SolrCloud controller - Backup Repositories", func() { + var ( + ctx context.Context + + solrCloud *solrv1beta1.SolrCloud + ) + + BeforeEach(func() { + ctx = context.Background() + + solrCloud = &solrv1beta1.SolrCloud{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + }, + Spec: solrv1beta1.SolrCloudSpec{}, + } + }) + + JustBeforeEach(func() { + By("creating the SolrCloud") + Expect(k8sClient.Create(ctx, solrCloud)).To(Succeed()) + + By("defaulting the missing SolrCloud values") + expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + g.Expect(found.WithDefaults()).To(BeFalse(), "The SolrCloud spec should not need to be defaulted eventually") + }) + }) + + AfterEach(func() { + cleanupTest(ctx, solrCloud) + }) + + FContext("S3 Repository", func() { + BeforeEach(func() { + solrCloud.Spec = solrv1beta1.SolrCloudSpec{ + ZookeeperRef: &solrv1beta1.ZookeeperRef{ + ConnectionInfo: &solrv1beta1.ZookeeperConnectionInfo{ + InternalConnectionString: "host:7271", + }, + }, + CustomSolrKubeOptions: solrv1beta1.CustomSolrKubeOptions{ + PodOptions: &solrv1beta1.PodOptions{ + EnvVariables: extraVars, + Volumes: extraVolumes, + }, + }, + BackupRepositories: []solrv1beta1.SolrBackupRepository{ + { + Name: "test-repo", + S3: &solrv1beta1.S3Repository{ + Region: "test-region", + Bucket: "test-bucket", + }, + }, + }, + } + }) + FIt("has the correct resources", func() { + By("testing the Solr ConfigMap") + configMap := expectConfigMap(ctx, solrCloud, solrCloud.ConfigMapName(), map[string]string{"solr.xml": util.GenerateSolrXMLStringForCloud(solrCloud)}) + + By("testing the Solr StatefulSet with explicit volumes and envVars before adding S3Repo credentials") + // Make sure envVars and Volumes are correct be + statefulSet := expectStatefulSet(ctx, solrCloud, solrCloud.StatefulSetName()) + + // Annotations for the solrxml configMap + solrXmlMd5 := fmt.Sprintf("%x", md5.Sum([]byte(configMap.Data[util.SolrXmlFile]))) + Expect(statefulSet.Spec.Template.Annotations).To(HaveKeyWithValue(util.SolrXmlMd5Annotation, solrXmlMd5), "Wrong solr.xml MD5 annotation in the pod template!") + + // Env Variable Tests + expectedEnvVars := map[string]string{ + "ZK_HOST": "host:7271/", + "SOLR_HOST": "$(POD_HOSTNAME)." + solrCloud.HeadlessServiceName() + "." + solrCloud.Namespace, + "SOLR_PORT": "8983", + "SOLR_NODE_PORT": "8983", + "SOLR_OPTS": "-DhostPort=$(SOLR_NODE_PORT)", + } + foundEnv := statefulSet.Spec.Template.Spec.Containers[0].Env + + // First test the extraVars, then the default Solr Operator vars + Expect(foundEnv[len(foundEnv)-3:len(foundEnv)-1]).To(Equal(extraVars), "Extra Env Vars are not the same as the ones provided in podOptions") + testPodEnvVariables(expectedEnvVars, append(foundEnv[:len(foundEnv)-3], foundEnv[len(foundEnv)-1])) + + // Check Volumes + extraVolumes[0].DefaultContainerMount.Name = extraVolumes[0].Name + Expect(statefulSet.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(len(extraVolumes)+1), "Container has wrong number of volumeMounts") + Expect(statefulSet.Spec.Template.Spec.Containers[0].VolumeMounts[1]).To(Equal(*extraVolumes[0].DefaultContainerMount), "Additional Volume from podOptions not mounted into container properly.") + Expect(statefulSet.Spec.Template.Spec.Volumes).To(HaveLen(len(extraVolumes)+2), "Pod has wrong number of volumes") + Expect(statefulSet.Spec.Template.Spec.Volumes[2].Name).To(Equal(extraVolumes[0].Name), "Additional Volume from podOptions not loaded into pod properly.") + Expect(statefulSet.Spec.Template.Spec.Volumes[2].VolumeSource).To(Equal(extraVolumes[0].Source), "Additional Volume from podOptions not loaded into pod properly.") + + By("adding credentials to the S3 repository (envVars)") + s3Credentials := &solrv1beta1.S3Credentials{ + AccessKeyIdSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "aws-secret-1"}, + Key: "accessKeyId", + }, + SecretAccessKeySecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "aws-secret-2"}, + Key: "secretAccessKey", + }, + SessionTokenSecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "aws-secret-3"}, + Key: "sessionToken", + }, + } + foundSolrCloud := expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + found.Spec.BackupRepositories[0].S3.Credentials = s3Credentials + g.Expect(k8sClient.Update(ctx, found)).To(Succeed(), "Change the s3 credentials for the SolrCloud") + }) + + By("testing the Solr StatefulSet after adding S3Repo envVar credentials") + // Make sure envVars and Volumes are correct be + statefulSet = expectStatefulSetWithChecks(ctx, foundSolrCloud, foundSolrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) { + // Annotations for the solrxml configMap + g.Expect(found.Spec.Template.Annotations).To(HaveKeyWithValue(util.SolrXmlMd5Annotation, fmt.Sprintf("%x", md5.Sum([]byte(configMap.Data[util.SolrXmlFile])))), "Wrong solr.xml MD5 annotation in the pod template!") + + // Env Variable Tests + expectedEnvVars := map[string]string{ + "ZK_HOST": "host:7271/", + "SOLR_HOST": "$(POD_HOSTNAME)." + foundSolrCloud.HeadlessServiceName() + "." + foundSolrCloud.Namespace, + "SOLR_PORT": "8983", + "SOLR_NODE_PORT": "8983", + "SOLR_LOG_LEVEL": "INFO", + "SOLR_OPTS": "-DhostPort=$(SOLR_NODE_PORT)", + } + foundEnv := found.Spec.Template.Spec.Containers[0].Env + + // First test the extraVars, then the default Solr Operator vars, then the S3 vars + g.Expect(foundEnv[len(foundEnv)-3:len(foundEnv)-1]).To(Equal(extraVars), "Extra Env Vars are not the same as the ones provided in podOptions") + testPodEnvVariablesWithGomega(g, expectedEnvVars, append(append([]corev1.EnvVar{}, foundEnv[:len(foundEnv)-6]...), foundEnv[len(foundEnv)-1])) + g.Expect(foundEnv[len(foundEnv)-6:len(foundEnv)-3]).To(Equal([]corev1.EnvVar{ + { + Name: "AWS_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: s3Credentials.AccessKeyIdSecret}, + }, + { + Name: "AWS_SECRET_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: s3Credentials.SecretAccessKeySecret}, + }, + { + Name: "AWS_SESSION_TOKEN", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: s3Credentials.SessionTokenSecret}, + }, + }), "Wrong envVars for the S3 Credentials") + + // Check that no additional volumes have been added + extraVolumes[0].DefaultContainerMount.Name = extraVolumes[0].Name + g.Expect(found.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(len(extraVolumes)+1), "Container has wrong number of volumeMounts") + g.Expect(found.Spec.Template.Spec.Containers[0].VolumeMounts[1]).To(Equal(*extraVolumes[0].DefaultContainerMount), "Additional Volume from podOptions not mounted into container properly.") + g.Expect(found.Spec.Template.Spec.Volumes).To(HaveLen(len(extraVolumes)+2), "Pod has wrong number of volumes") + g.Expect(found.Spec.Template.Spec.Volumes[2].Name).To(Equal(extraVolumes[0].Name), "Additional Volume from podOptions not loaded into pod properly.") + g.Expect(found.Spec.Template.Spec.Volumes[2].VolumeSource).To(Equal(extraVolumes[0].Source), "Additional Volume from podOptions not loaded into pod properly.") + }) + + By("adding credentials to the S3 repository (envVars & credentials file)") + s3Credentials.CredentialsFileSecret = &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "aws-credentials"}, + Key: "credentials-file", + } + foundSolrCloud = expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + found.Spec.BackupRepositories[0].S3.Credentials = s3Credentials + g.Expect(k8sClient.Update(ctx, found)).To(Succeed(), "Change the s3 credentials for the SolrCloud") + }) + + By("testing the Solr StatefulSet after adding S3Repo envVar credentials & credentials file") + // Make sure envVars and Volumes are correct be + statefulSet = expectStatefulSetWithChecks(ctx, foundSolrCloud, foundSolrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) { + // Annotations for the solrxml configMap + g.Expect(found.Spec.Template.Annotations).To(HaveKeyWithValue(util.SolrXmlMd5Annotation, fmt.Sprintf("%x", md5.Sum([]byte(configMap.Data[util.SolrXmlFile])))), "Wrong solr.xml MD5 annotation in the pod template!") + + // Env Variable Tests + expectedEnvVars := map[string]string{ + "ZK_HOST": "host:7271/", + "SOLR_HOST": "$(POD_HOSTNAME)." + foundSolrCloud.HeadlessServiceName() + "." + foundSolrCloud.Namespace, + "SOLR_PORT": "8983", + "SOLR_NODE_PORT": "8983", + "SOLR_LOG_LEVEL": "INFO", + "SOLR_OPTS": "-DhostPort=$(SOLR_NODE_PORT)", + } + foundEnv := found.Spec.Template.Spec.Containers[0].Env + + // First test the extraVars, then the default Solr Operator vars, then the S3 vars + g.Expect(foundEnv[len(foundEnv)-3:len(foundEnv)-1]).To(Equal(extraVars), "Extra Env Vars are not the same as the ones provided in podOptions") + testPodEnvVariablesWithGomega(g, expectedEnvVars, append(append([]corev1.EnvVar{}, foundEnv[:len(foundEnv)-7]...), foundEnv[len(foundEnv)-1])) + g.Expect(foundEnv[len(foundEnv)-7:len(foundEnv)-3]).To(Equal([]corev1.EnvVar{ + { + Name: "AWS_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: s3Credentials.AccessKeyIdSecret}, + }, + { + Name: "AWS_SECRET_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: s3Credentials.SecretAccessKeySecret}, + }, + { + Name: "AWS_SESSION_TOKEN", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: s3Credentials.SessionTokenSecret}, + }, + { + Name: "AWS_SHARED_CREDENTIALS_FILE", + Value: "/var/solr/data/backup-restore/test-repo/s3credential/credentials", + }, + }), "Wrong envVars for the S3 Credentials") + + // Check that no additional volumes have been added + extraVolumes[0].DefaultContainerMount.Name = extraVolumes[0].Name + g.Expect(found.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(len(extraVolumes)+2), "Container has wrong number of volumeMounts") + g.Expect(found.Spec.Template.Spec.Containers[0].VolumeMounts[1].Name).To(Equal("backup-repository-test-repo"), "S3Credentials file volumeMount has wrong name.") + g.Expect(found.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath).To(Equal("/var/solr/data/backup-restore/test-repo/s3credential"), "S3Credentials file volumeMount has wrong mount path.") + g.Expect(found.Spec.Template.Spec.Containers[0].VolumeMounts[1].ReadOnly).To(BeTrue(), "S3Credentials file volumeMount must be read-only.") + g.Expect(found.Spec.Template.Spec.Containers[0].VolumeMounts[2]).To(Equal(*extraVolumes[0].DefaultContainerMount), "Additional Volume from podOptions not mounted into container properly.") + g.Expect(found.Spec.Template.Spec.Volumes).To(HaveLen(len(extraVolumes)+3), "Pod has wrong number of volumes") + g.Expect(found.Spec.Template.Spec.Volumes[2].Name).To(Equal("backup-repository-test-repo"), "S3Credentials file volume has wrong name.") + g.Expect(found.Spec.Template.Spec.Volumes[2].VolumeSource.Secret).To(Not(BeNil()), "S3Credentials file has to be loaded via a secret volume.") + g.Expect(found.Spec.Template.Spec.Volumes[2].VolumeSource.Secret.SecretName).To(Equal(s3Credentials.CredentialsFileSecret.Name), "S3Credentials file is loaded into the pod using the wrong secret name.") + g.Expect(found.Spec.Template.Spec.Volumes[2].VolumeSource.Secret.Items).To(Equal( + []corev1.KeyToPath{{Key: s3Credentials.CredentialsFileSecret.Key, Path: util.S3CredentialFileName}}), "S3Credentials file pod volume has the wrong items.") + g.Expect(found.Spec.Template.Spec.Volumes[2].VolumeSource.Secret.DefaultMode).To(BeEquivalentTo(&util.SecretReadOnlyPermissions), "S3Credentials file pod volume has the wrong default mode.") + g.Expect(found.Spec.Template.Spec.Volumes[3].Name).To(Equal(extraVolumes[0].Name), "Additional Volume from podOptions not loaded into pod properly.") + g.Expect(found.Spec.Template.Spec.Volumes[3].VolumeSource).To(Equal(extraVolumes[0].Source), "Additional Volume from podOptions not loaded into pod properly.") + }) + + By("adding extra options to the S3 repository") + s3Credentials.CredentialsFileSecret = &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "aws-credentials"}, + Key: "credentials-file", + } + foundSolrCloud = expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, found *solrv1beta1.SolrCloud) { + found.Spec.BackupRepositories[0].S3.Endpoint = "https://test-endpoint:3223" + g.Expect(k8sClient.Update(ctx, found)).To(Succeed(), "Change the s3 endpoint for the SolrCloud") + }) + + By("checking the solr.xml configMap is updated with the new S3 options, and a solrCloud rolling update follows") + newConfigMap := expectConfigMap(ctx, foundSolrCloud, foundSolrCloud.ConfigMapName(), map[string]string{"solr.xml": util.GenerateSolrXMLStringForCloud(foundSolrCloud)}) + + updateSolrXmlMd5 := fmt.Sprintf("%x", md5.Sum([]byte(newConfigMap.Data[util.SolrXmlFile]))) + Expect(updateSolrXmlMd5).To(Not(Equal(solrXmlMd5)), "New configMap hash should not equal initial configMap hash") + expectStatefulSetWithChecks(ctx, solrCloud, solrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) { + g.Expect(found.Spec.Template.Annotations).To(HaveKeyWithValue(util.SolrXmlMd5Annotation, updateSolrXmlMd5), "Custom solr.xml MD5 annotation should be updated on the pod template.") + }) + }) + }) +}) diff --git a/controllers/util/backup_util.go b/controllers/util/backup_util.go index 7e247010..b06a8c54 100644 --- a/controllers/util/backup_util.go +++ b/controllers/util/backup_util.go @@ -57,6 +57,10 @@ func GetBackupRepositoryByName(backupRepos []solr.SolrBackupRepository, reposito return nil } +func FullCollectionBackupName(collection string, backupName string) string { + return fmt.Sprintf("%s-%s", backupName, collection) +} + func AsyncIdForCollectionBackup(collection string, backupName string) string { return fmt.Sprintf("%s-%s", backupName, collection) } @@ -309,9 +313,9 @@ func GenerateQueryParamsForBackup(backupRepository *solr.SolrBackupRepository, b queryParams := url.Values{} queryParams.Add("action", "BACKUP") queryParams.Add("collection", collection) - queryParams.Add("name", collection) + queryParams.Add("name", FullCollectionBackupName(collection, backup.Name)) queryParams.Add("async", AsyncIdForCollectionBackup(collection, backup.Name)) - queryParams.Add("location", BackupLocationPath(backupRepository, backup.Name)) + queryParams.Add("location", BackupLocationPath(backupRepository, backup.Spec.Location)) queryParams.Add("repository", backup.Spec.RepositoryName) return queryParams } @@ -379,10 +383,10 @@ func DeleteAsyncInfoForBackup(cloud *solr.SolrCloud, collection string, backupNa return err } -func EnsureDirectoryForBackup(solrCloud *solr.SolrCloud, backupRepository *solr.SolrBackupRepository, backupName string, config *rest.Config) (err error) { +func EnsureDirectoryForBackup(solrCloud *solr.SolrCloud, backupRepository *solr.SolrBackupRepository, backup *solr.SolrBackup, config *rest.Config) (err error) { // Directory creation only required/possible for managed (i.e. local) backups if IsRepoManaged(backupRepository) { - backupPath := BackupLocationPath(backupRepository, backupName) + backupPath := BackupLocationPath(backupRepository, backup.Spec.Location) return RunExecForPod( solrCloud.GetAllSolrNodeNames()[0], solrCloud.Namespace, diff --git a/controllers/util/backup_util_test.go b/controllers/util/backup_util_test.go index acc9b190..18e9cdc5 100644 --- a/controllers/util/backup_util_test.go +++ b/controllers/util/backup_util_test.go @@ -48,9 +48,39 @@ func TestSolrBackupApiParamsForManagedRepositoryBackup(t *testing.T) { assert.Equalf(t, "BACKUP", queryParams.Get("action"), "Wrong %s for Collections API Call", "action") assert.Equalf(t, "col2", queryParams.Get("collection"), "Wrong %s for Collections API Call", "collection name") - assert.Equalf(t, "col2", queryParams.Get("name"), "Wrong %s for Collections API Call", "backup name") + assert.Equalf(t, "somebackupname-col2", queryParams.Get("name"), "Wrong %s for Collections API Call", "backup name") assert.Equalf(t, "somebackupname-col2", queryParams.Get("async"), "Wrong %s for Collections API Call", "async id") - assert.Equalf(t, "/var/solr/data/backup-restore/somemanagedrepository/backups/somebackupname", queryParams.Get("location"), "Wrong %s for Collections API Call", "backup location") + assert.Equalf(t, "/var/solr/data/backup-restore/somemanagedrepository/backups", queryParams.Get("location"), "Wrong %s for Collections API Call", "backup location") + assert.Equalf(t, "somemanagedrepository", queryParams.Get("repository"), "Wrong %s for Collections API Call", "repository") +} + +func TestSolrBackupApiParamsForManagedRepositoryBackupWithLocation(t *testing.T) { + managedRepository := &solr.SolrBackupRepository{ + Name: "somemanagedrepository", + Managed: &solr.ManagedRepository{ + Volume: corev1.VolumeSource{}, // Actual volume info doesn't matter here + Directory: "/somedirectory", + }, + } + backupConfig := solr.SolrBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "somebackupname", + }, + Spec: solr.SolrBackupSpec{ + SolrCloud: "solrcloudcluster", + RepositoryName: "somemanagedrepository", + Collections: []string{"col1", "col2"}, + Location: "test/location", + }, + } + + queryParams := GenerateQueryParamsForBackup(managedRepository, &backupConfig, "col2") + + assert.Equalf(t, "BACKUP", queryParams.Get("action"), "Wrong %s for Collections API Call", "action") + assert.Equalf(t, "col2", queryParams.Get("collection"), "Wrong %s for Collections API Call", "collection name") + assert.Equalf(t, "somebackupname-col2", queryParams.Get("name"), "Wrong %s for Collections API Call", "backup name") + assert.Equalf(t, "somebackupname-col2", queryParams.Get("async"), "Wrong %s for Collections API Call", "async id") + assert.Equalf(t, "/var/solr/data/backup-restore/somemanagedrepository/test/location", queryParams.Get("location"), "Wrong %s for Collections API Call", "backup location") assert.Equalf(t, "somemanagedrepository", queryParams.Get("repository"), "Wrong %s for Collections API Call", "repository") } @@ -81,12 +111,137 @@ func TestSolrBackupApiParamsForGcsRepositoryBackup(t *testing.T) { assert.Equalf(t, "BACKUP", queryParams.Get("action"), "Wrong %s for Collections API Call", "action") assert.Equalf(t, "col2", queryParams.Get("collection"), "Wrong %s for Collections API Call", "collection name") - assert.Equalf(t, "col2", queryParams.Get("name"), "Wrong %s for Collections API Call", "backup name") + assert.Equalf(t, "somebackupname-col2", queryParams.Get("name"), "Wrong %s for Collections API Call", "backup name") assert.Equalf(t, "somebackupname-col2", queryParams.Get("async"), "Wrong %s for Collections API Call", "async id") assert.Equalf(t, "/some/gcs/path", queryParams.Get("location"), "Wrong %s for Collections API Call", "backup location") assert.Equalf(t, "somegcsrepository", queryParams.Get("repository"), "Wrong %s for Collections API Call", "repository") } +func TestSolrBackupApiParamsForGcsRepositoryBackupWithLocation(t *testing.T) { + gcsRepository := &solr.SolrBackupRepository{ + Name: "somegcsrepository", + GCS: &solr.GcsRepository{ + Bucket: "some-gcs-bucket", + GcsCredentialSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "some-secret-name"}, + Key: "some-secret-key", + }, + BaseLocation: "/some/gcs/path", + }, + } + backupConfig := solr.SolrBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "somebackupname", + }, + Spec: solr.SolrBackupSpec{ + SolrCloud: "solrcloudcluster", + RepositoryName: "somegcsrepository", + Collections: []string{"col1", "col2"}, + Location: "/another/gcs/path/test", + }, + } + + queryParams := GenerateQueryParamsForBackup(gcsRepository, &backupConfig, "col2") + + assert.Equalf(t, "BACKUP", queryParams.Get("action"), "Wrong %s for Collections API Call", "action") + assert.Equalf(t, "col2", queryParams.Get("collection"), "Wrong %s for Collections API Call", "collection name") + assert.Equalf(t, "somebackupname-col2", queryParams.Get("name"), "Wrong %s for Collections API Call", "backup name") + assert.Equalf(t, "somebackupname-col2", queryParams.Get("async"), "Wrong %s for Collections API Call", "async id") + assert.Equalf(t, "/another/gcs/path/test", queryParams.Get("location"), "Wrong %s for Collections API Call", "backup location") + assert.Equalf(t, "somegcsrepository", queryParams.Get("repository"), "Wrong %s for Collections API Call", "repository") +} + +func TestSolrBackupApiParamsForGcsRepositoryBackupWithNoLocations(t *testing.T) { + gcsRepository := &solr.SolrBackupRepository{ + Name: "somegcsrepository", + GCS: &solr.GcsRepository{ + Bucket: "some-gcs-bucket", + GcsCredentialSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "some-secret-name"}, + Key: "some-secret-key", + }, + }, + } + backupConfig := solr.SolrBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "somebackupname", + }, + Spec: solr.SolrBackupSpec{ + SolrCloud: "solrcloudcluster", + RepositoryName: "somegcsrepository", + Collections: []string{"col1", "col2"}, + }, + } + + queryParams := GenerateQueryParamsForBackup(gcsRepository, &backupConfig, "col2") + + assert.Equalf(t, "BACKUP", queryParams.Get("action"), "Wrong %s for Collections API Call", "action") + assert.Equalf(t, "col2", queryParams.Get("collection"), "Wrong %s for Collections API Call", "collection name") + assert.Equalf(t, "somebackupname-col2", queryParams.Get("name"), "Wrong %s for Collections API Call", "backup name") + assert.Equalf(t, "somebackupname-col2", queryParams.Get("async"), "Wrong %s for Collections API Call", "async id") + assert.Equalf(t, "/", queryParams.Get("location"), "Wrong %s for Collections API Call", "backup location") + assert.Equalf(t, "somegcsrepository", queryParams.Get("repository"), "Wrong %s for Collections API Call", "repository") +} + +func TestSolrBackupApiParamsForS3RepositoryBackup(t *testing.T) { + s3Repository := &solr.SolrBackupRepository{ + Name: "somes3repository", + S3: &solr.S3Repository{ + Bucket: "some-s3-bucket", + Region: "us-west-2", + }, + } + backupConfig := solr.SolrBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "somebackupname", + }, + Spec: solr.SolrBackupSpec{ + SolrCloud: "solrcloudcluster", + RepositoryName: "somes3repository", + Collections: []string{"col1", "col2"}, + }, + } + + queryParams := GenerateQueryParamsForBackup(s3Repository, &backupConfig, "col2") + + assert.Equalf(t, "BACKUP", queryParams.Get("action"), "Wrong %s for Collections API Call", "action") + assert.Equalf(t, "col2", queryParams.Get("collection"), "Wrong %s for Collections API Call", "collection name") + assert.Equalf(t, "somebackupname-col2", queryParams.Get("name"), "Wrong %s for Collections API Call", "backup name") + assert.Equalf(t, "somebackupname-col2", queryParams.Get("async"), "Wrong %s for Collections API Call", "async id") + assert.Equalf(t, "/", queryParams.Get("location"), "Wrong %s for Collections API Call", "backup location") + assert.Equalf(t, "somes3repository", queryParams.Get("repository"), "Wrong %s for Collections API Call", "repository") +} + +func TestSolrBackupApiParamsForS3RepositoryBackupWithLocation(t *testing.T) { + s3Repository := &solr.SolrBackupRepository{ + Name: "somes3repository", + S3: &solr.S3Repository{ + Bucket: "some-gcs-bucket", + Region: "us-west-2", + }, + } + backupConfig := solr.SolrBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "somebackupname", + }, + Spec: solr.SolrBackupSpec{ + SolrCloud: "solrcloudcluster", + RepositoryName: "somes3repository", + Collections: []string{"col1", "col2"}, + Location: "/another/path", + }, + } + + queryParams := GenerateQueryParamsForBackup(s3Repository, &backupConfig, "col2") + + assert.Equalf(t, "BACKUP", queryParams.Get("action"), "Wrong %s for Collections API Call", "action") + assert.Equalf(t, "col2", queryParams.Get("collection"), "Wrong %s for Collections API Call", "collection name") + assert.Equalf(t, "somebackupname-col2", queryParams.Get("name"), "Wrong %s for Collections API Call", "backup name") + assert.Equalf(t, "somebackupname-col2", queryParams.Get("async"), "Wrong %s for Collections API Call", "async id") + assert.Equalf(t, "/another/path", queryParams.Get("location"), "Wrong %s for Collections API Call", "backup location") + assert.Equalf(t, "somes3repository", queryParams.Get("repository"), "Wrong %s for Collections API Call", "repository") +} + func TestReportsFailureWhenBackupRepositoryCannotBeFoundByName(t *testing.T) { repos := []solr.SolrBackupRepository{ { diff --git a/controllers/util/solr_backup_repo_util.go b/controllers/util/solr_backup_repo_util.go index f7b9b93c..ced61cf3 100644 --- a/controllers/util/solr_backup_repo_util.go +++ b/controllers/util/solr_backup_repo_util.go @@ -29,6 +29,7 @@ const ( BaseBackupRestorePath = "/var/solr/data/backup-restore" GCSCredentialSecretKey = "service-account-key.json" + S3CredentialFileName = "credentials" ) func RepoVolumeName(repo *solrv1beta1.SolrBackupRepository) string { @@ -54,6 +55,10 @@ func GcsRepoSecretMountPath(repo *solrv1beta1.SolrBackupRepository) string { return fmt.Sprintf("%s/%s/%s", BaseBackupRestorePath, repo.Name, "gcscredential") } +func S3RepoSecretMountPath(repo *solrv1beta1.SolrBackupRepository) string { + return fmt.Sprintf("%s/%s/%s", BaseBackupRestorePath, repo.Name, "s3credential") +} + func ManagedRepoVolumeMountPath(repo *solrv1beta1.SolrBackupRepository) string { return fmt.Sprintf("%s/%s", BaseBackupRestorePath, repo.Name) } @@ -80,6 +85,19 @@ func RepoVolumeSourceAndMount(repo *solrv1beta1.SolrBackupRepository, solrCloudN MountPath: GcsRepoSecretMountPath(repo), ReadOnly: true, } + } else if repo.S3 != nil && repo.S3.Credentials != nil && repo.S3.Credentials.CredentialsFileSecret != nil { + source = &corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: repo.S3.Credentials.CredentialsFileSecret.Name, + Items: []corev1.KeyToPath{{Key: repo.S3.Credentials.CredentialsFileSecret.Key, Path: S3CredentialFileName}}, + DefaultMode: &SecretReadOnlyPermissions, + Optional: &f, + }, + } + mount = &corev1.VolumeMount{ + MountPath: S3RepoSecretMountPath(repo), + ReadOnly: true, + } } return } @@ -87,6 +105,8 @@ func RepoVolumeSourceAndMount(repo *solrv1beta1.SolrBackupRepository, solrCloudN func RepoSolrModules(repo *solrv1beta1.SolrBackupRepository) (libs []string) { if repo.GCS != nil { libs = []string{"gcs-repository"} + } else if repo.S3 != nil { + libs = []string{"s3-repository"} } return } @@ -104,11 +124,54 @@ func RepoXML(repo *solrv1beta1.SolrBackupRepository) (xml string) { %s %s/%s `, repo.Name, repo.GCS.Bucket, GcsRepoSecretMountPath(repo), GCSCredentialSecretKey) + } else if repo.S3 != nil { + s3Extras := make([]string, 0) + if repo.S3.Endpoint != "" { + s3Extras = append(s3Extras, fmt.Sprintf("%s", repo.S3.Endpoint)) + } + if repo.S3.ProxyUrl != "" { + s3Extras = append(s3Extras, fmt.Sprintf("%s", repo.S3.ProxyUrl)) + } + xml = fmt.Sprintf(` + + %s + %s + %s +`, repo.Name, repo.S3.Bucket, repo.S3.Region, strings.Join(s3Extras, ` + `)) } return } func RepoEnvVars(repo *solrv1beta1.SolrBackupRepository) (envVars []corev1.EnvVar) { + if repo.S3 != nil && repo.S3.Credentials != nil { + // Env Var names sourced from: https://docs.aws.amazon.com/sdk-for-java/v2/developer-guide/credentials.html + if repo.S3.Credentials.AccessKeyIdSecret != nil { + envVars = append(envVars, corev1.EnvVar{ + Name: "AWS_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: repo.S3.Credentials.AccessKeyIdSecret}, + }) + } + if repo.S3.Credentials.SecretAccessKeySecret != nil { + envVars = append(envVars, corev1.EnvVar{ + Name: "AWS_SECRET_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: repo.S3.Credentials.SecretAccessKeySecret}, + }) + } + if repo.S3.Credentials.SessionTokenSecret != nil { + envVars = append(envVars, corev1.EnvVar{ + Name: "AWS_SESSION_TOKEN", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: repo.S3.Credentials.SessionTokenSecret}, + }) + } + // Env Var name sourced from: https://docs.aws.amazon.com/sdkref/latest/guide/file-location.html + if repo.S3.Credentials.CredentialsFileSecret != nil { + envVars = append(envVars, corev1.EnvVar{ + Name: "AWS_SHARED_CREDENTIALS_FILE", + Value: fmt.Sprintf("%s/%s", S3RepoSecretMountPath(repo), S3CredentialFileName), + }) + } + } return envVars } @@ -143,14 +206,26 @@ func IsBackupVolumePresent(repo *solrv1beta1.SolrBackupRepository, pod *corev1.P return false } -func BackupLocationPath(repo *solrv1beta1.SolrBackupRepository, backupName string) string { +func BackupLocationPath(repo *solrv1beta1.SolrBackupRepository, backupLocation string) string { if repo.Managed != nil { - return fmt.Sprintf("%s/backups/%s", ManagedRepoVolumeMountPath(repo), backupName) + if backupLocation == "" { + backupLocation = "backups" + } + return fmt.Sprintf("%s/%s", ManagedRepoVolumeMountPath(repo), backupLocation) } else if repo.GCS != nil { - if repo.GCS.BaseLocation != "" { + if backupLocation != "" { + return backupLocation + } else if repo.GCS.BaseLocation != "" { return repo.GCS.BaseLocation + } else { + return "/" + } + } else if repo.S3 != nil { + if backupLocation != "" { + return backupLocation + } else { + return "/" } - return "/" } - return "" + return backupLocation } diff --git a/controllers/util/solr_backup_repo_util_test.go b/controllers/util/solr_backup_repo_util_test.go index b294ce9a..e33de2be 100644 --- a/controllers/util/solr_backup_repo_util_test.go +++ b/controllers/util/solr_backup_repo_util_test.go @@ -50,16 +50,6 @@ func TestGCSRepoXML(t *testing.T) { `, RepoXML(repo), "Wrong SolrXML entry for the GCS Repo with a base location set") } -func TestManagedRepoXML(t *testing.T) { - repo := &solr.SolrBackupRepository{ - Name: "managedrepository2", - Managed: &solr.ManagedRepository{ - Volume: corev1.VolumeSource{}, - }, - } - assert.EqualValuesf(t, "", RepoXML(repo), "Wrong SolrXML entry for the Managed Repo") -} - func TestGCSRepoAdditionalLibs(t *testing.T) { repo := &solr.SolrBackupRepository{ Name: "gcsrepository1", @@ -88,6 +78,74 @@ func TestGCSRepoSolrModules(t *testing.T) { assert.EqualValues(t, []string{"gcs-repository"}, RepoSolrModules(repo), "GCS Repos require the gcs-repository solr module") } +func TestS3RepoXML(t *testing.T) { + repo := &solr.SolrBackupRepository{ + Name: "repo1", + S3: &solr.S3Repository{ + Bucket: "some-bucket-name1", + Region: "ap-northeast-2", + }, + } + assert.EqualValuesf(t, ` + + some-bucket-name1 + ap-northeast-2 + +`, RepoXML(repo), "Wrong SolrXML entry for the S3 Repo") + + // Test with an endpoint + repo.S3.Endpoint = "http://other.s3.location:3242" + assert.EqualValuesf(t, ` + + some-bucket-name1 + ap-northeast-2 + http://other.s3.location:3242 +`, RepoXML(repo), "Wrong SolrXML entry for the S3 Repo with an endpoint set") + + // Test with a proxy url and endpoint + repo.S3.Endpoint = "http://other.s3.location:3242" + repo.S3.ProxyUrl = "https://proxy.url:3242" + assert.EqualValuesf(t, ` + + some-bucket-name1 + ap-northeast-2 + http://other.s3.location:3242 + https://proxy.url:3242 +`, RepoXML(repo), "Wrong SolrXML entry for the S3 Repo with an endpoint and proxy url set") +} + +func TestS3RepoAdditionalLibs(t *testing.T) { + repo := &solr.SolrBackupRepository{ + Name: "repo1", + S3: &solr.S3Repository{ + Bucket: "some-bucket-name1", + Region: "us-west-2", + }, + } + assert.Empty(t, AdditionalRepoLibs(repo), "S3 Repos require no additional libraries for Solr") +} + +func TestS3RepoSolrModules(t *testing.T) { + repo := &solr.SolrBackupRepository{ + Name: "repo1", + S3: &solr.S3Repository{ + Bucket: "some-bucket-name1", + Region: "us-west-2", + }, + } + assert.EqualValues(t, []string{"s3-repository"}, RepoSolrModules(repo), "S3 Repos require the s3-repository solr module") +} + +func TestManagedRepoXML(t *testing.T) { + repo := &solr.SolrBackupRepository{ + Name: "managedrepository2", + Managed: &solr.ManagedRepository{ + Volume: corev1.VolumeSource{}, + }, + } + assert.EqualValuesf(t, "", RepoXML(repo), "Wrong SolrXML entry for the Managed Repo") +} + func TestManagedRepoAdditionalLibs(t *testing.T) { repo := &solr.SolrBackupRepository{ Name: "managedrepository2", diff --git a/controllers/util/solr_util.go b/controllers/util/solr_util.go index db4455b3..34886e8e 100644 --- a/controllers/util/solr_util.go +++ b/controllers/util/solr_util.go @@ -195,6 +195,7 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl } // Add necessary specs for backupRepos + backupEnvVars := make([]corev1.EnvVar, 0) for _, repo := range solrCloud.Spec.BackupRepositories { volumeSource, mount := RepoVolumeSourceAndMount(&repo, solrCloud.Name) if volumeSource != nil { @@ -205,6 +206,10 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl mount.Name = RepoVolumeName(&repo) volumeMounts = append(volumeMounts, *mount) } + repoEnvVars := RepoEnvVars(&repo) + if len(repoEnvVars) > 0 { + backupEnvVars = append(backupEnvVars, repoEnvVars...) + } } if nil != customPodOptions { @@ -309,6 +314,11 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl } envVars = append(envVars, zkEnvVars...) + // Add envVars for backupRepos if any are needed + if len(backupEnvVars) > 0 { + envVars = append(envVars, backupEnvVars...) + } + // Only have a postStart command to create the chRoot, if it is not '/' (which does not need to be created) var postStart *corev1.Handler if hasChroot { diff --git a/controllers/util/solr_util_test.go b/controllers/util/solr_util_test.go index 5ffa8a92..333609c7 100644 --- a/controllers/util/solr_util_test.go +++ b/controllers/util/solr_util_test.go @@ -49,6 +49,13 @@ func TestGeneratedSolrXmlContainsEntryForEachRepository(t *testing.T) { }, }, }, + { + Name: "s3repository1", + S3: &solr.S3Repository{ + Bucket: "some-bucket-name1", + Region: "us-west-2", + }, + }, { Name: "managedrepository2", Managed: &solr.ManagedRepository{ @@ -66,6 +73,13 @@ func TestGeneratedSolrXmlContainsEntryForEachRepository(t *testing.T) { BaseLocation: "location-2", }, }, + { + Name: "s3repository2", + S3: &solr.S3Repository{ + Bucket: "some-bucket-name2", + Region: "ap-northeast-2", + }, + }, } xmlString, modules, libs := GenerateBackupRepositoriesForSolrXml(repos) @@ -75,8 +89,11 @@ func TestGeneratedSolrXmlContainsEntryForEachRepository(t *testing.T) { assert.Containsf(t, xmlString, "", "Did not find '%s' in the list of backup repositories", "managedrepository2") assert.Containsf(t, xmlString, "", "Did not find '%s' in the list of backup repositories", "gcsrepository1") assert.Containsf(t, xmlString, "", "Did not find '%s' in the list of backup repositories", "gcsrepository2") + assert.Containsf(t, xmlString, "", "Did not find '%s' in the list of backup repositories", "s3repository1") + assert.Containsf(t, xmlString, "", "Did not find '%s' in the list of backup repositories", "s3repository2") assert.Contains(t, modules, "gcs-repository", "The modules for the backupRepos should contain gcs-repository") + assert.Contains(t, modules, "s3-repository", "The modules for the backupRepos should contain s3-repository") assert.Empty(t, libs, "There should be no libs for the backupRepos") } diff --git a/docs/solr-backup/README.md b/docs/solr-backup/README.md index 077cd425..e0e9bea5 100644 --- a/docs/solr-backup/README.md +++ b/docs/solr-backup/README.md @@ -30,6 +30,8 @@ This page outlines how to create and delete a Kubernetes SolrBackup - [Creation](#creating-an-example-solrbackup) - [Deletion](#deleting-an-example-solrbackup) - [Repository Types](#supported-repository-types) + - [GCS](#gcs-backup-repositories) + - [S3](#s3-backup-repositories) ## Creating an example SolrBackup @@ -127,12 +129,14 @@ kubectl exec example-solrcloud-0 -- rm -r /var/solr/data/backup-restore-managed- ``` ## Supported Repository Types +_Since v0.5.0_ Note all repositories are defined in the `SolrCloud` specification. In order to use a repository in the `SolrBackup` CRD, it must be defined in the `SolrCloud` spec. All yaml examples below are `SolrCloud` resources, not `SolrBackup` resources. -The Solr-operator currently supports two different backup repository types: managed ("local") and Google Cloud Storage ("GCS") +The Solr-operator currently supports three different backup repository types: Google Cloud Storage ("GCS"), AWS S3 ("S3"), and managed ("local"). +The cloud backup solutions (GCS and S3) are strongly suggested over the managed option, though they require newer Solr releases. Multiple repositories can be defined under the `SolrCloud.spec.backupRepositories` field. Specify a unique name and repo type that you want to connect to. @@ -151,27 +155,8 @@ spec: ... ``` -### Managed ("Local") Backup Repositories - -Managed repositories store backup data "locally" on a Kubernetes volume mounted by each Solr pod. -Managed repositories are so called because with the data stored in and managed by Kubernetes, the operator is able to offer a few advanced post-processing features that are unavailable for other repository types. - -The main example of this currently is the operator's "persistence" feature, which upon completion of the backup will compress the backup files and optionally relocate the archive to a more permanent volume. See [the example here](../../example/test_backup_managed_with_persistence.yaml) for more details. - -An example of a SolrCloud spec with only one backup repository, with type Managed: - -```yaml -spec: - backupRepositories: - - name: "local-collection-backups-1" - managed: - volume: # Required - persistentVolumeClaim: - claimName: "collection-backup-pvc" - directory: "store/here" # Optional -``` - ### GCS Backup Repositories +_Since v0.5.0_ GCS Repositories store backup data remotely in Google Cloud Storage. This repository type is only supported in deployments that use a Solr version >= `8.9.0`. @@ -197,3 +182,95 @@ spec: key: "service-account-key.json" baseLocation: "/store/here" # Optional ``` + + +### S3 Backup Repositories +_Since v0.5.0_ + +S3 Repositories store backup data remotely in AWS S3 (or a supported S3 compatible interface). +This repository type is only supported in deployments that use a Solr version >= `8.10.0`. + +Each repository must specify an S3 bucket and region to store data in (the `bucket` and `region` properties). +Users will want to setup credentials so that the SolrCloud can connect to the S3 bucket and region, more information can be found in the [credentials section](#s3-credentials). + +```yaml +spec: + backupRepositories: + - name: "s3-backups-1" + s3: + region: "us-west-2" # Required + bucket: "backup-bucket" # Required + credentials: {} # Optional + proxyUrl: "https://proxy-url-for-s3:3242" # Optional + endpoint: "https://custom-s3-endpoint:3242" # Optional +``` + +Users can also optionally set a `proxyUrl` or `endpoint` for the S3Repository. +More information on these settings can be found in the [Ref Guide](https://solr.apache.org/guide/8_10/making-and-restoring-backups.html#s3backuprepository). + +#### S3 Credentials + +The Solr `S3Repository` module uses the [default credential chain for AWS](https://docs.aws.amazon.com/sdk-for-java/v2/developer-guide/credentials.html#credentials-chain). +All of the options below are designed to be utilized by this credential chain. + +There are a few options for giving a SolrCloud the credentials for connecting to S3. +The two most straightforward ways can be used via the `spec.backupRepositories.s3.credentials` property. + +```yaml +spec: + backupRepositories: + - name: "s3-backups-1" + s3: + region: "us-west-2" + bucket: "backup-bucket" + credentials: + accessKeyIdSecret: # Optional + name: aws-secrets + key: access-key-id + secretAccessKeySecret: # Optional + name: aws-secrets + key: secret-access-key + sessionTokenSecret: # Optional + name: aws-secrets + key: session-token + credentialsFileSecret: # Optional + name: aws-credentials + key: credentials +``` + +All options in the `credentials` property are optional, as users can pick and choose which ones to use. +If you have all of your credentials setup in an [AWS Credentials File](https://docs.aws.amazon.com/sdkref/latest/guide/file-format.html#file-format-creds), +then `credentialsFileSecret` will be the only property you need to set. +However, if you don't have a credentials file, you will likely need to set at least the `accessKeyIdSecret` and `secretAccessKeySecret` properties. +All of these options require the referenced Kuberentes secrets to already exist before creating the SolrCloud resource. +_(If desired, all options can be combined. e.g. Use `accessKeyIdSecret` and `credentialsFileSecret` together. The ordering of the default credentials chain will determine which options are used.)_ + +The options in the credentials file above merely set environment variables on the pod, or in the case of `credentialsFileSecret` use an environment variable and a volume mount. +Users can decide to not use the `credentials` section of the s3 repository config, and instead set these environment variables themselves via `spec.customSolrKubeOptions.podOptions.env`. + +Lastly, if running in EKS, it is possible to add [IAM information to Kubernetes serviceAccounts](https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/). +If this is done correctly, you will only need to specify the serviceAccount for the SolrCloud pods via `spec.customSolrKubeOptions.podOptions.serviceAccount`. + +_NOTE: Because the Solr S3 Repository is using system-wide settings for AWS credentials, you cannot specify different credentials for different S3 repositories. +This may be addressed in future Solr versions, but for now use the same credentials for all s3 repos._ + +### Managed ("Local") Backup Repositories +_Since v0.5.0_ + +Managed repositories store backup data "locally" on a Kubernetes volume mounted by each Solr pod. +Managed repositories are so called because with the data stored in and managed by Kubernetes, the operator is able to offer a few advanced post-processing features that are unavailable for other repository types. + +The main example of this currently is the operator's "persistence" feature, which upon completion of the backup will compress the backup files and optionally relocate the archive to a more permanent volume. See [the example here](../../example/test_backup_managed_with_persistence.yaml) for more details. + +An example of a SolrCloud spec with only one backup repository, with type Managed: + +```yaml +spec: + backupRepositories: + - name: "local-collection-backups-1" + managed: + volume: # Required + persistentVolumeClaim: + claimName: "collection-backup-pvc" + directory: "store/here" # Optional +``` diff --git a/docs/upgrade-notes.md b/docs/upgrade-notes.md index cb096c63..d9a8924f 100644 --- a/docs/upgrade-notes.md +++ b/docs/upgrade-notes.md @@ -106,6 +106,14 @@ _Note that the Helm chart version does not contain a `v` prefix, which the downl **Note**: Do not take backups while upgrading from the Solr Operator `v0.4.0` to `v0.5.0`. Wait for the SolrClouds to be updated, after the Solr Operator is upgraded, and complete their rolling restarts before continuing to use the Backup functionality. +- The location of Solr backup data as well as the name of the Solr backups have been changed, when using managed repositories. + Previously the name of the backup (in solr) was set to the name of the collection. + Now the name given to the backup in Solr will be set to `-`, without the `<` or `>` characters, where the `backup-resource-name` is the name of the SolrBackup resource. + + The directory in the Read-Write-Many Volume, required for managed repositories, that backups are written to is now `/backups` by default, instead of `/backups/`. + Because the backup name in Solr uses both the SolrBackup resource name and the collection name, there should be no collisions in this directory. + However, this can be overridden using the `SolrBackup.spec.location` option. + - Default ports when using TLS are now set to 443 instead of 80. This affects `solrCloud.Spec.SolrAddressability.CommonServicePort` and `solrCloud.Spec.SolrAddressability.CommonServicePort` field defaulting. Users already explicitly setting these values will not be affected. diff --git a/example/test_solrcloud.yaml b/example/test_solrcloud.yaml index 1757bb07..b8ba2471 100644 --- a/example/test_solrcloud.yaml +++ b/example/test_solrcloud.yaml @@ -26,12 +26,6 @@ spec: resources: requests: storage: "5Gi" - backupRestoreOptions: - managedRepositories: - - name: "local_backup_repo" - volume: - persistentVolumeClaim: - claimName: "pvc-test" replicas: 3 solrImage: tag: 8.7.0 diff --git a/example/test_solrcloud_backuprepos.yaml b/example/test_solrcloud_backuprepos.yaml index d8f8597d..498e07a7 100644 --- a/example/test_solrcloud_backuprepos.yaml +++ b/example/test_solrcloud_backuprepos.yaml @@ -45,6 +45,29 @@ spec: gcsCredentialSecret: name: "some-secret-name" key: "service-account-key.json" + + # Creates repositories that backup data to AWS S3. + # + # Requires Solr >= 8.9. + - name: "s3-backup-repo" + s3: + region: "us-west-2" + bucket: "product-catalog" + credentials: + credentials: + accessKeyIdSecret: # Optional + name: aws-secrets + key: access-key-id + secretAccessKeySecret: # Optional + name: aws-secrets + key: secret-access-key + sessionTokenSecret: # Optional + name: aws-secrets + key: session-token + credentialsFileSecret: # Optional + name: aws-credentials + key: credentials + - name: "main_collection_backup_repository_log" gcs: bucket: "log_data" diff --git a/helm/solr-operator/Chart.yaml b/helm/solr-operator/Chart.yaml index 4d97f4fb..96f1ff4e 100644 --- a/helm/solr-operator/Chart.yaml +++ b/helm/solr-operator/Chart.yaml @@ -85,14 +85,25 @@ annotations: - name: Backup Documentation url: https://apache.github.io/solr-operator/docs/solr-backup/ - kind: added - description: The ability to use GCS Repositories for the Solr Backup. + description: Introduced the ability to use GCS Backup Repositories with SolrCloud and SolrBackup. links: - name: Github Issue url: https://github.com/apache/solr-operator/issues/301 - name: Github PR url: https://github.com/apache/solr-operator/pull/302 - name: Backup Documentation - url: https://apache.github.io/solr-operator/docs/solr-backup/ + url: https://apache.github.io/solr-operator/docs/solr-backup#gcs-backup-repositories + - kind: added + description: Introduced the ability to use S3 Backup Repositories with SolrCloud and SolrBackup. + links: + - name: Github Issue + url: https://github.com/apache/solr-operator/issues/328 + - name: Github PR + url: https://github.com/apache/solr-operator/pull/345 + - name: Solr S3 Repository Documentation + url: https://solr.apache.org/guide/8_10/making-and-restoring-backups.html#s3backuprepository + - name: Backup Documentation + url: https://apache.github.io/solr-operator/docs/solr-backup#s3-backup-repositories - kind: added description: Customize the Lifecycle for Solr and PrometheusExporter containers links: @@ -109,6 +120,11 @@ annotations: url: https://github.com/apache/solr-operator/pull/332 - name: Solr Modules url: https://github.com/apache/solr/tree/main/solr/contrib + - kind: added + description: SolrBackups can now have a custom location specified to store the backup + links: + - name: Github PR + url: https://github.com/apache/solr-operator/pull/345 artifacthub.io/images: | - name: solr-operator image: apache/solr-operator:v0.5.0-prerelease @@ -157,13 +173,24 @@ annotations: solrOpts: "-Dsolr.autoSoftCommit.maxTime=10000" solrGCTune: "-XX:SurvivorRatio=4 -XX:TargetSurvivorRatio=90 -XX:MaxTenuringThreshold=8" backupRepositories: - - name: default + - name: default-gcs gcs: bucket: solr-gcs-backups gcsCredentialSecret: # Required name: gcs-credentials key: "service-account-key.json" baseLocation: "/solrcloud/backups" + - name: default-s3 + s3: + region: us-west-2 + bucket: solr-s3-backups + credentials: + accessKeyIdSecret: # Optional + name: aws-secrets + key: access-key-id + secretAccessKeySecret: # Optional + name: aws-secrets + key: secret-access-key - apiVersion: solr.apache.org/v1beta1 kind: SolrPrometheusExporter metadata: @@ -196,4 +223,5 @@ annotations: collections: - techproducts - books + location: "/this/location" artifacthub.io/containsSecurityUpdates: "false" diff --git a/helm/solr-operator/crds/crds.yaml b/helm/solr-operator/crds/crds.yaml index d3648166..fa8e34ad 100644 --- a/helm/solr-operator/crds/crds.yaml +++ b/helm/solr-operator/crds/crds.yaml @@ -67,6 +67,9 @@ spec: items: type: string type: array + location: + description: The location to store the backup in the specified backup repository. + type: string persistence: description: Persistence is the specification on how to persist the backup data. properties: @@ -1068,6 +1071,9 @@ spec: asyncBackupStatus: description: The status of the asynchronous backup call to solr type: string + backupName: + description: BackupName of this collection's backup in Solr + type: string collection: description: Solr Collection name type: string @@ -1127,7 +1133,6 @@ spec: description: Whether the backup was successful type: boolean required: - - persistenceStatus - solrVersion type: object type: object @@ -2151,6 +2156,92 @@ spec: name: description: 'A name used to identify this local storage profile. Values should follow RFC-1123. (See here for more details: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names)' type: string + s3: + description: An S3Repository for Solr to use when backing up and restoring collections. + properties: + baseLocation: + description: An already-created chroot within the bucket to store data in. Defaults to the root path "/" if not specified. + type: string + bucket: + description: The name of the S3 bucket that all backup data will be stored in + type: string + credentials: + description: "Options for specifying S3Credentials. This is optional in case you want to mount this information yourself. However, if you do not include these credentials, and you do not load them yourself via a mount or EnvVars, you will likely see errors when taking s3 backups. \n If running in EKS, you can create an IAMServiceAccount that uses a role permissioned for this S3 bucket. Then use that serviceAccountName for your SolrCloud, and the credentials should be auto-populated." + properties: + accessKeyIdSecret: + description: The name & key of a Kubernetes secret holding an AWS Access Key ID + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + credentialsFileSecret: + description: The name & key of a Kubernetes secret holding an AWS credentials file + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + secretAccessKeySecret: + description: The name & key of a Kubernetes secret holding an AWS Secret Access Key + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + sessionTokenSecret: + description: The name & key of a Kubernetes secret holding an AWS Session Token + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + type: object + endpoint: + description: The full endpoint URL to use when connecting with S3 (or a supported S3 compatible interface) + type: string + proxyUrl: + description: The full proxy URL to use when connecting with S3 + type: string + region: + description: The S3 region to store the backup data in + type: string + required: + - bucket + - region + type: object required: - name type: object