Allow users to take GCS-based backups

api/v1beta1/solr_cloud_type_helpers.go

@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package v1beta1
+
+import (
+	"fmt"
+	"k8s.io/api/core/v1"
+)
+
+func (backupOptions *SolrBackupRestoreOptions) IsManaged() bool {
+	return true
+}
+
+func (backupOptions *SolrBackupRestoreOptions) GetVolumeSource() v1.VolumeSource {
+	return *backupOptions.Volume
+}
+
+func (backupOptions *SolrBackupRestoreOptions) GetDirectory() string {
+	return backupOptions.Directory
+}
+
+func (backupOptions *SolrBackupRestoreOptions) GetSolrMountPath() string {
+	return BaseBackupRestorePath
+}
+
+func (backupOptions *SolrBackupRestoreOptions) GetInitPodMountPath() string {
+	return BackupRestoreInitContainerPath
+}
+
+func (backupOptions *SolrBackupRestoreOptions) GetBackupPath(backupName string) string {
+	return backupOptions.GetSolrMountPath() + "/backups/" + backupName
+}
+
+func (backupOptions *SolrBackupRestoreOptions) GetVolumeName() string {
+	return BackupRestoreVolume
+}
+
+func (gcsRepository *GcsStorage) IsManaged() bool { return false }
+
+func (gcsRepository *GcsStorage) GetSolrMountPath() string {
+	return fmt.Sprintf("%s-%s-%s", BaseBackupRestorePath, "gcscredential", gcsRepository.Name)
+}
+
+func (gcsRepository *GcsStorage) GetInitPodMountPath() string {
+	return ""
+}
+
+func (gcsRepository *GcsStorage) GetBackupPath(backupName string) string {
+	if gcsRepository.BaseLocation != "" {
+		return gcsRepository.BaseLocation
+	}
+	return "/"
+}
+
+func (gcsRepository *GcsStorage) GetVolumeName() string {
+	return fmt.Sprintf("%s-%s", gcsRepository.Name, BackupRestoreCredentialVolume)
+}
+
+func (managedRepository *ManagedStorage) IsManaged() bool { return true }
+
+func (managedRepository *ManagedStorage) GetVolumeSource() v1.VolumeSource {
+	return *managedRepository.Volume
+}
+
+func (managedRepository *ManagedStorage) GetDirectory() string {
+	return managedRepository.Directory
+}
+
+func (managedRepository *ManagedStorage) GetSolrMountPath() string {
+	return fmt.Sprintf("%s-%s-%s", BaseBackupRestorePath, "managed", managedRepository.Name)
+}
+
+func (managedRepository *ManagedStorage) GetInitPodMountPath() string {
+	return fmt.Sprintf("/backup-restore-managed-%s", managedRepository.Name)
+}
+
+func (managedRepository *ManagedStorage) GetBackupPath(backupName string) string {
+	return managedRepository.GetSolrMountPath() + "/backups/" + backupName
+}
+
+func (managedRepository *ManagedStorage) GetVolumeName() string {
+	return fmt.Sprintf("%s-%s", managedRepository.Name, BackupRestoreVolume)
+}
+
+func (gcs *GcsStorage) IsCredentialVolumePresent(volumes []v1.Volume) bool {
+	expectedVolumeName := gcs.GetVolumeName()
+	return VolumeExistsWithName(expectedVolumeName, volumes)
+}
+
+func (managedRepository *ManagedStorage) IsBackupVolumePresent(volumes []v1.Volume) bool {
+	expectedVolumeName := managedRepository.GetVolumeName()
+	return VolumeExistsWithName(expectedVolumeName, volumes)
+}
+
+func VolumeExistsWithName(needle string, haystack []v1.Volume) bool {
+	for _, volume := range haystack {
+		if volume.Name == needle {
+			return true
+		}
+	}
+	return false
+}

api/v1beta1/solrbackup_types.go

@@ -38,16 +38,24 @@ type SolrBackupSpec struct {
 	// A reference to the SolrCloud to create a backup for
 	SolrCloud string `json:"solrCloud"`
 
+	// The name of the repository to use for the backup.  Defaults to "legacy_local_repository" if not specified (the
+	// auto-configured repository for legacy singleton volumes).
+	// +optional
+	RepositoryName string `json:"repositoryName,omitempty"`
+
 	// The list of collections to backup. If empty, all collections in the cloud will be backed up.
 	// +optional
 	Collections []string `json:"collections,omitempty"`
 
 	// Persistence is the specification on how to persist the backup data.
-	Persistence PersistenceSource `json:"persistence"`
+	// +optional
+	Persistence *PersistenceSource `json:"persistence,omitempty"`
 }
 
 func (spec *SolrBackupSpec) withDefaults(backupName string) (changed bool) {
-	changed = spec.Persistence.withDefaults(backupName) || changed
+	if spec.Persistence != nil {
+		changed = spec.Persistence.withDefaults(backupName) || changed
+	}
 
 	return changed
 }

api/v1beta1/solrcloud_types.go

@@ -19,15 +19,13 @@ package v1beta1
 
 import (
 	"fmt"
-	"strconv"
-	"strings"
-
-	"k8s.io/apimachinery/pkg/util/intstr"
-
 	zk "github.com/pravega/zookeeper-operator/pkg/apis/zookeeper/v1beta1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"strconv"
+	"strings"
 )
 
 const (
@@ -55,6 +53,12 @@ const (
 	ZookeeperTechnologyLabel = "zookeeper"
 
 	DefaultBasicAuthUsername = "k8s-oper"
+
+	DefaultBackupRepositoryName    = "legacy_local_repository"
+	BaseBackupRestorePath          = "/var/solr/data/backup-restore"
+	BackupRestoreInitContainerPath = "/backup-restore"
+	BackupRestoreVolume            = "backup-restore"
+	BackupRestoreCredentialVolume  = "backup-restore-credential"
 )
 
 // SolrCloudSpec defines the desired state of SolrCloud
@@ -217,7 +221,7 @@ type SolrDataStorageOptions struct {
 	// +optional
 	EphemeralStorage *SolrEphemeralDataStorageOptions `json:"ephemeral,omitempty"`
 
-	// Options required for backups & restores to be enabled for this solrCloud.
+	// Options required for backups to be enabled for this solrCloud.
 	// +optional
 	BackupRestoreOptions *SolrBackupRestoreOptions `json:"backupRestoreOptions,omitempty"`
 }
@@ -331,7 +335,50 @@ type SolrBackupRestoreOptions struct {
 	// Since the volume will be mounted to all solrNodes, it must be able to be written from multiple pods.
 	// If a PVC reference is given, the PVC must have `accessModes: - ReadWriteMany`.
 	// Other options are to use a NFS volume.
-	Volume corev1.VolumeSource `json:"volume"`
+	// Deprecated: Create an explicit 'managedRepositories' entry instead.
+	Volume *corev1.VolumeSource `json:"volume,omitempty"`
+
+	// Allows specification of multiple different "repositories" for Solr to use when backing up data to GCS.
+	GcsRepositories *[]GcsStorage `json:"gcsRepositories,omitempty"`
+
+	// Allows specification of multiple different "repositories" for Solr to use when backing up data "locally".
+	// Repositories defined here are considered "managed" and can take advantage of special operator features, such as
+	// post-backup compression.
+	ManagedRepositories *[]ManagedStorage `json:"managedRepositories,omitempty"`
+
+	// Select a custom directory name to mount the backup/restore data from the given volume.
+	// If not specified, then the name of the solrcloud will be used by default.
+	// Deprecated: Create an explicit 'managedRepositories' entry instead.
+	// +optional
+	Directory string `json:"directory,omitempty"`
+}
+
+type GcsStorage struct {
+	// A name used to identify this GCS storage profile.  Values should follow RFC-1123.  (See here for more details:
+	// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names)
+	Name string `json:"name"`
+
+	// The name of the GCS bucket that all backup data will be stored in
+	Bucket string `json:"bucket"`
+
+	// The name of a Kubernetes secret holding a Google cloud service account key
+	GcsCredentialSecret string `json:"gcsCredentialSecret"`
+	// An already-created chroot within the bucket to store data in. Defaults to the root path "/" if not specified.
+	// +optional
+	BaseLocation string `json:"baseLocation,omitempty"`
+}
+
+type ManagedStorage struct {
+	// A name used to identify this local storage profile.  Values should follow RFC-1123.  (See here for more details:
+	// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names)
+	Name string `json:"name"`
+
+	// This is a volumeSource for a volume that will be mounted to all solrNodes to store backups and load restores.
+	// The data within the volume will be namespaced for this instance, so feel free to use the same volume for multiple clouds.
+	// Since the volume will be mounted to all solrNodes, it must be able to be written from multiple pods.
+	// If a PVC reference is given, the PVC must have `accessModes: - ReadWriteMany`.
+	// Other options are to use a NFS volume.
+	Volume *corev1.VolumeSource `json:"volume"`
 
 	// Select a custom directory name to mount the backup/restore data from the given volume.
 	// If not specified, then the name of the solrcloud will be used by default.

config/crd/bases/solr.apache.org_solrbackups.yaml

@@ -1048,11 +1048,13 @@ spec:
                     - source
                     type: object
                 type: object
+              repositoryName:
+                description: The name of the repository to use for the backup.  Defaults to "legacy_local_repository" if not specified (the auto-configured repository for legacy singleton volumes).
+                type: string
               solrCloud:
                 description: A reference to the SolrCloud to create a backup for
                 type: string
             required:
-            - persistence
             - solrCloud
             type: object
           status: