Allow users to take GCS-based backups

api/v1beta1/solrbackup_types.go

@@ -38,12 +38,16 @@ type SolrBackupSpec struct {
 	// A reference to the SolrCloud to create a backup for
 	SolrCloud string `json:"solrCloud"`
 
+	// The name of the repository to use for the backup.  Defaults to "legacy_local_repository" if not specified (the
+	// auto-configured repository for legacy singleton volumes).
+	RepositoryName string `json:"repositoryName,omitempty"`
+
 	// The list of collections to backup. If empty, all collections in the cloud will be backed up.
 	// +optional
 	Collections []string `json:"collections,omitempty"`
 
 	// Persistence is the specification on how to persist the backup data.
-	Persistence PersistenceSource `json:"persistence"`
+	Persistence PersistenceSource `json:"persistence,omitempty"`
 }
 
 func (spec *SolrBackupSpec) withDefaults(backupName string) (changed bool) {

api/v1beta1/solrcloud_types.go

@@ -19,15 +19,13 @@ package v1beta1
 
 import (
 	"fmt"
-	"strconv"
-	"strings"
-
-	"k8s.io/apimachinery/pkg/util/intstr"
-
 	zk "github.com/pravega/zookeeper-operator/pkg/apis/zookeeper/v1beta1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"strconv"
+	"strings"
 )
 
 const (
@@ -55,6 +53,12 @@ const (
 	ZookeeperTechnologyLabel = "zookeeper"
 
 	DefaultBasicAuthUsername = "k8s-oper"
+
+	DefaultBackupRepositoryName = "legacy_local_repository"
+	BaseBackupRestorePath = "/var/solr/data/backup-restore"
+	BackupRestoreInitContainerPath  = "/backup-restore"
+	BackupRestoreVolume = "backup-restore"
+	BackupRestoreCredentialVolume = "backup-restore-credential"
 )
 
 // SolrCloudSpec defines the desired state of SolrCloud
@@ -217,7 +221,7 @@ type SolrDataStorageOptions struct {
 	// +optional
 	EphemeralStorage *SolrEphemeralDataStorageOptions `json:"ephemeral,omitempty"`
 
-	// Options required for backups & restores to be enabled for this solrCloud.
+	// Options required for backups to be enabled for this solrCloud.
 	// +optional
 	BackupRestoreOptions *SolrBackupRestoreOptions `json:"backupRestoreOptions,omitempty"`
 }
@@ -325,20 +329,158 @@ type SolrEphemeralDataStorageOptions struct {
 	EmptyDir *corev1.EmptyDirVolumeSource `json:"emptyDir,omitempty"`
 }
 
+func (backupOptions *SolrBackupRestoreOptions) IsManaged() bool {
+	return true
+}
+
+func (backupOptions *SolrBackupRestoreOptions) GetVolumeSource() corev1.VolumeSource {
+	return *backupOptions.Volume
+}
+
+func (backupOptions *SolrBackupRestoreOptions) GetDirectory() string {
+	return backupOptions.Directory
+}
+
+func (backupOptions *SolrBackupRestoreOptions) GetSolrMountPath() string {
+	return BaseBackupRestorePath
+}
+
+func (backupOptions *SolrBackupRestoreOptions) GetInitPodMountPath() string {
+	return BackupRestoreInitContainerPath
+}
+
+func (backupOptions *SolrBackupRestoreOptions) GetBackupPath(backupName string) string {
+	return backupOptions.GetSolrMountPath() + "/backups/" + backupName
+}
+
+func (backupOptions *SolrBackupRestoreOptions) GetVolumeName() string {
+	return BackupRestoreVolume
+}
+
 type SolrBackupRestoreOptions struct {
+	// TODO Do we need to support this (Volume) here for backcompat, or can it live exclusively in ManagedStorage?
+
 	// This is a volumeSource for a volume that will be mounted to all solrNodes to store backups and load restores.
 	// The data within the volume will be namespaces for this instance, so feel free to use the same volume for multiple clouds.
 	// Since the volume will be mounted to all solrNodes, it must be able to be written from multiple pods.
 	// If a PVC reference is given, the PVC must have `accessModes: - ReadWriteMany`.
 	// Other options are to use a NFS volume.
-	Volume corev1.VolumeSource `json:"volume"`
+	Volume *corev1.VolumeSource `json:"volume,omitempty"`
+
+	// Allows specification of multiple different "repositories" for Solr to use when backing up data to GCS.
+	GcsRepositories *[]GcsStorage `json:"gcsRepositories,omitempty"`
 
+	// Allows specification of multiple different "repositories" for Solr to use when backing up data "locally".
+	// Repositories defined here are considered "managed" and can take advantage of special operator features, such as
+	// post-backup compression.
+	ManagedRepositories *[]ManagedStorage `json:"managedRepositories,omitempty""`
+
+	// TODO Do we need to support this here for backcompat, or can it live exclusively in ManagedStorage
 	// Select a custom directory name to mount the backup/restore data from the given volume.
 	// If not specified, then the name of the solrcloud will be used by default.
 	// +optional
 	Directory string `json:"directory,omitempty"`
 }
 
+type GcsStorage struct {
+	// A name used to identify this GCS storage profile.
+	Name string `json:"name"`
+
+	// The name of the GCS bucket that all backup data will be stored in
+	Bucket string `json:"bucket"`
+
+	// The name of a Kubernetes secret holding a Google cloud service account key
+	GcsCredentialSecret string `json:"gcsCredentialSecret"`
+	// JEGERLOW TODO Should 'baseLocation' be optional?
+	// A chroot within the bucket to store data in.  If specified this should already exist
+	BaseLocation string `json:"baseLocation"`
+}
+
+func VolumeExistsWithName(needle string, haystack []corev1.Volume) bool {
+	for _, volume := range haystack {
+		if volume.Name == needle {
+			return true
+		}
+	}
+	return false
+}
+
+func (gcsRepository *GcsStorage) IsManaged() bool { return false }
+
+func (gcsRepository *GcsStorage) GetSolrMountPath() string {
+	return fmt.Sprintf("%s-%s-%s", BaseBackupRestorePath, "gcscredential", gcsRepository.Name)
+}
+
+func (gcsRepository *GcsStorage) GetInitPodMountPath() string {
+	return ""
+}
+
+func (gcsRepository *GcsStorage) GetBackupPath(backupName string) string {
+	if gcsRepository.BaseLocation != "" {
+		return gcsRepository.BaseLocation
+	}
+	return "/"
+}
+
+func (gcsRepository *GcsStorage) GetVolumeName() string {
+	return fmt.Sprintf("%s-%s", gcsRepository.Name, BackupRestoreCredentialVolume)
+}
+
+func (gcs *GcsStorage) IsCredentialVolumePresent(volumes []corev1.Volume) bool {
+	expectedVolumeName := gcs.GetVolumeName()
+	return VolumeExistsWithName(expectedVolumeName, volumes)
+}
+
+type ManagedStorage struct {
+	// A name used to identify this local storage profile.
+	Name string `json:"name"`
+
+	// This is a volumeSource for a volume that will be mounted to all solrNodes to store backups and load restores.
+	// The data within the volume will be namespaces for this instance, so feel free to use the same volume for multiple clouds.
+	// Since the volume will be mounted to all solrNodes, it must be able to be written from multiple pods.
+	// If a PVC reference is given, the PVC must have `accessModes: - ReadWriteMany`.
+	// Other options are to use a NFS volume.
+	Volume *corev1.VolumeSource `json:"volume,omitempty"`
+
+	// Select a custom directory name to mount the backup/restore data from the given volume.
+	// If not specified, then the name of the solrcloud will be used by default.
+	// +optional
+	Directory string `json:"directory,omitempty"`
+}
+
+func (managedRepository *ManagedStorage) IsManaged() bool { return true }
+
+func (managedRepository *ManagedStorage) GetVolumeSource() corev1.VolumeSource {
+	return *managedRepository.Volume
+}
+
+func (managedRepository *ManagedStorage) GetDirectory() string {
+	return managedRepository.Directory
+}
+
+func (managedRepository *ManagedStorage) GetSolrMountPath() string {
+	return fmt.Sprintf("%s-%s-%s", BaseBackupRestorePath, "managed", managedRepository.Name)
+}
+
+func (managedRepository *ManagedStorage) GetInitPodMountPath() string {
+	return fmt.Sprintf("/backup-restore-managed-%s", managedRepository.Name)
+}
+
+func (managedRepository *ManagedStorage) GetBackupPath(backupName string) string {
+	return managedRepository.GetSolrMountPath() + "/backups/" + backupName
+}
+
+func (managedRepository *ManagedStorage) GetVolumeName() string {
+	return fmt.Sprintf("%s-%s", managedRepository.Name, BackupRestoreVolume)
+}
+
+// TODO JEGERLOW: Is this the most natural place for this?  'IsDataVolumePresent' seems like a method that'd fit
+// better on 'pod' and take in a ManagedStorage
+func (managedRepository *ManagedStorage) IsBackupVolumePresent(volumes []corev1.Volume) bool {
+	expectedVolumeName := managedRepository.GetVolumeName()
+	return VolumeExistsWithName(expectedVolumeName, volumes)
+}
+
 type SolrAddressabilityOptions struct {
 	// External defines the way in which this SolrCloud nodes should be made addressable externally, from outside the Kubernetes cluster.
 	// If none is provided, the Solr Cloud will not be made addressable externally.
@@ -1140,4 +1282,4 @@ type SolrSecurityOptions struct {
 	// endpoints with credentials sourced from an env var instead of HTTP directly.
 	// +optional
 	ProbesRequireAuth bool `json:"probesRequireAuth,omitempty"`
-}
+}

config/crd/bases/solr.apache.org_solrbackups.yaml

@@ -1048,11 +1048,13 @@ spec:
                     - source
                     type: object
                 type: object
+              repositoryName:
+                description: The name of the repository to use for the backup.  Defaults to "legacy_local_repository" if not specified (the auto-configured repository for legacy singleton volumes).
+                type: string
               solrCloud:
                 description: A reference to the SolrCloud to create a backup for
                 type: string
             required:
-            - persistence
             - solrCloud
             type: object
           status: