feat: backup and restore sqlite database

.gitignore

@@ -13,6 +13,7 @@ vendor
 # binary databases
 influxd.bolt
 *.db
+*.sqlite
 
 # GPG private keys
 private.key

CHANGELOG.md

@@ -13,6 +13,7 @@ This release adds an embedded SQLite database for storing metadata required by t
 1. [21543](https://github.com/influxdata/influxdb/pull/21543): Added `influxd` configuration flag `--sqlite-path` for specifying a user-defined path to the SQLite database file
 1. [21543](https://github.com/influxdata/influxdb/pull/21543): Updated `influxd` configuration flag `--store` to work with string values `disk` or `memory`. Memory continues to store metadata in-memory for testing; `disk` will persist metadata to disk via bolt and SQLite
 1. [21547](https://github.com/influxdata/influxdb/pull/21547): Allow hiding the tooltip independently of the static legend
+1. [21584](https://github.com/influxdata/influxdb/pull/21584): Added the `api/v2/backup/metadata` endpoint for backing up both KV and SQL metadata, and the `api/v2/restore/sql` for restoring SQL metadata.
 
 ### Bug Fixes
 

authorizer/backup.go

@@ -43,3 +43,16 @@ func (b BackupService) BackupShard(ctx context.Context, w io.Writer, shardID uin
 	}
 	return b.s.BackupShard(ctx, w, shardID, since)
 }
+
+// The Lock and Unlock methods below do not have authorization checks and should only be used
+// when appropriate authorization has already been confirmed, such as behind a middleware. They
+// are intended to be used for coordinating the locking and unlocking of the kv and sql metadata
+// databases during a backup. They are made available here to allow the calls to pass-through to the
+// underlying service.
+func (b BackupService) LockKVStore() {
+	b.s.LockKVStore()
+}
+
+func (b BackupService) UnlockKVStore() {
+	b.s.UnlockKVStore()
+}

authorizer/sql_backup_restore.go

@@ -0,0 +1,57 @@
+package authorizer
+
+import (
+	"context"
+	"io"
+
+	"github.com/influxdata/influxdb/v2"
+	"github.com/influxdata/influxdb/v2/kit/tracing"
+)
+
+var _ influxdb.SqlBackupRestoreService = (*SqlBackupRestoreService)(nil)
+
+// SqlBackupRestoreService wraps a influxdb.SqlBackupRestoreService and authorizes actions
+// against it appropriately.
+type SqlBackupRestoreService struct {
+	s influxdb.SqlBackupRestoreService
+}
+
+// NewSqlBackupRestoreService constructs an instance of an authorizing backup service.
+func NewSqlBackupRestoreService(s influxdb.SqlBackupRestoreService) *SqlBackupRestoreService {
+	return &SqlBackupRestoreService{
+		s: s,
+	}
+}
+
+func (s SqlBackupRestoreService) BackupSqlStore(ctx context.Context, w io.Writer) error {
+	span, ctx := tracing.StartSpanFromContext(ctx)
+	defer span.Finish()
+
+	if err := IsAllowedAll(ctx, influxdb.OperPermissions()); err != nil {
+		return err
+	}
+	return s.s.BackupSqlStore(ctx, w)
+}
+
+func (s SqlBackupRestoreService) RestoreSqlStore(ctx context.Context, r io.Reader) error {
+	span, ctx := tracing.StartSpanFromContext(ctx)
+	defer span.Finish()
+
+	if err := IsAllowedAll(ctx, influxdb.OperPermissions()); err != nil {
+		return err
+	}
+	return s.s.RestoreSqlStore(ctx, r)
+}
+
+// The Lock and Unlock methods below do not have authorization checks and should only be used
+// when appropriate authorization has already been confirmed, such as behind a middleware. They
+// are intended to be used for coordinating the locking and unlocking of the kv and sql metadata
+// databases during a backup. They are made available here to allow the calls to pass-through to the
+// underlying service.
+func (s SqlBackupRestoreService) LockSqlStore() {
+	s.s.LockSqlStore()
+}
+
+func (s SqlBackupRestoreService) UnlockSqlStore() {
+	s.s.UnlockSqlStore()
+}

authorizer/sql_backup_restore_test.go

@@ -0,0 +1,104 @@
+package authorizer_test
+
+import (
+	"bytes"
+	"context"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/influxdata/influxdb/v2/kit/platform/errors"
+	"github.com/stretchr/testify/require"
+
+	"github.com/influxdata/influxdb/v2"
+	"github.com/influxdata/influxdb/v2/authorizer"
+	influxdbcontext "github.com/influxdata/influxdb/v2/context"
+	"github.com/influxdata/influxdb/v2/mock"
+)
+
+func Test_BackupSqlStore(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		permList []influxdb.Permission
+		wantErr  error
+	}{
+		{
+			"authorized to do the backup",
+			influxdb.OperPermissions(),
+			nil,
+		},
+		{
+			"not authorized to do the backup",
+			influxdb.ReadAllPermissions(),
+			&errors.Error{
+				Msg:  "write:authorizations is unauthorized",
+				Code: errors.EUnauthorized,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctrlr := gomock.NewController(t)
+			svc := mock.NewMockSqlBackupRestoreService(ctrlr)
+			s := authorizer.NewSqlBackupRestoreService(svc)
+
+			w := bytes.NewBuffer([]byte{})
+
+			if tt.wantErr == nil {
+				svc.EXPECT().
+					BackupSqlStore(gomock.Any(), w).
+					Return(nil)
+			}
+
+			ctx := influxdbcontext.SetAuthorizer(context.Background(), mock.NewMockAuthorizer(false, tt.permList))
+			err := s.BackupSqlStore(ctx, w)
+			require.Equal(t, tt.wantErr, err)
+		})
+	}
+}
+
+func Test_RestoreSqlStore(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		permList []influxdb.Permission
+		wantErr  error
+	}{
+		{
+			"authorized to do the restore",
+			influxdb.OperPermissions(),
+			nil,
+		},
+		{
+			"not authorized to do the restore",
+			influxdb.ReadAllPermissions(),
+			&errors.Error{
+				Msg:  "write:authorizations is unauthorized",
+				Code: errors.EUnauthorized,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctrlr := gomock.NewController(t)
+			svc := mock.NewMockSqlBackupRestoreService(ctrlr)
+			s := authorizer.NewSqlBackupRestoreService(svc)
+
+			w := bytes.NewBuffer([]byte{})
+
+			if tt.wantErr == nil {
+				svc.EXPECT().
+					RestoreSqlStore(gomock.Any(), w).
+					Return(nil)
+			}
+
+			ctx := influxdbcontext.SetAuthorizer(context.Background(), mock.NewMockAuthorizer(false, tt.permList))
+			err := s.RestoreSqlStore(ctx, w)
+			require.Equal(t, tt.wantErr, err)
+		})
+	}
+}

backup.go

@@ -19,6 +19,27 @@ type BackupService interface {
 
 	// BackupShard downloads a backup file for a single shard.
 	BackupShard(ctx context.Context, w io.Writer, shardID uint64, since time.Time) error
+
+	// LockKVStore locks the database.
+	LockKVStore()
+
+	// UnlockKVStore unlocks the database.
+	UnlockKVStore()
+}
+
+// SqlBackupRestoreService represents the backup and restore functions for the sqlite database.
+type SqlBackupRestoreService interface {
+	// BackupSqlStore creates a live backup copy of the sqlite database.
+	BackupSqlStore(ctx context.Context, w io.Writer) error
+
+	// RestoreSqlStore restores & replaces the sqlite database.
+	RestoreSqlStore(ctx context.Context, r io.Reader) error
+
+	// LockSqlStore locks the database.
+	LockSqlStore()
+
+	// UnlockSqlStore unlocks the database.
+	UnlockSqlStore()
 }
 
 // RestoreService represents the data restore functions of InfluxDB.

bolt/kv.go

@@ -105,6 +105,16 @@ func (s *KVStore) Close() error {
 	return nil
 }
 
+// LockKVStore locks the database for reading during a backup.
+func (s *KVStore) Lock() {
+	s.mu.RLock()
+}
+
+// UnlockKVStore removes the read lock used during a backup.
+func (s *KVStore) Unlock() {
+	s.mu.RUnlock()
+}
+
 // DB returns a reference to the current Bolt database.
 func (s *KVStore) DB() *bolt.DB {
 	s.mu.RLock()

cmd/influxd/launcher/engine.go

@@ -164,6 +164,14 @@ func (t *TemporaryEngine) BackupKVStore(ctx context.Context, w io.Writer) error
 	return t.engine.BackupKVStore(ctx, w)
 }
 
+func (t *TemporaryEngine) LockKVStore() {
+	t.engine.LockKVStore()
+}
+
+func (t *TemporaryEngine) UnlockKVStore() {
+	t.engine.UnlockKVStore()
+}
+
 func (t *TemporaryEngine) RestoreKVStore(ctx context.Context, r io.Reader) error {
 	return t.engine.RestoreKVStore(ctx, r)
 }

cmd/influxd/launcher/launcher.go

@@ -771,12 +771,13 @@ func (m *Launcher) run(ctx context.Context, opts *InfluxdOpts) (err error) {
 			BucketFinder:  ts.BucketService,
 			LogBucketName: platform.MonitoringSystemBucketName,
 		},
-		DeleteService:          deleteService,
-		BackupService:          backupService,
-		RestoreService:         restoreService,
-		AuthorizationService:   authSvc,
-		AuthorizationV1Service: authSvcV1,
-		PasswordV1Service:      passwordV1,
+		DeleteService:           deleteService,
+		BackupService:           backupService,
+		SqlBackupRestoreService: m.sqlStore,
+		RestoreService:          restoreService,
+		AuthorizationService:    authSvc,
+		AuthorizationV1Service:  authSvcV1,
+		PasswordV1Service:       passwordV1,
 		AuthorizerV1: &authv1.Authorizer{
 			AuthV1:   authSvcV1,
 			AuthV2:   authSvc,

http/api_handler.go

@@ -64,6 +64,7 @@ type APIBackend struct {
 	PointsWriter                    storage.PointsWriter
 	DeleteService                   influxdb.DeleteService
 	BackupService                   influxdb.BackupService
+	SqlBackupRestoreService         influxdb.SqlBackupRestoreService
 	RestoreService                  influxdb.RestoreService
 	AuthorizationService            influxdb.AuthorizationService
 	AuthorizationV1Service          influxdb.AuthorizationService
@@ -199,10 +200,12 @@ func NewAPIHandler(b *APIBackend, opts ...APIHandlerOptFn) *APIHandler {
 
 	backupBackend := NewBackupBackend(b)
 	backupBackend.BackupService = authorizer.NewBackupService(backupBackend.BackupService)
+	backupBackend.SqlBackupRestoreService = authorizer.NewSqlBackupRestoreService(backupBackend.SqlBackupRestoreService)
 	h.Mount(prefixBackup, NewBackupHandler(backupBackend))
 
 	restoreBackend := NewRestoreBackend(b)
 	restoreBackend.RestoreService = authorizer.NewRestoreService(restoreBackend.RestoreService)
+	restoreBackend.SqlBackupRestoreService = authorizer.NewSqlBackupRestoreService(restoreBackend.SqlBackupRestoreService)
 	h.Mount(prefixRestore, NewRestoreHandler(restoreBackend))
 
 	h.Mount(dbrp.PrefixDBRP, dbrp.NewHTTPHandler(b.Logger, b.DBRPService, b.OrganizationService))