Automate README example generation

cmd/readme/readme_example_generator.go

@@ -0,0 +1,196 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"regexp"
+	"time"
+
+	"github.com/aerospike/aerospike-backup-service/v2/pkg/dto"
+	"github.com/aerospike/aerospike-backup-service/v2/pkg/util"
+	"gopkg.in/yaml.v3"
+)
+
+var allStorageTypes = map[string]dto.Storage{
+	"local": {
+		LocalStorage: &dto.LocalStorage{
+			Path: "backups",
+		},
+	},
+	"aws-s3": {
+		S3Storage: &dto.S3Storage{
+			Bucket:   "as-backup-bucket",
+			Path:     "backups",
+			S3Region: "eu-central-1",
+		},
+	},
+	"gcp-gcs": {
+		GcpStorage: &dto.GcpStorage{
+			Path:       "backups",
+			KeyFile:    "key-file.json",
+			BucketName: "gcp-backup-bucket",
+			Endpoint:   "http://127.0.0.1:9020",
+		},
+	},
+	"azure-blob-storage": {
+		AzureStorage: &dto.AzureStorage{
+			Path:          "backups",
+			Endpoint:      "http://127.0.0.1:6000/devstoreaccount1",
+			AccountName:   "devstoreaccount1",
+			ContainerName: "testcontainer",
+			AccountKey:    "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
+		},
+	},
+}
+
+var cluster = dto.AerospikeCluster{
+	SeedNodes: []dto.SeedNode{{
+		HostName: "host.docker.internal", Port: 3000},
+	},
+	Credentials: &dto.Credentials{
+		User:     util.Ptr("user"),
+		Password: util.Ptr("password"),
+	},
+}
+
+var jsonExamples = map[string]any{
+	"ClustersResponse": []dto.AerospikeCluster{cluster},
+	"RoutinesResponse": map[string]dto.BackupRoutine{
+		"routine1": {
+			BackupPolicy:  "keepFilesPolicy",
+			SourceCluster: "absDefaultCluster",
+			Storage:       "local",
+			IntervalCron:  "@yearly",
+			Namespaces:    []string{"test-namespace"},
+		},
+		"routine2": {
+			BackupPolicy:     "removeFilesPolicy",
+			SourceCluster:    "absDefaultCluster",
+			Storage:          "local",
+			IntervalCron:     "@monthly",
+			IncrIntervalCron: "@daily",
+			Namespaces:       []string{"test-namespace"},
+			SetList:          []string{"backupSet"},
+			BinList:          []string{"backupBin"},
+		},
+	},
+	"StorageResponse": allStorageTypes,
+	"FullBackupsResponse": map[string][]dto.BackupDetails{
+		"routine1": {{
+			BackupMetadata: dto.BackupMetadata{
+				Created:             time.Date(2024, 01, 01, 12, 0, 0, 0, time.UTC),
+				Namespace:           "source-ns1",
+				ByteCount:           480_000,
+				FileCount:           1,
+				UDFCount:            1,
+				RecordCount:         42,
+				SecondaryIndexCount: 5,
+			},
+			Key: "routine1/backup/1704110400000/source-ns1",
+			Storage: &dto.Storage{
+				S3Storage: &dto.S3Storage{
+					Bucket:   "as-backup-bucket",
+					Path:     "backups",
+					S3Region: "eu-central-1",
+				},
+			}},
+		},
+		"routine2": {{
+			BackupMetadata: dto.BackupMetadata{
+				Created:     time.Date(2024, 01, 01, 12, 0, 0, 0, time.UTC),
+				Namespace:   "source-ns2",
+				ByteCount:   1_234_567_890,
+				RecordCount: 1890,
+				FileCount:   4,
+			},
+			Key: "routine2/backup/1704110400000/source-ns2",
+			Storage: &dto.Storage{
+				S3Storage: &dto.S3Storage{
+					Bucket:   "as-backup-bucket",
+					Path:     "backups",
+					S3Region: "eu-central-1",
+				},
+			},
+		}},
+	},
+	"RestoreFullRequest": dto.RestoreRequest{
+		DestinationCluster: &cluster,
+		Policy: &dto.RestorePolicy{
+			NoGeneration: util.Ptr(true),
+		},
+		SourceStorage: &dto.Storage{
+			S3Storage: &dto.S3Storage{
+				Bucket:   "as-backup-bucket",
+				Path:     "backups",
+				S3Region: "eu-central-1",
+			},
+		},
+		BackupDataPath: "routine1/backup/1704110400000/source-ns1",
+	},
+	"RestoreTimestampRequest": dto.RestoreTimestampRequest{
+		DestinationCluster: &cluster,
+		Policy: &dto.RestorePolicy{
+			NoGeneration: util.Ptr(true),
+		},
+		Time:    1704110400000,
+		Routine: "routine1",
+	},
+}
+
+var yamlExamples = map[string]any{
+	"Storage": allStorageTypes,
+}
+
+func main() {
+	_ = dto.AerospikeCluster{}
+	readme, err := os.ReadFile("README.md")
+	if err != nil {
+		panic(err)
+	}
+
+	// comment containing an example name (e.g.,key from jsonExamples)
+	// followed by ```json/```yaml and the example code block.
+	re := regexp.MustCompile("<!--\\s*(\\w+)\\s*-->\\s*```(json|yaml)[\\s\\S]*?```")
+
+	updatedReadme := re.ReplaceAllFunc(readme, func(match []byte) []byte {
+		submatches := re.FindSubmatch(match)
+		if len(submatches) < 3 {
+			return match
+		}
+
+		name := string(submatches[1])
+		format := string(submatches[2])
+
+		var example any
+		var exists bool
+		var formattedExample []byte
+
+		if format == "json" {
+			example, exists = jsonExamples[name]
+			if exists {
+				formattedExample, err = json.MarshalIndent(example, "", "  ")
+			}
+		} else if format == "yaml" {
+			example, exists = yamlExamples[name]
+			if exists {
+				formattedExample, err = yaml.Marshal(example)
+			}
+		}
+
+		if !exists || err != nil {
+			return match
+		}
+
+		var buffer bytes.Buffer
+		buffer.WriteString(fmt.Sprintf("<!-- %s -->\n```%s\n", name, format))
+		buffer.Write(formattedExample)
+		buffer.WriteString("\n```")
+		return buffer.Bytes()
+	})
+	err = os.WriteFile("README.md", updatedReadme, 0600)
+	if err != nil {
+		panic(err)
+	}
+}

internal/server/handlers/restore_test.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/aerospike/aerospike-backup-service/v2/pkg/dto"
 	"github.com/aerospike/aerospike-backup-service/v2/pkg/util"
-	"github.com/aws/smithy-go/ptr"
 	"github.com/gorilla/mux"
 	"github.com/steinfletcher/apitest"
 	"github.com/stretchr/testify/require"
@@ -30,7 +29,7 @@ func testRestoreRequest() dto.RestoreRequest {
 		Policy:             policy,
 		SourceStorage:      storage,
 		SecretAgent:        nil,
-		BackupDataPath:     ptr.String(testDir),
+		BackupDataPath:     testDir,
 	}
 }
 

pkg/dto/backup_routine.go

@@ -26,7 +26,7 @@ type BackupRoutine struct {
 	// The interval for full backup as a cron expression string.
 	IntervalCron string `yaml:"interval-cron" json:"interval-cron" example:"0 0 * * * *" validate:"required"`
 	// The interval for incremental backup as a cron expression string (optional).
-	IncrIntervalCron string `yaml:"incr-interval-cron" json:"incr-interval-cron" example:"*/10 * * * * *"`
+	IncrIntervalCron string `yaml:"incr-interval-cron,omitempty" json:"incr-interval-cron,omitempty" example:"*/10 * * * * *"`
 	// The list of the namespaces to back up (optional, empty list implies backup up whole cluster).
 	Namespaces []string `yaml:"namespaces,omitempty" json:"namespaces,omitempty" example:"source-ns1"`
 	// The list of backup set names (optional, an empty list implies backing up all sets).

pkg/dto/restore_request.go

@@ -6,7 +6,6 @@ import (
 	"time"
 
 	"github.com/aerospike/aerospike-backup-service/v2/pkg/model"
-	"github.com/aerospike/aerospike-backup-service/v2/pkg/util"
 )
 
 // RestoreRequest represents a restore operation request from custom storage
@@ -17,7 +16,7 @@ type RestoreRequest struct {
 	SourceStorage      *Storage          `json:"source,omitempty" validate:"required"`
 	SecretAgent        *SecretAgent      `json:"secret-agent,omitempty"`
 	// Path to the data from storage root.
-	BackupDataPath *string `json:"backup-data-path" validate:"required"`
+	BackupDataPath string `json:"backup-data-path" validate:"required"`
 }
 
 // RestoreTimestampRequest represents a restore by timestamp operation request.
@@ -37,7 +36,7 @@ type RestoreTimestampRequest struct {
 
 // Validate validates the restore operation request.
 func (r *RestoreRequest) Validate() error {
-	if r.BackupDataPath == nil {
+	if len(r.BackupDataPath) == 0 {
 		return errors.New("path is not specified")
 	}
 	if err := r.DestinationCluster.Validate(); err != nil {
@@ -106,6 +105,6 @@ func (r *RestoreRequest) ToModel(config *model.Config) (*model.RestoreRequest, e
 		Policy:             r.Policy.ToModel(),
 		SourceStorage:      storage,
 		SecretAgent:        r.SecretAgent.ToModel(),
-		BackupDataPath:     util.ValueOrZero(r.BackupDataPath),
+		BackupDataPath:     r.BackupDataPath,
 	}, nil
 }