diff --git a/Dockerfile b/Dockerfile index e9a68535..8ee1da58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,8 @@ ARG VERSION ENV MONGODB_TOOLS_VERSION 4.2.1-r1 ENV GOOGLE_CLOUD_SDK_VERSION 276.0.0 -ENV AZURE_CLI_VERSION 2.5.1 +ENV AZURE_CLI_VERSION 2.13.0 +ENV AWS_CLI_VERSION 1.18.159 ENV PATH /root/google-cloud-sdk/bin:$PATH LABEL org.label-schema.build-date=$BUILD_DATE \ @@ -55,9 +56,9 @@ RUN apk --no-cache add \ libc6-compat \ openssh-client \ git \ - && pip3 install --upgrade pip && \ - pip install wheel && \ - pip install crcmod && \ + && pip3 --no-cache-dir install --upgrade pip && \ + pip --no-cache-dir install wheel && \ + pip --no-cache-dir install crcmod && \ curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GOOGLE_CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ tar xzf google-cloud-sdk-${GOOGLE_CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ rm google-cloud-sdk-${GOOGLE_CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ @@ -67,10 +68,11 @@ RUN apk --no-cache add \ gcloud config set metrics/environment github_docker_image && \ gcloud --version -# install azure-cli -RUN apk add --virtual=build gcc libffi-dev musl-dev openssl-dev python3-dev make && \ - pip install cffi && \ - pip install azure-cli==${AZURE_CLI_VERSION} && \ +# install azure-cli and aws-cli +RUN apk --no-cache add --virtual=build gcc libffi-dev musl-dev openssl-dev python3-dev make && \ + pip --no-cache-dir install cffi && \ + pip --no-cache-dir install azure-cli==${AZURE_CLI_VERSION} && \ + pip --no-cache-dir install awscli==${AWS_CLI_VERSION} && \ apk del --purge build COPY --from=0 /go/src/github.com/stefanprodan/mgob/mgob . diff --git a/README.md b/README.md index 37428aa5..40559bca 100644 --- a/README.md +++ b/README.md @@ -83,8 +83,18 @@ target: s3: url: "https://play.minio.io:9000" bucket: "backup" + # accessKey and secretKey are optional for AWS, if your Docker image has awscli accessKey: "Q3AM3UQ867SPQQA43P2F" secretKey: "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + # Optional, only used for AWS (when awscli is present) + # The customer-managed AWS Key Management Service (KMS) key ID that should be used to + # server-side encrypt the backup in S3 + #kmsKeyId: + # Optional, only used for AWS (when awscli is present) + # Valid choices are: STANDARD | REDUCED_REDUNDANCY | STANDARD_IA | ONE- + # ZONE_IA | INTELLIGENT_TIERING | GLACIER | DEEP_ARCHIVE. + # Defaults to 'STANDARD' + #storageClass: STANDARD # For Minio and AWS use S3v4 for GCP use S3v2 api: "S3v4" # GCloud upload (optional) diff --git a/cmd/mgob/mgob.go b/cmd/mgob/mgob.go index 6d94967a..d226f23e 100644 --- a/cmd/mgob/mgob.go +++ b/cmd/mgob/mgob.go @@ -18,7 +18,7 @@ import ( var ( appConfig = &config.AppConfig{} - version = "v1.1.0-dev" + version = "v1.2.0-dev" ) func beforeApp(c *cli.Context) error { @@ -95,6 +95,7 @@ func start(c *cli.Context) error { appConfig.TmpPath = c.String("TmpPath") appConfig.DataPath = c.String("DataPath") appConfig.Version = version + appConfig.UseAwsCli = true log.Infof("starting with config: %+v", appConfig) @@ -110,6 +111,13 @@ func start(c *cli.Context) error { } log.Info(info) + info, err = backup.CheckAWSClient() + if err != nil { + log.Warn(err) + appConfig.UseAwsCli = false + } + log.Info(info) + info, err = backup.CheckGCloudClient() if err != nil { log.Fatal(err) diff --git a/pkg/api/backup.go b/pkg/api/backup.go index 693b1a85..b879b324 100644 --- a/pkg/api/backup.go +++ b/pkg/api/backup.go @@ -37,7 +37,7 @@ func postBackup(w http.ResponseWriter, r *http.Request) { log.WithField("plan", planID).Info("On demand backup started") - res, err := backup.Run(plan, cfg.TmpPath, cfg.StoragePath) + res, err := backup.Run(plan, &cfg) if err != nil { log.WithField("plan", planID).Errorf("On demand backup failed %v", err) if err := notifier.SendNotification(fmt.Sprintf("%v on demand backup failed", planID), diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index 276b5d00..994f0d0f 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -13,7 +13,9 @@ import ( "github.com/stefanprodan/mgob/pkg/config" ) -func Run(plan config.Plan, tmpPath string, storagePath string) (Result, error) { +func Run(plan config.Plan, conf *config.AppConfig) (Result, error) { + tmpPath := conf.TmpPath + storagePath := conf.StoragePath t1 := time.Now() planDir := fmt.Sprintf("%v/%v", storagePath, plan.Name) @@ -81,7 +83,7 @@ func Run(plan config.Plan, tmpPath string, storagePath string) (Result, error) { } if plan.S3 != nil { - s3Output, err := s3Upload(file, plan) + s3Output, err := s3Upload(file, plan, conf.UseAwsCli) if err != nil { return res, err } else { diff --git a/pkg/backup/checks.go b/pkg/backup/checks.go index 3efda933..aa77ef32 100644 --- a/pkg/backup/checks.go +++ b/pkg/backup/checks.go @@ -33,6 +33,19 @@ func CheckMinioClient() (string, error) { return strings.Replace(string(output), "\n", " ", -1), nil } +func CheckAWSClient() (string, error) { + output, err := sh.Command("/bin/sh", "-c", "aws --version").CombinedOutput() + if err != nil { + ex := "" + if len(output) > 0 { + ex = strings.Replace(string(output), "\n", " ", -1) + } + return "", errors.Wrapf(err, "aws failed %v", ex) + } + + return strings.Replace(string(output), "\n", " ", -1), nil +} + func CheckGCloudClient() (string, error) { output, err := sh.Command("/bin/sh", "-c", "gcloud --version").CombinedOutput() if err != nil { diff --git a/pkg/backup/s3.go b/pkg/backup/s3.go index 42223612..bd7c795a 100644 --- a/pkg/backup/s3.go +++ b/pkg/backup/s3.go @@ -5,6 +5,7 @@ import ( "path/filepath" "strings" "time" + "net/url" "github.com/codeskyblue/go-sh" "github.com/pkg/errors" @@ -12,7 +13,69 @@ import ( "github.com/stefanprodan/mgob/pkg/config" ) -func s3Upload(file string, plan config.Plan) (string, error) { +func s3Upload(file string, plan config.Plan, useAwsCli bool) (string, error) { + + s3Url, err := url.Parse(plan.S3.URL) + + if err != nil { + return "", errors.Wrapf(err, "invalid S3 url for plan %v: %s", plan.Name, plan.S3.URL) + } + + if useAwsCli && strings.HasSuffix(s3Url.Hostname(), "amazonaws.com") { + return awsUpload(file, plan) + } + + return minioUpload(file, plan) +} + +func awsUpload(file string, plan config.Plan) (string, error) { + + output := "" + if len(plan.S3.AccessKey) > 0 && len(plan.S3.SecretKey) > 0 { + // Let's use credentials given + configure := fmt.Sprintf("aws configure set aws_access_key_id %v && aws configure set aws_secret_access_key %v", + plan.S3.AccessKey, plan.S3.SecretKey) + + result, err := sh.Command("/bin/sh", "-c", configure).CombinedOutput() + if len(result) > 0 { + output += strings.Replace(string(result), "\n", " ", -1) + } + if err != nil { + return "", errors.Wrapf(err, "aws configure for plan %v failed %s", plan.Name, output) + } + } + + fileName := filepath.Base(file) + + encrypt := "" + if len(plan.S3.KmsKeyId) > 0 { + encrypt = fmt.Sprintf(" --sse aws:kms --sse-kms-key-id %v", plan.S3.KmsKeyId) + } + + storage := "" + if len(plan.S3.StorageClass) > 0 { + storage = fmt.Sprintf(" --storage-class %v", plan.S3.StorageClass) + } + + upload := fmt.Sprintf("aws --quiet s3 cp %v s3://%v/%v%v%v", + file, plan.S3.Bucket, fileName, encrypt, storage) + + result, err := sh.Command("/bin/sh", "-c", upload).SetTimeout(time.Duration(plan.Scheduler.Timeout) * time.Minute).CombinedOutput() + if len(result) > 0 { + output += strings.Replace(string(result), "\n", " ", -1) + } + if err != nil { + return "", errors.Wrapf(err, "S3 uploading %v to %v/%v failed %v", file, plan.Name, plan.S3.Bucket, output) + } + + if strings.Contains(output, "") { + return "", errors.Errorf("S3 upload failed %v", output) + } + + return strings.Replace(output, "\n", " ", -1), nil +} + +func minioUpload(file string, plan config.Plan) (string, error) { register := fmt.Sprintf("mc config host add %v %v %v %v --api %v", plan.Name, plan.S3.URL, plan.S3.AccessKey, plan.S3.SecretKey, plan.S3.API) diff --git a/pkg/config/app.go b/pkg/config/app.go index d0d936e7..033444fe 100644 --- a/pkg/config/app.go +++ b/pkg/config/app.go @@ -9,4 +9,5 @@ type AppConfig struct { TmpPath string `json:"tmp_path"` DataPath string `json:"data_path"` Version string `json:"version"` + UseAwsCli bool `json:"use_aws_cli"` } diff --git a/pkg/config/plan.go b/pkg/config/plan.go index 34d76c3c..82eb9282 100644 --- a/pkg/config/plan.go +++ b/pkg/config/plan.go @@ -40,11 +40,13 @@ type Scheduler struct { } type S3 struct { - Bucket string `yaml:"bucket"` - AccessKey string `yaml:"accessKey"` - API string `yaml:"api"` - SecretKey string `yaml:"secretKey"` - URL string `yaml:"url"` + Bucket string `yaml:"bucket"` + AccessKey string `yaml:"accessKey"` + API string `yaml:"api"` + SecretKey string `yaml:"secretKey"` + URL string `yaml:"url"` + KmsKeyId string `yaml:"kmsKeyId"` + StorageClass string `yaml:"storageClass" validate:"omitempty,oneof=STANDARD REDUCED_REDUNDANCY STANDARD_IA ONE-ZONE_IA INTELLIGENT_TIERING GLACIER DEEP_ARCHIVE` } type GCloud struct { diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index b6f7c54b..a37b9322 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -86,7 +86,7 @@ func (b backupJob) Run() { var backupLog string t1 := time.Now() - res, err := backup.Run(b.plan, b.conf.TmpPath, b.conf.StoragePath) + res, err := backup.Run(b.plan, b.conf) if err != nil { status = "500" backupLog = fmt.Sprintf("Backup failed %v", err)