Merge pull request #108 from wonko/feature/rclone

Add rclone as upload mech

Dockerfile

@@ -37,6 +37,12 @@ RUN apk add --no-cache ca-certificates tzdata mongodb-tools=${MONGODB_TOOLS_VERS
 ADD https://dl.minio.io/client/mc/release/linux-amd64/mc /usr/bin
 RUN chmod u+x /usr/bin/mc
 
+ADD https://downloads.rclone.org/rclone-current-linux-amd64.zip /tmp
+RUN cd /tmp \
+  && unzip rclone-current-linux-amd64.zip \
+  && cp rclone-*-linux-amd64/rclone /usr/bin/ \
+  && chmod u+x /usr/bin/rclone
+
 WORKDIR /root/
 
 #install gcloud

README.md

@@ -12,6 +12,7 @@ MGOB is a MongoDB backup automation tool built with Go.
 * upload to S3 Object Storage (Minio, AWS, Google Cloud, Azure)
 * upload to gcloud storage
 * upload to SFTP
+* upload to any [Rclone](https://rclone.org/) supported storage
 * notifications (Email, Slack)
 * instrumentation with Prometheus
 * http file server for local backups and logs
@@ -94,6 +95,11 @@ gcloud:
 azure:
   containerName: "backup"
   connectionString: "DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=core.windows.net"
+# Rclone upload (optional)
+rclone:
+  bucket: "my-backup-bucket"
+  # See https://rclone.org/docs/ for details on how to configure rclone
+  configFilePath: /etc/rclone.conf
 # SFTP upload (optional)
 sftp:
   host: sftp.company.com

pkg/backup/backup.go

@@ -99,11 +99,20 @@ func Run(plan config.Plan, tmpPath string, storagePath string) (Result, error) {
 	}
 
 	if plan.Azure != nil {
-		azureOutout, err := azureUpload(file, plan)
+		azureOutput, err := azureUpload(file, plan)
 		if err != nil {
 			return res, err
 		} else {
-			log.WithField("plan", plan.Name).Infof("Azure upload finished %v", azureOutout)
+			log.WithField("plan", plan.Name).Infof("Azure upload finished %v", azureOutput)
+		}
+	}
+
+	if plan.Rclone != nil {
+		rcloneOutput, err := rcloneUpload(file, plan)
+		if err != nil {
+			return res, err
+		} else {
+			log.WithField("plan", plan.Name).Infof("Rclone upload finished %v", rcloneOutput)
 		}
 	}
 

pkg/backup/rclone.go

@@ -0,0 +1,33 @@
+package backup
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/codeskyblue/go-sh"
+	"github.com/pkg/errors"
+
+	"github.com/stefanprodan/mgob/pkg/config"
+)
+
+func rcloneUpload(file string, plan config.Plan) (string, error) {
+
+	fileName := filepath.Base(file)
+
+	upload := fmt.Sprintf("rclone --config=\"%v\" copy %v %v:%v/%v",
+		plan.Rclone.ConfigFilePath, plan.Name, plan.Rclone.Bucket, fileName)
+
+	result, err := sh.Command("/bin/sh", "-c", upload).SetTimeout(time.Duration(plan.Scheduler.Timeout) * time.Minute).CombinedOutput()
+	output := ""
+	if len(result) > 0 {
+		output = strings.Replace(string(result), "\n", " ", -1)
+	}
+
+	if err != nil {
+		return "", errors.Wrapf(err, "Rclone uploading %v to %v:%v failed %v", file, plan.Name, plan.Rclone.Bucket, output)
+	}
+
+	return strings.Replace(output, "\n", " ", -1), nil
+}

pkg/config/plan.go

@@ -16,6 +16,7 @@ type Plan struct {
 	Scheduler Scheduler `yaml:"scheduler"`
 	S3        *S3       `yaml:"s3"`
 	GCloud    *GCloud   `yaml:"gcloud"`
+	Rclone    *Rclone   `yaml:"rclone"`
 	Azure     *Azure    `yaml:"azure"`
 	SFTP      *SFTP     `yaml:"sftp"`
 	SMTP      *SMTP     `yaml:"smtp"`
@@ -51,6 +52,11 @@ type GCloud struct {
 	KeyFilePath string `yaml:"keyFilePath"`
 }
 
+type Rclone struct {
+	Bucket         string `yaml:"bucket"`
+	ConfigFilePath string `yaml:"configFilePath"`
+}
+
 type Azure struct {
 	ContainerName    string `yaml:"containerName"`
 	ConnectionString string `yaml:"connectionString"`