Skip to content
This repository has been archived by the owner on Nov 3, 2022. It is now read-only.

Add rclone as upload mech #108

Merged
merged 2 commits into from
May 1, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ RUN apk add --no-cache ca-certificates tzdata mongodb-tools=${MONGODB_TOOLS_VERS
ADD https://dl.minio.io/client/mc/release/linux-amd64/mc /usr/bin
RUN chmod u+x /usr/bin/mc

ADD https://downloads.rclone.org/rclone-current-linux-amd64.zip /tmp
RUN cd /tmp \
&& unzip rclone-current-linux-amd64.zip \
&& cp rclone-*-linux-amd64/rclone /usr/bin/ \
&& chmod u+x /usr/bin/rclone

WORKDIR /root/

#install gcloud
Expand Down
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ MGOB is a MongoDB backup automation tool built with Go.
* upload to S3 Object Storage (Minio, AWS, Google Cloud, Azure)
* upload to gcloud storage
* upload to SFTP
* upload to any [Rclone](https://rclone.org/) supported storage
* notifications (Email, Slack)
* instrumentation with Prometheus
* http file server for local backups and logs
Expand Down Expand Up @@ -94,6 +95,11 @@ gcloud:
azure:
containerName: "backup"
connectionString: "DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=core.windows.net"
# Rclone upload (optional)
rclone:
bucket: "my-backup-bucket"
# See https://rclone.org/docs/ for details on how to configure rclone
configFilePath: /etc/rclone.conf
# SFTP upload (optional)
sftp:
host: sftp.company.com
Expand Down
13 changes: 11 additions & 2 deletions pkg/backup/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,20 @@ func Run(plan config.Plan, tmpPath string, storagePath string) (Result, error) {
}

if plan.Azure != nil {
azureOutout, err := azureUpload(file, plan)
azureOutput, err := azureUpload(file, plan)
if err != nil {
return res, err
} else {
log.WithField("plan", plan.Name).Infof("Azure upload finished %v", azureOutout)
log.WithField("plan", plan.Name).Infof("Azure upload finished %v", azureOutput)
}
}

if plan.Rclone != nil {
rcloneOutput, err := rcloneUpload(file, plan)
if err != nil {
return res, err
} else {
log.WithField("plan", plan.Name).Infof("Rclone upload finished %v", rcloneOutput)
}
}

Expand Down
33 changes: 33 additions & 0 deletions pkg/backup/rclone.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package backup

import (
"fmt"
"path/filepath"
"strings"
"time"

"github.com/codeskyblue/go-sh"
"github.com/pkg/errors"

"github.com/stefanprodan/mgob/pkg/config"
)

func rcloneUpload(file string, plan config.Plan) (string, error) {

fileName := filepath.Base(file)

upload := fmt.Sprintf("rclone --config=\"%v\" copy %v %v:%v/%v",
plan.Rclone.ConfigFilePath, plan.Name, plan.Rclone.Bucket, fileName)

result, err := sh.Command("/bin/sh", "-c", upload).SetTimeout(time.Duration(plan.Scheduler.Timeout) * time.Minute).CombinedOutput()
output := ""
if len(result) > 0 {
output = strings.Replace(string(result), "\n", " ", -1)
}

if err != nil {
return "", errors.Wrapf(err, "Rclone uploading %v to %v:%v failed %v", file, plan.Name, plan.Rclone.Bucket, output)
}

return strings.Replace(output, "\n", " ", -1), nil
}
6 changes: 6 additions & 0 deletions pkg/config/plan.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ type Plan struct {
Scheduler Scheduler `yaml:"scheduler"`
S3 *S3 `yaml:"s3"`
GCloud *GCloud `yaml:"gcloud"`
Rclone *Rclone `yaml:"rclone"`
Azure *Azure `yaml:"azure"`
SFTP *SFTP `yaml:"sftp"`
SMTP *SMTP `yaml:"smtp"`
Expand Down Expand Up @@ -51,6 +52,11 @@ type GCloud struct {
KeyFilePath string `yaml:"keyFilePath"`
}

type Rclone struct {
Bucket string `yaml:"bucket"`
ConfigFilePath string `yaml:"configFilePath"`
}

type Azure struct {
ContainerName string `yaml:"containerName"`
ConnectionString string `yaml:"connectionString"`
Expand Down