Skip to content

Commit

Permalink
Dump: add output format tar and output to stdout (go-gitea#10376)
Browse files Browse the repository at this point in the history 
* Dump: Use mholt/archive/v3 to support tar including many compressions

Signed-off-by: Philipp Homann <[email protected]>

* Dump: Allow dump output to stdout

Signed-off-by: Philipp Homann <[email protected]>

* Dump: Fixed bug present since go-gitea#6677 where SessionConfig.Provider is never "file"

Signed-off-by: Philipp Homann <[email protected]>

* Dump: never pack RepoRootPath, LFS.ContentPath and LogRootPath when they are below AppDataPath

Signed-off-by: Philipp Homann <[email protected]>

* Dump: also dump LFS (fixes go-gitea#10058)

Signed-off-by: Philipp Homann <[email protected]>

* Dump: never dump CustomPath if CustomPath is a subdir of or equal to AppDataPath (fixes go-gitea#10365)

Signed-off-by: Philipp Homann <[email protected]>

* Use log.Info instead of fmt.Fprintf

Signed-off-by: Philipp Homann <[email protected]>

* import ordering

* make fmt

Co-authored-by: zeripath <[email protected]>
Co-authored-by: techknowlogick <[email protected]>
Co-authored-by: Matti R <[email protected]>
  • Loading branch information
@PhilippHomann @zeripath @techknowlogick
4 people authored and Yohann Delafollye committed Jul 31, 2020
1 parent f7583ca commit 40bd1d5
Show file tree
Hide file tree
Showing 303 changed files with 301,315 additions and 1,181 deletions.
269 changes: 212 additions & 57 deletions cmd/dump.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,120 @@
package cmd

import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"time"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"

"github.com/unknwon/cae/zip"
"gitea.com/macaron/session"
archiver "github.com/mholt/archiver/v3"
"github.com/unknwon/com"
"github.com/urfave/cli"
)

func addFile(w archiver.Writer, filePath string, absPath string, verbose bool) error {
if verbose {
log.Info("Adding file %s\n", filePath)
}
file, err := os.Open(absPath)
if err != nil {
return err
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return err
}

return w.Write(archiver.File{
FileInfo: archiver.FileInfo{
FileInfo: fileInfo,
CustomName: filePath,
},
ReadCloser: file,
})
}

func addRecursive(w archiver.Writer, dirPath string, absPath string, verbose bool) error {
if verbose {
log.Info("Adding dir %s\n", dirPath)
}
dir, err := os.Open(absPath)
if err != nil {
return fmt.Errorf("Could not open directory %s: %s", absPath, err)
}
files, err := dir.Readdir(0)
if err != nil {
return fmt.Errorf("Unable to list files in %s: %s", absPath, err)
}

if err := addFile(w, dirPath, absPath, false); err != nil {
return err
}

for _, fileInfo := range files {
if fileInfo.IsDir() {
err = addRecursive(w, filepath.Join(dirPath, fileInfo.Name()), filepath.Join(absPath, fileInfo.Name()), verbose)
} else {
err = addFile(w, filepath.Join(dirPath, fileInfo.Name()), filepath.Join(absPath, fileInfo.Name()), verbose)
}
if err != nil {
return err
}
}
return nil
}

func isSubdir(upper string, lower string) (bool, error) {
if relPath, err := filepath.Rel(upper, lower); err != nil {
return false, err
} else if relPath == "." || !strings.HasPrefix(relPath, ".") {
return true, nil
}
return false, nil
}

type outputType struct {
Enum []string
Default string
selected string
}

func (o outputType) Join() string {
return strings.Join(o.Enum, ", ")
}

func (o *outputType) Set(value string) error {
for _, enum := range o.Enum {
if enum == value {
o.selected = value
return nil
}
}

return fmt.Errorf("allowed values are %s", o.Join())
}

func (o outputType) String() string {
if o.selected == "" {
return o.Default
}
return o.selected
}

var outputTypeEnum = &outputType{
Enum: []string{"zip", "tar", "tar.gz", "tar.xz", "tar.bz2"},
Default: "zip",
}

// CmdDump represents the available dump sub-command.
var CmdDump = cli.Command{
Name: "dump",
Expand All @@ -33,7 +131,7 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
cli.StringFlag{
Name: "file, f",
Value: fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix()),
Usage: "Name of the dump file which will be created.",
Usage: "Name of the dump file which will be created. Supply '-' for stdout. See type for available types.",
},
cli.BoolFlag{
Name: "verbose, V",
Expand All @@ -56,6 +154,11 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
Name: "skip-log, L",
Usage: "Skip the log dumping",
},
cli.GenericFlag{
Name: "type",
Value: outputTypeEnum,
Usage: fmt.Sprintf("Dump output format: %s", outputTypeEnum.Join()),
},
},
}

Expand All @@ -65,79 +168,113 @@ func fatal(format string, args ...interface{}) {
}

func runDump(ctx *cli.Context) error {
var file *os.File
fileName := ctx.String("file")
if fileName == "-" {
file = os.Stdout
err := log.DelLogger("console")
if err != nil {
fatal("Deleting default logger failed. Can not write to stdout: %v", err)
}
}
setting.NewContext()
// make sure we are logging to the console no matter what the configuration tells us do to
if _, err := setting.Cfg.Section("log").NewKey("MODE", "console"); err != nil {
fatal("Setting logging mode to console failed: %v", err)
}
if _, err := setting.Cfg.Section("log.console").NewKey("STDERR", "true"); err != nil {
fatal("Setting console logger to stderr failed: %v", err)
}
setting.NewServices() // cannot access session settings otherwise

err := models.SetEngine()
if err != nil {
return err
}

tmpDir := ctx.String("tempdir")
if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
fatal("Path does not exist: %s", tmpDir)
}
tmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-dump-")
if err != nil {
fatal("Failed to create tmp work directory: %v", err)
if file == nil {
file, err = os.Create(fileName)
if err != nil {
fatal("Unable to open %s: %v", fileName, err)
}
}
log.Info("Creating tmp work dir: %s", tmpWorkDir)
defer file.Close()

// work-around #1103
if os.Getenv("TMPDIR") == "" {
os.Setenv("TMPDIR", tmpWorkDir)
verbose := ctx.Bool("verbose")
outType := ctx.String("type")
var iface interface{}
if fileName == "-" {
iface, err = archiver.ByExtension(fmt.Sprintf(".%s", outType))
} else {
iface, err = archiver.ByExtension(fileName)
}

dbDump := path.Join(tmpWorkDir, "gitea-db.sql")

fileName := ctx.String("file")
log.Info("Packing dump files...")
z, err := zip.Create(fileName)
if err != nil {
fatal("Failed to create %s: %v", fileName, err)
fatal("Unable to get archiver for extension: %v", err)
}

zip.Verbose = ctx.Bool("verbose")
w, _ := iface.(archiver.Writer)
if err := w.Create(file); err != nil {
fatal("Creating archiver.Writer failed: %v", err)
}
defer w.Close()

if ctx.IsSet("skip-repository") && ctx.Bool("skip-repository") {
log.Info("Skip dumping local repositories")
} else {
log.Info("Dumping local repositories...%s", setting.RepoRootPath)
reposDump := path.Join(tmpWorkDir, "gitea-repo.zip")
if err := zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil {
fatal("Failed to dump local repositories: %v", err)
log.Info("Dumping local repositories... %s", setting.RepoRootPath)
if err := addRecursive(w, "repos", setting.RepoRootPath, verbose); err != nil {
fatal("Failed to include repositories: %v", err)
}
if err := z.AddFile("gitea-repo.zip", reposDump); err != nil {
fatal("Failed to include gitea-repo.zip: %v", err)

if _, err := os.Stat(setting.LFS.ContentPath); !os.IsNotExist(err) {
log.Info("Dumping lfs... %s", setting.LFS.ContentPath)
if err := addRecursive(w, "lfs", setting.LFS.ContentPath, verbose); err != nil {
fatal("Failed to include lfs: %v", err)
}
}
}

tmpDir := ctx.String("tempdir")
if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
fatal("Path does not exist: %s", tmpDir)
}

dbDump, err := ioutil.TempFile(tmpDir, "gitea-db.sql")
if err != nil {
fatal("Failed to create tmp file: %v", err)
}
defer os.Remove(dbDump.Name())

targetDBType := ctx.String("database")
if len(targetDBType) > 0 && targetDBType != setting.Database.Type {
log.Info("Dumping database %s => %s...", setting.Database.Type, targetDBType)
} else {
log.Info("Dumping database...")
}

if err := models.DumpDatabase(dbDump, targetDBType); err != nil {
if err := models.DumpDatabase(dbDump.Name(), targetDBType); err != nil {
fatal("Failed to dump database: %v", err)
}

if err := z.AddFile("gitea-db.sql", dbDump); err != nil {
if err := addFile(w, "gitea-db.sql", dbDump.Name(), verbose); err != nil {
fatal("Failed to include gitea-db.sql: %v", err)
}

if len(setting.CustomConf) > 0 {
log.Info("Adding custom configuration file from %s", setting.CustomConf)
if err := z.AddFile("app.ini", setting.CustomConf); err != nil {
if err := addFile(w, "app.ini", setting.CustomConf, verbose); err != nil {
fatal("Failed to include specified app.ini: %v", err)
}
}

customDir, err := os.Stat(setting.CustomPath)
if err == nil && customDir.IsDir() {
if err := z.AddDir("custom", setting.CustomPath); err != nil {
fatal("Failed to include custom: %v", err)
if is, _ := isSubdir(setting.AppDataPath, setting.CustomPath); !is {
if err := addRecursive(w, "custom", setting.CustomPath, verbose); err != nil {
fatal("Failed to include custom: %v", err)
}
} else {
log.Info("Custom dir %s is inside data dir %s, skipped", setting.CustomPath, setting.AppDataPath)
}
} else {
log.Info("Custom dir %s doesn't exist, skipped", setting.CustomPath)
Expand All @@ -146,11 +283,19 @@ func runDump(ctx *cli.Context) error {
if com.IsExist(setting.AppDataPath) {
log.Info("Packing data directory...%s", setting.AppDataPath)

var sessionAbsPath string
if setting.SessionConfig.Provider == "file" {
sessionAbsPath = setting.SessionConfig.ProviderConfig
var excludes []string
if setting.Cfg.Section("session").Key("PROVIDER").Value() == "file" {
var opts session.Options
if err = json.Unmarshal([]byte(setting.SessionConfig.ProviderConfig), &opts); err != nil {
return err
}
excludes = append(excludes, opts.ProviderConfig)
}
if err := zipAddDirectoryExclude(z, "data", setting.AppDataPath, sessionAbsPath); err != nil {

excludes = append(excludes, setting.RepoRootPath)
excludes = append(excludes, setting.LFS.ContentPath)
excludes = append(excludes, setting.LogRootPath)
if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {
fatal("Failed to include data directory: %v", err)
}
}
Expand All @@ -161,32 +306,42 @@ func runDump(ctx *cli.Context) error {
if ctx.IsSet("skip-log") && ctx.Bool("skip-log") {
log.Info("Skip dumping log files")
} else if com.IsExist(setting.LogRootPath) {
if err := z.AddDir("log", setting.LogRootPath); err != nil {
if err := addRecursive(w, "log", setting.LogRootPath, verbose); err != nil {
fatal("Failed to include log: %v", err)
}
}

if err = z.Close(); err != nil {
_ = os.Remove(fileName)
fatal("Failed to save %s: %v", fileName, err)
}
if fileName != "-" {
if err = w.Close(); err != nil {
_ = os.Remove(fileName)
fatal("Failed to save %s: %v", fileName, err)
}

if err := os.Chmod(fileName, 0600); err != nil {
log.Info("Can't change file access permissions mask to 0600: %v", err)
if err := os.Chmod(fileName, 0600); err != nil {
log.Info("Can't change file access permissions mask to 0600: %v", err)
}
}

log.Info("Removing tmp work dir: %s", tmpWorkDir)

if err := os.RemoveAll(tmpWorkDir); err != nil {
fatal("Failed to remove %s: %v", tmpWorkDir, err)
if fileName != "-" {
log.Info("Finish dumping in file %s", fileName)
} else {
log.Info("Finish dumping to stdout")
}
log.Info("Finish dumping in file %s", fileName)

return nil
}

// zipAddDirectoryExclude zips absPath to specified zipPath inside z excluding excludeAbsPath
func zipAddDirectoryExclude(zip *zip.ZipArchive, zipPath, absPath string, excludeAbsPath string) error {
func contains(slice []string, s string) bool {
for _, v := range slice {
if v == s {
return true
}
}
return false
}

// addRecursiveExclude zips absPath to specified insidePath inside writer excluding excludeAbsPath
func addRecursiveExclude(w archiver.Writer, insidePath, absPath string, excludeAbsPath []string, verbose bool) error {
absPath, err := filepath.Abs(absPath)
if err != nil {
return err
Expand All @@ -197,24 +352,24 @@ func zipAddDirectoryExclude(zip *zip.ZipArchive, zipPath, absPath string, exclud
}
defer dir.Close()

zip.AddEmptyDir(zipPath)

files, err := dir.Readdir(0)
if err != nil {
return err
}
for _, file := range files {
currentAbsPath := path.Join(absPath, file.Name())
currentZipPath := path.Join(zipPath, file.Name())
currentInsidePath := path.Join(insidePath, file.Name())
if file.IsDir() {
if currentAbsPath != excludeAbsPath {
if err = zipAddDirectoryExclude(zip, currentZipPath, currentAbsPath, excludeAbsPath); err != nil {
if !contains(excludeAbsPath, currentAbsPath) {
if err := addFile(w, currentInsidePath, currentAbsPath, false); err != nil {
return err
}
if err = addRecursiveExclude(w, currentInsidePath, currentAbsPath, excludeAbsPath, verbose); err != nil {
return err
}
}

} else {
if err = zip.AddFile(currentZipPath, currentAbsPath); err != nil {
if err = addFile(w, currentInsidePath, currentAbsPath, verbose); err != nil {
return err
}
}
Expand Down
Loading

0 comments on commit 40bd1d5

Please sign in to comment.