Merge pull request #3 from lomorage/dev

support listing google drive files in tree view and also fix create time

cmd/lomob/list-cloud.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"fmt"
+	"strconv"
 
 	"github.com/lomorage/lomo-backup/common/gcloud"
 	"github.com/urfave/cli"
+	"github.com/xlab/treeprint"
 )
 
 func listFilesInGDrive(ctx *cli.Context) error {
@@ -24,8 +26,38 @@ func listFilesInGDrive(ctx *cli.Context) error {
 
 	if folderID == "" {
 		fmt.Printf("Folder '%s' not found.\n", folder)
+		return nil
 	} else {
 		fmt.Printf("File Name: %s, File ID: %s\n", folder, folderID)
 	}
+	rootNode := treeprint.NewWithRoot(folder)
+	err = listFileTreeInGDrive(client, rootNode, folderID)
+	if err != nil {
+		return err
+	}
+	fmt.Println(rootNode.String())
+	return nil
+}
+
+func listFileTreeInGDrive(client *gcloud.Client, currNode treeprint.Tree, folderID string) error {
+	folders, files, err := client.ListFiles(folderID)
+	if err != nil {
+		return err
+	}
+	for _, folder := range folders {
+		t := folder.ModTime
+
+		childNode := currNode.AddMetaBranch(
+			fmt.Sprintf("\t%02d/%02d/%d", t.Month(), t.Day(), t.Year()), folder.Path)
+		err = listFileTreeInGDrive(client, childNode, folder.RefID)
+		if err != nil {
+			return err
+		}
+	}
+	for _, file := range files {
+		t := file.ModTime
+		currNode.AddMetaNode(fmt.Sprintf("\t%12s\t%02d/%02d/%d", strconv.Itoa(file.Size),
+			t.Month(), t.Day(), t.Year()), file.Name)
+	}
 	return nil
 }

cmd/lomob/main.go

@@ -231,12 +231,6 @@ func main() {
 						},
 					},
 				},
-				{
-					Name:      "parts",
-					Action:    calculatePartHash,
-					Usage:     "Calculate given files base64 hash",
-					ArgsUsage: "[filename]",
-				},
 				{
 					Name:   "gdrive",
 					Action: listFilesInGDrive,

cmd/lomob/upload-files.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/lomorage/lomo-backup/common/gcloud"
 	"github.com/sirupsen/logrus"
@@ -34,7 +35,10 @@ func uploadFiles(ctx *cli.Context) error {
 	}
 
 	uploadRootFolder := ctx.String("folder")
-	exist, uploadRootFolderID, err := client.GetAndCreateFileIDIfNotExist(uploadRootFolder, "", nil)
+	exist, uploadRootFolderID, err := client.GetAndCreateFileIDIfNotExist(uploadRootFolder, "", nil,
+		gcloud.FileMetadata{
+			ModTime: time.Now(),
+		})
 	if err != nil {
 		return err
 	}
@@ -58,21 +62,29 @@ func uploadFiles(ctx *cli.Context) error {
 		parentFolderID string
 	}
 	existingDirsInCloud := map[string]dirInfoInCloud{
-		"": dirInfoInCloud{folderID: uploadRootFolderID},
+		"": {folderID: uploadRootFolderID},
 	}
 	for _, f := range fileInfos {
 		scanRoot, ok := scanRootDirs[f.DirID]
 		if !ok {
 			return fmt.Errorf("unable to find scan root directory whose ID is %d", f.DirID)
 		}
 
+		origFolder := scanRoot
+
 		// flatten scan root dir so as to have only one folder
 		// find its folder ID in google cloud, and create one if not exist, then add into local map
 		scanRootFolderInCloud := flattenScanRootDir(strings.Trim(scanRoot, string(os.PathSeparator)))
 		info, ok := existingDirsInCloud[scanRootFolderInCloud]
 		if !ok {
+			stat, err := os.Stat(origFolder)
+			if err != nil {
+				return err
+			}
+
 			info = dirInfoInCloud{parentFolderID: uploadRootFolderID}
-			ok, info.folderID, err = client.GetAndCreateFileIDIfNotExist(scanRootFolderInCloud, uploadRootFolderID, nil)
+			ok, info.folderID, err = client.GetAndCreateFileIDIfNotExist(scanRootFolderInCloud, uploadRootFolderID, nil,
+				gcloud.FileMetadata{ModTime: stat.ModTime()})
 			if err != nil {
 				return err
 			}
@@ -83,17 +95,23 @@ func uploadFiles(ctx *cli.Context) error {
 		}
 		scanRootFolderIDInCloud := info.folderID
 
-		// recursive check all directories' existance in cloud, and create if not exist
+		// check all directories' existence in cloud, and create if not exist
 		dir, filename := filepath.Split(f.Name)
 		dir = strings.Trim(dir, string(os.PathSeparator))
 		folderKey := scanRootFolderInCloud
 		parentID := scanRootFolderIDInCloud
 		for _, p := range strings.Split(dir, string(os.PathSeparator)) {
+			origFolder = filepath.Join(origFolder, p)
 			folderKey += "/" + p
 			info, ok = existingDirsInCloud[folderKey]
 			if !ok {
+				stat, err := os.Stat(origFolder)
+				if err != nil {
+					return err
+				}
 				info = dirInfoInCloud{parentFolderID: parentID}
-				ok, info.folderID, err = client.GetAndCreateFileIDIfNotExist(p, parentID, nil)
+				ok, info.folderID, err = client.GetAndCreateFileIDIfNotExist(p, parentID, nil,
+					gcloud.FileMetadata{ModTime: stat.ModTime()})
 				if err != nil {
 					return err
 				}
@@ -112,12 +130,19 @@ func uploadFiles(ctx *cli.Context) error {
 		if err != nil {
 			return err
 		}
-		logrus.Debugf("Uploading: %s into %s (%s):%s\n", fullLocalPath, folderKey, parentID, filename)
-		_, err = client.CreateFile(filename, parentID, reader)
+		logrus.Infof("Uploading: %s into %s (%s):%s\n", fullLocalPath, folderKey, parentID, filename)
+		stat, err := reader.Stat()
+		if err != nil {
+			return err
+		}
+		_, err = client.CreateFile(filename, parentID, reader, gcloud.FileMetadata{
+			ModTime: stat.ModTime(),
+			Hash:    f.Hash,
+		})
 		if err != nil {
 			return err
 		}
-		logrus.Debugf("Uploading success")
+		logrus.Infof("Uploading success")
 		err = reader.Close()
 		if err != nil {
 			logrus.Warnf("Close %s: %s", fullLocalPath, err)

common/gcloud/upload.go

@@ -5,18 +5,27 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"time"
 
+	"github.com/lomorage/lomo-backup/common/types"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/drive/v3"
 	"google.golang.org/api/option"
 )
 
+const mimiTypeFolder = "application/vnd.google-apps.folder"
+
 type Config struct {
 	CredFilename  string
 	TokenFilename string
 }
 
+type FileMetadata struct {
+	Hash    string
+	ModTime time.Time
+}
+
 type Client struct {
 	srv *drive.Service
 }
@@ -49,7 +58,6 @@ func (c *Client) GetFileID(filename, parentFolderID string) (string, string, err
 	query := fmt.Sprintf("name = '%s'", filename)
 	if parentFolderID != "" {
 		query += fmt.Sprintf(" and '%s' in parents", parentFolderID)
-
 	}
 	files, err := c.srv.Files.List().Q(query).PageSize(1).Fields("files(id, name, parents)").Do()
 	if err != nil {
@@ -68,14 +76,14 @@ func (c *Client) GetFileID(filename, parentFolderID string) (string, string, err
 	return files.Files[0].Id, parentID, nil
 }
 
-func (c *Client) GetAndCreateFileIDIfNotExist(filename, parentFolderID string, r io.Reader) (bool, string, error) {
+func (c *Client) GetAndCreateFileIDIfNotExist(filename, parentFolderID string, r io.Reader, m FileMetadata) (bool, string, error) {
 	fileID, pid, err := c.GetFileID(filename, parentFolderID)
 	if err != nil {
 		return false, "", err
 	}
 	if fileID == "" {
 		// not exist, create new one
-		fileID, err = c.CreateFile(filename, parentFolderID, r)
+		fileID, err = c.CreateFile(filename, parentFolderID, r, m)
 		return false, fileID, err
 	}
 	if parentFolderID != "" && pid != parentFolderID {
@@ -84,14 +92,22 @@ func (c *Client) GetAndCreateFileIDIfNotExist(filename, parentFolderID string, r
 	return true, fileID, nil
 }
 
-func (c *Client) CreateFile(filename, parentFolderID string, r io.Reader) (string, error) {
-	file := &drive.File{Name: filename}
+func (c *Client) CreateFile(filename, parentFolderID string, r io.Reader, m FileMetadata) (string, error) {
+	file := &drive.File{
+		Name:        filename,
+		CreatedTime: m.ModTime.Format(time.RFC3339),
+	}
+	if m.Hash != "" {
+		file.AppProperties = map[string]string{
+			"hash": m.Hash,
+		}
+	}
 	if parentFolderID != "" {
 		file.Parents = []string{parentFolderID}
 	}
 	if r == nil {
 		// it is a folder
-		file.MimeType = "application/vnd.google-apps.folder"
+		file.MimeType = mimiTypeFolder
 		f, err := c.srv.Files.Create(file).Do()
 		if err != nil {
 			return "", err
@@ -104,3 +120,35 @@ func (c *Client) CreateFile(filename, parentFolderID string, r io.Reader) (strin
 	}
 	return f.Id, nil
 }
+
+func (c *Client) ListFiles(folderID string) ([]*types.DirInfo, []*types.FileInfo, error) {
+	query := fmt.Sprintf("'%s' in parents", folderID)
+	list, err := c.srv.Files.List().Q(query).Fields("files(id, name, size, mimeType, createdTime)").Do()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	files := []*types.FileInfo{}
+	folders := []*types.DirInfo{}
+	for _, f := range list.Files {
+		t, err := time.Parse(time.RFC3339, f.CreatedTime)
+		if err != nil {
+			logrus.Warnf("Parse %s create time: %s", f.Name, err)
+		}
+		if f.MimeType == mimiTypeFolder {
+			folders = append(folders, &types.DirInfo{
+				RefID:   f.Id,
+				Path:    f.Name,
+				ModTime: t,
+			})
+			continue
+		}
+		files = append(files, &types.FileInfo{
+			RefID:   f.Id,
+			Name:    f.Name,
+			Size:    int(f.Size),
+			ModTime: t,
+		})
+	}
+	return folders, files, nil
+}

common/types/types.go

@@ -50,6 +50,7 @@ type DirInfo struct {
 	NumberOfFiles int
 	NumberOfDirs  int
 	TotalFileSize int
+	RefID         string // ID in cloud
 	Path          string
 	ModTime       time.Time
 	CreateTime    time.Time
@@ -60,6 +61,7 @@ type FileInfo struct {
 	ID      int
 	DirID   int
 	IsoID   int
+	RefID   string // ID in cloud
 	Name    string
 	Hash    string
 	Size    int