From ba59924d2cc314495144208984607c2f5f32375e Mon Sep 17 00:00:00 2001 From: Dweb Fan Date: Fri, 3 May 2024 21:26:23 -0700 Subject: [PATCH] support listing google drive files in tree view and also fix create time Signed-off-by: Dweb Fan --- cmd/lomob/list-cloud.go | 32 +++++++++++++++++++++ cmd/lomob/main.go | 6 ---- cmd/lomob/upload-files.go | 41 ++++++++++++++++++++------ common/gcloud/upload.go | 60 +++++++++++++++++++++++++++++++++++---- common/types/types.go | 2 ++ 5 files changed, 121 insertions(+), 20 deletions(-) diff --git a/cmd/lomob/list-cloud.go b/cmd/lomob/list-cloud.go index c11c178..25e3f64 100644 --- a/cmd/lomob/list-cloud.go +++ b/cmd/lomob/list-cloud.go @@ -2,9 +2,11 @@ package main import ( "fmt" + "strconv" "github.com/lomorage/lomo-backup/common/gcloud" "github.com/urfave/cli" + "github.com/xlab/treeprint" ) func listFilesInGDrive(ctx *cli.Context) error { @@ -24,8 +26,38 @@ func listFilesInGDrive(ctx *cli.Context) error { if folderID == "" { fmt.Printf("Folder '%s' not found.\n", folder) + return nil } else { fmt.Printf("File Name: %s, File ID: %s\n", folder, folderID) } + rootNode := treeprint.NewWithRoot(folder) + err = listFileTreeInGDrive(client, rootNode, folderID) + if err != nil { + return err + } + fmt.Println(rootNode.String()) + return nil +} + +func listFileTreeInGDrive(client *gcloud.Client, currNode treeprint.Tree, folderID string) error { + folders, files, err := client.ListFiles(folderID) + if err != nil { + return err + } + for _, folder := range folders { + t := folder.ModTime + + childNode := currNode.AddMetaBranch( + fmt.Sprintf("\t%02d/%02d/%d", t.Month(), t.Day(), t.Year()), folder.Path) + err = listFileTreeInGDrive(client, childNode, folder.RefID) + if err != nil { + return err + } + } + for _, file := range files { + t := file.ModTime + currNode.AddMetaNode(fmt.Sprintf("\t%12s\t%02d/%02d/%d", strconv.Itoa(file.Size), + t.Month(), t.Day(), t.Year()), file.Name) + } return nil } diff --git a/cmd/lomob/main.go b/cmd/lomob/main.go index 9ed65b8..2b94e20 100644 --- a/cmd/lomob/main.go +++ b/cmd/lomob/main.go @@ -231,12 +231,6 @@ func main() { }, }, }, - { - Name: "parts", - Action: calculatePartHash, - Usage: "Calculate given files base64 hash", - ArgsUsage: "[filename]", - }, { Name: "gdrive", Action: listFilesInGDrive, diff --git a/cmd/lomob/upload-files.go b/cmd/lomob/upload-files.go index 6cbbade..d832380 100644 --- a/cmd/lomob/upload-files.go +++ b/cmd/lomob/upload-files.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/lomorage/lomo-backup/common/gcloud" "github.com/sirupsen/logrus" @@ -34,7 +35,10 @@ func uploadFiles(ctx *cli.Context) error { } uploadRootFolder := ctx.String("folder") - exist, uploadRootFolderID, err := client.GetAndCreateFileIDIfNotExist(uploadRootFolder, "", nil) + exist, uploadRootFolderID, err := client.GetAndCreateFileIDIfNotExist(uploadRootFolder, "", nil, + gcloud.FileMetadata{ + ModTime: time.Now(), + }) if err != nil { return err } @@ -58,7 +62,7 @@ func uploadFiles(ctx *cli.Context) error { parentFolderID string } existingDirsInCloud := map[string]dirInfoInCloud{ - "": dirInfoInCloud{folderID: uploadRootFolderID}, + "": {folderID: uploadRootFolderID}, } for _, f := range fileInfos { scanRoot, ok := scanRootDirs[f.DirID] @@ -66,13 +70,21 @@ func uploadFiles(ctx *cli.Context) error { return fmt.Errorf("unable to find scan root directory whose ID is %d", f.DirID) } + origFolder := scanRoot + // flatten scan root dir so as to have only one folder // find its folder ID in google cloud, and create one if not exist, then add into local map scanRootFolderInCloud := flattenScanRootDir(strings.Trim(scanRoot, string(os.PathSeparator))) info, ok := existingDirsInCloud[scanRootFolderInCloud] if !ok { + stat, err := os.Stat(origFolder) + if err != nil { + return err + } + info = dirInfoInCloud{parentFolderID: uploadRootFolderID} - ok, info.folderID, err = client.GetAndCreateFileIDIfNotExist(scanRootFolderInCloud, uploadRootFolderID, nil) + ok, info.folderID, err = client.GetAndCreateFileIDIfNotExist(scanRootFolderInCloud, uploadRootFolderID, nil, + gcloud.FileMetadata{ModTime: stat.ModTime()}) if err != nil { return err } @@ -83,17 +95,23 @@ func uploadFiles(ctx *cli.Context) error { } scanRootFolderIDInCloud := info.folderID - // recursive check all directories' existance in cloud, and create if not exist + // check all directories' existence in cloud, and create if not exist dir, filename := filepath.Split(f.Name) dir = strings.Trim(dir, string(os.PathSeparator)) folderKey := scanRootFolderInCloud parentID := scanRootFolderIDInCloud for _, p := range strings.Split(dir, string(os.PathSeparator)) { + origFolder = filepath.Join(origFolder, p) folderKey += "/" + p info, ok = existingDirsInCloud[folderKey] if !ok { + stat, err := os.Stat(origFolder) + if err != nil { + return err + } info = dirInfoInCloud{parentFolderID: parentID} - ok, info.folderID, err = client.GetAndCreateFileIDIfNotExist(p, parentID, nil) + ok, info.folderID, err = client.GetAndCreateFileIDIfNotExist(p, parentID, nil, + gcloud.FileMetadata{ModTime: stat.ModTime()}) if err != nil { return err } @@ -112,12 +130,19 @@ func uploadFiles(ctx *cli.Context) error { if err != nil { return err } - logrus.Debugf("Uploading: %s into %s (%s):%s\n", fullLocalPath, folderKey, parentID, filename) - _, err = client.CreateFile(filename, parentID, reader) + logrus.Infof("Uploading: %s into %s (%s):%s\n", fullLocalPath, folderKey, parentID, filename) + stat, err := reader.Stat() + if err != nil { + return err + } + _, err = client.CreateFile(filename, parentID, reader, gcloud.FileMetadata{ + ModTime: stat.ModTime(), + Hash: f.Hash, + }) if err != nil { return err } - logrus.Debugf("Uploading success") + logrus.Infof("Uploading success") err = reader.Close() if err != nil { logrus.Warnf("Close %s: %s", fullLocalPath, err) diff --git a/common/gcloud/upload.go b/common/gcloud/upload.go index a5b1c75..27f5867 100644 --- a/common/gcloud/upload.go +++ b/common/gcloud/upload.go @@ -5,18 +5,27 @@ import ( "fmt" "io" "os" + "time" + "github.com/lomorage/lomo-backup/common/types" "github.com/sirupsen/logrus" "golang.org/x/oauth2/google" "google.golang.org/api/drive/v3" "google.golang.org/api/option" ) +const mimiTypeFolder = "application/vnd.google-apps.folder" + type Config struct { CredFilename string TokenFilename string } +type FileMetadata struct { + Hash string + ModTime time.Time +} + type Client struct { srv *drive.Service } @@ -49,7 +58,6 @@ func (c *Client) GetFileID(filename, parentFolderID string) (string, string, err query := fmt.Sprintf("name = '%s'", filename) if parentFolderID != "" { query += fmt.Sprintf(" and '%s' in parents", parentFolderID) - } files, err := c.srv.Files.List().Q(query).PageSize(1).Fields("files(id, name, parents)").Do() if err != nil { @@ -68,14 +76,14 @@ func (c *Client) GetFileID(filename, parentFolderID string) (string, string, err return files.Files[0].Id, parentID, nil } -func (c *Client) GetAndCreateFileIDIfNotExist(filename, parentFolderID string, r io.Reader) (bool, string, error) { +func (c *Client) GetAndCreateFileIDIfNotExist(filename, parentFolderID string, r io.Reader, m FileMetadata) (bool, string, error) { fileID, pid, err := c.GetFileID(filename, parentFolderID) if err != nil { return false, "", err } if fileID == "" { // not exist, create new one - fileID, err = c.CreateFile(filename, parentFolderID, r) + fileID, err = c.CreateFile(filename, parentFolderID, r, m) return false, fileID, err } if parentFolderID != "" && pid != parentFolderID { @@ -84,14 +92,22 @@ func (c *Client) GetAndCreateFileIDIfNotExist(filename, parentFolderID string, r return true, fileID, nil } -func (c *Client) CreateFile(filename, parentFolderID string, r io.Reader) (string, error) { - file := &drive.File{Name: filename} +func (c *Client) CreateFile(filename, parentFolderID string, r io.Reader, m FileMetadata) (string, error) { + file := &drive.File{ + Name: filename, + CreatedTime: m.ModTime.Format(time.RFC3339), + } + if m.Hash != "" { + file.AppProperties = map[string]string{ + "hash": m.Hash, + } + } if parentFolderID != "" { file.Parents = []string{parentFolderID} } if r == nil { // it is a folder - file.MimeType = "application/vnd.google-apps.folder" + file.MimeType = mimiTypeFolder f, err := c.srv.Files.Create(file).Do() if err != nil { return "", err @@ -104,3 +120,35 @@ func (c *Client) CreateFile(filename, parentFolderID string, r io.Reader) (strin } return f.Id, nil } + +func (c *Client) ListFiles(folderID string) ([]*types.DirInfo, []*types.FileInfo, error) { + query := fmt.Sprintf("'%s' in parents", folderID) + list, err := c.srv.Files.List().Q(query).Fields("files(id, name, size, mimeType, createdTime)").Do() + if err != nil { + return nil, nil, err + } + + files := []*types.FileInfo{} + folders := []*types.DirInfo{} + for _, f := range list.Files { + t, err := time.Parse(time.RFC3339, f.CreatedTime) + if err != nil { + logrus.Warnf("Parse %s create time: %s", f.Name, err) + } + if f.MimeType == mimiTypeFolder { + folders = append(folders, &types.DirInfo{ + RefID: f.Id, + Path: f.Name, + ModTime: t, + }) + continue + } + files = append(files, &types.FileInfo{ + RefID: f.Id, + Name: f.Name, + Size: int(f.Size), + ModTime: t, + }) + } + return folders, files, nil +} diff --git a/common/types/types.go b/common/types/types.go index 5f6904d..3c69093 100644 --- a/common/types/types.go +++ b/common/types/types.go @@ -50,6 +50,7 @@ type DirInfo struct { NumberOfFiles int NumberOfDirs int TotalFileSize int + RefID string // ID in cloud Path string ModTime time.Time CreateTime time.Time @@ -60,6 +61,7 @@ type FileInfo struct { ID int DirID int IsoID int + RefID string // ID in cloud Name string Hash string Size int