Add user-oriented file transfer logs (#463)

Co-authored-by: Casper da Costa-Luis <[email protected]>
iterative · Mar 30, 2022 · cd41ef3 · cd41ef3
1 parent 64c8280
commit cd41ef3
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 0 deletions.
diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md
@@ -85,6 +85,8 @@ This command will:
 
 With spot/preemptible instances (`spot >= 0`), auto-recovery logic and persistent storage will be used to relaunch interrupted tasks.
 
+-> **Note:** A large `workdir` may take a long time to upload.
+
 ## Query Status
 
 ```console
@@ -108,3 +110,5 @@ This command will:
 2. Delete all the cloud resources created by `terraform apply`.
 
 In this example, after running `terraform destroy`, the `results` directory should contain a file named `greeting.txt` with the text `Hello, World!`
+
+-> **Note:** A large `output` directory may take a long time to download.
diff --git a/go.mod b/go.mod
@@ -21,6 +21,7 @@ require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/brianvoe/gofakeit/v6 v6.9.0
 	github.com/cloudflare/gokey v0.1.0
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/gobwas/glob v0.2.3
 	github.com/google/go-github/v42 v42.0.0
 	github.com/hashicorp/hcl/v2 v2.8.0 // indirect

diff --git a/go.sum b/go.sum
@@ -295,6 +295,8 @@ github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQ
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dop251/scsu v0.0.0-20200422003335-8fadfb689669/go.mod h1:Gth7Xev0h28tuTayG4HlTZy90IXhiDgV2+MLtJzjpP0=
 github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.3 h1:h71/Ky9+298V45NSkxjhFv0aGsFOtPvqr25vbr5yMB0=

diff --git a/task/common/machine/storage.go b/task/common/machine/storage.go
@@ -9,6 +9,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	units "github.com/docker/go-units"
+
 	_ "github.com/rclone/rclone/backend/azureblob"
 	_ "github.com/rclone/rclone/backend/googlecloudstorage"
 	_ "github.com/rclone/rclone/backend/local"
@@ -19,6 +21,8 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/sync"
 
+	"github.com/sirupsen/logrus"
+
 	"terraform-provider-iterative/task/common"
 )
 
@@ -120,6 +124,12 @@ func Transfer(ctx context.Context, source, destination string, include string) e
 		return err
 	}
 
+	if count, size, err := operations.Count(ctx, sourceFileSystem); err == nil {
+		logrus.Infof("Transferring %s (%d files)...", units.HumanSize(float64(size)), count)
+	} else {
+		return err
+	}
+
 	return sync.CopyDir(ctx, destinationFileSystem, sourceFileSystem, true)
 }