From 52ee432a98b647ca518f95572b3f4c5a49234dc1 Mon Sep 17 00:00:00 2001 From: Shivam Srivastava Date: Fri, 19 Jan 2024 13:57:48 +0530 Subject: [PATCH 01/12] Download function added --- go.mod | 1 + go.sum | 2 ++ main.go | 6 +++++ plugin.go | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+) diff --git a/go.mod b/go.mod index 9b309d6..d015580 100644 --- a/go.mod +++ b/go.mod @@ -14,5 +14,6 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + golang.org/x/sync v0.6.0 golang.org/x/sys v0.1.0 // indirect ) diff --git a/go.sum b/go.sum index bcfd899..f8de94e 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,8 @@ golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/main.go b/main.go index cf452dd..f625b69 100644 --- a/main.go +++ b/main.go @@ -94,6 +94,11 @@ func main() { Usage: "server-side encryption algorithm, defaults to none", EnvVar: "PLUGIN_ENCRYPTION", }, + cli.BoolFlag{ + Name: "download", + Usage: "switch to download mode, which will fetch `target`'s files from s3 bucket and place them according to `strip-prefix`", + EnvVar: "PLUGIN_DOWNLOAD", + }, cli.BoolFlag{ Name: "dry-run", Usage: "dry run for debug purposes", @@ -164,6 +169,7 @@ func run(c *cli.Context) error { Exclude: c.StringSlice("exclude"), Encryption: c.String("encryption"), ContentType: c.Generic("content-type").(*StringMapFlag).Get(), + Download: c.Bool("download"), ContentEncoding: c.Generic("content-encoding").(*StringMapFlag).Get(), CacheControl: c.Generic("cache-control").(*StringMapFlag).Get(), StorageClass: c.String("storage-class"), diff --git a/plugin.go b/plugin.go index 7b0bed6..715b2a1 100644 --- a/plugin.go +++ b/plugin.go @@ -1,6 +1,7 @@ package main import ( + "io" "mime" "os" "path/filepath" @@ -16,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go/service/sts" "github.com/mattn/go-zglob" log "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" ) // Plugin defines the S3 plugin parameters. @@ -44,6 +46,9 @@ type Plugin struct { // sa-east-1 Region string + // if true, plugin is set to download mode, which means `target` from the bucket will be downloaded + Download bool + // Indicates the files ACL, which should be one // of the following: // private @@ -135,6 +140,77 @@ func (p *Plugin) Exec() error { client = s3.New(sess) } + if p.Download { + targetDir := strings.TrimPrefix(filepath.ToSlash(p.Target), "/") + log.WithFields(log.Fields{ + "bucket": p.Bucket, + "dir": targetDir, + }).Info("Listing S3 directory") + + list, err := client.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: &p.Bucket, + Prefix: &targetDir, + }) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "bucket": p.Bucket, + "dir": targetDir, + }).Error("Cannot list S3 directory") + return err + } + + g := errgroup.Group{} + + for _, item := range list.Contents { + log.WithFields(log.Fields{ + "bucket": p.Bucket, + "key": *item.Key, + }).Info("Getting S3 object") + + item := item + g.Go(func() error { + obj, err := client.GetObject(&s3.GetObjectInput{ + Bucket: &p.Bucket, + Key: item.Key, + }) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "bucket": p.Bucket, + "key": *item.Key, + }).Error("Cannot get S3 object") + return err + } + + source := resolveSource(targetDir, *item.Key, p.StripPrefix) + + f, err := os.Create(source) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": source, + }).Error("Problem opening file for writing") + return err + } + defer f.Close() + + _, err = io.Copy(f, obj.Body) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": source, + }).Error("Failed to write file") + return err + } + + return nil + }) + } + + return g.Wait() + } + // find the bucket log.WithFields(log.Fields{ "region": p.Region, @@ -322,6 +398,11 @@ func resolveKey(target, srcPath, stripPrefix string) string { return key } +func resolveSource(targetDir, target, stripPrefix string) string { + path := strings.TrimPrefix(strings.TrimPrefix(target, targetDir), "/") + return stripPrefix + path +} + // checks if the source path is a dir func isDir(source string, matches []string) bool { stat, err := os.Stat(source) From f4ec4fde87208277eb78ec02ee0c4d80de954b21 Mon Sep 17 00:00:00 2001 From: Shivam Srivastava Date: Mon, 22 Jan 2024 12:44:07 +0530 Subject: [PATCH 02/12] Tests added for download function --- plugin.go | 145 +++++++++++++++++++++++--------------------- plugin_unix_test.go | 35 +++++++++++ 2 files changed, 112 insertions(+), 68 deletions(-) diff --git a/plugin.go b/plugin.go index 715b2a1..dfd60b7 100644 --- a/plugin.go +++ b/plugin.go @@ -141,74 +141,7 @@ func (p *Plugin) Exec() error { } if p.Download { - targetDir := strings.TrimPrefix(filepath.ToSlash(p.Target), "/") - log.WithFields(log.Fields{ - "bucket": p.Bucket, - "dir": targetDir, - }).Info("Listing S3 directory") - - list, err := client.ListObjectsV2(&s3.ListObjectsV2Input{ - Bucket: &p.Bucket, - Prefix: &targetDir, - }) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "bucket": p.Bucket, - "dir": targetDir, - }).Error("Cannot list S3 directory") - return err - } - - g := errgroup.Group{} - - for _, item := range list.Contents { - log.WithFields(log.Fields{ - "bucket": p.Bucket, - "key": *item.Key, - }).Info("Getting S3 object") - - item := item - g.Go(func() error { - obj, err := client.GetObject(&s3.GetObjectInput{ - Bucket: &p.Bucket, - Key: item.Key, - }) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "bucket": p.Bucket, - "key": *item.Key, - }).Error("Cannot get S3 object") - return err - } - - source := resolveSource(targetDir, *item.Key, p.StripPrefix) - - f, err := os.Create(source) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "file": source, - }).Error("Problem opening file for writing") - return err - } - defer f.Close() - - _, err = io.Copy(f, obj.Body) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "file": source, - }).Error("Failed to write file") - return err - } - - return nil - }) - } - - return g.Wait() + downloadFromS3(p, client) } // find the bucket @@ -423,3 +356,79 @@ func isDir(source string, matches []string) bool { } return false } + +func resolveTargetDir(s string) string { + res := strings.TrimPrefix(filepath.ToSlash(s), "/") + return res +} + +func downloadFromS3(p *Plugin, client *s3.S3) error { + targetDir := resolveTargetDir(p.Target) + log.WithFields(log.Fields{ + "bucket": p.Bucket, + "dir": targetDir, + }).Info("Listing S3 directory") + + list, err := client.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: &p.Bucket, + Prefix: &targetDir, + }) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "bucket": p.Bucket, + "dir": targetDir, + }).Error("Cannot list S3 directory") + return err + } + + g := errgroup.Group{} + + for _, item := range list.Contents { + log.WithFields(log.Fields{ + "bucket": p.Bucket, + "key": *item.Key, + }).Info("Getting S3 object") + + item := item + g.Go(func() error { + obj, err := client.GetObject(&s3.GetObjectInput{ + Bucket: &p.Bucket, + Key: item.Key, + }) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "bucket": p.Bucket, + "key": *item.Key, + }).Error("Cannot get S3 object") + return err + } + + source := resolveSource(targetDir, *item.Key, p.StripPrefix) + + f, err := os.Create(source) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": source, + }).Error("Problem opening file for writing") + return err + } + defer f.Close() + + _, err = io.Copy(f, obj.Body) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": source, + }).Error("Failed to write file") + return err + } + + return nil + }) + } + + return g.Wait() +} diff --git a/plugin_unix_test.go b/plugin_unix_test.go index 517b1f4..9ecb960 100644 --- a/plugin_unix_test.go +++ b/plugin_unix_test.go @@ -44,3 +44,38 @@ func TestResolveUnixKey(t *testing.T) { } } } + +func TestResolveTargetDir(t *testing.T) { + tests := []struct { + input string + expected string + }{ + { + input: "example-string", + expected: "example-string", + }, + { + input: "/path/to/file", + expected: "path/to/file", + }, + { + input: "12345", + expected: "12345", + }, + { + input: "/root/directory", + expected: "root/directory", + }, + { + input: "no_slash", + expected: "no_slash", + }, + } + + for _, tc := range tests { + result := resolveTargetDir(tc.input) + if result != tc.expected { + t.Errorf("Expected: %s, Got: %s", tc.expected, result) + } + } +} From a18136d67039fe27715c610e4881228c9a7b50d5 Mon Sep 17 00:00:00 2001 From: Shivam Srivastava Date: Mon, 22 Jan 2024 15:31:27 +0530 Subject: [PATCH 03/12] Minor optimizations --- plugin.go | 142 ++++++++++++++++++++++++++---------------------------- 1 file changed, 69 insertions(+), 73 deletions(-) diff --git a/plugin.go b/plugin.go index dfd60b7..5095a72 100644 --- a/plugin.go +++ b/plugin.go @@ -103,7 +103,7 @@ type Plugin struct { // Exec runs the plugin func (p *Plugin) Exec() error { // normalize the target URL - p.Target = strings.TrimPrefix(p.Target, "/") + p.Target = resolveTargetDir(p.Target) // create the client conf := &aws.Config{ @@ -141,7 +141,74 @@ func (p *Plugin) Exec() error { } if p.Download { - downloadFromS3(p, client) + targetDir := strings.TrimPrefix(filepath.ToSlash(p.Target), "/") + log.WithFields(log.Fields{ + "bucket": p.Bucket, + "dir": targetDir, + }).Info("Listing S3 directory") + + list, err := client.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: &p.Bucket, + Prefix: &targetDir, + }) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "bucket": p.Bucket, + "dir": targetDir, + }).Error("Cannot list S3 directory") + return err + } + + g := errgroup.Group{} + + for _, item := range list.Contents { + log.WithFields(log.Fields{ + "bucket": p.Bucket, + "key": *item.Key, + }).Info("Getting S3 object") + + item := item + g.Go(func() error { + obj, err := client.GetObject(&s3.GetObjectInput{ + Bucket: &p.Bucket, + Key: item.Key, + }) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "bucket": p.Bucket, + "key": *item.Key, + }).Error("Cannot get S3 object") + return err + } + + source := resolveSource(targetDir, *item.Key, p.StripPrefix) + + f, err := os.Create(source) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": source, + }).Error("Problem opening file for writing") + return err + } + defer f.Close() + + _, err = io.Copy(f, obj.Body) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": source, + }).Error("Failed to write file") + return err + } + + return nil + }) + } + + return g.Wait() } // find the bucket @@ -361,74 +428,3 @@ func resolveTargetDir(s string) string { res := strings.TrimPrefix(filepath.ToSlash(s), "/") return res } - -func downloadFromS3(p *Plugin, client *s3.S3) error { - targetDir := resolveTargetDir(p.Target) - log.WithFields(log.Fields{ - "bucket": p.Bucket, - "dir": targetDir, - }).Info("Listing S3 directory") - - list, err := client.ListObjectsV2(&s3.ListObjectsV2Input{ - Bucket: &p.Bucket, - Prefix: &targetDir, - }) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "bucket": p.Bucket, - "dir": targetDir, - }).Error("Cannot list S3 directory") - return err - } - - g := errgroup.Group{} - - for _, item := range list.Contents { - log.WithFields(log.Fields{ - "bucket": p.Bucket, - "key": *item.Key, - }).Info("Getting S3 object") - - item := item - g.Go(func() error { - obj, err := client.GetObject(&s3.GetObjectInput{ - Bucket: &p.Bucket, - Key: item.Key, - }) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "bucket": p.Bucket, - "key": *item.Key, - }).Error("Cannot get S3 object") - return err - } - - source := resolveSource(targetDir, *item.Key, p.StripPrefix) - - f, err := os.Create(source) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "file": source, - }).Error("Problem opening file for writing") - return err - } - defer f.Close() - - _, err = io.Copy(f, obj.Body) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "file": source, - }).Error("Failed to write file") - return err - } - - return nil - }) - } - - return g.Wait() -} From d6f79f023d4fbcff05945596590acbfe70017ac9 Mon Sep 17 00:00:00 2001 From: Shivam Srivastava Date: Tue, 23 Jan 2024 11:49:00 +0530 Subject: [PATCH 04/12] Variable name optimized --- plugin.go | 31 +++++++++++++++++++++---------- plugin_unix_test.go | 4 ++-- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/plugin.go b/plugin.go index 5095a72..54a3dad 100644 --- a/plugin.go +++ b/plugin.go @@ -103,7 +103,11 @@ type Plugin struct { // Exec runs the plugin func (p *Plugin) Exec() error { // normalize the target URL - p.Target = resolveTargetDir(p.Target) + if p.Download { + p.Source = resolveDir(p.Source) + } else { + p.Target = resolveDir(p.Target) + } // create the client conf := &aws.Config{ @@ -141,21 +145,23 @@ func (p *Plugin) Exec() error { } if p.Download { - targetDir := strings.TrimPrefix(filepath.ToSlash(p.Target), "/") + // sourceDir := strings.TrimPrefix(filepath.ToSlash(p.Source), "/") + sourceDir := normalizePath(p.Source) + log.WithFields(log.Fields{ "bucket": p.Bucket, - "dir": targetDir, + "dir": sourceDir, }).Info("Listing S3 directory") list, err := client.ListObjectsV2(&s3.ListObjectsV2Input{ Bucket: &p.Bucket, - Prefix: &targetDir, + Prefix: &sourceDir, }) if err != nil { log.WithFields(log.Fields{ "error": err, "bucket": p.Bucket, - "dir": targetDir, + "dir": sourceDir, }).Error("Cannot list S3 directory") return err } @@ -183,13 +189,13 @@ func (p *Plugin) Exec() error { return err } - source := resolveSource(targetDir, *item.Key, p.StripPrefix) + target := resolveSource(sourceDir, *item.Key, p.StripPrefix) - f, err := os.Create(source) + f, err := os.Create(target) if err != nil { log.WithFields(log.Fields{ "error": err, - "file": source, + "file": target, }).Error("Problem opening file for writing") return err } @@ -199,7 +205,7 @@ func (p *Plugin) Exec() error { if err != nil { log.WithFields(log.Fields{ "error": err, - "file": source, + "file": target, }).Error("Failed to write file") return err } @@ -424,7 +430,12 @@ func isDir(source string, matches []string) bool { return false } -func resolveTargetDir(s string) string { +func resolveDir(s string) string { res := strings.TrimPrefix(filepath.ToSlash(s), "/") return res } + +// normalizePath converts the path to a forward slash format and trims the prefix. +func normalizePath(source string) string { + return strings.TrimPrefix(filepath.ToSlash(source), "/") +} diff --git a/plugin_unix_test.go b/plugin_unix_test.go index 9ecb960..5067474 100644 --- a/plugin_unix_test.go +++ b/plugin_unix_test.go @@ -45,7 +45,7 @@ func TestResolveUnixKey(t *testing.T) { } } -func TestResolveTargetDir(t *testing.T) { +func TestResolveDir(t *testing.T) { tests := []struct { input string expected string @@ -73,7 +73,7 @@ func TestResolveTargetDir(t *testing.T) { } for _, tc := range tests { - result := resolveTargetDir(tc.input) + result := resolveDir(tc.input) if result != tc.expected { t.Errorf("Expected: %s, Got: %s", tc.expected, result) } From ee30ef01a7c5189b80a8afebb68bf2ea3536cc75 Mon Sep 17 00:00:00 2001 From: Shivam Srivastava Date: Tue, 23 Jan 2024 11:54:31 +0530 Subject: [PATCH 05/12] Unit tests added --- plugin_unix_test.go | 39 ++++++++++++++++++++++ plugin_windows_test.go | 74 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/plugin_unix_test.go b/plugin_unix_test.go index 5067474..288c1e9 100644 --- a/plugin_unix_test.go +++ b/plugin_unix_test.go @@ -79,3 +79,42 @@ func TestResolveDir(t *testing.T) { } } } + +func TestNormalizePath(t *testing.T) { + tests := []struct { + input string + expected string + }{ + { + input: "/path/to/file.txt", + expected: "path/to/file.txt", + }, + { + input: "C:\\Users\\username\\Documents\\file.doc", + expected: "C:\\Users\\username\\Documents\\file.doc", + }, + { + input: "relative/path/to/file", + expected: "relative/path/to/file", + }, + { + input: "file.txt", + expected: "file.txt", + }, + { + input: "/root/directory/", + expected: "root/directory/", + }, + { + input: "no_slash", + expected: "no_slash", + }, + } + + for _, tc := range tests { + result := normalizePath(tc.input) + if result != tc.expected { + t.Errorf("Expected: %s, Got: %s", tc.expected, result) + } + } +} diff --git a/plugin_windows_test.go b/plugin_windows_test.go index 27379d6..c679d36 100644 --- a/plugin_windows_test.go +++ b/plugin_windows_test.go @@ -58,3 +58,77 @@ func TestResolveWinKey(t *testing.T) { } } } + +func TestResolveDir(t *testing.T) { + tests := []struct { + input string + expected string + }{ + { + input: "example-string", + expected: "example-string", + }, + { + input: "/path/to/file", + expected: "path/to/file", + }, + { + input: "12345", + expected: "12345", + }, + { + input: "/root/directory", + expected: "root/directory", + }, + { + input: "no_slash", + expected: "no_slash", + }, + } + + for _, tc := range tests { + result := resolveDir(tc.input) + if result != tc.expected { + t.Errorf("Expected: %s, Got: %s", tc.expected, result) + } + } +} + +func TestNormalizePath(t *testing.T) { + tests := []struct { + input string + expected string + }{ + { + input: "/path/to/file.txt", + expected: "path/to/file.txt", + }, + { + input: "C:\\Users\\username\\Documents\\file.doc", + expected: "C:\\Users\\username\\Documents\\file.doc", + }, + { + input: "relative/path/to/file", + expected: "relative/path/to/file", + }, + { + input: "file.txt", + expected: "file.txt", + }, + { + input: "/root/directory/", + expected: "root/directory/", + }, + { + input: "no_slash", + expected: "no_slash", + }, + } + + for _, tc := range tests { + result := normalizePath(tc.input) + if result != tc.expected { + t.Errorf("Expected: %s, Got: %s", tc.expected, result) + } + } +} From 4a459fa610e6484010e340d4512d98aa1d7e4e84 Mon Sep 17 00:00:00 2001 From: Shivam Srivastava Date: Tue, 30 Jan 2024 17:19:28 +0530 Subject: [PATCH 06/12] Migrated base image to scratch --- docker/Dockerfile.linux.arm64 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.linux.arm64 b/docker/Dockerfile.linux.arm64 index 337e207..269905c 100644 --- a/docker/Dockerfile.linux.arm64 +++ b/docker/Dockerfile.linux.arm64 @@ -1,4 +1,6 @@ -FROM plugins/base:multiarch +FROM plugins/base:multiarch as builder + +FROM scratch LABEL maintainer="Drone.IO Community " \ org.label-schema.name="Drone S3" \ @@ -6,4 +8,7 @@ LABEL maintainer="Drone.IO Community " \ org.label-schema.schema-version="1.0" ADD release/linux/arm64/drone-s3 /bin/ + +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ + ENTRYPOINT ["/bin/drone-s3"] From 7abb0a98e9fec4ce2f22151f63f72751fde00fb8 Mon Sep 17 00:00:00 2001 From: Shivam Srivastava Date: Fri, 2 Feb 2024 12:08:42 +0530 Subject: [PATCH 07/12] README updated --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 238462c..a901af6 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ docker: Error response from daemon: Container command Execute from the working directory: +* For upload ``` docker run --rm \ -e PLUGIN_SOURCE= \ @@ -53,3 +54,17 @@ docker run --rm \ -w $(pwd) \ plugins/s3 --dry-run ``` + +* For download +``` +docker run --rm \ + -e PLUGIN_SOURCE= \ + -e PLUGIN_BUCKET= \ + -e AWS_ACCESS_KEY_ID= \ + -e AWS_SECRET_ACCESS_KEY= \ + -e PLUGIN_REGION= \ + -e PLUGIN_DOWNLOAD="true" \ + -v $(pwd):$(pwd) \ + -w $(pwd) \ + plugins/s3 --dry-run +``` From f5e42841bdc0b28d9992fb09188f182f5bb6ce9b Mon Sep 17 00:00:00 2001 From: Shivam Srivastava Date: Fri, 2 Feb 2024 17:39:59 +0530 Subject: [PATCH 08/12] Minor changes --- docker/Dockerfile.linux.arm64 | 9 ++------- plugin.go | 5 ++--- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile.linux.arm64 b/docker/Dockerfile.linux.arm64 index 269905c..f6f8b2b 100644 --- a/docker/Dockerfile.linux.arm64 +++ b/docker/Dockerfile.linux.arm64 @@ -1,6 +1,4 @@ -FROM plugins/base:multiarch as builder - -FROM scratch +FROM plugins/base:multiarch LABEL maintainer="Drone.IO Community " \ org.label-schema.name="Drone S3" \ @@ -8,7 +6,4 @@ LABEL maintainer="Drone.IO Community " \ org.label-schema.schema-version="1.0" ADD release/linux/arm64/drone-s3 /bin/ - -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ - -ENTRYPOINT ["/bin/drone-s3"] +ENTRYPOINT ["/bin/drone-s3"] \ No newline at end of file diff --git a/plugin.go b/plugin.go index 54a3dad..b804d4a 100644 --- a/plugin.go +++ b/plugin.go @@ -46,7 +46,7 @@ type Plugin struct { // sa-east-1 Region string - // if true, plugin is set to download mode, which means `target` from the bucket will be downloaded + // if true, plugin is set to download mode, which means `source` from the bucket will be downloaded Download bool // Indicates the files ACL, which should be one @@ -145,7 +145,6 @@ func (p *Plugin) Exec() error { } if p.Download { - // sourceDir := strings.TrimPrefix(filepath.ToSlash(p.Source), "/") sourceDir := normalizePath(p.Source) log.WithFields(log.Fields{ @@ -196,7 +195,7 @@ func (p *Plugin) Exec() error { log.WithFields(log.Fields{ "error": err, "file": target, - }).Error("Problem opening file for writing") + }).Error("Failed to create file") return err } defer f.Close() From b8c5bf42438d37d1d6c43c6719067219398fcd08 Mon Sep 17 00:00:00 2001 From: Shivam Srivastava Date: Fri, 2 Feb 2024 18:09:04 +0530 Subject: [PATCH 09/12] Comments added --- plugin.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugin.go b/plugin.go index b804d4a..fc3932e 100644 --- a/plugin.go +++ b/plugin.go @@ -188,6 +188,9 @@ func (p *Plugin) Exec() error { return err } + // resolveSource takes a target directory, a target path, and a prefix to strip, + // and returns a resolved source path by removing the targetDir from the target + // and appending the stripPrefix. target := resolveSource(sourceDir, *item.Key, p.StripPrefix) f, err := os.Create(target) @@ -404,7 +407,10 @@ func resolveKey(target, srcPath, stripPrefix string) string { } func resolveSource(targetDir, target, stripPrefix string) string { + // Remove the leading targetDir from the target path path := strings.TrimPrefix(strings.TrimPrefix(target, targetDir), "/") + + // Add the specified stripPrefix to the resulting path return stripPrefix + path } From 56cdc0b3689a8fd899a296effc2fa457ff4192de Mon Sep 17 00:00:00 2001 From: Shivam Srivastava Date: Fri, 2 Feb 2024 21:27:15 +0530 Subject: [PATCH 10/12] Minor changes --- plugin.go | 90 ++++++++++++++++++++------------------------- plugin_unix_test.go | 35 ------------------ 2 files changed, 39 insertions(+), 86 deletions(-) diff --git a/plugin.go b/plugin.go index fc3932e..4095b3e 100644 --- a/plugin.go +++ b/plugin.go @@ -17,7 +17,6 @@ import ( "github.com/aws/aws-sdk-go/service/sts" "github.com/mattn/go-zglob" log "github.com/sirupsen/logrus" - "golang.org/x/sync/errgroup" ) // Plugin defines the S3 plugin parameters. @@ -104,9 +103,9 @@ type Plugin struct { func (p *Plugin) Exec() error { // normalize the target URL if p.Download { - p.Source = resolveDir(p.Source) + p.Source = normalizePath(p.Source) } else { - p.Target = resolveDir(p.Target) + p.Target = normalizePath(p.Target) } // create the client @@ -165,58 +164,52 @@ func (p *Plugin) Exec() error { return err } - g := errgroup.Group{} - for _, item := range list.Contents { log.WithFields(log.Fields{ "bucket": p.Bucket, "key": *item.Key, }).Info("Getting S3 object") - item := item - g.Go(func() error { - obj, err := client.GetObject(&s3.GetObjectInput{ - Bucket: &p.Bucket, - Key: item.Key, - }) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "bucket": p.Bucket, - "key": *item.Key, - }).Error("Cannot get S3 object") - return err - } - - // resolveSource takes a target directory, a target path, and a prefix to strip, - // and returns a resolved source path by removing the targetDir from the target - // and appending the stripPrefix. - target := resolveSource(sourceDir, *item.Key, p.StripPrefix) - - f, err := os.Create(target) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "file": target, - }).Error("Failed to create file") - return err - } - defer f.Close() - - _, err = io.Copy(f, obj.Body) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "file": target, - }).Error("Failed to write file") - return err - } - - return nil + obj, err := client.GetObject(&s3.GetObjectInput{ + Bucket: &p.Bucket, + Key: item.Key, }) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "bucket": p.Bucket, + "key": *item.Key, + }).Error("Cannot get S3 object") + return err + } + defer obj.Body.Close() + + // resolveSource takes a target directory, a target path, and a prefix to strip, + // and returns a resolved source path by removing the targetDir from the target + // and appending the stripPrefix. + target := resolveSource(sourceDir, *item.Key, p.StripPrefix) + + f, err := os.Create(target) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": target, + }).Error("Failed to create file") + return err + } + defer f.Close() + + _, err = io.Copy(f, obj.Body) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": target, + }).Error("Failed to write file") + return err + } } - return g.Wait() + return nil } // find the bucket @@ -435,11 +428,6 @@ func isDir(source string, matches []string) bool { return false } -func resolveDir(s string) string { - res := strings.TrimPrefix(filepath.ToSlash(s), "/") - return res -} - // normalizePath converts the path to a forward slash format and trims the prefix. func normalizePath(source string) string { return strings.TrimPrefix(filepath.ToSlash(source), "/") diff --git a/plugin_unix_test.go b/plugin_unix_test.go index 288c1e9..8699520 100644 --- a/plugin_unix_test.go +++ b/plugin_unix_test.go @@ -45,41 +45,6 @@ func TestResolveUnixKey(t *testing.T) { } } -func TestResolveDir(t *testing.T) { - tests := []struct { - input string - expected string - }{ - { - input: "example-string", - expected: "example-string", - }, - { - input: "/path/to/file", - expected: "path/to/file", - }, - { - input: "12345", - expected: "12345", - }, - { - input: "/root/directory", - expected: "root/directory", - }, - { - input: "no_slash", - expected: "no_slash", - }, - } - - for _, tc := range tests { - result := resolveDir(tc.input) - if result != tc.expected { - t.Errorf("Expected: %s, Got: %s", tc.expected, result) - } - } -} - func TestNormalizePath(t *testing.T) { tests := []struct { input string From fbde487bf99a31b2408f618de7a3c81be57718b0 Mon Sep 17 00:00:00 2001 From: Shivam Srivastava Date: Mon, 5 Feb 2024 14:08:39 +0530 Subject: [PATCH 11/12] Changes and fixes --- go.mod | 1 + go.sum | 1 + plugin.go | 202 ++++++++++++++++++++++++++--------------- plugin_windows_test.go | 35 ------- 4 files changed, 133 insertions(+), 106 deletions(-) diff --git a/go.mod b/go.mod index d015580..c47aa27 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( require ( github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/pkg/errors v0.9.1 github.com/russross/blackfriday/v2 v2.1.0 // indirect golang.org/x/sync v0.6.0 golang.org/x/sys v0.1.0 // indirect diff --git a/go.sum b/go.sum index f8de94e..9fe4b78 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,7 @@ github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/mattn/go-zglob v0.0.4 h1:LQi2iOm0/fGgu80AioIJ/1j9w9Oh+9DZ39J4VAGzHQM= github.com/mattn/go-zglob v0.0.4/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/plugin.go b/plugin.go index 4095b3e..cbabc52 100644 --- a/plugin.go +++ b/plugin.go @@ -16,6 +16,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/sts" "github.com/mattn/go-zglob" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -101,11 +102,11 @@ type Plugin struct { // Exec runs the plugin func (p *Plugin) Exec() error { - // normalize the target URL if p.Download { p.Source = normalizePath(p.Source) - } else { p.Target = normalizePath(p.Target) + } else { + p.Target = strings.TrimPrefix(p.Target, "/") } // create the client @@ -143,73 +144,12 @@ func (p *Plugin) Exec() error { client = s3.New(sess) } + // If in download mode, call the downloadS3Objects method if p.Download { sourceDir := normalizePath(p.Source) + client := p.createS3Client() - log.WithFields(log.Fields{ - "bucket": p.Bucket, - "dir": sourceDir, - }).Info("Listing S3 directory") - - list, err := client.ListObjectsV2(&s3.ListObjectsV2Input{ - Bucket: &p.Bucket, - Prefix: &sourceDir, - }) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "bucket": p.Bucket, - "dir": sourceDir, - }).Error("Cannot list S3 directory") - return err - } - - for _, item := range list.Contents { - log.WithFields(log.Fields{ - "bucket": p.Bucket, - "key": *item.Key, - }).Info("Getting S3 object") - - obj, err := client.GetObject(&s3.GetObjectInput{ - Bucket: &p.Bucket, - Key: item.Key, - }) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "bucket": p.Bucket, - "key": *item.Key, - }).Error("Cannot get S3 object") - return err - } - defer obj.Body.Close() - - // resolveSource takes a target directory, a target path, and a prefix to strip, - // and returns a resolved source path by removing the targetDir from the target - // and appending the stripPrefix. - target := resolveSource(sourceDir, *item.Key, p.StripPrefix) - - f, err := os.Create(target) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "file": target, - }).Error("Failed to create file") - return err - } - defer f.Close() - - _, err = io.Copy(f, obj.Body) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "file": target, - }).Error("Failed to write file") - return err - } - } - - return nil + return p.downloadS3Objects(client, sourceDir) } // find the bucket @@ -399,9 +339,9 @@ func resolveKey(target, srcPath, stripPrefix string) string { return key } -func resolveSource(targetDir, target, stripPrefix string) string { - // Remove the leading targetDir from the target path - path := strings.TrimPrefix(strings.TrimPrefix(target, targetDir), "/") +func resolveSource(sourceDir, source, stripPrefix string) string { + // Remove the leading sourceDir from the source path + path := strings.TrimPrefix(strings.TrimPrefix(source, sourceDir), "/") // Add the specified stripPrefix to the resulting path return stripPrefix + path @@ -429,6 +369,126 @@ func isDir(source string, matches []string) bool { } // normalizePath converts the path to a forward slash format and trims the prefix. -func normalizePath(source string) string { - return strings.TrimPrefix(filepath.ToSlash(source), "/") +func normalizePath(path string) string { + return strings.TrimPrefix(filepath.ToSlash(path), "/") +} + +// downloadS3Object downloads a single object from S3 +func (p *Plugin) downloadS3Object(client *s3.S3, sourceDir, key, target string) error { + log.WithFields(log.Fields{ + "bucket": p.Bucket, + "key": key, + }).Info("Getting S3 object") + + obj, err := client.GetObject(&s3.GetObjectInput{ + Bucket: &p.Bucket, + Key: &key, + }) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "bucket": p.Bucket, + "key": key, + }).Error("Cannot get S3 object") + return err + } + defer obj.Body.Close() + + // Create the destination file path + destination := filepath.Join(p.Target, target) + log.Println("Destination: ", destination) + + // Extract the directory from the destination path + dir := filepath.Dir(destination) + + // Create the directory and any necessary parent directories + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return errors.Wrap(err, "error creating directories") + } + + f, err := os.Create(destination) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": destination, + }).Error("Failed to create file") + return err + } + defer f.Close() + + _, err = io.Copy(f, obj.Body) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": destination, + }).Error("Failed to write file") + return err + } + + return nil +} + +// downloadS3Objects downloads all objects in the specified S3 bucket path +func (p *Plugin) downloadS3Objects(client *s3.S3, sourceDir string) error { + log.WithFields(log.Fields{ + "bucket": p.Bucket, + "dir": sourceDir, + }).Info("Listing S3 directory") + + list, err := client.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: &p.Bucket, + Prefix: &sourceDir, + }) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "bucket": p.Bucket, + "dir": sourceDir, + }).Error("Cannot list S3 directory") + return err + } + + for _, item := range list.Contents { + // resolveSource takes a source directory, a source path, and a prefix to strip, + // and returns a resolved target path by removing the sourceDir from the source + // and appending the stripPrefix. + target := resolveSource(sourceDir, *item.Key, p.StripPrefix) + + if err := p.downloadS3Object(client, sourceDir, *item.Key, target); err != nil { + return err + } + } + + return nil +} + +// createS3Client creates and returns an S3 client based on the plugin configuration +func (p *Plugin) createS3Client() *s3.S3 { + conf := &aws.Config{ + Region: aws.String(p.Region), + Endpoint: &p.Endpoint, + DisableSSL: aws.Bool(strings.HasPrefix(p.Endpoint, "http://")), + S3ForcePathStyle: aws.Bool(p.PathStyle), + } + + if p.Key != "" && p.Secret != "" { + conf.Credentials = credentials.NewStaticCredentials(p.Key, p.Secret, "") + } else if p.AssumeRole != "" { + conf.Credentials = assumeRole(p.AssumeRole, p.AssumeRoleSessionName, p.ExternalID) + } else { + log.Warn("AWS Key and/or Secret not provided (falling back to ec2 instance profile)") + } + + sess, _ := session.NewSession(conf) + client := s3.New(sess) + + if len(p.UserRoleArn) > 0 { + confRoleArn := aws.Config{ + Region: aws.String(p.Region), + Credentials: stscreds.NewCredentials(sess, p.UserRoleArn), + } + client = s3.New(sess, &confRoleArn) + } + + return client } diff --git a/plugin_windows_test.go b/plugin_windows_test.go index c679d36..ea1b170 100644 --- a/plugin_windows_test.go +++ b/plugin_windows_test.go @@ -59,41 +59,6 @@ func TestResolveWinKey(t *testing.T) { } } -func TestResolveDir(t *testing.T) { - tests := []struct { - input string - expected string - }{ - { - input: "example-string", - expected: "example-string", - }, - { - input: "/path/to/file", - expected: "path/to/file", - }, - { - input: "12345", - expected: "12345", - }, - { - input: "/root/directory", - expected: "root/directory", - }, - { - input: "no_slash", - expected: "no_slash", - }, - } - - for _, tc := range tests { - result := resolveDir(tc.input) - if result != tc.expected { - t.Errorf("Expected: %s, Got: %s", tc.expected, result) - } - } -} - func TestNormalizePath(t *testing.T) { tests := []struct { input string From 7eefd11859a548ad1d7fb0848262090a44e92f2e Mon Sep 17 00:00:00 2001 From: Shivam Srivastava Date: Tue, 6 Feb 2024 17:28:10 +0530 Subject: [PATCH 12/12] Changes and tests --- main.go | 4 ++-- plugin.go | 35 +------------------------------- plugin_unix_test.go | 45 ++++++++++++++++++++++++++++++++++++++++++ plugin_windows_test.go | 45 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 36 deletions(-) diff --git a/main.go b/main.go index f625b69..a1114f4 100644 --- a/main.go +++ b/main.go @@ -81,7 +81,7 @@ func main() { }, cli.StringFlag{ Name: "strip-prefix", - Usage: "strip the prefix from the target", + Usage: "used to add or remove a prefix from the source/target path", EnvVar: "PLUGIN_STRIP_PREFIX", }, cli.StringSliceFlag{ @@ -96,7 +96,7 @@ func main() { }, cli.BoolFlag{ Name: "download", - Usage: "switch to download mode, which will fetch `target`'s files from s3 bucket and place them according to `strip-prefix`", + Usage: "switch to download mode, which will fetch `source`'s files from s3 bucket", EnvVar: "PLUGIN_DOWNLOAD", }, cli.BoolFlag{ diff --git a/plugin.go b/plugin.go index cbabc52..d7b3387 100644 --- a/plugin.go +++ b/plugin.go @@ -110,44 +110,11 @@ func (p *Plugin) Exec() error { } // create the client - conf := &aws.Config{ - Region: aws.String(p.Region), - Endpoint: &p.Endpoint, - DisableSSL: aws.Bool(strings.HasPrefix(p.Endpoint, "http://")), - S3ForcePathStyle: aws.Bool(p.PathStyle), - } - - if p.Key != "" && p.Secret != "" { - conf.Credentials = credentials.NewStaticCredentials(p.Key, p.Secret, "") - } else if p.AssumeRole != "" { - conf.Credentials = assumeRole(p.AssumeRole, p.AssumeRoleSessionName, p.ExternalID) - } else { - log.Warn("AWS Key and/or Secret not provided (falling back to ec2 instance profile)") - } - - var client *s3.S3 - sess, err := session.NewSession(conf) - if err != nil { - log.WithError(err).Errorln("could not instantiate session") - return err - } - - // If user role ARN is set then assume role here - if len(p.UserRoleArn) > 0 { - confRoleArn := aws.Config{ - Region: aws.String(p.Region), - Credentials: stscreds.NewCredentials(sess, p.UserRoleArn), - } - - client = s3.New(sess, &confRoleArn) - } else { - client = s3.New(sess) - } + client := p.createS3Client() // If in download mode, call the downloadS3Objects method if p.Download { sourceDir := normalizePath(p.Source) - client := p.createS3Client() return p.downloadS3Objects(client, sourceDir) } diff --git a/plugin_unix_test.go b/plugin_unix_test.go index 8699520..336080d 100644 --- a/plugin_unix_test.go +++ b/plugin_unix_test.go @@ -83,3 +83,48 @@ func TestNormalizePath(t *testing.T) { } } } + +func TestResolveSource(t *testing.T) { + tests := []struct { + sourceDir string + source string + stripPrefix string + expected string + }{ + // Test case 1 + { + sourceDir: "/home/user/documents", + source: "/home/user/documents/file.txt", + stripPrefix: "output-", + expected: "output-file.txt", + }, + // Test case 2 + { + sourceDir: "assets", + source: "assets/images/logo.png", + stripPrefix: "", + expected: "images/logo.png", + }, + // Test case 3 + { + sourceDir: "/var/www/html", + source: "/var/www/html/pages/index.html", + stripPrefix: "web", + expected: "webpages/index.html", + }, + // Test case 4 + { + sourceDir: "dist", + source: "dist/js/app.js", + stripPrefix: "public", + expected: "publicjs/app.js", + }, + } + + for _, tc := range tests { + result := resolveSource(tc.sourceDir, tc.source, tc.stripPrefix) + if result != tc.expected { + t.Errorf("Expected: %s, Got: %s", tc.expected, result) + } + } +} diff --git a/plugin_windows_test.go b/plugin_windows_test.go index ea1b170..f86a85e 100644 --- a/plugin_windows_test.go +++ b/plugin_windows_test.go @@ -97,3 +97,48 @@ func TestNormalizePath(t *testing.T) { } } } + +func TestResolveSource(t *testing.T) { + tests := []struct { + sourceDir string + source string + stripPrefix string + expected string + }{ + // Test case 1 + { + sourceDir: "/home/user/documents", + source: "/home/user/documents/file.txt", + stripPrefix: "output-", + expected: "output-file.txt", + }, + // Test case 2 + { + sourceDir: "assets", + source: "assets/images/logo.png", + stripPrefix: "", + expected: "images/logo.png", + }, + // Test case 3 + { + sourceDir: "/var/www/html", + source: "/var/www/html/pages/index.html", + stripPrefix: "web", + expected: "webpages/index.html", + }, + // Test case 4 + { + sourceDir: "dist", + source: "dist/js/app.js", + stripPrefix: "public", + expected: "publicjs/app.js", + }, + } + + for _, tc := range tests { + result := resolveSource(tc.sourceDir, tc.source, tc.stripPrefix) + if result != tc.expected { + t.Errorf("Expected: %s, Got: %s", tc.expected, result) + } + } +}