diff --git a/README.md b/README.md index 238462c..a901af6 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ docker: Error response from daemon: Container command Execute from the working directory: +* For upload ``` docker run --rm \ -e PLUGIN_SOURCE= \ @@ -53,3 +54,17 @@ docker run --rm \ -w $(pwd) \ plugins/s3 --dry-run ``` + +* For download +``` +docker run --rm \ + -e PLUGIN_SOURCE= \ + -e PLUGIN_BUCKET= \ + -e AWS_ACCESS_KEY_ID= \ + -e AWS_SECRET_ACCESS_KEY= \ + -e PLUGIN_REGION= \ + -e PLUGIN_DOWNLOAD="true" \ + -v $(pwd):$(pwd) \ + -w $(pwd) \ + plugins/s3 --dry-run +``` diff --git a/docker/Dockerfile.linux.arm64 b/docker/Dockerfile.linux.arm64 index 337e207..f6f8b2b 100644 --- a/docker/Dockerfile.linux.arm64 +++ b/docker/Dockerfile.linux.arm64 @@ -6,4 +6,4 @@ LABEL maintainer="Drone.IO Community " \ org.label-schema.schema-version="1.0" ADD release/linux/arm64/drone-s3 /bin/ -ENTRYPOINT ["/bin/drone-s3"] +ENTRYPOINT ["/bin/drone-s3"] \ No newline at end of file diff --git a/go.mod b/go.mod index 9b309d6..c47aa27 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,8 @@ require ( require ( github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/pkg/errors v0.9.1 github.com/russross/blackfriday/v2 v2.1.0 // indirect + golang.org/x/sync v0.6.0 golang.org/x/sys v0.1.0 // indirect ) diff --git a/go.sum b/go.sum index bcfd899..9fe4b78 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,7 @@ github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/mattn/go-zglob v0.0.4 h1:LQi2iOm0/fGgu80AioIJ/1j9w9Oh+9DZ39J4VAGzHQM= github.com/mattn/go-zglob v0.0.4/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -40,6 +41,8 @@ golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/main.go b/main.go index cf452dd..a1114f4 100644 --- a/main.go +++ b/main.go @@ -81,7 +81,7 @@ func main() { }, cli.StringFlag{ Name: "strip-prefix", - Usage: "strip the prefix from the target", + Usage: "used to add or remove a prefix from the source/target path", EnvVar: "PLUGIN_STRIP_PREFIX", }, cli.StringSliceFlag{ @@ -94,6 +94,11 @@ func main() { Usage: "server-side encryption algorithm, defaults to none", EnvVar: "PLUGIN_ENCRYPTION", }, + cli.BoolFlag{ + Name: "download", + Usage: "switch to download mode, which will fetch `source`'s files from s3 bucket", + EnvVar: "PLUGIN_DOWNLOAD", + }, cli.BoolFlag{ Name: "dry-run", Usage: "dry run for debug purposes", @@ -164,6 +169,7 @@ func run(c *cli.Context) error { Exclude: c.StringSlice("exclude"), Encryption: c.String("encryption"), ContentType: c.Generic("content-type").(*StringMapFlag).Get(), + Download: c.Bool("download"), ContentEncoding: c.Generic("content-encoding").(*StringMapFlag).Get(), CacheControl: c.Generic("cache-control").(*StringMapFlag).Get(), StorageClass: c.String("storage-class"), diff --git a/plugin.go b/plugin.go index 7b0bed6..d7b3387 100644 --- a/plugin.go +++ b/plugin.go @@ -1,6 +1,7 @@ package main import ( + "io" "mime" "os" "path/filepath" @@ -15,6 +16,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/sts" "github.com/mattn/go-zglob" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -44,6 +46,9 @@ type Plugin struct { // sa-east-1 Region string + // if true, plugin is set to download mode, which means `source` from the bucket will be downloaded + Download bool + // Indicates the files ACL, which should be one // of the following: // private @@ -97,42 +102,21 @@ type Plugin struct { // Exec runs the plugin func (p *Plugin) Exec() error { - // normalize the target URL - p.Target = strings.TrimPrefix(p.Target, "/") - - // create the client - conf := &aws.Config{ - Region: aws.String(p.Region), - Endpoint: &p.Endpoint, - DisableSSL: aws.Bool(strings.HasPrefix(p.Endpoint, "http://")), - S3ForcePathStyle: aws.Bool(p.PathStyle), - } - - if p.Key != "" && p.Secret != "" { - conf.Credentials = credentials.NewStaticCredentials(p.Key, p.Secret, "") - } else if p.AssumeRole != "" { - conf.Credentials = assumeRole(p.AssumeRole, p.AssumeRoleSessionName, p.ExternalID) + if p.Download { + p.Source = normalizePath(p.Source) + p.Target = normalizePath(p.Target) } else { - log.Warn("AWS Key and/or Secret not provided (falling back to ec2 instance profile)") + p.Target = strings.TrimPrefix(p.Target, "/") } - var client *s3.S3 - sess, err := session.NewSession(conf) - if err != nil { - log.WithError(err).Errorln("could not instantiate session") - return err - } + // create the client + client := p.createS3Client() - // If user role ARN is set then assume role here - if len(p.UserRoleArn) > 0 { - confRoleArn := aws.Config{ - Region: aws.String(p.Region), - Credentials: stscreds.NewCredentials(sess, p.UserRoleArn), - } + // If in download mode, call the downloadS3Objects method + if p.Download { + sourceDir := normalizePath(p.Source) - client = s3.New(sess, &confRoleArn) - } else { - client = s3.New(sess) + return p.downloadS3Objects(client, sourceDir) } // find the bucket @@ -322,6 +306,14 @@ func resolveKey(target, srcPath, stripPrefix string) string { return key } +func resolveSource(sourceDir, source, stripPrefix string) string { + // Remove the leading sourceDir from the source path + path := strings.TrimPrefix(strings.TrimPrefix(source, sourceDir), "/") + + // Add the specified stripPrefix to the resulting path + return stripPrefix + path +} + // checks if the source path is a dir func isDir(source string, matches []string) bool { stat, err := os.Stat(source) @@ -342,3 +334,128 @@ func isDir(source string, matches []string) bool { } return false } + +// normalizePath converts the path to a forward slash format and trims the prefix. +func normalizePath(path string) string { + return strings.TrimPrefix(filepath.ToSlash(path), "/") +} + +// downloadS3Object downloads a single object from S3 +func (p *Plugin) downloadS3Object(client *s3.S3, sourceDir, key, target string) error { + log.WithFields(log.Fields{ + "bucket": p.Bucket, + "key": key, + }).Info("Getting S3 object") + + obj, err := client.GetObject(&s3.GetObjectInput{ + Bucket: &p.Bucket, + Key: &key, + }) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "bucket": p.Bucket, + "key": key, + }).Error("Cannot get S3 object") + return err + } + defer obj.Body.Close() + + // Create the destination file path + destination := filepath.Join(p.Target, target) + log.Println("Destination: ", destination) + + // Extract the directory from the destination path + dir := filepath.Dir(destination) + + // Create the directory and any necessary parent directories + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return errors.Wrap(err, "error creating directories") + } + + f, err := os.Create(destination) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": destination, + }).Error("Failed to create file") + return err + } + defer f.Close() + + _, err = io.Copy(f, obj.Body) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": destination, + }).Error("Failed to write file") + return err + } + + return nil +} + +// downloadS3Objects downloads all objects in the specified S3 bucket path +func (p *Plugin) downloadS3Objects(client *s3.S3, sourceDir string) error { + log.WithFields(log.Fields{ + "bucket": p.Bucket, + "dir": sourceDir, + }).Info("Listing S3 directory") + + list, err := client.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: &p.Bucket, + Prefix: &sourceDir, + }) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "bucket": p.Bucket, + "dir": sourceDir, + }).Error("Cannot list S3 directory") + return err + } + + for _, item := range list.Contents { + // resolveSource takes a source directory, a source path, and a prefix to strip, + // and returns a resolved target path by removing the sourceDir from the source + // and appending the stripPrefix. + target := resolveSource(sourceDir, *item.Key, p.StripPrefix) + + if err := p.downloadS3Object(client, sourceDir, *item.Key, target); err != nil { + return err + } + } + + return nil +} + +// createS3Client creates and returns an S3 client based on the plugin configuration +func (p *Plugin) createS3Client() *s3.S3 { + conf := &aws.Config{ + Region: aws.String(p.Region), + Endpoint: &p.Endpoint, + DisableSSL: aws.Bool(strings.HasPrefix(p.Endpoint, "http://")), + S3ForcePathStyle: aws.Bool(p.PathStyle), + } + + if p.Key != "" && p.Secret != "" { + conf.Credentials = credentials.NewStaticCredentials(p.Key, p.Secret, "") + } else if p.AssumeRole != "" { + conf.Credentials = assumeRole(p.AssumeRole, p.AssumeRoleSessionName, p.ExternalID) + } else { + log.Warn("AWS Key and/or Secret not provided (falling back to ec2 instance profile)") + } + + sess, _ := session.NewSession(conf) + client := s3.New(sess) + + if len(p.UserRoleArn) > 0 { + confRoleArn := aws.Config{ + Region: aws.String(p.Region), + Credentials: stscreds.NewCredentials(sess, p.UserRoleArn), + } + client = s3.New(sess, &confRoleArn) + } + + return client +} diff --git a/plugin_unix_test.go b/plugin_unix_test.go index 517b1f4..336080d 100644 --- a/plugin_unix_test.go +++ b/plugin_unix_test.go @@ -44,3 +44,87 @@ func TestResolveUnixKey(t *testing.T) { } } } + +func TestNormalizePath(t *testing.T) { + tests := []struct { + input string + expected string + }{ + { + input: "/path/to/file.txt", + expected: "path/to/file.txt", + }, + { + input: "C:\\Users\\username\\Documents\\file.doc", + expected: "C:\\Users\\username\\Documents\\file.doc", + }, + { + input: "relative/path/to/file", + expected: "relative/path/to/file", + }, + { + input: "file.txt", + expected: "file.txt", + }, + { + input: "/root/directory/", + expected: "root/directory/", + }, + { + input: "no_slash", + expected: "no_slash", + }, + } + + for _, tc := range tests { + result := normalizePath(tc.input) + if result != tc.expected { + t.Errorf("Expected: %s, Got: %s", tc.expected, result) + } + } +} + +func TestResolveSource(t *testing.T) { + tests := []struct { + sourceDir string + source string + stripPrefix string + expected string + }{ + // Test case 1 + { + sourceDir: "/home/user/documents", + source: "/home/user/documents/file.txt", + stripPrefix: "output-", + expected: "output-file.txt", + }, + // Test case 2 + { + sourceDir: "assets", + source: "assets/images/logo.png", + stripPrefix: "", + expected: "images/logo.png", + }, + // Test case 3 + { + sourceDir: "/var/www/html", + source: "/var/www/html/pages/index.html", + stripPrefix: "web", + expected: "webpages/index.html", + }, + // Test case 4 + { + sourceDir: "dist", + source: "dist/js/app.js", + stripPrefix: "public", + expected: "publicjs/app.js", + }, + } + + for _, tc := range tests { + result := resolveSource(tc.sourceDir, tc.source, tc.stripPrefix) + if result != tc.expected { + t.Errorf("Expected: %s, Got: %s", tc.expected, result) + } + } +} diff --git a/plugin_windows_test.go b/plugin_windows_test.go index 27379d6..f86a85e 100644 --- a/plugin_windows_test.go +++ b/plugin_windows_test.go @@ -58,3 +58,87 @@ func TestResolveWinKey(t *testing.T) { } } } + +func TestNormalizePath(t *testing.T) { + tests := []struct { + input string + expected string + }{ + { + input: "/path/to/file.txt", + expected: "path/to/file.txt", + }, + { + input: "C:\\Users\\username\\Documents\\file.doc", + expected: "C:\\Users\\username\\Documents\\file.doc", + }, + { + input: "relative/path/to/file", + expected: "relative/path/to/file", + }, + { + input: "file.txt", + expected: "file.txt", + }, + { + input: "/root/directory/", + expected: "root/directory/", + }, + { + input: "no_slash", + expected: "no_slash", + }, + } + + for _, tc := range tests { + result := normalizePath(tc.input) + if result != tc.expected { + t.Errorf("Expected: %s, Got: %s", tc.expected, result) + } + } +} + +func TestResolveSource(t *testing.T) { + tests := []struct { + sourceDir string + source string + stripPrefix string + expected string + }{ + // Test case 1 + { + sourceDir: "/home/user/documents", + source: "/home/user/documents/file.txt", + stripPrefix: "output-", + expected: "output-file.txt", + }, + // Test case 2 + { + sourceDir: "assets", + source: "assets/images/logo.png", + stripPrefix: "", + expected: "images/logo.png", + }, + // Test case 3 + { + sourceDir: "/var/www/html", + source: "/var/www/html/pages/index.html", + stripPrefix: "web", + expected: "webpages/index.html", + }, + // Test case 4 + { + sourceDir: "dist", + source: "dist/js/app.js", + stripPrefix: "public", + expected: "publicjs/app.js", + }, + } + + for _, tc := range tests { + result := resolveSource(tc.sourceDir, tc.source, tc.stripPrefix) + if result != tc.expected { + t.Errorf("Expected: %s, Got: %s", tc.expected, result) + } + } +}