Merge pull request #136 from shiv-am0/dev-s3-artifacts-download

Download artifacts functionality added

README.md

@@ -42,6 +42,7 @@ docker: Error response from daemon: Container command
 
 Execute from the working directory:
 
+* For upload
 ```
 docker run --rm \
   -e PLUGIN_SOURCE=<source> \
@@ -53,3 +54,17 @@ docker run --rm \
   -w $(pwd) \
   plugins/s3 --dry-run
 ```
+
+* For download
+```
+docker run --rm \
+  -e PLUGIN_SOURCE=<source directory to be downloaded from bucket> \
+  -e PLUGIN_BUCKET=<bucket> \
+  -e AWS_ACCESS_KEY_ID=<token> \
+  -e AWS_SECRET_ACCESS_KEY=<secret> \
+  -e PLUGIN_REGION=<region where the bucket is deployed> \
+  -e PLUGIN_DOWNLOAD="true" \
+  -v $(pwd):$(pwd) \
+  -w $(pwd) \
+  plugins/s3 --dry-run
+```

docker/Dockerfile.linux.arm64

@@ -6,4 +6,4 @@ LABEL maintainer="Drone.IO Community <[email protected]>" \
   org.label-schema.schema-version="1.0"
 
 ADD release/linux/arm64/drone-s3 /bin/
-ENTRYPOINT ["/bin/drone-s3"]
+ENTRYPOINT ["/bin/drone-s3"]

go.mod

@@ -13,6 +13,8 @@ require (
 require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/pkg/errors v0.9.1
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	golang.org/x/sync v0.6.0
 	golang.org/x/sys v0.1.0 // indirect
 )

go.sum

@@ -15,6 +15,7 @@ github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
 github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/mattn/go-zglob v0.0.4 h1:LQi2iOm0/fGgu80AioIJ/1j9w9Oh+9DZ39J4VAGzHQM=
 github.com/mattn/go-zglob v0.0.4/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -40,6 +41,8 @@ golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

main.go

@@ -81,7 +81,7 @@ func main() {
 		},
 		cli.StringFlag{
 			Name:   "strip-prefix",
-			Usage:  "strip the prefix from the target",
+			Usage:  "used to add or remove a prefix from the source/target path",
 			EnvVar: "PLUGIN_STRIP_PREFIX",
 		},
 		cli.StringSliceFlag{
@@ -94,6 +94,11 @@ func main() {
 			Usage:  "server-side encryption algorithm, defaults to none",
 			EnvVar: "PLUGIN_ENCRYPTION",
 		},
+		cli.BoolFlag{
+			Name:   "download",
+			Usage:  "switch to download mode, which will fetch `source`'s files from s3 bucket",
+			EnvVar: "PLUGIN_DOWNLOAD",
+		},
 		cli.BoolFlag{
 			Name:   "dry-run",
 			Usage:  "dry run for debug purposes",
@@ -164,6 +169,7 @@ func run(c *cli.Context) error {
 		Exclude:               c.StringSlice("exclude"),
 		Encryption:            c.String("encryption"),
 		ContentType:           c.Generic("content-type").(*StringMapFlag).Get(),
+		Download:              c.Bool("download"),
 		ContentEncoding:       c.Generic("content-encoding").(*StringMapFlag).Get(),
 		CacheControl:          c.Generic("cache-control").(*StringMapFlag).Get(),
 		StorageClass:          c.String("storage-class"),

plugin.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"io"
 	"mime"
 	"os"
 	"path/filepath"
@@ -15,6 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/sts"
 	"github.com/mattn/go-zglob"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -44,6 +46,9 @@ type Plugin struct {
 	// sa-east-1
 	Region string
 
+	// if true, plugin is set to download mode, which means `source` from the bucket will be downloaded
+	Download bool
+
 	// Indicates the files ACL, which should be one
 	// of the following:
 	//     private
@@ -97,42 +102,21 @@ type Plugin struct {
 
 // Exec runs the plugin
 func (p *Plugin) Exec() error {
-	// normalize the target URL
-	p.Target = strings.TrimPrefix(p.Target, "/")
-
-	// create the client
-	conf := &aws.Config{
-		Region:           aws.String(p.Region),
-		Endpoint:         &p.Endpoint,
-		DisableSSL:       aws.Bool(strings.HasPrefix(p.Endpoint, "http://")),
-		S3ForcePathStyle: aws.Bool(p.PathStyle),
-	}
-
-	if p.Key != "" && p.Secret != "" {
-		conf.Credentials = credentials.NewStaticCredentials(p.Key, p.Secret, "")
-	} else if p.AssumeRole != "" {
-		conf.Credentials = assumeRole(p.AssumeRole, p.AssumeRoleSessionName, p.ExternalID)
+	if p.Download {
+		p.Source = normalizePath(p.Source)
+		p.Target = normalizePath(p.Target)
 	} else {
-		log.Warn("AWS Key and/or Secret not provided (falling back to ec2 instance profile)")
+		p.Target = strings.TrimPrefix(p.Target, "/")
 	}
 
-	var client *s3.S3
-	sess, err := session.NewSession(conf)
-	if err != nil {
-		log.WithError(err).Errorln("could not instantiate session")
-		return err
-	}
+	// create the client
+	client := p.createS3Client()
 
-	// If user role ARN is set then assume role here
-	if len(p.UserRoleArn) > 0 {
-		confRoleArn := aws.Config{
-			Region:      aws.String(p.Region),
-			Credentials: stscreds.NewCredentials(sess, p.UserRoleArn),
-		}
+	// If in download mode, call the downloadS3Objects method
+	if p.Download {
+		sourceDir := normalizePath(p.Source)
 
-		client = s3.New(sess, &confRoleArn)
-	} else {
-		client = s3.New(sess)
+		return p.downloadS3Objects(client, sourceDir)
 	}
 
 	// find the bucket
@@ -322,6 +306,14 @@ func resolveKey(target, srcPath, stripPrefix string) string {
 	return key
 }
 
+func resolveSource(sourceDir, source, stripPrefix string) string {
+	// Remove the leading sourceDir from the source path
+	path := strings.TrimPrefix(strings.TrimPrefix(source, sourceDir), "/")
+
+	// Add the specified stripPrefix to the resulting path
+	return stripPrefix + path
+}
+
 // checks if the source path is a dir
 func isDir(source string, matches []string) bool {
 	stat, err := os.Stat(source)
@@ -342,3 +334,128 @@ func isDir(source string, matches []string) bool {
 	}
 	return false
 }
+
+// normalizePath converts the path to a forward slash format and trims the prefix.
+func normalizePath(path string) string {
+	return strings.TrimPrefix(filepath.ToSlash(path), "/")
+}
+
+// downloadS3Object downloads a single object from S3
+func (p *Plugin) downloadS3Object(client *s3.S3, sourceDir, key, target string) error {
+	log.WithFields(log.Fields{
+		"bucket": p.Bucket,
+		"key":    key,
+	}).Info("Getting S3 object")
+
+	obj, err := client.GetObject(&s3.GetObjectInput{
+		Bucket: &p.Bucket,
+		Key:    &key,
+	})
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":  err,
+			"bucket": p.Bucket,
+			"key":    key,
+		}).Error("Cannot get S3 object")
+		return err
+	}
+	defer obj.Body.Close()
+
+	// Create the destination file path
+	destination := filepath.Join(p.Target, target)
+	log.Println("Destination: ", destination)
+
+	// Extract the directory from the destination path
+	dir := filepath.Dir(destination)
+
+	// Create the directory and any necessary parent directories
+	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+		return errors.Wrap(err, "error creating directories")
+	}
+
+	f, err := os.Create(destination)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+			"file":  destination,
+		}).Error("Failed to create file")
+		return err
+	}
+	defer f.Close()
+
+	_, err = io.Copy(f, obj.Body)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+			"file":  destination,
+		}).Error("Failed to write file")
+		return err
+	}
+
+	return nil
+}
+
+// downloadS3Objects downloads all objects in the specified S3 bucket path
+func (p *Plugin) downloadS3Objects(client *s3.S3, sourceDir string) error {
+	log.WithFields(log.Fields{
+		"bucket": p.Bucket,
+		"dir":    sourceDir,
+	}).Info("Listing S3 directory")
+
+	list, err := client.ListObjectsV2(&s3.ListObjectsV2Input{
+		Bucket: &p.Bucket,
+		Prefix: &sourceDir,
+	})
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":  err,
+			"bucket": p.Bucket,
+			"dir":    sourceDir,
+		}).Error("Cannot list S3 directory")
+		return err
+	}
+
+	for _, item := range list.Contents {
+		// resolveSource takes a source directory, a source path, and a prefix to strip,
+		// and returns a resolved target path by removing the sourceDir from the source
+		// and appending the stripPrefix.
+		target := resolveSource(sourceDir, *item.Key, p.StripPrefix)
+
+		if err := p.downloadS3Object(client, sourceDir, *item.Key, target); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// createS3Client creates and returns an S3 client based on the plugin configuration
+func (p *Plugin) createS3Client() *s3.S3 {
+	conf := &aws.Config{
+		Region:           aws.String(p.Region),
+		Endpoint:         &p.Endpoint,
+		DisableSSL:       aws.Bool(strings.HasPrefix(p.Endpoint, "http://")),
+		S3ForcePathStyle: aws.Bool(p.PathStyle),
+	}
+
+	if p.Key != "" && p.Secret != "" {
+		conf.Credentials = credentials.NewStaticCredentials(p.Key, p.Secret, "")
+	} else if p.AssumeRole != "" {
+		conf.Credentials = assumeRole(p.AssumeRole, p.AssumeRoleSessionName, p.ExternalID)
+	} else {
+		log.Warn("AWS Key and/or Secret not provided (falling back to ec2 instance profile)")
+	}
+
+	sess, _ := session.NewSession(conf)
+	client := s3.New(sess)
+
+	if len(p.UserRoleArn) > 0 {
+		confRoleArn := aws.Config{
+			Region:      aws.String(p.Region),
+			Credentials: stscreds.NewCredentials(sess, p.UserRoleArn),
+		}
+		client = s3.New(sess, &confRoleArn)
+	}
+
+	return client
+}

plugin_unix_test.go

@@ -44,3 +44,87 @@ func TestResolveUnixKey(t *testing.T) {
 		}
 	}
 }
+
+func TestNormalizePath(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "/path/to/file.txt",
+			expected: "path/to/file.txt",
+		},
+		{
+			input:    "C:\\Users\\username\\Documents\\file.doc",
+			expected: "C:\\Users\\username\\Documents\\file.doc",
+		},
+		{
+			input:    "relative/path/to/file",
+			expected: "relative/path/to/file",
+		},
+		{
+			input:    "file.txt",
+			expected: "file.txt",
+		},
+		{
+			input:    "/root/directory/",
+			expected: "root/directory/",
+		},
+		{
+			input:    "no_slash",
+			expected: "no_slash",
+		},
+	}
+
+	for _, tc := range tests {
+		result := normalizePath(tc.input)
+		if result != tc.expected {
+			t.Errorf("Expected: %s, Got: %s", tc.expected, result)
+		}
+	}
+}
+
+func TestResolveSource(t *testing.T) {
+	tests := []struct {
+		sourceDir   string
+		source      string
+		stripPrefix string
+		expected    string
+	}{
+		// Test case 1
+		{
+			sourceDir:   "/home/user/documents",
+			source:      "/home/user/documents/file.txt",
+			stripPrefix: "output-",
+			expected:    "output-file.txt",
+		},
+		// Test case 2
+		{
+			sourceDir:   "assets",
+			source:      "assets/images/logo.png",
+			stripPrefix: "",
+			expected:    "images/logo.png",
+		},
+		// Test case 3
+		{
+			sourceDir:   "/var/www/html",
+			source:      "/var/www/html/pages/index.html",
+			stripPrefix: "web",
+			expected:    "webpages/index.html",
+		},
+		// Test case 4
+		{
+			sourceDir:   "dist",
+			source:      "dist/js/app.js",
+			stripPrefix: "public",
+			expected:    "publicjs/app.js",
+		},
+	}
+
+	for _, tc := range tests {
+		result := resolveSource(tc.sourceDir, tc.source, tc.stripPrefix)
+		if result != tc.expected {
+			t.Errorf("Expected: %s, Got: %s", tc.expected, result)
+		}
+	}
+}