command/cp: KMS encryption support (#185)

Resolves #18

CHANGELOG.md

@@ -7,11 +7,14 @@
 - Dropped storage class short codes display from default behaviour of `ls` operation. Instead, use `-s` flag with `ls`
 to see full names of the storage classes when listing objects.
 
+
 #### Features
+- Added Server-side Encryption (SSE) support for mv/cp operations. It uses customer master keys (CMKs) managed by AWS Key Management Service. ([#18](https://github.com/peak/s5cmd/issues/18))
 - Added an option to show full form of [storage class](https://aws.amazon.com/s3/storage-classes/) when listing objects. ([#165](https://github.com/peak/s5cmd/issues/165))
 - Add [access control lists (ACLs)](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html)
 support to enable managing access to buckets and objects. ([#26](https://github.com/peak/s5cmd/issues/26))
 
+
 #### Bugfixes
 
 - Fixed infinite repetition issue on mv/cp operations which would occur

README.md

@@ -12,6 +12,8 @@ storage services and local filesystems.
 - List buckets and objects
 - Upload, download or delete objects
 - Move, copy or rename objects
+- Set Server Side Encryption using AWS Key Management Service (KMS)
+- Set Access Control List (ACL) for objects/files on the upload, copy, move. 
 - Print object contents to stdout
 - Create buckets
 - Summarize objects sizes, grouping by storage class
@@ -117,7 +119,15 @@ $ tree
 #### Upload a file to S3
 
     s5cmd cp object.gz s3://bucket/
-
+
+ by setting server side encryption (*aws kms*) of the file:
+
+    s5cmd cp -sse aws:kms -sse-kms-key-id <your-kms-key-id> object.gz s3://bucket/
+
+ by setting Access Control List (*acl*) policy of the object:
+
+    s5cmd cp -acl bucket-owner-full-control object.gz s3://bucket/
+
 #### Upload multiple files to S3
 
     s5cmd cp directory/ s3://bucket/

command/cp.go

@@ -64,6 +64,12 @@ Examples:
 
 	10. Mirror an S3 prefix to target S3 prefix
 		 > s5cmd {{.HelpName}} -n -s -u s3://bucket/source-prefix/* s3://bucket/target-prefix/
+
+	11. Perform KMS Server Side Encryption of the object(s) at the destination
+		> s5cmd {{.HelpName}} -sse aws:kms s3://bucket/object s3://target-bucket/prefix/object
+
+	12. Perform KMS-SSE of the object(s) at the destination using customer managed Customer Master Key (CMK) key id
+		> s5cmd {{.HelpName}} -sse aws:kms -sse-kms-key-id <your-kms-key-id> s3://bucket/object s3://target-bucket/prefix/object
 `
 
 var copyCommandFlags = []cli.Flag{
@@ -107,6 +113,14 @@ var copyCommandFlags = []cli.Flag{
 		Value:   defaultPartSize,
 		Usage:   "size of each part transferred between host and remote server, in MiB",
 	},
+	&cli.StringFlag{
+		Name:  "sse",
+		Usage: "perform server side encryption of the data at its destination, e.g. aws:kms",
+	},
+	&cli.StringFlag{
+		Name:  "sse-kms-key-id",
+		Usage: "customer master key (CMK) id for SSE-KMS encryption; leave it out if server-side generated key is desired",
+	},
 	&cli.StringFlag{
 		Name:  "acl",
 		Usage: "set acl for target: defines granted accesses and their types on different accounts/groups",
@@ -130,15 +144,17 @@ var copyCommand = &cli.Command{
 			fullCommand:  givenCommand(c),
 			deleteSource: false, // don't delete source
 			// flags
-			noClobber:      c.Bool("no-clobber"),
-			ifSizeDiffer:   c.Bool("if-size-differ"),
-			ifSourceNewer:  c.Bool("if-source-newer"),
-			flatten:        c.Bool("flatten"),
-			followSymlinks: !c.Bool("no-follow-symlinks"),
-			storageClass:   storage.StorageClass(c.String("storage-class")),
-			concurrency:    c.Int("concurrency"),
-			partSize:       c.Int64("part-size") * megabytes,
-			acl:            c.String("acl"),
+			noClobber:        c.Bool("no-clobber"),
+			ifSizeDiffer:     c.Bool("if-size-differ"),
+			ifSourceNewer:    c.Bool("if-source-newer"),
+			flatten:          c.Bool("flatten"),
+			followSymlinks:   !c.Bool("no-follow-symlinks"),
+			storageClass:     storage.StorageClass(c.String("storage-class")),
+			concurrency:      c.Int("concurrency"),
+			partSize:         c.Int64("part-size") * megabytes,
+			encryptionMethod: c.String("sse"),
+			encryptionKeyID:  c.String("sse-kms-key-id"),
+			acl:              c.String("acl"),
 		}.Run(c.Context)
 	},
 }
@@ -153,13 +169,15 @@ type Copy struct {
 	deleteSource bool
 
 	// flags
-	noClobber      bool
-	ifSizeDiffer   bool
-	ifSourceNewer  bool
-	flatten        bool
-	followSymlinks bool
-	storageClass   storage.StorageClass
-	acl            string
+	noClobber        bool
+	ifSizeDiffer     bool
+	ifSourceNewer    bool
+	flatten          bool
+	followSymlinks   bool
+	storageClass     storage.StorageClass
+	encryptionMethod string
+	encryptionKeyID  string
+	acl              string
 
 	// s3 options
 	concurrency int
@@ -386,8 +404,10 @@ func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL) er
 	dstClient := storage.NewClient(dsturl)
 
 	metadata := storage.NewMetadata().
-		SetStorageClass(string(c.storageClass)).
 		SetContentType(guessContentType(f)).
+		SetStorageClass(string(c.storageClass)).
+		SetSSE(c.encryptionMethod).
+		SetSSEKeyID(c.encryptionKeyID).
 		SetACL(c.acl)
 
 	err = dstClient.Put(ctx, f, dsturl, metadata, c.concurrency, c.partSize)
@@ -427,6 +447,8 @@ func (c Copy) doCopy(ctx context.Context, srcurl *url.URL, dsturl *url.URL) erro
 
 	metadata := storage.NewMetadata().
 		SetStorageClass(string(c.storageClass)).
+		SetSSE(c.encryptionMethod).
+		SetSSEKeyID(c.encryptionKeyID).
 		SetACL(c.acl)
 
 	err := c.shouldOverride(ctx, srcurl, dsturl)

command/mv.go

@@ -48,12 +48,14 @@ var moveCommand = &cli.Command{
 			fullCommand:  givenCommand(c),
 			deleteSource: true, // delete source
 			// flags
-			noClobber:     c.Bool("no-clobber"),
-			ifSizeDiffer:  c.Bool("if-size-differ"),
-			ifSourceNewer: c.Bool("if-source-newer"),
-			flatten:       c.Bool("flatten"),
-			storageClass:  storage.StorageClass(c.String("storage-class")),
-			acl:           c.String("acl"),
+			noClobber:        c.Bool("no-clobber"),
+			ifSizeDiffer:     c.Bool("if-size-differ"),
+			ifSourceNewer:    c.Bool("if-source-newer"),
+			flatten:          c.Bool("flatten"),
+			storageClass:     storage.StorageClass(c.String("storage-class")),
+			encryptionMethod: c.String("sse"),
+			encryptionKeyID:  c.String("sse-kms-key-id"),
+			acl:              c.String("acl"),
 		}
 
 		return copyCommand.Run(c.Context)

storage/s3.go

@@ -326,11 +326,21 @@ func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata Metadata) err
 		Key:        aws.String(to.Path),
 		CopySource: aws.String(copySource),
 	}
+
 	storageClass := metadata.StorageClass()
 	if storageClass != "" {
 		input.StorageClass = aws.String(storageClass)
 	}
 
+	sseEncryption := metadata.SSE()
+	if sseEncryption != "" {
+		input.ServerSideEncryption = aws.String(sseEncryption)
+		sseKmsKeyID := metadata.SSEKeyID()
+		if sseKmsKeyID != "" {
+			input.SSEKMSKeyId = aws.String(sseKmsKeyID)
+		}
+	}
+
 	acl := metadata.ACL()
 	if acl != "" {
 		input.ACL = aws.String(acl)
@@ -403,6 +413,15 @@ func (s *S3) Put(
 		input.ACL = aws.String(acl)
 	}
 
+	sseEncryption := metadata.SSE()
+	if sseEncryption != "" {
+		input.ServerSideEncryption = aws.String(sseEncryption)
+		sseKmsKeyID := metadata.SSEKeyID()
+		if sseKmsKeyID != "" {
+			input.SSEKMSKeyId = aws.String(sseKmsKeyID)
+		}
+	}
+
 	_, err := s.uploader.UploadWithContext(ctx, input, func(u *s3manager.Uploader) {
 		u.PartSize = partSize
 		u.Concurrency = concurrency

storage/s3_test.go

@@ -430,22 +430,45 @@ func val(i interface{}, s string) interface{} {
 	return v[0]
 }
 
-func TestS3AclFlagOnCopy(t *testing.T) {
+func TestS3CopyEncryptionRequest(t *testing.T) {
 	testcases := []struct {
-		name string
-		acl  string
-
-		expectedAcl string
+		name     string
+		sse      string
+		sseKeyID string
+		acl      string
+
+		expectedSSE      string
+		expectedSSEKeyID string
+		expectedAcl      string
 	}{
 		{
-			name: "no acl flag",
+			name: "no encryption/no acl, by default",
+		},
+		{
+			name: "aws:kms encryption with server side generated keys",
+			sse:  "aws:kms",
+
+			expectedSSE: "aws:kms",
+		},
+		{
+			name:     "aws:kms encryption with user provided key",
+			sse:      "aws:kms",
+			sseKeyID: "sdkjn12SDdci#@#EFRFERTqW/ke",
+
+			expectedSSE:      "aws:kms",
+			expectedSSEKeyID: "sdkjn12SDdci#@#EFRFERTqW/ke",
+		},
+		{
+			name:     "provide key without encryption flag, shall be ignored",
+			sseKeyID: "1234567890",
 		},
 		{
 			name:        "acl flag with a value",
 			acl:         "bucket-owner-full-control",
 			expectedAcl: "bucket-owner-full-control",
 		},
 	}
+
 	u, err := url.New("s3://bucket/key")
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
@@ -468,20 +491,30 @@ func TestS3AclFlagOnCopy(t *testing.T) {
 					Body:       ioutil.NopCloser(strings.NewReader("")),
 				}
 
+				params := r.Params
+				sse := val(params, "ServerSideEncryption")
+				key := val(params, "SSEKMSKeyId")
+
+				if !(sse == nil && tc.expectedSSE == "") {
+					assert.Equal(t, sse, tc.expectedSSE)
+				}
+				if !(key == nil && tc.expectedSSEKeyID == "") {
+					assert.Equal(t, key, tc.expectedSSEKeyID)
+				}
+
 				aclVal := val(r.Params, "ACL")
 
 				if aclVal == nil && tc.expectedAcl == "" {
 					return
 				}
 				assert.Equal(t, aclVal, tc.expectedAcl)
-
 			})
 
 			mockS3 := &S3{
 				api: mockApi,
 			}
 
-			metadata := NewMetadata().SetACL(tc.acl)
+			metadata := NewMetadata().SetSSE(tc.sse).SetSSEKeyID(tc.sseKeyID).SetACL(tc.acl)
 
 			err = mockS3.Copy(context.Background(), u, u, metadata)
 
@@ -492,15 +525,36 @@ func TestS3AclFlagOnCopy(t *testing.T) {
 	}
 }
 
-func TestS3AclFlagOnPut(t *testing.T) {
+func TestS3PutEncryptionRequest(t *testing.T) {
 	testcases := []struct {
-		name string
-		acl  string
-
-		expectedAcl string
+		name     string
+		sse      string
+		sseKeyID string
+		acl      string
+
+		expectedSSE      string
+		expectedSSEKeyID string
+		expectedAcl      string
 	}{
 		{
-			name: "no acl flag",
+			name: "no encryption, no acl flag",
+		},
+		{
+			name:        "aws:kms encryption with server side generated keys",
+			sse:         "aws:kms",
+			expectedSSE: "aws:kms",
+		},
+		{
+			name:     "aws:kms encryption with user provided key",
+			sse:      "aws:kms",
+			sseKeyID: "sdkjn12SDdci#@#EFRFERTqW/ke",
+
+			expectedSSE:      "aws:kms",
+			expectedSSEKeyID: "sdkjn12SDdci#@#EFRFERTqW/ke",
+		},
+		{
+			name:     "provide key without encryption flag, shall be ignored",
+			sseKeyID: "1234567890",
 		},
 		{
 			name:        "acl flag with a value",
@@ -530,6 +584,17 @@ func TestS3AclFlagOnPut(t *testing.T) {
 					Body:       ioutil.NopCloser(strings.NewReader("")),
 				}
 
+				params := r.Params
+				sse := val(params, "ServerSideEncryption")
+				key := val(params, "SSEKMSKeyId")
+
+				if !(sse == nil && tc.expectedSSE == "") {
+					assert.Equal(t, sse, tc.expectedSSE)
+				}
+				if !(key == nil && tc.expectedSSEKeyID == "") {
+					assert.Equal(t, key, tc.expectedSSEKeyID)
+				}
+
 				aclVal := val(r.Params, "ACL")
 
 				if aclVal == nil && tc.expectedAcl == "" {
@@ -542,7 +607,7 @@ func TestS3AclFlagOnPut(t *testing.T) {
 				uploader: s3manager.NewUploaderWithClient(mockApi),
 			}
 
-			metadata := NewMetadata().SetACL(tc.acl)
+			metadata := NewMetadata().SetSSE(tc.sse).SetSSEKeyID(tc.sseKeyID).SetACL(tc.acl)
 
 			err = mockS3.Put(context.Background(), bytes.NewReader([]byte("")), u, metadata, 1, 5242880)