storage: allow custom file encryption (#1364)

* storage: allow custom file encryption

* npm publish woes.

lib/common/service-object.js

@@ -20,6 +20,7 @@
 
 'use strict';
 
+var arrify = require('arrify');
 var exec = require('methmeth');
 var extend = require('extend');
 var is = require('is');
@@ -301,12 +302,20 @@ ServiceObject.prototype.setMetadata = function(metadata, callback) {
  * @param {function} callback - The callback function passed to `request`.
  */
 ServiceObject.prototype.request = function(reqOpts, callback) {
+  reqOpts = extend(true, {}, reqOpts);
+
+  var isAbsoluteUrl = reqOpts.uri.indexOf('http') === 0;
+
   var uriComponents = [
     this.baseUrl,
     this.id,
     reqOpts.uri
   ];
 
+  if (isAbsoluteUrl) {
+    uriComponents.splice(0, uriComponents.indexOf(reqOpts.uri));
+  }
+
   reqOpts.uri = uriComponents
     .filter(exec('trim')) // Limit to non-empty strings.
     .map(function(uriComponent) {
@@ -315,7 +324,10 @@ ServiceObject.prototype.request = function(reqOpts, callback) {
     })
     .join('/');
 
-  reqOpts.interceptors_ = [].slice.call(this.interceptors);
+  var childInterceptors = arrify(reqOpts.interceptors_);
+  var localInterceptors = [].slice.call(this.interceptors);
+
+  reqOpts.interceptors_ = childInterceptors.concat(localInterceptors);
 
   return this.parent.request(reqOpts, callback);
 };

lib/common/service.js

@@ -71,6 +71,10 @@ function Service(config, options) {
  * @param {function} callback - The callback function passed to `request`.
  */
 Service.prototype.request = function(reqOpts, callback) {
+  reqOpts = extend(true, {}, reqOpts);
+
+  var isAbsoluteUrl = reqOpts.uri.indexOf('http') === 0;
+
   var uriComponents = [
     this.baseUrl
   ];
@@ -82,6 +86,10 @@ Service.prototype.request = function(reqOpts, callback) {
 
   uriComponents.push(reqOpts.uri);
 
+  if (isAbsoluteUrl) {
+    uriComponents.splice(0, uriComponents.indexOf(reqOpts.uri));
+  }
+
   reqOpts.uri = uriComponents
     .map(function(uriComponent) {
       var trimSlashesRegex = /^\/*|\/*$/g;

lib/storage/bucket.js

@@ -548,6 +548,8 @@ Bucket.prototype.deleteFiles = function(query, callback) {
  * @param {object=} options - Configuration options.
  * @param {string|number} options.generation - Only use a specific revision of
  *     this file.
+ * @param {string} options.key - A custom encryption key. See
+ *     [Customer-supplied Encryption Keys](https://cloud.google.com/storage/docs/encryption#customer-supplied).
  * @return {module:storage/file}
  *
  * @example
@@ -942,6 +944,8 @@ Bucket.prototype.makePublic = function(options, callback) {
  *     bucket using the name of the local file.
  * @param {boolean} options.gzip - Automatically gzip the file. This will set
  *     `options.metadata.contentEncoding` to `gzip`.
+ * @param {string} options.key - A custom encryption key. See
+ *     [Customer-supplied Encryption Keys](https://cloud.google.com/storage/docs/encryption#customer-supplied).
  * @param {object} options.metadata - See an
  *     [Objects: insert request body](https://cloud.google.com/storage/docs/json_api/v1/objects/insert#request_properties_JSON).
  * @param {string} options.offset - The starting byte of the upload stream, for
@@ -972,17 +976,17 @@ Bucket.prototype.makePublic = function(options, callback) {
  *     `options.predefinedAcl = 'publicRead'`)
  * @param {boolean} options.resumable - Force a resumable upload. (default:
  *     true for files larger than 5 MB).
- * @param {function} callback - The callback function.
- * @param {?error} callback.err - An error returned while making this request
- * @param {module:storage/file} callback.file - The uploaded File.
- * @param {object} callback.apiResponse - The full API response.
  * @param {string} options.uri - The URI for an already-created resumable
  *     upload. See {module:storage/file#createResumableUpload}.
  * @param {string|boolean} options.validation - Possible values: `"md5"`,
  *     `"crc32c"`, or `false`. By default, data integrity is validated with an
  *     MD5 checksum for maximum reliability. CRC32c will provide better
  *     performance with less reliability. You may also choose to skip validation
  *     completely, however this is **not recommended**.
+ * @param {function} callback - The callback function.
+ * @param {?error} callback.err - An error returned while making this request
+ * @param {module:storage/file} callback.file - The uploaded File.
+ * @param {object} callback.apiResponse - The full API response.
  *
  * @example
  * //-
@@ -1044,6 +1048,32 @@ Bucket.prototype.makePublic = function(options, callback) {
  *   // Note:
  *   // The `newFile` parameter is equal to `file`.
  * });
+ *
+ * //-
+ * // To use [Customer-supplied Encryption Keys](https://cloud.google.com/storage/docs/encryption#customer-supplied),
+ * // provide the `key` option.
+ * //-
+ * var crypto = require('crypto');
+ * var encryptionKey = crypto.randomBytes(32);
+ *
+ * bucket.upload('img.png', {
+ *   key: encryptionKey
+ * }, function(err, newFile) {
+ *   // `img.png` was uploaded with your custom encryption key.
+ *
+ *   // `newFile` is already configured to use the encryption key when making
+ *   // operations on the remote object.
+ *
+ *   // However, to use your encryption key later, you must create a `File`
+ *   // instance with the `key` supplied:
+ *   var file = bucket.file('img.png', {
+ *     key: encryptionKey
+ *   });
+ *
+ *   // Or with `file#setKey`:
+ *   var file = bucket.file('img.png');
+ *   file.setKey(encryptionKey);
+ * });
  */
 Bucket.prototype.upload = function(localPath, options, callback) {
   if (is.fn(options)) {
@@ -1060,10 +1090,14 @@ Bucket.prototype.upload = function(localPath, options, callback) {
     newFile = options.destination;
   } else if (is.string(options.destination)) {
     // Use the string as the name of the file.
-    newFile = this.file(options.destination);
+    newFile = this.file(options.destination, {
+      key: options.key
+    });
   } else {
     // Resort to using the name of the incoming file.
-    newFile = this.file(path.basename(localPath));
+    newFile = this.file(path.basename(localPath), {
+      key: options.key
+    });
   }
 
   var contentType = mime.contentType(path.basename(localPath));

lib/storage/file.js

@@ -86,6 +86,7 @@ var STORAGE_UPLOAD_BASE_URL = 'https://www.googleapis.com/upload/storage/v1/b';
  * @param {string} name - The name of the remote file.
  * @param {object=} options - Configuration object.
  * @param {number} options.generation - Generation to scope the file to.
+ * @param {string} options.key - A custom encryption key.
  */
 /**
  * A File object is created from your Bucket object using
@@ -241,6 +242,10 @@ function File(bucket, name, options) {
     methods: methods
   });
 
+  if (options.key) {
+    this.setKey(options.key);
+  }
+
   /**
    * Google Cloud Storage uses access control lists (ACLs) to manage object and
    * bucket access. ACLs are the mechanism you use to share objects with other
@@ -531,7 +536,7 @@ File.prototype.createReadStream = function(options) {
       };
     }
 
-    var requestStream = self.storage.makeAuthenticatedRequest(reqOpts);
+    var requestStream = self.request(reqOpts);
     var validateStream;
 
     // We listen to the response event from the request stream so that we can...
@@ -990,6 +995,55 @@ File.prototype.download = function(options, callback) {
   }
 };
 
+/**
+ * The Storage API allows you to use a custom key for server-side encryption.
+ * Supply this method with a passphrase and the correct key (AES-256) will be
+ * generated and used for you.
+ *
+ * @resource [Customer-supplied Encryption Keys]{@link https://cloud.google.com/storage/docs/encryption#customer-supplied}
+ *
+ * @param {string|buffer} key - An AES-256 encryption key.
+ * @return {module:storage/file}
+ *
+ * @example
+ * var crypto = require('crypto');
+ * var encryptionKey = crypto.randomBytes(32);
+ *
+ * var fileWithCustomEncryption = myBucket.file('my-file');
+ * fileWithCustomEncryption.setKey(encryptionKey);
+ *
+ * var fileWithoutCustomEncryption = myBucket.file('my-file');
+ *
+ * fileWithCustomEncryption.save('data', function(err) {
+ *   // Try to download with the File object that hasn't had `setKey()` called:
+ *   fileWithoutCustomEncryption.download(function(err) {
+ *     // We will receive an error:
+ *     //   err.message === 'Bad Request'
+ *
+ *     // Try again with the File object we called `setKey()` on:
+ *     fileWithCustomEncryption.download(function(err, contents) {
+ *       // contents.toString() === 'data'
+ *     });
+ *   });
+ * });
+ */
+File.prototype.setKey = function(key) {
+  this.key = key;
+
+  key = new Buffer(key).toString('base64');
+  var hash = crypto.createHash('sha256').update(key, 'base64').digest('base64');
+
+  this.interceptors.push({
+    request: function(reqOpts) {
+      reqOpts.headers = reqOpts.headers || {};
+      reqOpts.headers['x-goog-encryption-algorithm'] = 'AES256';
+      reqOpts.headers['x-goog-encryption-key'] = key;
+      reqOpts.headers['x-goog-encryption-key-sha256'] = hash;
+      return reqOpts;
+    }
+  });
+};
+
 /**
  * Get a signed policy document to allow a user to upload data with a POST
  * request.
@@ -1559,6 +1613,7 @@ File.prototype.startResumableUpload_ = function(dup, options) {
     bucket: this.bucket.name,
     file: this.name,
     generation: this.generation,
+    key: this.key,
     metadata: options.metadata,
     offset: options.offset,
     predefinedAcl: options.predefinedAcl,
@@ -1621,12 +1676,20 @@ File.prototype.startSimpleUpload_ = function(dup, options) {
   }
 
   util.makeWritableStream(dup, {
-    makeAuthenticatedRequest: this.storage.makeAuthenticatedRequest,
+    makeAuthenticatedRequest: function(reqOpts) {
+      self.request(reqOpts, function(err, body, resp) {
+        if (err) {
+          dup.destroy(err);
+          return;
+        }
+
+        self.metadata = body;
+        dup.emit('response', resp);
+        dup.emit('complete');
+      });
+    },
     metadata: options.metadata,
     request: reqOpts
-  }, function(data) {
-    self.metadata = data;
-    dup.emit('complete');
   });
 };
 

package.json

@@ -100,7 +100,7 @@
     "ent": "^2.2.0",
     "extend": "^3.0.0",
     "gce-images": "^0.2.0",
-    "gcs-resumable-upload": "^0.6.0",
+    "gcs-resumable-upload": "^0.7.1",
     "google-auto-auth": "^0.2.4",
     "google-proto-files": "^0.2.1",
     "grpc": "^0.14.1",

system-test/storage.js

@@ -379,6 +379,46 @@ describe('storage', function() {
         });
       });
 
+      it('should set custom encryption during the upload', function(done) {
+        var key = crypto.randomBytes(32);
+
+        bucket.upload(FILES.big.path, {
+          key: key,
+          resumable: false
+        }, function(err, file) {
+          assert.ifError(err);
+
+          file.getMetadata(function(err, metadata) {
+            assert.ifError(err);
+            assert.strictEqual(
+              metadata.customerEncryption.encryptionAlgorithm,
+              'AES256'
+            );
+            done();
+          });
+        });
+      });
+
+      it('should set custom encryption in a resumable upload', function(done) {
+        var key = crypto.randomBytes(32);
+
+        bucket.upload(FILES.big.path, {
+          key: key,
+          resumable: true
+        }, function(err, file) {
+          assert.ifError(err);
+
+          file.getMetadata(function(err, metadata) {
+            assert.ifError(err);
+            assert.strictEqual(
+              metadata.customerEncryption.encryptionAlgorithm,
+              'AES256'
+            );
+            done();
+          });
+        });
+      });
+
       it('should make a file public during the upload', function(done) {
         bucket.upload(FILES.big.path, {
           resumable: false,
@@ -759,6 +799,48 @@ describe('storage', function() {
       });
     });
 
+    describe('customer-supplied encryption keys', function() {
+      var encryptionKey = crypto.randomBytes(32);
+
+      var file = bucket.file('encrypted-file', { key: encryptionKey });
+      var unencryptedFile = bucket.file(file.name);
+
+      before(function(done) {
+        file.save('secret data', { resumable: false }, done);
+      });
+
+      it('should not get the hashes from the unencrypted file', function(done) {
+        unencryptedFile.getMetadata(function(err, metadata) {
+          assert.ifError(err);
+          assert.strictEqual(metadata.crc32c, undefined);
+          done();
+        });
+      });
+
+      it('should get the hashes from the encrypted file', function(done) {
+        file.getMetadata(function(err, metadata) {
+          assert.ifError(err);
+          assert.notStrictEqual(metadata.crc32c, undefined);
+          done();
+        });
+      });
+
+      it('should not download from the unencrypted file', function(done) {
+        unencryptedFile.download(function(err) {
+          assert.strictEqual(err.message, 'Bad Request');
+          done();
+        });
+      });
+
+      it('should download from the encrytped file', function(done) {
+        file.download(function(err, contents) {
+          assert.ifError(err);
+          assert.strictEqual(contents.toString(), 'secret data');
+          done();
+        });
+      });
+    });
+
     it('should copy an existing file', function(done) {
       var opts = { destination: 'CloudLogo' };
       bucket.upload(FILES.logo.path, opts, function(err, file) {