Skip to content

Commit

Permalink
TritonDataCenter#296 add client encryption support
Browse files Browse the repository at this point in the history
  • Loading branch information
@geek
geek committed Jan 26, 2017
1 parent 23cee39 commit 358670b
Show file tree
Hide file tree
Showing 9 changed files with 1,680 additions and 15 deletions.
21 changes: 21 additions & 0 deletions bin/mget
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ var OPTIONS_PARSER = dashdash.createParser({
help: 'HTTP headers to include',
helpArg: 'HEADER'
},
{
names: ['enc_key'],
type: 'string',
help: 'base64 secret key for decrypting remote encrypted objects.' +
' Defaults to environment variable: MANTA_ENCRYPTION_KEY_BYTES'
},
{
names: ['quiet', 'q'],
type: 'bool',
Expand Down Expand Up @@ -114,6 +120,17 @@ function parseOptions() {
opts.headers[tmp[0]] = tmp[1].trim();
});

opts.encrypt = opts.encrypt || {};
opts.encrypt.getKey = function (keyId, cb) {
var key = opts.enc_key || process.env.MANTA_ENCRYPTION_KEY_BYTES;
if (!key) {
cb(new Error('--enc_key or MANTA_ENCRYPTION_KEY_BYTES not found'));
return;
}

cb(null, new Buffer(key, 'base64'));
};

return (opts);
}

Expand Down Expand Up @@ -162,6 +179,10 @@ function printEntry(obj) {
src = stream.pipe(bar.stream());
}

src.on('error', function (srcErr) {
ifError(srcErr);
});

src.pipe(out);

src.on('end', function () {
Expand Down
60 changes: 55 additions & 5 deletions bin/mput
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,25 @@ var OPTIONS_PARSER = dashdash.createParser({
{
group: NAME + ' options'
},
{
names: ['enc_alg'],
type: 'string',
help: 'algorithm to use for encrypting file. ' +
'Defaults to environment variable: MANTA_ENCRYPTION_ALGORITHM'
},
{
names: ['copies', 'c'],
type: 'positiveInteger',
help: 'number of copies to make',
helpArg: 'COPIES',
'default': 2
},
{
names: ['encrypt'],
type: 'bool',
help: 'encrypt the file before storing it in Manta, using ' +
'--algorithm, --key, and --keyid options'
},
{
names: ['file', 'f'],
type: 'string',
Expand All @@ -51,6 +63,19 @@ var OPTIONS_PARSER = dashdash.createParser({
help: 'HTTP headers to include',
helpArg: 'HEADER'
},
{
names: ['enc_key'],
type: 'string',
help: 'base64 secret key for encrypting file. ' +
'Defaults to environment variable: MANTA_ENCRYPTION_KEY_BYTES'
},
{
names: ['enc_keyid'],
type: 'string',
help: 'key identifier, used for locating correct key during ' +
'decryption. Defaults to environment variable: ' +
'MANTA_CLIENT_ENCRYPTION_KEY_ID'
},
{
names: ['md5', 'm'],
type: 'bool',
Expand Down Expand Up @@ -146,6 +171,24 @@ function parseOptions() {
opts['role-tag'] = opts['role-tag'][0].split(/\s*,\s*/);
}

if (opts.encrypt) {
var key = opts.enc_key || process.env.MANTA_ENCRYPTION_KEY_BYTES;
var keyId = opts.enc_keyid ||
process.env.MANTA_CLIENT_ENCRYPTION_KEY_ID;
var algorithm = opts.enc_alg || process.env.MANTA_ENCRYPTION_ALGORITHM;

if (!key || !keyId || !algorithm) {
manta.cli_usage(OPTIONS_PARSER, 'encrypt requires --enc_key, ' +
'--enc_keyid, and --enc_alg or related environment variables');
}

opts.encrypt = {
key: (new Buffer(key, 'base64').toString()),
keyId: keyId,
cipher: algorithm
};
}

return (opts);
}

Expand Down Expand Up @@ -209,18 +252,25 @@ function printEntry(obj) {
});
}

client.put(options.path, stream, opts, function onPut(err) {
client.put(options.path, stream, opts, function onPut(err, res) {
if (err) {
if (cb) {
cb(err);
} else {
ifError(err);
}
return;
}

client.close();
if (cb)
cb();
res.on('error', function (resErr) {
ifError(resErr);
});

res.once('end', function () {
client.close();
if (cb)
cb();
});
});
});
}
Expand All @@ -244,7 +294,7 @@ function printEntry(obj) {
};
var fstream = fs.createReadStream(options.file, f_opts);
fstream.pause();
fstream.on('open', function () {
fstream.once('open', function () {
put(fstream, stats, cb);
});
}
Expand Down
84 changes: 76 additions & 8 deletions lib/client.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ var Watershed = require('watershed').Watershed;
var LOMStream = require('lomstream').LOMStream;

var auth = require('smartdc-auth');
var cse = require('./cse');
var jobshare = require('./jobshare');
var Queue = require('./queue');
var trackmarker = require('./trackmarker');
Expand Down Expand Up @@ -165,13 +166,29 @@ function createOptions(opts, userOpts) {
assert.object(opts, 'options');
assert.string(opts.path, 'options.path');
assert.object(userOpts, 'userOptions');

assert.ok(!userOpts.encrypt || typeof (userOpts.encrypt) === 'object',
'options.encrypt must be false, an object, or not set');

// userEncrypt defaults to false if client.encrypt is false
var userEncrypt = (opts.encrypt === false) || (userOpts.encrypt === false) ?
false : (userOpts.encrypt || {});
var optsEncrypt = opts.encrypt || {};

var encrypt = (userEncrypt === false) ? false : {
cipher: userEncrypt.cipher || optsEncrypt.cipher,
keyId: userEncrypt.keyId || optsEncrypt.keyId,
key: userEncrypt.key || optsEncrypt.key,
getKey: userEncrypt.getKey || optsEncrypt.getKey,
hmacType: userEncrypt.hmacType || optsEncrypt.hmacType,
authMode: userEncrypt.authMode || optsEncrypt.authMode
};
var id = opts.req_id || libuuid.v4();
var options = {
headers: normalizeHeaders(userOpts.headers),
id: id,
path: opts.path.replace(/\/$/, ''),
query: clone(userOpts.query || {})
query: clone(userOpts.query || {}),
encrypt: encrypt
};

if (userOpts.role)
Expand Down Expand Up @@ -492,6 +509,10 @@ function resultToInfoCb(_path, cb) {
* - user : optional user to create jobs under
* - subuser: optional subuser under the user
* - role: optional array of roles that are active for requests
* - encrypt: optional, when false then client-side encryption is disabled.
* When set to an object it can contain defaults for the client-side
* encryption cipher, keyId, key, getKey, and hmacType options provided
* to get()/put()
*
* Throws TypeError's if you pass bad arguments.
*/
Expand All @@ -510,7 +531,8 @@ function MantaClient(options) {
assert.optionalFunc(options.sign, 'options.sign');
assert.ok(options.url || options.socketPath,
'one of options.url or options.socketPath is required');

assert.ok(!options.encrypt || typeof (options.encrypt) === 'object',
'options.encrypt must be false, an object, or not set');
EventEmitter.call(this);

var self = this;
Expand All @@ -522,6 +544,19 @@ function MantaClient(options) {

this.user = options.user;
this.subuser = options.subuser;
this.encrypt = (options.encrypt === false) ? false :
(options.encrypt || {});

if (this.encrypt) {
assert.optionalString(this.encrypt.cipher, 'options.encrypt.cipher');
assert.optionalFunc(this.encrypt.getKey, 'options.encrypt.getKey');
assert.optionalString(this.encrypt.hmacType,
'options.encrypt.hmacType');
assert.optionalString(this.encrypt.key, 'options.encrypt.key');
assert.optionalString(this.encrypt.keyId, 'options.encrypt.keyId');
assert.optionalString(this.encrypt.authMode,
'options.encrypt.authMode');
}

if (options.role) {
options.headers = options.headers || {};
Expand Down Expand Up @@ -784,7 +819,8 @@ MantaClient.prototype.get = function get(p, opts, cb) {
var length = false;
var options = createOptions({
accept: opts.accept || '*/*',
path: _path
path: _path,
encrypt: this.encrypt
}, opts);
var log = this.log.child({
path: _path,
Expand Down Expand Up @@ -827,7 +863,13 @@ MantaClient.prototype.get = function get(p, opts, cb) {

res.pause();

cb(null, stream, res);
// Not encrypted, return original file stream
if (!options.encrypt || !cse.isSupported(res.headers)) {
cb(null, stream, res);
} else {
cse.decrypt({getKey: options.encrypt.getKey,
authMode: options.encrypt.authMode}, stream, res, cb);
}

if (length === false &&
res.headers['content-length'] &&
Expand Down Expand Up @@ -1808,6 +1850,8 @@ MantaClient.prototype.mkdirp = function mkdirp(dir, opts, cb) {
* - cb: callback of the form f(err)
*/
MantaClient.prototype.put = function put(p, input, opts, cb) {
var self = this;

assert.string(p, 'path');
assert.stream(input, 'input');
if (typeof (opts) === 'function') {
Expand All @@ -1831,7 +1875,8 @@ MantaClient.prototype.put = function put(p, input, opts, cb) {
mime.lookup(_path)),
contentLength: opts.size,
expect: '100-continue',
path: _path
path: _path,
encrypt: this.encrypt
}, opts);
var log = this.log.child({
path: _path,
Expand All @@ -1843,17 +1888,40 @@ MantaClient.prototype.put = function put(p, input, opts, cb) {
parseInt(opts.copies, 10);
}

if (options.headers['content-length'] === undefined)
if (options.headers['content-length'] === undefined) {
options.headers['transfer-encoding'] = 'chunked';
}

options._original_path = p; // needed for mkdirp case

log.debug(options, 'put: entered');
if (options.encrypt && options.encrypt.cipher) {
options.headers['e-content-type'] = options.contentType;
options.headers['content-type'] = 'application/octet-stream';
var encOptions = {
cipher: options.encrypt.cipher,
key: options.encrypt.key,
keyId: options.encrypt.keyId,
hmacType: options.encrypt.hmacType,
contentLength: options.contentLength,
headers: options.headers
};
cse.encrypt(encOptions, input, function (err, encrypted) {
if (err) {
cb(err);
return;
}

doPut(self, log, options, encrypted, cb, opts.mkdirs);
});
return;
}

doPut(this, log, options, input, cb, opts.mkdirs);
};


function doPut(self, log, options, input, cb, allowretry) {
log.debug(options, 'put: entered');
self.signRequest({
headers: options.headers
}, function onSignRequest(err) {
Expand Down
11 changes: 10 additions & 1 deletion lib/create_client.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ var DEFAULT_OPTIONS = [

function cloneOptions(options) {
assert.object(options, 'options');
var encrypt = (options.encrypt === false) ? false : (options.encrypt || {});

return ({
agent: options.agent,
Expand All @@ -112,7 +113,15 @@ function cloneOptions(options) {
user: options.user,
subuser: options.subuser,
role: options.role,
url: options.url
url: options.url,
encrypt: (encrypt === false) ? false : {
getKey: encrypt.getKey,
keyId: encrypt.keyId,
key: encrypt.key,
cipher: encrypt.cipher,
hmac: encrypt.hmac,
authMode: encrypt.authMode
}
});
}

Expand Down
Loading

0 comments on commit 358670b

Please sign in to comment.