From bef7df20b4baab053cca833cd6018403714156e5 Mon Sep 17 00:00:00 2001 From: "Jeong, Heon" Date: Tue, 15 Dec 2020 19:47:46 -0800 Subject: [PATCH] Support range header to support partial downloads Range header is important to support download resume, and media streaming. As (GetObject supports Range)[1], this commit proxy the range header from the request to Range parameter of the GetObject. Also found some non-ascii filename caused header to be broken. Added another option to suppress passing the proxied file path. [1]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax --- lib/s3.js | 38 +++++++++------- test/s3.js | 129 +++++++++++++++++++++++++++++++++++------------------ 2 files changed, 107 insertions(+), 60 deletions(-) diff --git a/lib/s3.js b/lib/s3.js index 2085385..3aae921 100644 --- a/lib/s3.js +++ b/lib/s3.js @@ -14,9 +14,10 @@ var debug = require('debug')('s3-proxy'); require('simple-errors'); // HTTP headers from the AWS request to forward along -var awsForwardHeaders = ['content-type', 'last-modified', 'etag', 'cache-control']; +var awsForwardHeaders = ['content-type', 'last-modified', 'etag', 'cache-control', + 'content-length', 'accept-ranges', 'content-range']; -module.exports = function(options) { +module.exports = function (options) { var s3 = new AWS.S3(assign(awsConfig(options), pick(options, 'endpoint', 's3ForcePathStyle'))); @@ -29,7 +30,7 @@ module.exports = function(options) { }; debug('list s3 keys at', s3Params.Prefix); - s3.listObjects(s3Params, function(err, data) { + s3.listObjects(s3Params, function (err, data) { if (err) { return next(Error.create('Could not read S3 keys', { prefix: s3Params.prefix, @@ -38,7 +39,7 @@ module.exports = function(options) { } var keys = []; - map(data.Contents, 'Key').forEach(function(key) { + map(data.Contents, 'Key').forEach(function (key) { // Chop off the prefix path if (key !== s3Params.Prefix) { if (isEmpty(s3Params.Prefix)) { @@ -60,7 +61,7 @@ module.exports = function(options) { // If the key is empty (this occurs if a request comes in for a url ending in '/'), and there is a defaultKey // option present on options, use the default key // E.g. if someone wants to route '/' to '/index.html' - if ( s3Key === '' && options.defaultKey ) s3Key = options.defaultKey; + if (s3Key === '' && options.defaultKey) s3Key = options.defaultKey; // Chop off the querystring, it causes problems with SDK. var queryIndex = s3Key.indexOf('?'); @@ -70,13 +71,14 @@ module.exports = function(options) { // Strip out any path segments that start with a double dash '--'. This is just used // to force a cache invalidation. - s3Key = reject(s3Key.split('/'), function(segment) { + s3Key = reject(s3Key.split('/'), function (segment) { return segment.slice(0, 2) === '--'; }).join('/'); var s3Params = { Bucket: options.bucket, - Key: options.prefix ? urljoin(options.prefix, s3Key) : s3Key + Key: options.prefix ? urljoin(options.prefix, s3Key) : s3Key, + Range: req.headers['range'] || null, }; debug('get s3 object with key %s', s3Params.Key); @@ -92,15 +94,18 @@ module.exports = function(options) { var s3Request = s3.getObject(s3Params); // Write a custom http header with the path to the S3 object being proxied - var headerPrefix = req.app.settings.customHttpHeaderPrefix || 'x-4front-'; - res.setHeader(headerPrefix + 's3-proxy-key', s3Params.Key); + if (options.proxyKey != false) { + var headerPrefix = req.app.settings.customHttpHeaderPrefix || 'x-4front-'; + res.setHeader(headerPrefix + 's3-proxy-key', s3Params.Key); + } - s3Request.on('httpHeaders', function(statusCode, s3Headers) { - debug('received httpHeaders'); + s3Request.on('httpHeaders', function (statusCode, s3Headers) { + debug('received httpHeaders', s3Headers); // Get the contentType from the headers - awsForwardHeaders.forEach(function(header) { + awsForwardHeaders.forEach(function (header) { var headerValue = s3Headers[header]; + debug('got header from s3: ', header, headerValue); if (header === 'content-type') { if (headerValue === 'application/octet-stream') { @@ -120,6 +125,7 @@ module.exports = function(options) { headerValue = '"' + trim(headerValue, '"') + '_base64' + '"'; } else if (header === 'content-length' && base64Encode) { // Clear out the content-length if we are going to base64 encode the response + debug('clearing out content-length as base64Encode is enabled'); headerValue = null; } @@ -133,8 +139,8 @@ module.exports = function(options) { debug('read stream %s', s3Params.Key); var readStream = s3Request.createReadStream() - .on('error', function(err) { - debug('readStream error'); + .on('error', function (err) { + debug('readStream error', err); // If the code is PreconditionFailed and we passed an IfNoneMatch param // the object has not changed, so just return a 304 Not Modified response. if (err.code === 'NotModified' || @@ -142,7 +148,7 @@ module.exports = function(options) { return res.status(304).end(); } if (err.code === 'NoSuchKey') { - return next(Error.http(404, 'Missing S3 key', {code: 'missingS3Key', key: s3Params.Key})); + return next(Error.http(404, 'Missing S3 key', { code: 'missingS3Key', key: s3Params.Key })); } return next(err); }); @@ -156,7 +162,7 @@ module.exports = function(options) { readStream.pipe(res); } - return function(req, res, next) { + return function (req, res, next) { if (req.method !== 'GET') return next(); //If a request is made to a url ending in '/', but there isn't a default file name, diff --git a/test/s3.js b/test/s3.js index fc244bc..a68ad7a 100644 --- a/test/s3.js +++ b/test/s3.js @@ -29,26 +29,26 @@ var S3_OPTIONS = { s3ForcePathStyle: true }; -describe('s3-proxy', function() { +describe('s3-proxy', function () { var self; - beforeEach(function(done) { + beforeEach(function (done) { self = this; this.app = express(); this.s3 = express(); this.pluginOptions = assign({}, S3_OPTIONS); - this.s3.use(function(req, res, next) { + this.s3.use(function (req, res, next) { debug('request to fake S3 server', req.url); next(); }); - this.app.use('/s3-proxy', function(req, res, next) { + this.app.use('/s3-proxy', function (req, res, next) { require('../lib/s3')(self.pluginOptions)(req, res, next); }); - this.app.use(function(err, req, res, next) { + this.app.use(function (err, req, res, next) { if (!err.status) err.status = 500; if (err.status === 500) { @@ -58,19 +58,19 @@ describe('s3-proxy', function() { res.status(err.status).json(Error.toJson(err)); }); - this.s3Server = http.createServer(this.s3).listen(S3_PORT, function() { + this.s3Server = http.createServer(this.s3).listen(S3_PORT, function () { debug('fake s3 server listening'); done(); }); }); - afterEach(function() { + afterEach(function () { if (this.s3Server) { this.s3Server.close(); } }); - it('returns existing json file', function(done) { + it('returns existing json file', function (done) { var jsonFile = [ { name: 'joe', @@ -90,7 +90,7 @@ describe('s3-proxy', function() { prefix: prefix }); - this.s3.get('/' + BUCKET_NAME + '/' + prefix + '/' + key, function(req, res, next) { + this.s3.get('/' + BUCKET_NAME + '/' + prefix + '/' + key, function (req, res, next) { res.set('etag', etag); res.json(jsonFile); }); @@ -101,14 +101,14 @@ describe('s3-proxy', function() { .expect('content-type', 'application/json; charset=utf-8') .expect('x-4front-s3-proxy-key', prefix + '/' + key) .expect('etag', etag) - .expect(function(res) { + .expect(function (res) { assert.deepEqual(res.body, jsonFile); }) .end(done); }); - it('returns 404 for missing file', function(done) { - this.s3.use(function(req, res, next) { + it('returns 404 for missing file', function (done) { + this.s3.use(function (req, res, next) { debug('return 404 error'); sendS3Error(res, 404, 'NoSuchKey'); }); @@ -120,10 +120,10 @@ describe('s3-proxy', function() { .end(done); }); - it('returns 304 for matching etag', function(done) { + it('returns 304 for matching etag', function (done) { var etag = Date.now().toString(); - this.s3.get('/' + BUCKET_NAME + '/' + this.key, function(req, res, next) { + this.s3.get('/' + BUCKET_NAME + '/' + this.key, function (req, res, next) { if (req.headers['if-none-match'] === etag) { return sendS3Error(res, 412, 'PreconditionFailed'); } @@ -138,8 +138,8 @@ describe('s3-proxy', function() { .end(done); }); - it('returns 304 for S3 error NotModified', function(done) { - this.s3.get('/' + BUCKET_NAME + '/' + this.key, function(req, res, next) { + it('returns 304 for S3 error NotModified', function (done) { + this.s3.get('/' + BUCKET_NAME + '/' + this.key, function (req, res, next) { return sendS3Error(res, 304, 'NotModified'); }); @@ -149,9 +149,9 @@ describe('s3-proxy', function() { .end(done); }); - it('sets content-type header based on file path', function(done) { + it('sets content-type header based on file path', function (done) { var key = urljoin('subfolder', 'data.csv'); - this.s3.get('/' + BUCKET_NAME + '/' + key, function(req, res, next) { + this.s3.get('/' + BUCKET_NAME + '/' + key, function (req, res, next) { res.set('content-type', 'application/octet-stream'); res.end('some text'); }); @@ -163,9 +163,9 @@ describe('s3-proxy', function() { .end(done); }); - it('streams an image', function(done) { + it('streams an image', function (done) { var key = urljoin('images', 's3.png'); - this.s3.get('/' + BUCKET_NAME + '/' + key, function(req, res, next) { + this.s3.get('/' + BUCKET_NAME + '/' + key, function (req, res, next) { res.set('content-type', 'image/png'); res.sendFile(path.join(__dirname, './fixtures/s3.png')); }); @@ -177,15 +177,15 @@ describe('s3-proxy', function() { .end(done); }); - describe('cacheControl', function() { - beforeEach(function() { + describe('cacheControl', function () { + beforeEach(function () { self = this; this.s3CacheControl = null; this.etag = null; this.key = urljoin('images', 's3.png'); - this.s3.get('/' + BUCKET_NAME + '/' + this.key, function(req, res, next) { + this.s3.get('/' + BUCKET_NAME + '/' + this.key, function (req, res, next) { res.set('content-type', 'image/png'); if (self.s3CacheControl) { res.set('cache-control', self.s3CacheControl); @@ -200,7 +200,7 @@ describe('s3-proxy', function() { }); }); - it('overrides cache-control', function(done) { + it('overrides cache-control', function (done) { this.s3CacheControl = 'nocache'; this.pluginOptions.overrideCacheControl = 'max-age=10000'; @@ -211,7 +211,7 @@ describe('s3-proxy', function() { .end(done); }); - it('uses default cache control option if no cache-control from S3', function(done) { + it('uses default cache control option if no cache-control from S3', function (done) { this.pluginOptions.defaultCacheControl = 'max-age=1000'; supertest(self.app) @@ -221,7 +221,7 @@ describe('s3-proxy', function() { .end(done); }); - it('uses S3 cache-control rather than defaultCacheControl option', function(done) { + it('uses S3 cache-control rather than defaultCacheControl option', function (done) { this.s3CacheControl = 'private, max-age=0'; this.pluginOptions.defaultCacheControl = 'max-age=1000'; @@ -233,22 +233,22 @@ describe('s3-proxy', function() { }); }); - describe('lists keys', function() { - beforeEach(function() { + describe('lists keys', function () { + beforeEach(function () { self = this; this.s3Keys = ['file1.txt', 'file2.xml', 'file3.json']; - this.s3.get('/' + BUCKET_NAME, function(req, res, next) { + this.s3.get('/' + BUCKET_NAME, function (req, res, next) { var actualKeys; if (req.query.prefix) { - actualKeys = [req.query.prefix].concat(map(self.s3Keys, function(key) { + actualKeys = [req.query.prefix].concat(map(self.s3Keys, function (key) { return urljoin(req.query.prefix, key); })); } else { actualKeys = self.s3Keys; } - var contentsXml = map(actualKeys, function(key) { + var contentsXml = map(actualKeys, function (key) { return '' + key + ''; }); @@ -259,18 +259,18 @@ describe('s3-proxy', function() { }); }); - it('without prefix', function(done) { + it('without prefix', function (done) { supertest(self.app) .get('/s3-proxy/metadata/') .expect(200) .expect('content-type', 'application/json; charset=utf-8') - .expect(function(res) { + .expect(function (res) { assert.deepEqual(res.body, self.s3Keys); }) .end(done); }); - it('with prefix', function(done) { + it('with prefix', function (done) { var prefix = 'folder-name'; this.pluginOptions = assign({}, S3_OPTIONS, { @@ -281,16 +281,16 @@ describe('s3-proxy', function() { .get('/s3-proxy/metadata/') .expect(200) .expect('content-type', 'application/json; charset=utf-8') - .expect(function(res) { + .expect(function (res) { assert.deepEqual(res.body, self.s3Keys); }) .end(done); }); }); - it('strips out path segments starting with a double dash', function(done) { + it('strips out path segments starting with a double dash', function (done) { var key = urljoin('images', 'screenshot.png'); - this.s3.get('/' + BUCKET_NAME + '/' + key, function(req, res, next) { + this.s3.get('/' + BUCKET_NAME + '/' + key, function (req, res, next) { res.set('content-type', 'image/png'); res.sendFile(path.join(__dirname, './fixtures/s3.png')); }); @@ -301,9 +301,9 @@ describe('s3-proxy', function() { .end(done); }); - it('strips off querystring', function(done) { + it('strips off querystring', function (done) { var key = urljoin('images', 'screenshot.png'); - this.s3.get('/' + BUCKET_NAME + '/' + key, function(req, res, next) { + this.s3.get('/' + BUCKET_NAME + '/' + key, function (req, res, next) { if (!isEmpty(req.query)) { return sendS3Error(res, 400, 'invalidQuerystring'); } @@ -318,9 +318,9 @@ describe('s3-proxy', function() { .end(done); }); - it('handles folders and files with spaces', function(done) { + it('handles folders and files with spaces', function (done) { var key = urljoin(encodeURIComponent('sample pdfs'), encodeURIComponent('pdf sample.pdf')); - this.s3.get('/' + BUCKET_NAME + '/' + key, function(req, res, next) { + this.s3.get('/' + BUCKET_NAME + '/' + key, function (req, res, next) { res.set('content-type', 'application/pdf'); res.sendFile(path.join(__dirname, './fixtures/pdf sample.pdf')); }); @@ -331,12 +331,12 @@ describe('s3-proxy', function() { .end(done); }); - it('base64 encodes response', function(done) { + it('base64 encodes response', function (done) { var key = urljoin('files', 'hello.pdf'); var etag = shortid.generate(); var pdfFile = path.join(__dirname, './fixtures/pdf sample.pdf'); - this.s3.get('/' + BUCKET_NAME + '/' + key, function(req, res, next) { + this.s3.get('/' + BUCKET_NAME + '/' + key, function (req, res, next) { res.set('content-type', 'application/pdf'); res.set('etag', '"' + etag + '"'); res.set('Content-Length', 1000); @@ -349,12 +349,53 @@ describe('s3-proxy', function() { .expect(200) .expect('Content-Encoding', 'base64') .expect('ETag', '"' + etag + '_base64"') - .expect(function(res) { + .expect(function (res) { assert.isUndefined(res.headers['content-length']); assert.equal(res.text, fs.readFileSync(pdfFile).toString('base64')); }) .end(done); }); + + it('delegates range header', function (done) { + var prefix = 'datadumps'; + var etag = 'asdfasdfasdf'; + var key = urljoin('subfolder', 'data.json'); + + this.pluginOptions = assign({}, S3_OPTIONS, { + prefix: prefix + }); + + this.s3.get('/' + BUCKET_NAME + '/' + prefix + '/' + key, function (req, res, next) { + res.set('content-type', 'text/plain'); + res.set('etag', '"' + etag + '"'); + if (req.headers['range'] === 'bytes=1-') { + res.end('SDF'); + } else { + res.end('ASDF'); + } + }); + + supertest(self.app) + .get(urljoin('/s3-proxy', key)) + .expect(200) + .expect('content-type', 'text/plain; charset=utf-8') + .expect('x-4front-s3-proxy-key', prefix + '/' + key) + .expect('etag', '"' + etag + '"') + .expect(function (res) { + assert.deepEqual(res.text, "ASDF"); + }) + .get(urljoin('/s3-proxy', key)) + .set('range', 'bytes=1-') + .expect(200) + .expect('content-type', 'text/plain; charset=utf-8') + .expect('x-4front-s3-proxy-key', prefix + '/' + key) + .expect('etag', '"' + etag + '"') + .expect(function (res) { + assert.deepEqual(res.text, "SDF"); + }) + .end(done); + }); + }); function sendS3Error(res, status, code) {