From 5e2f4e27c71b05f702024b12e4e27155f76688dd Mon Sep 17 00:00:00 2001 From: Blake Embrey Date: Sun, 10 Jul 2016 17:10:37 -0700 Subject: [PATCH] Implement a max buffer size for node responses --- README.md | 1 + lib/index.ts | 32 ++++++++++++++++++++++---------- lib/test/index.ts | 9 +++++++++ scripts/server.js | 5 +++++ 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index cad84b6..15390d4 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ Popsicle comes with two built-in transports, one for node (using `{http,https}.r * **jar** An instance of a cookie jar (`popsicle.jar()`) (default: `null`) * **agent** Custom HTTP pooling agent (default: [infinity-agent](https://github.com/floatdrop/infinity-agent)) * **maxRedirects** Override the number of redirects allowed (default: `5`) +* **maxBufferSize** The maximum size of the buffered response body (default: `2000000`) * **rejectUnauthorized** Reject invalid SSL certificates (default: `true`) * **confirmRedirect** Confirm redirects on `307` and `308` status codes (default: `() => false`) * **ca** A string, `Buffer` or array of strings or `Buffers` of trusted certificates in PEM format diff --git a/lib/index.ts b/lib/index.ts index f01d9bd..7fdb700 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -36,6 +36,7 @@ export interface Options { ca?: string | Buffer | Array cert?: string | Buffer key?: string | Buffer + maxBufferSize?: number } /** @@ -84,6 +85,7 @@ function handle (request: Request, options: Options) { const { followRedirects, type, unzip, rejectUnauthorized, ca, key, cert, agent } = options const { url, method, body } = request const maxRedirects = num(options.maxRedirects, 5) + const maxBufferSize = num(options.maxBufferSize, type === 'stream' ? Infinity : 2 * 1000 * 1000) const storeCookies = getStoreCookies(request, options) const attachCookies = getAttachCookies(request, options) const confirmRedirect = options.confirmRedirect || falsey @@ -132,21 +134,31 @@ function handle (request: Request, options: Options) { // Track upload/download progress through a stream. const requestStream = new PassThrough() const responseStream = new PassThrough() + let uploadedBytes = 0 + let downloadedBytes = 0 requestStream.on('data', function (chunk: Buffer) { - request.uploadedBytes += chunk.length + uploadedBytes += chunk.length + request.uploadedBytes = uploadedBytes }) requestStream.on('end', function () { - request.uploadedBytes = request.uploadLength + request.uploadedBytes = request.uploadLength = uploadedBytes }) responseStream.on('data', function (chunk: Buffer) { - request.downloadedBytes += chunk.length + downloadedBytes += chunk.length + request.downloadedBytes = downloadedBytes + + // Abort on the max buffer size. + if (downloadedBytes > maxBufferSize) { + rawRequest.abort() + responseStream.emit('error', request.error('Response too large', 'ETOOLARGE')) + } }) responseStream.on('end', function () { - request.downloadedBytes = request.downloadLength + request.downloadedBytes = request.downloadLength = downloadedBytes }) // Handle the HTTP response. @@ -204,24 +216,24 @@ function handle (request: Request, options: Options) { reject(error) } - rawRequest.once('response', function (message: IncomingMessage) { + rawRequest.on('response', function (message: IncomingMessage) { resolve(storeCookies(url, message.headers).then(() => response(message))) }) - rawRequest.once('error', function (error: Error) { + rawRequest.on('error', function (error: Error) { emitError(request.error(`Unable to connect to "${url}"`, 'EUNAVAILABLE', error)) }) request._raw = rawRequest request.uploadLength = num(rawRequest.getHeader('content-length'), 0) requestStream.pipe(rawRequest) - requestStream.once('error', emitError) + requestStream.on('error', emitError) // Pipe the body to the stream. if (body) { if (typeof body.pipe === 'function') { body.pipe(requestStream) - body.once('error', emitError) + body.on('error', emitError) } else { requestStream.end(body) } @@ -329,11 +341,11 @@ function handleResponse ( options: Options ) { const type = options.type || 'text' - const { unzip } = options + const unzip = options.unzip !== false const isText = textTypes.indexOf(type) > -1 const result = new Promise((resolve, reject) => { - if (unzip !== false) { + if (unzip) { const enc = headers['content-encoding'] if (enc === 'deflate' || enc === 'gzip') { diff --git a/lib/test/index.ts b/lib/test/index.ts index 6de75b0..7f3c5f4 100644 --- a/lib/test/index.ts +++ b/lib/test/index.ts @@ -622,6 +622,15 @@ test('response body', function (t) { }) }) + t.test('should break when response body is bigger than buffer size', function (t) { + t.plan(1) + + return popsicle.request(REMOTE_URL + '/urandom') + .catch(function (err) { + t.equal(err.code, 'ETOOLARGE') + }) + }) + t.test('pipe streams', function (t) { return popsicle.request({ url: REMOTE_URL + '/echo', diff --git a/scripts/server.js b/scripts/server.js index aa3515e..c66660f 100644 --- a/scripts/server.js +++ b/scripts/server.js @@ -2,6 +2,7 @@ var express = require('express') var bodyParser = require('body-parser') var zlib = require('zlib') var extend = require('xtend') +var fs = require('fs') var app = module.exports = express() @@ -123,6 +124,10 @@ app.all('/destination', function (req, res) { res.send('welcome ' + req.method.toLowerCase()) }) +app.all('/urandom', function (req, res) { + fs.createReadStream('/dev/urandom').pipe(res) +}) + app.get('/json', function (req, res) { res.send({ username: 'blakeembrey'