diff --git a/fixtures/.gitkeep b/fixtures/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/lib/customErrors/requestManagerErrors.ts b/src/lib/customErrors/requestManagerErrors.ts index cd029b7..6c2f20d 100644 --- a/src/lib/customErrors/requestManagerErrors.ts +++ b/src/lib/customErrors/requestManagerErrors.ts @@ -1,4 +1,8 @@ -import {ApiError} from './apiError' +import {ApiError, + ApiAuthenticationError, + NotFoundError, + GenericError + } from './apiError' const requestsManagerErrorOverload = (err: Error, channel: string, requestId: string): Error => { switch(err?.name){ @@ -10,7 +14,6 @@ const requestsManagerErrorOverload = (err: Error, channel: string, requestId: st return new RequestsManagerNotFoundError(err.message, channel, requestId) case 'Unknown': return new RequestsManagerGenericError(err.message, channel, requestId) - break; default: } return new RequestsManagerGenericError("Unclassified", channel, requestId) } @@ -27,7 +30,7 @@ class RequestsManagerApiError extends ApiError { } } -class RequestsManagerApiAuthenticationError extends ApiError { +class RequestsManagerApiAuthenticationError extends ApiAuthenticationError { channel: string requestId: string constructor(message: any, channel: string, requestId: string){ @@ -39,7 +42,7 @@ class RequestsManagerApiAuthenticationError extends ApiError { } } -class RequestsManagerNotFoundError extends ApiError { +class RequestsManagerNotFoundError extends NotFoundError { channel: string requestId: string constructor(message: any, channel: string, requestId: string){ @@ -51,7 +54,7 @@ class RequestsManagerNotFoundError extends ApiError { } } -class RequestsManagerGenericError extends ApiError { +class RequestsManagerGenericError extends GenericError { channel: string requestId: string constructor(message: any, channel: string, requestId: string){ diff --git a/src/lib/index.ts b/src/lib/index.ts index 07f6a9c..b9b20a6 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -1,5 +1,5 @@ import 'source-map-support/register'; -import { requestsManager } from './requests/requestsManager' +import { requestsManager } from './request/requestManager' const run = async () => { const manager = new requestsManager() diff --git a/src/lib/request/request.ts b/src/lib/request/request.ts index 938c748..5ac1bf3 100644 --- a/src/lib/request/request.ts +++ b/src/lib/request/request.ts @@ -6,7 +6,8 @@ interface snykRequest { verb: string, url: string, body?: string, - headers?: Object + headers?: Object, + requestId?: string } const makeSnykRequest = async (request: snykRequest) => { diff --git a/src/lib/request/requestManager.ts b/src/lib/request/requestManager.ts index 8d8466a..820f268 100644 --- a/src/lib/request/requestManager.ts +++ b/src/lib/request/requestManager.ts @@ -28,11 +28,15 @@ class requestsManager { _requestsQueue: LeakyBucketQueue // TODO: Type _events rather than plain obscure object structure _events: any + _retryCounter: Map + _MAX_RETRY_COUNT: number - constructor() { - this._requestsQueue = new LeakyBucketQueue({ burstSize: 10, period: 500 }); + constructor(burstSize = 10, period = 500, maxRetryCount = 5) { + this._requestsQueue = new LeakyBucketQueue({ burstSize: burstSize, period: period }); this._setupQueueExecutors(this._requestsQueue) this._events = {} + this._retryCounter = new Map() + this._MAX_RETRY_COUNT = maxRetryCount } _setupQueueExecutors = (queue: LeakyBucketQueue) => { @@ -52,7 +56,16 @@ class requestsManager { this._emit({eventType: eventType.data, channel: request.channel, requestId: requestId, data: response }) } catch (err) { let overloadedError = requestsManagerError.requestsManagerErrorOverload(err, request.channel, requestId) - this._emit({eventType: eventType.error, channel: request.channel, requestId: requestId, data: overloadedError }) + let alreadyRetriedCount = this._retryCounter.get(requestId) || 0 + if(alreadyRetriedCount >= this._MAX_RETRY_COUNT){ + this._emit({eventType: eventType.error, channel: request.channel, requestId: requestId, data: overloadedError }) + } else { + this._retryCounter.set(requestId, alreadyRetriedCount+1) + // Throw it back into the queue + this.requestStream(request.snykRequest,request.channel, request.id) + } + + } } @@ -168,8 +181,8 @@ class requestsManager { }) } - requestStream = (request: snykRequest, channel: string = 'stream'): string => { - let requestId = uuidv4() + requestStream = (request: snykRequest, channel: string = 'stream', id: string = ''): string => { + let requestId = id ? id : uuidv4() let requestForQueue: queuedRequest = {id: requestId, channel: channel, snykRequest: request} this._requestsQueue.enqueue(requestForQueue) if(!this._doesChannelHaveListeners(channel)){ diff --git a/test/fixtures/apiResponses/general-doc.json b/test/fixtures/apiResponses/general-doc.json new file mode 100644 index 0000000..542c5e4 --- /dev/null +++ b/test/fixtures/apiResponses/general-doc.json @@ -0,0 +1,5 @@ +{ + "what orgs can the current token access?": "https://snyk.io/api/v1/orgs", + "what projects are owned by this org?": "https://snyk.io/api/v1/org/:id/projects", + "test a package for issues": "https://snyk.io/api/v1/test/:packageManager/:packageName/:packageVersion" +} \ No newline at end of file diff --git a/test/fixtures/apiResponses/projectIssues.json b/test/fixtures/apiResponses/projectIssues.json new file mode 100644 index 0000000..5cd2a65 --- /dev/null +++ b/test/fixtures/apiResponses/projectIssues.json @@ -0,0 +1,310 @@ +{ + "ok": false, + "issues": { + "vulnerabilities": [ + { + "id": "npm:ms:20170412", + "url": "https://snyk.io/vuln/npm:ms:20170412", + "title": "Regular Expression Denial of Service (ReDoS)", + "type": "vuln", + "description": "## Overview\n[`ms`](https://www.npmjs.com/package/ms) is a tiny millisecond conversion utility.\n\nAffected versions of this package are vulnerable to Regular Expression Denial of Service (ReDoS) due to an incomplete fix for previously reported vulnerability [npm:ms:20151024](https://snyk.io/vuln/npm:ms:20151024). The fix limited the length of accepted input string to 10,000 characters, and turned to be insufficient making it possible to block the event loop for 0.3 seconds (on a typical laptop) with a specially crafted string passed to `ms()` function.\n\n*Proof of concept*\n```js\nms = require('ms');\nms('1'.repeat(9998) + 'Q') // Takes about ~0.3s\n```\n\n**Note:** Snyk's patch for this vulnerability limits input length to 100 characters. This new limit was deemed to be a breaking change by the author.\nBased on user feedback, we believe the risk of breakage is _very_ low, while the value to your security is much greater, and therefore opted to still capture this change in a patch for earlier versions as well. Whenever patching security issues, we always suggest to run tests on your code to validate that nothing has been broken.\n\nFor more information on `Regular Expression Denial of Service (ReDoS)` attacks, go to our [blog](https://snyk.io/blog/redos-and-catastrophic-backtracking/).\n\n## Disclosure Timeline\n- Feb 9th, 2017 - Reported the issue to package owner.\n- Feb 11th, 2017 - Issue acknowledged by package owner.\n- April 12th, 2017 - Fix PR opened by Snyk Security Team.\n- May 15th, 2017 - Vulnerability published.\n- May 16th, 2017 - Issue fixed and version `2.0.0` released.\n- May 21th, 2017 - Patches released for versions `>=0.7.1, <=1.0.0`.\n\n## Remediation\nUpgrade `ms` to version 2.0.0 or higher.\n\n## References\n- [GitHub PR](https://github.com/zeit/ms/pull/89)\n- [GitHub Commit](https://github.com/zeit/ms/pull/89/commits/305f2ddcd4eff7cc7c518aca6bb2b2d2daad8fef)\n", + "from": [ + "mongoose@4.2.4", + "mquery@1.6.3", + "debug@2.2.0", + "ms@0.7.1" + ], + "package": "ms", + "version": "0.7.1", + "severity": "low", + "exploitMaturity": "no-known-exploit", + "language": "js", + "packageManager": "npm", + "semver": { + "unaffected": ">=2.0.0", + "vulnerable": "<2.0.0" + }, + "publicationTime": "2017-05-15T06:02:45.497Z", + "disclosureTime": "2017-04-11T21:00:00.000Z", + "isUpgradable": true, + "isPinnable": false, + "isPatchable": true, + "identifiers": { + "CVE": [], + "CWE": [ + "CWE-400" + ], + "ALTERNATIVE": [ + "SNYK-JS-MS-10509" + ] + }, + "credit": [ + "Snyk Security Research Team" + ], + "CVSSv3": "CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:L", + "cvssScore": 3.7, + "patches": [ + { + "id": "patch:npm:ms:20170412:0", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/ms/20170412/ms_100.patch" + ], + "version": "=1.0.0", + "comments": [], + "modificationTime": "2017-05-16T10:12:18.990Z" + }, + { + "id": "patch:npm:ms:20170412:1", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/ms/20170412/ms_072-073.patch" + ], + "version": "=0.7.2 || =0.7.3", + "comments": [], + "modificationTime": "2017-05-16T10:12:18.990Z" + }, + { + "id": "patch:npm:ms:20170412:2", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/ms/20170412/ms_071.patch" + ], + "version": "=0.7.1", + "comments": [], + "modificationTime": "2017-05-16T10:12:18.990Z" + } + ], + "isIgnored": true, + "isPatched": false, + "upgradePath": [ + "mongoose@4.10.2", + "mquery@2.3.1", + "debug@2.6.8", + "ms@2.0.0" + ] + }, + { + "id": "npm:qs:20170213", + "url": "https://snyk.io/vuln/npm:qs:20170213", + "title": "Prototype Override Protection Bypass", + "type": "vuln", + "description": "## Overview\n[`qs`](https://www.npmjs.com/package/qs) is a querystring parser that supports nesting and arrays, with a depth limit.\n\nBy default `qs` protects against attacks that attempt to overwrite an object's existing prototype properties, such as `toString()`, `hasOwnProperty()`,etc.\n\nFrom [`qs` documentation](https://github.com/ljharb/qs):\n> By default parameters that would overwrite properties on the object prototype are ignored, if you wish to keep the data from those fields either use plainObjects as mentioned above, or set allowPrototypes to true which will allow user input to overwrite those properties. WARNING It is generally a bad idea to enable this option as it can cause problems when attempting to use the properties that have been overwritten. Always be careful with this option.\n\nOverwriting these properties can impact application logic, potentially allowing attackers to work around security controls, modify data, make the application unstable and more.\n\nIn versions of the package affected by this vulnerability, it is possible to circumvent this protection and overwrite prototype properties and functions by prefixing the name of the parameter with `[` or `]`. e.g. `qs.parse(\"]=toString\")` will return `{toString = true}`, as a result, calling `toString()` on the object will throw an exception.\n\n**Example:**\n```js\nqs.parse('toString=foo', { allowPrototypes: false })\n// {}\n\nqs.parse(\"]=toString\", { allowPrototypes: false })\n// {toString = true} <== prototype overwritten\n```\n\nFor more information, you can check out our [blog](https://snyk.io/blog/high-severity-vulnerability-qs/).\n\n## Disclosure Timeline\n- February 13th, 2017 - Reported the issue to package owner.\n- February 13th, 2017 - Issue acknowledged by package owner.\n- February 16th, 2017 - Partial fix released in versions `6.0.3`, `6.1.1`, `6.2.2`, `6.3.1`.\n- March 6th, 2017 - Final fix released in versions `6.4.0`,`6.3.2`, `6.2.3`, `6.1.2` and `6.0.4`\n\n## Remediation\nUpgrade `qs` to version `6.4.0` or higher.\n**Note:** The fix was backported to the following versions `6.3.2`, `6.2.3`, `6.1.2`, `6.0.4`.\n\n## References\n- [GitHub Commit](https://github.com/ljharb/qs/commit/beade029171b8cef9cee0d03ebe577e2dd84976d)\n- [Report of an insufficient fix](https://github.com/ljharb/qs/issues/200)\n", + "from": [ + "qs@0.0.6" + ], + "package": "qs", + "version": "0.0.6", + "severity": "high", + "exploitMaturity": "mature", + "language": "js", + "packageManager": "npm", + "semver": { + "unaffected": ">=6.4.0 || ~6.3.2 || ~6.2.3 || ~6.1.2 || ~6.0.4", + "vulnerable": "<6.3.2 >=6.3.0 || <6.2.3 >=6.2.0 || <6.1.2 >=6.1.0 || <6.0.4" + }, + "publicationTime": "2017-03-01T10:00:54.163Z", + "disclosureTime": "2017-02-13T00:00:00.000Z", + "isUpgradable": true, + "isPinnable": false, + "isPatchable": false, + "identifiers": { + "CVE": [], + "CWE": [ + "CWE-20" + ], + "ALTERNATIVE": [ + "SNYK-JS-QS-10407" + ] + }, + "credit": [ + "Snyk Security Research Team" + ], + "CVSSv3": "CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:N", + "cvssScore": 7.4, + "patches": [ + { + "id": "patch:npm:qs:20170213:0", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/qs/20170213/630_632.patch" + ], + "version": "=6.3.0", + "comments": [], + "modificationTime": "2017-03-09T00:00:00.000Z" + }, + { + "id": "patch:npm:qs:20170213:1", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/qs/20170213/631_632.patch" + ], + "version": "=6.3.1", + "comments": [], + "modificationTime": "2017-03-09T00:00:00.000Z" + }, + { + "id": "patch:npm:qs:20170213:2", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/qs/20170213/621_623.patch" + ], + "version": "=6.2.1", + "comments": [], + "modificationTime": "2017-03-09T00:00:00.000Z" + }, + { + "id": "patch:npm:qs:20170213:3", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/qs/20170213/622_623.patch" + ], + "version": "=6.2.2", + "comments": [], + "modificationTime": "2017-03-09T00:00:00.000Z" + }, + { + "id": "patch:npm:qs:20170213:4", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/qs/20170213/610_612.patch" + ], + "version": "=6.1.0", + "comments": [], + "modificationTime": "2017-03-09T00:00:00.000Z" + }, + { + "id": "patch:npm:qs:20170213:5", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/qs/20170213/611_612.patch" + ], + "version": "=6.1.1", + "comments": [], + "modificationTime": "2017-03-09T00:00:00.000Z" + }, + { + "id": "patch:npm:qs:20170213:6", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/qs/20170213/602_604.patch" + ], + "version": "=6.0.2", + "comments": [], + "modificationTime": "2017-03-09T00:00:00.000Z" + }, + { + "id": "patch:npm:qs:20170213:7", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/qs/20170213/603_604.patch" + ], + "version": "=6.0.3", + "comments": [], + "modificationTime": "2017-03-09T00:00:00.000Z" + } + ], + "isIgnored": true, + "isPatched": false, + "upgradePath": [ + "qs@6.0.4" + ], + "ignored": [ + { + "reason": "Test reason", + "expires": "2100-01-01T00:00:00.000Z", + "source": "cli" + } + ] + }, + { + "id": "npm:mongoose:20160116", + "url": "https://snyk.io/vuln/npm:mongoose:20160116", + "title": "Remote Memory Exposure", + "type": "vuln", + "description": "## Overview\nA potential memory disclosure vulnerability exists in mongoose.\nA `Buffer` field in a MongoDB document can be used to expose sensitive\ninformation such as code, runtime memory and user data into MongoDB.\n\n### Details\nInitializing a `Buffer` field in a document with integer `N` creates a `Buffer`\nof length `N` with non zero-ed out memory.\n**Example:**\n```\nvar x = new Buffer(100); // uninitialized Buffer of length 100\n// vs\nvar x = new Buffer('100'); // initialized Buffer with value of '100'\n```\nInitializing a MongoDB document field in such manner will dump uninitialized\nmemory into MongoDB.\nThe patch wraps `Buffer` field initialization in mongoose by converting a\n`number` value `N` to array `[N]`, initializing the `Buffer` with `N` in its\nbinary form.\n\n#### Proof of concept\n```javascript\nvar mongoose = require('mongoose');\nmongoose.connect('mongodb://localhost/bufftest');\n\n// data: Buffer is not uncommon, taken straight from the docs: http://mongoosejs.com/docs/schematypes.html\nmongoose.model('Item', new mongoose.Schema({id: String, data: Buffer}));\n\nvar Item = mongoose.model('Item');\n\nvar sample = new Item();\nsample.id = 'item1';\n\n// This will create an uninitialized buffer of size 100\nsample.data = 100;\nsample.save(function () {\n Item.findOne(function (err, result) {\n // Print out the data (exposed memory)\n console.log(result.data.toString('ascii'))\n mongoose.connection.db.dropDatabase(); // Clean up everything\n process.exit();\n });\n});\n```\n\n## Remediation\nUpgrade `mongoose` to version >= 3.8.39 or >= 4.3.6.\n\nIf a direct dependency update is not possible, use [`snyk wizard`](https://snyk.io/docs/using-snyk#wizard) to patch this vulnerability.\n\n## References\n- [GitHub Issue](https://github.com/Automattic/mongoose/issues/3764)\n- [Blog: Node Buffer API fix](https://github.com/ChALkeR/notes/blob/master/Lets-fix-Buffer-API.md#previous-materials)\n- [Blog: Information about Buffer](https://github.com/ChALkeR/notes/blob/master/Buffer-knows-everything.md)\n", + "from": [ + "mongoose@4.2.4" + ], + "package": "mongoose", + "version": "4.2.4", + "severity": "medium", + "exploitMaturity": "proof-of-concept", + "language": "js", + "packageManager": "npm", + "semver": { + "unaffected": "<3.5.5 || >=4.3.6", + "vulnerable": "<3.8.39 >=3.5.5 || <4.3.6 >=4.0.0" + }, + "publicationTime": "2016-01-23T12:00:05.158Z", + "disclosureTime": "2016-01-23T12:00:05.158Z", + "isUpgradable": true, + "isPinnable": false, + "isPatchable": true, + "identifiers": { + "CVE": [], + "CWE": [ + "CWE-201" + ], + "ALTERNATIVE": [ + "SNYK-JS-MONGOOSE-10081" + ] + }, + "credit": [ + "ChALkeR" + ], + "CVSSv3": "CVSS:3.0/AV:L/AC:H/PR:N/UI:N/S:U/C:H/I:N/A:N", + "cvssScore": 5.1, + "patches": [ + { + "id": "patch:npm:mongoose:20160116:0", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/mongoose/20160116/20160116_0_0_mongoose_8066b145c07984c8b7e56dbb51721c0a3d48e18a.patch" + ], + "version": "<4.3.6 >=4.1.2", + "comments": [], + "modificationTime": "2016-01-23T12:00:05.158Z" + }, + { + "id": "patch:npm:mongoose:20160116:1", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/mongoose/20160116/20160116_0_1_mongoose_8066b145c07984c8b7e56dbb51721c0a3d48e18a.patch" + ], + "version": "<4.1.2 >=4.0.0", + "comments": [], + "modificationTime": "2016-01-23T12:00:05.158Z" + }, + { + "id": "patch:npm:mongoose:20160116:2", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/mongoose/20160116/20160116_0_3_mongoose_2ff7d36c5e52270211b17f3a84c8a47c6f4d8c1f.patch" + ], + "version": "<3.8.39 >=3.6.11", + "comments": [], + "modificationTime": "2016-01-23T12:00:05.158Z" + }, + { + "id": "patch:npm:mongoose:20160116:3", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/mongoose/20160116/20160116_0_5_mongoose_2ff7d36c5e52270211b17f3a84c8a47c6f4d8c1f.patch" + ], + "version": "=3.6.11", + "comments": [], + "modificationTime": "2016-01-23T12:00:05.158Z" + }, + { + "id": "patch:npm:mongoose:20160116:4", + "urls": [ + "https://snyk-patches.s3.amazonaws.com/npm/mongoose/20160116/20160116_0_4_mongoose_2ff7d36c5e52270211b17f3a84c8a47c6f4d8c1f.patch" + ], + "version": "<3.6.10 >=3.5.5", + "comments": [], + "modificationTime": "2016-01-23T12:00:05.158Z" + } + ], + "isIgnored": false, + "isPatched": true, + "upgradePath": [ + "mongoose@4.3.6" + ], + "patched": [ + { + "patched": "2016-10-24T10:50:51.980Z" + } + ] + } + ], + "licenses": [] + }, + "dependencyCount": 250, + "packageManager": "npm" + } \ No newline at end of file diff --git a/test/lib/index.test.ts b/test/lib/index.test.ts new file mode 100644 index 0000000..ddd39d1 --- /dev/null +++ b/test/lib/index.test.ts @@ -0,0 +1,3 @@ +test('Is everything ready for the development?', async () => { + expect(true).toBeTruthy(); +}); diff --git a/test/lib/request/request.test.ts b/test/lib/request/request.test.ts new file mode 100644 index 0000000..71eb5d1 --- /dev/null +++ b/test/lib/request/request.test.ts @@ -0,0 +1,194 @@ +import { makeSnykRequest, getConfig } from '../../../src/lib/request/request'; +import * as fs from 'fs'; +import * as nock from 'nock'; +import * as _ from 'lodash'; +import * as path from 'path'; +import { + NotFoundError, + ApiError, + ApiAuthenticationError, + GenericError, +} from '../../../src/lib/customErrors/apiError'; + +const fixturesFolderPath = path.resolve(__dirname, '../..') + '/fixtures/'; +beforeEach(() => { + return nock('https://snyk.io') + .get(/\/xyz/) + .reply(404, '404') + .post(/\/xyz/) + .reply(404, '404') + .get(/\/apierror/) + .reply(500, '500') + .post(/\/apierror/) + .reply(500, '500') + .get(/\/genericerror/) + .reply(512, '512') + .post(/\/genericerror/) + .reply(512, '512') + .get(/\/apiautherror/) + .reply(401, '401') + .post(/\/apiautherror/) + .reply(401, '401') + .post(/^(?!.*xyz).*$/) + .reply(200, (uri, requestBody) => { + switch (uri) { + case '/api/v1/': + return requestBody; + break; + default: + } + }) + .get(/^(?!.*xyz).*$/) + .reply(200, (uri) => { + switch (uri) { + case '/api/v1/': + return fs.readFileSync( + fixturesFolderPath + 'apiResponses/general-doc.json', + ); + break; + default: + } + }); +}); + +const OLD_ENV = process.env; +beforeEach(() => { + jest.resetModules(); // this is important - it clears the cache + process.env = { ...OLD_ENV }; + delete process.env.SNYK_TOKEN; +}); + +afterEach(() => { + process.env = OLD_ENV; +}); + +describe('Test Snyk Utils make request properly', () => { + it('Test GET command on /', async () => { + const response = await makeSnykRequest({ verb: 'GET', url: '/' }); + const fixturesJSON = JSON.parse( + fs + .readFileSync(fixturesFolderPath + 'apiResponses/general-doc.json') + .toString(), + ); + + expect(_.isEqual(response, fixturesJSON)).toBeTruthy(); + }); + it('Test POST command on /', async () => { + const bodyToSend = { + testbody: {}, + }; + const response = await makeSnykRequest({ + verb: 'POST', + url: '/', + body: JSON.stringify(bodyToSend), + }); + expect(_.isEqual(response, bodyToSend)).toBeTruthy(); + }); +}); + +describe('Test Snyk Utils error handling/classification', () => { + it('Test NotFoundError on GET command', async () => { + try { + await makeSnykRequest({ verb: 'GET', url: '/xyz', body: '' }); + } catch (err) { + expect(err).toBeInstanceOf(NotFoundError); + } + }); + + it('Test NotFoundError on POST command', async () => { + try { + const bodyToSend = { + testbody: {}, + }; + await makeSnykRequest({ + verb: 'POST', + url: '/xyz', + body: JSON.stringify(bodyToSend), + }); + } catch (err) { + expect(err).toBeInstanceOf(NotFoundError); + } + }); + + it('Test ApiError on GET command', async () => { + try { + await makeSnykRequest({ verb: 'GET', url: '/apierror' }); + } catch (err) { + expect(err).toBeInstanceOf(ApiError); + } + }); + it('Test ApiError on POST command', async () => { + try { + const bodyToSend = { + testbody: {}, + }; + await makeSnykRequest({ + verb: 'POST', + url: '/apierror', + body: JSON.stringify(bodyToSend), + }); + } catch (err) { + expect(err).toBeInstanceOf(ApiError); + } + }); + + it('Test ApiAuthenticationError on GET command', async () => { + try { + await makeSnykRequest({ verb: 'GET', url: '/apiautherror' }); + } catch (err) { + expect(err).toBeInstanceOf(ApiAuthenticationError); + } + }); + it('Test ApiAuthenticationError on POST command', async () => { + try { + const bodyToSend = { + testbody: {}, + }; + await makeSnykRequest({ + verb: 'POST', + url: '/apiautherror', + body: JSON.stringify(bodyToSend), + }); + } catch (err) { + expect(err).toBeInstanceOf(ApiAuthenticationError); + } + }); + + it('Test GenericError on GET command', async () => { + try { + await makeSnykRequest({ verb: 'GET', url: '/genericerror' }); + } catch (err) { + expect(err).toBeInstanceOf(GenericError); + } + }); + it('Test GenericError on POST command', async () => { + try { + const bodyToSend = { + testbody: {}, + }; + await makeSnykRequest({ + verb: 'POST', + url: '/genericerror', + body: JSON.stringify(bodyToSend), + }); + } catch (err) { + expect(err).toBeInstanceOf(GenericError); + } + }); +}); + +describe('Test getConfig function', () => { + it('Get snyk token via env var', async () => { + process.env.SNYK_TOKEN = '123'; + expect(getConfig().token).toEqual('123'); + }); + + it('Get snyk.io api endpoint default', async () => { + expect(getConfig().endpoint).toEqual('https://snyk.io/api/v1'); + }); + + it('Get snyk api endpoint via env var', async () => { + process.env.SNYK_API = 'API'; + expect(getConfig().endpoint).toEqual('API'); + }); +}); diff --git a/test/lib/requestManager/normal-flows.test.ts b/test/lib/requestManager/normal-flows.test.ts new file mode 100644 index 0000000..2d8ccd1 --- /dev/null +++ b/test/lib/requestManager/normal-flows.test.ts @@ -0,0 +1,321 @@ +import { requestsManager } from '../../../src/lib/request/requestManager'; +import * as fs from 'fs'; +import * as nock from 'nock'; +import * as _ from 'lodash'; +import * as path from 'path'; +import { RequestsManagerNotFoundError } from '../../../src/lib/customErrors/requestManagerErrors'; + +const fixturesFolderPath = path.resolve(__dirname, '../..') + '/fixtures/'; +beforeAll(() => { + return nock('https://snyk.io') + .persist() + .get(/\/xyz/) + .reply(404, '404') + .post(/\/xyz/) + .reply(404, '404') + .get(/\/apierror/) + .reply(500, '500') + .post(/\/apierror/) + .reply(500, '500') + .get(/\/genericerror/) + .reply(512, '512') + .post(/\/genericerror/) + .reply(512, '512') + .get(/\/apiautherror/) + .reply(401, '401') + .post(/\/apiautherror/) + .reply(401, '401') + .post(/^(?!.*xyz).*$/) + .reply(200, (uri, requestBody) => { + switch (uri) { + case '/api/v1/': + return requestBody; + case '/api/v1/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues': + return fs.readFileSync( + fixturesFolderPath + 'apiResponses/projectIssues.json', + ); + default: + } + }) + .get(/\/api\/v1\/dummypath/) + .delay(1000) + .reply(200, () => { + return 'dummypath slowed down'; + }) + .get(/^(?!.*xyz).*$/) + .reply(200, (uri) => { + switch (uri) { + case '/api/v1/': + return fs.readFileSync( + fixturesFolderPath + 'apiResponses/general-doc.json', + ); + default: + } + }); +}); + +const requestManager = new requestsManager(); + +describe('Testing Request Flows', () => { + it('Single Sync request', async () => { + try { + const responseSync = await requestManager.request({ + verb: 'GET', + url: '/', + }); + const fixturesJSON = JSON.parse( + fs + .readFileSync(fixturesFolderPath + 'apiResponses/general-doc.json') + .toString(), + ); + + expect(_.isEqual(responseSync, fixturesJSON)).toBeTruthy(); + } catch (err) { + throw new Error(err); + } + }); + + it('2 successive Sync requests', async () => { + try { + const responseSync1 = await requestManager.request({ + verb: 'GET', + url: '/', + }); + const responseSync2 = await requestManager.request({ + verb: 'GET', + url: '/', + }); + const fixturesJSON = JSON.parse( + fs + .readFileSync(fixturesFolderPath + 'apiResponses/general-doc.json') + .toString(), + ); + + expect(_.isEqual(responseSync1, fixturesJSON)).toBeTruthy(); + expect(_.isEqual(responseSync2, fixturesJSON)).toBeTruthy(); + } catch (err) { + console.log(err); + throw new Error(err); + } + }); + + it('Single Sync request fail not found', async () => { + try { + await requestManager.request({ verb: 'GET', url: '/xyz' }); + } catch (err) { + expect(err).toBeInstanceOf(RequestsManagerNotFoundError); + } + }); + + it('Bulk Requests Array Sync request', async () => { + try { + // dummypath is slowed down 1sec to verify that the response array respect the order of request + // waits for all request to be done and return an array of response in the same order. + const results = await requestManager.requestBulk([ + { verb: 'GET', url: '/dummypath' }, + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + { verb: 'GET', url: '/' }, + ]); + const fixturesJSON1 = JSON.parse( + fs + .readFileSync(fixturesFolderPath + 'apiResponses/general-doc.json') + .toString(), + ); + const fixturesJSON2 = JSON.parse( + fs + .readFileSync(fixturesFolderPath + 'apiResponses/projectIssues.json') + .toString(), + ); + + expect(results[0]).toEqual('dummypath slowed down'); + expect(_.isEqual(results[2], fixturesJSON1)).toBeTruthy(); + expect(_.isEqual(results[1], fixturesJSON2)).toBeTruthy(); + } catch (resultsWithError) { + console.log(resultsWithError); + } + }); + + it('Bulk Requests Array Sync request fail not found', async () => { + try { + // xyz serves 404, expecting to return an array with error in it. + await requestManager.requestBulk([ + { verb: 'GET', url: '/xyz' }, + { verb: 'GET', url: '/dummypath' }, + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + ]); + } catch (resultsWithError) { + const fixturesJSON2 = JSON.parse( + fs + .readFileSync(fixturesFolderPath + 'apiResponses/projectIssues.json') + .toString(), + ); + expect(_.isEqual(resultsWithError[2], fixturesJSON2)).toBeTruthy(); + expect(resultsWithError[1]).toEqual('dummypath slowed down'); + expect(resultsWithError[0]).toBeInstanceOf(RequestsManagerNotFoundError); + } + }); + + it('Request Stream request return as soon as done', async (done) => { + const responseMap = new Map(); + + const expectedResponse = [ + { + 'what orgs can the current token access?': + 'https://snyk.io/api/v1/orgs', + 'what projects are owned by this org?': + 'https://snyk.io/api/v1/org/:id/projects', + 'test a package for issues': + 'https://snyk.io/api/v1/test/:packageManager/:packageName/:packageVersion', + }, + + 'dummypath slowed down', + + JSON.parse( + fs + .readFileSync(fixturesFolderPath + 'apiResponses/projectIssues.json') + .toString(), + ), + ]; + + requestManager.on('data', { + callback: (requestId, data) => { + responseMap.set(requestId, data); + + if ( + Array.from(responseMap.values()).filter((value) => value != '') + .length == 3 + ) { + try { + Array.from(responseMap.values()).forEach((response, index) => { + expect(response).toEqual(expectedResponse[index]); + }); + done(); + } catch (err) { + done(err); + } + } + }, + channel: 'test-channel', + }); + requestManager.on('error', { + callback: () => { + done.fail(); + }, + }); + + try { + responseMap.set( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + '', + ); + responseMap.set( + requestManager.requestStream( + { verb: 'GET', url: '/dummypath', body: '' }, + 'test-channel', + ), + '', + ); + responseMap.set( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + '', + ); + } catch (err) { + console.log(err); + } + }); +}); + +// const run = async () => { +// const manager = new requestsManager() +// manager.on('data', { +// callback:(requestId, data) => { +// console.log("response for request ", requestId) +// console.log(data) +// } +// }) + +// manager.on('error', { +// callback:(requestId, data) => { +// console.log("response for request ", requestId) +// console.log(data) +// } +// }) + +// try{ +// let requestSync = await manager.request({verb: "GET", url: '/', body: ''}) +// console.log(requestSync) +// console.log('done with synced request') +// } catch (err) { +// console.log('error') +// console.log(err) +// } + +// manager.on('data', { +// callback:(requestId, data) => { +// console.log("response for request on test-channel ", requestId) +// console.log(data) +// }, +// channel: 'test-channel' +// }) + +// try { +// console.log('1',manager.requestStream({verb: "GET", url: '/', body: ''})) +// console.log('1-channel',manager.requestStream({verb: "GET", url: '/', body: ''}, 'test-channel')) +// console.log('2',manager.requestStream({verb: "GET", url: '/', body: ''})) +// console.log('2-channel',manager.requestStream({verb: "GET", url: '/', body: ''}, 'test-channel')) +// console.log('3',manager.requestStream({verb: "GET", url: '/', body: ''})) +// console.log('3-channel',manager.requestStream({verb: "GET", url: '/', body: ''}, 'test-channel')) +// } catch (err) { +// console.log(err) +// } + +// const filters = `{ +// "filters": { +// "severities": [ +// "high", +// "medium", +// "low" +// ], +// "exploitMaturity": [ +// "mature", +// "proof-of-concept", +// "no-known-exploit", +// "no-data" +// ], +// "types": [ +// "vuln", +// "license" +// ], +// "ignored": false +// } +// } +// ` +// try { +// const results = await manager.requestBulk([{verb: "GET", url: '/', body: ''}, {verb: "POST", url: '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', body: filters}, {verb: "GET", url: '/', body: ''}]) +// console.log(results) +// } catch(resultsWithError) { +// console.log(resultsWithError) +// } + +// } diff --git a/test/lib/requestManager/rate-limits.test.ts b/test/lib/requestManager/rate-limits.test.ts new file mode 100644 index 0000000..339b86f --- /dev/null +++ b/test/lib/requestManager/rate-limits.test.ts @@ -0,0 +1,753 @@ +import { requestsManager } from '../../../src/lib/request/requestManager'; +import * as fs from 'fs'; +import * as nock from 'nock'; +import * as path from 'path'; + +const fixturesFolderPath = path.resolve(__dirname, '../..') + '/fixtures/'; +beforeAll(() => { + return nock('https://snyk.io') + .persist() + .get(/\/xyz/) + .reply(404, '404') + .post(/\/xyz/) + .reply(404, '404') + .get(/\/apierror/) + .reply(500, '500') + .post(/\/apierror/) + .reply(500, '500') + .get(/\/genericerror/) + .reply(512, '512') + .post(/\/genericerror/) + .reply(512, '512') + .get(/\/apiautherror/) + .reply(401, '401') + .post(/\/apiautherror/) + .reply(401, '401') + .post(/^(?!.*xyz).*$/) + .reply(200, (uri, requestBody) => { + switch (uri) { + case '/api/v1/': + return requestBody; + case '/api/v1/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues': + return fs.readFileSync( + fixturesFolderPath + 'apiResponses/projectIssues.json', + ); + default: + } + }) + .get(/\/api\/v1\/dummypath/) + .delay(1000) + .reply(200, () => { + return 'dummypath slowed down'; + }) + .get(/^(?!.*xyz).*$/) + .reply(200, (uri) => { + switch (uri) { + case '/api/v1/': + return fs.readFileSync( + fixturesFolderPath + 'apiResponses/general-doc.json', + ); + default: + } + }); +}); + +describe('Testing Request Rate limiting', () => { + describe('Testing Sync requests', () => { + it('Overall rate limiting in sync requests - burst size 1', async () => { + const requestManager = new requestsManager(1, 200); + const t0 = Date.now(); + + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }); + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(600); + expect(t1 - t0).toBeLessThan(800); + }); + + it('Overall rate limiting in sync requests - burst size 1 - with slow request', async () => { + const requestManager = new requestsManager(1, 200); + const t0 = Date.now(); + + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/dummypath' }); + await requestManager.request({ + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }); + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(1400); + expect(t1 - t0).toBeLessThan(1600); + }); + + it('Overall rate limiting in sync requests - burst size 2', async () => { + const requestManager = new requestsManager(2, 200); + const t0 = Date.now(); + + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }); + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(400); + expect(t1 - t0).toBeLessThan(600); + }); + + it('Overall rate limiting in sync requests - burst size 4', async () => { + const requestManager = new requestsManager(4, 200); + const t0 = Date.now(); + + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }); + const t1 = Date.now(); + + expect(t1 - t0).toBeLessThan(200); + }); + + it('Overall rate limiting in bulk sync requests - burst size 1', async () => { + const requestManager = new requestsManager(1, 200); + const t0 = Date.now(); + + await requestManager.requestBulk([ + { verb: 'GET', url: '/' }, + { verb: 'GET', url: '/' }, + { verb: 'GET', url: '/' }, + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }, + ]); + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(600); + expect(t1 - t0).toBeLessThan(800); + }); + + it('Overall rate limiting in bulk sync requests - burst size 1 with slow request', async () => { + const requestManager = new requestsManager(1, 200); + const t0 = Date.now(); + + await requestManager.requestBulk([ + { verb: 'GET', url: '/' }, + { verb: 'GET', url: '/' }, + { verb: 'GET', url: '/dummypath' }, + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }, + ]); + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(1400); + expect(t1 - t0).toBeLessThan(1600); + }); + + it('Overall rate limiting in bulk sync requests - burst size 2', async () => { + const requestManager = new requestsManager(2, 200); + const t0 = Date.now(); + + await requestManager.requestBulk([ + { verb: 'GET', url: '/' }, + { verb: 'GET', url: '/' }, + { verb: 'GET', url: '/' }, + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }, + ]); + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(400); + expect(t1 - t0).toBeLessThan(600); + }); + + it('Overall rate limiting in bulk sync requests - burst size 4', async () => { + const requestManager = new requestsManager(4, 200); + const t0 = Date.now(); + + await requestManager.requestBulk([ + { verb: 'GET', url: '/' }, + { verb: 'GET', url: '/' }, + { verb: 'GET', url: '/' }, + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }, + ]); + const t1 = Date.now(); + + expect(t1 - t0).toBeLessThan(200); + }); + }); + + describe('Testing Stream requests', () => { + it('Overall rate limiting in sync requests - burst size 1', async (done) => { + const requestManager = new requestsManager(1, 200); + const responseIdArray: Array = []; + const t0 = Date.now(); + + requestManager.on('data', { + callback: (requestId) => { + responseIdArray.splice(responseIdArray.indexOf(requestId), 1); + + if (responseIdArray.length == 0) { + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(600); + expect(t1 - t0).toBeLessThan(800); + + done(); + } + }, + channel: 'test-channel', + }); + + requestManager.on('error', { + callback: () => { + done.fail(); + }, + }); + + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + }); + + it('Overall rate limiting in sync requests - burst size 1 with slow request', async (done) => { + const requestManager = new requestsManager(1, 200); + const responseIdArray: Array = []; + const t0 = Date.now(); + + requestManager.on('data', { + callback: (requestId) => { + responseIdArray.splice(responseIdArray.indexOf(requestId), 1); + if (responseIdArray.length == 0) { + const t1 = Date.now(); + try { + expect(t1 - t0).toBeGreaterThan(1200); + expect(t1 - t0).toBeLessThan(1400); + } catch (err) { + done(err); + } + done(); + } + }, + channel: 'test-channel', + }); + + requestManager.on('error', { + callback: () => { + done.fail(); + }, + }); + + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/dummypath', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + }); + + it('Overall rate limiting in sync requests - burst size 2', async (done) => { + const requestManager = new requestsManager(2, 200); + const responseIdArray: Array = []; + const t0 = Date.now(); + + requestManager.on('data', { + callback: (requestId) => { + responseIdArray.splice(responseIdArray.indexOf(requestId), 1); + + if (responseIdArray.length == 0) { + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(400); + expect(t1 - t0).toBeLessThan(600); + + done(); + } + }, + channel: 'test-channel', + }); + + requestManager.on('error', { + callback: () => { + done.fail(); + }, + }); + + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + }); + + it('Overall rate limiting in sync requests - burst size 4', async (done) => { + const requestManager = new requestsManager(4, 200); + const responseIdArray: Array = []; + const t0 = Date.now(); + + requestManager.on('data', { + callback: (requestId) => { + responseIdArray.splice(responseIdArray.indexOf(requestId), 1); + + if (responseIdArray.length == 0) { + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(0); + expect(t1 - t0).toBeLessThan(200); + + done(); + } + }, + channel: 'test-channel', + }); + + requestManager.on('error', { + callback: () => { + done.fail(); + }, + }); + + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + }); + }); + + describe('Testing Stream + Sync requests', () => { + it('Overall rate limiting in mixed (sync+stream) requests - burst size 1', async (done) => { + const requestManager = new requestsManager(1, 200); + const responseIdArray: Array = []; + const t0 = Date.now(); + + requestManager.on('data', { + callback: (requestId) => { + responseIdArray.splice(responseIdArray.indexOf(requestId), 1); + }, + channel: 'test-channel', + }); + + requestManager.on('error', { + callback: () => { + done.fail(); + }, + }); + + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }); + + if (responseIdArray.length == 0) { + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(1400); + expect(t1 - t0).toBeLessThan(1600); + done(); + } else { + done.fail(); + } + }); + + it('Overall rate limiting in mixed (sync+stream) requests - burst size 2', async (done) => { + const requestManager = new requestsManager(2, 200); + const responseIdArray: Array = []; + const t0 = Date.now(); + + requestManager.on('data', { + callback: (requestId) => { + responseIdArray.splice(responseIdArray.indexOf(requestId), 1); + }, + channel: 'test-channel', + }); + + requestManager.on('error', { + callback: () => { + done.fail(); + }, + }); + + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }); + + if (responseIdArray.length == 0) { + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(1200); + expect(t1 - t0).toBeLessThan(1400); + done(); + } else { + done.fail(); + } + }); + + it('Overall rate limiting in mixed (sync+stream) requests - burst size 4', async (done) => { + const requestManager = new requestsManager(4, 200); + const responseIdArray: Array = []; + const t0 = Date.now(); + + requestManager.on('data', { + callback: (requestId) => { + responseIdArray.splice(responseIdArray.indexOf(requestId), 1); + }, + channel: 'test-channel', + }); + + requestManager.on('error', { + callback: () => { + done.fail(); + }, + }); + + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }); + + if (responseIdArray.length == 0) { + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(800); + expect(t1 - t0).toBeLessThan(1000); + done(); + } else { + done.fail(); + } + }); + + it('Overall rate limiting in mixed (sync+bulk+stream) requests - burst size 4', async (done) => { + const requestManager = new requestsManager(4, 200); + const responseIdArray: Array = []; + const t0 = Date.now(); + + requestManager.on('data', { + callback: (requestId) => { + responseIdArray.splice(responseIdArray.indexOf(requestId), 1); + }, + channel: 'test-channel', + }); + + requestManager.on('error', { + callback: () => { + done.fail(); + }, + }); + + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + responseIdArray.push( + requestManager.requestStream( + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: '{}', + }, + 'test-channel', + ), + ); + + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ verb: 'GET', url: '/' }); + await requestManager.request({ + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }); + + await requestManager.requestBulk([ + { verb: 'GET', url: '/' }, + { verb: 'GET', url: '/' }, + { verb: 'GET', url: '/' }, + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + }, + ]); + + if (responseIdArray.length == 0) { + const t1 = Date.now(); + expect(t1 - t0).toBeGreaterThan(1600); + expect(t1 - t0).toBeLessThan(1800); + done(); + } else { + done.fail(); + } + }); + }); + // it('Burst size respected', async () => { + // const requestManager = new requestsManager(2,200) + + // }) +}); diff --git a/test/lib/requestManager/retries.test.ts b/test/lib/requestManager/retries.test.ts new file mode 100644 index 0000000..a3315db --- /dev/null +++ b/test/lib/requestManager/retries.test.ts @@ -0,0 +1,114 @@ +import { requestsManager } from '../../../src/lib/request/requestManager'; +import * as fs from 'fs'; +import * as nock from 'nock'; +import * as path from 'path'; +import { RequestsManagerApiError } from '../../../src/lib/customErrors/requestManagerErrors'; + +const fixturesFolderPath = path.resolve(__dirname, '../..') + '/fixtures/'; + +const requestManager = new requestsManager(); + +describe('Testing Request Retries', () => { + it('Retry on 500 - success after 1 retry', async () => { + nock('https://snyk.io') + .post(/\/apierror/) + .reply(500, '500'); + nock('https://snyk.io') + .post(/\/apierror/) + .reply(200, () => { + return fs.readFileSync( + fixturesFolderPath + 'apiResponses/general-doc.json', + ); + }); + + try { + const response = await requestManager.request({ + verb: 'POST', + url: '/apierror', + }); + expect(response).toEqual( + JSON.parse( + fs + .readFileSync(fixturesFolderPath + 'apiResponses/general-doc.json') + .toString(), + ), + ); + } catch (err) { + console.log(err); + } + }); + + it('Retry on 500 - success after 4 retries', async () => { + nock('https://snyk.io') + .post(/\/apierror/) + .reply(500, '500'); + nock('https://snyk.io') + .post(/\/apierror/) + .reply(500, '500'); + nock('https://snyk.io') + .post(/\/apierror/) + .reply(500, '500'); + nock('https://snyk.io') + .post(/\/apierror/) + .reply(500, '500'); + nock('https://snyk.io') + .post(/\/apierror/) + .reply(200, () => { + return fs.readFileSync( + fixturesFolderPath + 'apiResponses/general-doc.json', + ); + }); + + try { + const response = await requestManager.request({ + verb: 'POST', + url: '/apierror', + }); + expect(response).toEqual( + JSON.parse( + fs + .readFileSync(fixturesFolderPath + 'apiResponses/general-doc.json') + .toString(), + ), + ); + } catch (err) { + console.log(err); + } + }); + + it('Retry on 500 - fail after 5 retries', async () => { + let hasReached5thTime = false; + nock('https://snyk.io') + .post(/\/apierror/) + .reply(500, '500'); + nock('https://snyk.io') + .post(/\/apierror/) + .reply(500, '500'); + nock('https://snyk.io') + .post(/\/apierror/) + .reply(500, '500'); + nock('https://snyk.io') + .post(/\/apierror/) + .reply(500, '500'); + nock('https://snyk.io') + .post(/\/apierror/) + .reply(500, '500'); + nock('https://snyk.io') + .post(/\/apierror/) + .reply(500, () => { + hasReached5thTime = true; + return '500'; + }); + + try { + const response = await requestManager.request({ + verb: 'POST', + url: '/apierror', + }); + expect(response).toThrowError(); + } catch (err) { + expect(err).toBeInstanceOf(RequestsManagerApiError); + expect(hasReached5thTime).toBeTruthy(); + } + }); +}); diff --git a/test/tsconfig.json b/test/tsconfig.json new file mode 100644 index 0000000..e516b7f --- /dev/null +++ b/test/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../tsconfig.json", + "include": ["**/*.ts"] +}