From 42096c65baac35a9f2b7183de7a4009eefff5abf Mon Sep 17 00:00:00 2001 From: ghe Date: Mon, 31 Jan 2022 12:46:47 +0000 Subject: [PATCH 1/2] chore: types & formatting --- package.json | 6 +- src/lib/customErrors/apiError.ts | 65 ++- src/lib/customErrors/inputError.ts | 16 +- src/lib/customErrors/requestManagerErrors.ts | 136 +++--- src/lib/error.ts | 59 ++- src/lib/examples.ts | 123 ++++-- src/lib/index.ts | 6 +- src/lib/request/request.ts | 3 +- src/lib/request/requestManager.ts | 433 ++++++++++--------- 9 files changed, 457 insertions(+), 390 deletions(-) diff --git a/package.json b/package.json index 17db606..15550f9 100644 --- a/package.json +++ b/package.json @@ -3,10 +3,10 @@ "description": "Rate controlled and retry enabled request manager to interact with Snyk APIs", "main": "dist/index.js", "scripts": { - "format:check": "prettier --check '{''{lib,test}/!(fixtures)/**/*,*}.{js,ts,json,yml}'", - "format": "prettier --write '{''{lib,test}/!(fixtures)/**/*,*}.{js,ts,json,yml}'", + "format:check": "prettier --check '{''{src,test}/!(fixtures)/**/*,*}.{js,ts,json,yml}'", + "format": "prettier --write '{''{src,test}/!(fixtures)/**/*,*}.{js,ts,json,yml}'", "lint": "npm run format:check && npm run lint:eslint", - "lint:eslint": "eslint --cache '{lib,test}/**/*.ts'", + "lint:eslint": "eslint --cache '{src,test}/**/*.ts'", "test": "npm run lint && npm run test:unit", "test:unit": "jest", "test:coverage": "npm run test:unit -- --coverage", diff --git a/src/lib/customErrors/apiError.ts b/src/lib/customErrors/apiError.ts index 96c1339..db6aec3 100644 --- a/src/lib/customErrors/apiError.ts +++ b/src/lib/customErrors/apiError.ts @@ -1,48 +1,41 @@ - - class ApiError extends Error { - data: {} - constructor(message: any){ - super(message) - this.name = "ApiError" - this.message = (message || "") - this.data = (message.response?.data || "") - } + data: {}; + constructor(message: any) { + super(message); + this.name = 'ApiError'; + this.message = message || ''; + this.data = message.response?.data || ''; + } } class ApiAuthenticationError extends Error { - data: {} - constructor(message: any){ - super(message) - this.name = "ApiAuthenticationError" - this.message = (message || "") - this.data = (message.response?.data || "") - } + data: {}; + constructor(message: any) { + super(message); + this.name = 'ApiAuthenticationError'; + this.message = message || ''; + this.data = message.response?.data || ''; + } } class NotFoundError extends Error { - data: {} - constructor(message: any){ - super(message) - this.name = "NotFoundError" - this.message = (message || "") - this.data = (message.response?.data || "") - } + data: {}; + constructor(message: any) { + super(message); + this.name = 'NotFoundError'; + this.message = message || ''; + this.data = message.response?.data || ''; + } } class GenericError extends Error { - data: {} - constructor(message: any){ - super(message) - this.name = "Unknown" - this.message = (message || "") - this.data = (message.response?.data || "") - } + data: {}; + constructor(message: any) { + super(message); + this.name = 'Unknown'; + this.message = message || ''; + this.data = message.response?.data || ''; + } } -export { - ApiError, - ApiAuthenticationError, - NotFoundError, - GenericError -} \ No newline at end of file +export { ApiError, ApiAuthenticationError, NotFoundError, GenericError }; diff --git a/src/lib/customErrors/inputError.ts b/src/lib/customErrors/inputError.ts index 52b7a2c..33b5c20 100644 --- a/src/lib/customErrors/inputError.ts +++ b/src/lib/customErrors/inputError.ts @@ -1,13 +1,9 @@ - - class BadInputError extends Error { - constructor(message: any){ - super(message) - this.name = "BadInputError" - this.message = (message || "") - } + constructor(message: any) { + super(message); + this.name = 'BadInputError'; + this.message = message || ''; + } } -export { - BadInputError -} \ No newline at end of file +export { BadInputError }; diff --git a/src/lib/customErrors/requestManagerErrors.ts b/src/lib/customErrors/requestManagerErrors.ts index d7bc373..f62fc31 100644 --- a/src/lib/customErrors/requestManagerErrors.ts +++ b/src/lib/customErrors/requestManagerErrors.ts @@ -1,79 +1,89 @@ import * as debugModule from 'debug'; -import {ApiError, - ApiAuthenticationError, - NotFoundError, - GenericError - } from './apiError' -const debug = debugModule('snyk') - -const requestsManagerErrorOverload = (err: Error, channel: string, requestId: string): Error => { - debug('ERROR:', err); - switch(err?.name){ - case 'ApiError': - return new RequestsManagerApiError(err.message, channel, requestId) - case 'ApiAuthenticationError': - return new RequestsManagerApiAuthenticationError(err.message, channel, requestId) - case 'NotFoundError': - return new RequestsManagerNotFoundError(err.message, channel, requestId) - case 'Unknown': - return new RequestsManagerGenericError(err.message, channel, requestId) - default: - } return new RequestsManagerGenericError("Unclassified", channel, requestId) -} +import { + ApiError, + ApiAuthenticationError, + NotFoundError, + GenericError, +} from './apiError'; +const debug = debugModule('snyk'); class RequestsManagerApiError extends ApiError { - channel: string - requestId: string - constructor(message: any, channel: string, requestId: string){ - super(message) - this.name = "ApiError" - this.channel = channel - this.requestId = requestId - this.message = (message || "") - } + channel: string; + requestId: string; + constructor(message: string, channel: string, requestId: string) { + super(message); + this.name = 'ApiError'; + this.channel = channel; + this.requestId = requestId; + this.message = message || ''; + } } class RequestsManagerApiAuthenticationError extends ApiAuthenticationError { - channel: string - requestId: string - constructor(message: any, channel: string, requestId: string){ - super(message) - this.name = "ApiAuthenticationError" - this.channel = channel - this.requestId = requestId - this.message = (message || "") - } + channel: string; + requestId: string; + constructor(message: string, channel: string, requestId: string) { + super(message); + this.name = 'ApiAuthenticationError'; + this.channel = channel; + this.requestId = requestId; + this.message = message || ''; + } } class RequestsManagerNotFoundError extends NotFoundError { - channel: string - requestId: string - constructor(message: any, channel: string, requestId: string){ - super(message) - this.name = "NotFoundError" - this.channel = channel - this.requestId = requestId - this.message = (message || "") - } + channel: string; + requestId: string; + constructor(message: string, channel: string, requestId: string) { + super(message); + this.name = 'NotFoundError'; + this.channel = channel; + this.requestId = requestId; + this.message = message || ''; + } } class RequestsManagerGenericError extends GenericError { - channel: string - requestId: string - constructor(message: any, channel: string, requestId: string){ - super(message) - this.name = "Unknown" - this.channel = channel - this.requestId = requestId - this.message = (message || "") - } + channel: string; + requestId: string; + constructor(message: string, channel: string, requestId: string) { + super(message); + this.name = 'Unknown'; + this.channel = channel; + this.requestId = requestId; + this.message = message || ''; + } } +const requestsManagerErrorOverload = ( + err: Error, + channel: string, + requestId: string, +): Error => { + debug('ERROR:', err); + switch (err?.name) { + case 'ApiError': + return new RequestsManagerApiError(err.message, channel, requestId); + case 'ApiAuthenticationError': + return new RequestsManagerApiAuthenticationError( + err.message, + channel, + requestId, + ); + case 'NotFoundError': + return new RequestsManagerNotFoundError(err.message, channel, requestId); + case 'Unknown': + return new RequestsManagerGenericError(err.message, channel, requestId); + default: + } + return new RequestsManagerGenericError('Unclassified', channel, requestId); +}; + export { - RequestsManagerApiError, - RequestsManagerApiAuthenticationError, - RequestsManagerNotFoundError, - RequestsManagerGenericError, - requestsManagerErrorOverload -} + RequestsManagerApiError, + RequestsManagerApiAuthenticationError, + RequestsManagerNotFoundError, + RequestsManagerGenericError, + requestsManagerErrorOverload, +}; diff --git a/src/lib/error.ts b/src/lib/error.ts index 5ad9342..6e21a80 100644 --- a/src/lib/error.ts +++ b/src/lib/error.ts @@ -1,33 +1,32 @@ -import * as chalk from 'chalk' +import * as chalk from 'chalk'; import debugModule = require('debug'); +const handleError = (error: Error): void => { + const debug = debugModule('snyk'); + if (!process.env.DEBUG) { + console.log(chalk.hex('#316fcc')('hint: Check debug mode -d')); + } + switch (error.name) { + case 'ApiError': + console.log('Uh oh, seems like we messed something up?'); + debug(error); + break; + case 'ApiAuthenticationError': + console.log('Hum, looks like we have a wrong token?'); + debug(error); + break; + case 'NotFoundError': + console.log("Couldn't find find this resource"); + debug(error); + break; + case 'BadInputError': + console.log('Bad input. Please check the --help'); + debug(error); + break; + default: + //console.log("Unknown error") + debug(error); + } +}; -const handleError = (error: Error) => { - const debug = debugModule('snyk') - if(!process.env.DEBUG) { - console.log(chalk.hex("#316fcc")("hint: Check debug mode -d")) - } - switch(error.name){ - case 'ApiError': - console.log("Uh oh, seems like we messed something up?") - debug(error) - break; - case 'ApiAuthenticationError': - console.log("Hum, looks like we have a wrong token?") - debug(error) - break; - case 'NotFoundError': - console.log("Couldn't find find this resource") - debug(error) - break; - case 'BadInputError': - console.log("Bad input. Please check the --help") - debug(error) - break; - default: - //console.log("Unknown error") - debug(error) - } -} - -export default handleError +export default handleError; diff --git a/src/lib/examples.ts b/src/lib/examples.ts index 8aee24a..419925d 100644 --- a/src/lib/examples.ts +++ b/src/lib/examples.ts @@ -1,53 +1,80 @@ -import { requestsManager } from './request/requestManager' +import { requestsManager } from './request/requestManager'; -const run = async () => { - const manager = new requestsManager() +const run = async (): Promise => { + const manager = new requestsManager(); manager.on('data', { - callback:(requestId, data) => { - console.log("response for request ", requestId) - console.log(data) - } - }) + callback: (requestId, data) => { + console.log('response for request ', requestId); + console.log(data); + }, + }); manager.on('error', { - callback:(requestId, data) => { - console.log("response for request ", requestId) - console.log(data) - } - }) + callback: (requestId, data) => { + console.log('response for request ', requestId); + console.log(data); + }, + }); - try{ - let requestSync = await manager.request({verb: "GET", url: '/', body: ''}) - console.log(requestSync) - console.log('done with synced request') + try { + const requestSync = await manager.request({ + verb: 'GET', + url: '/', + body: '', + }); + console.log(requestSync); + console.log('done with synced request'); } catch (err) { - console.log('error') - console.log(err) + console.log('error'); + console.log(err); } - manager.on('data', { - callback:(requestId, data) => { - console.log("response for request on test-channel ", requestId) - console.log(data) - }, - channel: 'test-channel' - }) + callback: (requestId, data) => { + console.log('response for request on test-channel ', requestId); + console.log(data); + }, + channel: 'test-channel', + }); try { - console.log('1',manager.requestStream({verb: "GET", url: '/', body: ''})) - console.log('1-channel',manager.requestStream({verb: "GET", url: '/', body: ''}, 'test-channel')) - console.log('2',manager.requestStream({verb: "GET", url: '/', body: ''})) - console.log('2-channel',manager.requestStream({verb: "GET", url: '/', body: ''}, 'test-channel')) - console.log('3',manager.requestStream({verb: "GET", url: '/', body: ''})) - console.log('3-channel',manager.requestStream({verb: "GET", url: '/', body: ''}, 'test-channel')) + console.log( + '1', + manager.requestStream({ verb: 'GET', url: '/', body: '' }), + ); + console.log( + '1-channel', + manager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + console.log( + '2', + manager.requestStream({ verb: 'GET', url: '/', body: '' }), + ); + console.log( + '2-channel', + manager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); + console.log( + '3', + manager.requestStream({ verb: 'GET', url: '/', body: '' }), + ); + console.log( + '3-channel', + manager.requestStream( + { verb: 'GET', url: '/', body: '' }, + 'test-channel', + ), + ); } catch (err) { - console.log(err) + console.log(err); } - - - const filters = `{ "filters": { "severities": [ @@ -69,14 +96,22 @@ const run = async () => { "ignored": false } } -` +`; try { - const results = await manager.requestBulk([{verb: "GET", url: '/', body: ''}, {verb: "POST", url: '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', body: filters}, {verb: "GET", url: '/', body: ''}]) - console.log(results) - } catch(resultsWithError) { - console.log(resultsWithError) + const results = await manager.requestBulk([ + { verb: 'GET', url: '/', body: '' }, + { + verb: 'POST', + url: + '/org/334e0c45-5d3d-40f6-b882-ae82a164b317/project/0bbbfee1-2138-4322-80d4-4166d1259ae5/issues', + body: filters, + }, + { verb: 'GET', url: '/', body: '' }, + ]); + console.log(results); + } catch (resultsWithError) { + console.log(resultsWithError); } - -} +}; -run() +run(); diff --git a/src/lib/index.ts b/src/lib/index.ts index f9459c1..2e2b279 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -1,6 +1,4 @@ import 'source-map-support/register'; -import { requestsManager } from './request/requestManager' +import { requestsManager } from './request/requestManager'; -export { - requestsManager -} \ No newline at end of file +export { requestsManager }; diff --git a/src/lib/request/request.ts b/src/lib/request/request.ts index e6a2f46..f58e33d 100644 --- a/src/lib/request/request.ts +++ b/src/lib/request/request.ts @@ -1,3 +1,4 @@ +// eslint-disable-next-line @typescript-eslint/no-var-requires const Configstore = require('@snyk/configstore'); import axios, { AxiosResponse } from 'axios'; import * as Error from '../customErrors/apiError'; @@ -93,4 +94,4 @@ const makeSnykRequest = async ( } }; -export { makeSnykRequest, getConfig, SnykRequest as snykRequest }; +export { makeSnykRequest, getConfig, SnykRequest }; diff --git a/src/lib/request/requestManager.ts b/src/lib/request/requestManager.ts index 1148fef..ff204cb 100644 --- a/src/lib/request/requestManager.ts +++ b/src/lib/request/requestManager.ts @@ -1,229 +1,264 @@ import { LeakyBucketQueue } from 'leaky-bucket-queue'; -import { snykRequest, makeSnykRequest } from './request' +import { SnykRequest, makeSnykRequest } from './request'; import { v4 as uuidv4 } from 'uuid'; -import * as requestsManagerError from '../customErrors/requestManagerErrors' +import * as requestsManagerError from '../customErrors/requestManagerErrors'; -interface queuedRequest { - id: string, - channel: string, - snykRequest: snykRequest +interface QueuedRequest { + id: string; + channel: string; + snykRequest: SnykRequest; } -interface queueCallbackListenerBundle { - callback(requestId: string, data: any): void, - channel?: string +interface QueueCallbackListenerBundle { + callback(requestId: string, data: any): void; + channel?: string; } enum eventType { - data = "data", - error = "error", + data = 'data', + error = 'error', } -interface responseEvent { - eventType: eventType, - channel: string, - requestId: string, - data: any +interface ResponseEvent { + eventType: eventType; + channel: string; + requestId: string; + data: any; } -interface requestsManagerParams { - snykToken?: string, - burstSize?: number, - period?: number, - maxRetryCount?: number, - userAgentPrefix?: string +interface RequestsManagerParams { + snykToken?: string; + burstSize?: number; + period?: number; + maxRetryCount?: number; + userAgentPrefix?: string; } -class requestsManager { - _requestsQueue: LeakyBucketQueue - // TODO: Type _events rather than plain obscure object structure - _events: any - _retryCounter: Map - _MAX_RETRY_COUNT: number - _snykToken: string - _userAgentPrefix: string - - //snykToken = '', burstSize = 10, period = 500, maxRetryCount = 5 - constructor(params: requestsManagerParams = {}) { - this._requestsQueue = new LeakyBucketQueue({ burstSize: params?.burstSize || 10, period: params?.period || 500 }); - this._setupQueueExecutors(this._requestsQueue) - this._events = {} - this._retryCounter = new Map() - this._MAX_RETRY_COUNT = params?.maxRetryCount || 5 - this._snykToken = params?.snykToken || '' - this._userAgentPrefix = params?.userAgentPrefix || '' - } +class RequestsManager { + _requestsQueue: LeakyBucketQueue; + // TODO: Type _events rather than plain obscure object structure + _events: any; + _retryCounter: Map; + _MAX_RETRY_COUNT: number; + _snykToken: string; + _userAgentPrefix: string; - _setupQueueExecutors = (queue: LeakyBucketQueue) => { - queue.consume().subscribe({ - next: this._makeRequest, - error: this._queueErrorHandler, - complete: () => { - console.log("Stopped queue") - } - }) - } + //snykToken = '', burstSize = 10, period = 500, maxRetryCount = 5 + constructor(params: RequestsManagerParams = {}) { + this._requestsQueue = new LeakyBucketQueue({ + burstSize: params?.burstSize || 10, + period: params?.period || 500, + }); + this._setupQueueExecutors(this._requestsQueue); + this._events = {}; + this._retryCounter = new Map(); + this._MAX_RETRY_COUNT = params?.maxRetryCount || 5; + this._snykToken = params?.snykToken || ''; + this._userAgentPrefix = params?.userAgentPrefix || ''; + } - _makeRequest = async (request: queuedRequest) => { - let requestId = request.id - try { - let response = await makeSnykRequest(request.snykRequest, this._snykToken, this._userAgentPrefix) - this._emit({eventType: eventType.data, channel: request.channel, requestId: requestId, data: response }) - } catch (err) { - let overloadedError = requestsManagerError.requestsManagerErrorOverload(err, request.channel, requestId) - let alreadyRetriedCount = this._retryCounter.get(requestId) || 0 - if(alreadyRetriedCount >= this._MAX_RETRY_COUNT){ - this._emit({eventType: eventType.error, channel: request.channel, requestId: requestId, data: overloadedError }) - } else { - this._retryCounter.set(requestId, alreadyRetriedCount+1) - // Throw it back into the queue - this.requestStream(request.snykRequest,request.channel, request.id) - } - - - } - - } + _setupQueueExecutors = (queue: LeakyBucketQueue): void => { + queue.consume().subscribe({ + next: this._makeRequest, + error: this._queueErrorHandler, + complete: () => { + console.log('Stopped queue'); + }, + }); + }; - _queueErrorHandler = (err: Error) => { - //debug(err) - // TODO: Add retry logic - // Track request ID count and throw it back into the queue - // Throw error when count > MAX_RETRIES_LIMIT - throw new Error(err.stack) + _makeRequest = async (request: QueuedRequest): Promise => { + const requestId = request.id; + try { + const response = await makeSnykRequest( + request.snykRequest, + this._snykToken, + this._userAgentPrefix, + ); + this._emit({ + eventType: eventType.data, + channel: request.channel, + requestId, + data: response, + }); + } catch (err) { + const overloadedError = requestsManagerError.requestsManagerErrorOverload( + err, + request.channel, + requestId, + ); + const alreadyRetriedCount = this._retryCounter.get(requestId) || 0; + if (alreadyRetriedCount >= this._MAX_RETRY_COUNT) { + this._emit({ + eventType: eventType.error, + channel: request.channel, + requestId: requestId, + data: overloadedError, + }); + } else { + this._retryCounter.set(requestId, alreadyRetriedCount + 1); + // Throw it back into the queue + this.requestStream(request.snykRequest, request.channel, request.id); + } } + }; - - _emit = (response: responseEvent) => { - if (!this._events[response.eventType]) { - throw new Error(`Can't emit an event. Event "${eventType}" doesn't exits.`); - } - - const fireCallbacks = (listenerBundle: queueCallbackListenerBundle) => { - if(response.channel == listenerBundle.channel){ - - listenerBundle.callback(response.requestId, response.data); - } - - }; - - this._events[response.eventType].forEach(fireCallbacks); - } + _queueErrorHandler = (err: Error): void => { + //debug(err) + // TODO: Add retry logic + // Track request ID count and throw it back into the queue + // Throw error when count > MAX_RETRIES_LIMIT + throw new Error(err.stack); + }; - _removeAllListenersForChannel = (channel: string) => { - Object.keys(eventType).forEach(typeOfEvent => { - if (!this._events[typeOfEvent]) { - throw new Error(`Can't remove a listener. Event "${typeOfEvent}" doesn't exits.`); - } - const filterListeners = (callbackListener: queueCallbackListenerBundle) => callbackListener.channel !== channel; - - this._events[typeOfEvent] = this._events[typeOfEvent].filter(filterListeners); - }) + _emit = (response: ResponseEvent): void => { + if (!this._events[response.eventType]) { + throw new Error( + `Can't emit an event. Event "${eventType}" doesn't exits.`, + ); } - _doesChannelHaveListeners = (channel: string) => { - let dataEventListeners = this._events['data'] as Array - return dataEventListeners.some(listener => listener.channel == channel) - } - - request = (request: snykRequest): Promise => { - return new Promise((resolve,reject) => { - let syncRequestChannel = uuidv4() - - const callbackBundle = { - callback: (originalRequestId: string, data: any) => { - if(requestId == originalRequestId){ - this._removeAllListenersForChannel(syncRequestChannel) - resolve(data) - } - }, - channel: syncRequestChannel - } - const errorCallbackBundle = { - callback:(originalRequestId: string, data: any) => { - if(requestId == originalRequestId){ - this._removeAllListenersForChannel(syncRequestChannel) - reject(data) - } - }, - channel: syncRequestChannel - } - - this.on('data', callbackBundle) - this.on('error', errorCallbackBundle) - let requestId = this.requestStream(request, syncRequestChannel) - }) - - } - - - requestBulk = (snykRequestsArray: Array): Promise> => { - return new Promise((resolve,reject) => { - // Fire off all requests in Array and return only when responses are all returned - // Must return array of responses in the same order. - let requestsMap: Map = new Map() - let bulkRequestChannel = uuidv4() - let isErrorInAtLeastOneRequest = false - let requestRemainingCount = snykRequestsArray.length - const callbackBundle = { - callback: (originalRequestId: string, data: any) => { - requestsMap.set(originalRequestId, data) - requestRemainingCount-- - if(requestRemainingCount <= 0){ - let responsesArray: Array = [] - requestsMap.forEach((value) => { - responsesArray.push(value) - }) - isErrorInAtLeastOneRequest? reject(responsesArray) : resolve(responsesArray) - } - }, - channel: bulkRequestChannel - } - const errorCallbackBundle = { - callback:(originalRequestId: string, data: any) => { - isErrorInAtLeastOneRequest = true - callbackBundle.callback(originalRequestId,data) - }, - channel: bulkRequestChannel - } - - this.on('data', callbackBundle) - this.on('error', errorCallbackBundle) - - snykRequestsArray.forEach(snykRequest => { - requestsMap.set(this.requestStream(snykRequest, bulkRequestChannel), {}) - }) - }) - } + const fireCallbacks = ( + listenerBundle: QueueCallbackListenerBundle, + ): void => { + if (response.channel == listenerBundle.channel) { + listenerBundle.callback(response.requestId, response.data); + } + }; - requestStream = (request: snykRequest, channel: string = 'stream', id: string = ''): string => { - let requestId = id ? id : uuidv4() - let requestForQueue: queuedRequest = {id: requestId, channel: channel, snykRequest: request} - this._requestsQueue.enqueue(requestForQueue) - if(!this._doesChannelHaveListeners(channel)){ - throw new Error(`Not listener(s) setup for channel ${channel}`) - } - return requestId - } - + this._events[response.eventType].forEach(fireCallbacks); + }; + _removeAllListenersForChannel = (channel: string): void => { + Object.keys(eventType).forEach((typeOfEvent) => { + if (!this._events[typeOfEvent]) { + throw new Error( + `Can't remove a listener. Event "${typeOfEvent}" doesn't exits.`, + ); + } + const filterListeners = ( + callbackListener: QueueCallbackListenerBundle, + ): boolean => callbackListener.channel !== channel; - on = (eventType: string, listenerBundle: queueCallbackListenerBundle) => { - if (!this._events[eventType]) { - this._events[eventType] = []; - } - if(!listenerBundle.channel) { - listenerBundle.channel = 'stream' - } - this._events[eventType].push(listenerBundle); - } + this._events[typeOfEvent] = this._events[typeOfEvent].filter( + filterListeners, + ); + }); + }; -} + _doesChannelHaveListeners = (channel: string): boolean => { + const dataEventListeners = this._events['data'] as Array< + QueueCallbackListenerBundle + >; + return dataEventListeners.some((listener) => listener.channel == channel); + }; + request = (request: SnykRequest): Promise => { + return new Promise((resolve, reject) => { + const syncRequestChannel = uuidv4(); -export { - requestsManager -} - + const callbackBundle = { + callback: (originalRequestId: string, data: any) => { + // TODO: couble check this is ok + // eslint-disable-next-line @typescript-eslint/no-use-before-define + if (requestId == originalRequestId) { + this._removeAllListenersForChannel(syncRequestChannel); + resolve(data); + } + }, + channel: syncRequestChannel, + }; + const errorCallbackBundle = { + callback: (originalRequestId: string, data: any) => { + // TODO: couble check this is ok + // eslint-disable-next-line @typescript-eslint/no-use-before-define + if (requestId == originalRequestId) { + this._removeAllListenersForChannel(syncRequestChannel); + reject(data); + } + }, + channel: syncRequestChannel, + }; + + this.on('data', callbackBundle); + this.on('error', errorCallbackBundle); + const requestId = this.requestStream(request, syncRequestChannel); + }); + }; + + requestBulk = ( + snykRequestsArray: Array, + ): Promise>> => { + return new Promise((resolve, reject) => { + // Fire off all requests in Array and return only when responses are all returned + // Must return array of responses in the same order. + const requestsMap: Map = new Map(); + const bulkRequestChannel = uuidv4(); + let isErrorInAtLeastOneRequest = false; + let requestRemainingCount = snykRequestsArray.length; + const callbackBundle = { + callback: (originalRequestId: string, data: any) => { + requestsMap.set(originalRequestId, data); + requestRemainingCount--; + if (requestRemainingCount <= 0) { + const responsesArray: Array> = []; + requestsMap.forEach((value) => { + responsesArray.push(value); + }); + isErrorInAtLeastOneRequest + ? reject(responsesArray) + : resolve(responsesArray); + } + }, + channel: bulkRequestChannel, + }; + const errorCallbackBundle = { + callback: (originalRequestId: string, data: any) => { + isErrorInAtLeastOneRequest = true; + callbackBundle.callback(originalRequestId, data); + }, + channel: bulkRequestChannel, + }; + this.on('data', callbackBundle); + this.on('error', errorCallbackBundle); + snykRequestsArray.forEach((snykRequest) => { + requestsMap.set( + this.requestStream(snykRequest, bulkRequestChannel), + {}, + ); + }); + }); + }; + requestStream = ( + request: SnykRequest, + channel = 'stream', + id = '', + ): string => { + const requestId = id ? id : uuidv4(); + const requestForQueue: QueuedRequest = { + id: requestId, + channel: channel, + snykRequest: request, + }; + this._requestsQueue.enqueue(requestForQueue); + if (!this._doesChannelHaveListeners(channel)) { + throw new Error(`Not listener(s) setup for channel ${channel}`); + } + return requestId; + }; + on = ( + eventType: string, + listenerBundle: QueueCallbackListenerBundle, + ): void => { + if (!this._events[eventType]) { + this._events[eventType] = []; + } + if (!listenerBundle.channel) { + listenerBundle.channel = 'stream'; + } + this._events[eventType].push(listenerBundle); + }; +} +export { RequestsManager as requestsManager }; From 8a09bc19584b7bc57063b363202cdd77c6b02fe6 Mon Sep 17 00:00:00 2001 From: ghe Date: Mon, 31 Jan 2022 17:25:06 +0000 Subject: [PATCH 2/2] feat: read user config at requestManager class creation once only --- src/lib/request/request.ts | 23 +--- src/lib/request/requestManager.ts | 35 ++++-- test/lib/request/request.test.ts | 116 ++++++++----------- test/lib/requestManager/normal-flows.test.ts | 40 +++++-- 4 files changed, 113 insertions(+), 101 deletions(-) diff --git a/src/lib/request/request.ts b/src/lib/request/request.ts index f58e33d..489b20b 100644 --- a/src/lib/request/request.ts +++ b/src/lib/request/request.ts @@ -1,5 +1,3 @@ -// eslint-disable-next-line @typescript-eslint/no-var-requires -const Configstore = require('@snyk/configstore'); import axios, { AxiosResponse } from 'axios'; import * as Error from '../customErrors/apiError'; @@ -7,15 +5,8 @@ import * as Error from '../customErrors/apiError'; // where HTTPS over HTTP Proxy Fails with 500 handshakefailed on mcafee proxy import 'global-agent/bootstrap'; -const getConfig = (): { endpoint: string; token: string } => { - const snykApiEndpoint: string = - process.env.SNYK_API || - new Configstore('snyk').get('endpoint') || - 'https://snyk.io/api/v1'; - const snykToken = - process.env.SNYK_TOKEN || new Configstore('snyk').get('api'); - return { endpoint: snykApiEndpoint, token: snykToken }; -}; +const DEFAULT_API = 'https://snyk.io/api/v1'; + interface SnykRequest { verb: string; url: string; @@ -39,11 +30,9 @@ const getTopParentModuleName = (parent: NodeModule | null): string => { const makeSnykRequest = async ( request: SnykRequest, snykToken = '', + apiUrl = DEFAULT_API, userAgentPrefix = '', ): Promise> => { - const userConfig = getConfig(); - const token = snykToken == '' ? userConfig.token : snykToken; - const topParentModuleName = getTopParentModuleName(module.parent as any); const userAgentPrefixChecked = userAgentPrefix != '' && !userAgentPrefix.endsWith('/') @@ -51,12 +40,12 @@ const makeSnykRequest = async ( : userAgentPrefix; const requestHeaders: Record = { 'Content-Type': 'application/json', - Authorization: 'token ' + token, + Authorization: 'token ' + snykToken, 'User-Agent': `${topParentModuleName}${userAgentPrefixChecked}tech-services/snyk-request-manager/1.0`, }; const apiClient = axios.create({ - baseURL: userConfig.endpoint, + baseURL: apiUrl, responseType: 'json', headers: { ...requestHeaders, ...request.headers }, }); @@ -94,4 +83,4 @@ const makeSnykRequest = async ( } }; -export { makeSnykRequest, getConfig, SnykRequest }; +export { makeSnykRequest, SnykRequest, DEFAULT_API }; diff --git a/src/lib/request/requestManager.ts b/src/lib/request/requestManager.ts index ff204cb..982295b 100644 --- a/src/lib/request/requestManager.ts +++ b/src/lib/request/requestManager.ts @@ -1,5 +1,8 @@ +// eslint-disable-next-line @typescript-eslint/no-var-requires +const Configstore = require('@snyk/configstore'); + import { LeakyBucketQueue } from 'leaky-bucket-queue'; -import { SnykRequest, makeSnykRequest } from './request'; +import { SnykRequest, makeSnykRequest, DEFAULT_API } from './request'; import { v4 as uuidv4 } from 'uuid'; import * as requestsManagerError from '../customErrors/requestManagerErrors'; @@ -31,17 +34,33 @@ interface RequestsManagerParams { userAgentPrefix?: string; } +const getConfig = (): { endpoint: string; token: string } => { + const snykApiEndpoint: string = + process.env.SNYK_API || + new Configstore('snyk').get('endpoint') || + DEFAULT_API; + const snykToken = + process.env.SNYK_TOKEN || new Configstore('snyk').get('api'); + return { endpoint: snykApiEndpoint, token: snykToken }; +}; + class RequestsManager { _requestsQueue: LeakyBucketQueue; // TODO: Type _events rather than plain obscure object structure _events: any; + _userConfig: { + endpoint: string; + token: string; + }; // loaded user config from configstore + _apiUrl: string; _retryCounter: Map; _MAX_RETRY_COUNT: number; _snykToken: string; - _userAgentPrefix: string; + _userAgentPrefix?: string; //snykToken = '', burstSize = 10, period = 500, maxRetryCount = 5 constructor(params: RequestsManagerParams = {}) { + this._userConfig = getConfig(); this._requestsQueue = new LeakyBucketQueue({ burstSize: params?.burstSize || 10, period: params?.period || 500, @@ -50,8 +69,9 @@ class RequestsManager { this._events = {}; this._retryCounter = new Map(); this._MAX_RETRY_COUNT = params?.maxRetryCount || 5; - this._snykToken = params?.snykToken || ''; - this._userAgentPrefix = params?.userAgentPrefix || ''; + this._snykToken = params?.snykToken ?? this._userConfig.token; + this._apiUrl = this._userConfig.endpoint; + this._userAgentPrefix = params?.userAgentPrefix; } _setupQueueExecutors = (queue: LeakyBucketQueue): void => { @@ -70,6 +90,7 @@ class RequestsManager { const response = await makeSnykRequest( request.snykRequest, this._snykToken, + this._apiUrl, this._userAgentPrefix, ); this._emit({ @@ -156,7 +177,7 @@ class RequestsManager { const callbackBundle = { callback: (originalRequestId: string, data: any) => { - // TODO: couble check this is ok + // TODO: double check this is ok // eslint-disable-next-line @typescript-eslint/no-use-before-define if (requestId == originalRequestId) { this._removeAllListenersForChannel(syncRequestChannel); @@ -167,7 +188,7 @@ class RequestsManager { }; const errorCallbackBundle = { callback: (originalRequestId: string, data: any) => { - // TODO: couble check this is ok + // TODO: double check this is ok // eslint-disable-next-line @typescript-eslint/no-use-before-define if (requestId == originalRequestId) { this._removeAllListenersForChannel(syncRequestChannel); @@ -261,4 +282,4 @@ class RequestsManager { }; } -export { RequestsManager as requestsManager }; +export { RequestsManager as requestsManager, getConfig }; diff --git a/test/lib/request/request.test.ts b/test/lib/request/request.test.ts index 1152042..517926b 100644 --- a/test/lib/request/request.test.ts +++ b/test/lib/request/request.test.ts @@ -1,4 +1,4 @@ -import { makeSnykRequest, getConfig } from '../../../src/lib/request/request'; +import { makeSnykRequest } from '../../../src/lib/request/request'; import * as fs from 'fs'; import * as nock from 'nock'; import * as _ from 'lodash'; @@ -69,7 +69,10 @@ afterEach(() => { describe('Test Snyk Utils make request properly', () => { it('Test GET command on /', async () => { - const response = await makeSnykRequest({ verb: 'GET', url: '/' }); + const response = await makeSnykRequest( + { verb: 'GET', url: '/' }, + 'token123', + ); const fixturesJSON = JSON.parse( fs .readFileSync(fixturesFolderPath + 'apiResponses/general-doc.json') @@ -82,11 +85,14 @@ describe('Test Snyk Utils make request properly', () => { const bodyToSend = { testbody: {}, }; - const response = await makeSnykRequest({ - verb: 'POST', - url: '/', - body: JSON.stringify(bodyToSend), - }); + const response = await makeSnykRequest( + { + verb: 'POST', + url: '/', + body: JSON.stringify(bodyToSend), + }, + 'token123', + ); expect(_.isEqual(response.data, bodyToSend)).toBeTruthy(); }); }); @@ -94,7 +100,7 @@ describe('Test Snyk Utils make request properly', () => { describe('Test Snyk Utils error handling/classification', () => { it('Test NotFoundError on GET command', async () => { try { - await makeSnykRequest({ verb: 'GET', url: '/xyz', body: '' }); + await makeSnykRequest({ verb: 'GET', url: '/xyz', body: '' }, 'token123'); } catch (err) { expect(err.data).toEqual(404); expect(err).toBeInstanceOf(NotFoundError); @@ -106,11 +112,14 @@ describe('Test Snyk Utils error handling/classification', () => { const bodyToSend = { testbody: {}, }; - await makeSnykRequest({ - verb: 'POST', - url: '/xyz', - body: JSON.stringify(bodyToSend), - }); + await makeSnykRequest( + { + verb: 'POST', + url: '/xyz', + body: JSON.stringify(bodyToSend), + }, + 'token123', + ); } catch (err) { expect(err.data).toEqual(404); expect(err).toBeInstanceOf(NotFoundError); @@ -119,7 +128,7 @@ describe('Test Snyk Utils error handling/classification', () => { it('Test ApiError on GET command', async () => { try { - await makeSnykRequest({ verb: 'GET', url: '/apierror' }); + await makeSnykRequest({ verb: 'GET', url: '/apierror' }, 'token123'); } catch (err) { expect(err.data).toEqual(500); expect(err).toBeInstanceOf(ApiError); @@ -130,11 +139,14 @@ describe('Test Snyk Utils error handling/classification', () => { const bodyToSend = { testbody: {}, }; - await makeSnykRequest({ - verb: 'POST', - url: '/apierror', - body: JSON.stringify(bodyToSend), - }); + await makeSnykRequest( + { + verb: 'POST', + url: '/apierror', + body: JSON.stringify(bodyToSend), + }, + 'token123', + ); } catch (err) { expect(err.data).toEqual(500); expect(err).toBeInstanceOf(ApiError); @@ -143,7 +155,7 @@ describe('Test Snyk Utils error handling/classification', () => { it('Test ApiAuthenticationError on GET command', async () => { try { - await makeSnykRequest({ verb: 'GET', url: '/apiautherror' }); + await makeSnykRequest({ verb: 'GET', url: '/apiautherror' }, 'token123'); } catch (err) { expect(err.data).toEqual(401); expect(err).toBeInstanceOf(ApiAuthenticationError); @@ -154,11 +166,14 @@ describe('Test Snyk Utils error handling/classification', () => { const bodyToSend = { testbody: {}, }; - await makeSnykRequest({ - verb: 'POST', - url: '/apiautherror', - body: JSON.stringify(bodyToSend), - }); + await makeSnykRequest( + { + verb: 'POST', + url: '/apiautherror', + body: JSON.stringify(bodyToSend), + }, + 'token123', + ); } catch (err) { expect(err.data).toEqual(401); expect(err).toBeInstanceOf(ApiAuthenticationError); @@ -167,7 +182,7 @@ describe('Test Snyk Utils error handling/classification', () => { it('Test GenericError on GET command', async () => { try { - await makeSnykRequest({ verb: 'GET', url: '/genericerror' }); + await makeSnykRequest({ verb: 'GET', url: '/genericerror' }, 'token123'); } catch (err) { expect(err.data).toEqual(512); expect(err).toBeInstanceOf(GenericError); @@ -178,50 +193,17 @@ describe('Test Snyk Utils error handling/classification', () => { const bodyToSend = { testbody: {}, }; - await makeSnykRequest({ - verb: 'POST', - url: '/genericerror', - body: JSON.stringify(bodyToSend), - }); + await makeSnykRequest( + { + verb: 'POST', + url: '/genericerror', + body: JSON.stringify(bodyToSend), + }, + 'token123', + ); } catch (err) { expect(err.data).toEqual(512); expect(err).toBeInstanceOf(GenericError); } }); }); - -describe('Test getConfig function', () => { - it('Get snyk token via env var', async () => { - process.env.SNYK_TOKEN = '123'; - expect(getConfig().token).toEqual('123'); - }); - - it('Get snyk.io api endpoint default', async () => { - expect(getConfig().endpoint).toEqual('https://snyk.io/api/v1'); - }); - - it('Get snyk api endpoint via env var', async () => { - process.env.SNYK_API = 'API'; - expect(getConfig().endpoint).toEqual('API'); - }); -}); - -describe('Test snykToken override', () => { - it('Test GET command on / with token override', async () => { - process.env.SNYK_TOKEN = '123'; - const response = await makeSnykRequest( - { verb: 'GET', url: '/customtoken' }, - '0987654321', - ); - expect(_.isEqual(response.data, 'token 0987654321')).toBeTruthy(); - }); - - it('Test GET command on / without token override', async () => { - process.env.SNYK_TOKEN = '123'; - const response = await makeSnykRequest({ - verb: 'GET', - url: '/customtoken', - }); - expect(_.isEqual(response.data, 'token 123')).toBeTruthy(); - }); -}); diff --git a/test/lib/requestManager/normal-flows.test.ts b/test/lib/requestManager/normal-flows.test.ts index f5a3b62..214c181 100644 --- a/test/lib/requestManager/normal-flows.test.ts +++ b/test/lib/requestManager/normal-flows.test.ts @@ -1,4 +1,7 @@ -import { requestsManager } from '../../../src/lib/request/requestManager'; +import { + getConfig, + requestsManager, +} from '../../../src/lib/request/requestManager'; import * as fs from 'fs'; import * as nock from 'nock'; import * as _ from 'lodash'; @@ -258,33 +261,50 @@ describe('Testing Request Flows', () => { console.log(err); } }); +}); - it('Single Sync request with no token override', async () => { - process.env.SNYK_TOKEN = '123'; +describe('Testing Request Flows', () => { + it('Single Sync request with token override', async () => { try { + const requestManager = new requestsManager({ snykToken: '0987654321' }); const responseSync = await requestManager.request({ verb: 'GET', url: '/customtoken', }); - expect(responseSync.data).toEqual('token 123'); + + expect(responseSync.data).toEqual('token 0987654321'); } catch (err) { throw new Error(err); } }); -}); -describe('Testing Request Flows', () => { - const requestManager = new requestsManager({ snykToken: '0987654321' }); - it('Single Sync request with token override', async () => { + it('Single Sync request without token override', async () => { + process.env.SNYK_TOKEN = '123'; + const requestManager = new requestsManager(); try { const responseSync = await requestManager.request({ verb: 'GET', url: '/customtoken', }); - - expect(responseSync.data).toEqual('token 0987654321'); + expect(responseSync.data).toEqual('token 123'); } catch (err) { throw new Error(err); } }); }); + +describe('Test getConfig function', () => { + it('Get snyk token via env var', async () => { + process.env.SNYK_TOKEN = '123'; + expect(getConfig().token).toEqual('123'); + }); + + it('Get snyk.io api endpoint default', async () => { + expect(getConfig().endpoint).toEqual('https://snyk.io/api/v1'); + }); + + it('Get snyk api endpoint via env var', async () => { + process.env.SNYK_API = 'API'; + expect(getConfig().endpoint).toEqual('API'); + }); +});