From 2c12831a65a656a83e088eea7a06d312b815234f Mon Sep 17 00:00:00 2001 From: Nikhil Shahi Date: Wed, 19 Oct 2022 15:52:00 -0500 Subject: [PATCH] fix endpoint, data field, and spec generation (#40) --- backend/src/services/data-field/index.ts | 36 ++++++++++--------- .../jobs/generate-endpoints-traces.ts | 2 +- .../services/jobs/generate-openapi-spec.ts | 36 ++++++++++--------- backend/src/utils/index.ts | 5 +-- 4 files changed, 43 insertions(+), 36 deletions(-) diff --git a/backend/src/services/data-field/index.ts b/backend/src/services/data-field/index.ts index ce4b8096..c94eb2bd 100644 --- a/backend/src/services/data-field/index.ts +++ b/backend/src/services/data-field/index.ts @@ -52,7 +52,7 @@ export class DataFieldService { apiEndpoint: ApiEndpoint, dataValue: any, ): void { - const existingMatch = `${dataSection}.${dataPath}` + const existingMatch = `${dataSection}${dataPath ? `.${dataPath}` : ""}` const dataType = getDataType(dataValue) if (!this.dataFields[existingMatch]) { const dataField = new DataField() @@ -214,31 +214,33 @@ export class DataFieldService { this.dataFields = apiEndpoint.dataFields.reduce((obj, item) => { return { ...obj, - [`${item.dataSection}.${item.dataPath}`]: item, + [`${item.dataSection}${item.dataPath ? `.${item.dataPath}` : ""}`]: item, } }, {}) this.updatedFields = {} this.findPathDataFields(apiTrace.path, apiEndpoint) - this.findPairObjectDataFields( - DataSection.REQUEST_QUERY, - apiTrace.requestParameters, - apiEndpoint, - ) - this.findPairObjectDataFields( - DataSection.REQUEST_HEADER, - apiTrace.requestHeaders, - apiEndpoint, - ) + if (apiTrace.responseStatus < 400) { + this.findPairObjectDataFields( + DataSection.REQUEST_QUERY, + apiTrace.requestParameters, + apiEndpoint, + ) + this.findPairObjectDataFields( + DataSection.REQUEST_HEADER, + apiTrace.requestHeaders, + apiEndpoint, + ) + this.findBodyDataFields( + DataSection.REQUEST_BODY, + apiTrace.requestBody, + apiEndpoint, + ) + } this.findPairObjectDataFields( DataSection.RESPONSE_HEADER, apiTrace.responseHeaders, apiEndpoint, ) - this.findBodyDataFields( - DataSection.REQUEST_BODY, - apiTrace.requestBody, - apiEndpoint, - ) this.findBodyDataFields( DataSection.RESPONSE_BODY, apiTrace.responseBody, diff --git a/backend/src/services/jobs/generate-endpoints-traces.ts b/backend/src/services/jobs/generate-endpoints-traces.ts index 41520545..9220f86d 100644 --- a/backend/src/services/jobs/generate-endpoints-traces.ts +++ b/backend/src/services/jobs/generate-endpoints-traces.ts @@ -112,7 +112,7 @@ const generateEndpointsFromTraces = async (): Promise => { } } if (pathRegex.length > 0) { - pathRegex = String.raw`^${pathRegex}$` + pathRegex = String.raw`^${pathRegex}(/)*$` const regexKey = `${trace.host}-${trace.method}-${pathRegex}` if (regexToTracesMap[regexKey]) { regexToTracesMap[regexKey].traces.push(trace) diff --git a/backend/src/services/jobs/generate-openapi-spec.ts b/backend/src/services/jobs/generate-openapi-spec.ts index 90721c10..80f65533 100644 --- a/backend/src/services/jobs/generate-openapi-spec.ts +++ b/backend/src/services/jobs/generate-openapi-spec.ts @@ -120,21 +120,23 @@ const generateOpenApiSpec = async (): Promise => { ) } } - for (const requestParameter of requestParamters) { - const key = `${requestParameter.name}<>query` - parameters[key] = parseSchema( - parameters[key] ?? {}, - parsedJsonNonNull(requestParameter.value, true), - ) - } - for (const requestHeader of requestHeaders) { - const key = `${requestHeader.name}<>header` - parameters[key] = parseSchema( - parameters[key] ?? {}, - parsedJsonNonNull(requestHeader.value, true), - ) - if (requestHeader.name.toLowerCase() === "content-type") { - requestContentType = requestHeader.value.toLowerCase() + if (trace.responseStatus < 400) { + for (const requestParameter of requestParamters) { + const key = `${requestParameter.name}<>query` + parameters[key] = parseSchema( + parameters[key] ?? {}, + parsedJsonNonNull(requestParameter.value, true), + ) + } + for (const requestHeader of requestHeaders) { + const key = `${requestHeader.name}<>header` + parameters[key] = parseSchema( + parameters[key] ?? {}, + parsedJsonNonNull(requestHeader.value, true), + ) + if (requestHeader.name.toLowerCase() === "content-type") { + requestContentType = requestHeader.value.toLowerCase() + } } } for (const responseHeader of responseHeaders) { @@ -155,7 +157,9 @@ const generateOpenApiSpec = async (): Promise => { ) } - parseContent(requestBodySpec, requestBody, requestContentType) + if (trace.responseStatus < 400) { + parseContent(requestBodySpec, requestBody, requestContentType) + } if (responseBody) { if (!responses[responseStatusString]?.content) { responses[responseStatusString] = { diff --git a/backend/src/utils/index.ts b/backend/src/utils/index.ts index cca12cce..4d669456 100644 --- a/backend/src/utils/index.ts +++ b/backend/src/utils/index.ts @@ -70,12 +70,13 @@ export const getRiskScore = (dataFields: DataField[]): RiskScore => { if (!dataFields) { return RiskScore.NONE } - let numRiskySensitiveDataClasses = 0 + let uniqueSensitiveDataClasses = new Set() for (const dataField of dataFields) { if (dataField.dataClasses) { - numRiskySensitiveDataClasses += dataField.dataClasses?.length ?? 0 + dataField.dataClasses.forEach(e => uniqueSensitiveDataClasses.add(e)) } } + const numRiskySensitiveDataClasses = uniqueSensitiveDataClasses.size switch (true) { case numRiskySensitiveDataClasses >= 3: return RiskScore.HIGH