From 777fa33982a3a8a4c12893922abaec3749d26596 Mon Sep 17 00:00:00 2001 From: Nikhil Shahi Date: Fri, 9 Sep 2022 00:16:26 -0500 Subject: [PATCH] (chore) add '[REDACTED]' to blocked fields instead of removing --- backend/src/services/block-fields/index.ts | 151 +++++++++++++-------- backend/src/services/log-request/index.ts | 2 +- 2 files changed, 97 insertions(+), 56 deletions(-) diff --git a/backend/src/services/block-fields/index.ts b/backend/src/services/block-fields/index.ts index c9bed11a..01f91a9f 100644 --- a/backend/src/services/block-fields/index.ts +++ b/backend/src/services/block-fields/index.ts @@ -1,11 +1,25 @@ import { In, Raw } from "typeorm" import { DataType, DisableRestMethod } from "@common/enums" -import { getDataType, parsedJson } from "utils" +import { getDataType, parsedJson, parsedJsonNonNull } from "utils" import { PairObject } from "@common/types" import { ApiTrace, BlockFields } from "models" import { AppDataSource } from "data-source" export class BlockFieldsService { + static async getBlockFieldsEntry(apiTrace: ApiTrace) { + const blockFieldsRepo = AppDataSource.getRepository(BlockFields) + return await blockFieldsRepo.findOne({ + where: { + host: apiTrace.host, + method: In([apiTrace.method, DisableRestMethod.ALL]), + pathRegex: Raw(alias => `:path ~ ${alias}`, { path: apiTrace.path }), + }, + order: { + numberParams: "ASC", + }, + }) + } + static isContained(arr: string[], str: string) { return arr.some(e => e.toLowerCase() === str.toLowerCase()) } @@ -15,30 +29,44 @@ export class BlockFieldsService { dataSection: string, jsonBody: any, disabledPaths: string[], - ): void { + redacted: boolean, + ): any { const dataType = getDataType(jsonBody) const path = dataPath ? `${dataSection}.${dataPath}` : dataSection if (dataType === DataType.OBJECT) { for (const key in jsonBody) { - if (this.isContained(disabledPaths, `${path}.${key}`)) { - delete jsonBody[key] - } else { - this.recursiveParseBody( - `${dataPath}.${key}`, - dataSection, - jsonBody[key], - disabledPaths, - ) + let tempRedacted = false + if (redacted || this.isContained(disabledPaths, `${path}.${key}`)) { + tempRedacted = true } + jsonBody[key] = this.recursiveParseBody( + `${dataPath}.${key}`, + dataSection, + jsonBody[key], + disabledPaths, + tempRedacted, + ) } } else if (dataType === DataType.ARRAY) { - for (const item of jsonBody) { - this.recursiveParseBody(dataPath, dataSection, item, disabledPaths) + ;(jsonBody as any[]).forEach((item, idx) => { + jsonBody[idx] = this.recursiveParseBody( + dataPath, + dataSection, + item, + disabledPaths, + redacted, + ) + }) + } else { + if (redacted) { + return "[REDACTED]" } + return jsonBody } + return jsonBody } - static removeBlockedFieldsBodyData( + static redactBlockedFieldsBodyData( body: string, dataSection: string, disabledPaths: string[], @@ -46,35 +74,54 @@ export class BlockFieldsService { if (!body) { return } + let redacted = false if (this.isContained(disabledPaths, dataSection)) { - return {} + redacted = true } - const jsonBody = parsedJson(body) + let jsonBody = parsedJson(body) if (jsonBody) { const dataType = getDataType(jsonBody) if (dataType === DataType.OBJECT) { for (let key in jsonBody) { - if (this.isContained(disabledPaths, `${dataSection}.${key}`)) { - delete jsonBody[key] - } else { - this.recursiveParseBody( - key, - dataSection, - jsonBody[key], - disabledPaths, - ) + let tempRedacted = false + if ( + redacted || + this.isContained(disabledPaths, `${dataSection}.${key}`) + ) { + tempRedacted = true } + jsonBody[key] = this.recursiveParseBody( + key, + dataSection, + jsonBody[key], + disabledPaths, + tempRedacted, + ) } } else if (dataType === DataType.ARRAY) { - for (let item of jsonBody) { - this.recursiveParseBody("", dataSection, item, disabledPaths) + ;(jsonBody as any[]).forEach((item, idx) => { + jsonBody[idx] = this.recursiveParseBody( + "", + dataSection, + item, + disabledPaths, + redacted, + ) + }) + } else { + if (redacted) { + jsonBody = "[REDACTED]" } } + } else { + if (redacted) { + jsonBody = "[REDACTED]" + } } return jsonBody ?? body } - static removeBlockedFieldsPairObject( + static redactBlockedFieldsPairObject( data: PairObject[], dataSection: string, disabledPaths: string[], @@ -82,54 +129,48 @@ export class BlockFieldsService { if (!data) { return data } + let redacted = false if (this.isContained(disabledPaths, dataSection)) { - return [] + redacted = true } - let res: PairObject[] = [] - for (const item of data) { - const field = item.name - if (!this.isContained(disabledPaths, `${dataSection}.${field}`)) { - res.push(item) - } - } - return res + return data.map(item => ({ + name: item.name, + value: this.recursiveParseBody( + item.name, + dataSection, + parsedJsonNonNull(item.value, true), + disabledPaths, + redacted || + this.isContained(disabledPaths, `${dataSection}.${item.name}`), + ), + })) } - static async removeBlockedFields(apiTrace: ApiTrace) { - const blockFieldsRepo = AppDataSource.getRepository(BlockFields) - const blockFieldEntry = await blockFieldsRepo.findOne({ - where: { - host: apiTrace.host, - method: In([apiTrace.method, DisableRestMethod.ALL]), - pathRegex: Raw(alias => `:path ~ ${alias}`, { path: apiTrace.path }), - }, - order: { - numberParams: "ASC", - }, - }) + static async redactBlockedFields(apiTrace: ApiTrace) { + const blockFieldEntry = await this.getBlockFieldsEntry(apiTrace) if (blockFieldEntry) { const disabledPaths = blockFieldEntry.disabledPaths - const validRequestParams = this.removeBlockedFieldsPairObject( + const validRequestParams = this.redactBlockedFieldsPairObject( apiTrace.requestParameters, - "req.params", + "req.query", disabledPaths, ) - const validRequestHeaders = this.removeBlockedFieldsPairObject( + const validRequestHeaders = this.redactBlockedFieldsPairObject( apiTrace.requestHeaders, "req.headers", disabledPaths, ) - const validRequestBody = this.removeBlockedFieldsBodyData( + const validRequestBody = this.redactBlockedFieldsBodyData( apiTrace.requestBody, "req.body", disabledPaths, ) - const validResponseHeaders = this.removeBlockedFieldsPairObject( + const validResponseHeaders = this.redactBlockedFieldsPairObject( apiTrace.responseHeaders, "res.headers", disabledPaths, ) - const validResponseBody = this.removeBlockedFieldsBodyData( + const validResponseBody = this.redactBlockedFieldsBodyData( apiTrace.responseBody, "res.body", disabledPaths, diff --git a/backend/src/services/log-request/index.ts b/backend/src/services/log-request/index.ts index 4cded35b..0f07ff8f 100644 --- a/backend/src/services/log-request/index.ts +++ b/backend/src/services/log-request/index.ts @@ -33,7 +33,7 @@ export class LogRequestService { apiTraceObj.responseBody = responseBody apiTraceObj.meta = traceParams?.meta - await BlockFieldsService.removeBlockedFields(apiTraceObj) + await BlockFieldsService.redactBlockedFields(apiTraceObj) /** Update existing endpoint record if exists */ const apiEndpointRepository = AppDataSource.getRepository(ApiEndpoint) const apiEndpoint = await apiEndpointRepository.findOne({