(chore) add '[REDACTED]' to blocked fields instead of removing

metlo-labs · Sep 9, 2022 · 777fa33 · 777fa33
1 parent be01d10
commit 777fa33
Show file tree

Hide file tree

Showing 2 changed files with 97 additions and 56 deletions.
diff --git a/backend/src/services/block-fields/index.ts b/backend/src/services/block-fields/index.ts
@@ -1,11 +1,25 @@
 import { In, Raw } from "typeorm"
 import { DataType, DisableRestMethod } from "@common/enums"
-import { getDataType, parsedJson } from "utils"
+import { getDataType, parsedJson, parsedJsonNonNull } from "utils"
 import { PairObject } from "@common/types"
 import { ApiTrace, BlockFields } from "models"
 import { AppDataSource } from "data-source"
 
 export class BlockFieldsService {
+  static async getBlockFieldsEntry(apiTrace: ApiTrace) {
+    const blockFieldsRepo = AppDataSource.getRepository(BlockFields)
+    return await blockFieldsRepo.findOne({
+      where: {
+        host: apiTrace.host,
+        method: In([apiTrace.method, DisableRestMethod.ALL]),
+        pathRegex: Raw(alias => `:path ~ ${alias}`, { path: apiTrace.path }),
+      },
+      order: {
+        numberParams: "ASC",
+      },
+    })
+  }
+
   static isContained(arr: string[], str: string) {
     return arr.some(e => e.toLowerCase() === str.toLowerCase())
   }
@@ -15,121 +29,148 @@ export class BlockFieldsService {
     dataSection: string,
     jsonBody: any,
     disabledPaths: string[],
-  ): void {
+    redacted: boolean,
+  ): any {
     const dataType = getDataType(jsonBody)
     const path = dataPath ? `${dataSection}.${dataPath}` : dataSection
     if (dataType === DataType.OBJECT) {
       for (const key in jsonBody) {
-        if (this.isContained(disabledPaths, `${path}.${key}`)) {
-          delete jsonBody[key]
-        } else {
-          this.recursiveParseBody(
-            `${dataPath}.${key}`,
-            dataSection,
-            jsonBody[key],
-            disabledPaths,
-          )
+        let tempRedacted = false
+        if (redacted || this.isContained(disabledPaths, `${path}.${key}`)) {
+          tempRedacted = true
         }
+        jsonBody[key] = this.recursiveParseBody(
+          `${dataPath}.${key}`,
+          dataSection,
+          jsonBody[key],
+          disabledPaths,
+          tempRedacted,
+        )
       }
     } else if (dataType === DataType.ARRAY) {
-      for (const item of jsonBody) {
-        this.recursiveParseBody(dataPath, dataSection, item, disabledPaths)
+      ;(jsonBody as any[]).forEach((item, idx) => {
+        jsonBody[idx] = this.recursiveParseBody(
+          dataPath,
+          dataSection,
+          item,
+          disabledPaths,
+          redacted,
+        )
+      })
+    } else {
+      if (redacted) {
+        return "[REDACTED]"
       }
+      return jsonBody
     }
+    return jsonBody
   }
 
-  static removeBlockedFieldsBodyData(
+  static redactBlockedFieldsBodyData(
     body: string,
     dataSection: string,
     disabledPaths: string[],
   ) {
     if (!body) {
       return
     }
+    let redacted = false
     if (this.isContained(disabledPaths, dataSection)) {
-      return {}
+      redacted = true
     }
-    const jsonBody = parsedJson(body)
+    let jsonBody = parsedJson(body)
     if (jsonBody) {
       const dataType = getDataType(jsonBody)
       if (dataType === DataType.OBJECT) {
         for (let key in jsonBody) {
-          if (this.isContained(disabledPaths, `${dataSection}.${key}`)) {
-            delete jsonBody[key]
-          } else {
-            this.recursiveParseBody(
-              key,
-              dataSection,
-              jsonBody[key],
-              disabledPaths,
-            )
+          let tempRedacted = false
+          if (
+            redacted ||
+            this.isContained(disabledPaths, `${dataSection}.${key}`)
+          ) {
+            tempRedacted = true
           }
+          jsonBody[key] = this.recursiveParseBody(
+            key,
+            dataSection,
+            jsonBody[key],
+            disabledPaths,
+            tempRedacted,
+          )
         }
       } else if (dataType === DataType.ARRAY) {
-        for (let item of jsonBody) {
-          this.recursiveParseBody("", dataSection, item, disabledPaths)
+        ;(jsonBody as any[]).forEach((item, idx) => {
+          jsonBody[idx] = this.recursiveParseBody(
+            "",
+            dataSection,
+            item,
+            disabledPaths,
+            redacted,
+          )
+        })
+      } else {
+        if (redacted) {
+          jsonBody = "[REDACTED]"
         }
       }
+    } else {
+      if (redacted) {
+        jsonBody = "[REDACTED]"
+      }
     }
     return jsonBody ?? body
   }
 
-  static removeBlockedFieldsPairObject(
+  static redactBlockedFieldsPairObject(
     data: PairObject[],
     dataSection: string,
     disabledPaths: string[],
   ): PairObject[] {
     if (!data) {
       return data
     }
+    let redacted = false
     if (this.isContained(disabledPaths, dataSection)) {
-      return []
+      redacted = true
     }
-    let res: PairObject[] = []
-    for (const item of data) {
-      const field = item.name
-      if (!this.isContained(disabledPaths, `${dataSection}.${field}`)) {
-        res.push(item)
-      }
-    }
-    return res
+    return data.map(item => ({
+      name: item.name,
+      value: this.recursiveParseBody(
+        item.name,
+        dataSection,
+        parsedJsonNonNull(item.value, true),
+        disabledPaths,
+        redacted ||
+          this.isContained(disabledPaths, `${dataSection}.${item.name}`),
+      ),
+    }))
   }
 
-  static async removeBlockedFields(apiTrace: ApiTrace) {
-    const blockFieldsRepo = AppDataSource.getRepository(BlockFields)
-    const blockFieldEntry = await blockFieldsRepo.findOne({
-      where: {
-        host: apiTrace.host,
-        method: In([apiTrace.method, DisableRestMethod.ALL]),
-        pathRegex: Raw(alias => `:path ~ ${alias}`, { path: apiTrace.path }),
-      },
-      order: {
-        numberParams: "ASC",
-      },
-    })
+  static async redactBlockedFields(apiTrace: ApiTrace) {
+    const blockFieldEntry = await this.getBlockFieldsEntry(apiTrace)
     if (blockFieldEntry) {
       const disabledPaths = blockFieldEntry.disabledPaths
-      const validRequestParams = this.removeBlockedFieldsPairObject(
+      const validRequestParams = this.redactBlockedFieldsPairObject(
         apiTrace.requestParameters,
-        "req.params",
+        "req.query",
         disabledPaths,
       )
-      const validRequestHeaders = this.removeBlockedFieldsPairObject(
+      const validRequestHeaders = this.redactBlockedFieldsPairObject(
         apiTrace.requestHeaders,
         "req.headers",
         disabledPaths,
       )
-      const validRequestBody = this.removeBlockedFieldsBodyData(
+      const validRequestBody = this.redactBlockedFieldsBodyData(
         apiTrace.requestBody,
         "req.body",
         disabledPaths,
       )
-      const validResponseHeaders = this.removeBlockedFieldsPairObject(
+      const validResponseHeaders = this.redactBlockedFieldsPairObject(
         apiTrace.responseHeaders,
         "res.headers",
         disabledPaths,
       )
-      const validResponseBody = this.removeBlockedFieldsBodyData(
+      const validResponseBody = this.redactBlockedFieldsBodyData(
         apiTrace.responseBody,
         "res.body",
         disabledPaths,

diff --git a/backend/src/services/log-request/index.ts b/backend/src/services/log-request/index.ts
@@ -33,7 +33,7 @@ export class LogRequestService {
       apiTraceObj.responseBody = responseBody
       apiTraceObj.meta = traceParams?.meta
 
-      await BlockFieldsService.removeBlockedFields(apiTraceObj)
+      await BlockFieldsService.redactBlockedFields(apiTraceObj)
       /** Update existing endpoint record if exists */
       const apiEndpointRepository = AppDataSource.getRepository(ApiEndpoint)
       const apiEndpoint = await apiEndpointRepository.findOne({