add endpoints from traces

backend/models/api-trace.ts

@@ -1,6 +1,7 @@
-import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn } from "typeorm";
+import { BaseEntity, Column, CreateDateColumn, Entity, JoinColumn, ManyToOne, OneToOne, PrimaryGeneratedColumn } from "typeorm";
 import { Meta, PairObject } from "../src/types";
 import { RestMethod } from "../src/enums";
+import { ApiEndpoint } from "./api-endpoint";
 
 @Entity()
 export class ApiTrace extends BaseEntity {
@@ -45,4 +46,11 @@ export class ApiTrace extends BaseEntity {
 
   @Column({ type: "jsonb", nullable: true})
   meta: Meta
+
+  @Column({ nullable: true })
+  apiEndpointUuid: string
+
+  @ManyToOne(() => ApiEndpoint)
+  @JoinColumn()
+  apiEndpoint: ApiEndpoint
 }

backend/package.json

@@ -20,11 +20,13 @@
     "express": "^4.18.1",
     "pg": "^8.7.3",
     "reflect-metadata": "^0.1.13",
-    "typeorm": "^0.3.7"
+    "typeorm": "^0.3.7",
+    "validator": "^13.7.0"
   },
   "devDependencies": {
     "@types/express": "^4.17.13",
     "@types/node": "^18.6.1",
+    "@types/validator": "^13.7.4",
     "nodemon": "^2.0.19",
     "prettier": "^2.7.1",
     "ts-node": "^10.9.1",

backend/src/services/endpoints/index.ts

@@ -0,0 +1,77 @@
+import { IsNull } from "typeorm";
+import { isSuspectedParamater } from "../../utils";
+import { ApiEndpoint, ApiTrace } from "../../../models";
+import { AppDataSource } from "../../data-source";
+import { RestMethod } from "../../enums";
+
+interface GenerateEndpoint {
+  parameterizedPath: string;
+  host: string;
+  method: RestMethod;
+  traces: ApiTrace[];
+}
+
+export class EndpointsService {
+  static async generateEndpointsFromTraces() {
+    const apiTraceRepository = AppDataSource.getRepository(ApiTrace);
+    const apiEndpointRepository = AppDataSource.getRepository(ApiEndpoint);
+    const regexToTracesMap: Record<string, GenerateEndpoint> = {};
+    const traces = await apiTraceRepository.findBy({ apiEndpointUuid: IsNull() });
+    if (traces?.length > 0) {
+      for (let i = 0; i < traces.length; i++) {
+        const trace = traces[i];
+        let found = false;
+        const regexes = Object.keys(regexToTracesMap);
+        for (let x = 0; x < regexes.length && !found; x++) {
+          const regex = regexes[x];
+          const curr = regexToTracesMap[regex];
+          if (RegExp(regex).test(trace.path) && trace.host === curr.host && trace.method === curr.method) {
+            found = true;
+            regexToTracesMap[regex] = { ...regexToTracesMap[regex], traces: [...regexToTracesMap[regex].traces, trace]};
+          }
+        }
+        if (!found) {
+          const pathTokens = trace.path.split("/");
+          let paramNum = 1;
+          let parameterizedPath = "";
+          let pathRegex = String.raw``;
+          for (let j = 0; j < pathTokens.length; j++) {
+            const tokenString = pathTokens[j];
+            if (isSuspectedParamater(tokenString)) {
+              parameterizedPath += `/{param${paramNum}}`;
+              pathRegex += String.raw`/[^/]+`;
+              paramNum += 1;
+            } else {
+              parameterizedPath += `/${tokenString}`;
+              pathRegex += String.raw`/${tokenString}`;
+            }
+          }
+          if (pathRegex.length > 0) {
+            if (regexToTracesMap[pathRegex]) {
+              regexToTracesMap[pathRegex] = { ...regexToTracesMap[pathRegex], traces: [...regexToTracesMap[pathRegex].traces, trace]};
+            } else {
+              regexToTracesMap[pathRegex] = { parameterizedPath, host: trace.host, method: trace.method, traces: [trace]};
+            }
+          }
+        }
+      }
+      console.log(regexToTracesMap);
+      Object.entries(regexToTracesMap).map(async ([regex, value], idx) => {
+        const apiEndpoint = new ApiEndpoint();
+        apiEndpoint.path =  value.parameterizedPath;
+        apiEndpoint.host = value.traces[0].host;
+        apiEndpoint.environment = value.traces[0].environment;
+        apiEndpoint.totalCalls = value.traces.length;
+        apiEndpoint.method = value.traces[0].method;
+        apiEndpoint.owner = value.traces[0].owner;
+        await apiEndpointRepository.save(apiEndpoint);
+        // TODO: Do something with setting sensitive data classes during iteration of traces
+        for (let i = 0; i < value.traces.length; i++) {
+          const trace = value.traces[i];
+          trace.apiEndpoint = apiEndpoint;
+          await apiTraceRepository.save(trace);
+        }
+      })
+    }
+  }
+}

backend/src/utils/index.ts

@@ -0,0 +1,11 @@
+import validator from "validator";
+
+export const isSuspectedParamater = (value: string) => {
+  if (!isNaN(Number(value))) {
+    return true;
+  }
+  if (validator.isUUID(value)) {
+    return true;
+  }
+  return false;
+}

backend/yarn.lock

@@ -114,6 +114,11 @@
     "@types/mime" "^1"
     "@types/node" "*"
 
+"@types/validator@^13.7.4":
+  version "13.7.4"
+  resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.7.4.tgz#33cc949ee87dd47c63e35ba4ad94f6888852be04"
+  integrity sha512-uAaSWegu2lymY18l+s5nmcXu3sFeeTOl1zhSGoYzcr6T3wz1M+3OcW4UjfPhIhHGd13tIMRDsEpR+d8w/MexwQ==
+
 abbrev@1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
@@ -1187,6 +1192,11 @@ v8-compile-cache-lib@^3.0.1:
   resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"
   integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==
 
+validator@^13.7.0:
+  version "13.7.0"
+  resolved "https://registry.yarnpkg.com/validator/-/validator-13.7.0.tgz#4f9658ba13ba8f3d82ee881d3516489ea85c0857"
+  integrity sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw==
+
 vary@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"