redo home page

backend/package.json

@@ -36,6 +36,8 @@
     "express-session": "^1.17.3",
     "ioredis": "^5.2.3",
     "js-yaml": "^4.1.0",
+    "lodash": "^4.17.21",
+    "memory-cache": "^0.2.0",
     "multer": "^1.4.5-lts.1",
     "newman": "^5.3.2",
     "node-schedule": "^2.1.0",
@@ -54,6 +56,8 @@
     "@types/express": "^4.17.13",
     "@types/express-session": "^1.17.5",
     "@types/js-yaml": "^4.0.5",
+    "@types/lodash": "^4.14.184",
+    "@types/memory-cache": "^0.2.2",
     "@types/multer": "^1.4.7",
     "@types/node": "^18.6.1",
     "@types/node-schedule": "^2.1.0",

backend/src/api/alert/index.ts

@@ -32,15 +32,3 @@ export const updateAlertHandler = async (
     await ApiResponseHandler.error(res, err)
   }
 }
-
-export const getTopAlertsHandler = async (
-  req: Request,
-  res: Response,
-): Promise<void> => {
-  try {
-    const topAlerts = await AlertService.getTopAlerts()
-    await ApiResponseHandler.success(res, topAlerts)
-  } catch (err) {
-    await ApiResponseHandler.error(res, err)
-  }
-}

backend/src/index.ts

@@ -21,11 +21,7 @@ import {
   updateSpecHandler,
   uploadNewSpecHandler,
 } from "api/spec"
-import {
-  getAlertsHandler,
-  getTopAlertsHandler,
-  updateAlertHandler,
-} from "api/alert"
+import { getAlertsHandler, updateAlertHandler } from "api/alert"
 import { updateDataFieldClasses } from "api/data-field"
 import { getSummaryHandler } from "api/summary"
 import { AppDataSource } from "data-source"
@@ -97,7 +93,6 @@ app.post(
 )
 
 app.get("/api/v1/alerts", getAlertsHandler)
-app.get("/api/v1/topAlerts", getTopAlertsHandler)
 app.put("/api/v1/alert/:alertId", updateAlertHandler)
 
 app.post("/api/v1/setup_connection", setup_connection)

backend/src/models/api-endpoint.ts

@@ -13,7 +13,7 @@ import {
 import { DataField } from "models/data-field"
 import { Alert } from "models/alert"
 import { OpenApiSpec } from "models/openapi-spec"
-import { DataClass, DataSection, RestMethod, RiskScore } from "@common/enums"
+import { RestMethod, RiskScore } from "@common/enums"
 import { getPathTokens, isParameter } from "utils"
 
 @Entity()

backend/src/services/alert/index.ts

@@ -146,23 +146,6 @@ export class AlertService {
     return alerts
   }
 
-  static async getTopAlerts(): Promise<AlertResponse[]> {
-    const alertRepository = AppDataSource.getRepository(Alert)
-    return await alertRepository.find({
-      where: {
-        status: Status.OPEN,
-      },
-      relations: {
-        apiEndpoint: true,
-      },
-      order: {
-        riskScore: "DESC",
-        createdAt: "DESC",
-      },
-      take: 20,
-    })
-  }
-
   static async getAlert(alertId: string): Promise<AlertResponse> {
     const alertRepository = AppDataSource.getRepository(Alert)
     return await alertRepository.findOneBy({ uuid: alertId })

backend/src/services/summary/alerts.ts

@@ -0,0 +1,52 @@
+import { Status, AlertType } from "@common/enums"
+import { AppDataSource } from "data-source"
+import { Alert } from "models"
+import cache from "memory-cache"
+
+export const getAlertTypeAgg = async () => {
+  const queryRunner = AppDataSource.createQueryRunner()
+  const alertTypeAggRes: { type: AlertType; count: number }[] =
+    await queryRunner.manager.query(`
+    SELECT type, CAST(COUNT(*) AS INTEGER) as count
+    FROM alert WHERE status = 'Open'
+    GROUP BY 1
+  `)
+  return Object.fromEntries(alertTypeAggRes.map(e => [e.type, e.count]))
+}
+
+export const getAlertTypeAggCached = async () => {
+  const cacheRes: Record<AlertType, number> | null = cache.get("alertTypeAgg")
+  if (cacheRes) {
+    return cacheRes
+  }
+  const realRes = await getAlertTypeAgg()
+  cache.put("alertTypeAgg", realRes, 5000)
+  return realRes
+}
+
+export const getTopAlerts = async () => {
+  const alertRepository = AppDataSource.getRepository(Alert)
+  return await alertRepository.find({
+    where: {
+      status: Status.OPEN,
+    },
+    relations: {
+      apiEndpoint: true,
+    },
+    order: {
+      riskScore: "DESC",
+      createdAt: "DESC",
+    },
+    take: 10,
+  })
+}
+
+export const getTopAlertsCached = async () => {
+  const cacheRes: Alert[] | null = cache.get("topAlertsCached")
+  if (cacheRes) {
+    return cacheRes
+  }
+  const realRes = await getTopAlerts()
+  cache.put("topAlertsCached", realRes, 5000)
+  return realRes
+}

backend/src/services/summary/endpoints.ts

@@ -0,0 +1,78 @@
+import _ from "lodash"
+import { In } from "typeorm"
+import { AppDataSource } from "data-source"
+import { ApiEndpoint, ApiTrace } from "models"
+import { EndpointAndUsage } from "@common/types"
+import cache from "memory-cache"
+
+export const getTopEndpoints = async () => {
+  const queryRunner = AppDataSource.createQueryRunner()
+  const apiTraceRepository = AppDataSource.getRepository(ApiTrace)
+  const apiEndpointRepository = AppDataSource.getRepository(ApiEndpoint)
+  const endpointStats: {
+    endpoint: string
+    last1MinCnt: number
+    last5MinCnt: number
+    last30MinCnt: number
+  }[] = await queryRunner.manager.query(`
+      SELECT
+        traces."apiEndpointUuid" as endpoint,
+        CAST(COUNT(*) AS INTEGER) as "last30MinCnt",
+        CAST(SUM(CASE WHEN traces."createdAt" > (NOW() - INTERVAL '5 minutes') THEN 1 ELSE 0 END) AS INTEGER) as "last5MinCnt",
+        CAST(SUM(CASE WHEN traces."createdAt" > (NOW() - INTERVAL '1 minutes') THEN 1 ELSE 0 END) AS INTEGER) as "last1MinCnt"
+      FROM
+        api_trace traces
+      WHERE
+        traces."apiEndpointUuid" IS NOT NULL
+        AND traces."createdAt" > (NOW() - INTERVAL '30 minutes')
+      GROUP BY 1
+      ORDER BY 4 DESC
+      LIMIT 10
+    `)
+
+  const endpoints = await apiEndpointRepository.find({
+    where: { uuid: In(endpointStats.map(e => e.endpoint)) },
+    relations: {
+      dataFields: true,
+    },
+    order: {
+      dataFields: {
+        dataTag: "ASC",
+        dataPath: "ASC",
+      },
+    },
+  })
+
+  const traces = await Promise.all(
+    endpointStats.map(e =>
+      apiTraceRepository.find({
+        where: { apiEndpointUuid: e.endpoint },
+        order: { createdAt: "DESC" },
+        take: 25,
+      }),
+    ),
+  )
+  const traceMap = _.groupBy(traces.flat(), e => e.apiEndpointUuid)
+
+  return endpointStats.map(
+    stats =>
+      ({
+        ...endpoints.find(e => e.uuid == stats.endpoint),
+        last30MinCnt: stats.last30MinCnt,
+        last5MinCnt: stats.last5MinCnt,
+        last1MinCnt: stats.last1MinCnt,
+        traces: traceMap[stats.endpoint],
+        tests: [],
+      } as EndpointAndUsage),
+  )
+}
+
+export const getTopEndpointsCached = async () => {
+  const cacheRes: EndpointAndUsage[] | null = cache.get("endpointUsageStats")
+  if (cacheRes) {
+    return cacheRes
+  }
+  const realRes = await getTopEndpoints()
+  cache.put("endpointUsageStats", realRes, 15000)
+  return realRes
+}

backend/src/services/summary/index.ts

@@ -1,27 +1,24 @@
-import { DataTag, RiskScore, Status } from "@common/enums"
-import { Alert, ApiEndpoint, DataField } from "models"
-import { AppDataSource } from "data-source"
 import { Summary as SummaryResponse } from "@common/types"
+import { getAlertTypeAggCached, getTopAlertsCached } from "./alerts"
+import { getTopEndpointsCached } from "./endpoints"
+import { getPIIDataTypeCountCached } from "./piiData"
+import { getCountsCached, getUsageStatsCached } from "./usageStats"
 
 export class SummaryService {
   static async getSummaryData(): Promise<SummaryResponse> {
-    const alertRepository = AppDataSource.getRepository(Alert)
-    const apiEndpointRepository = AppDataSource.getRepository(ApiEndpoint)
-    const dataFieldRepository = AppDataSource.getRepository(DataField)
-    const highRiskAlerts = await alertRepository.countBy({
-      riskScore: RiskScore.HIGH,
-      status: Status.OPEN,
-    })
-    const newAlerts = await alertRepository.countBy({ status: Status.OPEN })
-    const endpointsTracked = await apiEndpointRepository.count({})
-    const piiDataFields = await dataFieldRepository.countBy({
-      dataTag: DataTag.PII,
-    })
+    const topEndpoints = await getTopEndpointsCached()
+    const alertTypeCount = await getAlertTypeAggCached()
+    const topAlerts = await getTopAlertsCached()
+    const piiDataTypeCount = await getPIIDataTypeCountCached()
+    const usageStats = await getUsageStatsCached()
+    const counts = await getCountsCached()
     return {
-      highRiskAlerts,
-      newAlerts,
-      endpointsTracked,
-      piiDataFields,
+      piiDataTypeCount: piiDataTypeCount as any,
+      alertTypeCount: alertTypeCount as any,
+      topAlerts,
+      topEndpoints,
+      usageStats,
+      ...counts,
     }
   }
 }

backend/src/services/summary/piiData.ts

@@ -0,0 +1,24 @@
+import { DataClass } from "@common/enums"
+import { AppDataSource } from "data-source"
+import cache from "memory-cache"
+
+export const getPIIDataTypeCount = async () => {
+  const queryRunner = AppDataSource.createQueryRunner()
+  const piiDataTypeCountRes: { type: DataClass; cnt: number }[] =
+    await queryRunner.manager.query(`
+    SELECT data_class as type, CAST(COUNT(*) AS INTEGER) as cnt
+    FROM (SELECT UNNEST("dataClasses") as data_class FROM data_field) tbl
+    GROUP BY 1
+  `)
+  return Object.fromEntries(piiDataTypeCountRes.map(e => [e.type, e.cnt]))
+}
+
+export const getPIIDataTypeCountCached = async () => {
+  const cacheRes: Record<DataClass, number> | null = cache.get("PIIDataTypeCount")
+  if (cacheRes) {
+    return cacheRes
+  }
+  const realRes = await getPIIDataTypeCount()
+  cache.put("PIIDataTypeCount", realRes, 5000)
+  return realRes
+}

backend/src/services/summary/usageStats.ts

@@ -0,0 +1,94 @@
+import { AppDataSource } from "data-source"
+import { UsageStats } from "@common/types"
+import cache from "memory-cache"
+
+export const getUsageStats = async () => {
+  const queryRunner = AppDataSource.createQueryRunner()
+  const stats: {
+    day: string
+    cnt: number
+  }[] = await queryRunner.manager.query(`
+  SELECT
+    DATE_TRUNC('day', traces."createdAt") as day,
+    COUNT(*) as cnt
+  FROM api_trace traces
+  WHERE traces."createdAt" > (NOW() - INTERVAL '15 days')
+  GROUP BY 1
+  ORDER BY 1
+  `)
+  const lastNRequests: {
+    last1MinCnt: number
+    last60MinCnt: number
+  } = await queryRunner.manager.query(`
+  SELECT
+    CAST(SUM(CASE WHEN traces."createdAt" > (NOW() - INTERVAL '1 minutes') THEN 1 ELSE 0 END) AS INTEGER) as "last1MinCnt",
+    CAST(COUNT(*) AS INTEGER) as "last60MinCnt"
+  FROM api_trace traces
+  WHERE traces."createdAt" > (NOW() - INTERVAL '60 minutes')
+  `)
+  return {
+    dailyUsage: stats,
+    last1MinCnt: lastNRequests[0].last1MinCnt,
+    last60MinCnt: lastNRequests[0].last60MinCnt,
+  } as UsageStats
+}
+
+export const getUsageStatsCached = async () => {
+  const cacheRes: UsageStats | null = cache.get("usageStats")
+  if (cacheRes) {
+    return cacheRes
+  }
+  const realRes = await getUsageStats()
+  cache.put("usageStats", realRes, 60000)
+  return realRes
+}
+
+interface CountsResponse {
+  newAlerts: number
+  endpointsTracked: number
+  piiDataFields: number
+  hostCount: number
+  highRiskAlerts: number
+}
+
+export const getCounts = async () => {
+  const queryRunner = AppDataSource.createQueryRunner()
+  const newAlertQueryRes = await queryRunner.manager.query(`
+    SELECT
+      CAST(COUNT(*) AS INTEGER) as count,
+      CAST(SUM(CASE WHEN "riskScore" = 'high' THEN 1 ELSE 0 END) AS INTEGER) as high_risk_count
+    FROM alert WHERE status = 'Open'
+  `)
+  const newAlerts = newAlertQueryRes[0].count
+  const highRiskAlerts = newAlertQueryRes[0].high_risk_count
+  const endpointsTrackedQueryRes = await queryRunner.manager.query(`
+    SELECT
+      CAST(COUNT(*) AS INTEGER) as endpoint_count,
+      CAST(COUNT(DISTINCT(host)) AS INTEGER) as host_count
+    FROM api_endpoint
+  `)
+  const endpointsTracked = endpointsTrackedQueryRes[0].endpoint_count
+  const hostCount = endpointsTrackedQueryRes[0].host_count
+  const piiDataFieldsQueryRes = await queryRunner.manager.query(`
+      SELECT CAST(COUNT(*) AS INTEGER) as count
+      FROM data_field WHERE "dataTag" = 'PII'
+  `)
+  const piiDataFields = piiDataFieldsQueryRes[0].count
+  return {
+    newAlerts,
+    endpointsTracked,
+    piiDataFields,
+    hostCount,
+    highRiskAlerts,
+  }
+}
+
+export const getCountsCached = async () => {
+  const cacheRes: CountsResponse | null = cache.get("usageCounts")
+  if (cacheRes) {
+    return cacheRes
+  }
+  const realRes = await getCounts()
+  cache.put("usageCounts", realRes, 5000)
+  return realRes
+}