split up hourly and minutely aggregate trace data tables

backend/src/data-source.ts

@@ -12,8 +12,9 @@ import {
   ApiKey,
   BlockFields,
   InstanceSettings,
-  AggregateTraceData,
+  AggregateTraceDataMinutely,
   AuthenticationConfig,
+  AggregateTraceDataHourly,
 } from "models"
 
 export const AppDataSource: DataSource = new DataSource({
@@ -32,8 +33,9 @@ export const AppDataSource: DataSource = new DataSource({
     ApiKey,
     BlockFields,
     InstanceSettings,
-    AggregateTraceData,
+    AggregateTraceDataMinutely,
     AuthenticationConfig,
+    AggregateTraceDataHourly,
   ],
   migrations: [],
   logging: false,

backend/src/models/aggregate-trace-data-hourly.ts

@@ -0,0 +1,30 @@
+import {
+  Entity,
+  Unique,
+  BaseEntity,
+  PrimaryGeneratedColumn,
+  Column,
+  ManyToOne,
+  JoinColumn,
+} from "typeorm"
+import { ApiEndpoint } from "./api-endpoint"
+
+@Entity()
+@Unique("unique_constraint_hourly", ["apiEndpoint", "hour"])
+export class AggregateTraceDataHourly extends BaseEntity {
+  @PrimaryGeneratedColumn("uuid")
+  uuid: string
+
+  @Column()
+  numCalls: number
+
+  @Column({ type: "timestamptz" })
+  hour: Date
+
+  @Column()
+  apiEndpointUuid: string
+
+  @ManyToOne(() => ApiEndpoint)
+  @JoinColumn()
+  apiEndpoint: ApiEndpoint
+}

backend/src/models/aggregate-trace-data.ts → backend/src/models/aggregate-trace-data-minutely.ts

@@ -10,8 +10,8 @@ import {
 import { ApiEndpoint } from "./api-endpoint"
 
 @Entity()
-@Unique("unique_constraint", ["apiEndpoint", "minute"])
-export class AggregateTraceData extends BaseEntity {
+@Unique("unique_constraint_minutely", ["apiEndpoint", "minute"])
+export class AggregateTraceDataMinutely extends BaseEntity {
   @PrimaryGeneratedColumn("uuid")
   uuid: string
 

backend/src/models/index.ts

@@ -9,8 +9,9 @@ import { ApiEndpointTest } from "./api-endpoint-test"
 import { ApiKey } from "./keys"
 import { BlockFields } from "./block-fields"
 import { InstanceSettings } from "./instance-settings"
-import { AggregateTraceData } from "./aggregate-trace-data"
+import { AggregateTraceDataMinutely } from "./aggregate-trace-data-minutely"
 import { AuthenticationConfig } from "./authentication-config"
+import { AggregateTraceDataHourly } from "./aggregate-trace-data-hourly"
 
 export type DatabaseModel =
   | ApiEndpoint
@@ -24,8 +25,9 @@ export type DatabaseModel =
   | ApiKey
   | BlockFields
   | InstanceSettings
-  | AggregateTraceData
+  | AggregateTraceDataMinutely
   | AuthenticationConfig
+  | AggregateTraceDataHourly
 
 export {
   ApiEndpoint,
@@ -39,6 +41,7 @@ export {
   ApiKey,
   BlockFields,
   InstanceSettings,
-  AggregateTraceData,
+  AggregateTraceDataMinutely,
   AuthenticationConfig,
+  AggregateTraceDataHourly,
 }

backend/src/services/get-endpoints/index.ts

@@ -1,10 +1,10 @@
 import { FindOptionsWhere, In, ILike, Raw } from "typeorm"
 import { AppDataSource } from "data-source"
 import {
-  AggregateTraceData,
   ApiEndpoint,
   ApiEndpointTest,
   ApiTrace,
+  AggregateTraceDataHourly,
 } from "models"
 import {
   GetEndpointParams,
@@ -165,8 +165,9 @@ export class GetEndpointsService {
 
   static async getUsage(endpointId: string): Promise<UsageResponse[]> {
     try {
-      const aggregateTraceDataRepo =
-        AppDataSource.getRepository(AggregateTraceData)
+      const aggregateTraceDataRepo = AppDataSource.getRepository(
+        AggregateTraceDataHourly,
+      )
       const usage = await aggregateTraceDataRepo
         .createQueryBuilder("trace")
         .select([`DATE_TRUNC('day', hour) AS date`, `SUM("numCalls") AS count`])

backend/src/services/jobs/index.ts

@@ -9,13 +9,7 @@ import {
   parsedJson,
   parsedJsonNonNull,
 } from "utils"
-import {
-  ApiEndpoint,
-  ApiTrace,
-  OpenApiSpec,
-  Alert,
-  AggregateTraceData,
-} from "models"
+import { ApiEndpoint, ApiTrace, OpenApiSpec, Alert } from "models"
 import { AppDataSource } from "data-source"
 import { AlertType, DataType, RestMethod, SpecExtension } from "@common/enums"
 import { getPathTokens } from "@common/utils"
@@ -123,6 +117,18 @@ export class JobsService {
         .where('"apiEndpointUuid" IS NOT NULL')
         .andWhere('"createdAt" < :oneHourAgo', { oneHourAgo })
 
+      const aggregateTracesDataHourlyQb = await queryRunner.manager
+        .createQueryBuilder(ApiTrace, "trace")
+        .select([
+          '"apiEndpointUuid"',
+          `DATE_TRUNC('hour', "createdAt") as hour`,
+          'COUNT(*) as "numTraces"',
+        ])
+        .where('"apiEndpointUuid" IS NOT NULL')
+        .andWhere('"createdAt" < :oneHourAgo', { oneHourAgo })
+        .groupBy('"apiEndpointUuid"')
+        .addGroupBy("hour")
+
       const tracesBySecondStatus = `
         WITH traces_by_second_status AS (
           SELECT
@@ -176,7 +182,7 @@ export class JobsService {
           GROUP BY 1, 2
         )
       `
-      const aggregateTracesDataQuery = `
+      const aggregateTracesDataMinutelyQuery = `
         ${tracesBySecondStatus},
         ${tracesByMinuteStatus},
         ${tracesByMinute},
@@ -194,15 +200,15 @@ export class JobsService {
           traces.minute = status_code_map.minute AND traces."apiEndpointUuid" = status_code_map."apiEndpointUuid"
       `
 
-      const aggregateTracesData: any[] = await queryRunner.query(
-        aggregateTracesDataQuery,
+      const aggregateTracesDataMinutely: any[] = await queryRunner.query(
+        aggregateTracesDataMinutelyQuery,
         [oneHourAgo],
       )
-      const parameters: any[] = []
-      const argArray: string[] = []
+      const parametersMinutely: any[] = []
+      const argArrayMinutely: string[] = []
       let argNumber = 1
-      aggregateTracesData.forEach(data => {
-        parameters.push(
+      aggregateTracesDataMinutely.forEach(data => {
+        parametersMinutely.push(
           uuidv4(),
           data.numTraces,
           data.minute,
@@ -212,21 +218,48 @@ export class JobsService {
           data.countByStatusCode,
           data.apiEndpointUuid,
         )
-        argArray.push(
+        argArrayMinutely.push(
           `($${argNumber++}, $${argNumber++}, $${argNumber++}, $${argNumber++}, $${argNumber++}, $${argNumber++}, $${argNumber++}, $${argNumber++})`,
         )
       })
 
-      const argString = argArray.join(",")
-      const insertQuery = `
-        INSERT INTO aggregate_trace_data ("uuid", "numCalls", "minute", "maxRPS", "minRPS", "meanRPS", "countByStatusCode", "apiEndpointUuid")
-        VALUES ${argString}
-        ON CONFLICT ON CONSTRAINT unique_constraint
-        DO UPDATE SET "numCalls" = EXCLUDED."numCalls" + aggregate_trace_data."numCalls";
+      const aggregateTracesDataHourly =
+        await aggregateTracesDataHourlyQb.getRawMany()
+      const parametersHourly: any[] = []
+      const argArrayHourly: string[] = []
+      argNumber = 1
+      aggregateTracesDataHourly.forEach(data => {
+        parametersHourly.push(
+          uuidv4(),
+          data.numTraces,
+          data.hour,
+          data.apiEndpointUuid,
+        )
+        argArrayHourly.push(
+          `($${argNumber++}, $${argNumber++}, $${argNumber++}, $${argNumber++})`,
+        )
+      })
+
+      const argStringMinutely = argArrayMinutely.join(",")
+      const insertQueryMinutely = `
+        INSERT INTO aggregate_trace_data_minutely ("uuid", "numCalls", "minute", "maxRPS", "minRPS", "meanRPS", "countByStatusCode", "apiEndpointUuid")
+        VALUES ${argStringMinutely}
+        ON CONFLICT ON CONSTRAINT unique_constraint_minutely
+        DO UPDATE SET "numCalls" = EXCLUDED."numCalls" + aggregate_trace_data_minutely."numCalls";
+      `
+      const argStringHourly = argArrayHourly.join(",")
+      const insertQueryHourly = `
+        INSERT INTO aggregate_trace_data_hourly ("uuid", "numCalls", "hour", "apiEndpointUuid")
+        VALUES ${argStringHourly}
+        ON CONFLICT ON CONSTRAINT unique_constraint_hourly
+        DO UPDATE SET "numCalls" = EXCLUDED."numCalls" + aggregate_trace_data_hourly."numCalls";
       `
       await deleteTracesQb.execute()
-      if (parameters.length > 0) {
-        await queryRunner.query(insertQuery, parameters)
+      if (parametersMinutely.length > 0) {
+        await queryRunner.query(insertQueryMinutely, parametersMinutely)
+      }
+      if (parametersHourly.length > 0) {
+        await queryRunner.query(insertQueryHourly, parametersHourly)
       }
     } catch (err) {
       console.error(`Encountered error while clearing trace data: ${err}`)

backend/src/services/spec/index.ts

@@ -15,7 +15,8 @@ import {
   DataField,
   OpenApiSpec,
   Alert,
-  AggregateTraceData,
+  AggregateTraceDataMinutely,
+  AggregateTraceDataHourly,
 } from "models"
 import { JSONValue, OpenApiSpec as OpenApiSpecResponse } from "@common/types"
 import { getPathTokens } from "@common/utils"
@@ -148,8 +149,12 @@ export class SpecService {
     const apiEndpointRepository = AppDataSource.getRepository(ApiEndpoint)
     const openApiSpecRepository = AppDataSource.getRepository(OpenApiSpec)
     const apiTraceRepository = AppDataSource.getRepository(ApiTrace)
-    const aggregateTraceDataRepository =
-      AppDataSource.getRepository(AggregateTraceData)
+    const aggregateTraceDataMinutelyRepository = AppDataSource.getRepository(
+      AggregateTraceDataMinutely,
+    )
+    const aggregateTraceDataHourlyRepository = AppDataSource.getRepository(
+      AggregateTraceDataHourly,
+    )
 
     let existingSpec = await openApiSpecRepository.findOneBy({
       name: fileName,
@@ -165,15 +170,17 @@ export class SpecService {
       similarEndpoints: ApiEndpoint[]
       apiEndpoints: ApiEndpoint[]
       traces: ApiTrace[]
-      aggregateData: AggregateTraceData[]
+      aggregateDataMinutely: AggregateTraceDataMinutely[]
+      aggregateDataHourly: AggregateTraceDataHourly[]
       dataFields: DataField[]
       alertsToKeep: Alert[]
       alertsToRemove: Alert[]
     } = {
       similarEndpoints: [],
       apiEndpoints: [],
       traces: [],
-      aggregateData: [],
+      aggregateDataMinutely: [],
+      aggregateDataHourly: [],
       dataFields: [],
       alertsToKeep: [],
       alertsToRemove: [],
@@ -253,19 +260,25 @@ export class SpecService {
                 const traces = await apiTraceRepository.findBy({
                   apiEndpointUuid: endpoint.uuid,
                 })
-                const aggregateData = await aggregateTraceDataRepository.findBy(
-                  {
+                const aggregateDataMinutely =
+                  await aggregateTraceDataMinutelyRepository.findBy({
                     apiEndpointUuid: endpoint.uuid,
-                  },
-                )
+                  })
+                const aggregateDataHourly =
+                  await aggregateTraceDataHourlyRepository.findBy({
+                    apiEndpointUuid: endpoint.uuid,
+                  })
                 endpoint.dataFields.forEach(dataField => {
                   dataField.apiEndpointUuid = apiEndpoint.uuid
                 })
                 traces.forEach(trace => {
                   trace.apiEndpointUuid = apiEndpoint.uuid
                   apiEndpoint.updateDates(trace.createdAt)
                 })
-                aggregateData.forEach(
+                aggregateDataMinutely.forEach(
+                  data => (data.apiEndpointUuid = apiEndpoint.uuid),
+                )
+                aggregateDataHourly.forEach(
                   data => (data.apiEndpointUuid = apiEndpoint.uuid),
                 )
                 endpoint.alerts.forEach(alert => {
@@ -283,7 +296,8 @@ export class SpecService {
                   }
                 })
                 endpoints.traces.push(...traces)
-                endpoints.aggregateData.push(...aggregateData)
+                endpoints.aggregateDataMinutely.push(...aggregateDataMinutely)
+                endpoints.aggregateDataHourly.push(...aggregateDataHourly)
                 endpoints.dataFields.push(...endpoint.dataFields)
               }
               endpoints.similarEndpoints.push(...similarEndpoints)
@@ -298,7 +312,8 @@ export class SpecService {
         [existingSpec],
         endpoints.apiEndpoints,
         endpoints.traces,
-        endpoints.aggregateData,
+        endpoints.aggregateDataMinutely,
+        endpoints.aggregateDataHourly,
         endpoints.dataFields,
         endpoints.alertsToKeep,
       ],

backend/src/services/summary/usageStats.ts

@@ -7,7 +7,7 @@ export const getUsageStats = async () => {
     SELECT
       DATE_TRUNC('day', traces.hour) as day,
       SUM(traces."numCalls") as cnt
-    FROM aggregate_trace_data traces
+    FROM aggregate_trace_data_hourly traces
     WHERE traces.hour > (NOW() - INTERVAL '15 days')
     GROUP BY 1
     ORDER BY 1