From e793b59a4e9eb9347460031cf2694bd6c6ecee6f Mon Sep 17 00:00:00 2001 From: Nikhil Shahi Date: Wed, 21 Sep 2022 20:01:04 -0500 Subject: [PATCH] split up hourly and minutely aggregate trace data tables --- backend/src/data-source.ts | 6 +- .../src/models/aggregate-trace-data-hourly.ts | 30 +++++++ ...ta.ts => aggregate-trace-data-minutely.ts} | 4 +- backend/src/models/index.ts | 9 ++- backend/src/services/get-endpoints/index.ts | 7 +- backend/src/services/jobs/index.ts | 79 +++++++++++++------ backend/src/services/spec/index.ts | 39 ++++++--- backend/src/services/summary/usageStats.ts | 2 +- 8 files changed, 130 insertions(+), 46 deletions(-) create mode 100644 backend/src/models/aggregate-trace-data-hourly.ts rename backend/src/models/{aggregate-trace-data.ts => aggregate-trace-data-minutely.ts} (83%) diff --git a/backend/src/data-source.ts b/backend/src/data-source.ts index 1bffd046..fcc7d7eb 100644 --- a/backend/src/data-source.ts +++ b/backend/src/data-source.ts @@ -12,8 +12,9 @@ import { ApiKey, BlockFields, InstanceSettings, - AggregateTraceData, + AggregateTraceDataMinutely, AuthenticationConfig, + AggregateTraceDataHourly, } from "models" export const AppDataSource: DataSource = new DataSource({ @@ -32,8 +33,9 @@ export const AppDataSource: DataSource = new DataSource({ ApiKey, BlockFields, InstanceSettings, - AggregateTraceData, + AggregateTraceDataMinutely, AuthenticationConfig, + AggregateTraceDataHourly, ], migrations: [], logging: false, diff --git a/backend/src/models/aggregate-trace-data-hourly.ts b/backend/src/models/aggregate-trace-data-hourly.ts new file mode 100644 index 00000000..ffeba0c3 --- /dev/null +++ b/backend/src/models/aggregate-trace-data-hourly.ts @@ -0,0 +1,30 @@ +import { + Entity, + Unique, + BaseEntity, + PrimaryGeneratedColumn, + Column, + ManyToOne, + JoinColumn, +} from "typeorm" +import { ApiEndpoint } from "./api-endpoint" + +@Entity() +@Unique("unique_constraint_hourly", ["apiEndpoint", "hour"]) +export class AggregateTraceDataHourly extends BaseEntity { + @PrimaryGeneratedColumn("uuid") + uuid: string + + @Column() + numCalls: number + + @Column({ type: "timestamptz" }) + hour: Date + + @Column() + apiEndpointUuid: string + + @ManyToOne(() => ApiEndpoint) + @JoinColumn() + apiEndpoint: ApiEndpoint +} diff --git a/backend/src/models/aggregate-trace-data.ts b/backend/src/models/aggregate-trace-data-minutely.ts similarity index 83% rename from backend/src/models/aggregate-trace-data.ts rename to backend/src/models/aggregate-trace-data-minutely.ts index 57489e4e..ac81669e 100644 --- a/backend/src/models/aggregate-trace-data.ts +++ b/backend/src/models/aggregate-trace-data-minutely.ts @@ -10,8 +10,8 @@ import { import { ApiEndpoint } from "./api-endpoint" @Entity() -@Unique("unique_constraint", ["apiEndpoint", "minute"]) -export class AggregateTraceData extends BaseEntity { +@Unique("unique_constraint_minutely", ["apiEndpoint", "minute"]) +export class AggregateTraceDataMinutely extends BaseEntity { @PrimaryGeneratedColumn("uuid") uuid: string diff --git a/backend/src/models/index.ts b/backend/src/models/index.ts index 413b8aec..83bc058f 100644 --- a/backend/src/models/index.ts +++ b/backend/src/models/index.ts @@ -9,8 +9,9 @@ import { ApiEndpointTest } from "./api-endpoint-test" import { ApiKey } from "./keys" import { BlockFields } from "./block-fields" import { InstanceSettings } from "./instance-settings" -import { AggregateTraceData } from "./aggregate-trace-data" +import { AggregateTraceDataMinutely } from "./aggregate-trace-data-minutely" import { AuthenticationConfig } from "./authentication-config" +import { AggregateTraceDataHourly } from "./aggregate-trace-data-hourly" export type DatabaseModel = | ApiEndpoint @@ -24,8 +25,9 @@ export type DatabaseModel = | ApiKey | BlockFields | InstanceSettings - | AggregateTraceData + | AggregateTraceDataMinutely | AuthenticationConfig + | AggregateTraceDataHourly export { ApiEndpoint, @@ -39,6 +41,7 @@ export { ApiKey, BlockFields, InstanceSettings, - AggregateTraceData, + AggregateTraceDataMinutely, AuthenticationConfig, + AggregateTraceDataHourly, } diff --git a/backend/src/services/get-endpoints/index.ts b/backend/src/services/get-endpoints/index.ts index 9458cba4..6b741782 100644 --- a/backend/src/services/get-endpoints/index.ts +++ b/backend/src/services/get-endpoints/index.ts @@ -1,10 +1,10 @@ import { FindOptionsWhere, In, ILike, Raw } from "typeorm" import { AppDataSource } from "data-source" import { - AggregateTraceData, ApiEndpoint, ApiEndpointTest, ApiTrace, + AggregateTraceDataHourly, } from "models" import { GetEndpointParams, @@ -165,8 +165,9 @@ export class GetEndpointsService { static async getUsage(endpointId: string): Promise { try { - const aggregateTraceDataRepo = - AppDataSource.getRepository(AggregateTraceData) + const aggregateTraceDataRepo = AppDataSource.getRepository( + AggregateTraceDataHourly, + ) const usage = await aggregateTraceDataRepo .createQueryBuilder("trace") .select([`DATE_TRUNC('day', hour) AS date`, `SUM("numCalls") AS count`]) diff --git a/backend/src/services/jobs/index.ts b/backend/src/services/jobs/index.ts index 2b2a785a..ad27737c 100644 --- a/backend/src/services/jobs/index.ts +++ b/backend/src/services/jobs/index.ts @@ -9,13 +9,7 @@ import { parsedJson, parsedJsonNonNull, } from "utils" -import { - ApiEndpoint, - ApiTrace, - OpenApiSpec, - Alert, - AggregateTraceData, -} from "models" +import { ApiEndpoint, ApiTrace, OpenApiSpec, Alert } from "models" import { AppDataSource } from "data-source" import { AlertType, DataType, RestMethod, SpecExtension } from "@common/enums" import { getPathTokens } from "@common/utils" @@ -123,6 +117,18 @@ export class JobsService { .where('"apiEndpointUuid" IS NOT NULL') .andWhere('"createdAt" < :oneHourAgo', { oneHourAgo }) + const aggregateTracesDataHourlyQb = await queryRunner.manager + .createQueryBuilder(ApiTrace, "trace") + .select([ + '"apiEndpointUuid"', + `DATE_TRUNC('hour', "createdAt") as hour`, + 'COUNT(*) as "numTraces"', + ]) + .where('"apiEndpointUuid" IS NOT NULL') + .andWhere('"createdAt" < :oneHourAgo', { oneHourAgo }) + .groupBy('"apiEndpointUuid"') + .addGroupBy("hour") + const tracesBySecondStatus = ` WITH traces_by_second_status AS ( SELECT @@ -176,7 +182,7 @@ export class JobsService { GROUP BY 1, 2 ) ` - const aggregateTracesDataQuery = ` + const aggregateTracesDataMinutelyQuery = ` ${tracesBySecondStatus}, ${tracesByMinuteStatus}, ${tracesByMinute}, @@ -194,15 +200,15 @@ export class JobsService { traces.minute = status_code_map.minute AND traces."apiEndpointUuid" = status_code_map."apiEndpointUuid" ` - const aggregateTracesData: any[] = await queryRunner.query( - aggregateTracesDataQuery, + const aggregateTracesDataMinutely: any[] = await queryRunner.query( + aggregateTracesDataMinutelyQuery, [oneHourAgo], ) - const parameters: any[] = [] - const argArray: string[] = [] + const parametersMinutely: any[] = [] + const argArrayMinutely: string[] = [] let argNumber = 1 - aggregateTracesData.forEach(data => { - parameters.push( + aggregateTracesDataMinutely.forEach(data => { + parametersMinutely.push( uuidv4(), data.numTraces, data.minute, @@ -212,21 +218,48 @@ export class JobsService { data.countByStatusCode, data.apiEndpointUuid, ) - argArray.push( + argArrayMinutely.push( `($${argNumber++}, $${argNumber++}, $${argNumber++}, $${argNumber++}, $${argNumber++}, $${argNumber++}, $${argNumber++}, $${argNumber++})`, ) }) - const argString = argArray.join(",") - const insertQuery = ` - INSERT INTO aggregate_trace_data ("uuid", "numCalls", "minute", "maxRPS", "minRPS", "meanRPS", "countByStatusCode", "apiEndpointUuid") - VALUES ${argString} - ON CONFLICT ON CONSTRAINT unique_constraint - DO UPDATE SET "numCalls" = EXCLUDED."numCalls" + aggregate_trace_data."numCalls"; + const aggregateTracesDataHourly = + await aggregateTracesDataHourlyQb.getRawMany() + const parametersHourly: any[] = [] + const argArrayHourly: string[] = [] + argNumber = 1 + aggregateTracesDataHourly.forEach(data => { + parametersHourly.push( + uuidv4(), + data.numTraces, + data.hour, + data.apiEndpointUuid, + ) + argArrayHourly.push( + `($${argNumber++}, $${argNumber++}, $${argNumber++}, $${argNumber++})`, + ) + }) + + const argStringMinutely = argArrayMinutely.join(",") + const insertQueryMinutely = ` + INSERT INTO aggregate_trace_data_minutely ("uuid", "numCalls", "minute", "maxRPS", "minRPS", "meanRPS", "countByStatusCode", "apiEndpointUuid") + VALUES ${argStringMinutely} + ON CONFLICT ON CONSTRAINT unique_constraint_minutely + DO UPDATE SET "numCalls" = EXCLUDED."numCalls" + aggregate_trace_data_minutely."numCalls"; + ` + const argStringHourly = argArrayHourly.join(",") + const insertQueryHourly = ` + INSERT INTO aggregate_trace_data_hourly ("uuid", "numCalls", "hour", "apiEndpointUuid") + VALUES ${argStringHourly} + ON CONFLICT ON CONSTRAINT unique_constraint_hourly + DO UPDATE SET "numCalls" = EXCLUDED."numCalls" + aggregate_trace_data_hourly."numCalls"; ` await deleteTracesQb.execute() - if (parameters.length > 0) { - await queryRunner.query(insertQuery, parameters) + if (parametersMinutely.length > 0) { + await queryRunner.query(insertQueryMinutely, parametersMinutely) + } + if (parametersHourly.length > 0) { + await queryRunner.query(insertQueryHourly, parametersHourly) } } catch (err) { console.error(`Encountered error while clearing trace data: ${err}`) diff --git a/backend/src/services/spec/index.ts b/backend/src/services/spec/index.ts index c86da9f6..18b6c7fd 100644 --- a/backend/src/services/spec/index.ts +++ b/backend/src/services/spec/index.ts @@ -15,7 +15,8 @@ import { DataField, OpenApiSpec, Alert, - AggregateTraceData, + AggregateTraceDataMinutely, + AggregateTraceDataHourly, } from "models" import { JSONValue, OpenApiSpec as OpenApiSpecResponse } from "@common/types" import { getPathTokens } from "@common/utils" @@ -148,8 +149,12 @@ export class SpecService { const apiEndpointRepository = AppDataSource.getRepository(ApiEndpoint) const openApiSpecRepository = AppDataSource.getRepository(OpenApiSpec) const apiTraceRepository = AppDataSource.getRepository(ApiTrace) - const aggregateTraceDataRepository = - AppDataSource.getRepository(AggregateTraceData) + const aggregateTraceDataMinutelyRepository = AppDataSource.getRepository( + AggregateTraceDataMinutely, + ) + const aggregateTraceDataHourlyRepository = AppDataSource.getRepository( + AggregateTraceDataHourly, + ) let existingSpec = await openApiSpecRepository.findOneBy({ name: fileName, @@ -165,7 +170,8 @@ export class SpecService { similarEndpoints: ApiEndpoint[] apiEndpoints: ApiEndpoint[] traces: ApiTrace[] - aggregateData: AggregateTraceData[] + aggregateDataMinutely: AggregateTraceDataMinutely[] + aggregateDataHourly: AggregateTraceDataHourly[] dataFields: DataField[] alertsToKeep: Alert[] alertsToRemove: Alert[] @@ -173,7 +179,8 @@ export class SpecService { similarEndpoints: [], apiEndpoints: [], traces: [], - aggregateData: [], + aggregateDataMinutely: [], + aggregateDataHourly: [], dataFields: [], alertsToKeep: [], alertsToRemove: [], @@ -253,11 +260,14 @@ export class SpecService { const traces = await apiTraceRepository.findBy({ apiEndpointUuid: endpoint.uuid, }) - const aggregateData = await aggregateTraceDataRepository.findBy( - { + const aggregateDataMinutely = + await aggregateTraceDataMinutelyRepository.findBy({ apiEndpointUuid: endpoint.uuid, - }, - ) + }) + const aggregateDataHourly = + await aggregateTraceDataHourlyRepository.findBy({ + apiEndpointUuid: endpoint.uuid, + }) endpoint.dataFields.forEach(dataField => { dataField.apiEndpointUuid = apiEndpoint.uuid }) @@ -265,7 +275,10 @@ export class SpecService { trace.apiEndpointUuid = apiEndpoint.uuid apiEndpoint.updateDates(trace.createdAt) }) - aggregateData.forEach( + aggregateDataMinutely.forEach( + data => (data.apiEndpointUuid = apiEndpoint.uuid), + ) + aggregateDataHourly.forEach( data => (data.apiEndpointUuid = apiEndpoint.uuid), ) endpoint.alerts.forEach(alert => { @@ -283,7 +296,8 @@ export class SpecService { } }) endpoints.traces.push(...traces) - endpoints.aggregateData.push(...aggregateData) + endpoints.aggregateDataMinutely.push(...aggregateDataMinutely) + endpoints.aggregateDataHourly.push(...aggregateDataHourly) endpoints.dataFields.push(...endpoint.dataFields) } endpoints.similarEndpoints.push(...similarEndpoints) @@ -298,7 +312,8 @@ export class SpecService { [existingSpec], endpoints.apiEndpoints, endpoints.traces, - endpoints.aggregateData, + endpoints.aggregateDataMinutely, + endpoints.aggregateDataHourly, endpoints.dataFields, endpoints.alertsToKeep, ], diff --git a/backend/src/services/summary/usageStats.ts b/backend/src/services/summary/usageStats.ts index 85429d3f..e70522bf 100644 --- a/backend/src/services/summary/usageStats.ts +++ b/backend/src/services/summary/usageStats.ts @@ -7,7 +7,7 @@ export const getUsageStats = async () => { SELECT DATE_TRUNC('day', traces.hour) as day, SUM(traces."numCalls") as cnt - FROM aggregate_trace_data traces + FROM aggregate_trace_data_hourly traces WHERE traces.hour > (NOW() - INTERVAL '15 days') GROUP BY 1 ORDER BY 1