feature: editable metlo-config

backend/src/api/metlo-config/index.ts

@@ -0,0 +1,34 @@
+import { Response } from "express"
+import { UpdateMetloConfigParams } from "@common/types"
+import { getMetloConfig, updateMetloConfig } from "services/metlo-config"
+import ApiResponseHandler from "api-response-handler"
+import Error404NotFound from "errors/error-404-not-found"
+import { MetloRequest } from "types"
+
+export const updateMetloConfigHandler = async (
+  req: MetloRequest,
+  res: Response,
+): Promise<void> => {
+  try {
+    const updateMetloConfigParams: UpdateMetloConfigParams = req.body
+    await updateMetloConfig(req.ctx, updateMetloConfigParams)
+    await ApiResponseHandler.success(res, null)
+  } catch (err) {
+    await ApiResponseHandler.error(res, err)
+  }
+}
+
+export const getMetloConfigHandler = async (
+  req: MetloRequest,
+  res: Response,
+): Promise<void> => {
+  try {
+    const metloConfig = await getMetloConfig(req.ctx)
+    if (!metloConfig) {
+      throw new Error404NotFound("No config saved yet.")
+    }
+    await ApiResponseHandler.success(res, metloConfig)
+  } catch (err) {
+    await ApiResponseHandler.error(res, err)
+  }
+}

backend/src/collector.ts

@@ -2,16 +2,18 @@ import dotenv from "dotenv"
 dotenv.config()
 
 import express, { Express, Request, Response } from "express"
+import fs from "fs"
 import { AppDataSource } from "data-source"
 import {
   logRequestBatchHandler,
   logRequestSingleHandler,
 } from "collector_src/log-request"
 import { verifyApiKeyMiddleware } from "middleware/verify-api-key-middleware"
 import { bodyParserMiddleware } from "middleware/body-parser-middleware"
-import { populateBlockFields } from "collector_src/block-fields"
-import { populateAuthentication } from "collector_src/authentication"
 import { MetloContext, MetloRequest } from "types"
+import { populateMetloConfig } from "services/metlo-config"
+import { MetloConfig } from "models/metlo-config"
+import { getRepoQB } from "services/database/utils"
 
 const app: Express = express()
 const port = process.env.PORT || 8081
@@ -42,11 +44,19 @@ const main = async () => {
         datasource.isInitialized ? "Yes" : "No"
       }`,
     )
+    try {
+      const ctx: MetloContext = {}
+      const configString = fs.readFileSync("./metlo-config.yaml", "utf-8")
+      const existingMetloConfig = await getRepoQB(ctx, MetloConfig)
+        .select(["uuid"])
+        .getRawOne()
+      if (configString?.length > 0 && !existingMetloConfig) {
+        await populateMetloConfig(ctx, configString)
+      }
+    } catch (err) {}
     app.listen(port, () => {
       console.log(`⚡️[server]: Server is running at http://localhost:${port}`)
     })
-    const ctx: MetloContext = {}
-    await Promise.all([populateBlockFields(), populateAuthentication(ctx)])
   } catch (err) {
     console.error(`CatchBlockInsideMain: ${err}`)
   }

backend/src/constants.ts

@@ -33,6 +33,8 @@ export const EXTENSION_TO_MIME_TYPE: Record<SpecExtension, string[]> = {
 
 export const AUTH_CONFIG_LIST_KEY = "auth_config_list"
 
+export const BLOCK_FIELDS_LIST_KEY = "block_fields_list"
+
 export const BLOCK_FIELDS_ALL_REGEX = "^.*$"
 
 export const TRACES_QUEUE = "traces_queue"

backend/src/data-source.ts

@@ -23,6 +23,9 @@ import { dropAnalyzedColumnFromApiTrace1666752646836 } from "migrations/16667526
 import { addIndexForDataField1666941075032 } from "migrations/1666941075032-add-index-for-data-field"
 import { addIsgraphqlColumnApiEndpoint1667095325334 } from "migrations/1667095325334-add-isgraphql-column-api-endpoint"
 import { addApiEndpointUuidIndexForAlert1667259254414 } from "migrations/1667259254414-add-apiEndpointUuid-index-for-alert"
+import { MetloConfig } from "models/metlo-config"
+import { addMetloConfigTable1667599667595 } from "migrations/1667599667595-add-metlo-config-table"
+import { updateDisabledPathsColumnBlockFieldsTable1667606447208 } from "migrations/1667606447208-update-disabledPaths-column-blockFields-table"
 
 export const AppDataSource: DataSource = new DataSource({
   type: "postgres",
@@ -42,6 +45,7 @@ export const AppDataSource: DataSource = new DataSource({
     AuthenticationConfig,
     AggregateTraceDataHourly,
     Attack,
+    MetloConfig,
   ],
   synchronize: false,
   migrations: [
@@ -51,6 +55,8 @@ export const AppDataSource: DataSource = new DataSource({
     addIndexForDataField1666941075032,
     addIsgraphqlColumnApiEndpoint1667095325334,
     addApiEndpointUuidIndexForAlert1667259254414,
+    addMetloConfigTable1667599667595,
+    updateDisabledPathsColumnBlockFieldsTable1667606447208,
   ],
   migrationsRun: runMigration,
   logging: false,

backend/src/index.ts

@@ -56,6 +56,10 @@ import {
   getInstanceSettingsHandler,
   putInstanceSettingsHandler,
 } from "api/settings"
+import {
+  getMetloConfigHandler,
+  updateMetloConfigHandler,
+} from "api/metlo-config"
 
 const port = process.env.PORT || 8080
 RedisClient.getInstance()
@@ -149,6 +153,9 @@ app.get("/api/v1/keys/list", listKeys)
 app.post("/api/v1/keys/create", createKey)
 app.delete("/api/v1/keys/:name/delete", deleteKey)
 
+app.put("/api/v1/metlo-config", updateMetloConfigHandler)
+app.get("/api/v1/metlo-config", getMetloConfigHandler)
+
 const initInstanceSettings = async () => {
   const settingRepository = AppDataSource.getRepository(InstanceSettings)
   const numSettings = await settingRepository.count()

backend/src/migrations/1667599667595-add-metlo-config-table.ts

@@ -0,0 +1,13 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class addMetloConfigTable1667599667595 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE IF NOT EXISTS "metlo_config" ("uuid" uuid NOT NULL DEFAULT uuid_generate_v4(), "configString" character varying NOT NULL, CONSTRAINT "PK_af010f5deb8072a39d9aa13c89b" PRIMARY KEY ("uuid"))`,
+    )
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP TABLE IF EXISTS "metlo_config"`)
+  }
+}

backend/src/migrations/1667606447208-update-disabledPaths-column-blockFields-table.ts

@@ -0,0 +1,23 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class updateDisabledPathsColumnBlockFieldsTable1667606447208
+  implements MigrationInterface
+{
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "block_fields" DROP COLUMN "disabledPaths"`,
+    )
+    await queryRunner.query(
+      `ALTER TABLE "block_fields" ADD "disabledPaths" jsonb`,
+    )
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "block_fields" DROP COLUMN "disabledPaths"`,
+    )
+    await queryRunner.query(
+      `ALTER TABLE "block_fields" ADD "disabledPaths" character varying array NOT NULL DEFAULT '{}'`,
+    )
+  }
+}

backend/src/models/block-fields.ts

@@ -8,6 +8,7 @@ import { DisableRestMethod } from "@common/enums"
 import { isParameter } from "utils"
 import { getPathTokens } from "@common/utils"
 import MetloBaseEntity from "./metlo-base-entity"
+import { DisabledPathSection } from "@common/types"
 
 @Entity()
 export class BlockFields extends MetloBaseEntity {
@@ -33,8 +34,8 @@ export class BlockFields extends MetloBaseEntity {
   @Column({ type: "integer", nullable: false, default: 0 })
   numberParams: number
 
-  @Column("varchar", { array: true, default: [] })
-  disabledPaths: string[]
+  @Column({ type: "jsonb", nullable: true, default: null })
+  disabledPaths: DisabledPathSection
 
   @BeforeInsert()
   addNumberParams() {

backend/src/models/metlo-config.ts

@@ -0,0 +1,17 @@
+import {
+  BaseEntity,
+  Column,
+  CreateDateColumn,
+  Entity,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
+} from "typeorm"
+
+@Entity()
+export class MetloConfig extends BaseEntity {
+  @PrimaryGeneratedColumn("uuid")
+  uuid: string
+
+  @Column()
+  configString: string
+}