add onboarding (#155)

backend/src/api/keys/index.ts

@@ -7,6 +7,7 @@ import { createApiKey } from "./service"
 import Error400BadRequest from "errors/error-400-bad-request"
 import { createQB, getRepository } from "services/database/utils"
 import { MetloRequest } from "types"
+import { API_KEY_TYPE } from "@common/enums"
 
 export const listKeys = async (
   req: MetloRequest,
@@ -28,7 +29,7 @@ export const createKey = async (
   req: MetloRequest,
   res: Response,
 ): Promise<void> => {
-  const { name: keyName } = req.body
+  const { name: keyName, keyFor } = req.body
   const key_exists = await getRepository(req.ctx, ApiKey).countBy({
     name: keyName,
   })
@@ -44,7 +45,20 @@ export const createKey = async (
       new Error400BadRequest(`Key name is required.`),
     )
   }
+  if (keyFor && !API_KEY_TYPE[keyFor]) {
+    return ApiResponseHandler.error(
+      res,
+      new Error400BadRequest(
+        `The key must be for one of the following: ${Object.keys(
+          API_KEY_TYPE,
+        ).join(", ")}`,
+      ),
+    )
+  }
   const [key, rawKey] = createApiKey(keyName)
+  if (keyFor) {
+    key.for = API_KEY_TYPE[keyFor]
+  }
   await getRepository(req.ctx, ApiKey).save(key)
   return ApiResponseHandler.success(res, {
     apiKey: rawKey,
@@ -78,3 +92,28 @@ export const deleteKey = async (
     )
   }
 }
+
+export const getOnboardingKeys = async (
+  req: MetloRequest,
+  res: Response,
+): Promise<void> => {
+  try {
+    const keys = await getRepository(req.ctx, ApiKey).find({
+      where: { for: API_KEY_TYPE.ONBOARDING },
+      order: {
+        createdAt: "DESC",
+      },
+    })
+    const payload = keys
+      ? keys.map<ApiKeyType>(v => ({
+          name: v.name,
+          identifier: `metlo.${v.keyIdentifier}`,
+          created: v.createdAt.toISOString(),
+          for: v.for,
+        }))
+      : null
+    await ApiResponseHandler.success(res, payload)
+  } catch (err) {
+    await ApiResponseHandler.error(res, err)
+  }
+}

backend/src/api/summary/index.ts

@@ -1,5 +1,5 @@
 import { Response } from "express"
-import { getSummaryData } from "services/summary"
+import { getEndpointExists, getSummaryData } from "services/summary"
 import ApiResponseHandler from "api-response-handler"
 import { MetloRequest } from "types"
 
@@ -14,3 +14,15 @@ export const getSummaryHandler = async (
     await ApiResponseHandler.error(res, err)
   }
 }
+
+export const getEndpointTrackedHandler = async (
+  req: MetloRequest,
+  res: Response,
+): Promise<void> => {
+  try {
+    const exists = await getEndpointExists(req.ctx)
+    await ApiResponseHandler.success(res, { exists })
+  } catch (err) {
+    await ApiResponseHandler.error(res, err)
+  }
+}

backend/src/data-source.ts

@@ -26,6 +26,7 @@ import { addApiEndpointUuidIndexForAlert1667259254414 } from "migrations/1667259
 import { MetloConfig } from "models/metlo-config"
 import { addMetloConfigTable1667599667595 } from "migrations/1667599667595-add-metlo-config-table"
 import { updateDisabledPathsColumnBlockFieldsTable1667606447208 } from "migrations/1667606447208-update-disabledPaths-column-blockFields-table"
+import { removeApiKeyTypeEnum1669778297643 } from "migrations/1669778297643-remove-apiKeyType-enum"
 
 export const AppDataSource: DataSource = new DataSource({
   type: "postgres",
@@ -57,6 +58,7 @@ export const AppDataSource: DataSource = new DataSource({
     addApiEndpointUuidIndexForAlert1667259254414,
     addMetloConfigTable1667599667595,
     updateDisabledPathsColumnBlockFieldsTable1667606447208,
+    removeApiKeyTypeEnum1669778297643,
   ],
   migrationsRun: runMigration,
   logging: false,

backend/src/index.ts

@@ -22,7 +22,7 @@ import {
 } from "api/spec"
 import { getAlertsHandler, updateAlertHandler } from "api/alert"
 import { deleteDataFieldHandler, updateDataFieldClasses } from "api/data-field"
-import { getSummaryHandler } from "api/summary"
+import { getEndpointTrackedHandler, getSummaryHandler } from "api/summary"
 import { MetloRequest } from "types"
 import { AppDataSource } from "data-source"
 import { MulterSource } from "multer-source"
@@ -37,7 +37,7 @@ import { RedisClient } from "utils/redis"
 import { getSensitiveDataSummaryHandler } from "api/data-field/sensitive-data"
 import { getVulnerabilitySummaryHandler } from "api/alert/vulnerability"
 import { inSandboxMode } from "utils"
-import { createKey, deleteKey, listKeys } from "api/keys"
+import { createKey, deleteKey, getOnboardingKeys, listKeys } from "api/keys"
 import {
   getInstanceSettingsHandler,
   putInstanceSettingsHandler,
@@ -74,6 +74,7 @@ app.get("/api/v1", (req: MetloRequest, res: Response) => {
 
 const apiRouter = express.Router()
 apiRouter.get("/api/v1/summary", getSummaryHandler)
+apiRouter.get("/api/v1/summary/endpoint-tracked", getEndpointTrackedHandler)
 apiRouter.get("/api/v1/instance-settings", getInstanceSettingsHandler)
 apiRouter.put("/api/v1/instance-settings", putInstanceSettingsHandler)
 apiRouter.get("/api/v1/sensitive-data-summary", getSensitiveDataSummaryHandler)
@@ -122,6 +123,7 @@ apiRouter.delete("/api/v1/test/:uuid/delete", deleteTest)
 apiRouter.get("/api/v1/keys/list", listKeys)
 apiRouter.post("/api/v1/keys/create", createKey)
 apiRouter.delete("/api/v1/keys/:name/delete", deleteKey)
+apiRouter.get("/api/v1/keys/onboarding", getOnboardingKeys)
 
 apiRouter.put("/api/v1/metlo-config", updateMetloConfigHandler)
 apiRouter.get("/api/v1/metlo-config", getMetloConfigHandler)

backend/src/migrations/1669778297643-remove-apiKeyType-enum.ts

@@ -0,0 +1,43 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class removeApiKeyTypeEnum1669778297643 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "api_key" ALTER COLUMN "for" TYPE text`,
+    )
+    await queryRunner.query(
+      `ALTER TABLE "api_key" ALTER COLUMN "for" SET NOT NULL`,
+    )
+    await queryRunner.query(
+      `ALTER TABLE "api_key" ALTER COLUMN "for" SET DEFAULT 'GENERIC'`,
+    )
+    const existingApiKeyForEnum = await queryRunner.query(
+      "SELECT 1 FROM pg_type WHERE typname = 'api_key_for_enum'",
+    )
+    const existingApiKeyTypeEnum = await queryRunner.query(
+      "SELECT 1 FROM pg_type WHERE typname = 'api_key_type_enum'",
+    )
+    if (existingApiKeyForEnum) {
+      await queryRunner.query(`DROP TYPE IF EXISTS "public"."api_key_for_enum"`)
+    }
+    if (existingApiKeyTypeEnum) {
+      await queryRunner.query(
+        `DROP TYPE IF EXISTS "public"."api_key_type_enum"`,
+      )
+    }
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    const existingApiKeyTypeEnum = await queryRunner.query(
+      "SELECT 1 FROM pg_type WHERE typname = 'api_key_type_enum'",
+    )
+    if (!existingApiKeyTypeEnum[0]) {
+      await queryRunner.query(
+        `CREATE TYPE "public"."api_key_type_enum" AS ENUM('GCP', 'AWS', 'GENERIC')`,
+      )
+    }
+    await queryRunner.query(
+      `ALTER TABLE "api_key" ALTER COLUMN "for" "public"."api_key_type_enum" NOT NULL DEFAULT 'GENERIC'`,
+    )
+  }
+}

backend/src/services/summary/index.ts

@@ -1,5 +1,7 @@
 import { Summary as SummaryResponse } from "@common/types"
+import { ApiEndpoint } from "models"
 import { ConnectionsService } from "services/connections"
+import { createQB } from "services/database/utils"
 import { MetloContext } from "types"
 import { getAlertTypeAggCached, getTopAlertsCached } from "./alerts"
 import { getTopEndpointsCached } from "./endpoints"
@@ -26,3 +28,17 @@ export const getSummaryData = async (
     ...counts,
   }
 }
+
+export const getEndpointExists = async (
+  ctx: MetloContext,
+): Promise<boolean> => {
+  const existingEndpoint = await createQB(ctx)
+    .select(["uuid"])
+    .from(ApiEndpoint, "endpoint")
+    .limit(1)
+    .getRawMany()
+  if (existingEndpoint?.length > 0) {
+    return true
+  }
+  return false
+}

backend/src/services/summary/usageStats.ts

@@ -113,6 +113,8 @@ export const getCountsCached = async (ctx: MetloContext) => {
     return cacheRes
   }
   const realRes = await getCounts(ctx)
-  await RedisClient.addToRedis(ctx, "usageCounts", realRes, 60)
+  if (realRes.hostCount > 0) {
+    await RedisClient.addToRedis(ctx, "usageCounts", realRes, 60)
+  }
   return realRes
 }

common/src/enums.ts

@@ -82,7 +82,7 @@ export enum ConnectionType {
   NODE = "NODE",
   JAVA = "JAVA",
   GOLANG = "GOLANG",
-  KUBERNETES = "KUBERNETES"
+  KUBERNETES = "KUBERNETES",
 }
 
 export enum SpecExtension {
@@ -134,7 +134,7 @@ export enum DataType {
   BOOLEAN = "boolean",
   OBJECT = "object",
   ARRAY = "array",
-  UNKNOWN = "unknown"
+  UNKNOWN = "unknown",
 }
 
 export enum DataSection {
@@ -188,7 +188,8 @@ export enum GCP_SOURCE_TYPE {
 export enum API_KEY_TYPE {
   GCP = "GCP",
   AWS = "AWS",
-  GENERIC = "GENERIC"
+  GENERIC = "GENERIC",
+  ONBOARDING = "ONBOARDING",
 }
 
 export enum OperationType {

frontend/src/api/home/index.ts

@@ -31,3 +31,17 @@ export const getInstanceSettings = async (
     return null
   }
 }
+
+export const getEndpointTracked = async (
+  headers?: AxiosRequestHeaders,
+): Promise<{ exists: boolean }> => {
+  try {
+    const resp = await axios.get<{ exists: boolean }>(
+      `${getAPIURL()}/summary/endpoint-tracked`,
+      { headers },
+    )
+    return resp.data
+  } catch (err) {
+    return { exists: false }
+  }
+}

frontend/src/api/keys/index.ts

@@ -17,6 +17,7 @@ export const getKeys = async (
     throw err
   }
 }
+
 export const deleteKey = async (
   key_name: string,
   headers?: AxiosRequestHeaders,
@@ -41,15 +42,18 @@ export const deleteKey = async (
     throw err
   }
 }
+
 export const addKey = async (
   key_name: string,
+  keyFor?: string,
   headers?: AxiosRequestHeaders,
 ): Promise<ApiKey & { apiKey: string }> => {
   try {
     const resp = await axios.post<ApiKey & { apiKey: string }>(
       `${getAPIURL()}/keys/create`,
       {
         name: key_name,
+        keyFor,
       },
       { headers },
     )
@@ -66,3 +70,12 @@ export const addKey = async (
     throw err
   }
 }
+
+export const getOnboardingKeys = async (
+  headers?: AxiosRequestHeaders,
+): Promise<ApiKey[]> => {
+  const resp = await axios.get<ApiKey[]>(`${getAPIURL()}/keys/onboarding`, {
+    headers,
+  })
+  return resp.data
+}

frontend/src/components/ConnectionDocs/docs/aws.tsx

@@ -214,6 +214,7 @@ sudo systemctl start metlo-ingestor.service`}</SyntaxHighlighter>
                   href={getAWSIngestorLaunchStackURL(
                     selectedRegion,
                     metloAddress,
+                    apiKey,
                   )}
                   isDisabled={selectedRegion === ""}
                 >