feat: integration details page with logging

backend/src/ee/routes/v1/project-router.ts

@@ -146,12 +146,16 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
         actorId: req.permission.id,
         actorOrgId: req.permission.orgId,
         actorAuthMethod: req.permission.authMethod,
-        projectId: req.params.workspaceId,
-        ...req.query,
-        endDate: req.query.endDate,
-        startDate: req.query.startDate || getLastMidnightDateISO(),
-        auditLogActor: req.query.actor,
-        actor: req.permission.type
+        actor: req.permission.type,
+
+        filter: {
+          ...req.query,
+          projectId: req.params.workspaceId,
+          endDate: req.query.endDate,
+          startDate: req.query.startDate || getLastMidnightDateISO(),
+          auditLogActorId: req.query.actor,
+          eventType: req.query.eventType ? [req.query.eventType] : undefined
+        }
       });
       return { auditLogs };
     }

backend/src/ee/services/audit-log/audit-log-dal.ts

@@ -6,6 +6,9 @@ import { DatabaseError } from "@app/lib/errors";
 import { ormify, selectAllTableCols, stripUndefinedInWhere } from "@app/lib/knex";
 import { logger } from "@app/lib/logger";
 import { QueueName } from "@app/queue";
+import { ActorType } from "@app/services/auth/auth-type";
+
+import { EventType } from "./audit-log-types";
 
 export type TAuditLogDALFactory = ReturnType<typeof auditLogDALFactory>;
 
@@ -25,7 +28,24 @@ export const auditLogDALFactory = (db: TDbClient) => {
   const auditLogOrm = ormify(db, TableName.AuditLog);
 
   const find = async (
-    { orgId, projectId, userAgentType, startDate, endDate, limit = 20, offset = 0, actor, eventType }: TFindQuery,
+    {
+      orgId,
+      projectId,
+      userAgentType,
+      startDate,
+      endDate,
+      limit = 20,
+      offset = 0,
+      actorId,
+      actorType,
+      eventType,
+      eventMetadata
+    }: Omit<TFindQuery, "actor" | "eventType"> & {
+      actorId?: string;
+      actorType?: ActorType;
+      eventType?: EventType[];
+      eventMetadata?: Record<string, string>;
+    },
     tx?: Knex
   ) => {
     try {
@@ -34,7 +54,6 @@ export const auditLogDALFactory = (db: TDbClient) => {
           stripUndefinedInWhere({
             projectId,
             [`${TableName.AuditLog}.orgId`]: orgId,
-            eventType,
             userAgentType
           })
         )
@@ -52,8 +71,22 @@ export const auditLogDALFactory = (db: TDbClient) => {
         .offset(offset)
         .orderBy(`${TableName.AuditLog}.createdAt`, "desc");
 
-      if (actor) {
-        void sqlQuery.whereRaw(`"actorMetadata"->>'userId' = ?`, [actor]);
+      if (actorId) {
+        void sqlQuery.whereRaw(`"actorMetadata"->>'userId' = ?`, [actorId]);
+      }
+
+      if (eventMetadata && Object.keys(eventMetadata).length) {
+        Object.entries(eventMetadata).forEach(([key, value]) => {
+          void sqlQuery.whereRaw(`"eventMetadata"->>'${key}' = ?`, [value]);
+        });
+      }
+
+      if (actorType) {
+        void sqlQuery.where("actor", actorType);
+      }
+
+      if (eventType?.length) {
+        void sqlQuery.whereIn("eventType", eventType);
       }
 
       if (startDate) {

backend/src/ee/services/audit-log/audit-log-service.ts

@@ -23,25 +23,12 @@ export const auditLogServiceFactory = ({
   auditLogQueue,
   permissionService
 }: TAuditLogServiceFactoryDep) => {
-  const listAuditLogs = async ({
-    userAgentType,
-    eventType,
-    offset,
-    limit,
-    endDate,
-    startDate,
-    actor,
-    actorId,
-    actorOrgId,
-    actorAuthMethod,
-    projectId,
-    auditLogActor
-  }: TListProjectAuditLogDTO) => {
-    if (projectId) {
+  const listAuditLogs = async ({ actorAuthMethod, actorId, actorOrgId, actor, filter }: TListProjectAuditLogDTO) => {
+    if (filter.projectId) {
       const { permission } = await permissionService.getProjectPermission(
         actor,
         actorId,
-        projectId,
+        filter.projectId,
         actorAuthMethod,
         actorOrgId
       );
@@ -65,14 +52,16 @@ export const auditLogServiceFactory = ({
     // If project ID is not provided, then we need to return all the audit logs for the organization itself.
 
     const auditLogs = await auditLogDAL.find({
-      startDate,
-      endDate,
-      limit,
-      offset,
-      eventType,
-      userAgentType,
-      actor: auditLogActor,
-      ...(projectId ? { projectId } : { orgId: actorOrgId })
+      startDate: filter.startDate,
+      endDate: filter.endDate,
+      limit: filter.limit,
+      offset: filter.offset,
+      eventType: filter.eventType,
+      userAgentType: filter.userAgentType,
+      actorId: filter.auditLogActorId,
+      actorType: filter.actorType,
+      eventMetadata: filter.eventMetadata,
+      ...(filter.projectId ? { projectId: filter.projectId } : { orgId: actorOrgId })
     });
 
     return auditLogs.map(({ eventType: logEventType, actor: eActor, actorMetadata, eventMetadata, ...el }) => ({

backend/src/ee/services/audit-log/audit-log-types.ts

@@ -5,19 +5,23 @@ import { TIdentityTrustedIp } from "@app/services/identity/identity-types";
 import { PkiItemType } from "@app/services/pki-collection/pki-collection-types";
 
 export type TListProjectAuditLogDTO = {
-  auditLogActor?: string;
-  projectId?: string;
-  eventType?: string;
-  startDate?: string;
-  endDate?: string;
-  userAgentType?: string;
-  limit?: number;
-  offset?: number;
+  filter: {
+    userAgentType?: UserAgentType;
+    eventType?: EventType[];
+    offset?: number;
+    limit: number;
+    endDate?: string;
+    startDate?: string;
+    projectId?: string;
+    auditLogActorId?: string;
+    actorType?: ActorType;
+    eventMetadata?: Record<string, string>;
+  };
 } & Omit<TProjectPermission, "projectId">;
 
 export type TCreateAuditLogDTO = {
   event: Event;
-  actor: UserActor | IdentityActor | ServiceActor | ScimClientActor;
+  actor: UserActor | IdentityActor | ServiceActor | ScimClientActor | PlatformActor;
   orgId?: string;
   projectId?: string;
 } & BaseAuthData;
@@ -177,7 +181,8 @@ export enum EventType {
   UPDATE_SLACK_INTEGRATION = "update-slack-integration",
   DELETE_SLACK_INTEGRATION = "delete-slack-integration",
   GET_PROJECT_SLACK_CONFIG = "get-project-slack-config",
-  UPDATE_PROJECT_SLACK_CONFIG = "update-project-slack-config"
+  UPDATE_PROJECT_SLACK_CONFIG = "update-project-slack-config",
+  INTEGRATION_SYNCED = "integration-synced"
 }
 
 interface UserActorMetadata {
@@ -198,6 +203,8 @@ interface IdentityActorMetadata {
 
 interface ScimClientActorMetadata {}
 
+interface PlatformActorMetadata {}
+
 export interface UserActor {
   type: ActorType.USER;
   metadata: UserActorMetadata;
@@ -208,6 +215,11 @@ export interface ServiceActor {
   metadata: ServiceActorMetadata;
 }
 
+export interface PlatformActor {
+  type: ActorType.PLATFORM;
+  metadata: PlatformActorMetadata;
+}
+
 export interface IdentityActor {
   type: ActorType.IDENTITY;
   metadata: IdentityActorMetadata;
@@ -218,7 +230,7 @@ export interface ScimClientActor {
   metadata: ScimClientActorMetadata;
 }
 
-export type Actor = UserActor | ServiceActor | IdentityActor | ScimClientActor;
+export type Actor = UserActor | ServiceActor | IdentityActor | ScimClientActor | PlatformActor;
 
 interface GetSecretsEvent {
   type: EventType.GET_SECRETS;
@@ -1518,6 +1530,16 @@ interface GetProjectSlackConfig {
     id: string;
   };
 }
+interface IntegrationSyncedEvent {
+  type: EventType.INTEGRATION_SYNCED;
+  metadata: {
+    integrationId: string;
+    lastSyncJobId: string;
+    lastUsed: Date;
+    syncMessage: string;
+    isSynced: boolean;
+  };
+}
 
 export type Event =
   | GetSecretsEvent
@@ -1657,4 +1679,5 @@ export type Event =
   | DeleteSlackIntegration
   | GetSlackIntegration
   | UpdateProjectSlackConfig
-  | GetProjectSlackConfig;
+  | GetProjectSlackConfig
+  | IntegrationSyncedEvent;

backend/src/queue/queue-service.ts

@@ -91,6 +91,8 @@ export type TQueueJobTypes = {
   [QueueName.IntegrationSync]: {
     name: QueueJobs.IntegrationSync;
     payload: {
+      isManual?: boolean;
+      actorId?: string;
       projectId: string;
       environment: string;
       secretPath: string;

backend/src/server/routes/index.ts

@@ -811,6 +811,8 @@ export const registerRoutes = async (
     projectEnvDAL,
     webhookDAL,
     orgDAL,
+    auditLogService,
+    userDAL,
     projectMembershipDAL,
     smtpService,
     projectDAL,

backend/src/server/routes/v1/integration-router.ts

@@ -4,7 +4,7 @@ import { IntegrationsSchema } from "@app/db/schemas";
 import { EventType } from "@app/ee/services/audit-log/audit-log-types";
 import { INTEGRATION } from "@app/lib/api-docs";
 import { removeTrailingSlash, shake } from "@app/lib/fn";
-import { writeLimit } from "@app/server/config/rateLimiter";
+import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
 import { getTelemetryDistinctId } from "@app/server/lib/telemetry";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
@@ -154,6 +154,48 @@ export const registerIntegrationRouter = async (server: FastifyZodProvider) => {
     }
   });
 
+  server.route({
+    method: "GET",
+    url: "/:integrationId",
+    config: {
+      rateLimit: readLimit
+    },
+    schema: {
+      description: "Get an integration by integration id",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
+      params: z.object({
+        integrationId: z.string().trim().describe(INTEGRATION.UPDATE.integrationId)
+      }),
+      response: {
+        200: z.object({
+          integration: IntegrationsSchema.extend({
+            environment: z.object({
+              slug: z.string().trim(),
+              name: z.string().trim(),
+              id: z.string().trim()
+            })
+          })
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
+    handler: async (req) => {
+      const integration = await server.services.integration.getIntegration({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        id: req.params.integrationId
+      });
+
+      return { integration };
+    }
+  });
+
   server.route({
     method: "DELETE",
     url: "/:integrationId",

backend/src/server/routes/v1/organization-router.ts

@@ -14,7 +14,7 @@ import { AUDIT_LOGS, ORGANIZATIONS } from "@app/lib/api-docs";
 import { getLastMidnightDateISO } from "@app/lib/fn";
 import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
-import { AuthMode } from "@app/services/auth/auth-type";
+import { ActorType, AuthMode } from "@app/services/auth/auth-type";
 
 export const registerOrgRouter = async (server: FastifyZodProvider) => {
   server.route({
@@ -74,8 +74,35 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
     schema: {
       description: "Get all audit logs for an organization",
       querystring: z.object({
-        eventType: z.nativeEnum(EventType).optional().describe(AUDIT_LOGS.EXPORT.eventType),
+        projectId: z.string().optional(),
+        actorType: z.nativeEnum(ActorType).optional(),
+        // eventType is split with , for multiple values, we need to transform it to array
+        eventType: z
+          .string()
+          .optional()
+          .transform((val) => (val ? val.split(",") : undefined)),
         userAgentType: z.nativeEnum(UserAgentType).optional().describe(AUDIT_LOGS.EXPORT.userAgentType),
+        eventMetadata: z
+          .string()
+          .optional()
+          .transform((val) => {
+            if (!val) {
+              return undefined;
+            }
+
+            const pairs = val.split(",");
+
+            return pairs.reduce(
+              (acc, pair) => {
+                const [key, value] = pair.split("=");
+                if (key && value) {
+                  acc[key] = value;
+                }
+                return acc;
+              },
+              {} as Record<string, string>
+            );
+          }),
         startDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.startDate),
         endDate: z.string().datetime().optional().describe(AUDIT_LOGS.EXPORT.endDate),
         offset: z.coerce.number().default(0).describe(AUDIT_LOGS.EXPORT.offset),
@@ -114,13 +141,19 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
     onRequest: verifyAuth([AuthMode.JWT]),
     handler: async (req) => {
       const auditLogs = await server.services.auditLog.listAuditLogs({
+        filter: {
+          ...req.query,
+          endDate: req.query.endDate,
+          projectId: req.query.projectId,
+          startDate: req.query.startDate || getLastMidnightDateISO(),
+          auditLogActorId: req.query.actor,
+          actorType: req.query.actorType,
+          eventType: req.query.eventType as EventType[] | undefined
+        },
+
         actorId: req.permission.id,
         actorOrgId: req.permission.orgId,
         actorAuthMethod: req.permission.authMethod,
-        ...req.query,
-        endDate: req.query.endDate,
-        startDate: req.query.startDate || getLastMidnightDateISO(),
-        auditLogActor: req.query.actor,
         actor: req.permission.type
       });
       return { auditLogs };

backend/src/services/auth/auth-type.ts

@@ -34,6 +34,7 @@ export enum AuthMode {
 }
 
 export enum ActorType { // would extend to AWS, Azure, ...
+  PLATFORM = "platform", // Useful for when we want to perform logging on automated actions such as integration syncs.
   USER = "user", // userIdentity
   SERVICE = "service",
   IDENTITY = "identity",