feat(mis): 新增数据统计功能 (#825)

### 新增： 1. 管理系统新增平台数据统计页面，统计内容如下： - 总用户数量，总账户数量，总租户数量，总作业数，总消费数 - 筛选时间内新增用户数，新增账户数，新增租户数，新增作业数，新增消费数 - 筛选时间内，新增用户数每日统计，活跃用户数每日统计（用户是否活跃根据是否产生操作日志判断） - 筛选时间内，消费账户top10统计，消费金额每日统计，充值账户top10统计，充值金额每日统计 - 筛选时间内，提交作业数top10用户统计，新增作业每日统计 - 筛选时间内，使用portal功能次数统计（维度仅限于操作日志统计的操作类型），使用mis功能次数统计 ![image](https://github.com/PKUHPC/SCOW/assets/130351655/4bb4f6f6-c690-4399-b7fd-3ad4a5bf43ab) ![image](https://github.com/PKUHPC/SCOW/assets/130351655/08d6461d-987b-49f0-8bd3-c3050e5ede61) ![image](https://github.com/PKUHPC/SCOW/assets/130351655/31f3aa3f-d18b-477e-b15c-634092c815a0) 2. scow db里新增TABLE `query_cache`，表结构如下 ![image](https://github.com/PKUHPC/SCOW/assets/130351655/a204b918-1e5c-4929-93e8-d56c84b49e7a) 该表主要用于缓存时间统计页面中时间消耗较长的接口, 如： - getTopChargeAccount - getDailyCharge - getTopPayAccount - getDailyPay - getTopSubmitJobUsers - getNewJobCount 缓存方案： 1. 调用以上接口时，将接口方法以及传参作为key查询query_cache中的query_key, 如果有结果，直接返回该条数据的query_result作为接口返回，如果没有，则查询真实的表，并将该结果缓存在query_cache表中 2. 设置定时任务，每天零点清除一次query_cache表，将缓存删除。
PKUHPC · Dec 25, 2023 · f023d52 · f023d52
1 parent cd252c1
commit f023d52
Show file tree

Hide file tree

Showing 59 changed files with 3,244 additions and 50 deletions.
diff --git a/.changeset/gold-insects-trade.md b/.changeset/gold-insects-trade.md
@@ -0,0 +1,5 @@
+---
+"@scow/grpc-api": minor
+---
+
+新增数据统计接口，audit 新增 GetActiveUserCount，GetPortalUsageCount，GetMisUsageCount, server 新增 GetTopChargeAccount，GetDailyCharge，GetTopPayAccount，GetDailyPay，GetStatisticInfo，GetTopSubmitJobUsers，GetNewJobCount，GetJobTotalCount，GetNewUserCount
diff --git a/.changeset/warm-islands-kneel.md b/.changeset/warm-islands-kneel.md
@@ -0,0 +1,8 @@
+---
+"@scow/portal-server": minor
+"@scow/audit-server": minor
+"@scow/mis-server": minor
+"@scow/mis-web": minor
+---
+
+管理系统新增数据统计功能，统计用户，账户，租户，作业，消费及功能使用次数
diff --git a/apps/audit-server/package.json b/apps/audit-server/package.json
@@ -33,8 +33,10 @@
     "@scow/config": "workspace:*",
     "@scow/lib-config": "workspace:*",
     "@scow/lib-decimal": "workspace:*",
+    "@scow/lib-operation-log": "workspace:*",
     "@scow/utils": "workspace:*",
     "@scow/protos": "workspace:*",
+    "dayjs": "1.11.9",
     "pino": "8.16.2",
     "pino-pretty": "10.2.3"
   },

diff --git a/apps/audit-server/src/app.ts b/apps/audit-server/src/app.ts
@@ -16,6 +16,7 @@ import { readVersionFile } from "@scow/utils/build/version";
 import { config } from "src/config/env";
 import { plugins } from "src/plugins";
 import { operationLogServiceServer } from "src/services/operationLog";
+import { statisticServiceServer } from "src/services/statistic";
 import { loggerOptions } from "src/utils/logger";
 
 export async function createServer() {
@@ -34,6 +35,6 @@ export async function createServer() {
     await server.register(plugin);
   }
   await server.register(operationLogServiceServer);
-
+  await server.register(statisticServiceServer);
   return server;
 }
diff --git a/apps/audit-server/src/entities/OperationLog.ts b/apps/audit-server/src/entities/OperationLog.ts
@@ -11,7 +11,9 @@
  */
 
 import { Entity, Enum, PrimaryKey, Property } from "@mikro-orm/core";
-import { CURRENT_TIMESTAMP, DATETIME_TYPE } from "src/utils/orm";
+import { OperationEvent } from "@scow/lib-operation-log";
+import { CURRENT_TIMESTAMP, DATETIME_TYPE } from "src/utils/orm"; ;
+
 
 export enum OperationResult {
   UNKNOWN = "UNKNOWN",
@@ -21,6 +23,7 @@ export enum OperationResult {
 
 @Entity()
 export class OperationLog {
+
   @PrimaryKey()
     id!: number;
 
@@ -37,15 +40,15 @@ export class OperationLog {
     operationResult: OperationResult;
 
   @Property({ type: "json", nullable: true })
-    metaData?: { [key: string]: any; };
+    metaData?: OperationEvent & { targetAccountName?: string };
 
   constructor(init: {
       operationLogId?: number;
       operatorUserId: string;
       operatorIp: string;
       operationTime?: Date;
       operationResult: OperationResult;
-      metaData: { [key: string]: any };
+      metaData: OperationEvent & { targetAccountName?: string };
     }) {
     if (init.operationLogId) {
       this.id = init.operationLogId;

diff --git a/apps/audit-server/src/services/operationLog.ts b/apps/audit-server/src/services/operationLog.ts
@@ -34,9 +34,11 @@ export const operationLogServiceServer = plugin((server) => {
         operationEvent,
       } = request;
 
-      const metaData = operationEvent || {};
+      if (!operationEvent) {
+        return [];
+      }
       const operationType = operationEvent?.$case;
-      const targetAccountName = (operationEvent && operationType)
+      const targetAccountName: string | undefined = (operationEvent && operationType)
         ? operationEvent[operationType].accountName
         : undefined;
 
@@ -46,7 +48,7 @@ export const operationLogServiceServer = plugin((server) => {
         operatorUserId,
         operatorIp,
         operationResult: dbOperationResult,
-        metaData: { ...metaData, targetAccountName },
+        metaData: targetAccountName ? { ...operationEvent, targetAccountName } : operationEvent,
       });
       await em.persistAndFlush(operationLog);
       return [];

diff --git a/apps/audit-server/src/services/statistic.ts b/apps/audit-server/src/services/statistic.ts
@@ -0,0 +1,142 @@
+/**
+ * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy
+ * SCOW is licensed under Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *          http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+import { ensureNotUndefined, plugin } from "@ddadaal/tsgrpc-server";
+import { QueryOrder } from "@mikro-orm/core";
+import { OperationType } from "@scow/lib-operation-log";
+import { StatisticServiceServer, StatisticServiceService } from "@scow/protos/build/audit/statistic";
+import { OperationLog } from "src/entities/OperationLog";
+
+
+export const statisticServiceServer = plugin((server) => {
+
+  server.addService<StatisticServiceServer>(StatisticServiceService, {
+
+    getActiveUserCount: async ({ request, em }) => {
+
+      const { startTime, endTime } = ensureNotUndefined(request, ["startTime", "endTime"]);
+
+      const qb = em.createQueryBuilder(OperationLog, "o");
+
+      qb
+        .select("DATE(o.operation_time) as date, COUNT(DISTINCT o.operator_user_id) as userCount")
+        .where({ operationTime: { $gte: startTime } })
+        .andWhere({ operationTime: { $lte: endTime } })
+        .groupBy("DATE(o.operation_time)")
+        .orderBy({ "DATE(o.operation_time)": QueryOrder.DESC });
+
+      const records: {date: string, userCount: number}[] = await qb.execute();
+
+      return [{
+        results: records.map((record) => ({
+          date: record.date,
+          count: record.userCount,
+        })),
+      }];
+    },
+
+    getPortalUsageCount: async ({ request, em }) => {
+      const { startTime, endTime } = ensureNotUndefined(request, ["startTime", "endTime"]);
+
+      const portalOperationType: OperationType[] = [
+        "submitJob",
+        "endJob",
+        "addJobTemplate",
+        "deleteJobTemplate",
+        "updateJobTemplate",
+        "shellLogin",
+        "createDesktop",
+        "deleteDesktop",
+        "createApp",
+        "createFile",
+        "deleteFile",
+        "uploadFile",
+        "createDirectory",
+        "deleteDirectory",
+        "moveFileItem",
+        "copyFileItem",
+      ];
+
+      const qb = em.createQueryBuilder(OperationLog, "o");
+      qb
+        .select("JSON_EXTRACT(meta_data, '$.$case') as operationType, COUNT(*) as count")
+        .where({ operationTime: { $gte: startTime } })
+        .andWhere({ operationTime: { $lte: endTime } })
+        .andWhere({ "JSON_EXTRACT(meta_data, '$.$case')": { $in: portalOperationType } })
+        .groupBy("JSON_EXTRACT(meta_data, '$.$case')")
+        .orderBy({ "COUNT(*)": QueryOrder.DESC });
+
+      const results: {operationType: string, count: number}[] = await qb.execute();
+
+      return [{
+        results,
+      }];
+
+
+    },
+    getMisUsageCount: async ({ request, em }) => {
+      const { startTime, endTime } = ensureNotUndefined(request, ["startTime", "endTime"]);
+
+      const misOperationType: OperationType[] = [
+        "setJobTimeLimit",
+        "createUser",
+        "addUserToAccount",
+        "removeUserFromAccount",
+        "setAccountAdmin",
+        "unsetAccountAdmin",
+        "blockUser",
+        "unblockUser",
+        "accountSetChargeLimit",
+        "accountUnsetChargeLimit",
+        "setTenantBilling",
+        "setTenantAdmin",
+        "unsetTenantAdmin",
+        "setTenantFinance",
+        "unsetTenantFinance",
+        "tenantChangePassword",
+        "createAccount",
+        "addAccountToWhitelist",
+        "removeAccountFromWhitelist",
+        "accountPay",
+        "blockAccount",
+        "unblockAccount",
+        "importUsers",
+        "setPlatformAdmin",
+        "unsetPlatformAdmin",
+        "setPlatformFinance",
+        "unsetPlatformFinance",
+        "platformChangePassword",
+        "setPlatformBilling",
+        "createTenant",
+        "tenantPay",
+      ];
+
+      const qb = em.createQueryBuilder(OperationLog, "o");
+      qb
+        .select("JSON_EXTRACT(meta_data, '$.$case') as operationType, COUNT(*) as count")
+        .where({ operationTime: { $gte: startTime } })
+        .andWhere({ operationTime: { $lte: endTime } })
+        .andWhere({ "JSON_EXTRACT(meta_data, '$.$case')": { $in: misOperationType } })
+        .groupBy("JSON_EXTRACT(meta_data, '$.$case')")
+        .orderBy({ "COUNT(*)": QueryOrder.DESC });
+
+      const results: {operationType: string, count: number}[] = await qb.execute();
+
+      return [{
+        results,
+      }];
+
+
+    },
+  });
+
+});
diff --git a/apps/audit-server/src/utils/operationLogs.ts b/apps/audit-server/src/utils/operationLogs.ts
@@ -11,6 +11,7 @@
  */
 
 import { FilterQuery } from "@mikro-orm/core";
+import { OperationEvent } from "@scow/lib-operation-log";
 import {
   OperationLog,
   OperationLogFilter,
@@ -35,7 +36,7 @@ export async function filterOperationLogs(
     $and: [
       ...(startTime ? [{ operationTime: { $gte: startTime } }] : []),
       ...(endTime ? [{ operationTime: { $lte: endTime } }] : []),
-      ...((operationType) ? [{ metaData: { $case: operationType } }] : []),
+      ...((operationType) ? [{ metaData: { $case: operationType } as OperationEvent }] : []),
       ...((operationTargetAccountName) ? [{ metaData: { targetAccountName: operationTargetAccountName } }] : []),
       ...(operationDetail ? [ { metaData: { $like: `%${operationDetail}%` } }] : []),
     ],
@@ -52,15 +53,7 @@ export function toGrpcOperationLog(x: OperationLogEntity): OperationLog {
     operatorIp: x.operatorIp,
     operationTime: x.operationTime?.toISOString(),
     operationResult: operationResultFromJSON(x.operationResult),
+    operationEvent: (x.metaData),
   };
-  if (x.metaData && x.metaData.$case) {
-    // @ts-ignore
-    grpcOperationLog.operationEvent = {
-      $case: x.metaData.$case,
-      [x.metaData.$case]: x.metaData[x.metaData.$case],
-    };
-
-  }
-
   return grpcOperationLog;
 }
diff --git a/apps/audit-server/tests/log/operationLogs.test.ts b/apps/audit-server/tests/log/operationLogs.test.ts
@@ -58,7 +58,7 @@ it("create operation log", async () => {
   expect(operationLogs[0].operatorIp).toEqual(operationLog.operatorIp);
   expect(operationLogs[0].operationResult).toEqual(operationLog.operationResult);
   expect(operationLogs[0].metaData?.$case).toEqual(operationLog.operationEvent.$case);
-  expect(operationLogs[0].metaData?.submitJob).toEqual(operationLog.operationEvent.submitJob);
+  expect(operationLogs[0].metaData?.[operationLogs[0].metaData?.$case]).toEqual(operationLog.operationEvent.submitJob);
   expect(operationLogs[0].metaData?.targetAccountName).toEqual(operationLog.operationEvent.submitJob.accountName);
 });