[serverless] add cspm metering functionality (#162019)

x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metring.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getCspmUsageRecord } from './cspm_metring_task';
+import type { MeteringCallbackInput, UsageRecord } from '../types';
+
+export const CLOUD_SECURITY_TASK_TYPE = 'Cloud_Security';
+
+export const cloudSecurityMetringCallback = async ({
+  esClient,
+  cloudSetup,
+  logger,
+  taskId,
+  lastSuccessfulReport,
+}: MeteringCallbackInput): Promise<UsageRecord[]> => {
+  const projectId = cloudSetup?.serverless?.projectId || 'missing project id';
+
+  if (!cloudSetup?.serverless?.projectId) {
+    logger.error('no project id found');
+  }
+
+  try {
+    const cloudSecurityUsageRecords: UsageRecord[] = [];
+
+    const cspmUsageRecord = await getCspmUsageRecord({
+      esClient,
+      projectId,
+      logger,
+      taskId,
+      lastSuccessfulReport,
+    });
+
+    if (cspmUsageRecord) {
+      cloudSecurityUsageRecords.push(cspmUsageRecord);
+    }
+
+    return cloudSecurityUsageRecords;
+  } catch (err) {
+    logger.error(`Failed to fetch Cloud Security metering data ${err}`);
+    return [];
+  }
+};

x-pack/plugins/security_solution_serverless/server/cloud_security/cspm_metring_task.ts

@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  CSPM_POLICY_TEMPLATE,
+  CSP_LATEST_FINDINGS_DATA_VIEW,
+} from '@kbn/cloud-security-posture-plugin/common/constants';
+import { CLOUD_SECURITY_TASK_TYPE } from './cloud_security_metring';
+import { cloudSecurityMetringTaskProperties } from './metering_tasks_configs';
+
+import type { CloudSecurityMeteringCallbackInput, UsageRecord } from '../types';
+
+const CSPM_CYCLE_SCAN_FREQUENT = '24h';
+const CSPM_BUCKET_SUB_TYPE_NAME = 'CSPM';
+
+interface ResourceCountAggregation {
+  min_timestamp: MinTimestamp;
+  unique_resources: {
+    value: number;
+  };
+}
+
+interface MinTimestamp {
+  value: number;
+  value_as_string: string;
+}
+
+export const getCspmUsageRecord = async ({
+  esClient,
+  projectId,
+  logger,
+  taskId,
+}: CloudSecurityMeteringCallbackInput): Promise<UsageRecord | undefined> => {
+  try {
+    const response = await esClient.search<unknown, ResourceCountAggregation>(
+      getFindingsByResourceAggQuery()
+    );
+
+    if (!response.aggregations) {
+      return;
+    }
+    const cspmResourceCount = response.aggregations.unique_resources.value;
+
+    const minTimestamp = response.aggregations
+      ? new Date(response.aggregations.min_timestamp.value_as_string).toISOString()
+      : new Date().toISOString();
+
+    const usageRecords = {
+      id: `${CLOUD_SECURITY_TASK_TYPE}:${CLOUD_SECURITY_TASK_TYPE}`,
+      usage_timestamp: minTimestamp,
+      creation_timestamp: new Date().toISOString(),
+      usage: {
+        type: CLOUD_SECURITY_TASK_TYPE,
+        sub_type: CSPM_BUCKET_SUB_TYPE_NAME,
+        quantity: cspmResourceCount,
+        period_seconds: cloudSecurityMetringTaskProperties.periodSeconds,
+      },
+      source: {
+        id: taskId,
+        instance_group_id: projectId,
+      },
+    };
+
+    logger.debug(`Fetched CSPM metring data`);
+
+    return usageRecords;
+  } catch (err) {
+    logger.error(`Failed to fetch CSPM metering data ${err}`);
+  }
+};
+
+export const getFindingsByResourceAggQuery = () => ({
+  index: CSP_LATEST_FINDINGS_DATA_VIEW,
+  query: {
+    bool: {
+      must: [
+        {
+          term: {
+            'rule.benchmark.posture_type': CSPM_POLICY_TEMPLATE,
+          },
+        },
+        {
+          range: {
+            '@timestamp': {
+              gte: `now-${CSPM_CYCLE_SCAN_FREQUENT}`, // the "look back" period should be the same as the scan interval
+            },
+          },
+        },
+      ],
+    },
+  },
+  size: 0,
+  aggs: {
+    unique_resources: {
+      cardinality: {
+        field: 'resource.id',
+        precision_threshold: 3000, // default = 3000  note note that even with a threshold as low as 100, the error remains very low 1-6% even when counting millions of items. https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-cardinality-aggregation.html#_counts_are_approximate
+      },
+    },
+    min_timestamp: {
+      min: {
+        field: '@timestamp',
+      },
+    },
+  },
+});

x-pack/plugins/security_solution_serverless/server/cloud_security/metering_tasks_configs.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { cloudSecurityMetringCallback } from './cloud_security_metring';
+import type { MetringTaskProperties } from '../types';
+
+const TASK_INTERVAL = 3600; // 1 hour
+
+export const cloudSecurityMetringTaskProperties: MetringTaskProperties = {
+  taskType: 'cloud-security-usage-reporting-task',
+  taskTitle: 'Cloud Security Metring Periodic Tasks',
+  meteringCallback: cloudSecurityMetringCallback,
+  interval: `${TASK_INTERVAL.toString()}s`,
+  periodSeconds: TASK_INTERVAL,
+  version: '1',
+};

x-pack/plugins/security_solution_serverless/server/common/services/usage_reporting_service.ts

@@ -15,7 +15,7 @@ export class UsageReportingService {
   public async reportUsage(records: UsageRecord[]): Promise<Response> {
     return fetch(USAGE_SERVICE_USAGE_URL, {
       method: 'post',
-      body: JSON.stringify([records]),
+      body: JSON.stringify(records),
       headers: { 'Content-Type': 'application/json' },
     });
   }

x-pack/plugins/security_solution_serverless/server/plugin.ts

@@ -15,6 +15,8 @@ import type {
   SecuritySolutionServerlessPluginSetupDeps,
   SecuritySolutionServerlessPluginStartDeps,
 } from './types';
+import { SecurityUsageReportingTask } from './task_manager/usage_reporting_task';
+import { cloudSecurityMetringTaskProperties } from './cloud_security/metering_tasks_configs';
 
 export class SecuritySolutionServerlessPlugin
   implements
@@ -26,6 +28,7 @@ export class SecuritySolutionServerlessPlugin
     >
 {
   private config: ServerlessSecurityConfig;
+  private cspmUsageReportingTask: SecurityUsageReportingTask | undefined;
 
   constructor(private readonly initializerContext: PluginInitializerContext) {
     this.config = this.initializerContext.config.get<ServerlessSecurityConfig>();
@@ -35,17 +38,33 @@ export class SecuritySolutionServerlessPlugin
     // securitySolutionEss plugin should always be disabled when securitySolutionServerless is enabled.
     // This check is an additional layer of security to prevent double registrations when
     // `plugins.forceEnableAllPlugins` flag is enabled).
+
     const shouldRegister = pluginsSetup.securitySolutionEss == null;
 
     if (shouldRegister) {
       pluginsSetup.securitySolution.setAppFeatures(getProductAppFeatures(this.config.productTypes));
     }
-
     pluginsSetup.ml.setFeaturesEnabled({ ad: true, dfa: true, nlp: false });
+
+    this.cspmUsageReportingTask = new SecurityUsageReportingTask({
+      core: _coreSetup,
+      logFactory: this.initializerContext.logger,
+      taskManager: pluginsSetup.taskManager,
+      cloudSetup: pluginsSetup.cloudSetup,
+      taskType: cloudSecurityMetringTaskProperties.taskType,
+      taskTitle: cloudSecurityMetringTaskProperties.taskTitle,
+      version: cloudSecurityMetringTaskProperties.version,
+      meteringCallback: cloudSecurityMetringTaskProperties.meteringCallback,
+    });
+
     return {};
   }
 
   public start(_coreStart: CoreStart, pluginsSetup: SecuritySolutionServerlessPluginStartDeps) {
+    this.cspmUsageReportingTask?.start({
+      taskManager: pluginsSetup.taskManager,
+      interval: cloudSecurityMetringTaskProperties.interval,
+    });
     return {};
   }