Annotate our anomalies with @timestamp field

We were notably lacking this ECS field in our post-conversion anomalies, and typescript was rightly complaining about it.
elastic · Mar 17, 2020 · 5e85c7e · 5e85c7e
1 parent e80dd2c
commit 5e85c7e
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 14 deletions.
diff --git a/...ck/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts b/...ck/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { convertAnomalyFieldsToECS } from './bulk_create_ml_signals';
+import { transformAnomalyFieldsToEcs } from './bulk_create_ml_signals';
 
 const buildMockAnomaly = () => ({
   job_id: 'rare_process_by_host_linux_ecs',
@@ -45,10 +45,18 @@ const buildMockAnomaly = () => ({
   'host.name': ['rock01'],
 });
 
-describe('convertAnomalyFieldsToECS', () => {
+describe('transformAnomalyFieldsToEcs', () => {
+  it('adds a @timestamp field based on timestamp', () => {
+    const anomaly = buildMockAnomaly();
+    const result = transformAnomalyFieldsToEcs(anomaly);
+    const expectedTime = '2020-03-17T22:00:00.000Z';
+
+    expect(result['@timestamp']).toEqual(expectedTime);
+  });
+
   it('deletes dotted influencer fields', () => {
     const anomaly = buildMockAnomaly();
-    const result = convertAnomalyFieldsToECS(anomaly);
+    const result = transformAnomalyFieldsToEcs(anomaly);
 
     const ecsKeys = Object.keys(result);
     expect(ecsKeys).not.toContain('user.name');
@@ -58,15 +66,15 @@ describe('convertAnomalyFieldsToECS', () => {
 
   it('deletes dotted entity field', () => {
     const anomaly = buildMockAnomaly();
-    const result = convertAnomalyFieldsToECS(anomaly);
+    const result = transformAnomalyFieldsToEcs(anomaly);
 
     const ecsKeys = Object.keys(result);
     expect(ecsKeys).not.toContain('process.name');
   });
 
   it('creates nested influencer fields', () => {
     const anomaly = buildMockAnomaly();
-    const result = convertAnomalyFieldsToECS(anomaly);
+    const result = transformAnomalyFieldsToEcs(anomaly);
 
     expect(result.process.pid).toEqual(['123']);
     expect(result.user.name).toEqual(['root']);
@@ -75,7 +83,7 @@ describe('convertAnomalyFieldsToECS', () => {
 
   it('creates nested entity field', () => {
     const anomaly = buildMockAnomaly();
-    const result = convertAnomalyFieldsToECS(anomaly);
+    const result = transformAnomalyFieldsToEcs(anomaly);
 
     expect(result.process.name).toEqual(['gzip']);
   });

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts
@@ -5,6 +5,7 @@
  */
 
 import { flow, set, omit } from 'lodash/fp';
+import { SearchResponse } from 'elasticsearch';
 
 import { Logger } from '../../../../../../../../src/core/server';
 import { AlertServices } from '../../../../../../../plugins/alerting/server';
@@ -29,8 +30,17 @@ interface BulkCreateMlSignalsParams {
   tags: string[];
 }
 
-export const convertAnomalyFieldsToECS = (anomaly: Anomaly): Anomaly => {
-  const { by_field_name: entityName, by_field_value: entityValue, influencers } = anomaly;
+interface EcsAnomaly extends Anomaly {
+  '@timestamp': string;
+}
+
+export const transformAnomalyFieldsToEcs = (anomaly: Anomaly): EcsAnomaly => {
+  const {
+    by_field_name: entityName,
+    by_field_value: entityValue,
+    influencers,
+    timestamp,
+  } = anomaly;
   let errantFields = (influencers ?? []).map(influencer => ({
     name: influencer.influencer_field_name,
     value: influencer.influencer_field_values,
@@ -42,16 +52,29 @@ export const convertAnomalyFieldsToECS = (anomaly: Anomaly): Anomaly => {
 
   const omitDottedFields = omit(errantFields.map(field => field.name));
   const setNestedFields = errantFields.map(field => set(field.name, field.value));
+  const setTimestamp = set('@timestamp', new Date(timestamp).toISOString());
 
-  return flow(omitDottedFields, setNestedFields)(anomaly);
+  return flow(omitDottedFields, setNestedFields, setTimestamp)(anomaly);
 };
 
-export const bulkCreateMlSignals = async (params: BulkCreateMlSignalsParams) => {
-  const anomalies = params.someResult;
-  anomalies.hits.hits = anomalies.hits.hits.map(({ _source, ...rest }) => ({
+const transformAnomalyResultsToEcs = (results: AnomalyResults): SearchResponse<EcsAnomaly> => {
+  const transformedHits = results.hits.hits.map(({ _source, ...rest }) => ({
     ...rest,
-    _source: convertAnomalyFieldsToECS(_source),
+    _source: transformAnomalyFieldsToEcs(_source),
   }));
 
-  return singleBulkCreate({ ...params, someResult: anomalies });
+  return {
+    ...results,
+    hits: {
+      ...results.hits,
+      hits: transformedHits,
+    },
+  };
+};
+
+export const bulkCreateMlSignals = async (params: BulkCreateMlSignalsParams) => {
+  const anomalyResults = params.someResult;
+  const ecsResults = transformAnomalyResultsToEcs(anomalyResults);
+
+  return singleBulkCreate({ ...params, someResult: ecsResults });
 };