[Security Solution] Fixes data normalization in diff algorithms for `…

…threat` and `rule_schedule` fields (elastic#200105)

**Fixes elastic#199629

## Summary

Fixes the data normalization we do before comparison for the `threat`
and `rule_schedule` fields so that they align with our prebuilt rule
specs. Specifically:

- Trims any extra optional nested fields in the `threat` field that were
left as empty arrays
- Removes the logic to use the `from` value in the `meta` field if it
existed, so that we can normalize the time strings for `rule_schedule`

These errors were occurring when a rule was saved via the Rule Editing
form in the UI and extra fields were added in the update API call. This
PR makes the diff algorithms more robust against different field values
that are represented differently but are logically the same.

This extra data added in the Rule Edit UI form was also causing rules to
appear as modified when saved from the form, even if no fields had been
modified.



### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts

@@ -53,6 +53,7 @@ import { extractRuleNameOverrideObject } from './extract_rule_name_override_obje
 import { extractRuleSchedule } from './extract_rule_schedule';
 import { extractTimelineTemplateReference } from './extract_timeline_template_reference';
 import { extractTimestampOverrideObject } from './extract_timestamp_override_object';
+import { extractThreatArray } from './extract_threat_array';
 
 /**
  * Normalizes a given rule to the form which is suitable for passing to the diff algorithm.
@@ -128,7 +129,7 @@ const extractDiffableCommonFields = (
     // About -> Advanced settings
     references: rule.references ?? [],
     false_positives: rule.false_positives ?? [],
-    threat: rule.threat ?? [],
+    threat: extractThreatArray(rule),
     note: rule.note ?? '',
     setup: rule.setup ?? '',
     related_integrations: rule.related_integrations ?? [],

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_schedule.test.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getRulesSchemaMock } from '../../../api/detection_engine/model/rule_schema/mocks';
+import { extractRuleSchedule } from './extract_rule_schedule';
+
+describe('extractRuleSchedule', () => {
+  it('normalizes lookback strings to seconds', () => {
+    const mockRule = { ...getRulesSchemaMock(), from: 'now-6m', interval: '5m', to: 'now' };
+    const normalizedRuleSchedule = extractRuleSchedule(mockRule);
+
+    expect(normalizedRuleSchedule).toEqual({ interval: '5m', lookback: '60s' });
+  });
+});

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_schedule.ts

@@ -9,34 +9,17 @@ import moment from 'moment';
 import dateMath from '@elastic/datemath';
 import { parseDuration } from '@kbn/alerting-plugin/common';
 
-import type { RuleMetadata, RuleResponse } from '../../../api/detection_engine/model/rule_schema';
+import type { RuleResponse } from '../../../api/detection_engine/model/rule_schema';
 import type { RuleSchedule } from '../../../api/detection_engine/prebuilt_rules';
 
 export const extractRuleSchedule = (rule: RuleResponse): RuleSchedule => {
   const interval = rule.interval ?? '5m';
   const from = rule.from ?? 'now-6m';
   const to = rule.to ?? 'now';
 
-  const ruleMeta: RuleMetadata = ('meta' in rule ? rule.meta : undefined) ?? {};
-  const lookbackFromMeta = String(ruleMeta.from ?? '');
-
   const intervalDuration = parseInterval(interval);
-  const lookbackFromMetaDuration = parseInterval(lookbackFromMeta);
   const driftToleranceDuration = parseDriftTolerance(from, to);
 
-  if (lookbackFromMetaDuration != null) {
-    if (intervalDuration != null) {
-      return {
-        interval,
-        lookback: lookbackFromMeta,
-      };
-    }
-    return {
-      interval: `Cannot parse: interval="${interval}"`,
-      lookback: lookbackFromMeta,
-    };
-  }
-
   if (intervalDuration == null) {
     return {
       interval: `Cannot parse: interval="${interval}"`,

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_threat_array.test.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getRulesSchemaMock } from '../../../api/detection_engine/model/rule_schema/mocks';
+import { getThreatMock } from '../../schemas/types/threat.mock';
+import { extractThreatArray } from './extract_threat_array';
+
+const mockThreat = getThreatMock()[0];
+
+describe('extractThreatArray', () => {
+  it('trims empty technique fields from threat object', () => {
+    const mockRule = { ...getRulesSchemaMock(), threat: [{ ...mockThreat, technique: [] }] };
+    const normalizedThreatArray = extractThreatArray(mockRule);
+
+    expect(normalizedThreatArray).toEqual([
+      {
+        framework: 'MITRE ATT&CK',
+        tactic: {
+          id: 'TA0000',
+          name: 'test tactic',
+          reference: 'https://attack.mitre.org/tactics/TA0000/',
+        },
+      },
+    ]);
+  });
+
+  it('trims empty subtechnique fields from threat object', () => {
+    const mockRule = {
+      ...getRulesSchemaMock(),
+      threat: [{ ...mockThreat, technique: [{ ...mockThreat.technique![0], subtechnique: [] }] }],
+    };
+    const normalizedThreatArray = extractThreatArray(mockRule);
+
+    expect(normalizedThreatArray).toEqual([
+      {
+        framework: 'MITRE ATT&CK',
+        tactic: {
+          id: 'TA0000',
+          name: 'test tactic',
+          reference: 'https://attack.mitre.org/tactics/TA0000/',
+        },
+        technique: [
+          {
+            id: 'T0000',
+            name: 'test technique',
+            reference: 'https://attack.mitre.org/techniques/T0000/',
+          },
+        ],
+      },
+    ]);
+  });
+});

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_threat_array.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  RuleResponse,
+  ThreatArray,
+  ThreatTechnique,
+} from '../../../api/detection_engine/model/rule_schema';
+
+export const extractThreatArray = (rule: RuleResponse): ThreatArray =>
+  rule.threat.map((threat) => {
+    if (threat.technique && threat.technique.length) {
+      return { ...threat, technique: trimTechniqueArray(threat.technique) };
+    }
+    return { ...threat, technique: undefined }; // If `technique` is an empty array, remove the field from the `threat` object
+  });
+
+const trimTechniqueArray = (techniqueArray: ThreatTechnique[]): ThreatTechnique[] => {
+  return techniqueArray.map((technique) => ({
+    ...technique,
+    subtechnique:
+      technique.subtechnique && technique.subtechnique.length ? technique.subtechnique : undefined, // If `subtechnique` is an empty array, remove the field from the `technique` object
+  }));
+};

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts

@@ -385,7 +385,9 @@ export const filterEmptyThreats = (threats: Threats): Threats => {
           return {
             ...technique,
             subtechnique:
-              technique.subtechnique != null ? trimThreatsWithNoName(technique.subtechnique) : [],
+              technique.subtechnique != null
+                ? trimThreatsWithNoName(technique.subtechnique)
+                : undefined,
           };
         }),
       };