From a61dd5e9e5565a57a0076df58ff95e3cb4260b5c Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 19 Feb 2020 11:01:16 -0600
Subject: [PATCH 01/63] Remove unnecessary linter exceptions

Not sure what was causing issues here, but it's gone now.
---
 .../siem/public/pages/detection_engine/rules/create/index.tsx  | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/index.tsx
index d816c7e867057..ff25094ff9330 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/index.tsx
@@ -98,7 +98,6 @@ const CreateRulePageComponent: React.FC = () => {
   const userHasNoPermissions =
     canUserCRUD != null && hasManageApiKey != null ? !canUserCRUD || !hasManageApiKey : false;
 
-  // eslint-disable-next-line react-hooks/rules-of-hooks
   const setStepData = useCallback(
     (step: RuleStep, data: unknown, isValid: boolean) => {
       stepsData.current[step] = { ...stepsData.current[step], data, isValid };
@@ -138,12 +137,10 @@ const CreateRulePageComponent: React.FC = () => {
     [isStepRuleInReadOnlyView, openAccordionId, stepsData.current, setRule]
   );
 
-  // eslint-disable-next-line react-hooks/rules-of-hooks
   const setStepsForm = useCallback((step: RuleStep, form: FormHook<FormData>) => {
     stepsForm.current[step] = form;
   }, []);
 
-  // eslint-disable-next-line react-hooks/rules-of-hooks
   const getAccordionType = useCallback(
     (accordionId: RuleStep) => {
       if (accordionId === openAccordionId) {

From 5843a9ade02eb267c552dc8cadfd11fbc05cd5d3 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 19 Feb 2020 13:45:59 -0600
Subject: [PATCH 02/63] WIP: Simple form to test creation of ML rules

This will be integrated into the regular rule creation workflow, but for
now this simple form should allow us to exercise the full ML rule
workflow.
---
 .../public/pages/detection_engine/index.tsx   |  4 ++
 .../rules/create_ml/index.tsx                 | 60 +++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create_ml/index.tsx

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/index.tsx
index 1509348819510..873b948609b7e 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/index.tsx
@@ -9,6 +9,7 @@ import { Redirect, Route, Switch, RouteComponentProps } from 'react-router-dom';
 
 import { ManageUserInfo } from './components/user_info';
 import { CreateRulePage } from './rules/create';
+import { CreateMLRulePage } from './rules/create_ml';
 import { DetectionEnginePage } from './detection_engine';
 import { EditRulePage } from './rules/edit';
 import { RuleDetailsPage } from './rules/details';
@@ -35,6 +36,9 @@ const DetectionEngineContainerComponent: React.FC<Props> = () => (
       <Route exact path={`${detectionEnginePath}/rules/create`}>
         <CreateRulePage />
       </Route>
+      <Route exact path={`${detectionEnginePath}/rules/create_ml`}>
+        <CreateMLRulePage />
+      </Route>
       <Route exact path={`${detectionEnginePath}/rules/id/:detailName`}>
         <RuleDetailsPage />
       </Route>
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create_ml/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create_ml/index.tsx
new file mode 100644
index 0000000000000..a4c16c38d0473
--- /dev/null
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create_ml/index.tsx
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { useState, useCallback } from 'react';
+import { EuiButton, EuiRange, EuiSuperSelect, EuiText } from '@elastic/eui';
+
+import { useSiemJobs } from '../../../../components/ml_popover/hooks/use_siem_jobs';
+
+interface JobDisplayProps {
+  title: string;
+  description: string;
+}
+const JobDisplay = ({ title, description }: JobDisplayProps) => (
+  <>
+    <strong>{title}</strong>
+    <EuiText size="xs" color="subdued">
+      <p>{description}</p>
+    </EuiText>
+  </>
+);
+
+export const CreateMLRulePage = () => {
+  const [isLoading, siemJobs] = useSiemJobs(false);
+  const [jobId, setJobId] = useState<string>('');
+  const [threshold, setThreshold] = useState<number>(50);
+
+  const submit = useCallback(() => {
+    console.log(jobId, threshold);
+  }, [jobId, threshold]);
+
+  const onThresholdChange = useCallback(
+    (event: React.ChangeEvent<HTMLInputElement> | React.MouseEvent<HTMLButtonElement>) => {
+      setThreshold(Number((event as React.ChangeEvent<HTMLInputElement>).target.value));
+    },
+    []
+  );
+
+  const options = siemJobs.map(job => ({
+    value: job.id,
+    inputDisplay: job.id,
+    dropdownDisplay: <JobDisplay title={job.id} description={job.description} />,
+  }));
+
+  return (
+    <>
+      <EuiSuperSelect
+        hasDividers
+        isLoading={isLoading}
+        onChange={setJobId}
+        options={options}
+        valueOfSelected={jobId}
+      />
+      <EuiRange min={0} max={100} value={threshold} onChange={onThresholdChange} showInput />
+      <EuiButton onClick={submit}>Create ML Rule</EuiButton>
+    </>
+  );
+};

From 34bf43281c6d413005b854adad9e8c37e8638b66 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 19 Feb 2020 15:16:42 -0600
Subject: [PATCH 03/63] WIP: Adds POST to backend, and type/payload changes
 necessary to make that work

---
 .../detection_engine/rules/types.ts           |  4 +-
 .../rules/create_ml/index.tsx                 | 84 +++++++++++++++----
 .../routes/schemas/create_rules_schema.ts     | 12 +++
 .../routes/schemas/schemas.ts                 |  4 +-
 .../siem/server/lib/detection_engine/types.ts |  4 +-
 5 files changed, 88 insertions(+), 20 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
index 4d2aec4ee8740..e7988381a430b 100644
--- a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
@@ -18,13 +18,15 @@ export const NewRuleSchema = t.intersection([
     query: t.string,
     risk_score: t.number,
     severity: t.string,
-    type: t.union([t.literal('query'), t.literal('saved_query')]),
+    type: t.union([t.literal('query'), t.literal('saved_query'), t.literal('machine_learning')]),
   }),
   t.partial({
+    anomaly_threshold: t.number,
     created_by: t.string,
     false_positives: t.array(t.string),
     from: t.string,
     id: t.string,
+    ml_job_id: t.string,
     max_signals: t.number,
     references: t.array(t.string),
     rule_id: t.string,
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create_ml/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create_ml/index.tsx
index a4c16c38d0473..82b9b62d12abe 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create_ml/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create_ml/index.tsx
@@ -5,15 +5,18 @@
  */
 
 import React, { useState, useCallback } from 'react';
-import { EuiButton, EuiRange, EuiSuperSelect, EuiText } from '@elastic/eui';
+import { Redirect } from 'react-router-dom';
+import { EuiButton, EuiFieldText, EuiRange, EuiSuperSelect, EuiText } from '@elastic/eui';
 
+import { DETECTION_ENGINE_PAGE_NAME } from '../../../../components/link_to/redirect_to_detection_engine';
 import { useSiemJobs } from '../../../../components/ml_popover/hooks/use_siem_jobs';
+import { usePersistRule } from '../../../../containers/detection_engine/rules';
+import { RuleType } from '../../../../../server/lib/detection_engine/types';
 
-interface JobDisplayProps {
-  title: string;
-  description: string;
-}
-const JobDisplay = ({ title, description }: JobDisplayProps) => (
+type Event = React.ChangeEvent<HTMLInputElement>;
+type EventArg = Event | React.MouseEvent<HTMLButtonElement>;
+
+const JobDisplay = ({ title, description }: { title: string; description: string }) => (
   <>
     <strong>{title}</strong>
     <EuiText size="xs" color="subdued">
@@ -22,21 +25,61 @@ const JobDisplay = ({ title, description }: JobDisplayProps) => (
   </>
 );
 
+const formatRule = ({
+  jobId,
+  name,
+  threshold,
+}: {
+  jobId: string;
+  name: string;
+  threshold: number;
+}) => ({
+  ml_job_id: jobId,
+  anomaly_threshold: threshold,
+  name,
+  index: [],
+  language: 'kuery',
+  query: '',
+  filters: [],
+  false_positives: [],
+  references: [],
+  risk_score: 50,
+  threat: [],
+  severity: 'low',
+  tags: [],
+  interval: '5m',
+  from: 'now-360s',
+  enabled: false,
+  to: 'now',
+  type: 'machine_learning' as RuleType,
+  description: 'Test ML Rule',
+});
+
 export const CreateMLRulePage = () => {
-  const [isLoading, siemJobs] = useSiemJobs(false);
+  const [jobsLoading, siemJobs] = useSiemJobs(false);
+  const [{ isLoading: ruleLoading, isSaved }, setRule] = usePersistRule();
   const [jobId, setJobId] = useState<string>('');
+  const [name, setName] = useState<string>('');
   const [threshold, setThreshold] = useState<number>(50);
+  const isLoading = jobsLoading || ruleLoading;
 
   const submit = useCallback(() => {
-    console.log(jobId, threshold);
-  }, [jobId, threshold]);
-
-  const onThresholdChange = useCallback(
-    (event: React.ChangeEvent<HTMLInputElement> | React.MouseEvent<HTMLButtonElement>) => {
-      setThreshold(Number((event as React.ChangeEvent<HTMLInputElement>).target.value));
-    },
-    []
-  );
+    console.log(name, jobId, threshold);
+    setRule(
+      formatRule({
+        jobId,
+        name,
+        threshold,
+      })
+    );
+  }, [name, jobId, threshold]);
+
+  const onNameChange = useCallback((event: Event) => {
+    setName(event.target.value);
+  }, []);
+  const onThresholdChange = useCallback((event: EventArg) => {
+    setThreshold(Number((event as Event).target.value));
+  }, []);
 
   const options = siemJobs.map(job => ({
     value: job.id,
@@ -44,8 +87,13 @@ export const CreateMLRulePage = () => {
     dropdownDisplay: <JobDisplay title={job.id} description={job.description} />,
   }));
 
+  if (isSaved) {
+    return <Redirect to={`/${DETECTION_ENGINE_PAGE_NAME}/rules`} />;
+  }
+
   return (
     <>
+      <EuiFieldText onChange={onNameChange} placeholder="Rule name" value={name}></EuiFieldText>
       <EuiSuperSelect
         hasDividers
         isLoading={isLoading}
@@ -54,7 +102,9 @@ export const CreateMLRulePage = () => {
         valueOfSelected={jobId}
       />
       <EuiRange min={0} max={100} value={threshold} onChange={onThresholdChange} showInput />
-      <EuiButton onClick={submit}>Create ML Rule</EuiButton>
+      <EuiButton onClick={submit} disabled={isLoading}>
+        Create ML Rule
+      </EuiButton>
     </>
   );
 };
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts
index c9b380d3c67e1..c25438578b6c7 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts
@@ -8,6 +8,7 @@ import Joi from 'joi';
 
 /* eslint-disable @typescript-eslint/camelcase */
 import {
+  anomaly_threshold,
   enabled,
   description,
   false_positives,
@@ -34,12 +35,18 @@ import {
   references,
   note,
   version,
+  ml_job_id,
 } from './schemas';
 /* eslint-enable @typescript-eslint/camelcase */
 
 import { DEFAULT_MAX_SIGNALS } from '../../../../../common/constants';
 
 export const createRulesSchema = Joi.object({
+  anomaly_threshold: anomaly_threshold.when('type', {
+    is: 'machine_learning',
+    then: Joi.required(),
+    otherwise: Joi.forbidden(),
+  }),
   description: description.required(),
   enabled: enabled.default(true),
   false_positives: false_positives.default([]),
@@ -59,6 +66,11 @@ export const createRulesSchema = Joi.object({
   timeline_id,
   timeline_title,
   meta,
+  ml_job_id: ml_job_id.when('type', {
+    is: 'machine_learning',
+    then: Joi.required(),
+    otherwise: Joi.forbidden(),
+  }),
   risk_score: risk_score.required(),
   max_signals: max_signals.default(DEFAULT_MAX_SIGNALS),
   name: name.required(),
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts
index 2ba9ec7f83253..5049cbf73b116 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts
@@ -7,6 +7,7 @@
 import Joi from 'joi';
 
 /* eslint-disable @typescript-eslint/camelcase */
+export const anomaly_threshold = Joi.number();
 export const description = Joi.string();
 export const enabled = Joi.boolean();
 export const exclude_export_details = Joi.boolean();
@@ -48,7 +49,8 @@ export const risk_score = Joi.number()
 export const severity = Joi.string().valid('low', 'medium', 'high', 'critical');
 export const status = Joi.string().valid('open', 'closed');
 export const to = Joi.string();
-export const type = Joi.string().valid('query', 'saved_query');
+export const type = Joi.string().valid('query', 'saved_query', 'machine_learning');
+export const ml_job_id = Joi.string();
 export const queryFilter = Joi.string();
 export const references = Joi.array()
   .items(Joi.string())
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
index fa43ac1debb92..9109e793d5883 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
@@ -22,6 +22,8 @@ export interface ThreatParams {
   technique: IMitreAttack[];
 }
 
+export type RuleType = 'query' | 'saved_query' | 'machine_learning';
+
 export interface RuleAlertParams {
   description: string;
   note: string | undefined | null;
@@ -48,7 +50,7 @@ export interface RuleAlertParams {
   timelineId: string | undefined | null;
   timelineTitle: string | undefined | null;
   threat: ThreatParams[] | undefined | null;
-  type: 'query' | 'saved_query';
+  type: RuleType;
   version: number;
   throttle?: string;
 }

From 56a07666d52096852a11dfcb0b3a646a58316b9c Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 19 Feb 2020 17:04:58 -0600
Subject: [PATCH 04/63] Simplify logic with Math.min

---
 .../lib/detection_engine/signals/signal_rule_alert_type.ts | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index b467dfdaff305..5c5afdd33d931 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -176,11 +176,8 @@ export const signalRulesAlertType = ({
           );
         }
       }
-      // set searchAfter page size to be the lesser of default page size or maxSignals.
-      const searchAfterSize =
-        DEFAULT_SEARCH_AFTER_PAGE_SIZE <= params.maxSignals
-          ? DEFAULT_SEARCH_AFTER_PAGE_SIZE
-          : params.maxSignals;
+
+      const searchAfterSize = Math.min(params.maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE);
       try {
         const inputIndex = await getInputIndex(services, version, index);
         const esFilter = await getFilter({

From 0b19ddc1aa1b544580f5a898fb014b4d0299c392 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 19 Feb 2020 17:58:11 -0600
Subject: [PATCH 05/63] WIP: Failed spike of making an http call

---
 .../routes/__mocks__/request_responses.ts     | 12 +++++++
 .../routes/rules/create_rules_route.test.ts   | 10 +++++-
 .../signals/find_ml_signals.ts                | 10 ++++++
 .../signals/signal_rule_alert_type.ts         | 12 +++++++
 .../siem/server/lib/machine_learning/index.ts | 32 +++++++++++++++++++
 5 files changed, 75 insertions(+), 1 deletion(-)
 create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
 create mode 100644 x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
index 54ed42a1d2b6c..3091d3e31cc46 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
@@ -293,6 +293,18 @@ export const getCreateRequest = () =>
     body: typicalPayload(),
   });
 
+export const createMlRuleRequest = () =>
+  requestMock.create({
+    method: 'post',
+    path: DETECTION_ENGINE_RULES_URL,
+    body: {
+      ...typicalPayload(),
+      type: 'machine_learning',
+      anomaly_threshold: 50,
+      ml_job_id: 'some-uuid',
+    },
+  });
+
 export const getSetSignalStatusByIdsRequest = () =>
   requestMock.create({
     method: 'post',
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts
index d019668e2a8d1..0ae09c5cb1661 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts
@@ -14,6 +14,7 @@ import {
   getNonEmptyIndex,
   getEmptyIndex,
   getFindResultWithSingleHit,
+  createMlRuleRequest,
 } from '../__mocks__/request_responses';
 import { requestContextMock, serverMock, requestMock } from '../__mocks__';
 import { createRulesRoute } from './create_rules_route';
@@ -48,6 +49,13 @@ describe('create_rules', () => {
     });
   });
 
+  describe('creating an ML Rule', () => {
+    it('works', async () => {
+      const response = await server.inject(createMlRuleRequest(), context);
+      expect(response.status).toEqual(200);
+    });
+  });
+
   describe('unhappy paths', () => {
     test('it returns a 400 if the index does not exist', async () => {
       clients.clusterClient.callAsCurrentUser.mockResolvedValue(getEmptyIndex());
@@ -111,7 +119,7 @@ describe('create_rules', () => {
       const result = server.validate(request);
 
       expect(result.badRequest).toHaveBeenCalledWith(
-        'child "type" fails because ["type" must be one of [query, saved_query]]'
+        'child "type" fails because ["type" must be one of [query, saved_query, machine_learning]]'
       );
     });
   });
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
new file mode 100644
index 0000000000000..a2500173d73d7
--- /dev/null
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { getAnomaliesTableData } from '../../machine_learning';
+
+export const findMlSignals = () => {
+  return [];
+};
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index 5c5afdd33d931..c5e03c0112e41 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -22,6 +22,8 @@ import { SignalRuleAlertTypeDefinition } from './types';
 import { getGapBetweenRuns } from './utils';
 import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings';
 import { IRuleSavedAttributesSavedObjectAttributes } from '../rules/types';
+import { findMlSignals } from './find_ml_signals';
+
 interface AlertAttributes {
   enabled: boolean;
   name: string;
@@ -33,6 +35,7 @@ interface AlertAttributes {
     interval: string;
   };
 }
+
 export const signalRulesAlertType = ({
   logger,
   version,
@@ -177,6 +180,15 @@ export const signalRulesAlertType = ({
         }
       }
 
+      if (type === 'machine_learning') {
+        const signals = await findMlSignals(savedObject);
+        if (signals.length) {
+          logger.debug(`Found ${signals.length} signals`);
+        }
+
+        return;
+      }
+
       const searchAfterSize = Math.min(params.maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE);
       try {
         const inputIndex = await getInputIndex(services, version, index);
diff --git a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
new file mode 100644
index 0000000000000..7982fe79e16b7
--- /dev/null
+++ b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+type MlService = any;
+
+export interface Params {
+  jobIds: string[];
+  criteriaFields: CriteriaFields[];
+  influencers: InfluencerInput[];
+  aggregationInterval: string;
+  threshold: number;
+  earliestMs: number;
+  latestMs: number;
+  dateFormatTz: string;
+  maxRecords: number;
+  maxExamples: number;
+}
+
+export const anomaliesTableData = async (params: Params): Promise<Anomalies> => {
+  const response = await MlService.get().fetch<Anomalies>('/api/ml/results/anomalies_table_data', {
+    method: 'POST',
+    body: JSON.stringify(body),
+    asSystemRequest: true,
+  });
+
+  return response;
+};
+
+export const getAnomaliesTableData = () => {};

From 28ede95bdc9c787f1d845830fb1cc39f2548d187 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 10 Mar 2020 13:40:40 -0500
Subject: [PATCH 06/63] WIP: Hacking together an ML client

The rest of this is going to be easier if I have actual data. For now
this is mostly copy/pasted and simplified ML code. I've hardcoded time
ranges to a period I know has data for a particular job.
---
 .../signals/find_ml_signals.ts                |  27 ++-
 .../siem/server/lib/machine_learning/index.ts | 160 ++++++++++++++++--
 2 files changed, 169 insertions(+), 18 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
index a2500173d73d7..49c29b221aa84 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
@@ -3,8 +3,29 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { getAnomaliesTableData } from '../../machine_learning';
 
-export const findMlSignals = () => {
-  return [];
+import dateMath from '@elastic/datemath';
+
+import { AlertServices } from '../../../../../../../plugins/alerting/server';
+
+import { anomaliesTableData } from '../../machine_learning';
+
+export const findMlSignals = async (
+  jobId: string,
+  anomalyThreshold: number,
+  from: string,
+  to: string,
+  callCluster: AlertServices['callCluster']
+) => {
+  const params = {
+    jobIds: [jobId],
+    threshold: anomalyThreshold,
+    // earliestMs: dateMath.parse(from)!.valueOf(),
+    // latestMs: dateMath.parse(to)!.valueOf(),
+    earliestMs: 1583431900000,
+    latestMs: 1583431900000 + 86400000,
+  };
+  const relevantAnomalies = await anomaliesTableData(params, callCluster);
+
+  return relevantAnomalies;
 };
diff --git a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
index 7982fe79e16b7..dd5eb5cdd0b5e 100644
--- a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
@@ -4,29 +4,159 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-type MlService = any;
+import _ from 'lodash';
+import moment from 'moment';
+import { SearchResponse } from 'elasticsearch';
 
-export interface Params {
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { buildAnomalyTableItems } from '../../../../../../plugins/ml/server/models/results_service/build_anomaly_table_items';
+import { CriteriaFields, InfluencerInput, Anomalies } from '../../../public/components/ml/types';
+import { AlertServices } from '../../../../../../plugins/alerting/server';
+
+export interface AnomaliesSearchParams {
   jobIds: string[];
-  criteriaFields: CriteriaFields[];
-  influencers: InfluencerInput[];
-  aggregationInterval: string;
   threshold: number;
   earliestMs: number;
   latestMs: number;
-  dateFormatTz: string;
-  maxRecords: number;
-  maxExamples: number;
+  maxRecords?: number;
 }
 
-export const anomaliesTableData = async (params: Params): Promise<Anomalies> => {
-  const response = await MlService.get().fetch<Anomalies>('/api/ml/results/anomalies_table_data', {
-    method: 'POST',
-    body: JSON.stringify(body),
-    asSystemRequest: true,
+export const anomaliesTableData = async (
+  params: AnomaliesSearchParams,
+  callCluster: AlertServices['callCluster']
+): Promise<Anomalies> => {
+  const boolCriteria = buildCriteria(params);
+
+  const anomalies: SearchResponse<any> = await callCluster('search', {
+    index: '.ml-anomalies-*',
+    rest_total_hits_as_int: true,
+    size: params.maxRecords || 100,
+    body: {
+      query: {
+        bool: {
+          filter: [
+            {
+              query_string: {
+                query: 'result_type:record',
+                analyze_wildcard: false,
+              },
+            },
+            {
+              bool: {
+                must: boolCriteria,
+              },
+            },
+          ],
+        },
+      },
+      sort: [{ record_score: { order: 'desc' } }],
+    },
   });
 
-  return response;
+  return aggregate(anomalies);
+};
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const buildCriteria = (params: AnomaliesSearchParams): any => {
+  const { earliestMs, jobIds, latestMs, threshold } = params;
+  const jobIdsFilterable =
+    jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*');
+
+  const boolCriteria: object[] = [
+    {
+      range: {
+        timestamp: {
+          gte: earliestMs,
+          lte: latestMs,
+          format: 'epoch_millis',
+        },
+      },
+    },
+    {
+      range: {
+        record_score: {
+          gte: threshold,
+        },
+      },
+    },
+  ];
+
+  if (jobIdsFilterable) {
+    const jobIdFilter = jobIds.map(jobId => `job_id:${jobId}`).join(' OR ');
+
+    boolCriteria.push({
+      query_string: {
+        analyze_wildcard: false,
+        query: jobIdFilter,
+      },
+    });
+  }
+
+  return boolCriteria;
 };
 
-export const getAnomaliesTableData = () => {};
+const aggregate = (anomalies: SearchResponse<any>): Anomalies => {
+  const aggregationInterval = 'auto';
+  const tableData: {
+    anomalies: AnomaliesTableRecord[];
+    interval: string;
+    examplesByJobId?: { [key: string]: any };
+  } = {
+    anomalies: [],
+    interval: 'second',
+  };
+  if (anomalies.hits.total !== 0) {
+    let records: AnomalyRecordDoc[] = [];
+    anomalies.hits.hits.forEach(hit => {
+      records.push(hit._source);
+    });
+
+    // Sort anomalies in ascending time order.
+    records = _.sortBy(records, 'timestamp');
+    tableData.interval = aggregationInterval;
+    if (aggregationInterval === 'auto') {
+      // Determine the actual interval to use if aggregating.
+      const earliest = moment(records[0].timestamp);
+      const latest = moment(records[records.length - 1].timestamp);
+
+      const daysDiff = latest.diff(earliest, 'days');
+      tableData.interval = daysDiff < 2 ? 'hour' : 'day';
+    }
+
+    tableData.anomalies = buildAnomalyTableItems(records, tableData.interval, 'America/Chicago');
+
+    // Load examples for any categorization anomalies.
+    const categoryAnomalies = tableData.anomalies.filter(
+      (item: any) => item.entityName === 'mlcategory'
+    );
+    if (categoryAnomalies.length > 0) {
+      tableData.examplesByJobId = {};
+
+      const categoryIdsByJobId: { [key: string]: any } = {};
+      categoryAnomalies.forEach(anomaly => {
+        if (!_.has(categoryIdsByJobId, anomaly.jobId)) {
+          categoryIdsByJobId[anomaly.jobId] = [];
+        }
+        if (categoryIdsByJobId[anomaly.jobId].indexOf(anomaly.entityValue) === -1) {
+          categoryIdsByJobId[anomaly.jobId].push(anomaly.entityValue);
+        }
+      });
+
+      // const categoryJobIds = Object.keys(categoryIdsByJobId);
+      // await Promise.all(
+      //   categoryJobIds.map(async jobId => {
+      //     const examplesByCategoryId = await getCategoryExamples(
+      //       jobId,
+      //       categoryIdsByJobId[jobId],
+      //       maxExamples
+      //     );
+      //     if (tableData.examplesByJobId !== undefined) {
+      //       tableData.examplesByJobId[jobId] = examplesByCategoryId;
+      //     }
+      //   })
+      // );
+    }
+  }
+
+  return tableData;
+};

From bec1a90f3437d05c9663e297bf05d3ae3b8ee481 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 10 Mar 2020 13:42:28 -0500
Subject: [PATCH 07/63] Threading through our new ML Rule params

It's a bummer that we normalize our rule alert params across all rule
types currently, but that's the deal.
---
 .../routes/rules/create_rules_route.ts        |  4 ++++
 .../routes/schemas/response/schemas.ts        |  2 +-
 .../detection_engine/rules/create_rules.ts    |  4 ++++
 .../lib/detection_engine/rules/types.ts       | 20 +++++++++++++++-
 .../signals/signal_rule_alert_type.ts         | 23 ++++++++-----------
 .../siem/server/lib/detection_engine/types.ts |  2 ++
 6 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts
index fcfcee99f369e..04a7336e451e1 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts
@@ -31,6 +31,7 @@ export const createRulesRoute = (router: IRouter): void => {
     },
     async (context, request, response) => {
       const {
+        anomaly_threshold: anomalyThreshold,
         description,
         enabled,
         false_positives: falsePositives,
@@ -42,6 +43,7 @@ export const createRulesRoute = (router: IRouter): void => {
         timeline_id: timelineId,
         timeline_title: timelineTitle,
         meta,
+        ml_job_id: mlJobId,
         filters,
         rule_id: ruleId,
         index,
@@ -93,6 +95,7 @@ export const createRulesRoute = (router: IRouter): void => {
         const createdRule = await createRules({
           alertsClient,
           actionsClient,
+          anomalyThreshold,
           description,
           enabled,
           falsePositives,
@@ -105,6 +108,7 @@ export const createRulesRoute = (router: IRouter): void => {
           timelineId,
           timelineTitle,
           meta,
+          mlJobId,
           filters,
           ruleId: ruleId ?? uuid.v4(),
           index,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts
index 16f6c0fd6b8b4..c4ed3db677aa8 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts
@@ -64,7 +64,7 @@ export const job_status = t.keyof({ succeeded: null, failed: null, 'going to run
 // TODO: Create a regular expression type or custom date math part type here
 export const to = t.string;
 
-export const type = t.keyof({ query: null, saved_query: null });
+export const type = t.keyof({ machine_learning: null, query: null, saved_query: null });
 export const queryFilter = t.string;
 export const references = t.array(t.string);
 export const per_page = PositiveInteger;
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts
index ea87950a59b78..5885653051d69 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts
@@ -12,6 +12,7 @@ import { addTags } from './add_tags';
 export const createRules = ({
   alertsClient,
   actionsClient, // TODO: Use this actionsClient once we have actions such as email, etc...
+  anomalyThreshold,
   description,
   enabled,
   falsePositives,
@@ -22,6 +23,7 @@ export const createRules = ({
   timelineId,
   timelineTitle,
   meta,
+  mlJobId,
   filters,
   ruleId,
   immutable,
@@ -47,6 +49,7 @@ export const createRules = ({
       alertTypeId: SIGNALS_ID,
       consumer: APP_ID,
       params: {
+        anomalyThreshold,
         description,
         ruleId,
         index,
@@ -60,6 +63,7 @@ export const createRules = ({
         timelineId,
         timelineTitle,
         meta,
+        mlJobId,
         filters,
         maxSignals,
         riskScore,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts
index 1efa46c6b8b57..713ce214e41da 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts
@@ -147,6 +147,11 @@ export interface Clients {
   actionsClient: ActionsClient;
 }
 
+export interface MlRuleParams {
+  anomalyThreshold: number;
+  mlJobId: string;
+}
+
 export type PatchRuleParams = Partial<RuleAlertParams> & {
   id: string | undefined | null;
   savedObjectsClient: SavedObjectsClientContract;
@@ -162,7 +167,8 @@ export type DeleteRuleParams = Clients & {
   ruleId: string | undefined | null;
 };
 
-export type CreateRuleParams = Omit<RuleAlertParams, 'ruleId'> & { ruleId: string } & Clients;
+export type CreateRuleParams = Omit<RuleAlertParams, 'ruleId'> & { ruleId: string } & Clients &
+  Partial<MlRuleParams>;
 
 export interface ReadRuleParams {
   alertsClient: AlertsClient;
@@ -195,3 +201,15 @@ export const isRuleStatusFindTypes = (
 ): obj is Array<SavedObjectsFindResponse<IRuleSavedAttributesSavedObjectAttributes>> => {
   return obj ? obj.every(ruleStatus => isRuleStatusFindType(ruleStatus)) : false;
 };
+
+export interface RuleAlertAttributes {
+  enabled: boolean;
+  name: string;
+  tags: string[];
+  createdBy: string;
+  createdAt: string;
+  updatedBy: string;
+  schedule: {
+    interval: string;
+  };
+}
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index c5e03c0112e41..e84119a957c24 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -21,21 +21,9 @@ import { getFilter } from './get_filter';
 import { SignalRuleAlertTypeDefinition } from './types';
 import { getGapBetweenRuns } from './utils';
 import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings';
-import { IRuleSavedAttributesSavedObjectAttributes } from '../rules/types';
+import { IRuleSavedAttributesSavedObjectAttributes, RuleAlertAttributes } from '../rules/types';
 import { findMlSignals } from './find_ml_signals';
 
-interface AlertAttributes {
-  enabled: boolean;
-  name: string;
-  tags: string[];
-  createdBy: string;
-  createdAt: string;
-  updatedBy: string;
-  schedule: {
-    interval: string;
-  };
-}
-
 export const signalRulesAlertType = ({
   logger,
   version,
@@ -57,6 +45,7 @@ export const signalRulesAlertType = ({
     defaultActionGroupId: 'default',
     validate: {
       params: schema.object({
+        anomalyThreshold: schema.nullable(schema.number()),
         description: schema.string(),
         note: schema.nullable(schema.string()),
         falsePositives: schema.arrayOf(schema.string(), { defaultValue: [] }),
@@ -73,6 +62,7 @@ export const signalRulesAlertType = ({
         query: schema.nullable(schema.string()),
         filters: schema.nullable(schema.arrayOf(schema.object({}, { allowUnknowns: true }))),
         maxSignals: schema.number({ defaultValue: DEFAULT_MAX_SIGNALS }),
+        mlJobId: schema.nullable(schema.string()),
         riskScore: schema.number(),
         severity: schema.string(),
         threat: schema.nullable(schema.arrayOf(schema.object({}, { allowUnknowns: true }))),
@@ -85,11 +75,13 @@ export const signalRulesAlertType = ({
     // fun fact: previousStartedAt is not actually a Date but a String of a date
     async executor({ previousStartedAt, alertId, services, params }) {
       const {
+        anomalyThreshold,
         from,
         ruleId,
         index,
         filters,
         language,
+        mlJobId,
         outputIndex,
         savedId,
         query,
@@ -97,7 +89,10 @@ export const signalRulesAlertType = ({
         type,
       } = params;
       // TODO: Remove this hard extraction of name once this is fixed: https://github.com/elastic/kibana/issues/50522
-      const savedObject = await services.savedObjectsClient.get<AlertAttributes>('alert', alertId);
+      const savedObject = await services.savedObjectsClient.get<RuleAlertAttributes>(
+        'alert',
+        alertId
+      );
       const ruleStatusSavedObjects = await services.savedObjectsClient.find<
         IRuleSavedAttributesSavedObjectAttributes
       >({
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
index 9109e793d5883..8c23c407f22f0 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
@@ -25,6 +25,7 @@ export interface ThreatParams {
 export type RuleType = 'query' | 'saved_query' | 'machine_learning';
 
 export interface RuleAlertParams {
+  anomalyThreshold?: number;
   description: string;
   note: string | undefined | null;
   enabled: boolean;
@@ -37,6 +38,7 @@ export interface RuleAlertParams {
   ruleId: string | undefined | null;
   language: string | undefined | null;
   maxSignals: number;
+  mlJobId?: string;
   riskScore: number;
   outputIndex: string;
   name: string;

From d9639af6ea056acbf8d2d05ef9193b7d97ea8e6f Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 10 Mar 2020 13:43:37 -0500
Subject: [PATCH 08/63] Retrieve our anomalies during rule execution

Next step: generate signals
---
 .../signals/signal_rule_alert_type.ts            | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index e84119a957c24..ffeb40e67e08c 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -176,9 +176,19 @@ export const signalRulesAlertType = ({
       }
 
       if (type === 'machine_learning') {
-        const signals = await findMlSignals(savedObject);
-        if (signals.length) {
-          logger.debug(`Found ${signals.length} signals`);
+        const { anomalies, interval: _interval } = await findMlSignals(
+          mlJobId!,
+          anomalyThreshold!,
+          from,
+          to,
+          services.callCluster
+        );
+
+        console.log('foundMlSignals', anomalies);
+        if (anomalies.length) {
+          logger.info(
+            `Found ${anomalies.length} anomalies in interval ${_interval}; generating signals`
+          );
         }
 
         return;

From 190c7bb6ea4376baea01922f5a093a4aad03728e Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 11 Mar 2020 21:25:59 -0500
Subject: [PATCH 09/63] WIP: Generate ECS-compatible ML Signals

This uses as much of the existing signal-creation code as possible. I
skipped the search_after stuff for now because it would require us
recreating the anomalies query which we really shouldn't own. For now,
here's how it works:

* Adds a separate branch of the rule executor for machine_learning rules
* In that branch, we call our new bulkCreateMlSignal function
  * This function first transforms the anomaly document into ECS fields
  * We then pass the transformed documents to singleBulkCreate, which
  does the rest
* After both branches, we update the rule's status appropriately.

We need to do some more work on the anomaly transformation, but this
works!
---
 .../detection_engine/signals/build_rule.ts    |   2 +
 .../signals/bulk_create_ml_signals.ts         |  72 ++++++++
 .../signals/search_after_bulk_create.ts       |   2 +-
 .../signals/signal_rule_alert_type.ts         | 171 ++++++++++--------
 .../signals/single_bulk_create.ts             |   4 +-
 .../lib/detection_engine/signals/types.ts     |   2 +-
 .../siem/server/lib/machine_learning/index.ts |  79 +-------
 7 files changed, 180 insertions(+), 152 deletions(-)
 create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts
index 9baf6a55b7f48..9a94d647aee5d 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts
@@ -65,5 +65,7 @@ export const buildRule = ({
     version: ruleParams.version,
     created_at: createdAt,
     updated_at: updatedAt,
+    ml_job_id: ruleParams.mlJobId,
+    anomaly_threshold: ruleParams.anomalyThreshold,
   });
 };
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts
new file mode 100644
index 0000000000000..060fd4da16ac3
--- /dev/null
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SearchResponse } from 'elasticsearch';
+import { flow, set, omit } from 'lodash/fp';
+
+import { Logger } from '../../../../../../../../src/core/server';
+import { AlertServices } from '../../../../../../../plugins/alerting/server';
+import { RuleTypeParams } from '../types';
+import { singleBulkCreate } from './single_bulk_create';
+import { Influencer } from '../../../../public/components/ml/types';
+
+interface Anomaly {
+  job_id: string;
+  record_score: number;
+  timestamp: number;
+  by_field_name: string;
+  by_field_value: string;
+  influencers?: Influencer[];
+}
+
+type AnomalyResults = SearchResponse<Anomaly>;
+
+interface BulkCreateMlSignalsParams {
+  someResult: AnomalyResults;
+  ruleParams: RuleTypeParams;
+  services: AlertServices;
+  logger: Logger;
+  id: string;
+  signalsIndex: string;
+  name: string;
+  createdAt: string;
+  createdBy: string;
+  updatedAt: string;
+  updatedBy: string;
+  interval: string;
+  enabled: boolean;
+  tags: string[];
+}
+
+const convertAnomalyFieldsToECS = (anomaly: Anomaly): Anomaly => {
+  const {
+    by_field_name: entityName,
+    by_field_value: entityValue,
+    influencers: maybeInfluencers,
+  } = anomaly;
+  const influencers = maybeInfluencers ?? [];
+
+  const setEntityField = set(entityName, entityValue);
+  const setInfluencerFields = influencers.map(influencer =>
+    set(influencer.influencer_field_name, influencer.influencer_field_values)
+  );
+  const omitDottedFields = omit([
+    entityName,
+    ...influencers.map(influencer => influencer.influencer_field_name),
+  ]);
+
+  return flow(omitDottedFields, setEntityField, setInfluencerFields)(anomaly);
+};
+
+export const bulkCreateMlSignals = async (params: BulkCreateMlSignalsParams) => {
+  const anomalies = params.someResult;
+  anomalies.hits.hits = anomalies.hits.hits.map(({ _source, ...rest }) => ({
+    ...rest,
+    _source: convertAnomalyFieldsToECS(_source),
+  }));
+
+  return singleBulkCreate({ ...params, someResult: anomalies });
+};
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
index 1cfd2f812a195..31c16a87fa0ab 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
@@ -77,7 +77,7 @@ export const searchAfterAndBulkCreate = async ({
   // If the total number of hits for the overall search result is greater than
   // maxSignals, default to requesting a total of maxSignals, otherwise use the
   // totalHits in the response from the searchAfter query.
-  const maxTotalHitsSize = totalHits >= ruleParams.maxSignals ? ruleParams.maxSignals : totalHits;
+  const maxTotalHitsSize = Math.max(totalHits, ruleParams.maxSignals);
 
   // number of docs in the current search result
   let hitsSize = someResult.hits.hits.length;
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index ffeb40e67e08c..9580b4f2409ce 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -23,6 +23,7 @@ import { getGapBetweenRuns } from './utils';
 import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings';
 import { IRuleSavedAttributesSavedObjectAttributes, RuleAlertAttributes } from '../rules/types';
 import { findMlSignals } from './find_ml_signals';
+import { bulkCreateMlSignals } from './bulk_create_ml_signals';
 
 export const signalRulesAlertType = ({
   logger,
@@ -175,73 +176,36 @@ export const signalRulesAlertType = ({
         }
       }
 
-      if (type === 'machine_learning') {
-        const { anomalies, interval: _interval } = await findMlSignals(
-          mlJobId!,
-          anomalyThreshold!,
-          from,
-          to,
-          services.callCluster
-        );
-
-        console.log('foundMlSignals', anomalies);
-        if (anomalies.length) {
-          logger.info(
-            `Found ${anomalies.length} anomalies in interval ${_interval}; generating signals`
-          );
-        }
-
-        return;
-      }
-
       const searchAfterSize = Math.min(params.maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE);
-      try {
-        const inputIndex = await getInputIndex(services, version, index);
-        const esFilter = await getFilter({
-          type,
-          filters,
-          language,
-          query,
-          savedId,
-          services,
-          index: inputIndex,
-        });
 
-        const noReIndex = buildEventsSearchQuery({
-          index: inputIndex,
-          from,
-          to,
-          filter: esFilter,
-          size: searchAfterSize,
-          searchAfterSortId: undefined,
-        });
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      let bulkIndexResult: any = null;
 
-        try {
-          logger.debug(
-            `Starting signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
-          );
-          logger.debug(
-            `[+] Initial search call of signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
+      try {
+        if (type === 'machine_learning') {
+          const anomalyResults = await findMlSignals(
+            mlJobId!,
+            anomalyThreshold!,
+            from,
+            to,
+            services.callCluster
           );
-          const noReIndexResult = await services.callCluster('search', noReIndex);
-          if (noReIndexResult.hits.total.value !== 0) {
+
+          // console.log('foundMlSignals', JSON.stringify(anomalyResults, null, 2));
+          const anomalyCount = anomalyResults.hits.hits.length;
+          if (anomalyCount) {
             logger.info(
-              `Found ${
-                noReIndexResult.hits.total.value
-              } signals from the indexes of "[${inputIndex.join(
-                ', '
-              )}]" using signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", pushing signals to index "${outputIndex}"`
+              `Found ${anomalyCount} anomalies in interval [${from}, ${to}]; generating signals`
             );
           }
 
-          const bulkIndexResult = await searchAfterAndBulkCreate({
-            someResult: noReIndexResult,
+          bulkIndexResult = await bulkCreateMlSignals({
+            someResult: anomalyResults,
             ruleParams: params,
             services,
             logger,
             id: alertId,
             signalsIndex: outputIndex,
-            filter: esFilter,
             name,
             createdBy,
             createdAt,
@@ -249,35 +213,74 @@ export const signalRulesAlertType = ({
             updatedAt,
             interval,
             enabled,
-            pageSize: searchAfterSize,
             tags,
           });
+        } else {
+          const inputIndex = await getInputIndex(services, version, index);
+          const esFilter = await getFilter({
+            type,
+            filters,
+            language,
+            query,
+            savedId,
+            services,
+            index: inputIndex,
+          });
 
-          if (bulkIndexResult) {
+          const noReIndex = buildEventsSearchQuery({
+            index: inputIndex,
+            from,
+            to,
+            filter: esFilter,
+            size: searchAfterSize,
+            searchAfterSortId: undefined,
+          });
+
+          try {
             logger.debug(
-              `Finished signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
+              `Starting signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
             );
-            const sDate = new Date().toISOString();
-            currentStatusSavedObject.attributes.status = 'succeeded';
-            currentStatusSavedObject.attributes.statusDate = sDate;
-            currentStatusSavedObject.attributes.lastSuccessAt = sDate;
-            currentStatusSavedObject.attributes.lastSuccessMessage = 'succeeded';
-            await services.savedObjectsClient.update(
-              ruleStatusSavedObjectType,
-              currentStatusSavedObject.id,
-              {
-                ...currentStatusSavedObject.attributes,
-              }
+            logger.debug(
+              `[+] Initial search call of signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
             );
-          } else {
+            const noReIndexResult = await services.callCluster('search', noReIndex);
+            if (noReIndexResult.hits.total.value !== 0) {
+              logger.info(
+                `Found ${
+                  noReIndexResult.hits.total.value
+                } signals from the indexes of "[${inputIndex.join(
+                  ', '
+                )}]" using signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", pushing signals to index "${outputIndex}"`
+              );
+            }
+
+            bulkIndexResult = await searchAfterAndBulkCreate({
+              someResult: noReIndexResult,
+              ruleParams: params,
+              services,
+              logger,
+              id: alertId,
+              signalsIndex: outputIndex,
+              filter: esFilter,
+              name,
+              createdBy,
+              createdAt,
+              updatedBy,
+              updatedAt,
+              interval,
+              enabled,
+              pageSize: searchAfterSize,
+              tags,
+            });
+          } catch (err) {
             logger.error(
-              `Error processing signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
+              `Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", ${err.message}\n${err.stack}`
             );
             const sDate = new Date().toISOString();
             currentStatusSavedObject.attributes.status = 'failed';
             currentStatusSavedObject.attributes.statusDate = sDate;
             currentStatusSavedObject.attributes.lastFailureAt = sDate;
-            currentStatusSavedObject.attributes.lastFailureMessage = `Bulk Indexing signals failed. Check logs for further details \nRule name: "${name}"\nid: "${alertId}"\nrule_id: "${ruleId}"\n`;
+            currentStatusSavedObject.attributes.lastFailureMessage = err.message;
             // current status is failing
             await services.savedObjectsClient.update(
               ruleStatusSavedObjectType,
@@ -299,15 +302,33 @@ export const signalRulesAlertType = ({
               );
             }
           }
-        } catch (err) {
+        }
+
+        if (bulkIndexResult) {
+          logger.debug(
+            `Finished signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
+          );
+          const sDate = new Date().toISOString();
+          currentStatusSavedObject.attributes.status = 'succeeded';
+          currentStatusSavedObject.attributes.statusDate = sDate;
+          currentStatusSavedObject.attributes.lastSuccessAt = sDate;
+          currentStatusSavedObject.attributes.lastSuccessMessage = 'succeeded';
+          await services.savedObjectsClient.update(
+            ruleStatusSavedObjectType,
+            currentStatusSavedObject.id,
+            {
+              ...currentStatusSavedObject.attributes,
+            }
+          );
+        } else {
           logger.error(
-            `Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", ${err.message}`
+            `Error processing signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
           );
           const sDate = new Date().toISOString();
           currentStatusSavedObject.attributes.status = 'failed';
           currentStatusSavedObject.attributes.statusDate = sDate;
           currentStatusSavedObject.attributes.lastFailureAt = sDate;
-          currentStatusSavedObject.attributes.lastFailureMessage = err.message;
+          currentStatusSavedObject.attributes.lastFailureMessage = `Bulk Indexing signals failed. Check logs for further details \nRule name: "${name}"\nid: "${alertId}"\nrule_id: "${ruleId}"\n`;
           // current status is failing
           await services.savedObjectsClient.update(
             ruleStatusSavedObjectType,
@@ -331,7 +352,7 @@ export const signalRulesAlertType = ({
         }
       } catch (exception) {
         logger.error(
-          `Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}" message: ${exception.message}`
+          `Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}" message: ${exception.message}, trace: \n${exception.stack}`
         );
         const sDate = new Date().toISOString();
         currentStatusSavedObject.attributes.status = 'failed';
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts
index 7d6d6d99fa422..642329f480dfa 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts
@@ -12,9 +12,11 @@ import { RuleTypeParams } from '../types';
 import { generateId } from './utils';
 import { buildBulkBody } from './build_bulk_body';
 import { Logger } from '../../../../../../../../src/core/server';
+import { SearchResponse } from '../../types';
 
 interface SingleBulkCreateParams {
-  someResult: SignalSearchResponse;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  someResult: SearchResponse<any>;
   ruleParams: RuleTypeParams;
   services: AlertServices;
   logger: Logger;
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts
index 7442545117310..b7cebd57ce2d1 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts
@@ -104,7 +104,7 @@ export interface GetResponse {
 }
 
 export type SignalSearchResponse = SearchResponse<SignalSource>;
-export type SignalSourceHit = SignalSearchResponse['hits']['hits'][0];
+export type SignalSourceHit = SignalSearchResponse['hits']['hits'][number];
 
 export type RuleExecutorOptions = Omit<AlertExecutorOptions, 'params'> & {
   params: RuleAlertParams & {
diff --git a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
index dd5eb5cdd0b5e..b9cd494ae450a 100644
--- a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
@@ -5,12 +5,8 @@
  */
 
 import _ from 'lodash';
-import moment from 'moment';
 import { SearchResponse } from 'elasticsearch';
 
-// eslint-disable-next-line @kbn/eslint/no-restricted-paths
-import { buildAnomalyTableItems } from '../../../../../../plugins/ml/server/models/results_service/build_anomaly_table_items';
-import { CriteriaFields, InfluencerInput, Anomalies } from '../../../public/components/ml/types';
 import { AlertServices } from '../../../../../../plugins/alerting/server';
 
 export interface AnomaliesSearchParams {
@@ -21,13 +17,16 @@ export interface AnomaliesSearchParams {
   maxRecords?: number;
 }
 
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type Anomaly = any;
+
 export const anomaliesTableData = async (
   params: AnomaliesSearchParams,
   callCluster: AlertServices['callCluster']
-): Promise<Anomalies> => {
+): Promise<SearchResponse<Anomaly>> => {
   const boolCriteria = buildCriteria(params);
 
-  const anomalies: SearchResponse<any> = await callCluster('search', {
+  return callCluster('search', {
     index: '.ml-anomalies-*',
     rest_total_hits_as_int: true,
     size: params.maxRecords || 100,
@@ -52,8 +51,6 @@ export const anomaliesTableData = async (
       sort: [{ record_score: { order: 'desc' } }],
     },
   });
-
-  return aggregate(anomalies);
 };
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -94,69 +91,3 @@ const buildCriteria = (params: AnomaliesSearchParams): any => {
 
   return boolCriteria;
 };
-
-const aggregate = (anomalies: SearchResponse<any>): Anomalies => {
-  const aggregationInterval = 'auto';
-  const tableData: {
-    anomalies: AnomaliesTableRecord[];
-    interval: string;
-    examplesByJobId?: { [key: string]: any };
-  } = {
-    anomalies: [],
-    interval: 'second',
-  };
-  if (anomalies.hits.total !== 0) {
-    let records: AnomalyRecordDoc[] = [];
-    anomalies.hits.hits.forEach(hit => {
-      records.push(hit._source);
-    });
-
-    // Sort anomalies in ascending time order.
-    records = _.sortBy(records, 'timestamp');
-    tableData.interval = aggregationInterval;
-    if (aggregationInterval === 'auto') {
-      // Determine the actual interval to use if aggregating.
-      const earliest = moment(records[0].timestamp);
-      const latest = moment(records[records.length - 1].timestamp);
-
-      const daysDiff = latest.diff(earliest, 'days');
-      tableData.interval = daysDiff < 2 ? 'hour' : 'day';
-    }
-
-    tableData.anomalies = buildAnomalyTableItems(records, tableData.interval, 'America/Chicago');
-
-    // Load examples for any categorization anomalies.
-    const categoryAnomalies = tableData.anomalies.filter(
-      (item: any) => item.entityName === 'mlcategory'
-    );
-    if (categoryAnomalies.length > 0) {
-      tableData.examplesByJobId = {};
-
-      const categoryIdsByJobId: { [key: string]: any } = {};
-      categoryAnomalies.forEach(anomaly => {
-        if (!_.has(categoryIdsByJobId, anomaly.jobId)) {
-          categoryIdsByJobId[anomaly.jobId] = [];
-        }
-        if (categoryIdsByJobId[anomaly.jobId].indexOf(anomaly.entityValue) === -1) {
-          categoryIdsByJobId[anomaly.jobId].push(anomaly.entityValue);
-        }
-      });
-
-      // const categoryJobIds = Object.keys(categoryIdsByJobId);
-      // await Promise.all(
-      //   categoryJobIds.map(async jobId => {
-      //     const examplesByCategoryId = await getCategoryExamples(
-      //       jobId,
-      //       categoryIdsByJobId[jobId],
-      //       maxExamples
-      //     );
-      //     if (tableData.examplesByJobId !== undefined) {
-      //       tableData.examplesByJobId[jobId] = examplesByCategoryId;
-      //     }
-      //   })
-      // );
-    }
-  }
-
-  return tableData;
-};

From 919dfef5626fcc11e2a1deedab900341da5fdd04 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 11 Mar 2020 21:46:06 -0500
Subject: [PATCH 10/63] Extract setting of rule failure to helper function

We were doing this identically in three places.
---
 .../signals/signal_rule_alert_type.ts         | 120 +++++++-----------
 1 file changed, 44 insertions(+), 76 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index 9580b4f2409ce..5d48926358dcb 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -5,15 +5,16 @@
  */
 
 import { schema } from '@kbn/config-schema';
-import { Logger } from 'src/core/server';
+import { Logger, SavedObject, SavedObjectsFindResponse } from 'src/core/server';
 import moment from 'moment';
 import { i18n } from '@kbn/i18n';
+import { AlertServices } from '../../../../../../../plugins/alerting/server';
+
 import {
   SIGNALS_ID,
   DEFAULT_MAX_SIGNALS,
   DEFAULT_SEARCH_AFTER_PAGE_SIZE,
 } from '../../../../common/constants';
-
 import { buildEventsSearchQuery } from './build_events_query';
 import { getInputIndex } from './get_input_output_index';
 import { searchAfterAndBulkCreate } from './search_after_bulk_create';
@@ -104,7 +105,7 @@ export const signalRulesAlertType = ({
         search: `${alertId}`,
         searchFields: ['alertId'],
       });
-      let currentStatusSavedObject;
+      let currentStatusSavedObject: SavedObject<IRuleSavedAttributesSavedObjectAttributes>;
       if (ruleStatusSavedObjects.saved_objects.length === 0) {
         // create
         const date = new Date().toISOString();
@@ -276,31 +277,7 @@ export const signalRulesAlertType = ({
             logger.error(
               `Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", ${err.message}\n${err.stack}`
             );
-            const sDate = new Date().toISOString();
-            currentStatusSavedObject.attributes.status = 'failed';
-            currentStatusSavedObject.attributes.statusDate = sDate;
-            currentStatusSavedObject.attributes.lastFailureAt = sDate;
-            currentStatusSavedObject.attributes.lastFailureMessage = err.message;
-            // current status is failing
-            await services.savedObjectsClient.update(
-              ruleStatusSavedObjectType,
-              currentStatusSavedObject.id,
-              {
-                ...currentStatusSavedObject.attributes,
-              }
-            );
-            // create new status for historical purposes
-            await services.savedObjectsClient.create(ruleStatusSavedObjectType, {
-              ...currentStatusSavedObject.attributes,
-            });
-
-            if (ruleStatusSavedObjects.saved_objects.length >= 6) {
-              // delete fifth status and prepare to insert a newer one.
-              const toDelete = ruleStatusSavedObjects.saved_objects.slice(5);
-              await toDelete.forEach(async item =>
-                services.savedObjectsClient.delete(ruleStatusSavedObjectType, item.id)
-              );
-            }
+            markRuleFailed(currentStatusSavedObject, ruleStatusSavedObjects, err.message, services);
           }
         }
 
@@ -324,62 +301,53 @@ export const signalRulesAlertType = ({
           logger.error(
             `Error processing signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
           );
-          const sDate = new Date().toISOString();
-          currentStatusSavedObject.attributes.status = 'failed';
-          currentStatusSavedObject.attributes.statusDate = sDate;
-          currentStatusSavedObject.attributes.lastFailureAt = sDate;
-          currentStatusSavedObject.attributes.lastFailureMessage = `Bulk Indexing signals failed. Check logs for further details \nRule name: "${name}"\nid: "${alertId}"\nrule_id: "${ruleId}"\n`;
-          // current status is failing
-          await services.savedObjectsClient.update(
-            ruleStatusSavedObjectType,
-            currentStatusSavedObject.id,
-            {
-              ...currentStatusSavedObject.attributes,
-            }
+          markRuleFailed(
+            currentStatusSavedObject,
+            ruleStatusSavedObjects,
+            `Bulk Indexing signals failed. Check logs for further details \nRule name: "${name}"\nid: "${alertId}"\nrule_id: "${ruleId}"\n`,
+            services
           );
-          // create new status for historical purposes
-          await services.savedObjectsClient.create(ruleStatusSavedObjectType, {
-            ...currentStatusSavedObject.attributes,
-          });
-
-          if (ruleStatusSavedObjects.saved_objects.length >= 6) {
-            // delete fifth status and prepare to insert a newer one.
-            const toDelete = ruleStatusSavedObjects.saved_objects.slice(5);
-            await toDelete.forEach(async item =>
-              services.savedObjectsClient.delete(ruleStatusSavedObjectType, item.id)
-            );
-          }
         }
       } catch (exception) {
         logger.error(
           `Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}" message: ${exception.message}, trace: \n${exception.stack}`
         );
-        const sDate = new Date().toISOString();
-        currentStatusSavedObject.attributes.status = 'failed';
-        currentStatusSavedObject.attributes.statusDate = sDate;
-        currentStatusSavedObject.attributes.lastFailureAt = sDate;
-        currentStatusSavedObject.attributes.lastFailureMessage = exception.message;
-        // current status is failing
-        await services.savedObjectsClient.update(
-          ruleStatusSavedObjectType,
-          currentStatusSavedObject.id,
-          {
-            ...currentStatusSavedObject.attributes,
-          }
+        markRuleFailed(
+          currentStatusSavedObject,
+          ruleStatusSavedObjects,
+          exception.message,
+          services
         );
-        // create new status for historical purposes
-        await services.savedObjectsClient.create(ruleStatusSavedObjectType, {
-          ...currentStatusSavedObject.attributes,
-        });
-
-        if (ruleStatusSavedObjects.saved_objects.length >= 6) {
-          // delete fifth status and prepare to insert a newer one.
-          const toDelete = ruleStatusSavedObjects.saved_objects.slice(5);
-          await toDelete.forEach(async item =>
-            services.savedObjectsClient.delete(ruleStatusSavedObjectType, item.id)
-          );
-        }
       }
     },
   };
 };
+
+const markRuleFailed = async (
+  currentStatus: SavedObject<IRuleSavedAttributesSavedObjectAttributes>,
+  statuses: SavedObjectsFindResponse<IRuleSavedAttributesSavedObjectAttributes>,
+  message: string,
+  services: AlertServices
+) => {
+  const sDate = new Date().toISOString();
+  currentStatus.attributes.status = 'failed';
+  currentStatus.attributes.statusDate = sDate;
+  currentStatus.attributes.lastFailureAt = sDate;
+  currentStatus.attributes.lastFailureMessage = message;
+  // current status is failing
+  await services.savedObjectsClient.update(ruleStatusSavedObjectType, currentStatus.id, {
+    ...currentStatus.attributes,
+  });
+  // create new status for historical purposes
+  await services.savedObjectsClient.create(ruleStatusSavedObjectType, {
+    ...currentStatus.attributes,
+  });
+
+  if (statuses.saved_objects.length >= 6) {
+    // delete fifth status and prepare to insert a newer one.
+    const toDelete = statuses.saved_objects.slice(5);
+    await toDelete.forEach(async item =>
+      services.savedObjectsClient.delete(ruleStatusSavedObjectType, item.id)
+    );
+  }
+};

From 051f638316384dfbd91ac2f3512557859fbcf4b1 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 11 Mar 2020 21:48:08 -0500
Subject: [PATCH 11/63] Remove unused import

---
 x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
index b9cd494ae450a..53e55811dc4d5 100644
--- a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
@@ -4,7 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import _ from 'lodash';
 import { SearchResponse } from 'elasticsearch';
 
 import { AlertServices } from '../../../../../../plugins/alerting/server';

From 7ae29bf13d15c088119e0e23c9ad5c1f5c2ad0dd Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Thu, 12 Mar 2020 17:00:27 -0500
Subject: [PATCH 12/63] Define a field for our Rule Type selection

This adds most of the markup and logic to allow an ML rule type to be
selected. We still need to add things like license-checking and
showing/hiding of fields based on type.
---
 .../detection_engine/rules/types.ts           |  9 ++-
 .../components/select_rule_type/index.tsx     | 63 +++++++++++++++++++
 .../select_rule_type/translations.ts          | 42 +++++++++++++
 .../components/step_define_rule/index.tsx     |  6 +-
 .../components/step_define_rule/schema.tsx    |  9 +++
 .../detection_engine/rules/create/helpers.ts  | 21 +++----
 .../pages/detection_engine/rules/types.ts     |  5 +-
 7 files changed, 139 insertions(+), 16 deletions(-)
 create mode 100644 x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
 create mode 100644 x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/translations.ts

diff --git a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
index e7988381a430b..dcab62e82ccee 100644
--- a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
@@ -6,6 +6,13 @@
 
 import * as t from 'io-ts';
 
+export const RuleTypeSchema = t.union([
+  t.literal('query'),
+  t.literal('saved_query'),
+  t.literal('machine_learning'),
+]);
+export type RuleType = t.TypeOf<typeof RuleTypeSchema>;
+
 export const NewRuleSchema = t.intersection([
   t.type({
     description: t.string,
@@ -18,7 +25,7 @@ export const NewRuleSchema = t.intersection([
     query: t.string,
     risk_score: t.number,
     severity: t.string,
-    type: t.union([t.literal('query'), t.literal('saved_query'), t.literal('machine_learning')]),
+    type: RuleTypeSchema,
   }),
   t.partial({
     anomaly_threshold: t.number,
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
new file mode 100644
index 0000000000000..1c735904b96a3
--- /dev/null
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { useCallback, useState } from 'react';
+import styled from 'styled-components';
+import { EuiCard, EuiFlexGrid, EuiFlexItem, EuiIcon, EuiFormRow } from '@elastic/eui';
+
+import { FieldHook } from '../../../../../shared_imports';
+import { RuleType } from '../../../../../containers/detection_engine/rules/types';
+import * as i18n from './translations';
+
+interface SelectRuleTypeProps {
+  field: FieldHook;
+}
+
+const Wrapper = styled(EuiFormRow)``;
+
+export const SelectRuleType = ({ field }: SelectRuleTypeProps) => {
+  const [ruleType, setRuleType] = useState<RuleType>('query');
+  const setType = useCallback(
+    (type: RuleType) => {
+      setRuleType(type);
+      field.setValue(type);
+    },
+    [field]
+  );
+  const setMl = useCallback(() => setType('machine_learning'), [setType]);
+  const setQuery = useCallback(() => setType('query'), [setType]);
+  const license = true; // TODO
+
+  return (
+    <Wrapper label={field.label} fullWidth>
+      <EuiFlexGrid columns={4}>
+        <EuiFlexItem>
+          <EuiCard
+            title={i18n.QUERY_TYPE_TITLE}
+            description={i18n.QUERY_TYPE_DESCRIPTION}
+            icon={<EuiIcon size="l" type="search" />}
+            selectable={{
+              onClick: setQuery,
+              isSelected: ruleType === 'query',
+            }}
+          />
+        </EuiFlexItem>
+        <EuiFlexItem>
+          <EuiCard
+            title={i18n.ML_TYPE_TITLE}
+            description={license ? i18n.ML_TYPE_DESCRIPTION : i18n.ML_TYPE_DISABLED_DESCRIPTION}
+            isDisabled={!license}
+            icon={<EuiIcon size="l" type="machineLearningApp" />}
+            selectable={{
+              onClick: setMl,
+              isSelected: ruleType === 'machine_learning',
+            }}
+          />
+        </EuiFlexItem>
+      </EuiFlexGrid>
+    </Wrapper>
+  );
+};
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/translations.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/translations.ts
new file mode 100644
index 0000000000000..32b860e8f703e
--- /dev/null
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/translations.ts
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { i18n } from '@kbn/i18n';
+
+export const QUERY_TYPE_TITLE = i18n.translate(
+  'xpack.siem.detectionEngine.createRule.stepDefineRule.ruleTypeField.queryTypeTitle',
+  {
+    defaultMessage: 'Custom query',
+  }
+);
+
+export const QUERY_TYPE_DESCRIPTION = i18n.translate(
+  'xpack.siem.detectionEngine.createRule.stepDefineRule.ruleTypeField.queryTypeDescription',
+  {
+    defaultMessage: 'Use KQL or Lucene to detect issues across indices.',
+  }
+);
+
+export const ML_TYPE_TITLE = i18n.translate(
+  'xpack.siem.detectionEngine.createRule.stepDefineRule.ruleTypeField.mlTypeTitle',
+  {
+    defaultMessage: 'Machine Learning',
+  }
+);
+
+export const ML_TYPE_DESCRIPTION = i18n.translate(
+  'xpack.siem.detectionEngine.createRule.stepDefineRule.ruleTypeField.mlTypeDescription',
+  {
+    defaultMessage: 'Select ML job to detect anomalous activity.',
+  }
+);
+
+export const ML_TYPE_DISABLED_DESCRIPTION = i18n.translate(
+  'xpack.siem.detectionEngine.createRule.stepDefineRule.ruleTypeField.mlTypeDisabledDescription',
+  {
+    defaultMessage: 'Access to ML requires a Platinum subscription.',
+  }
+);
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
index 490a8d9d194cb..36d6a27b0cff8 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
@@ -4,6 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { i18n as I18n } from '@kbn/i18n';
 import {
   EuiButtonEmpty,
   EuiHorizontalRule,
@@ -25,6 +26,7 @@ import * as RuleI18n from '../../translations';
 import { DefineStepRule, RuleStep, RuleStepProps } from '../../types';
 import { StepRuleDescription } from '../description_step';
 import { QueryBarDefineRule } from '../query_bar';
+import { SelectRuleType } from '../select_rule_type';
 import { StepContentWrapper } from '../step_content_wrapper';
 import {
   Field,
@@ -43,9 +45,10 @@ interface StepDefineRuleProps extends RuleStepProps {
   defaultValues?: DefineStepRule | null;
 }
 
-const stepDefineDefaultValue = {
+const stepDefineDefaultValue: DefineStepRule = {
   index: [],
   isNew: true,
+  ruleType: 'query',
   queryBar: {
     query: { query: '', language: 'kuery' },
     filters: [],
@@ -167,6 +170,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
     <>
       <StepContentWrapper addPadding={!isUpdateView}>
         <Form form={form} data-test-subj="stepDefineRule">
+          <UseField path="ruleType" component={SelectRuleType} componentProps={{}} />
           <CommonUseField
             path="index"
             config={{
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
index e202ff030cd90..97eaacbbd7638 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
@@ -90,4 +90,13 @@ export const schema: FormSchema = {
       },
     ],
   },
+  ruleType: {
+    label: i18n.translate(
+      'xpack.siem.detectionEngine.createRule.stepDefineRule.fieldRuleTypeLabel',
+      {
+        defaultMessage: 'Rule type',
+      }
+    ),
+    validations: [],
+  },
 };
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
index de6678b42df6f..a3e2943e07e2a 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
@@ -16,7 +16,6 @@ import {
   DefineStepRuleJson,
   ScheduleStepRuleJson,
   AboutStepRuleJson,
-  FormatRuleType,
 } from '../types';
 
 const getTimeTypeValue = (time: string): { unit: string; value: number } => {
@@ -40,10 +39,11 @@ const getTimeTypeValue = (time: string): { unit: string; value: number } => {
 };
 
 const formatDefineStepData = (defineStepData: DefineStepRule): DefineStepRuleJson => {
-  const { queryBar, isNew, ...rest } = defineStepData;
+  const { queryBar, isNew, ruleType, ...rest } = defineStepData;
   const { filters, query, saved_id: savedId } = queryBar;
   return {
     ...rest,
+    type: ruleType,
     language: query.language,
     filters,
     query: query.query as string,
@@ -102,13 +102,10 @@ export const formatRule = (
   aboutStepData: AboutStepRule,
   scheduleData: ScheduleStepRule,
   ruleId?: string
-): NewRule => {
-  const type: FormatRuleType = !isEmpty(defineStepData.queryBar.saved_id) ? 'saved_query' : 'query';
-  const persistData = {
-    type,
-    ...formatDefineStepData(defineStepData),
-    ...formatAboutStepData(aboutStepData),
-    ...formatScheduleStepData(scheduleData),
-  };
-  return ruleId != null ? { id: ruleId, ...persistData } : persistData;
-};
+): NewRule => ({
+  ...(!isEmpty(defineStepData) && ruleId != null ? { id: ruleId } : {}),
+  ...formatDefineStepData(defineStepData),
+  ...formatAboutStepData(aboutStepData),
+  ...formatScheduleStepData(scheduleData),
+  ...(!isEmpty(defineStepData.queryBar.saved_id) ? { type: 'saved_query' } : {}),
+});
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
index 34df20de1e461..2abf0b3b44fdb 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
@@ -5,6 +5,7 @@
  */
 
 import { Filter } from '../../../../../../../../src/plugins/data/common';
+import { RuleType } from '../../../containers/detection_engine/rules/types';
 import { FieldValueQueryBar } from './components/query_bar';
 import { FormData, FormHook } from '../../../shared_imports';
 import { FieldValueTimeline } from './components/pick_timeline';
@@ -63,6 +64,7 @@ export interface AboutStepRule extends StepRuleData {
 export interface DefineStepRule extends StepRuleData {
   index: string[];
   queryBar: FieldValueQueryBar;
+  ruleType: RuleType;
 }
 
 export interface ScheduleStepRule extends StepRuleData {
@@ -78,6 +80,7 @@ export interface DefineStepRuleJson {
   saved_id?: string;
   query: string;
   language: string;
+  type: RuleType;
 }
 
 export interface AboutStepRuleJson {
@@ -105,8 +108,6 @@ export type MyRule = Omit<DefineStepRule & ScheduleStepRule & AboutStepRule, 'is
   immutable: boolean;
 };
 
-export type FormatRuleType = 'query' | 'saved_query';
-
 export interface IMitreAttack {
   id: string;
   name: string;

From 6b25593262dd5705fd5f71da32cd13c7120ed146 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Thu, 12 Mar 2020 19:56:33 -0500
Subject: [PATCH 13/63] Hide Query Fields when ML is selected

These are still getting set on the form. We'll need to filter these
fields before we send off the data, and not show them on the readonly
display either.

ALso, edit is majorly broken.
---
 .../components/step_define_rule/index.tsx     | 104 ++++++++++--------
 .../components/step_define_rule/schema.tsx    |  44 +++++---
 2 files changed, 89 insertions(+), 59 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
index 36d6a27b0cff8..8c9972b94de01 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
@@ -4,12 +4,12 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { i18n as I18n } from '@kbn/i18n';
 import {
   EuiButtonEmpty,
   EuiHorizontalRule,
   EuiFlexGroup,
   EuiFlexItem,
+  EuiFormRow,
   EuiButton,
 } from '@elastic/eui';
 import { isEmpty } from 'lodash/fp';
@@ -99,6 +99,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
 }) => {
   const [openTimelineSearch, setOpenTimelineSearch] = useState(false);
   const [localUseIndicesConfig, setLocalUseIndicesConfig] = useState(false);
+  const [isMlRule, setIsMlRule] = useState(false);
   const [indicesConfig] = useUiSetting$<string[]>(DEFAULT_INDEX_KEY);
   const [mylocalIndicesConfig, setMyLocalIndicesConfig] = useState(
     defaultValues != null ? defaultValues.index : indicesConfig ?? []
@@ -171,51 +172,55 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
       <StepContentWrapper addPadding={!isUpdateView}>
         <Form form={form} data-test-subj="stepDefineRule">
           <UseField path="ruleType" component={SelectRuleType} componentProps={{}} />
-          <CommonUseField
-            path="index"
-            config={{
-              ...schema.index,
-              labelAppend: !localUseIndicesConfig ? (
-                <MyLabelButton onClick={handleResetIndices} iconType="refresh">
-                  {i18n.RESET_DEFAULT_INDEX}
-                </MyLabelButton>
-              ) : null,
-            }}
-            componentProps={{
-              idAria: 'detectionEngineStepDefineRuleIndices',
-              'data-test-subj': 'detectionEngineStepDefineRuleIndices',
-              euiFieldProps: {
-                fullWidth: true,
-                isDisabled: isLoading,
-                placeholder: '',
-              },
-            }}
-          />
-          <UseField
-            path="queryBar"
-            config={{
-              ...schema.queryBar,
-              labelAppend: (
-                <MyLabelButton onClick={handleOpenTimelineSearch}>
-                  {i18n.IMPORT_TIMELINE_QUERY}
-                </MyLabelButton>
-              ),
-            }}
-            component={QueryBarDefineRule}
-            componentProps={{
-              browserFields,
-              loading: indexPatternLoadingQueryBar,
-              idAria: 'detectionEngineStepDefineRuleQueryBar',
-              indexPattern: indexPatternQueryBar,
-              isDisabled: isLoading,
-              isLoading: indexPatternLoadingQueryBar,
-              dataTestSubj: 'detectionEngineStepDefineRuleQueryBar',
-              openTimelineSearch,
-              onCloseTimelineSearch: handleCloseTimelineSearch,
-            }}
-          />
-          <FormDataProvider pathsToWatch="index">
-            {({ index }) => {
+          <EuiFormRow fullWidth style={{ display: isMlRule ? 'none' : 'inline-block' }}>
+            <>
+              <CommonUseField
+                path="index"
+                config={{
+                  ...schema.index,
+                  labelAppend: !localUseIndicesConfig ? (
+                    <MyLabelButton onClick={handleResetIndices} iconType="refresh">
+                      {i18n.RESET_DEFAULT_INDEX}
+                    </MyLabelButton>
+                  ) : null,
+                }}
+                componentProps={{
+                  idAria: 'detectionEngineStepDefineRuleIndices',
+                  'data-test-subj': 'detectionEngineStepDefineRuleIndices',
+                  euiFieldProps: {
+                    fullWidth: true,
+                    isDisabled: isLoading,
+                    placeholder: '',
+                  },
+                }}
+              />
+              <UseField
+                path="queryBar"
+                config={{
+                  ...schema.queryBar,
+                  labelAppend: (
+                    <MyLabelButton onClick={handleOpenTimelineSearch}>
+                      {i18n.IMPORT_TIMELINE_QUERY}
+                    </MyLabelButton>
+                  ),
+                }}
+                component={QueryBarDefineRule}
+                componentProps={{
+                  browserFields,
+                  loading: indexPatternLoadingQueryBar,
+                  idAria: 'detectionEngineStepDefineRuleQueryBar',
+                  indexPattern: indexPatternQueryBar,
+                  isDisabled: isLoading,
+                  isLoading: indexPatternLoadingQueryBar,
+                  dataTestSubj: 'detectionEngineStepDefineRuleQueryBar',
+                  openTimelineSearch,
+                  onCloseTimelineSearch: handleCloseTimelineSearch,
+                }}
+              />
+            </>
+          </EuiFormRow>
+          <FormDataProvider pathsToWatch={['index', 'ruleType']}>
+            {({ index, ruleType }) => {
               if (index != null) {
                 if (deepEqual(index, indicesConfig) && !localUseIndicesConfig) {
                   setLocalUseIndicesConfig(true);
@@ -227,6 +232,13 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
                   setMyLocalIndicesConfig(index);
                 }
               }
+
+              if (ruleType === 'machine_learning' && !isMlRule) {
+                setIsMlRule(true);
+              } else if (ruleType !== 'machine_learning' && isMlRule) {
+                setIsMlRule(false);
+              }
+
               return null;
             }}
           </FormDataProvider>
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
index 97eaacbbd7638..83d975f86aa5f 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
@@ -20,8 +20,6 @@ import {
 } from '../../../../../shared_imports';
 import { CUSTOM_QUERY_REQUIRED, INVALID_CUSTOM_QUERY, INDEX_HELPER_TEXT } from './translations';
 
-const { emptyField } = fieldValidators;
-
 export const schema: FormSchema = {
   index: {
     type: FIELD_TYPES.COMBO_BOX,
@@ -34,14 +32,25 @@ export const schema: FormSchema = {
     helpText: <EuiText size="xs">{INDEX_HELPER_TEXT}</EuiText>,
     validations: [
       {
-        validator: emptyField(
-          i18n.translate(
-            'xpack.siem.detectionEngine.createRule.stepDefineRule.outputIndiceNameFieldRequiredError',
-            {
-              defaultMessage: 'A minimum of one index pattern is required.',
-            }
-          )
-        ),
+        validator: (
+          ...args: Parameters<ValidationFunc>
+        ): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
+          const [{ formData }] = args;
+          const needsValidation = formData.ruleType !== 'machine_learning';
+
+          if (!needsValidation) {
+            return;
+          }
+
+          return fieldValidators.emptyField(
+            i18n.translate(
+              'xpack.siem.detectionEngine.createRule.stepDefineRule.outputIndiceNameFieldRequiredError',
+              {
+                defaultMessage: 'A minimum of one index pattern is required.',
+              }
+            )
+          )(...args);
+        },
       },
     ],
   },
@@ -57,8 +66,13 @@ export const schema: FormSchema = {
         validator: (
           ...args: Parameters<ValidationFunc>
         ): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
-          const [{ value, path }] = args;
+          const [{ value, path, formData }] = args;
           const { query, filters } = value as FieldValueQueryBar;
+          const needsValidation = formData.ruleType !== 'machine_learning';
+          if (!needsValidation) {
+            return;
+          }
+
           return isEmpty(query.query as string) && isEmpty(filters)
             ? {
                 code: 'ERR_FIELD_MISSING',
@@ -72,8 +86,13 @@ export const schema: FormSchema = {
         validator: (
           ...args: Parameters<ValidationFunc>
         ): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
-          const [{ value, path }] = args;
+          const [{ value, path, formData }] = args;
           const { query } = value as FieldValueQueryBar;
+          const needsValidation = formData.ruleType !== 'machine_learning';
+          if (!needsValidation) {
+            return;
+          }
+
           if (!isEmpty(query.query as string) && query.language === 'kuery') {
             try {
               esKuery.fromKueryExpression(query.query);
@@ -85,7 +104,6 @@ export const schema: FormSchema = {
               };
             }
           }
-          return undefined;
         },
       },
     ],

From d5b03a270d30549c8bd91658fa47e0b9e0013635 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Thu, 12 Mar 2020 22:25:28 -0500
Subject: [PATCH 14/63] Add input field for anomaly threshold

---
 .../anomaly_threshold_slider/index.tsx        | 49 +++++++++++++++++++
 .../components/description_step/index.tsx     | 19 +------
 .../components/step_define_rule/index.tsx     | 11 ++++-
 .../components/step_define_rule/schema.tsx    |  9 ++++
 .../pages/detection_engine/rules/types.ts     |  2 +
 5 files changed, 71 insertions(+), 19 deletions(-)
 create mode 100644 x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx
new file mode 100644
index 0000000000000..0d60344e22823
--- /dev/null
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { useCallback, useState } from 'react';
+import styled from 'styled-components';
+import { EuiFlexGrid, EuiFlexItem, EuiRange, EuiFormRow } from '@elastic/eui';
+
+import { FieldHook } from '../../../../../shared_imports';
+
+interface AnomalyThresholdSliderProps {
+  field: FieldHook;
+}
+type Event = React.ChangeEvent<HTMLInputElement>;
+type EventArg = Event | React.MouseEvent<HTMLButtonElement>;
+
+const Wrapper = styled(EuiFormRow)``;
+
+export const AnomalyThresholdSlider = ({ field }: AnomalyThresholdSliderProps) => {
+  const [localThreshold, setLocalThreshold] = useState<number>(50);
+  const onThresholdChange = useCallback(
+    (event: EventArg) => {
+      const threshold = Number((event as Event).target.value);
+      setLocalThreshold(threshold);
+      field.setValue(threshold);
+    },
+    [field]
+  );
+
+  return (
+    <Wrapper label={field.label} fullWidth>
+      <EuiFlexGrid columns={2}>
+        <EuiFlexItem>
+          <EuiRange
+            value={localThreshold}
+            onChange={onThresholdChange}
+            fullWidth
+            showInput
+            showRange
+            showTicks
+            tickInterval={25}
+          />
+        </EuiFlexItem>
+      </EuiFlexGrid>
+    </Wrapper>
+  );
+};
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx
index cb5c98bb23f07..6a01bbbed34c1 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx
@@ -132,13 +132,6 @@ const getDescriptionItem = (
       (singleThreat: IMitreEnterpriseAttack) => singleThreat.tactic.name !== 'none'
     );
     return buildThreatDescription({ label, threat });
-  } else if (field === 'description') {
-    return [
-      {
-        title: label,
-        description: get(field, value),
-      },
-    ];
   } else if (field === 'references') {
     const urls: string[] = get(field, value);
     return buildUrlsDescription(label, urls);
@@ -151,13 +144,6 @@ const getDescriptionItem = (
   } else if (field === 'severity') {
     const val: string = get(field, value);
     return buildSeverityDescription(label, val);
-  } else if (field === 'riskScore') {
-    return [
-      {
-        title: label,
-        description: get(field, value),
-      },
-    ];
   } else if (field === 'timeline') {
     const timeline = get(field, value) as FieldValueTimeline;
     return [
@@ -166,12 +152,11 @@ const getDescriptionItem = (
         description: timeline.title ?? DEFAULT_TIMELINE_TITLE,
       },
     ];
-  } else if (field === 'riskScore') {
-    const description: string = get(field, value);
+  } else if (['anomalyThreshold', 'description', 'riskScore'].includes(field)) {
     return [
       {
         title: label,
-        description,
+        description: get(field, value),
       },
     ];
   }
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
index 8c9972b94de01..103c30d1b243f 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
@@ -38,6 +38,7 @@ import {
 } from '../../../../../shared_imports';
 import { schema } from './schema';
 import * as i18n from './translations';
+import { AnomalyThresholdSlider } from '../anomaly_threshold_slider';
 
 const CommonUseField = getUseField({ component: Field });
 
@@ -46,6 +47,7 @@ interface StepDefineRuleProps extends RuleStepProps {
 }
 
 const stepDefineDefaultValue: DefineStepRule = {
+  anomalyThreshold: 50,
   index: [],
   isNew: true,
   ruleType: 'query',
@@ -171,8 +173,8 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
     <>
       <StepContentWrapper addPadding={!isUpdateView}>
         <Form form={form} data-test-subj="stepDefineRule">
-          <UseField path="ruleType" component={SelectRuleType} componentProps={{}} />
-          <EuiFormRow fullWidth style={{ display: isMlRule ? 'none' : 'inline-block' }}>
+          <UseField path="ruleType" component={SelectRuleType} />
+          <EuiFormRow fullWidth style={{ display: isMlRule ? 'none' : 'flex' }}>
             <>
               <CommonUseField
                 path="index"
@@ -219,6 +221,11 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
               />
             </>
           </EuiFormRow>
+          <EuiFormRow fullWidth style={{ display: isMlRule ? 'flex' : 'none' }}>
+            <>
+              <UseField path="anomalyThreshold" component={AnomalyThresholdSlider} />
+            </>
+          </EuiFormRow>
           <FormDataProvider pathsToWatch={['index', 'ruleType']}>
             {({ index, ruleType }) => {
               if (index != null) {
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
index 83d975f86aa5f..934d0c15b3996 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
@@ -117,4 +117,13 @@ export const schema: FormSchema = {
     ),
     validations: [],
   },
+  anomalyThreshold: {
+    label: i18n.translate(
+      'xpack.siem.detectionEngine.createRule.stepDefineRule.fieldAnomalyThresholdLabel',
+      {
+        defaultMessage: 'Anomaly score threshold',
+      }
+    ),
+    validations: [],
+  },
 };
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
index 2abf0b3b44fdb..373587936f1c1 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
@@ -62,6 +62,7 @@ export interface AboutStepRule extends StepRuleData {
 }
 
 export interface DefineStepRule extends StepRuleData {
+  anomalyThreshold: number;
   index: string[];
   queryBar: FieldValueQueryBar;
   ruleType: RuleType;
@@ -75,6 +76,7 @@ export interface ScheduleStepRule extends StepRuleData {
 }
 
 export interface DefineStepRuleJson {
+  anomalyThreshold?: number;
   index: string[];
   filters: Filter[];
   saved_id?: string;

From 82093ceaada388532a63561174410e17416abf0b Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Thu, 12 Mar 2020 22:56:59 -0500
Subject: [PATCH 15/63] Display numberic values in the readonly view of a step

TIL that isEmpty returns false for numbers and other non-iterable
values. I don't think it's exactly what we want here, but until I figure
out the intention this gets our anomalyThreshold showing up without a
separate logic branch here. Removes the unnecessary branch that was
redundant with the 'else' clause.
---
 .../rules/components/description_step/index.tsx      | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx
index 6a01bbbed34c1..108a20303f0c9 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx
@@ -5,7 +5,7 @@
  */
 
 import { EuiDescriptionList, EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
-import { isEmpty, chunk, get, pick } from 'lodash/fp';
+import { isEmpty, chunk, get, pick, isNumber } from 'lodash/fp';
 import React, { memo, useState } from 'react';
 
 import {
@@ -152,16 +152,10 @@ const getDescriptionItem = (
         description: timeline.title ?? DEFAULT_TIMELINE_TITLE,
       },
     ];
-  } else if (['anomalyThreshold', 'description', 'riskScore'].includes(field)) {
-    return [
-      {
-        title: label,
-        description: get(field, value),
-      },
-    ];
   }
+
   const description: string = get(field, value);
-  if (!isEmpty(description)) {
+  if (isNumber(description) || !isEmpty(description)) {
     return [
       {
         title: label,

From bb8abc231adc801ccb279ac472768dd09263bdcc Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Thu, 12 Mar 2020 22:59:08 -0500
Subject: [PATCH 16/63] Add field for selecting an ML job

This is not the same as the mockups and lacks some functionality, but
it'll allow us to select a job for now.
---
 .../rules/components/ml_job_select/index.tsx  | 60 +++++++++++++++++++
 .../components/step_define_rule/index.tsx     |  4 +-
 .../components/step_define_rule/schema.tsx    |  9 +++
 3 files changed, 72 insertions(+), 1 deletion(-)
 create mode 100644 x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
new file mode 100644
index 0000000000000..d83798f8fd5f0
--- /dev/null
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { useCallback, useState } from 'react';
+import styled from 'styled-components';
+import { EuiFlexGroup, EuiFlexItem, EuiFormRow, EuiSuperSelect, EuiText } from '@elastic/eui';
+
+import { FieldHook } from '../../../../../shared_imports';
+import { useSiemJobs } from '../../../../../components/ml_popover/hooks/use_siem_jobs';
+
+interface MlJobSelectProps {
+  field: FieldHook;
+}
+
+const Wrapper = styled(EuiFormRow)``;
+const JobDisplay = ({ title, description }: { title: string; description: string }) => (
+  <>
+    <strong>{title}</strong>
+    <EuiText size="xs" color="subdued">
+      <p>{description}</p>
+    </EuiText>
+  </>
+);
+
+export const MlJobSelect = ({ field }: MlJobSelectProps) => {
+  const [localJobId, setLocalJobId] = useState<string>('query');
+  const [isLoading, siemJobs] = useSiemJobs(false);
+  const handleJobChange = useCallback(
+    (jobId: string) => {
+      setLocalJobId(jobId);
+      field.setValue(jobId);
+    },
+    [field]
+  );
+
+  const options = siemJobs.map(job => ({
+    value: job.id,
+    inputDisplay: job.id,
+    dropdownDisplay: <JobDisplay title={job.id} description={job.description} />,
+  }));
+
+  return (
+    <Wrapper label={field.label} fullWidth>
+      <EuiFlexGroup>
+        <EuiFlexItem>
+          <EuiSuperSelect
+            hasDividers
+            isLoading={isLoading}
+            onChange={handleJobChange}
+            options={options}
+            valueOfSelected={localJobId}
+          />
+        </EuiFlexItem>
+      </EuiFlexGroup>
+    </Wrapper>
+  );
+};
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
index 103c30d1b243f..ff3b897d6496c 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
@@ -27,6 +27,8 @@ import { DefineStepRule, RuleStep, RuleStepProps } from '../../types';
 import { StepRuleDescription } from '../description_step';
 import { QueryBarDefineRule } from '../query_bar';
 import { SelectRuleType } from '../select_rule_type';
+import { AnomalyThresholdSlider } from '../anomaly_threshold_slider';
+import { MlJobSelect } from '../ml_job_select';
 import { StepContentWrapper } from '../step_content_wrapper';
 import {
   Field,
@@ -38,7 +40,6 @@ import {
 } from '../../../../../shared_imports';
 import { schema } from './schema';
 import * as i18n from './translations';
-import { AnomalyThresholdSlider } from '../anomaly_threshold_slider';
 
 const CommonUseField = getUseField({ component: Field });
 
@@ -223,6 +224,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
           </EuiFormRow>
           <EuiFormRow fullWidth style={{ display: isMlRule ? 'flex' : 'none' }}>
             <>
+              <UseField path="mlJobId" component={MlJobSelect} />
               <UseField path="anomalyThreshold" component={AnomalyThresholdSlider} />
             </>
           </EuiFormRow>
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
index 934d0c15b3996..2917d39521d29 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
@@ -126,4 +126,13 @@ export const schema: FormSchema = {
     ),
     validations: [],
   },
+  mlJobId: {
+    label: i18n.translate(
+      'xpack.siem.detectionEngine.createRule.stepDefineRule.fieldMlJobIdLabel',
+      {
+        defaultMessage: 'Machine Learning job',
+      }
+    ),
+    validations: [],
+  },
 };

From 526e8a33a14d4e66358e735ab77c5503608e21e0 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Thu, 12 Mar 2020 23:21:28 -0500
Subject: [PATCH 17/63] Format our new ML Fields when sending them to the
 server

So that we don't get rejected due to snake case vs camelcase.
---
 .../public/pages/detection_engine/rules/create/helpers.ts  | 7 +++++--
 .../siem/public/pages/detection_engine/rules/types.ts      | 4 +++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
index a3e2943e07e2a..539934688771e 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { isEmpty } from 'lodash/fp';
+import { isEmpty, isNumber } from 'lodash/fp';
 import moment from 'moment';
 
 import { NewRule } from '../../../../containers/detection_engine/rules';
@@ -39,7 +39,7 @@ const getTimeTypeValue = (time: string): { unit: string; value: number } => {
 };
 
 const formatDefineStepData = (defineStepData: DefineStepRule): DefineStepRuleJson => {
-  const { queryBar, isNew, ruleType, ...rest } = defineStepData;
+  const { anomalyThreshold, mlJobId, queryBar, isNew, ruleType, ...rest } = defineStepData;
   const { filters, query, saved_id: savedId } = queryBar;
   return {
     ...rest,
@@ -48,6 +48,8 @@ const formatDefineStepData = (defineStepData: DefineStepRule): DefineStepRuleJso
     filters,
     query: query.query as string,
     ...(savedId != null && savedId !== '' ? { saved_id: savedId } : {}),
+    ...(isNumber(anomalyThreshold) ? { anomaly_threshold: anomalyThreshold } : {}),
+    ...(mlJobId ? { ml_job_id: mlJobId } : {}),
   };
 };
 
@@ -108,4 +110,5 @@ export const formatRule = (
   ...formatAboutStepData(aboutStepData),
   ...formatScheduleStepData(scheduleData),
   ...(!isEmpty(defineStepData.queryBar.saved_id) ? { type: 'saved_query' } : {}),
+  // TODO: FILTER OUT MUTEX FIELDS
 });
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
index 373587936f1c1..3c649b306d433 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
@@ -64,6 +64,7 @@ export interface AboutStepRule extends StepRuleData {
 export interface DefineStepRule extends StepRuleData {
   anomalyThreshold: number;
   index: string[];
+  mlJobId: string;
   queryBar: FieldValueQueryBar;
   ruleType: RuleType;
 }
@@ -76,9 +77,10 @@ export interface ScheduleStepRule extends StepRuleData {
 }
 
 export interface DefineStepRuleJson {
-  anomalyThreshold?: number;
+  anomaly_threshold?: number;
   index: string[];
   filters: Filter[];
+  ml_job_id?: string;
   saved_id?: string;
   query: string;
   language: string;

From a5d05e02b33b390dcaf66bb3ba99907919f7fcc3 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Thu, 12 Mar 2020 23:22:30 -0500
Subject: [PATCH 18/63] Put back code that respects a rule's schedule

It was previously hardcoded to a time period I knew had anomalies.
---
 .../server/lib/detection_engine/signals/find_ml_signals.ts  | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
index 49c29b221aa84..ce70168e01b6a 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
@@ -20,10 +20,8 @@ export const findMlSignals = async (
   const params = {
     jobIds: [jobId],
     threshold: anomalyThreshold,
-    // earliestMs: dateMath.parse(from)!.valueOf(),
-    // latestMs: dateMath.parse(to)!.valueOf(),
-    earliestMs: 1583431900000,
-    latestMs: 1583431900000 + 86400000,
+    earliestMs: dateMath.parse(from)!.valueOf(),
+    latestMs: dateMath.parse(to)!.valueOf(),
   };
   const relevantAnomalies = await anomaliesTableData(params, callCluster);
 

From 75eaa12b9d699af2c5156e5ed05195ca55853094 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Fri, 13 Mar 2020 10:08:31 -0500
Subject: [PATCH 19/63] ML fields are optional in our creation step

In that we don't initialize them like we do the query (default) fields.
---
 .../rules/components/step_define_rule/index.tsx               | 1 -
 .../plugins/siem/public/pages/detection_engine/rules/types.ts | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
index ff3b897d6496c..0b0143d337563 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
@@ -48,7 +48,6 @@ interface StepDefineRuleProps extends RuleStepProps {
 }
 
 const stepDefineDefaultValue: DefineStepRule = {
-  anomalyThreshold: 50,
   index: [],
   isNew: true,
   ruleType: 'query',
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
index 3c649b306d433..ce3088aab0cef 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
@@ -62,9 +62,9 @@ export interface AboutStepRule extends StepRuleData {
 }
 
 export interface DefineStepRule extends StepRuleData {
-  anomalyThreshold: number;
+  anomalyThreshold?: number;
   index: string[];
-  mlJobId: string;
+  mlJobId?: string;
   queryBar: FieldValueQueryBar;
   ruleType: RuleType;
 }

From 68cbf21303c6062afa761a299bc7d25112ab39db Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Fri, 13 Mar 2020 10:08:43 -0500
Subject: [PATCH 20/63] Only send along type-specific Rule fields from form

This makes any query- or ML-specific fields optional on a Rule, and
performs some logic on the frontend to group and include these fieldsets
conditionally based on the user's selection. The one place we don't
handle this well is on the readonly view of a completed step in the
rules creation, but we'll address that.
---
 .../detection_engine/rules/types.ts           | 20 ++++++------
 .../rules/components/query_bar/index.tsx      |  2 +-
 .../detection_engine/rules/create/helpers.ts  | 31 +++++++++++--------
 .../detection_engine/rules/edit/index.tsx     | 12 +++----
 .../pages/detection_engine/rules/types.ts     |  8 ++---
 .../siem/server/lib/detection_engine/types.ts |  2 +-
 6 files changed, 41 insertions(+), 34 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
index dcab62e82ccee..46514935ab80e 100644
--- a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
@@ -17,12 +17,8 @@ export const NewRuleSchema = t.intersection([
   t.type({
     description: t.string,
     enabled: t.boolean,
-    filters: t.array(t.unknown),
-    index: t.array(t.string),
     interval: t.string,
-    language: t.string,
     name: t.string,
-    query: t.string,
     risk_score: t.number,
     severity: t.string,
     type: RuleTypeSchema,
@@ -31,10 +27,14 @@ export const NewRuleSchema = t.intersection([
     anomaly_threshold: t.number,
     created_by: t.string,
     false_positives: t.array(t.string),
+    filters: t.array(t.unknown),
     from: t.string,
     id: t.string,
+    index: t.array(t.string),
+    language: t.string,
     ml_job_id: t.string,
     max_signals: t.number,
+    query: t.string,
     references: t.array(t.string),
     rule_id: t.string,
     saved_id: t.string,
@@ -64,32 +64,34 @@ export const RuleSchema = t.intersection([
     description: t.string,
     enabled: t.boolean,
     false_positives: t.array(t.string),
-    filters: t.array(t.unknown),
     from: t.string,
     id: t.string,
-    index: t.array(t.string),
     interval: t.string,
     immutable: t.boolean,
-    language: t.string,
     name: t.string,
     max_signals: t.number,
-    query: t.string,
     references: t.array(t.string),
     risk_score: t.number,
     rule_id: t.string,
     severity: t.string,
     tags: t.array(t.string),
-    type: t.string,
+    type: RuleTypeSchema,
     to: t.string,
     threat: t.array(t.unknown),
     updated_at: t.string,
     updated_by: t.string,
   }),
   t.partial({
+    anomaly_threshold: t.number,
+    filters: t.array(t.unknown),
+    index: t.array(t.string),
+    language: t.string,
     last_failure_at: t.string,
     last_failure_message: t.string,
     meta: MetaRule,
+    ml_job_id: t.string,
     output_index: t.string,
+    query: t.string,
     saved_id: t.string,
     status: t.string,
     status_date: t.string,
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/query_bar/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/query_bar/index.tsx
index 5886a76182eec..d232c86c19e6f 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/query_bar/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/query_bar/index.tsx
@@ -35,7 +35,7 @@ import * as i18n from './translations';
 export interface FieldValueQueryBar {
   filters: Filter[];
   query: Query;
-  saved_id: string | null;
+  saved_id?: string;
 }
 interface QueryBarDefineRuleProps {
   browserFields: BrowserFields;
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
index 539934688771e..72641fb4df113 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
@@ -7,7 +7,7 @@
 import { isEmpty, isNumber } from 'lodash/fp';
 import moment from 'moment';
 
-import { NewRule } from '../../../../containers/detection_engine/rules';
+import { NewRule, RuleType } from '../../../../containers/detection_engine/rules';
 
 import {
   AboutStepRule,
@@ -39,17 +39,26 @@ const getTimeTypeValue = (time: string): { unit: string; value: number } => {
 };
 
 const formatDefineStepData = (defineStepData: DefineStepRule): DefineStepRuleJson => {
-  const { anomalyThreshold, mlJobId, queryBar, isNew, ruleType, ...rest } = defineStepData;
+  const { anomalyThreshold, mlJobId, queryBar, index, isNew, ruleType, ...rest } = defineStepData;
   const { filters, query, saved_id: savedId } = queryBar;
+  const typeProps =
+    ruleType === 'machine_learning'
+      ? {
+          ...(isNumber(anomalyThreshold) ? { anomaly_threshold: anomalyThreshold } : {}),
+          ...(mlJobId ? { ml_job_id: mlJobId } : {}),
+        }
+      : {
+          index,
+          filters,
+          language: query.language,
+          query: query.query as string,
+          ...(!isEmpty(savedId) ? { saved_id: savedId, type: 'saved_query' as RuleType } : {}),
+        };
+
   return {
     ...rest,
     type: ruleType,
-    language: query.language,
-    filters,
-    query: query.query as string,
-    ...(savedId != null && savedId !== '' ? { saved_id: savedId } : {}),
-    ...(isNumber(anomalyThreshold) ? { anomaly_threshold: anomalyThreshold } : {}),
-    ...(mlJobId ? { ml_job_id: mlJobId } : {}),
+    ...typeProps,
   };
 };
 
@@ -102,13 +111,9 @@ const formatAboutStepData = (aboutStepData: AboutStepRule): AboutStepRuleJson =>
 export const formatRule = (
   defineStepData: DefineStepRule,
   aboutStepData: AboutStepRule,
-  scheduleData: ScheduleStepRule,
-  ruleId?: string
+  scheduleData: ScheduleStepRule
 ): NewRule => ({
-  ...(!isEmpty(defineStepData) && ruleId != null ? { id: ruleId } : {}),
   ...formatDefineStepData(defineStepData),
   ...formatAboutStepData(aboutStepData),
   ...formatScheduleStepData(scheduleData),
-  ...(!isEmpty(defineStepData.queryBar.saved_id) ? { type: 'saved_query' } : {}),
-  // TODO: FILTER OUT MUTEX FIELDS
 });
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/edit/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/edit/index.tsx
index 5e0e4223e3e27..8618bf9504861 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/edit/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/edit/index.tsx
@@ -195,8 +195,8 @@ const EditRulePageComponent: FC = () => {
 
     if (invalidForms.length === 0 && activeForm != null) {
       setTabHasError([]);
-      setRule(
-        formatRule(
+      setRule({
+        ...formatRule(
           (activeFormId === RuleStep.defineRule
             ? activeForm.data
             : myDefineRuleForm.data) as DefineStepRule,
@@ -205,10 +205,10 @@ const EditRulePageComponent: FC = () => {
             : myAboutRuleForm.data) as AboutStepRule,
           (activeFormId === RuleStep.scheduleRule
             ? activeForm.data
-            : myScheduleRuleForm.data) as ScheduleStepRule,
-          ruleId
-        )
-      );
+            : myScheduleRuleForm.data) as ScheduleStepRule
+        ),
+        ...(ruleId ? { id: ruleId } : {}),
+      });
     } else {
       setTabHasError(invalidForms);
     }
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
index ce3088aab0cef..ef0c23faeba30 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
@@ -78,12 +78,12 @@ export interface ScheduleStepRule extends StepRuleData {
 
 export interface DefineStepRuleJson {
   anomaly_threshold?: number;
-  index: string[];
-  filters: Filter[];
+  index?: string[];
+  filters?: Filter[];
   ml_job_id?: string;
   saved_id?: string;
-  query: string;
-  language: string;
+  query?: string;
+  language?: string;
   type: RuleType;
 }
 
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
index 8c23c407f22f0..f57ce145c6a85 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
@@ -33,7 +33,7 @@ export interface RuleAlertParams {
   filters: PartialFilter[] | undefined | null;
   from: string;
   immutable: boolean;
-  index: string[];
+  index: string[] | undefined | null;
   interval: string;
   ruleId: string | undefined | null;
   language: string | undefined | null;

From 7a62dba020cc3382cd1b53edc448a523f7b0aeb6 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Fri, 13 Mar 2020 11:56:23 -0500
Subject: [PATCH 21/63] Rename anomalies query

It's no longer tabular data. If we need that, we can use the ML client.
---
 .../server/lib/detection_engine/signals/find_ml_signals.ts    | 4 ++--
 .../legacy/plugins/siem/server/lib/machine_learning/index.ts  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
index ce70168e01b6a..5a809ba32b124 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
@@ -8,7 +8,7 @@ import dateMath from '@elastic/datemath';
 
 import { AlertServices } from '../../../../../../../plugins/alerting/server';
 
-import { anomaliesTableData } from '../../machine_learning';
+import { getAnomalies } from '../../machine_learning';
 
 export const findMlSignals = async (
   jobId: string,
@@ -23,7 +23,7 @@ export const findMlSignals = async (
     earliestMs: dateMath.parse(from)!.valueOf(),
     latestMs: dateMath.parse(to)!.valueOf(),
   };
-  const relevantAnomalies = await anomaliesTableData(params, callCluster);
+  const relevantAnomalies = await getAnomalies(params, callCluster);
 
   return relevantAnomalies;
 };
diff --git a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
index 53e55811dc4d5..a0fcf21f0183c 100644
--- a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
@@ -19,7 +19,7 @@ export interface AnomaliesSearchParams {
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 type Anomaly = any;
 
-export const anomaliesTableData = async (
+export const getAnomalies = async (
   params: AnomaliesSearchParams,
   callCluster: AlertServices['callCluster']
 ): Promise<SearchResponse<Anomaly>> => {

From 31ac415c1eaa02c0b86fe9c08bc2a1d99ecfffdc Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Sat, 14 Mar 2020 14:26:04 -0500
Subject: [PATCH 22/63] Remove spike page with simple form

---
 .../public/pages/detection_engine/index.tsx   |   4 -
 .../rules/create_ml/index.tsx                 | 110 ------------------
 2 files changed, 114 deletions(-)
 delete mode 100644 x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create_ml/index.tsx

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/index.tsx
index 873b948609b7e..1509348819510 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/index.tsx
@@ -9,7 +9,6 @@ import { Redirect, Route, Switch, RouteComponentProps } from 'react-router-dom';
 
 import { ManageUserInfo } from './components/user_info';
 import { CreateRulePage } from './rules/create';
-import { CreateMLRulePage } from './rules/create_ml';
 import { DetectionEnginePage } from './detection_engine';
 import { EditRulePage } from './rules/edit';
 import { RuleDetailsPage } from './rules/details';
@@ -36,9 +35,6 @@ const DetectionEngineContainerComponent: React.FC<Props> = () => (
       <Route exact path={`${detectionEnginePath}/rules/create`}>
         <CreateRulePage />
       </Route>
-      <Route exact path={`${detectionEnginePath}/rules/create_ml`}>
-        <CreateMLRulePage />
-      </Route>
       <Route exact path={`${detectionEnginePath}/rules/id/:detailName`}>
         <RuleDetailsPage />
       </Route>
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create_ml/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create_ml/index.tsx
deleted file mode 100644
index 82b9b62d12abe..0000000000000
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create_ml/index.tsx
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import React, { useState, useCallback } from 'react';
-import { Redirect } from 'react-router-dom';
-import { EuiButton, EuiFieldText, EuiRange, EuiSuperSelect, EuiText } from '@elastic/eui';
-
-import { DETECTION_ENGINE_PAGE_NAME } from '../../../../components/link_to/redirect_to_detection_engine';
-import { useSiemJobs } from '../../../../components/ml_popover/hooks/use_siem_jobs';
-import { usePersistRule } from '../../../../containers/detection_engine/rules';
-import { RuleType } from '../../../../../server/lib/detection_engine/types';
-
-type Event = React.ChangeEvent<HTMLInputElement>;
-type EventArg = Event | React.MouseEvent<HTMLButtonElement>;
-
-const JobDisplay = ({ title, description }: { title: string; description: string }) => (
-  <>
-    <strong>{title}</strong>
-    <EuiText size="xs" color="subdued">
-      <p>{description}</p>
-    </EuiText>
-  </>
-);
-
-const formatRule = ({
-  jobId,
-  name,
-  threshold,
-}: {
-  jobId: string;
-  name: string;
-  threshold: number;
-}) => ({
-  ml_job_id: jobId,
-  anomaly_threshold: threshold,
-  name,
-  index: [],
-  language: 'kuery',
-  query: '',
-  filters: [],
-  false_positives: [],
-  references: [],
-  risk_score: 50,
-  threat: [],
-  severity: 'low',
-  tags: [],
-  interval: '5m',
-  from: 'now-360s',
-  enabled: false,
-  to: 'now',
-  type: 'machine_learning' as RuleType,
-  description: 'Test ML Rule',
-});
-
-export const CreateMLRulePage = () => {
-  const [jobsLoading, siemJobs] = useSiemJobs(false);
-  const [{ isLoading: ruleLoading, isSaved }, setRule] = usePersistRule();
-  const [jobId, setJobId] = useState<string>('');
-  const [name, setName] = useState<string>('');
-  const [threshold, setThreshold] = useState<number>(50);
-  const isLoading = jobsLoading || ruleLoading;
-
-  const submit = useCallback(() => {
-    console.log(name, jobId, threshold);
-    setRule(
-      formatRule({
-        jobId,
-        name,
-        threshold,
-      })
-    );
-  }, [name, jobId, threshold]);
-
-  const onNameChange = useCallback((event: Event) => {
-    setName(event.target.value);
-  }, []);
-  const onThresholdChange = useCallback((event: EventArg) => {
-    setThreshold(Number((event as Event).target.value));
-  }, []);
-
-  const options = siemJobs.map(job => ({
-    value: job.id,
-    inputDisplay: job.id,
-    dropdownDisplay: <JobDisplay title={job.id} description={job.description} />,
-  }));
-
-  if (isSaved) {
-    return <Redirect to={`/${DETECTION_ENGINE_PAGE_NAME}/rules`} />;
-  }
-
-  return (
-    <>
-      <EuiFieldText onChange={onNameChange} placeholder="Rule name" value={name}></EuiFieldText>
-      <EuiSuperSelect
-        hasDividers
-        isLoading={isLoading}
-        onChange={setJobId}
-        options={options}
-        valueOfSelected={jobId}
-      />
-      <EuiRange min={0} max={100} value={threshold} onChange={onThresholdChange} showInput />
-      <EuiButton onClick={submit} disabled={isLoading}>
-        Create ML Rule
-      </EuiButton>
-    </>
-  );
-};

From 26822ebab10ddeb32f64ba12e7696643c67fdfd5 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Sat, 14 Mar 2020 14:32:28 -0500
Subject: [PATCH 23/63] Remove unneeded ES option

This response isn't going to HTTP, which is where this option would
matter.
---
 x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
index a0fcf21f0183c..773de9f97bf8f 100644
--- a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
@@ -27,7 +27,6 @@ export const getAnomalies = async (
 
   return callCluster('search', {
     index: '.ml-anomalies-*',
-    rest_total_hits_as_int: true,
     size: params.maxRecords || 100,
     body: {
       query: {

From 7434bcb21bef936a87e9781ced8bd15d4e93eece Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Sat, 14 Mar 2020 15:14:25 -0500
Subject: [PATCH 24/63] Fix bulk create logic

I made a happy accident and flipped the logic here, which meant we
weren't capping the signals we created.
---
 .../lib/detection_engine/signals/search_after_bulk_create.ts    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
index 31c16a87fa0ab..4068665116b64 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
@@ -77,7 +77,7 @@ export const searchAfterAndBulkCreate = async ({
   // If the total number of hits for the overall search result is greater than
   // maxSignals, default to requesting a total of maxSignals, otherwise use the
   // totalHits in the response from the searchAfter query.
-  const maxTotalHitsSize = Math.max(totalHits, ruleParams.maxSignals);
+  const maxTotalHitsSize = Math.min(totalHits, ruleParams.maxSignals);
 
   // number of docs in the current search result
   let hitsSize = someResult.hits.hits.length;

From fb7c5f135a527c3c0917265dbea7cd25cf309627 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Sat, 14 Mar 2020 15:39:12 -0500
Subject: [PATCH 25/63] Rename argument

Value is a little more ambiguous than data, here: this is our step data.
---
 .../components/description_step/index.tsx     | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx
index 108a20303f0c9..cb87e848460a2 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx
@@ -111,14 +111,14 @@ export const addFilterStateIfNotThere = (filters: Filter[]): Filter[] => {
 const getDescriptionItem = (
   field: string,
   label: string,
-  value: unknown,
+  data: unknown,
   filterManager: FilterManager,
   indexPatterns?: IIndexPattern
 ): ListItems[] => {
   if (field === 'queryBar') {
-    const filters = addFilterStateIfNotThere(get('queryBar.filters', value) ?? []);
-    const query = get('queryBar.query', value) as Query;
-    const savedId = get('queryBar.saved_id', value);
+    const filters = addFilterStateIfNotThere(get('queryBar.filters', data) ?? []);
+    const query = get('queryBar.query', data) as Query;
+    const savedId = get('queryBar.saved_id', data);
     return buildQueryBarDescription({
       field,
       filters,
@@ -128,24 +128,24 @@ const getDescriptionItem = (
       indexPatterns,
     });
   } else if (field === 'threat') {
-    const threat: IMitreEnterpriseAttack[] = get(field, value).filter(
+    const threat: IMitreEnterpriseAttack[] = get(field, data).filter(
       (singleThreat: IMitreEnterpriseAttack) => singleThreat.tactic.name !== 'none'
     );
     return buildThreatDescription({ label, threat });
   } else if (field === 'references') {
-    const urls: string[] = get(field, value);
+    const urls: string[] = get(field, data);
     return buildUrlsDescription(label, urls);
   } else if (field === 'falsePositives') {
-    const values: string[] = get(field, value);
+    const values: string[] = get(field, data);
     return buildUnorderedListArrayDescription(label, field, values);
-  } else if (Array.isArray(get(field, value))) {
-    const values: string[] = get(field, value);
+  } else if (Array.isArray(get(field, data))) {
+    const values: string[] = get(field, data);
     return buildStringArrayDescription(label, field, values);
   } else if (field === 'severity') {
-    const val: string = get(field, value);
+    const val: string = get(field, data);
     return buildSeverityDescription(label, val);
   } else if (field === 'timeline') {
-    const timeline = get(field, value) as FieldValueTimeline;
+    const timeline = get(field, data) as FieldValueTimeline;
     return [
       {
         title: label,
@@ -154,7 +154,7 @@ const getDescriptionItem = (
     ];
   }
 
-  const description: string = get(field, value);
+  const description: string = get(field, data);
   if (isNumber(description) || !isEmpty(description)) {
     return [
       {

From cec8075dbe14c1f9a371a02506b20e44ae8f8779 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Sat, 14 Mar 2020 17:54:29 -0500
Subject: [PATCH 26/63] Create Rule form stores all values, but filters by type
 for use

When sending off to the backend, or displaying on the readonly view, we
inspect which rule type we've currently selected, and filter our form
values appropriately.
---
 .../components/description_step/helpers.tsx   |  4 +-
 .../components/description_step/index.tsx     |  3 +-
 .../components/description_step/types.ts      |  3 +-
 .../components/step_define_rule/index.tsx     | 12 ++--
 .../detection_engine/rules/create/helpers.ts  | 62 ++++++++++++-------
 .../pages/detection_engine/rules/types.ts     |  4 +-
 6 files changed, 55 insertions(+), 33 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.tsx
index df767fbd4ff8c..d9b87178f6e45 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.tsx
@@ -72,12 +72,12 @@ export const buildQueryBarDescription = ({
       },
     ];
   }
-  if (!isEmpty(query.query)) {
+  if (!isEmpty(query)) {
     items = [
       ...items,
       {
         title: <>{i18n.QUERY_LABEL} </>,
-        description: <>{query.query} </>,
+        description: <>{query} </>,
       },
     ];
   }
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx
index cb87e848460a2..4c57c0f84b4f4 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx
@@ -13,7 +13,6 @@ import {
   Filter,
   esFilters,
   FilterManager,
-  Query,
 } from '../../../../../../../../../../src/plugins/data/public';
 import { DEFAULT_TIMELINE_TITLE } from '../../../../../components/timeline/translations';
 import { useKibana } from '../../../../../lib/kibana';
@@ -117,7 +116,7 @@ const getDescriptionItem = (
 ): ListItems[] => {
   if (field === 'queryBar') {
     const filters = addFilterStateIfNotThere(get('queryBar.filters', data) ?? []);
-    const query = get('queryBar.query', data) as Query;
+    const query = get('queryBar.query.query', data);
     const savedId = get('queryBar.saved_id', data);
     return buildQueryBarDescription({
       field,
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/types.ts
index ab73c52ae9070..bfca6b2068443 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/types.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/types.ts
@@ -9,7 +9,6 @@ import {
   IIndexPattern,
   Filter,
   FilterManager,
-  Query,
 } from '../../../../../../../../../../src/plugins/data/public';
 import { IMitreEnterpriseAttack } from '../../types';
 
@@ -22,7 +21,7 @@ export interface BuildQueryBarDescription {
   field: string;
   filters: Filter[];
   filterManager: FilterManager;
-  query: Query;
+  query: string;
   savedId: string;
   indexPatterns?: IIndexPattern;
 }
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
index 0b0143d337563..8dcb5e584861d 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
@@ -40,6 +40,7 @@ import {
 } from '../../../../../shared_imports';
 import { schema } from './schema';
 import * as i18n from './translations';
+import { filterRuleFieldsForType } from '../../create/helpers';
 
 const CommonUseField = getUseField({ component: Field });
 
@@ -48,13 +49,15 @@ interface StepDefineRuleProps extends RuleStepProps {
 }
 
 const stepDefineDefaultValue: DefineStepRule = {
+  anomalyThreshold: 50,
   index: [],
   isNew: true,
+  mlJobId: '',
   ruleType: 'query',
   queryBar: {
     query: { query: '', language: 'kuery' },
     filters: [],
-    saved_id: null,
+    saved_id: undefined,
   },
 };
 
@@ -160,13 +163,14 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
     setOpenTimelineSearch(false);
   }, []);
 
-  return isReadOnlyView && myStepData?.queryBar != null ? (
+  return isReadOnlyView ? (
     <StepContentWrapper addPadding={addPadding}>
       <StepRuleDescription
         direction={descriptionDirection}
         indexPatterns={indexPatternQueryBar as IIndexPattern}
-        schema={schema}
-        data={myStepData}
+        // @ts-ignore schema is insufficiently typed, but this works
+        schema={filterRuleFieldsForType(schema, myStepData.ruleType)}
+        data={filterRuleFieldsForType(myStepData, myStepData.ruleType)}
       />
     </StepContentWrapper>
   ) : (
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
index 72641fb4df113..155a57d307a39 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { isEmpty, isNumber } from 'lodash/fp';
+import { has, omit, isEmpty } from 'lodash/fp';
 import moment from 'moment';
 
 import { NewRule, RuleType } from '../../../../containers/detection_engine/rules';
@@ -38,28 +38,48 @@ const getTimeTypeValue = (time: string): { unit: string; value: number } => {
   return timeObj;
 };
 
+interface RuleFields {
+  anomalyThreshold: unknown;
+  mlJobId: unknown;
+  queryBar: unknown;
+  index: unknown;
+  ruleType: unknown;
+}
+type QueryRuleFields<T> = Omit<T, 'anomalyThreshold' | 'mlJobId'>;
+type MlRuleFields<T> = Omit<T, 'queryBar' | 'index'>;
+
+const isMlFields = <T>(fields: QueryRuleFields<T> | MlRuleFields<T>): fields is MlRuleFields<T> =>
+  has('anomalyThreshold', fields);
+
+export const filterRuleFieldsForType = <T extends RuleFields>(fields: T, type: RuleType) => {
+  return type === 'machine_learning'
+    ? omit(['index', 'queryBar'], fields)
+    : omit(['anomalyThreshold', 'mlJobId'], fields);
+};
+
 const formatDefineStepData = (defineStepData: DefineStepRule): DefineStepRuleJson => {
-  const { anomalyThreshold, mlJobId, queryBar, index, isNew, ruleType, ...rest } = defineStepData;
-  const { filters, query, saved_id: savedId } = queryBar;
-  const typeProps =
-    ruleType === 'machine_learning'
-      ? {
-          ...(isNumber(anomalyThreshold) ? { anomaly_threshold: anomalyThreshold } : {}),
-          ...(mlJobId ? { ml_job_id: mlJobId } : {}),
-        }
-      : {
-          index,
-          filters,
-          language: query.language,
-          query: query.query as string,
-          ...(!isEmpty(savedId) ? { saved_id: savedId, type: 'saved_query' as RuleType } : {}),
-        };
+  const ruleFields = filterRuleFieldsForType(defineStepData, defineStepData.ruleType);
 
-  return {
-    ...rest,
-    type: ruleType,
-    ...typeProps,
-  };
+  if (isMlFields(ruleFields)) {
+    const { anomalyThreshold, mlJobId, isNew, ruleType, ...rest } = ruleFields;
+    return {
+      ...rest,
+      type: ruleType,
+      anomaly_threshold: anomalyThreshold,
+      ml_job_id: mlJobId,
+    };
+  } else {
+    const { index, queryBar, isNew, ruleType, ...rest } = ruleFields;
+    return {
+      ...rest,
+      type: ruleType,
+      filters: queryBar?.filters,
+      language: queryBar?.query?.language,
+      query: queryBar?.query?.query as string,
+      saved_id: queryBar?.saved_id,
+      ...(ruleType === 'query' && queryBar?.saved_id ? { type: 'saved_query' as RuleType } : {}),
+    };
+  }
 };
 
 const formatScheduleStepData = (scheduleData: ScheduleStepRule): ScheduleStepRuleJson => {
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
index ef0c23faeba30..b8aca075da3de 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
@@ -62,9 +62,9 @@ export interface AboutStepRule extends StepRuleData {
 }
 
 export interface DefineStepRule extends StepRuleData {
-  anomalyThreshold?: number;
+  anomalyThreshold: number;
   index: string[];
-  mlJobId?: string;
+  mlJobId: string;
   queryBar: FieldValueQueryBar;
   ruleType: RuleType;
 }

From b80d5d06c304fd295ef2f9bf373e4fe935233dc1 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Sat, 14 Mar 2020 18:53:28 -0500
Subject: [PATCH 27/63] Fix editing of ML fields on Rule Create

We need to inherit the field value from our form on initial render, and
everything works as expected.
---
 .../rules/components/anomaly_threshold_slider/index.tsx         | 2 +-
 .../detection_engine/rules/components/ml_job_select/index.tsx   | 2 +-
 .../rules/components/select_rule_type/index.tsx                 | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx
index 0d60344e22823..24556c62e1159 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx
@@ -19,7 +19,7 @@ type EventArg = Event | React.MouseEvent<HTMLButtonElement>;
 const Wrapper = styled(EuiFormRow)``;
 
 export const AnomalyThresholdSlider = ({ field }: AnomalyThresholdSliderProps) => {
-  const [localThreshold, setLocalThreshold] = useState<number>(50);
+  const [localThreshold, setLocalThreshold] = useState<number>(field.value as number);
   const onThresholdChange = useCallback(
     (event: EventArg) => {
       const threshold = Number((event as Event).target.value);
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
index d83798f8fd5f0..1039f522f7904 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
@@ -26,7 +26,7 @@ const JobDisplay = ({ title, description }: { title: string; description: string
 );
 
 export const MlJobSelect = ({ field }: MlJobSelectProps) => {
-  const [localJobId, setLocalJobId] = useState<string>('query');
+  const [localJobId, setLocalJobId] = useState<string>(field.value as string);
   const [isLoading, siemJobs] = useSiemJobs(false);
   const handleJobChange = useCallback(
     (jobId: string) => {
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
index 1c735904b96a3..57495d8a8b51a 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
@@ -19,7 +19,7 @@ interface SelectRuleTypeProps {
 const Wrapper = styled(EuiFormRow)``;
 
 export const SelectRuleType = ({ field }: SelectRuleTypeProps) => {
-  const [ruleType, setRuleType] = useState<RuleType>('query');
+  const [ruleType, setRuleType] = useState<RuleType>(field.value as RuleType);
   const setType = useCallback(
     (type: RuleType) => {
       setRuleType(type);

From 05d4dda1a9920d92d9dd86ffabc1843e9ab93e9f Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Sat, 14 Mar 2020 19:17:05 -0500
Subject: [PATCH 28/63] Clear form errors when switching between rule types

Validation errors prevent us from moving to the next step, so it was
previously possible to get an error for Query fields, switch to an ML
rule, and be unable to continue because the form had Query errors.

This also adds a helper for checking whether a ruleType is ML, to
prevent having to change all these references if the type string
changes.
---
 .../rules/components/select_rule_type/index.tsx   |  5 +++--
 .../rules/components/step_define_rule/index.tsx   | 15 +++++++++------
 .../rules/components/step_define_rule/schema.tsx  |  7 ++++---
 .../detection_engine/rules/create/helpers.ts      |  3 ++-
 .../pages/detection_engine/rules/helpers.tsx      |  4 +++-
 5 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
index 57495d8a8b51a..e6290214524a6 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
@@ -11,6 +11,7 @@ import { EuiCard, EuiFlexGrid, EuiFlexItem, EuiIcon, EuiFormRow } from '@elastic
 import { FieldHook } from '../../../../../shared_imports';
 import { RuleType } from '../../../../../containers/detection_engine/rules/types';
 import * as i18n from './translations';
+import { isMlRule } from '../../helpers';
 
 interface SelectRuleTypeProps {
   field: FieldHook;
@@ -41,7 +42,7 @@ export const SelectRuleType = ({ field }: SelectRuleTypeProps) => {
             icon={<EuiIcon size="l" type="search" />}
             selectable={{
               onClick: setQuery,
-              isSelected: ruleType === 'query',
+              isSelected: !isMlRule(ruleType),
             }}
           />
         </EuiFlexItem>
@@ -53,7 +54,7 @@ export const SelectRuleType = ({ field }: SelectRuleTypeProps) => {
             icon={<EuiIcon size="l" type="machineLearningApp" />}
             selectable={{
               onClick: setMl,
-              isSelected: ruleType === 'machine_learning',
+              isSelected: isMlRule(ruleType),
             }}
           />
         </EuiFlexItem>
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
index 8dcb5e584861d..3ee27b6023674 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
@@ -21,7 +21,7 @@ import { IIndexPattern } from '../../../../../../../../../../src/plugins/data/pu
 import { useFetchIndexPatterns } from '../../../../../containers/detection_engine/rules';
 import { DEFAULT_INDEX_KEY } from '../../../../../../common/constants';
 import { useUiSetting$ } from '../../../../../lib/kibana';
-import { setFieldValue } from '../../helpers';
+import { setFieldValue, isMlRule } from '../../helpers';
 import * as RuleI18n from '../../translations';
 import { DefineStepRule, RuleStep, RuleStepProps } from '../../types';
 import { StepRuleDescription } from '../description_step';
@@ -104,7 +104,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
 }) => {
   const [openTimelineSearch, setOpenTimelineSearch] = useState(false);
   const [localUseIndicesConfig, setLocalUseIndicesConfig] = useState(false);
-  const [isMlRule, setIsMlRule] = useState(false);
+  const [localIsMlRule, setIsMlRule] = useState(false);
   const [indicesConfig] = useUiSetting$<string[]>(DEFAULT_INDEX_KEY);
   const [mylocalIndicesConfig, setMyLocalIndicesConfig] = useState(
     defaultValues != null ? defaultValues.index : indicesConfig ?? []
@@ -121,6 +121,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
     options: { stripEmptyFields: false },
     schema,
   });
+  const clearErrors = useCallback(() => form.reset({ resetValues: false }), [form]);
 
   const onSubmit = useCallback(async () => {
     if (setStepData) {
@@ -178,7 +179,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
       <StepContentWrapper addPadding={!isUpdateView}>
         <Form form={form} data-test-subj="stepDefineRule">
           <UseField path="ruleType" component={SelectRuleType} />
-          <EuiFormRow fullWidth style={{ display: isMlRule ? 'none' : 'flex' }}>
+          <EuiFormRow fullWidth style={{ display: localIsMlRule ? 'none' : 'flex' }}>
             <>
               <CommonUseField
                 path="index"
@@ -225,7 +226,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
               />
             </>
           </EuiFormRow>
-          <EuiFormRow fullWidth style={{ display: isMlRule ? 'flex' : 'none' }}>
+          <EuiFormRow fullWidth style={{ display: localIsMlRule ? 'flex' : 'none' }}>
             <>
               <UseField path="mlJobId" component={MlJobSelect} />
               <UseField path="anomalyThreshold" component={AnomalyThresholdSlider} />
@@ -245,10 +246,12 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
                 }
               }
 
-              if (ruleType === 'machine_learning' && !isMlRule) {
+              if (isMlRule(ruleType) && !localIsMlRule) {
                 setIsMlRule(true);
-              } else if (ruleType !== 'machine_learning' && isMlRule) {
+                clearErrors();
+              } else if (!isMlRule(ruleType) && localIsMlRule) {
                 setIsMlRule(false);
+                clearErrors();
               }
 
               return null;
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
index 2917d39521d29..6d5167f8aead4 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
@@ -19,6 +19,7 @@ import {
   ValidationFunc,
 } from '../../../../../shared_imports';
 import { CUSTOM_QUERY_REQUIRED, INVALID_CUSTOM_QUERY, INDEX_HELPER_TEXT } from './translations';
+import { isMlRule } from '../../helpers';
 
 export const schema: FormSchema = {
   index: {
@@ -36,7 +37,7 @@ export const schema: FormSchema = {
           ...args: Parameters<ValidationFunc>
         ): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
           const [{ formData }] = args;
-          const needsValidation = formData.ruleType !== 'machine_learning';
+          const needsValidation = !isMlRule(formData.ruleType);
 
           if (!needsValidation) {
             return;
@@ -68,7 +69,7 @@ export const schema: FormSchema = {
         ): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
           const [{ value, path, formData }] = args;
           const { query, filters } = value as FieldValueQueryBar;
-          const needsValidation = formData.ruleType !== 'machine_learning';
+          const needsValidation = !isMlRule(formData.ruleType);
           if (!needsValidation) {
             return;
           }
@@ -88,7 +89,7 @@ export const schema: FormSchema = {
         ): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
           const [{ value, path, formData }] = args;
           const { query } = value as FieldValueQueryBar;
-          const needsValidation = formData.ruleType !== 'machine_learning';
+          const needsValidation = !isMlRule(formData.ruleType);
           if (!needsValidation) {
             return;
           }
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
index 155a57d307a39..654fed38080c9 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
@@ -17,6 +17,7 @@ import {
   ScheduleStepRuleJson,
   AboutStepRuleJson,
 } from '../types';
+import { isMlRule } from '../helpers';
 
 const getTimeTypeValue = (time: string): { unit: string; value: number } => {
   const timeObj = {
@@ -52,7 +53,7 @@ const isMlFields = <T>(fields: QueryRuleFields<T> | MlRuleFields<T>): fields is
   has('anomalyThreshold', fields);
 
 export const filterRuleFieldsForType = <T extends RuleFields>(fields: T, type: RuleType) => {
-  return type === 'machine_learning'
+  return isMlRule(type)
     ? omit(['index', 'queryBar'], fields)
     : omit(['anomalyThreshold', 'mlJobId'], fields);
 };
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx
index 85f3bcbd236e9..871eff2a18914 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx
@@ -10,7 +10,7 @@ import moment from 'moment';
 import { useLocation } from 'react-router-dom';
 
 import { Filter } from '../../../../../../../../src/plugins/data/public';
-import { Rule } from '../../../containers/detection_engine/rules';
+import { Rule, RuleType } from '../../../containers/detection_engine/rules';
 import { FormData, FormHook, FormSchema } from '../../../shared_imports';
 import { AboutStepRule, DefineStepRule, IMitreEnterpriseAttack, ScheduleStepRule } from './types';
 
@@ -139,6 +139,8 @@ export const setFieldValue = (
     }
   });
 
+export const isMlRule = (ruleType: RuleType) => ruleType === 'machine_learning';
+
 export const redirectToDetections = (
   isSignalIndexExists: boolean | null,
   isAuthenticated: boolean | null,

From 79d9cb44739dafb063be0f4ab1ff67d8e856a1e7 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Sat, 14 Mar 2020 19:42:18 -0500
Subject: [PATCH 29/63] Validate the selection of an ML Job

---
 .../rules/components/ml_job_select/index.tsx  |  5 +++--
 .../components/step_define_rule/schema.tsx    | 21 ++++++++++++++++++-
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
index 1039f522f7904..ee0d95fff2bb9 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
@@ -8,7 +8,7 @@ import React, { useCallback, useState } from 'react';
 import styled from 'styled-components';
 import { EuiFlexGroup, EuiFlexItem, EuiFormRow, EuiSuperSelect, EuiText } from '@elastic/eui';
 
-import { FieldHook } from '../../../../../shared_imports';
+import { FieldHook, getFieldValidityAndErrorMessage } from '../../../../../shared_imports';
 import { useSiemJobs } from '../../../../../components/ml_popover/hooks/use_siem_jobs';
 
 interface MlJobSelectProps {
@@ -26,6 +26,7 @@ const JobDisplay = ({ title, description }: { title: string; description: string
 );
 
 export const MlJobSelect = ({ field }: MlJobSelectProps) => {
+  const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(field);
   const [localJobId, setLocalJobId] = useState<string>(field.value as string);
   const [isLoading, siemJobs] = useSiemJobs(false);
   const handleJobChange = useCallback(
@@ -43,7 +44,7 @@ export const MlJobSelect = ({ field }: MlJobSelectProps) => {
   }));
 
   return (
-    <Wrapper label={field.label} fullWidth>
+    <Wrapper fullWidth label={field.label} isInvalid={isInvalid} error={errorMessage}>
       <EuiFlexGroup>
         <EuiFlexItem>
           <EuiSuperSelect
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
index 6d5167f8aead4..b50607f7a4c0f 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
@@ -134,6 +134,25 @@ export const schema: FormSchema = {
         defaultMessage: 'Machine Learning job',
       }
     ),
-    validations: [],
+    validations: [
+      {
+        validator: (
+          ...args: Parameters<ValidationFunc>
+        ): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
+          const [{ formData }] = args;
+          const needsValidation = isMlRule(formData.ruleType);
+
+          if (!needsValidation) {
+            return;
+          }
+
+          return fieldValidators.emptyField(
+            i18n.translate('xpack.siem.detectionEngine.createRule.stepDefineRule.mlJobIdRequired', {
+              defaultMessage: 'A Machine Learning job is required.',
+            })
+          )(...args);
+        },
+      },
+    ],
   },
 };

From 54247463a30aff0d9df40b8a8d66009d0864ba66 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Sat, 14 Mar 2020 20:04:34 -0500
Subject: [PATCH 30/63] Fix type errors on frontend

According to the types, this is essentially the opposite of formatRule,
so we need to reinflate all potential form values from the rule.
---
 .../public/pages/detection_engine/rules/helpers.tsx   | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx
index 871eff2a18914..b9a07202d7d3b 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx
@@ -31,11 +31,14 @@ export const getStepsData = ({
     rule != null
       ? {
           isNew: false,
-          index: rule.index,
+          ruleType: rule.type,
+          anomalyThreshold: rule.anomaly_threshold ?? 50,
+          mlJobId: rule.ml_job_id ?? '',
+          index: rule.index ?? [],
           queryBar: {
-            query: { query: rule.query as string, language: rule.language },
-            filters: rule.filters as Filter[],
-            saved_id: rule.saved_id ?? null,
+            query: { query: rule.query ?? '', language: rule.language ?? '' },
+            filters: (rule.filters ?? []) as Filter[],
+            saved_id: rule.saved_id,
           },
         }
       : null;

From a689d330fa627418d864e34146b95adabd5356d1 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Sun, 15 Mar 2020 17:30:16 -0500
Subject: [PATCH 31/63] Don't set defaults for query-specific rules

For ML rules these types should not be included.
---
 .../detection_engine/routes/schemas/create_rules_schema.ts    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts
index c25438578b6c7..d5132507e7476 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts
@@ -55,8 +55,8 @@ export const createRulesSchema = Joi.object({
   rule_id,
   index,
   interval: interval.default('5m'),
-  query: query.allow('').default(''),
-  language: language.default('kuery'),
+  query, // TODO conditional type/default?
+  language,
   output_index,
   saved_id: saved_id.when('type', {
     is: 'saved_query',

From 9cb88c782957ae3c92d0548add0d6e42b54b16f2 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Sun, 15 Mar 2020 17:35:52 -0500
Subject: [PATCH 32/63] Return ML Fields in Rule responses

This adds these fields to our rule serialization, and then adds
conditional validation around those fields if the rule type is ML.
Conversely, we moved the 'language' and 'query' fields to be
conditionally validated if the rule is a query/saved_query rule.
---
 .../detection_engine/routes/rules/utils.ts    |  2 +
 .../schemas/response/__mocks__/utils.ts       | 12 ++++
 .../response/check_type_dependents.test.ts    | 68 ++++++++++++++++++-
 .../schemas/response/check_type_dependents.ts | 24 +++++++
 .../routes/schemas/response/rules_schema.ts   | 12 +++-
 .../routes/schemas/response/schemas.ts        |  2 +
 6 files changed, 117 insertions(+), 3 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts
index ecf669b0106c3..a9794e1e82c5b 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts
@@ -106,6 +106,7 @@ export const transformAlertToRule = (
     created_by: alert.createdBy,
     description: alert.params.description,
     enabled: alert.enabled,
+    anomaly_threshold: alert.params.anomalyThreshold,
     false_positives: alert.params.falsePositives,
     filters: alert.params.filters,
     from: alert.params.from,
@@ -117,6 +118,7 @@ export const transformAlertToRule = (
     language: alert.params.language,
     output_index: alert.params.outputIndex,
     max_signals: alert.params.maxSignals,
+    ml_job_id: alert.params.mlJobId,
     risk_score: alert.params.riskScore,
     name: alert.name,
     query: alert.params.query,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/__mocks__/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/__mocks__/utils.ts
index 05b85ffab7263..cc6c94373934a 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/__mocks__/utils.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/__mocks__/utils.ts
@@ -67,6 +67,18 @@ export const getBaseResponsePayload = (anchorDate: string = ANCHOR_DATE): RulesS
 
 export const getRulesBulkPayload = (): RulesBulkSchema => [getBaseResponsePayload()];
 
+export const getMlRuleResponsePayload = (anchorDate: string = ANCHOR_DATE): RulesSchema => {
+  const basePayload = getBaseResponsePayload(anchorDate);
+  const { filters, index, query, language, ...rest } = basePayload;
+
+  return {
+    ...rest,
+    type: 'machine_learning',
+    anomaly_threshold: 59,
+    ml_job_id: 'some_ml_job_id',
+  };
+};
+
 export const getErrorPayload = (
   id: string = '819eded6-e9c8-445b-a647-519aea39e063'
 ): ErrorSchema => ({
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.test.ts
index fc1c019ff97b5..1a5ee793a25da 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.test.ts
@@ -12,8 +12,15 @@ import {
   getDependents,
   addSavedId,
   addTimelineTitle,
+  addQueryFields,
+  addMlFields,
 } from './check_type_dependents';
-import { foldLeftRight, getBaseResponsePayload, getPaths } from './__mocks__/utils';
+import {
+  foldLeftRight,
+  getBaseResponsePayload,
+  getPaths,
+  getMlRuleResponsePayload,
+} from './__mocks__/utils';
 import { left } from 'fp-ts/lib/Either';
 import { exactCheck } from './exact_check';
 import { RulesSchema } from './rules_schema';
@@ -375,6 +382,34 @@ describe('check_type_dependents', () => {
       ]);
       expect(message.schema).toEqual({});
     });
+
+    test('it validates an ML rule response', () => {
+      const payload = getMlRuleResponsePayload();
+
+      const dependents = getDependents(payload);
+      const decoded = dependents.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      const expected = getMlRuleResponsePayload();
+
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(expected);
+    });
+
+    test('it rejects a response with both ML and query properties', () => {
+      const payload = {
+        ...getBaseResponsePayload(),
+        ...getMlRuleResponsePayload(),
+      };
+
+      const dependents = getDependents(payload);
+      const decoded = dependents.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+
+      expect(getPaths(left(message.errors))).toEqual(['invalid keys "query,language"']);
+      expect(message.schema).toEqual({});
+    });
   });
 
   describe('addSavedId', () => {
@@ -402,4 +437,35 @@ describe('check_type_dependents', () => {
       expect(array.length).toEqual(2);
     });
   });
+
+  describe('addQueryFields', () => {
+    test('should return empty array if type is not "query"', () => {
+      const fields = addQueryFields({ type: 'machine_learning' });
+      const expected: t.Mixed[] = [];
+      expect(fields).toEqual(expected);
+    });
+
+    test('should return two fields for a rule of type "query"', () => {
+      const fields = addQueryFields({ type: 'query' });
+      expect(fields.length).toEqual(2);
+    });
+
+    test('should return two fields for a rule of type "saved_query"', () => {
+      const fields = addQueryFields({ type: 'saved_query' });
+      expect(fields.length).toEqual(2);
+    });
+  });
+
+  describe('addMlFields', () => {
+    test('should return empty array if type is not "machine_learning"', () => {
+      const fields = addMlFields({ type: 'query' });
+      const expected: t.Mixed[] = [];
+      expect(fields).toEqual(expected);
+    });
+
+    test('should return two fields for a rule of type "machine_learning"', () => {
+      const fields = addMlFields({ type: 'machine_learning' });
+      expect(fields.length).toEqual(2);
+    });
+  });
 });
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts
index 09142c8568b2d..0fac827c6e50e 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts
@@ -35,12 +35,36 @@ export const addTimelineTitle = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mi
   }
 };
 
+export const addQueryFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => {
+  if (typeAndTimelineOnly.type === 'query' || typeAndTimelineOnly.type === 'saved_query') {
+    return [
+      t.exact(t.type({ query: dependentRulesSchema.props.query })),
+      t.exact(t.type({ language: dependentRulesSchema.props.language })),
+    ];
+  } else {
+    return [];
+  }
+};
+
+export const addMlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => {
+  if (typeAndTimelineOnly.type === 'machine_learning') {
+    return [
+      t.exact(t.type({ anomaly_threshold: dependentRulesSchema.props.anomaly_threshold })),
+      t.exact(t.type({ ml_job_id: dependentRulesSchema.props.ml_job_id })),
+    ];
+  } else {
+    return [];
+  }
+};
+
 export const getDependents = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed => {
   const dependents: t.Mixed[] = [
     t.exact(requiredRulesSchema),
     t.exact(partialRulesSchema),
     ...addSavedId(typeAndTimelineOnly),
     ...addTimelineTitle(typeAndTimelineOnly),
+    ...addQueryFields(typeAndTimelineOnly),
+    ...addMlFields(typeAndTimelineOnly),
   ];
 
   if (dependents.length > 1) {
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/rules_schema.ts
index 945b5651be066..ca82d60ece755 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/rules_schema.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/rules_schema.ts
@@ -11,6 +11,7 @@ import { Either } from 'fp-ts/lib/Either';
 
 import { checkTypeDependents } from './check_type_dependents';
 import {
+  anomaly_threshold,
   description,
   enabled,
   false_positives,
@@ -24,6 +25,7 @@ import {
   name,
   output_index,
   max_signals,
+  ml_job_id,
   query,
   references,
   severity,
@@ -65,12 +67,10 @@ export const requiredRulesSchema = t.type({
   immutable,
   interval,
   rule_id,
-  language,
   output_index,
   max_signals,
   risk_score,
   name,
-  query,
   references,
   severity,
   updated_by,
@@ -91,12 +91,20 @@ export type RequiredRulesSchema = t.TypeOf<typeof requiredRulesSchema>;
  * check_type_dependents file for whichever REST flow it is going through.
  */
 export const dependentRulesSchema = t.partial({
+  // query fields
+  language,
+  query,
+
   // when type = saved_query, saved_is is required
   saved_id,
 
   // These two are required together or not at all.
   timeline_id,
   timeline_title,
+
+  // ML fields
+  anomaly_threshold,
+  ml_job_id,
 });
 
 /**
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts
index c4ed3db677aa8..59da24471c63e 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts
@@ -45,6 +45,8 @@ export const output_index = t.string;
 export const saved_id = t.string;
 export const timeline_id = t.string;
 export const timeline_title = t.string;
+export const anomaly_threshold = PositiveInteger;
+export const ml_job_id = t.string;
 
 /**
  * Note that this is a plain unknown object because we allow the UI

From 85678cb2e100862716b8a1bfde4aaa71af057b78 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 16 Mar 2020 12:49:31 -0500
Subject: [PATCH 33/63] Fix editing of ML rules by changing who controls the
 field values

The source of truth for their state is the parent form object; these
inputs should not have local state.
---
 .../components/anomaly_threshold_slider/index.tsx     | 11 +++++------
 .../rules/components/ml_job_select/index.tsx          | 11 +++++------
 .../rules/components/select_rule_type/index.tsx       |  5 ++---
 3 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx
index 24556c62e1159..be4fd27840830 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useCallback, useState } from 'react';
+import React, { useCallback } from 'react';
 import styled from 'styled-components';
 import { EuiFlexGrid, EuiFlexItem, EuiRange, EuiFormRow } from '@elastic/eui';
 
@@ -19,12 +19,11 @@ type EventArg = Event | React.MouseEvent<HTMLButtonElement>;
 const Wrapper = styled(EuiFormRow)``;
 
 export const AnomalyThresholdSlider = ({ field }: AnomalyThresholdSliderProps) => {
-  const [localThreshold, setLocalThreshold] = useState<number>(field.value as number);
+  const threshold = field.value as number;
   const onThresholdChange = useCallback(
     (event: EventArg) => {
-      const threshold = Number((event as Event).target.value);
-      setLocalThreshold(threshold);
-      field.setValue(threshold);
+      const thresholdValue = Number((event as Event).target.value);
+      field.setValue(thresholdValue);
     },
     [field]
   );
@@ -34,7 +33,7 @@ export const AnomalyThresholdSlider = ({ field }: AnomalyThresholdSliderProps) =
       <EuiFlexGrid columns={2}>
         <EuiFlexItem>
           <EuiRange
-            value={localThreshold}
+            value={threshold}
             onChange={onThresholdChange}
             fullWidth
             showInput
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
index ee0d95fff2bb9..ff9c64ac0915a 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useCallback, useState } from 'react';
+import React, { useCallback } from 'react';
 import styled from 'styled-components';
 import { EuiFlexGroup, EuiFlexItem, EuiFormRow, EuiSuperSelect, EuiText } from '@elastic/eui';
 
@@ -26,13 +26,12 @@ const JobDisplay = ({ title, description }: { title: string; description: string
 );
 
 export const MlJobSelect = ({ field }: MlJobSelectProps) => {
+  const jobId = field.value as string;
   const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(field);
-  const [localJobId, setLocalJobId] = useState<string>(field.value as string);
   const [isLoading, siemJobs] = useSiemJobs(false);
   const handleJobChange = useCallback(
-    (jobId: string) => {
-      setLocalJobId(jobId);
-      field.setValue(jobId);
+    (mlJobId: string) => {
+      field.setValue(mlJobId);
     },
     [field]
   );
@@ -52,7 +51,7 @@ export const MlJobSelect = ({ field }: MlJobSelectProps) => {
             isLoading={isLoading}
             onChange={handleJobChange}
             options={options}
-            valueOfSelected={localJobId}
+            valueOfSelected={jobId}
           />
         </EuiFlexItem>
       </EuiFlexGroup>
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
index e6290214524a6..77c2a053ea209 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useCallback, useState } from 'react';
+import React, { useCallback } from 'react';
 import styled from 'styled-components';
 import { EuiCard, EuiFlexGrid, EuiFlexItem, EuiIcon, EuiFormRow } from '@elastic/eui';
 
@@ -20,10 +20,9 @@ interface SelectRuleTypeProps {
 const Wrapper = styled(EuiFormRow)``;
 
 export const SelectRuleType = ({ field }: SelectRuleTypeProps) => {
-  const [ruleType, setRuleType] = useState<RuleType>(field.value as RuleType);
+  const ruleType = field.value as RuleType;
   const setType = useCallback(
     (type: RuleType) => {
-      setRuleType(type);
       field.setValue(type);
     },
     [field]

From 2cd97f1c87475288c314077026cac7a3e1fa942d Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 16 Mar 2020 19:31:17 -0500
Subject: [PATCH 34/63] Fix type errors related to new ML fields

In adding the new ML fields, some other fields (e.g. `query` and
`index`) that were previously required but implicitly part of Query
Rules are now marked as optional.

Consequently, any downstream code that actually required these fields
started to complain. In general, the fix was to verify that those fields
exist, and throw an error otherwise as to appease the linter.

Runtime-wise, the new ML rules/signals follow a separate code path and
both branches should be unaffected by these changes; the issue is simply
that our conditional types don't work well with Typescript.
---
 .../lib/detection_engine/signals/get_filter.ts   |  3 +++
 .../signals/search_after_bulk_create.ts          |  8 +++++++-
 .../signals/signal_rule_alert_type.ts            | 12 ++++++++++--
 .../signals/single_search_after.test.ts          | 16 +++++++++-------
 .../signals/single_search_after.ts               | 15 +++++++++------
 5 files changed, 38 insertions(+), 16 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_filter.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_filter.ts
index 9c3e15de7ce90..53750eec49fb0 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_filter.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_filter.ts
@@ -107,6 +107,9 @@ export const getFilter = async ({
         throw new BadRequestError('savedId parameter should be defined');
       }
     }
+    case 'machine_learning': {
+      throw new Error('getFilter called with a ML Rule');
+    }
   }
   return assertUnreachable(type);
 };
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
index 4068665116b64..b67e20248dc87 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
@@ -52,6 +52,10 @@ export const searchAfterAndBulkCreate = async ({
   if (someResult.hits.hits.length === 0) {
     return true;
   }
+  const { index, from, to } = ruleParams;
+  if (index == null) {
+    throw new Error('Attempted to bulk create signals, but rule had no indexPattern');
+  }
 
   logger.debug('[+] starting bulk insertion');
   await singleBulkCreate({
@@ -98,7 +102,9 @@ export const searchAfterAndBulkCreate = async ({
       logger.debug(`sortIds: ${sortIds}`);
       const searchAfterResult: SignalSearchResponse = await singleSearchAfter({
         searchAfterSortId: sortId,
-        ruleParams,
+        index,
+        from,
+        to,
         services,
         logger,
         filter,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index 9941e6fd13c72..14d3646a9c6ae 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -98,9 +98,13 @@ export const signalRulesAlertType = ({
 
       try {
         if (type === 'machine_learning') {
+          if (mlJobId == null || anomalyThreshold == null) {
+            throw new Error('Attempted to execute ML Rule, but missing jobId or anomalyThreshold');
+          }
+
           const anomalyResults = await findMlSignals(
-            mlJobId!,
-            anomalyThreshold!,
+            mlJobId,
+            anomalyThreshold,
             from,
             to,
             services.callCluster
@@ -130,6 +134,10 @@ export const signalRulesAlertType = ({
             tags,
           });
         } else {
+          if (index == null) {
+            throw new Error('Attempted to execute Query Rule, but no index was specified');
+          }
+
           const inputIndex = await getInputIndex(services, version, index);
           const esFilter = await getFilter({
             type,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts
index a5d1f66d3089e..1685c6518def3 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts
@@ -6,7 +6,6 @@
 
 import { savedObjectsClientMock } from 'src/core/server/mocks';
 import {
-  sampleRuleAlertParams,
   sampleDocSearchResultsNoSortId,
   mockLogger,
   sampleDocSearchResultsWithSortId,
@@ -26,12 +25,13 @@ describe('singleSearchAfter', () => {
 
   test('if singleSearchAfter works without a given sort id', async () => {
     let searchAfterSortId;
-    const sampleParams = sampleRuleAlertParams();
     mockService.callCluster.mockReturnValue(sampleDocSearchResultsNoSortId);
     await expect(
       singleSearchAfter({
         searchAfterSortId,
-        ruleParams: sampleParams,
+        index: [],
+        from: 'now-360s',
+        to: 'now',
         services: mockService,
         logger: mockLogger,
         pageSize: 1,
@@ -41,11 +41,12 @@ describe('singleSearchAfter', () => {
   });
   test('if singleSearchAfter works with a given sort id', async () => {
     const searchAfterSortId = '1234567891111';
-    const sampleParams = sampleRuleAlertParams();
     mockService.callCluster.mockReturnValue(sampleDocSearchResultsWithSortId);
     const searchAfterResult = await singleSearchAfter({
       searchAfterSortId,
-      ruleParams: sampleParams,
+      index: [],
+      from: 'now-360s',
+      to: 'now',
       services: mockService,
       logger: mockLogger,
       pageSize: 1,
@@ -55,14 +56,15 @@ describe('singleSearchAfter', () => {
   });
   test('if singleSearchAfter throws error', async () => {
     const searchAfterSortId = '1234567891111';
-    const sampleParams = sampleRuleAlertParams();
     mockService.callCluster.mockImplementation(async () => {
       throw Error('Fake Error');
     });
     await expect(
       singleSearchAfter({
         searchAfterSortId,
-        ruleParams: sampleParams,
+        index: [],
+        from: 'now-360s',
+        to: 'now',
         services: mockService,
         logger: mockLogger,
         pageSize: 1,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts
index a0e7047ad1cd6..bb12b5a802f8f 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts
@@ -5,14 +5,15 @@
  */
 
 import { AlertServices } from '../../../../../../../plugins/alerting/server';
-import { RuleTypeParams } from '../types';
 import { Logger } from '../../../../../../../../src/core/server';
 import { SignalSearchResponse } from './types';
 import { buildEventsSearchQuery } from './build_events_query';
 
 interface SingleSearchAfterParams {
   searchAfterSortId: string | undefined;
-  ruleParams: RuleTypeParams;
+  index: string[];
+  from: string;
+  to: string;
   services: AlertServices;
   logger: Logger;
   pageSize: number;
@@ -22,7 +23,9 @@ interface SingleSearchAfterParams {
 // utilize search_after for paging results into bulk.
 export const singleSearchAfter = async ({
   searchAfterSortId,
-  ruleParams,
+  index,
+  from,
+  to,
   services,
   filter,
   logger,
@@ -33,9 +36,9 @@ export const singleSearchAfter = async ({
   }
   try {
     const searchAfterQuery = buildEventsSearchQuery({
-      index: ruleParams.index,
-      from: ruleParams.from,
-      to: ruleParams.to,
+      index,
+      from,
+      to,
       filter,
       size: pageSize,
       searchAfterSortId,

From 23532ab7d709fcbc26004cc293a0aa8500f03b46 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 16 Mar 2020 19:44:47 -0500
Subject: [PATCH 35/63] Fix failing route tests

Error message changed.
---
 .../routes/rules/create_rules_bulk_route.test.ts                | 2 +-
 .../routes/rules/patch_rules_bulk_route.test.ts                 | 2 +-
 .../lib/detection_engine/routes/rules/patch_rules_route.test.ts | 2 +-
 .../routes/rules/update_rules_bulk_route.test.ts                | 2 +-
 .../detection_engine/routes/rules/update_rules_route.test.ts    | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.test.ts
index 6ad9efebce2dd..2b31d37dddddb 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.test.ts
@@ -137,7 +137,7 @@ describe('create_rules_bulk', () => {
       const result = server.validate(request);
 
       expect(result.badRequest).toHaveBeenCalledWith(
-        '"value" at position 0 fails because [child "type" fails because ["type" must be one of [query, saved_query]]]'
+        '"value" at position 0 fails because [child "type" fails because ["type" must be one of [query, saved_query, machine_learning]]]'
       );
     });
   });
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.test.ts
index 19bcd2e7f0596..967fd46f7e3da 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.test.ts
@@ -89,7 +89,7 @@ describe('patch_rules_bulk', () => {
       const result = server.validate(request);
 
       expect(result.badRequest).toHaveBeenCalledWith(
-        '"value" at position 0 fails because [child "type" fails because ["type" must be one of [query, saved_query]]]'
+        '"value" at position 0 fails because [child "type" fails because ["type" must be one of [query, saved_query, machine_learning]]]'
       );
     });
   });
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts
index 1658de77e3390..0c2ca882a5590 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts
@@ -112,7 +112,7 @@ describe('patch_rules', () => {
       const result = server.validate(request);
 
       expect(result.badRequest).toHaveBeenCalledWith(
-        'child "type" fails because ["type" must be one of [query, saved_query]]'
+        'child "type" fails because ["type" must be one of [query, saved_query, machine_learning]]'
       );
     });
   });
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.test.ts
index 7a9159ecc852b..46639e1fe3380 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.test.ts
@@ -110,7 +110,7 @@ describe('update_rules_bulk', () => {
       const result = server.validate(request);
 
       expect(result.badRequest).toHaveBeenCalledWith(
-        '"value" at position 0 fails because [child "type" fails because ["type" must be one of [query, saved_query]]]'
+        '"value" at position 0 fails because [child "type" fails because ["type" must be one of [query, saved_query, machine_learning]]]'
       );
     });
   });
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.test.ts
index 6ef508b817713..a6da8cd56ec17 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.test.ts
@@ -115,7 +115,7 @@ describe('update_rules', () => {
       const result = await server.validate(request);
 
       expect(result.badRequest).toHaveBeenCalledWith(
-        'child "type" fails because ["type" must be one of [query, saved_query]]'
+        'child "type" fails because ["type" must be one of [query, saved_query, machine_learning]]'
       );
     });
   });

From 41e5af171327cb7365d13bbeed03d8eacb3ff256 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 16 Mar 2020 21:14:21 -0500
Subject: [PATCH 36/63] Fix integration tests

We were not sending required properties when creating a rule(index and
language).
---
 .../security_and_spaces/tests/utils.ts                         | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts
index 1570124cdb92b..c332916c9f662 100644
--- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts
+++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts
@@ -49,8 +49,10 @@ export const getSimpleRule = (ruleId = 'rule-1'): Partial<OutputRuleAlertRest> =
   risk_score: 1,
   rule_id: ruleId,
   severity: 'high',
+  index: ['auditbeat-*'],
   type: 'query',
   query: 'user.name: root or user.name: admin',
+  language: 'kuery',
 });
 
 export const getSignalStatus = () => ({
@@ -118,6 +120,7 @@ export const getSimpleRuleOutput = (ruleId = 'rule-1'): Partial<OutputRuleAlertR
   false_positives: [],
   from: 'now-6m',
   immutable: false,
+  index: ['auditbeat-*'],
   interval: '5m',
   rule_id: ruleId,
   language: 'kuery',

From 0a8caf0c775096ab4d057698e710585aeb26ab1f Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 10:23:00 -0500
Subject: [PATCH 37/63] Fix non-ML Rule creation

I was accidentally dropping this parameter for our POST payload. Whoops.
---
 .../siem/public/pages/detection_engine/rules/create/helpers.ts  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
index 654fed38080c9..3f2d2f8065250 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
@@ -70,7 +70,7 @@ const formatDefineStepData = (defineStepData: DefineStepRule): DefineStepRuleJso
       ml_job_id: mlJobId,
     };
   } else {
-    const { index, queryBar, isNew, ruleType, ...rest } = ruleFields;
+    const { queryBar, isNew, ruleType, ...rest } = ruleFields;
     return {
       ...rest,
       type: ruleType,

From 9aca40cdcecbe5071bb5b8101fbc5b1b003c183e Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 10:36:25 -0500
Subject: [PATCH 38/63] More informative logging during ML signal generation

The messaging diverged from the normal path here because we don't have
index patterns to display. However, we have the rest of the rule
context, and should report it appropriately.
---
 .../lib/detection_engine/signals/signal_rule_alert_type.ts      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index 14d3646a9c6ae..1c8df3937a853 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -113,7 +113,7 @@ export const signalRulesAlertType = ({
           const anomalyCount = anomalyResults.hits.hits.length;
           if (anomalyCount) {
             logger.info(
-              `Found ${anomalyCount} anomalies in interval [${from}, ${to}]; generating signals`
+              `Found ${anomalyCount} signals from ML anomalies for signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", pushing signals to index "${outputIndex}"`
             );
           }
 

From 92f9c57f861e0cdbb4ab20d0f709dbbde02248c3 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 12:29:58 -0500
Subject: [PATCH 39/63] Prefer keyof for string union types

---
 .../public/containers/detection_engine/rules/types.ts  | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
index b52c074df7ab3..c87f40f0542e2 100644
--- a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
@@ -6,11 +6,11 @@
 
 import * as t from 'io-ts';
 
-export const RuleTypeSchema = t.union([
-  t.literal('query'),
-  t.literal('saved_query'),
-  t.literal('machine_learning'),
-]);
+export const RuleTypeSchema = t.keyof({
+  query: null,
+  saved_query: null,
+  machine_learning: null,
+});
 export type RuleType = t.TypeOf<typeof RuleTypeSchema>;
 
 export const NewRuleSchema = t.intersection([

From addc6acc555e419b752b787c0174a754ed8d1196 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 12:36:19 -0500
Subject: [PATCH 40/63] Tidy up our new form components

* Type them as React.FCs
* Remove unnecessary use of styled-components
---
 .../anomaly_threshold_slider/index.tsx           |  9 +++------
 .../rules/components/ml_job_select/index.tsx     | 16 +++++++---------
 .../rules/components/select_rule_type/index.tsx  |  9 +++------
 3 files changed, 13 insertions(+), 21 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx
index be4fd27840830..18970ff935b8d 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx
@@ -5,7 +5,6 @@
  */
 
 import React, { useCallback } from 'react';
-import styled from 'styled-components';
 import { EuiFlexGrid, EuiFlexItem, EuiRange, EuiFormRow } from '@elastic/eui';
 
 import { FieldHook } from '../../../../../shared_imports';
@@ -16,9 +15,7 @@ interface AnomalyThresholdSliderProps {
 type Event = React.ChangeEvent<HTMLInputElement>;
 type EventArg = Event | React.MouseEvent<HTMLButtonElement>;
 
-const Wrapper = styled(EuiFormRow)``;
-
-export const AnomalyThresholdSlider = ({ field }: AnomalyThresholdSliderProps) => {
+export const AnomalyThresholdSlider: React.FC<AnomalyThresholdSliderProps> = ({ field }) => {
   const threshold = field.value as number;
   const onThresholdChange = useCallback(
     (event: EventArg) => {
@@ -29,7 +26,7 @@ export const AnomalyThresholdSlider = ({ field }: AnomalyThresholdSliderProps) =
   );
 
   return (
-    <Wrapper label={field.label} fullWidth>
+    <EuiFormRow label={field.label} fullWidth>
       <EuiFlexGrid columns={2}>
         <EuiFlexItem>
           <EuiRange
@@ -43,6 +40,6 @@ export const AnomalyThresholdSlider = ({ field }: AnomalyThresholdSliderProps) =
           />
         </EuiFlexItem>
       </EuiFlexGrid>
-    </Wrapper>
+    </EuiFormRow>
   );
 };
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
index ff9c64ac0915a..9fc465443bcc8 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
@@ -5,17 +5,11 @@
  */
 
 import React, { useCallback } from 'react';
-import styled from 'styled-components';
 import { EuiFlexGroup, EuiFlexItem, EuiFormRow, EuiSuperSelect, EuiText } from '@elastic/eui';
 
 import { FieldHook, getFieldValidityAndErrorMessage } from '../../../../../shared_imports';
 import { useSiemJobs } from '../../../../../components/ml_popover/hooks/use_siem_jobs';
 
-interface MlJobSelectProps {
-  field: FieldHook;
-}
-
-const Wrapper = styled(EuiFormRow)``;
 const JobDisplay = ({ title, description }: { title: string; description: string }) => (
   <>
     <strong>{title}</strong>
@@ -25,7 +19,11 @@ const JobDisplay = ({ title, description }: { title: string; description: string
   </>
 );
 
-export const MlJobSelect = ({ field }: MlJobSelectProps) => {
+interface MlJobSelectProps {
+  field: FieldHook;
+}
+
+export const MlJobSelect: React.FC<MlJobSelectProps> = ({ field }) => {
   const jobId = field.value as string;
   const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(field);
   const [isLoading, siemJobs] = useSiemJobs(false);
@@ -43,7 +41,7 @@ export const MlJobSelect = ({ field }: MlJobSelectProps) => {
   }));
 
   return (
-    <Wrapper fullWidth label={field.label} isInvalid={isInvalid} error={errorMessage}>
+    <EuiFormRow fullWidth label={field.label} isInvalid={isInvalid} error={errorMessage}>
       <EuiFlexGroup>
         <EuiFlexItem>
           <EuiSuperSelect
@@ -55,6 +53,6 @@ export const MlJobSelect = ({ field }: MlJobSelectProps) => {
           />
         </EuiFlexItem>
       </EuiFlexGroup>
-    </Wrapper>
+    </EuiFormRow>
   );
 };
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
index 77c2a053ea209..b3b35699914f6 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx
@@ -5,7 +5,6 @@
  */
 
 import React, { useCallback } from 'react';
-import styled from 'styled-components';
 import { EuiCard, EuiFlexGrid, EuiFlexItem, EuiIcon, EuiFormRow } from '@elastic/eui';
 
 import { FieldHook } from '../../../../../shared_imports';
@@ -17,9 +16,7 @@ interface SelectRuleTypeProps {
   field: FieldHook;
 }
 
-const Wrapper = styled(EuiFormRow)``;
-
-export const SelectRuleType = ({ field }: SelectRuleTypeProps) => {
+export const SelectRuleType: React.FC<SelectRuleTypeProps> = ({ field }) => {
   const ruleType = field.value as RuleType;
   const setType = useCallback(
     (type: RuleType) => {
@@ -32,7 +29,7 @@ export const SelectRuleType = ({ field }: SelectRuleTypeProps) => {
   const license = true; // TODO
 
   return (
-    <Wrapper label={field.label} fullWidth>
+    <EuiFormRow label={field.label} fullWidth>
       <EuiFlexGrid columns={4}>
         <EuiFlexItem>
           <EuiCard
@@ -58,6 +55,6 @@ export const SelectRuleType = ({ field }: SelectRuleTypeProps) => {
           />
         </EuiFlexItem>
       </EuiFlexGrid>
-    </Wrapper>
+    </EuiFormRow>
   );
 };

From 0ca38583a4d3e1c60c1b7806cd9994a8d95fd5e1 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 12:59:18 -0500
Subject: [PATCH 41/63] Prefer destructuring to lodash's omit

---
 .../pages/detection_engine/rules/create/helpers.ts   | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
index 942791c34394c..fd32e9c84fecf 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { has, omit, isEmpty } from 'lodash/fp';
+import { has, isEmpty } from 'lodash/fp';
 import moment from 'moment';
 
 import { NewRule, RuleType } from '../../../../containers/detection_engine/rules';
@@ -53,9 +53,13 @@ const isMlFields = <T>(fields: QueryRuleFields<T> | MlRuleFields<T>): fields is
   has('anomalyThreshold', fields);
 
 export const filterRuleFieldsForType = <T extends RuleFields>(fields: T, type: RuleType) => {
-  return isMlRule(type)
-    ? omit(['index', 'queryBar'], fields)
-    : omit(['anomalyThreshold', 'mlJobId'], fields);
+  if (isMlRule(type)) {
+    const { index, queryBar, ...mlRuleFields } = fields;
+    return mlRuleFields;
+  } else {
+    const { anomalyThreshold, mlJobId, ...queryRuleFields } = fields;
+    return queryRuleFields;
+  }
 };
 
 export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStepRuleJson => {

From 190600ca2cf791815a6c1345a91dff0af66d607e Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 14:09:31 -0500
Subject: [PATCH 42/63] Fix mock params for helper functions

These were updated to take simpler parameters.
---
 .../description_step/helpers.test.tsx         | 25 ++++---------------
 1 file changed, 5 insertions(+), 20 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.test.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.test.tsx
index 56c9d6da15607..3f2523609cf04 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.test.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.test.tsx
@@ -38,10 +38,7 @@ setupMock.uiSettings.get.mockImplementation(uiSettingsMock(true));
 const mockFilterManager = new FilterManager(setupMock.uiSettings);
 
 const mockQueryBar = {
-  query: {
-    query: 'test query',
-    language: 'kuery',
-  },
+  query: 'test query',
   filters: [
     {
       $state: {
@@ -93,10 +90,7 @@ describe('helpers', () => {
   describe('buildQueryBarDescription', () => {
     test('returns empty array if no filters, query or savedId exist', () => {
       const emptyMockQueryBar = {
-        query: {
-          query: '',
-          language: 'kuery',
-        },
+        query: '',
         filters: [],
         saved_id: '',
       };
@@ -113,10 +107,7 @@ describe('helpers', () => {
     test('returns expected array of ListItems when filters exists, but no indexPatterns passed in', () => {
       const mockQueryBarWithFilters = {
         ...mockQueryBar,
-        query: {
-          query: '',
-          language: 'kuery',
-        },
+        query: '',
         saved_id: '',
       };
       const result: ListItems[] = buildQueryBarDescription({
@@ -135,10 +126,7 @@ describe('helpers', () => {
     test('returns expected array of ListItems when filters AND indexPatterns exist', () => {
       const mockQueryBarWithFilters = {
         ...mockQueryBar,
-        query: {
-          query: '',
-          language: 'kuery',
-        },
+        query: '',
         saved_id: '',
       };
       const result: ListItems[] = buildQueryBarDescription({
@@ -177,10 +165,7 @@ describe('helpers', () => {
     test('returns expected array of ListItems when "savedId" exists', () => {
       const mockQueryBarWithSavedId = {
         ...mockQueryBar,
-        query: {
-          query: '',
-          language: 'kuery',
-        },
+        query: '',
         filters: [],
       };
       const result: ListItems[] = buildQueryBarDescription({

From a683ef58c9b6d5a0c4bd06c64398060c7f9885df Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 14:13:54 -0500
Subject: [PATCH 43/63] Remove any type

This could have been a boolean all along, whoops
---
 .../detection_engine/signals/signal_rule_alert_type.ts | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index 1c8df3937a853..99424c6606539 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -92,9 +92,7 @@ export const signalRulesAlertType = ({
       });
 
       const searchAfterSize = Math.min(params.maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE);
-
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      let bulkIndexResult: any;
+      let creationSucceeded = false;
 
       try {
         if (type === 'machine_learning') {
@@ -117,7 +115,7 @@ export const signalRulesAlertType = ({
             );
           }
 
-          bulkIndexResult = await bulkCreateMlSignals({
+          creationSucceeded = await bulkCreateMlSignals({
             someResult: anomalyResults,
             ruleParams: params,
             services,
@@ -175,7 +173,7 @@ export const signalRulesAlertType = ({
             );
           }
 
-          bulkIndexResult = await searchAfterAndBulkCreate({
+          creationSucceeded = await searchAfterAndBulkCreate({
             someResult: noReIndexResult,
             ruleParams: params,
             services,
@@ -195,7 +193,7 @@ export const signalRulesAlertType = ({
           });
         }
 
-        if (bulkIndexResult) {
+        if (creationSucceeded) {
           logger.debug(
             `Finished signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
           );

From 94c177458f344f91070e1e08242381b18755e937 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 15:09:23 -0500
Subject: [PATCH 44/63] Fix mock types

---
 .../public/pages/detection_engine/rules/all/__mocks__/mock.ts  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts
index 5627d33818500..419f0068b5b6f 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts
@@ -181,6 +181,9 @@ export const mockAboutStepRule = (isNew = false): AboutStepRule => ({
 
 export const mockDefineStepRule = (isNew = false): DefineStepRule => ({
   isNew,
+  ruleType: 'query',
+  anomalyThreshold: 50,
+  mlJobId: '',
   index: ['filebeat-'],
   queryBar: mockQueryBar,
 });

From 327a877dd1c27ef38910bb7c1582d3dcd7468c98 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 16:49:42 -0500
Subject: [PATCH 45/63] Update outdated tests

These were added on master, but behavior has been changed on my branch.
---
 .../components/description_step/helpers.test.tsx    |  2 +-
 .../detection_engine/rules/create/helpers.test.ts   |  9 +++------
 .../pages/detection_engine/rules/helpers.test.tsx   | 13 +++++++++++--
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.test.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.test.tsx
index 3f2523609cf04..7a3f0105d3d15 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.test.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.test.tsx
@@ -159,7 +159,7 @@ describe('helpers', () => {
         savedId: mockQueryBarWithQuery.saved_id,
       });
       expect(result[0].title).toEqual(<>{i18n.QUERY_LABEL} </>);
-      expect(result[0].description).toEqual(<>{mockQueryBarWithQuery.query.query} </>);
+      expect(result[0].description).toEqual(<>{mockQueryBarWithQuery.query} </>);
     });
 
     test('returns expected array of ListItems when "savedId" exists', () => {
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.test.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.test.ts
index dbc5dd9bbe29a..ea6b02924cb3e 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.test.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.test.ts
@@ -87,6 +87,7 @@ describe('helpers', () => {
         query: 'test query',
         saved_id: 'test123',
         index: ['filebeat-'],
+        type: 'saved_query',
       };
 
       expect(result).toEqual(expected);
@@ -106,6 +107,8 @@ describe('helpers', () => {
         filters: mockQueryBar.filters,
         query: 'test query',
         index: ['filebeat-'],
+        saved_id: '',
+        type: 'query',
       };
 
       expect(result).toEqual(expected);
@@ -574,12 +577,6 @@ describe('helpers', () => {
       expect(result.type).toEqual('query');
     });
 
-    test('returns NewRule with id set to ruleId if ruleId exists', () => {
-      const result: NewRule = formatRule(mockDefine, mockAbout, mockSchedule, 'query-with-rule-id');
-
-      expect(result.id).toEqual('query-with-rule-id');
-    });
-
     test('returns NewRule without id if ruleId does not exist', () => {
       const result: NewRule = formatRule(mockDefine, mockAbout, mockSchedule);
 
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.test.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.test.tsx
index 0c29bc31cdebc..3f4a88845157f 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.test.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.test.tsx
@@ -32,7 +32,10 @@ describe('rule helpers', () => {
       });
       const defineRuleStepData = {
         isNew: false,
+        ruleType: 'saved_query',
+        anomalyThreshold: 50,
         index: ['auditbeat-*'],
+        mlJobId: '',
         queryBar: {
           query: {
             query: 'user.name: root or user.name: admin',
@@ -180,6 +183,9 @@ describe('rule helpers', () => {
       const result: DefineStepRule = getDefineStepsData(mockRule('test-id'));
       const expected = {
         isNew: false,
+        ruleType: 'saved_query',
+        anomalyThreshold: 50,
+        mlJobId: '',
         index: ['auditbeat-*'],
         queryBar: {
           query: {
@@ -194,7 +200,7 @@ describe('rule helpers', () => {
       expect(result).toEqual(expected);
     });
 
-    test('returns with saved_id of null if value does not exist on rule', () => {
+    test('returns with saved_id of undefined if value does not exist on rule', () => {
       const mockedRule = {
         ...mockRule('test-id'),
       };
@@ -202,6 +208,9 @@ describe('rule helpers', () => {
       const result: DefineStepRule = getDefineStepsData(mockedRule);
       const expected = {
         isNew: false,
+        ruleType: 'saved_query',
+        anomalyThreshold: 50,
+        mlJobId: '',
         index: ['auditbeat-*'],
         queryBar: {
           query: {
@@ -209,7 +218,7 @@ describe('rule helpers', () => {
             language: 'kuery',
           },
           filters: [],
-          saved_id: null,
+          saved_id: undefined,
         },
       };
 

From e85feb582f57679f164b138c6c1ed84a14aa99c1 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 17:42:33 -0500
Subject: [PATCH 46/63] Add some tests around our helper function

I need to refactor it, so this is as good a time as any to pin down the
behavior.
---
 .../signals/bulk_create_ml_signals.test.ts    | 82 +++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts
new file mode 100644
index 0000000000000..21b02e7900f37
--- /dev/null
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts
@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { convertAnomalyFieldsToECS } from './bulk_create_ml_signals';
+
+const buildMockAnomaly = () => ({
+  job_id: 'rare_process_by_host_linux_ecs',
+  result_type: 'record',
+  probability: 0.03406145177566593,
+  multi_bucket_impact: -0.0,
+  record_score: 10.86784984522809,
+  initial_record_score: 10.86784984522809,
+  bucket_span: 900,
+  detector_index: 0,
+  is_interim: false,
+  timestamp: 1584482400000,
+  by_field_name: 'process.name',
+  by_field_value: 'gzip',
+  partition_field_name: 'host.name',
+  partition_field_value: 'rock01',
+  function: 'rare',
+  function_description: 'rare',
+  typical: [0.03406145177566593],
+  actual: [1.0],
+  influencers: [
+    {
+      influencer_field_name: 'user.name',
+      influencer_field_values: ['root'],
+    },
+    {
+      influencer_field_name: 'process.pid',
+      influencer_field_values: ['123'],
+    },
+    {
+      influencer_field_name: 'host.name',
+      influencer_field_values: ['rock01'],
+    },
+  ],
+  'process.name': 'gzip',
+  'process.pid': ['123'],
+  'user.name': ['root'],
+  'host.name': ['rock01'],
+});
+
+describe('convertAnomalyFieldsToECS', () => {
+  it('deletes dotted influencer fields', () => {
+    const anomaly = buildMockAnomaly();
+    const result = convertAnomalyFieldsToECS(anomaly);
+
+    const ecsKeys = Object.keys(result);
+    expect(ecsKeys).not.toContain('user.name');
+    expect(ecsKeys).not.toContain('process.pid');
+    expect(ecsKeys).not.toContain('host.name');
+  });
+
+  it('deletes dotted entity field', () => {
+    const anomaly = buildMockAnomaly();
+    const result = convertAnomalyFieldsToECS(anomaly);
+
+    const ecsKeys = Object.keys(result);
+    expect(ecsKeys).not.toContain('process.name');
+  });
+
+  it('creates nested influencer fields', () => {
+    const anomaly = buildMockAnomaly();
+    const result = convertAnomalyFieldsToECS(anomaly);
+
+    expect(result.process.pid).toEqual(['123']);
+    expect(result.user.name).toEqual(['root']);
+    expect(result.host.name).toEqual(['rock01']);
+  });
+
+  it('creates nested entity field', () => {
+    const anomaly = buildMockAnomaly();
+    const result = convertAnomalyFieldsToECS(anomaly);
+
+    expect(result.process.name).toEqual('gzip');
+  });
+});

From e80dd2cead13887573f62d594fbdb1760eb8a239 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 18:05:20 -0500
Subject: [PATCH 47/63] Remove uses of any in favor of actual types

Mainly leverages ML typings instead of our placeholder types. This
required handling a null case in our formatting of anomalies.
---
 .../signals/bulk_create_ml_signals.test.ts    |  4 +-
 .../signals/bulk_create_ml_signals.ts         | 43 ++++++-------------
 .../signals/single_bulk_create.ts             |  4 +-
 .../siem/server/lib/machine_learning/index.ts | 12 +++---
 4 files changed, 23 insertions(+), 40 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts
index 21b02e7900f37..20067dbeaaaef 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts
@@ -39,7 +39,7 @@ const buildMockAnomaly = () => ({
       influencer_field_values: ['rock01'],
     },
   ],
-  'process.name': 'gzip',
+  'process.name': ['gzip'],
   'process.pid': ['123'],
   'user.name': ['root'],
   'host.name': ['rock01'],
@@ -77,6 +77,6 @@ describe('convertAnomalyFieldsToECS', () => {
     const anomaly = buildMockAnomaly();
     const result = convertAnomalyFieldsToECS(anomaly);
 
-    expect(result.process.name).toEqual('gzip');
+    expect(result.process.name).toEqual(['gzip']);
   });
 });
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts
index 060fd4da16ac3..eb4e5e50d6651 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts
@@ -4,25 +4,13 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { SearchResponse } from 'elasticsearch';
 import { flow, set, omit } from 'lodash/fp';
 
 import { Logger } from '../../../../../../../../src/core/server';
 import { AlertServices } from '../../../../../../../plugins/alerting/server';
 import { RuleTypeParams } from '../types';
 import { singleBulkCreate } from './single_bulk_create';
-import { Influencer } from '../../../../public/components/ml/types';
-
-interface Anomaly {
-  job_id: string;
-  record_score: number;
-  timestamp: number;
-  by_field_name: string;
-  by_field_value: string;
-  influencers?: Influencer[];
-}
-
-type AnomalyResults = SearchResponse<Anomaly>;
+import { AnomalyResults, Anomaly } from '../../machine_learning';
 
 interface BulkCreateMlSignalsParams {
   someResult: AnomalyResults;
@@ -41,24 +29,21 @@ interface BulkCreateMlSignalsParams {
   tags: string[];
 }
 
-const convertAnomalyFieldsToECS = (anomaly: Anomaly): Anomaly => {
-  const {
-    by_field_name: entityName,
-    by_field_value: entityValue,
-    influencers: maybeInfluencers,
-  } = anomaly;
-  const influencers = maybeInfluencers ?? [];
+export const convertAnomalyFieldsToECS = (anomaly: Anomaly): Anomaly => {
+  const { by_field_name: entityName, by_field_value: entityValue, influencers } = anomaly;
+  let errantFields = (influencers ?? []).map(influencer => ({
+    name: influencer.influencer_field_name,
+    value: influencer.influencer_field_values,
+  }));
+
+  if (entityName && entityValue) {
+    errantFields = [...errantFields, { name: entityName, value: [entityValue] }];
+  }
 
-  const setEntityField = set(entityName, entityValue);
-  const setInfluencerFields = influencers.map(influencer =>
-    set(influencer.influencer_field_name, influencer.influencer_field_values)
-  );
-  const omitDottedFields = omit([
-    entityName,
-    ...influencers.map(influencer => influencer.influencer_field_name),
-  ]);
+  const omitDottedFields = omit(errantFields.map(field => field.name));
+  const setNestedFields = errantFields.map(field => set(field.name, field.value));
 
-  return flow(omitDottedFields, setEntityField, setInfluencerFields)(anomaly);
+  return flow(omitDottedFields, setNestedFields)(anomaly);
 };
 
 export const bulkCreateMlSignals = async (params: BulkCreateMlSignalsParams) => {
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts
index 642329f480dfa..7d6d6d99fa422 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts
@@ -12,11 +12,9 @@ import { RuleTypeParams } from '../types';
 import { generateId } from './utils';
 import { buildBulkBody } from './build_bulk_body';
 import { Logger } from '../../../../../../../../src/core/server';
-import { SearchResponse } from '../../types';
 
 interface SingleBulkCreateParams {
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  someResult: SearchResponse<any>;
+  someResult: SignalSearchResponse;
   ruleParams: RuleTypeParams;
   services: AlertServices;
   logger: Logger;
diff --git a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
index 773de9f97bf8f..ab06ae25abb4a 100644
--- a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
@@ -7,6 +7,10 @@
 import { SearchResponse } from 'elasticsearch';
 
 import { AlertServices } from '../../../../../../plugins/alerting/server';
+import { AnomalyRecordDoc as Anomaly } from '../../../../../../plugins/ml/common/types/anomalies';
+
+export { Anomaly };
+export type AnomalyResults = SearchResponse<Anomaly>;
 
 export interface AnomaliesSearchParams {
   jobIds: string[];
@@ -16,13 +20,10 @@ export interface AnomaliesSearchParams {
   maxRecords?: number;
 }
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-type Anomaly = any;
-
 export const getAnomalies = async (
   params: AnomaliesSearchParams,
   callCluster: AlertServices['callCluster']
-): Promise<SearchResponse<Anomaly>> => {
+): Promise<AnomalyResults> => {
   const boolCriteria = buildCriteria(params);
 
   return callCluster('search', {
@@ -51,8 +52,7 @@ export const getAnomalies = async (
   });
 };
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-const buildCriteria = (params: AnomaliesSearchParams): any => {
+const buildCriteria = (params: AnomaliesSearchParams): object[] => {
   const { earliestMs, jobIds, latestMs, threshold } = params;
   const jobIdsFilterable =
     jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*');

From 5e85c7e47f20187dc74be56c6806ecceb5ac607b Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 18:49:47 -0500
Subject: [PATCH 48/63] Annotate our anomalies with @timestamp field

We were notably lacking this ECS field in our post-conversion anomalies,
and typescript was rightly complaining about it.
---
 .../signals/bulk_create_ml_signals.test.ts    | 20 +++++++---
 .../signals/bulk_create_ml_signals.ts         | 39 +++++++++++++++----
 2 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts
index 20067dbeaaaef..d9fb9d4bbabde 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { convertAnomalyFieldsToECS } from './bulk_create_ml_signals';
+import { transformAnomalyFieldsToEcs } from './bulk_create_ml_signals';
 
 const buildMockAnomaly = () => ({
   job_id: 'rare_process_by_host_linux_ecs',
@@ -45,10 +45,18 @@ const buildMockAnomaly = () => ({
   'host.name': ['rock01'],
 });
 
-describe('convertAnomalyFieldsToECS', () => {
+describe('transformAnomalyFieldsToEcs', () => {
+  it('adds a @timestamp field based on timestamp', () => {
+    const anomaly = buildMockAnomaly();
+    const result = transformAnomalyFieldsToEcs(anomaly);
+    const expectedTime = '2020-03-17T22:00:00.000Z';
+
+    expect(result['@timestamp']).toEqual(expectedTime);
+  });
+
   it('deletes dotted influencer fields', () => {
     const anomaly = buildMockAnomaly();
-    const result = convertAnomalyFieldsToECS(anomaly);
+    const result = transformAnomalyFieldsToEcs(anomaly);
 
     const ecsKeys = Object.keys(result);
     expect(ecsKeys).not.toContain('user.name');
@@ -58,7 +66,7 @@ describe('convertAnomalyFieldsToECS', () => {
 
   it('deletes dotted entity field', () => {
     const anomaly = buildMockAnomaly();
-    const result = convertAnomalyFieldsToECS(anomaly);
+    const result = transformAnomalyFieldsToEcs(anomaly);
 
     const ecsKeys = Object.keys(result);
     expect(ecsKeys).not.toContain('process.name');
@@ -66,7 +74,7 @@ describe('convertAnomalyFieldsToECS', () => {
 
   it('creates nested influencer fields', () => {
     const anomaly = buildMockAnomaly();
-    const result = convertAnomalyFieldsToECS(anomaly);
+    const result = transformAnomalyFieldsToEcs(anomaly);
 
     expect(result.process.pid).toEqual(['123']);
     expect(result.user.name).toEqual(['root']);
@@ -75,7 +83,7 @@ describe('convertAnomalyFieldsToECS', () => {
 
   it('creates nested entity field', () => {
     const anomaly = buildMockAnomaly();
-    const result = convertAnomalyFieldsToECS(anomaly);
+    const result = transformAnomalyFieldsToEcs(anomaly);
 
     expect(result.process.name).toEqual(['gzip']);
   });
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts
index eb4e5e50d6651..1ab34f26d4b70 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts
@@ -5,6 +5,7 @@
  */
 
 import { flow, set, omit } from 'lodash/fp';
+import { SearchResponse } from 'elasticsearch';
 
 import { Logger } from '../../../../../../../../src/core/server';
 import { AlertServices } from '../../../../../../../plugins/alerting/server';
@@ -29,8 +30,17 @@ interface BulkCreateMlSignalsParams {
   tags: string[];
 }
 
-export const convertAnomalyFieldsToECS = (anomaly: Anomaly): Anomaly => {
-  const { by_field_name: entityName, by_field_value: entityValue, influencers } = anomaly;
+interface EcsAnomaly extends Anomaly {
+  '@timestamp': string;
+}
+
+export const transformAnomalyFieldsToEcs = (anomaly: Anomaly): EcsAnomaly => {
+  const {
+    by_field_name: entityName,
+    by_field_value: entityValue,
+    influencers,
+    timestamp,
+  } = anomaly;
   let errantFields = (influencers ?? []).map(influencer => ({
     name: influencer.influencer_field_name,
     value: influencer.influencer_field_values,
@@ -42,16 +52,29 @@ export const convertAnomalyFieldsToECS = (anomaly: Anomaly): Anomaly => {
 
   const omitDottedFields = omit(errantFields.map(field => field.name));
   const setNestedFields = errantFields.map(field => set(field.name, field.value));
+  const setTimestamp = set('@timestamp', new Date(timestamp).toISOString());
 
-  return flow(omitDottedFields, setNestedFields)(anomaly);
+  return flow(omitDottedFields, setNestedFields, setTimestamp)(anomaly);
 };
 
-export const bulkCreateMlSignals = async (params: BulkCreateMlSignalsParams) => {
-  const anomalies = params.someResult;
-  anomalies.hits.hits = anomalies.hits.hits.map(({ _source, ...rest }) => ({
+const transformAnomalyResultsToEcs = (results: AnomalyResults): SearchResponse<EcsAnomaly> => {
+  const transformedHits = results.hits.hits.map(({ _source, ...rest }) => ({
     ...rest,
-    _source: convertAnomalyFieldsToECS(_source),
+    _source: transformAnomalyFieldsToEcs(_source),
   }));
 
-  return singleBulkCreate({ ...params, someResult: anomalies });
+  return {
+    ...results,
+    hits: {
+      ...results.hits,
+      hits: transformedHits,
+    },
+  };
+};
+
+export const bulkCreateMlSignals = async (params: BulkCreateMlSignalsParams) => {
+  const anomalyResults = params.someResult;
+  const ecsResults = transformAnomalyResultsToEcs(anomalyResults);
+
+  return singleBulkCreate({ ...params, someResult: ecsResults });
 };

From eed90df182c4180b09048d5aff351146e156b266 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 23:00:42 -0500
Subject: [PATCH 49/63] ml_job_id -> machine_learning_job_id

---
 .../siem/public/containers/detection_engine/rules/types.ts    | 4 ++--
 .../public/pages/detection_engine/rules/create/helpers.ts     | 2 +-
 .../siem/public/pages/detection_engine/rules/helpers.tsx      | 2 +-
 .../plugins/siem/public/pages/detection_engine/rules/types.ts | 2 +-
 .../detection_engine/routes/__mocks__/request_responses.ts    | 2 +-
 .../lib/detection_engine/routes/rules/create_rules_route.ts   | 2 +-
 .../siem/server/lib/detection_engine/routes/rules/utils.ts    | 2 +-
 .../detection_engine/routes/schemas/create_rules_schema.ts    | 4 ++--
 .../routes/schemas/response/__mocks__/utils.ts                | 2 +-
 .../routes/schemas/response/check_type_dependents.ts          | 4 +++-
 .../detection_engine/routes/schemas/response/rules_schema.ts  | 4 ++--
 .../lib/detection_engine/routes/schemas/response/schemas.ts   | 2 +-
 .../server/lib/detection_engine/routes/schemas/schemas.ts     | 2 +-
 .../siem/server/lib/detection_engine/signals/build_rule.ts    | 2 +-
 14 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
index c87f40f0542e2..5466ba2203714 100644
--- a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts
@@ -32,7 +32,7 @@ export const NewRuleSchema = t.intersection([
     id: t.string,
     index: t.array(t.string),
     language: t.string,
-    ml_job_id: t.string,
+    machine_learning_job_id: t.string,
     max_signals: t.number,
     query: t.string,
     references: t.array(t.string),
@@ -90,7 +90,7 @@ export const RuleSchema = t.intersection([
     last_failure_at: t.string,
     last_failure_message: t.string,
     meta: MetaRule,
-    ml_job_id: t.string,
+    machine_learning_job_id: t.string,
     output_index: t.string,
     query: t.string,
     saved_id: t.string,
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
index fd32e9c84fecf..14ec7d68cc296 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
@@ -71,7 +71,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep
       ...rest,
       type: ruleType,
       anomaly_threshold: anomalyThreshold,
-      ml_job_id: mlJobId,
+      machine_learning_job_id: mlJobId,
     };
   } else {
     const { queryBar, isNew, ruleType, ...rest } = ruleFields;
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx
index 3dadc9637ebaa..b4c58f3d02599 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx
@@ -47,7 +47,7 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => {
     isNew: false,
     ruleType: rule.type,
     anomalyThreshold: rule.anomaly_threshold ?? 50,
-    mlJobId: rule.ml_job_id ?? '',
+    mlJobId: rule.machine_learning_job_id ?? '',
     index: rule.index ?? [],
     queryBar: {
       query: { query: rule.query ?? '', language: rule.language ?? '' },
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
index c8939cf388d45..fb6f898ff5949 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
@@ -86,7 +86,7 @@ export interface DefineStepRuleJson {
   anomaly_threshold?: number;
   index?: string[];
   filters?: Filter[];
-  ml_job_id?: string;
+  machine_learning_job_id?: string;
   saved_id?: string;
   query?: string;
   language?: string;
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
index 3091d3e31cc46..07e15122a4bf5 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
@@ -301,7 +301,7 @@ export const createMlRuleRequest = () =>
       ...typicalPayload(),
       type: 'machine_learning',
       anomaly_threshold: 50,
-      ml_job_id: 'some-uuid',
+      machine_learning_job_id: 'some-uuid',
     },
   });
 
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts
index 04a7336e451e1..b2757d0ca6541 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts
@@ -43,7 +43,7 @@ export const createRulesRoute = (router: IRouter): void => {
         timeline_id: timelineId,
         timeline_title: timelineTitle,
         meta,
-        ml_job_id: mlJobId,
+        machine_learning_job_id: mlJobId,
         filters,
         rule_id: ruleId,
         index,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts
index a9794e1e82c5b..177dae461e715 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts
@@ -118,7 +118,7 @@ export const transformAlertToRule = (
     language: alert.params.language,
     output_index: alert.params.outputIndex,
     max_signals: alert.params.maxSignals,
-    ml_job_id: alert.params.mlJobId,
+    machine_learning_job_id: alert.params.mlJobId,
     risk_score: alert.params.riskScore,
     name: alert.name,
     query: alert.params.query,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts
index d5132507e7476..5aca09f587a0d 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts
@@ -35,7 +35,7 @@ import {
   references,
   note,
   version,
-  ml_job_id,
+  machine_learning_job_id,
 } from './schemas';
 /* eslint-enable @typescript-eslint/camelcase */
 
@@ -66,7 +66,7 @@ export const createRulesSchema = Joi.object({
   timeline_id,
   timeline_title,
   meta,
-  ml_job_id: ml_job_id.when('type', {
+  machine_learning_job_id: machine_learning_job_id.when('type', {
     is: 'machine_learning',
     then: Joi.required(),
     otherwise: Joi.forbidden(),
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/__mocks__/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/__mocks__/utils.ts
index cc6c94373934a..dd88bd80d5787 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/__mocks__/utils.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/__mocks__/utils.ts
@@ -75,7 +75,7 @@ export const getMlRuleResponsePayload = (anchorDate: string = ANCHOR_DATE): Rule
     ...rest,
     type: 'machine_learning',
     anomaly_threshold: 59,
-    ml_job_id: 'some_ml_job_id',
+    machine_learning_job_id: 'some_machine_learning_job_id',
   };
 };
 
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts
index 0fac827c6e50e..b5a01e3e5c6df 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts
@@ -50,7 +50,9 @@ export const addMlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[]
   if (typeAndTimelineOnly.type === 'machine_learning') {
     return [
       t.exact(t.type({ anomaly_threshold: dependentRulesSchema.props.anomaly_threshold })),
-      t.exact(t.type({ ml_job_id: dependentRulesSchema.props.ml_job_id })),
+      t.exact(
+        t.type({ machine_learning_job_id: dependentRulesSchema.props.machine_learning_job_id })
+      ),
     ];
   } else {
     return [];
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/rules_schema.ts
index ca82d60ece755..28b588a86aeb0 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/rules_schema.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/rules_schema.ts
@@ -25,7 +25,7 @@ import {
   name,
   output_index,
   max_signals,
-  ml_job_id,
+  machine_learning_job_id,
   query,
   references,
   severity,
@@ -104,7 +104,7 @@ export const dependentRulesSchema = t.partial({
 
   // ML fields
   anomaly_threshold,
-  ml_job_id,
+  machine_learning_job_id,
 });
 
 /**
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts
index 59da24471c63e..072e3f5beefe2 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts
@@ -46,7 +46,7 @@ export const saved_id = t.string;
 export const timeline_id = t.string;
 export const timeline_title = t.string;
 export const anomaly_threshold = PositiveInteger;
-export const ml_job_id = t.string;
+export const machine_learning_job_id = t.string;
 
 /**
  * Note that this is a plain unknown object because we allow the UI
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts
index 5049cbf73b116..9dffe8e508692 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts
@@ -50,7 +50,7 @@ export const severity = Joi.string().valid('low', 'medium', 'high', 'critical');
 export const status = Joi.string().valid('open', 'closed');
 export const to = Joi.string();
 export const type = Joi.string().valid('query', 'saved_query', 'machine_learning');
-export const ml_job_id = Joi.string();
+export const machine_learning_job_id = Joi.string();
 export const queryFilter = Joi.string();
 export const references = Joi.array()
   .items(Joi.string())
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts
index 9a94d647aee5d..e3cf6be3e8e00 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts
@@ -65,7 +65,7 @@ export const buildRule = ({
     version: ruleParams.version,
     created_at: createdAt,
     updated_at: updatedAt,
-    ml_job_id: ruleParams.mlJobId,
+    machine_learning_job_id: ruleParams.mlJobId,
     anomaly_threshold: ruleParams.anomalyThreshold,
   });
 };

From abfcfc1038348e90c1f6e886762b87f3cc563386 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 17 Mar 2020 23:14:49 -0500
Subject: [PATCH 50/63] PR Feedback

* Stricter threshold type
* More robust date parsing
* More informative log/error messages
* Remove redundant runtime checks
---
 .../lib/detection_engine/routes/schemas/schemas.ts   |  5 ++++-
 .../lib/detection_engine/signals/find_ml_signals.ts  |  4 ++--
 .../lib/detection_engine/signals/get_filter.ts       |  4 +++-
 .../signals/search_after_bulk_create.ts              |  4 +++-
 .../signals/signal_rule_alert_type.ts                | 12 ++++++++----
 .../siem/server/lib/machine_learning/index.ts        |  3 +--
 6 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts
index 9dffe8e508692..ad7050e8dd65c 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts
@@ -7,7 +7,10 @@
 import Joi from 'joi';
 
 /* eslint-disable @typescript-eslint/camelcase */
-export const anomaly_threshold = Joi.number();
+export const anomaly_threshold = Joi.number()
+  .integer()
+  .greater(-1)
+  .less(101);
 export const description = Joi.string();
 export const enabled = Joi.boolean();
 export const exclude_export_details = Joi.boolean();
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
index 5a809ba32b124..b7f752e6ba5e0 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts
@@ -20,8 +20,8 @@ export const findMlSignals = async (
   const params = {
     jobIds: [jobId],
     threshold: anomalyThreshold,
-    earliestMs: dateMath.parse(from)!.valueOf(),
-    latestMs: dateMath.parse(to)!.valueOf(),
+    earliestMs: dateMath.parse(from)?.valueOf() ?? 0,
+    latestMs: dateMath.parse(to)?.valueOf() ?? 0,
   };
   const relevantAnomalies = await getAnomalies(params, callCluster);
 
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_filter.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_filter.ts
index 53750eec49fb0..82a50222dc351 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_filter.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_filter.ts
@@ -108,7 +108,9 @@ export const getFilter = async ({
       }
     }
     case 'machine_learning': {
-      throw new Error('getFilter called with a ML Rule');
+      throw new BadRequestError(
+        'Unsupported Rule of type "machine_learning" supplied to getFilter'
+      );
     }
   }
   return assertUnreachable(type);
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
index b67e20248dc87..9c003d2d75268 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
@@ -54,7 +54,9 @@ export const searchAfterAndBulkCreate = async ({
   }
   const { index, from, to } = ruleParams;
   if (index == null) {
-    throw new Error('Attempted to bulk create signals, but rule had no indexPattern');
+    throw new Error(
+      `Attempted to bulk create signals, but rule id: ${id}, name: ${name}, signals index: ${signalsIndex} has no index pattern`
+    );
   }
 
   logger.debug('[+] starting bulk insertion');
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index 99424c6606539..dad5b48577dc9 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -97,7 +97,9 @@ export const signalRulesAlertType = ({
       try {
         if (type === 'machine_learning') {
           if (mlJobId == null || anomalyThreshold == null) {
-            throw new Error('Attempted to execute ML Rule, but missing jobId or anomalyThreshold');
+            throw new Error(
+              `Attempted to execute machine learning rule, but it is missing job id and/or anomaly threshold for rule id: "${ruleId}", name: "${name}", signals index: "${outputIndex}", job id: "${mlJobId}", anomaly threshold: "${anomalyThreshold}"`
+            );
           }
 
           const anomalyResults = await findMlSignals(
@@ -133,7 +135,9 @@ export const signalRulesAlertType = ({
           });
         } else {
           if (index == null) {
-            throw new Error('Attempted to execute Query Rule, but no index was specified');
+            throw new Error(
+              `Attempted to execute query rule, but it is missing index pattern for rule id: "${ruleId}", name: "${name}", signals index: "${outputIndex}", index pattern: "${index}"`
+            );
           }
 
           const inputIndex = await getInputIndex(services, version, index);
@@ -195,7 +199,7 @@ export const signalRulesAlertType = ({
 
         if (creationSucceeded) {
           logger.debug(
-            `Finished signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
+            `Finished signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", output_index: "${outputIndex}"`
           );
           await writeCurrentStatusSucceeded({
             services,
@@ -207,7 +211,7 @@ export const signalRulesAlertType = ({
             alertId,
             currentStatusSavedObject,
             logger,
-            message: `Bulk Indexing signals failed. Check logs for further details \nRule name: "${name}"\nid: "${alertId}"\nrule_id: "${ruleId}"\n`,
+            message: `Bulk Indexing signals failed. Check logs for further details Rule name: "${name}" id: "${alertId}" rule_id: "${ruleId}" output_index: "${outputIndex}"`,
             services,
             ruleStatusSavedObjects,
             ruleId: ruleId ?? '(unknown rule id)',
diff --git a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
index ab06ae25abb4a..aa83df15f68d4 100644
--- a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts
@@ -54,8 +54,7 @@ export const getAnomalies = async (
 
 const buildCriteria = (params: AnomaliesSearchParams): object[] => {
   const { earliestMs, jobIds, latestMs, threshold } = params;
-  const jobIdsFilterable =
-    jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*');
+  const jobIdsFilterable = jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*');
 
   const boolCriteria: object[] = [
     {

From f0a1f0f779ee5d2b25205fb63bbddf3136d3eaec Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 11:48:07 -0500
Subject: [PATCH 51/63] Cleaning up our new ML types

* Fix types on our Rest types
* Use less ambiguous machineLearningJobId over mlJobId
* Declare our ML params as required keys, and ensure we pass them around
everywhere we might need them (creating, importing, updating rules).
---
 .../rules/all/__mocks__/mock.ts               |  2 +-
 .../rules/components/ml_job_select/index.tsx  |  4 +--
 .../components/step_define_rule/index.tsx     |  4 +--
 .../components/step_define_rule/schema.tsx    | 13 ++++----
 .../detection_engine/rules/create/helpers.ts  | 10 +++----
 .../detection_engine/rules/helpers.test.tsx   |  6 ++--
 .../pages/detection_engine/rules/helpers.tsx  |  2 +-
 .../pages/detection_engine/rules/types.ts     |  2 +-
 .../routes/__mocks__/request_responses.ts     |  2 ++
 .../routes/rules/create_rules_bulk_route.ts   |  4 +++
 .../routes/rules/create_rules_route.ts        |  4 +--
 .../routes/rules/import_rules_route.ts        |  5 ++++
 .../routes/rules/update_rules_bulk_route.ts   |  4 +++
 .../routes/rules/update_rules_route.ts        |  4 +++
 .../routes/rules/utils.test.ts                | 30 ++++++++++++++-----
 .../detection_engine/routes/rules/utils.ts    |  2 +-
 .../detection_engine/rules/create_rules.ts    |  4 +--
 .../rules/install_prepacked_rules.ts          |  4 +++
 .../lib/detection_engine/rules/types.ts       |  8 +----
 .../signals/__mocks__/es_results.ts           |  2 ++
 .../detection_engine/signals/build_rule.ts    |  2 +-
 .../signals/signal_params_schema.ts           |  2 +-
 .../signals/signal_rule_alert_type.ts         |  8 ++---
 .../siem/server/lib/detection_engine/types.ts |  8 +++--
 24 files changed, 89 insertions(+), 47 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts
index 419f0068b5b6f..011a2614c1af9 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts
@@ -183,7 +183,7 @@ export const mockDefineStepRule = (isNew = false): DefineStepRule => ({
   isNew,
   ruleType: 'query',
   anomalyThreshold: 50,
-  mlJobId: '',
+  machineLearningJobId: '',
   index: ['filebeat-'],
   queryBar: mockQueryBar,
 });
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
index 9fc465443bcc8..627fa21cc2f61 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx
@@ -28,8 +28,8 @@ export const MlJobSelect: React.FC<MlJobSelectProps> = ({ field }) => {
   const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(field);
   const [isLoading, siemJobs] = useSiemJobs(false);
   const handleJobChange = useCallback(
-    (mlJobId: string) => {
-      field.setValue(mlJobId);
+    (machineLearningJobId: string) => {
+      field.setValue(machineLearningJobId);
     },
     [field]
   );
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
index c60a41fd680d8..55e4883eeb25d 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
@@ -52,7 +52,7 @@ const stepDefineDefaultValue: DefineStepRule = {
   anomalyThreshold: 50,
   index: [],
   isNew: true,
-  mlJobId: '',
+  machineLearningJobId: '',
   ruleType: 'query',
   queryBar: {
     query: { query: '', language: 'kuery' },
@@ -228,7 +228,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
           </EuiFormRow>
           <EuiFormRow fullWidth style={{ display: localIsMlRule ? 'flex' : 'none' }}>
             <>
-              <UseField path="mlJobId" component={MlJobSelect} />
+              <UseField path="machineLearningJobId" component={MlJobSelect} />
               <UseField path="anomalyThreshold" component={AnomalyThresholdSlider} />
             </>
           </EuiFormRow>
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
index b50607f7a4c0f..bcfcd4f4ee09d 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
@@ -127,9 +127,9 @@ export const schema: FormSchema = {
     ),
     validations: [],
   },
-  mlJobId: {
+  machineLearningJobId: {
     label: i18n.translate(
-      'xpack.siem.detectionEngine.createRule.stepDefineRule.fieldMlJobIdLabel',
+      'xpack.siem.detectionEngine.createRule.stepDefineRule.fieldMachineLearningJobIdLabel',
       {
         defaultMessage: 'Machine Learning job',
       }
@@ -147,9 +147,12 @@ export const schema: FormSchema = {
           }
 
           return fieldValidators.emptyField(
-            i18n.translate('xpack.siem.detectionEngine.createRule.stepDefineRule.mlJobIdRequired', {
-              defaultMessage: 'A Machine Learning job is required.',
-            })
+            i18n.translate(
+              'xpack.siem.detectionEngine.createRule.stepDefineRule.machineLearningJobIdRequired',
+              {
+                defaultMessage: 'A Machine Learning job is required.',
+              }
+            )
           )(...args);
         },
       },
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
index 14ec7d68cc296..6afd22b58bf84 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
@@ -41,12 +41,12 @@ export const getTimeTypeValue = (time: string): { unit: string; value: number }
 
 interface RuleFields {
   anomalyThreshold: unknown;
-  mlJobId: unknown;
+  machineLearningJobId: unknown;
   queryBar: unknown;
   index: unknown;
   ruleType: unknown;
 }
-type QueryRuleFields<T> = Omit<T, 'anomalyThreshold' | 'mlJobId'>;
+type QueryRuleFields<T> = Omit<T, 'anomalyThreshold' | 'machineLearningJobId'>;
 type MlRuleFields<T> = Omit<T, 'queryBar' | 'index'>;
 
 const isMlFields = <T>(fields: QueryRuleFields<T> | MlRuleFields<T>): fields is MlRuleFields<T> =>
@@ -57,7 +57,7 @@ export const filterRuleFieldsForType = <T extends RuleFields>(fields: T, type: R
     const { index, queryBar, ...mlRuleFields } = fields;
     return mlRuleFields;
   } else {
-    const { anomalyThreshold, mlJobId, ...queryRuleFields } = fields;
+    const { anomalyThreshold, machineLearningJobId, ...queryRuleFields } = fields;
     return queryRuleFields;
   }
 };
@@ -66,12 +66,12 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep
   const ruleFields = filterRuleFieldsForType(defineStepData, defineStepData.ruleType);
 
   if (isMlFields(ruleFields)) {
-    const { anomalyThreshold, mlJobId, isNew, ruleType, ...rest } = ruleFields;
+    const { anomalyThreshold, machineLearningJobId, isNew, ruleType, ...rest } = ruleFields;
     return {
       ...rest,
       type: ruleType,
       anomaly_threshold: anomalyThreshold,
-      machine_learning_job_id: mlJobId,
+      machine_learning_job_id: machineLearningJobId,
     };
   } else {
     const { queryBar, isNew, ruleType, ...rest } = ruleFields;
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.test.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.test.tsx
index 3f4a88845157f..ee43ae5f1d6e2 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.test.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.test.tsx
@@ -35,7 +35,7 @@ describe('rule helpers', () => {
         ruleType: 'saved_query',
         anomalyThreshold: 50,
         index: ['auditbeat-*'],
-        mlJobId: '',
+        machineLearningJobId: '',
         queryBar: {
           query: {
             query: 'user.name: root or user.name: admin',
@@ -185,7 +185,7 @@ describe('rule helpers', () => {
         isNew: false,
         ruleType: 'saved_query',
         anomalyThreshold: 50,
-        mlJobId: '',
+        machineLearningJobId: '',
         index: ['auditbeat-*'],
         queryBar: {
           query: {
@@ -210,7 +210,7 @@ describe('rule helpers', () => {
         isNew: false,
         ruleType: 'saved_query',
         anomalyThreshold: 50,
-        mlJobId: '',
+        machineLearningJobId: '',
         index: ['auditbeat-*'],
         queryBar: {
           query: {
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx
index b4c58f3d02599..e59ca5e7e14e5 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx
@@ -47,7 +47,7 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => {
     isNew: false,
     ruleType: rule.type,
     anomalyThreshold: rule.anomaly_threshold ?? 50,
-    mlJobId: rule.machine_learning_job_id ?? '',
+    machineLearningJobId: rule.machine_learning_job_id ?? '',
     index: rule.index ?? [],
     queryBar: {
       query: { query: rule.query ?? '', language: rule.language ?? '' },
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
index fb6f898ff5949..447b5dc6325ee 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts
@@ -70,7 +70,7 @@ export interface AboutStepRuleDetails {
 export interface DefineStepRule extends StepRuleData {
   anomalyThreshold: number;
   index: string[];
-  mlJobId: string;
+  machineLearningJobId: string;
   queryBar: FieldValueQueryBar;
   ruleType: RuleType;
 }
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
index 07e15122a4bf5..e9fda3a1da304 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
@@ -361,6 +361,7 @@ export const getResult = (): RuleAlertType => ({
   alertTypeId: 'siem.signals',
   consumer: 'siem',
   params: {
+    anomalyThreshold: null,
     description: 'Detecting root and admin users',
     ruleId: 'rule-1',
     index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
@@ -369,6 +370,7 @@ export const getResult = (): RuleAlertType => ({
     immutable: false,
     query: 'user.name: root or user.name: admin',
     language: 'kuery',
+    machineLearningJobId: null,
     outputIndex: '.siem-signals',
     timelineId: 'some-timeline-id',
     timelineTitle: 'some-timeline-title',
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts
index d727bbb953d2a..b819bc6919274 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts
@@ -56,12 +56,14 @@ export const createRulesBulkRoute = (router: IRouter) => {
           .filter(rule => rule.rule_id == null || !dupes.includes(rule.rule_id))
           .map(async payloadRule => {
             const {
+              anomaly_threshold: anomalyThreshold,
               description,
               enabled,
               false_positives: falsePositives,
               from,
               query,
               language,
+              machine_learning_job_id: machineLearningJobId,
               output_index: outputIndex,
               saved_id: savedId,
               meta,
@@ -107,6 +109,7 @@ export const createRulesBulkRoute = (router: IRouter) => {
               const createdRule = await createRules({
                 alertsClient,
                 actionsClient,
+                anomalyThreshold,
                 description,
                 enabled,
                 falsePositives,
@@ -114,6 +117,7 @@ export const createRulesBulkRoute = (router: IRouter) => {
                 immutable: false,
                 query,
                 language,
+                machineLearningJobId,
                 outputIndex: finalIndex,
                 savedId,
                 timelineId,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts
index b2757d0ca6541..42bade1ba0855 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts
@@ -43,7 +43,7 @@ export const createRulesRoute = (router: IRouter): void => {
         timeline_id: timelineId,
         timeline_title: timelineTitle,
         meta,
-        machine_learning_job_id: mlJobId,
+        machine_learning_job_id: machineLearningJobId,
         filters,
         rule_id: ruleId,
         index,
@@ -108,7 +108,7 @@ export const createRulesRoute = (router: IRouter): void => {
           timelineId,
           timelineTitle,
           meta,
-          mlJobId,
+          machineLearningJobId,
           filters,
           ruleId: ruleId ?? uuid.v4(),
           index,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts
index ec4e707f46e50..d92ef316aef0c 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts
@@ -111,6 +111,7 @@ export const importRulesRoute = (router: IRouter, config: LegacyServices['config
                     return null;
                   }
                   const {
+                    anomaly_threshold: anomalyThreshold,
                     description,
                     enabled,
                     false_positives: falsePositives,
@@ -118,6 +119,7 @@ export const importRulesRoute = (router: IRouter, config: LegacyServices['config
                     immutable,
                     query,
                     language,
+                    machine_learning_job_id: machineLearningJobId,
                     output_index: outputIndex,
                     saved_id: savedId,
                     meta,
@@ -139,6 +141,7 @@ export const importRulesRoute = (router: IRouter, config: LegacyServices['config
                     timeline_title: timelineTitle,
                     version,
                   } = parsedRule;
+
                   try {
                     const signalsIndex = siemClient.signalsIndex;
                     const indexExists = await getIndexExists(
@@ -159,6 +162,7 @@ export const importRulesRoute = (router: IRouter, config: LegacyServices['config
                       await createRules({
                         alertsClient,
                         actionsClient,
+                        anomalyThreshold,
                         description,
                         enabled,
                         falsePositives,
@@ -166,6 +170,7 @@ export const importRulesRoute = (router: IRouter, config: LegacyServices['config
                         immutable,
                         query,
                         language,
+                        machineLearningJobId,
                         outputIndex: signalsIndex,
                         savedId,
                         timelineId,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts
index 777b9f3cc7a9d..859935d851126 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts
@@ -47,12 +47,14 @@ export const updateRulesBulkRoute = (router: IRouter) => {
       const rules = await Promise.all(
         request.body.map(async payloadRule => {
           const {
+            anomaly_threshold: anomalyThreshold,
             description,
             enabled,
             false_positives: falsePositives,
             from,
             query,
             language,
+            machine_learning_job_id: machineLearningJobId,
             output_index: outputIndex,
             saved_id: savedId,
             timeline_id: timelineId,
@@ -81,6 +83,7 @@ export const updateRulesBulkRoute = (router: IRouter) => {
             const rule = await updateRules({
               alertsClient,
               actionsClient,
+              anomalyThreshold,
               description,
               enabled,
               immutable: false,
@@ -88,6 +91,7 @@ export const updateRulesBulkRoute = (router: IRouter) => {
               from,
               query,
               language,
+              machineLearningJobId,
               outputIndex: finalIndex,
               savedId,
               savedObjectsClient,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.ts
index 1393de8c725cb..a9982a9896633 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.ts
@@ -30,12 +30,14 @@ export const updateRulesRoute = (router: IRouter) => {
     },
     async (context, request, response) => {
       const {
+        anomaly_threshold: anomalyThreshold,
         description,
         enabled,
         false_positives: falsePositives,
         from,
         query,
         language,
+        machine_learning_job_id: machineLearningJobId,
         output_index: outputIndex,
         saved_id: savedId,
         timeline_id: timelineId,
@@ -77,6 +79,7 @@ export const updateRulesRoute = (router: IRouter) => {
         const rule = await updateRules({
           alertsClient,
           actionsClient,
+          anomalyThreshold,
           description,
           enabled,
           falsePositives,
@@ -84,6 +87,7 @@ export const updateRulesRoute = (router: IRouter) => {
           immutable: false,
           query,
           language,
+          machineLearningJobId,
           outputIndex: finalIndex,
           savedId,
           savedObjectsClient,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts
index 70fcbb2c163ca..3243ccb14f89c 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts
@@ -36,7 +36,7 @@ describe('utils', () => {
     test('should work with a full data set', () => {
       const fullRule = getResult();
       const rule = transformAlertToRule(fullRule);
-      const expected: OutputRuleAlertRest = {
+      const expected: Partial<OutputRuleAlertRest> = {
         created_by: 'elastic',
         created_at: '2019-12-13T16:40:33.400Z',
         updated_at: '2019-12-13T16:40:33.400Z',
@@ -358,7 +358,7 @@ describe('utils', () => {
       const fullRule = getResult();
       fullRule.enabled = false;
       const ruleWithEnabledFalse = transformAlertToRule(fullRule);
-      const expected: OutputRuleAlertRest = {
+      const expected: Partial<OutputRuleAlertRest> = {
         created_by: 'elastic',
         created_at: '2019-12-13T16:40:33.400Z',
         updated_at: '2019-12-13T16:40:33.400Z',
@@ -424,7 +424,7 @@ describe('utils', () => {
       const fullRule = getResult();
       fullRule.params.immutable = false;
       const ruleWithEnabledFalse = transformAlertToRule(fullRule);
-      const expected: OutputRuleAlertRest = {
+      const expected: Partial<OutputRuleAlertRest> = {
         created_by: 'elastic',
         created_at: '2019-12-13T16:40:33.400Z',
         updated_at: '2019-12-13T16:40:33.400Z',
@@ -490,7 +490,7 @@ describe('utils', () => {
       const fullRule = getResult();
       fullRule.tags = ['tag 1', 'tag 2', `${INTERNAL_IDENTIFIER}_some_other_value`];
       const rule = transformAlertToRule(fullRule);
-      const expected: OutputRuleAlertRest = {
+      const expected: Partial<OutputRuleAlertRest> = {
         created_at: '2019-12-13T16:40:33.400Z',
         updated_at: '2019-12-13T16:40:33.400Z',
         created_by: 'elastic',
@@ -551,6 +551,22 @@ describe('utils', () => {
       };
       expect(rule).toEqual(expected);
     });
+
+    it('transforms ML Rule fields', () => {
+      const mlRule = getResult();
+      mlRule.params.anomalyThreshold = 55;
+      mlRule.params.machineLearningJobId = 'some_job_id';
+      mlRule.params.type = 'machine_learning';
+
+      const rule = transformAlertToRule(mlRule);
+      expect(rule).toEqual(
+        expect.objectContaining({
+          anomaly_threshold: 55,
+          machine_learning_job_id: 'some_job_id',
+          type: 'machine_learning',
+        })
+      );
+    });
   });
 
   describe('getIdError', () => {
@@ -640,7 +656,7 @@ describe('utils', () => {
         total: 0,
         data: [getResult()],
       });
-      const expected: OutputRuleAlertRest = {
+      const expected: Partial<OutputRuleAlertRest> = {
         created_by: 'elastic',
         created_at: '2019-12-13T16:40:33.400Z',
         updated_at: '2019-12-13T16:40:33.400Z',
@@ -722,7 +738,7 @@ describe('utils', () => {
   describe('transform', () => {
     test('outputs 200 if the data is of type siem alert', () => {
       const output = transform(getResult());
-      const expected: OutputRuleAlertRest = {
+      const expected: Partial<OutputRuleAlertRest> = {
         created_by: 'elastic',
         created_at: '2019-12-13T16:40:33.400Z',
         updated_at: '2019-12-13T16:40:33.400Z',
@@ -895,7 +911,7 @@ describe('utils', () => {
   describe('transformOrBulkError', () => {
     test('outputs 200 if the data is of type siem alert', () => {
       const output = transformOrBulkError('rule-1', getResult());
-      const expected: OutputRuleAlertRest = {
+      const expected: Partial<OutputRuleAlertRest> = {
         created_by: 'elastic',
         created_at: '2019-12-13T16:40:33.400Z',
         updated_at: '2019-12-13T16:40:33.400Z',
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts
index 177dae461e715..abd8dd7e87f03 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts
@@ -118,7 +118,7 @@ export const transformAlertToRule = (
     language: alert.params.language,
     output_index: alert.params.outputIndex,
     max_signals: alert.params.maxSignals,
-    machine_learning_job_id: alert.params.mlJobId,
+    machine_learning_job_id: alert.params.machineLearningJobId,
     risk_score: alert.params.riskScore,
     name: alert.name,
     query: alert.params.query,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts
index 5885653051d69..1b4c06fb5d828 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts
@@ -23,7 +23,7 @@ export const createRules = ({
   timelineId,
   timelineTitle,
   meta,
-  mlJobId,
+  machineLearningJobId,
   filters,
   ruleId,
   immutable,
@@ -63,7 +63,7 @@ export const createRules = ({
         timelineId,
         timelineTitle,
         meta,
-        mlJobId,
+        machineLearningJobId,
         filters,
         maxSignals,
         riskScore,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/install_prepacked_rules.ts
index 3b5ef57d3dcb6..dc71ae3678f2e 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/install_prepacked_rules.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/install_prepacked_rules.ts
@@ -18,6 +18,7 @@ export const installPrepackagedRules = (
 ): Array<Promise<Alert>> =>
   rules.reduce<Array<Promise<Alert>>>((acc, rule) => {
     const {
+      anomaly_threshold: anomalyThreshold,
       description,
       enabled,
       false_positives: falsePositives,
@@ -25,6 +26,7 @@ export const installPrepackagedRules = (
       immutable,
       query,
       language,
+      machine_learning_job_id: machineLearningJobId,
       saved_id: savedId,
       timeline_id: timelineId,
       timeline_title: timelineTitle,
@@ -50,6 +52,7 @@ export const installPrepackagedRules = (
       createRules({
         alertsClient,
         actionsClient,
+        anomalyThreshold,
         description,
         enabled,
         falsePositives,
@@ -57,6 +60,7 @@ export const installPrepackagedRules = (
         immutable,
         query,
         language,
+        machineLearningJobId,
         outputIndex,
         savedId,
         timelineId,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts
index d84358a32c672..1efa46c6b8b57 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts
@@ -147,11 +147,6 @@ export interface Clients {
   actionsClient: ActionsClient;
 }
 
-export interface MlRuleParams {
-  anomalyThreshold: number;
-  mlJobId: string;
-}
-
 export type PatchRuleParams = Partial<RuleAlertParams> & {
   id: string | undefined | null;
   savedObjectsClient: SavedObjectsClientContract;
@@ -167,8 +162,7 @@ export type DeleteRuleParams = Clients & {
   ruleId: string | undefined | null;
 };
 
-export type CreateRuleParams = Omit<RuleAlertParams, 'ruleId'> & { ruleId: string } & Clients &
-  Partial<MlRuleParams>;
+export type CreateRuleParams = Omit<RuleAlertParams, 'ruleId'> & { ruleId: string } & Clients;
 
 export interface ReadRuleParams {
   alertsClient: AlertsClient;
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts
index 922651edc4082..65fa5e86478a8 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts
@@ -29,6 +29,8 @@ export const sampleRuleAlertParams = (
   riskScore: riskScore ? riskScore : 50,
   maxSignals: maxSignals ? maxSignals : 10000,
   note: '',
+  anomalyThreshold: null,
+  machineLearningJobId: null,
   filters: undefined,
   savedId: undefined,
   timelineId: undefined,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts
index e3cf6be3e8e00..a9ccda2efe99c 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts
@@ -65,7 +65,7 @@ export const buildRule = ({
     version: ruleParams.version,
     created_at: createdAt,
     updated_at: updatedAt,
-    machine_learning_job_id: ruleParams.mlJobId,
+    machine_learning_job_id: ruleParams.machineLearningJobId,
     anomaly_threshold: ruleParams.anomalyThreshold,
   });
 };
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_params_schema.ts
index 3519613f3cb3d..5d94117daba30 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_params_schema.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_params_schema.ts
@@ -28,7 +28,7 @@ export const signalParamsSchema = () =>
     timelineId: schema.nullable(schema.string()),
     timelineTitle: schema.nullable(schema.string()),
     meta: schema.nullable(schema.object({}, { unknowns: 'allow' })),
-    mlJobId: schema.nullable(schema.string()),
+    machineLearningJobId: schema.nullable(schema.string()),
     query: schema.nullable(schema.string()),
     filters: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))),
     maxSignals: schema.number({ defaultValue: DEFAULT_MAX_SIGNALS }),
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index dad5b48577dc9..3d4998e960a8e 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -46,7 +46,7 @@ export const signalRulesAlertType = ({
         index,
         filters,
         language,
-        mlJobId,
+        machineLearningJobId,
         outputIndex,
         savedId,
         query,
@@ -96,14 +96,14 @@ export const signalRulesAlertType = ({
 
       try {
         if (type === 'machine_learning') {
-          if (mlJobId == null || anomalyThreshold == null) {
+          if (machineLearningJobId == null || anomalyThreshold == null) {
             throw new Error(
-              `Attempted to execute machine learning rule, but it is missing job id and/or anomaly threshold for rule id: "${ruleId}", name: "${name}", signals index: "${outputIndex}", job id: "${mlJobId}", anomaly threshold: "${anomalyThreshold}"`
+              `Attempted to execute machine learning rule, but it is missing job id and/or anomaly threshold for rule id: "${ruleId}", name: "${name}", signals index: "${outputIndex}", job id: "${machineLearningJobId}", anomaly threshold: "${anomalyThreshold}"`
             );
           }
 
           const anomalyResults = await findMlSignals(
-            mlJobId,
+            machineLearningJobId,
             anomalyThreshold,
             from,
             to,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
index f57ce145c6a85..980daf746dee7 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
@@ -25,7 +25,7 @@ export interface ThreatParams {
 export type RuleType = 'query' | 'saved_query' | 'machine_learning';
 
 export interface RuleAlertParams {
-  anomalyThreshold?: number;
+  anomalyThreshold: number | undefined | null;
   description: string;
   note: string | undefined | null;
   enabled: boolean;
@@ -38,7 +38,7 @@ export interface RuleAlertParams {
   ruleId: string | undefined | null;
   language: string | undefined | null;
   maxSignals: number;
-  mlJobId?: string;
+  machineLearningJobId: string | undefined | null;
   riskScore: number;
   outputIndex: string;
   name: string;
@@ -61,10 +61,12 @@ export type RuleTypeParams = Omit<RuleAlertParams, 'name' | 'enabled' | 'interva
 
 export type RuleAlertParamsRest = Omit<
   RuleAlertParams,
+  | 'anomalyThreshold'
   | 'ruleId'
   | 'falsePositives'
   | 'immutable'
   | 'maxSignals'
+  | 'machineLearningJobId'
   | 'savedId'
   | 'riskScore'
   | 'timelineId'
@@ -81,12 +83,14 @@ export type RuleAlertParamsRest = Omit<
     | 'lastSuccessMessage'
     | 'lastFailureMessage'
   > & {
+    anomaly_threshold: RuleAlertParams['anomalyThreshold'];
     rule_id: RuleAlertParams['ruleId'];
     false_positives: RuleAlertParams['falsePositives'];
     saved_id?: RuleAlertParams['savedId'];
     timeline_id: RuleAlertParams['timelineId'];
     timeline_title: RuleAlertParams['timelineTitle'];
     max_signals: RuleAlertParams['maxSignals'];
+    machine_learning_job_id: RuleAlertParams['machineLearningJobId'];
     risk_score: RuleAlertParams['riskScore'];
     output_index: RuleAlertParams['outputIndex'];
     created_at: string;

From e703568bd41061685698f5fb6cd242a07017f3cf Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 12:03:57 -0500
Subject: [PATCH 52/63] Use implicit type to avoid the need for a ts-ignore

FormSchema has a very generic index signature such that our
filterRuleFieldsForType helper cannot infer that it has our necessary
rule fields (when in fact it does). By removing the FormSchema hint we
get the actual keys of our schema, and things work as expected.

All other uses of schema continue to work because they're expecting
FormSchema, which is effectively { [key: string]: any }.
---
 .../rules/components/step_define_rule/index.tsx                | 1 -
 .../rules/components/step_define_rule/schema.tsx               | 3 +--
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
index 55e4883eeb25d..dd0997a03ba6e 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
@@ -169,7 +169,6 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
       <StepRuleDescription
         columns={descriptionColumns}
         indexPatterns={indexPatternQueryBar as IIndexPattern}
-        // @ts-ignore schema is insufficiently typed, but this works
         schema={filterRuleFieldsForType(schema, myStepData.ruleType)}
         data={filterRuleFieldsForType(myStepData, myStepData.ruleType)}
       />
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
index bcfcd4f4ee09d..b91047cd0fd59 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
@@ -15,13 +15,12 @@ import {
   ERROR_CODE,
   FIELD_TYPES,
   fieldValidators,
-  FormSchema,
   ValidationFunc,
 } from '../../../../../shared_imports';
 import { CUSTOM_QUERY_REQUIRED, INVALID_CUSTOM_QUERY, INDEX_HELPER_TEXT } from './translations';
 import { isMlRule } from '../../helpers';
 
-export const schema: FormSchema = {
+export const schema = {
   index: {
     type: FIELD_TYPES.COMBO_BOX,
     label: i18n.translate(

From 8ff89b0a03d2e0028da51b63479aacb337686c72 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 13:28:29 -0500
Subject: [PATCH 53/63] New ML params are not nullable

Rather than setting a null and then never using it, let's just make it
truly optional in terms of default values.
---
 .../detection_engine/routes/__mocks__/request_responses.ts    | 4 ++--
 .../lib/detection_engine/signals/__mocks__/es_results.ts      | 4 ++--
 .../lib/detection_engine/signals/signal_params_schema.ts      | 4 ++--
 .../legacy/plugins/siem/server/lib/detection_engine/types.ts  | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
index e9fda3a1da304..ac281a4563432 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
@@ -361,7 +361,7 @@ export const getResult = (): RuleAlertType => ({
   alertTypeId: 'siem.signals',
   consumer: 'siem',
   params: {
-    anomalyThreshold: null,
+    anomalyThreshold: undefined,
     description: 'Detecting root and admin users',
     ruleId: 'rule-1',
     index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
@@ -370,7 +370,7 @@ export const getResult = (): RuleAlertType => ({
     immutable: false,
     query: 'user.name: root or user.name: admin',
     language: 'kuery',
-    machineLearningJobId: null,
+    machineLearningJobId: undefined,
     outputIndex: '.siem-signals',
     timelineId: 'some-timeline-id',
     timelineTitle: 'some-timeline-title',
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts
index 65fa5e86478a8..010f6b2ee98ff 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts
@@ -29,8 +29,8 @@ export const sampleRuleAlertParams = (
   riskScore: riskScore ? riskScore : 50,
   maxSignals: maxSignals ? maxSignals : 10000,
   note: '',
-  anomalyThreshold: null,
-  machineLearningJobId: null,
+  anomalyThreshold: undefined,
+  machineLearningJobId: undefined,
   filters: undefined,
   savedId: undefined,
   timelineId: undefined,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_params_schema.ts
index 5d94117daba30..7b0546f56dd15 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_params_schema.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_params_schema.ts
@@ -14,7 +14,7 @@ import { DEFAULT_MAX_SIGNALS } from '../../../../common/constants';
  */
 export const signalParamsSchema = () =>
   schema.object({
-    anomalyThreshold: schema.nullable(schema.number()),
+    anomalyThreshold: schema.maybe(schema.number()),
     description: schema.string(),
     note: schema.nullable(schema.string()),
     falsePositives: schema.arrayOf(schema.string(), { defaultValue: [] }),
@@ -28,7 +28,7 @@ export const signalParamsSchema = () =>
     timelineId: schema.nullable(schema.string()),
     timelineTitle: schema.nullable(schema.string()),
     meta: schema.nullable(schema.object({}, { unknowns: 'allow' })),
-    machineLearningJobId: schema.nullable(schema.string()),
+    machineLearningJobId: schema.maybe(schema.string()),
     query: schema.nullable(schema.string()),
     filters: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))),
     maxSignals: schema.number({ defaultValue: DEFAULT_MAX_SIGNALS }),
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
index 980daf746dee7..f77924aafadf8 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
@@ -25,7 +25,7 @@ export interface ThreatParams {
 export type RuleType = 'query' | 'saved_query' | 'machine_learning';
 
 export interface RuleAlertParams {
-  anomalyThreshold: number | undefined | null;
+  anomalyThreshold: number | undefined;
   description: string;
   note: string | undefined | null;
   enabled: boolean;
@@ -38,7 +38,7 @@ export interface RuleAlertParams {
   ruleId: string | undefined | null;
   language: string | undefined | null;
   maxSignals: number;
-  machineLearningJobId: string | undefined | null;
+  machineLearningJobId: string | undefined;
   riskScore: number;
   outputIndex: string;
   name: string;

From edf354ebd709e6babf76bd0c0cf3689541329363 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 13:56:37 -0500
Subject: [PATCH 54/63] Query and language are conditional based on rule type

For ML Rules, we don't use them.
---
 .../routes/schemas/create_rules_schema.ts            | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts
index 5aca09f587a0d..e86963fd4594c 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts
@@ -55,8 +55,16 @@ export const createRulesSchema = Joi.object({
   rule_id,
   index,
   interval: interval.default('5m'),
-  query, // TODO conditional type/default?
-  language,
+  query: query.when('type', {
+    is: 'machine_learning',
+    then: Joi.forbidden(),
+    otherwise: query.allow('').default(''),
+  }),
+  language: language.when('type', {
+    is: 'machine_learning',
+    then: Joi.forbidden(),
+    otherwise: language.default('kuery'),
+  }),
   output_index,
   saved_id: saved_id.when('type', {
     is: 'saved_query',

From a949ce5d78cc96882cd8a525d999401487537fc0 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 14:06:09 -0500
Subject: [PATCH 55/63] Remove defaulted parameter in API test

We don't need to specify this, and we should continue not to for
backwards compatibility.
---
 .../security_and_spaces/tests/utils.ts                           | 1 -
 1 file changed, 1 deletion(-)

diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts
index c332916c9f662..b00248ece4126 100644
--- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts
+++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts
@@ -52,7 +52,6 @@ export const getSimpleRule = (ruleId = 'rule-1'): Partial<OutputRuleAlertRest> =
   index: ['auditbeat-*'],
   type: 'query',
   query: 'user.name: root or user.name: admin',
-  language: 'kuery',
 });
 
 export const getSignalStatus = () => ({

From cf8d8c31a010f7551ce5dac44b36b8ef3bfbe2b0 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 15:18:30 -0500
Subject: [PATCH 56/63] Use explicit types over implicit ones

The concern is that not typing our schemae as FormSchema could break our
form if there are upstream changes. For now, we simply use the
intersection of FormSchema and our generic parameter to satisfy our use
within the function.
---
 .../rules/components/step_define_rule/index.tsx              | 5 +++--
 .../rules/components/step_define_rule/schema.tsx             | 3 ++-
 .../public/pages/detection_engine/rules/create/helpers.ts    | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
index dd0997a03ba6e..6b1a9a828d950 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx
@@ -37,10 +37,11 @@ import {
   getUseField,
   UseField,
   useForm,
+  FormSchema,
 } from '../../../../../shared_imports';
 import { schema } from './schema';
 import * as i18n from './translations';
-import { filterRuleFieldsForType } from '../../create/helpers';
+import { filterRuleFieldsForType, RuleFields } from '../../create/helpers';
 
 const CommonUseField = getUseField({ component: Field });
 
@@ -169,7 +170,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
       <StepRuleDescription
         columns={descriptionColumns}
         indexPatterns={indexPatternQueryBar as IIndexPattern}
-        schema={filterRuleFieldsForType(schema, myStepData.ruleType)}
+        schema={filterRuleFieldsForType(schema as FormSchema & RuleFields, myStepData.ruleType)}
         data={filterRuleFieldsForType(myStepData, myStepData.ruleType)}
       />
     </StepContentWrapper>
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
index b91047cd0fd59..bcfcd4f4ee09d 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx
@@ -15,12 +15,13 @@ import {
   ERROR_CODE,
   FIELD_TYPES,
   fieldValidators,
+  FormSchema,
   ValidationFunc,
 } from '../../../../../shared_imports';
 import { CUSTOM_QUERY_REQUIRED, INVALID_CUSTOM_QUERY, INDEX_HELPER_TEXT } from './translations';
 import { isMlRule } from '../../helpers';
 
-export const schema = {
+export const schema: FormSchema = {
   index: {
     type: FIELD_TYPES.COMBO_BOX,
     label: i18n.translate(
diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
index 6afd22b58bf84..1f3379bf681bb 100644
--- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
+++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts
@@ -39,7 +39,7 @@ export const getTimeTypeValue = (time: string): { unit: string; value: number }
   return timeObj;
 };
 
-interface RuleFields {
+export interface RuleFields {
   anomalyThreshold: unknown;
   machineLearningJobId: unknown;
   queryBar: unknown;

From f692e017be9d339e1105ec6cfce12d96d8a22b68 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 15:39:40 -0500
Subject: [PATCH 57/63] Add integration test for creation of ML Rule

---
 .../security_and_spaces/tests/create_rules.ts | 13 +++++++++
 .../security_and_spaces/tests/utils.ts        | 29 +++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts
index d6a238e5b0940..8f28afef3c1c7 100644
--- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts
+++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts
@@ -18,6 +18,8 @@ import {
   getSimpleRuleWithoutRuleId,
   removeServerGeneratedProperties,
   removeServerGeneratedPropertiesIncludingRuleId,
+  getSimpleMlRule,
+  getSimpleMlRuleOutput,
 } from './utils';
 
 // eslint-disable-next-line import/no-default-export
@@ -74,6 +76,17 @@ export default ({ getService }: FtrProviderContext) => {
         expect(bodyToCompare).to.eql(getSimpleRuleOutputWithoutRuleId());
       });
 
+      it('should create a single Machine Learning rule', async () => {
+        const { body } = await supertest
+          .post(DETECTION_ENGINE_RULES_URL)
+          .set('kbn-xsrf', 'true')
+          .send(getSimpleMlRule())
+          .expect(200);
+
+        const bodyToCompare = removeServerGeneratedProperties(body);
+        expect(bodyToCompare).to.eql(getSimpleMlRuleOutput());
+      });
+
       it('should cause a 409 conflict if we attempt to create the same rule_id twice', async () => {
         await supertest
           .post(DETECTION_ENGINE_RULES_URL)
diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts
index b00248ece4126..8847a2fdb21af 100644
--- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts
+++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts
@@ -54,6 +54,21 @@ export const getSimpleRule = (ruleId = 'rule-1'): Partial<OutputRuleAlertRest> =
   query: 'user.name: root or user.name: admin',
 });
 
+/**
+ * This is a representative ML rule payload as expected by the server
+ * @param ruleId
+ */
+export const getSimpleMlRule = (ruleId = 'rule-1'): Partial<OutputRuleAlertRest> => ({
+  name: 'Simple ML Rule',
+  description: 'Simple Machine Learning Rule',
+  anomaly_threshold: 44,
+  risk_score: 1,
+  rule_id: ruleId,
+  severity: 'high',
+  machine_learning_job_id: 'some_job_id',
+  type: 'machine_learning',
+});
+
 export const getSignalStatus = () => ({
   aggs: { statuses: { terms: { field: 'signal.status', size: 10 } } },
 });
@@ -149,6 +164,20 @@ export const getSimpleRuleOutputWithoutRuleId = (
   return ruleWithoutRuleId;
 };
 
+export const getSimpleMlRuleOutput = (ruleId = 'rule-1'): Partial<OutputRuleAlertRest> => {
+  const rule = getSimpleRuleOutput(ruleId);
+  const { query, language, index, ...rest } = rule;
+
+  return {
+    ...rest,
+    name: 'Simple ML Rule',
+    description: 'Simple Machine Learning Rule',
+    anomaly_threshold: 44,
+    machine_learning_job_id: 'some_job_id',
+    type: 'machine_learning',
+  };
+};
+
 /**
  * Remove all alerts from the .kibana index
  * @param es The ElasticSearch handle

From 1aa43217ac06ef59ff803c1adcd522de305713a4 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 16:02:04 -0500
Subject: [PATCH 58/63] Add ML fields to route schemae

* threshold and job id are conditional on type
* makes query and language mutually exclusive with above
---
 .../schemas/add_prepackaged_rules_schema.ts   | 24 +++++++++++++++++--
 .../routes/schemas/import_rules_schema.ts     | 24 +++++++++++++++++--
 .../routes/schemas/patch_rules_schema.ts      |  4 ++++
 .../routes/schemas/update_rules_schema.ts     | 24 +++++++++++++++++--
 4 files changed, 70 insertions(+), 6 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/add_prepackaged_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/add_prepackaged_rules_schema.ts
index 974ddcf35eeb4..ec0a8e7871b5b 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/add_prepackaged_rules_schema.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/add_prepackaged_rules_schema.ts
@@ -34,6 +34,8 @@ import {
   references,
   note,
   version,
+  anomaly_threshold,
+  machine_learning_job_id,
 } from './schemas';
 /* eslint-enable @typescript-eslint/camelcase */
 
@@ -49,6 +51,11 @@ import { DEFAULT_MAX_SIGNALS } from '../../../../../common/constants';
  *  - index is a required field that must exist
  */
 export const addPrepackagedRulesSchema = Joi.object({
+  anomaly_threshold: anomaly_threshold.when('type', {
+    is: 'machine_learning',
+    then: Joi.required(),
+    otherwise: Joi.forbidden(),
+  }),
   description: description.required(),
   enabled: enabled.default(false),
   false_positives: false_positives.default([]),
@@ -61,8 +68,21 @@ export const addPrepackagedRulesSchema = Joi.object({
     .valid(true),
   index: index.required(),
   interval: interval.default('5m'),
-  query: query.allow('').default(''),
-  language: language.default('kuery'),
+  query: query.when('type', {
+    is: 'machine_learning',
+    then: Joi.forbidden(),
+    otherwise: query.allow('').default(''),
+  }),
+  language: language.when('type', {
+    is: 'machine_learning',
+    then: Joi.forbidden(),
+    otherwise: language.default('kuery'),
+  }),
+  machine_learning_job_id: machine_learning_job_id.when('type', {
+    is: 'machine_learning',
+    then: Joi.required(),
+    otherwise: Joi.forbidden(),
+  }),
   saved_id: saved_id.when('type', {
     is: 'saved_query',
     then: Joi.required(),
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/import_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/import_rules_schema.ts
index bd12872c4dc72..92718b7ae71ba 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/import_rules_schema.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/import_rules_schema.ts
@@ -40,6 +40,8 @@ import {
   references,
   note,
   version,
+  anomaly_threshold,
+  machine_learning_job_id,
 } from './schemas';
 /* eslint-enable @typescript-eslint/camelcase */
 
@@ -55,6 +57,11 @@ import { DEFAULT_MAX_SIGNALS } from '../../../../../common/constants';
  *   - updated_by is optional (but ignored in the import code)
  */
 export const importRulesSchema = Joi.object({
+  anomaly_threshold: anomaly_threshold.when('type', {
+    is: 'machine_learning',
+    then: Joi.required(),
+    otherwise: Joi.forbidden(),
+  }),
   id,
   description: description.required(),
   enabled: enabled.default(true),
@@ -65,9 +72,22 @@ export const importRulesSchema = Joi.object({
   immutable: immutable.default(false).valid(false),
   index,
   interval: interval.default('5m'),
-  query: query.allow('').default(''),
-  language: language.default('kuery'),
+  query: query.when('type', {
+    is: 'machine_learning',
+    then: Joi.forbidden(),
+    otherwise: query.allow('').default(''),
+  }),
+  language: language.when('type', {
+    is: 'machine_learning',
+    then: Joi.forbidden(),
+    otherwise: language.default('kuery'),
+  }),
   output_index,
+  machine_learning_job_id: machine_learning_job_id.when('type', {
+    is: 'machine_learning',
+    then: Joi.required(),
+    otherwise: Joi.forbidden(),
+  }),
   saved_id: saved_id.when('type', {
     is: 'saved_query',
     then: Joi.required(),
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/patch_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/patch_rules_schema.ts
index 4d1b73fb69e5b..4496a808f6869 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/patch_rules_schema.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/patch_rules_schema.ts
@@ -35,10 +35,13 @@ import {
   note,
   id,
   version,
+  anomaly_threshold,
+  machine_learning_job_id,
 } from './schemas';
 /* eslint-enable @typescript-eslint/camelcase */
 
 export const patchRulesSchema = Joi.object({
+  anomaly_threshold,
   description,
   enabled,
   false_positives,
@@ -50,6 +53,7 @@ export const patchRulesSchema = Joi.object({
   interval,
   query: query.allow(''),
   language,
+  machine_learning_job_id,
   output_index,
   saved_id,
   timeline_id,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/update_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/update_rules_schema.ts
index a72105142d287..f7a53385200df 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/update_rules_schema.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/update_rules_schema.ts
@@ -35,6 +35,8 @@ import {
   id,
   note,
   version,
+  anomaly_threshold,
+  machine_learning_job_id,
 } from './schemas';
 /* eslint-enable @typescript-eslint/camelcase */
 
@@ -48,6 +50,11 @@ import { DEFAULT_MAX_SIGNALS } from '../../../../../common/constants';
  *   - id is on here because you can pass in an id to update using it instead of rule_id.
  */
 export const updateRulesSchema = Joi.object({
+  anomaly_threshold: anomaly_threshold.when('type', {
+    is: 'machine_learning',
+    then: Joi.required(),
+    otherwise: Joi.forbidden(),
+  }),
   description: description.required(),
   enabled: enabled.default(true),
   id,
@@ -57,8 +64,21 @@ export const updateRulesSchema = Joi.object({
   rule_id,
   index,
   interval: interval.default('5m'),
-  query: query.allow('').default(''),
-  language: language.default('kuery'),
+  query: query.when('type', {
+    is: 'machine_learning',
+    then: Joi.forbidden(),
+    otherwise: query.allow('').default(''),
+  }),
+  language: language.when('type', {
+    is: 'machine_learning',
+    then: Joi.forbidden(),
+    otherwise: language.default('kuery'),
+  }),
+  machine_learning_job_id: machine_learning_job_id.when('type', {
+    is: 'machine_learning',
+    then: Joi.required(),
+    otherwise: Joi.forbidden(),
+  }),
   output_index,
   saved_id: saved_id.when('type', {
     is: 'saved_query',

From 8ff01e94cd3f287d8ae32ba0b69b858897a2a738 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 16:02:59 -0500
Subject: [PATCH 59/63] Fix router test for creating an ML rule

We were sending invalid parameters.
---
 .../routes/__mocks__/request_responses.ts                | 9 ++++++---
 .../routes/rules/create_rules_route.test.ts              | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
index ac281a4563432..d90c8ea49a53f 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
@@ -293,17 +293,20 @@ export const getCreateRequest = () =>
     body: typicalPayload(),
   });
 
-export const createMlRuleRequest = () =>
-  requestMock.create({
+export const createMlRuleRequest = () => {
+  const { query, language, index, ...mlParams } = typicalPayload();
+
+  return requestMock.create({
     method: 'post',
     path: DETECTION_ENGINE_RULES_URL,
     body: {
-      ...typicalPayload(),
+      ...mlParams,
       type: 'machine_learning',
       anomaly_threshold: 50,
       machine_learning_job_id: 'some-uuid',
     },
   });
+};
 
 export const getSetSignalStatusByIdsRequest = () =>
   requestMock.create({
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts
index 0ae09c5cb1661..976f371c6b1a6 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts
@@ -50,7 +50,7 @@ describe('create_rules', () => {
   });
 
   describe('creating an ML Rule', () => {
-    it('works', async () => {
+    it('is successful', async () => {
       const response = await server.inject(createMlRuleRequest(), context);
       expect(response.status).toEqual(200);
     });

From 16b25d1abbb760530bd0184c17f91bb174c90af1 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 16:58:54 -0500
Subject: [PATCH 60/63] Remove null check against index for query rules

We support not having an index here, as getInputIndex will return the
current UI setting if none is specified.
---
 .../lib/detection_engine/signals/signal_rule_alert_type.ts  | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index 3d4998e960a8e..b28e3d7d47f2f 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -134,12 +134,6 @@ export const signalRulesAlertType = ({
             tags,
           });
         } else {
-          if (index == null) {
-            throw new Error(
-              `Attempted to execute query rule, but it is missing index pattern for rule id: "${ruleId}", name: "${name}", signals index: "${outputIndex}", index pattern: "${index}"`
-            );
-          }
-
           const inputIndex = await getInputIndex(services, version, index);
           const esFilter = await getFilter({
             type,

From b4fd572a61b9e2fea5831e4d929cb13ba3d05c90 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 17:06:53 -0500
Subject: [PATCH 61/63] Add regression test for API compatibility

We were previously able to create a rule without an input index; we
should continue to support that, as verified by this test!
---
 .../security_and_spaces/tests/create_rules.ts      | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts
index 8f28afef3c1c7..91088acb7a51c 100644
--- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts
+++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts
@@ -65,6 +65,20 @@ export default ({ getService }: FtrProviderContext) => {
         expect(bodyToCompare).to.eql(getSimpleRuleOutput());
       });
 
+      it('should create a single rule without an input index', async () => {
+        const { index, ...payload } = getSimpleRule();
+        const { index: _index, ...expected } = getSimpleRuleOutput();
+
+        const { body } = await supertest
+          .post(DETECTION_ENGINE_RULES_URL)
+          .set('kbn-xsrf', 'true')
+          .send(payload)
+          .expect(200);
+
+        const bodyToCompare = removeServerGeneratedProperties(body);
+        expect(bodyToCompare).to.eql(expected);
+      });
+
       it('should create a single rule without a rule_id', async () => {
         const { body } = await supertest
           .post(DETECTION_ENGINE_RULES_URL)

From 462d4106905e1a333863d4bc19137c1baa2ae454 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 17:26:35 -0500
Subject: [PATCH 62/63] Respect the index pattern determined at runtime when
 performing search_after

If a rule does not specify an input index pattern on creation, we use
the current UI default when the rule is evaluated. This ensures that any
subsequent searches use that same index.

We're not currently persisting that runtime index to the generated
signal, but we should.
---
 .../signals/search_after_bulk_create.ts            | 14 +++++---------
 .../signals/signal_rule_alert_type.ts              |  1 +
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
index 9c003d2d75268..f54ad67af4a48 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts
@@ -17,6 +17,7 @@ interface SearchAfterAndBulkCreateParams {
   services: AlertServices;
   logger: Logger;
   id: string;
+  inputIndexPattern: string[];
   signalsIndex: string;
   name: string;
   createdAt: string;
@@ -37,6 +38,7 @@ export const searchAfterAndBulkCreate = async ({
   services,
   logger,
   id,
+  inputIndexPattern,
   signalsIndex,
   filter,
   name,
@@ -52,12 +54,6 @@ export const searchAfterAndBulkCreate = async ({
   if (someResult.hits.hits.length === 0) {
     return true;
   }
-  const { index, from, to } = ruleParams;
-  if (index == null) {
-    throw new Error(
-      `Attempted to bulk create signals, but rule id: ${id}, name: ${name}, signals index: ${signalsIndex} has no index pattern`
-    );
-  }
 
   logger.debug('[+] starting bulk insertion');
   await singleBulkCreate({
@@ -104,9 +100,9 @@ export const searchAfterAndBulkCreate = async ({
       logger.debug(`sortIds: ${sortIds}`);
       const searchAfterResult: SignalSearchResponse = await singleSearchAfter({
         searchAfterSortId: sortId,
-        index,
-        from,
-        to,
+        index: inputIndexPattern,
+        from: ruleParams.from,
+        to: ruleParams.to,
         services,
         logger,
         filter,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index b28e3d7d47f2f..7a4dcf68e0ca9 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -177,6 +177,7 @@ export const signalRulesAlertType = ({
             services,
             logger,
             id: alertId,
+            inputIndexPattern: inputIndex,
             signalsIndex: outputIndex,
             filter: esFilter,
             name,

From 9f9f32465be75ff32a8785f7e09237dc9b5c9d8c Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 18 Mar 2020 17:44:29 -0500
Subject: [PATCH 63/63] Fix type errors in our bulk create tests

We added a new argument, but didn't update the tests.
---
 .../signals/search_after_bulk_create.test.ts           | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts
index bf7a97a29aef3..09daae8485381 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts
@@ -26,8 +26,10 @@ export const mockService = {
 };
 
 describe('searchAfterAndBulkCreate', () => {
+  let inputIndexPattern: string[] = [];
   beforeEach(() => {
     jest.clearAllMocks();
+    inputIndexPattern = ['auditbeat-*'];
   });
 
   test('if successful with empty search results', async () => {
@@ -38,6 +40,7 @@ describe('searchAfterAndBulkCreate', () => {
       services: mockService,
       logger: mockLogger,
       id: sampleRuleGuid,
+      inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
       name: 'rule-name',
       createdAt: '2020-01-28T15:58:34.810Z',
@@ -93,6 +96,7 @@ describe('searchAfterAndBulkCreate', () => {
       services: mockService,
       logger: mockLogger,
       id: sampleRuleGuid,
+      inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
       name: 'rule-name',
       createdAt: '2020-01-28T15:58:34.810Z',
@@ -119,6 +123,7 @@ describe('searchAfterAndBulkCreate', () => {
       services: mockService,
       logger: mockLogger,
       id: sampleRuleGuid,
+      inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
       name: 'rule-name',
       createdAt: '2020-01-28T15:58:34.810Z',
@@ -152,6 +157,7 @@ describe('searchAfterAndBulkCreate', () => {
       services: mockService,
       logger: mockLogger,
       id: sampleRuleGuid,
+      inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
       name: 'rule-name',
       createdAt: '2020-01-28T15:58:34.810Z',
@@ -185,6 +191,7 @@ describe('searchAfterAndBulkCreate', () => {
       services: mockService,
       logger: mockLogger,
       id: sampleRuleGuid,
+      inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
       name: 'rule-name',
       createdAt: '2020-01-28T15:58:34.810Z',
@@ -220,6 +227,7 @@ describe('searchAfterAndBulkCreate', () => {
       services: mockService,
       logger: mockLogger,
       id: sampleRuleGuid,
+      inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
       name: 'rule-name',
       createdAt: '2020-01-28T15:58:34.810Z',
@@ -255,6 +263,7 @@ describe('searchAfterAndBulkCreate', () => {
       services: mockService,
       logger: mockLogger,
       id: sampleRuleGuid,
+      inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
       name: 'rule-name',
       createdAt: '2020-01-28T15:58:34.810Z',
@@ -292,6 +301,7 @@ describe('searchAfterAndBulkCreate', () => {
       services: mockService,
       logger: mockLogger,
       id: sampleRuleGuid,
+      inputIndexPattern,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
       name: 'rule-name',
       createdAt: '2020-01-28T15:58:34.810Z',