[ML] Explain log rate spikes: Analysis API endpoint.

x-pack/.i18nrc.json

@@ -37,7 +37,10 @@
     "xpack.logstash": ["plugins/logstash"],
     "xpack.main": "legacy/plugins/xpack_main",
     "xpack.maps": ["plugins/maps"],
-    "xpack.aiops": ["plugins/aiops"],
+    "xpack.aiops": [
+      "packages/ml/aiops_utils",
+      "plugins/aiops"
+    ],
     "xpack.ml": ["plugins/ml"],
     "xpack.monitoring": ["plugins/monitoring"],
     "xpack.osquery": ["plugins/osquery"],

x-pack/packages/ml/aiops_utils/BUILD.bazel

@@ -37,8 +37,10 @@ NPM_MODULE_EXTRA_FILES = [
 #    "@npm//name-of-package"
 #    eg. "@npm//lodash"
 RUNTIME_DEPS = [
-  "//packages/kbn-logging",
-  "@npm//react"
+  "@npm//react",
+  "@npm//@elastic/eui",
+  "//packages/kbn-i18n-react",
+  "//packages/kbn-logging"
 ]
 
 # In this array place dependencies necessary to build the types, which will include the
@@ -51,10 +53,12 @@ RUNTIME_DEPS = [
 #
 #  References to NPM packages work the same as RUNTIME_DEPS
 TYPES_DEPS = [
-  "//packages/kbn-logging:npm_module_types",
   "@npm//@types/node",
   "@npm//@types/jest",
-  "@npm//@types/react"
+  "@npm//@types/react",
+  "@npm//@elastic/eui",
+  "//packages/kbn-i18n-react:npm_module_types",
+  "//packages/kbn-logging:npm_module_types"
 ]
 
 jsts_transpiler(

x-pack/packages/ml/aiops_utils/src/components/progress_controls.tsx

@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiProgress, EuiText } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n-react';
+import React from 'react';
+
+interface ProgressControlProps {
+  progress: number;
+  progressMessage: string;
+  onRefresh: () => void;
+  onCancel: () => void;
+  isRunning: boolean;
+}
+
+export function ProgressControls({
+  progress,
+  progressMessage,
+  onRefresh,
+  onCancel,
+  isRunning,
+}: ProgressControlProps) {
+  return (
+    <EuiFlexGroup>
+      <EuiFlexItem>
+        <EuiFlexGroup direction="column" gutterSize="none">
+          <EuiFlexItem data-test-subj="aiopProgressTitle">
+            <EuiText size="xs" color="subdued">
+              <FormattedMessage
+                data-test-subj="aiopsProgressTitleMessage"
+                id="xpack.aiops.progressTitle"
+                defaultMessage="Progress: {progress}% — {progressMessage}"
+                values={{ progress: Math.round(progress * 100), progressMessage }}
+              />
+            </EuiText>
+          </EuiFlexItem>
+          <EuiFlexItem>
+            <EuiProgress
+              aria-label={i18n.translate('xpack.aiops.progressAriaLabel', {
+                defaultMessage: 'Progress',
+              })}
+              value={Math.round(progress * 100)}
+              max={100}
+              size="m"
+            />
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      </EuiFlexItem>
+      <EuiFlexItem grow={false}>
+        {!isRunning && (
+          <EuiButton size="s" onClick={onRefresh}>
+            <FormattedMessage id="xpack.aiops.refreshButtonTitle" defaultMessage="Refresh" />
+          </EuiButton>
+        )}
+        {isRunning && (
+          <EuiButton size="s" onClick={onCancel}>
+            <FormattedMessage id="xpack.aiops.cancelButtonTitle" defaultMessage="Cancel" />
+          </EuiButton>
+        )}
+      </EuiFlexItem>
+    </EuiFlexGroup>
+  );
+}

x-pack/packages/ml/aiops_utils/src/index.ts

@@ -5,6 +5,9 @@
  * 2.0.
  */
 
+export { ProgressControls } from './components/progress_controls';
+export { getWindowParameters } from './lib/get_window_parameters';
+export type { WindowParameters } from './lib/get_window_parameters';
 export { streamFactory } from './lib/stream_factory';
 export { useFetchStream } from './lib/use_fetch_stream';
 export type {

x-pack/packages/ml/aiops_utils/src/lib/accept_compression.ts

@@ -5,9 +5,11 @@
  * 2.0.
  */
 
-// TODO: Replace these with kbn packaged versions once we have those available to us.
-// At the moment imports from runtime plugins into packages are not supported.
-// import type { Headers } from '@kbn/core/server';
+/**
+ * TODO: Replace these with kbn packaged versions once we have those available to us.
+ * At the moment imports from runtime plugins into packages are not supported.
+ * import type { Headers } from '@kbn/core/server';
+ */
 type Headers = Record<string, string | string[] | undefined>;
 
 function containsGzip(s: string) {

x-pack/packages/ml/aiops_utils/src/lib/get_window_parameters.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * Time range definition for baseline and deviation to be used by spike log analysis.
+ */
+export interface WindowParameters {
+  baselineMin: number;
+  baselineMax: number;
+  deviationMin: number;
+  deviationMax: number;
+}
+
+/**
+ * Given a point in time (e.g. where a user clicks), use simple heuristics to compute:
+ *
+ * 1. The time window around the click to evaluate for changes
+ * 2. The historical time window prior to the click to use as a baseline.
+ *
+ * The philosophy here is that charts are displayed with different granularities according to their
+ * overall time window. We select the change point and historical time windows inline with the
+ * overall time window.
+ *
+ * The algorithm for doing this is based on the typical granularities that exist in machine data.
+ *
+ * @param clickTime timestamp of the clicked log rate spike.
+ * @param minTime minimum timestamp of the time window to be analysed
+ * @param maxTime maximum timestamp of the time window to be analysed
+ * @returns WindowParameters
+ */
+export const getWindowParameters = (
+  clickTime: number,
+  minTime: number,
+  maxTime: number
+): WindowParameters => {
+  const totalWindow = maxTime - minTime;
+
+  // min deviation window
+  const minDeviationWindow = 10 * 60 * 1000; // 10min
+  const minBaselineWindow = 30 * 60 * 1000; // 30min
+  const minWindowGap = 5 * 60 * 1000; // 5min
+
+  // work out bounds
+  const deviationWindow = Math.max(totalWindow / 10, minDeviationWindow);
+  const baselineWindow = Math.max(totalWindow / 3.5, minBaselineWindow);
+  const windowGap = Math.max(totalWindow / 10, minWindowGap);
+
+  const deviationMin = clickTime - deviationWindow / 2;
+  const deviationMax = clickTime + deviationWindow / 2;
+
+  const baselineMax = deviationMin - windowGap;
+  const baselineMin = baselineMax - baselineWindow;
+
+  return {
+    baselineMin: Math.round(baselineMin),
+    baselineMax: Math.round(baselineMax),
+    deviationMin: Math.round(deviationMin),
+    deviationMax: Math.round(deviationMax),
+  };
+};

x-pack/packages/ml/aiops_utils/src/lib/stream_factory.ts

@@ -8,12 +8,13 @@
 import { Stream } from 'stream';
 import * as zlib from 'zlib';
 
-// TODO: Replace these with kbn packaged versions once we have those available to us.
-// At the moment imports from runtime plugins into packages are not supported.
-// import type { Headers } from '@kbn/core/server';
-
 import { acceptCompression } from './accept_compression';
 
+/**
+ * TODO: Replace these with kbn packaged versions once we have those available to us.
+ * At the moment imports from runtime plugins into packages are not supported.
+ * import type { Headers } from '@kbn/core/server';
+ */
 type Headers = Record<string, string | string[] | undefined>;
 
 // We need this otherwise Kibana server will crash with a 'ERR_METHOD_NOT_IMPLEMENTED' error.

x-pack/plugins/aiops/common/api/explain_log_rate_spikes.ts

@@ -7,28 +7,63 @@
 
 import { schema, TypeOf } from '@kbn/config-schema';
 
+import type { ChangePoint } from '../types';
+
 export const aiopsExplainLogRateSpikesSchema = schema.object({
+  start: schema.number(),
+  end: schema.number(),
+  kuery: schema.string(),
+  timeFieldName: schema.string(),
+  includeFrozen: schema.maybe(schema.boolean()),
+  /** Analysis selection time ranges */
+  baselineMin: schema.number(),
+  baselineMax: schema.number(),
+  deviationMin: schema.number(),
+  deviationMax: schema.number(),
   /** The index to query for log rate spikes */
   index: schema.string(),
 });
 
 export type AiopsExplainLogRateSpikesSchema = TypeOf<typeof aiopsExplainLogRateSpikesSchema>;
 
 export const API_ACTION_NAME = {
-  ADD_FIELDS: 'add_fields',
+  ADD_CHANGE_POINTS: 'add_change_points',
+  UPDATE_LOADING_STATE: 'update_loading_state',
 } as const;
 export type ApiActionName = typeof API_ACTION_NAME[keyof typeof API_ACTION_NAME];
 
-interface ApiActionAddFields {
-  type: typeof API_ACTION_NAME.ADD_FIELDS;
-  payload: string[];
+interface ApiActionAddChangePoints {
+  type: typeof API_ACTION_NAME.ADD_CHANGE_POINTS;
+  payload: ChangePoint[];
+}
+
+export function addChangePoints(
+  payload: ApiActionAddChangePoints['payload']
+): ApiActionAddChangePoints {
+  return {
+    type: API_ACTION_NAME.ADD_CHANGE_POINTS,
+    payload,
+  };
+}
+
+interface ApiActionUpdateLoadingState {
+  type: typeof API_ACTION_NAME.UPDATE_LOADING_STATE;
+  payload: {
+    ccsWarning: boolean;
+    loaded: number;
+    loadingState: string;
+  };
 }
 
-export function addFieldsAction(payload: string[]): ApiActionAddFields {
+export function updateLoadingStateAction(
+  payload: ApiActionUpdateLoadingState['payload']
+): ApiActionUpdateLoadingState {
   return {
-    type: API_ACTION_NAME.ADD_FIELDS,
+    type: API_ACTION_NAME.UPDATE_LOADING_STATE,
     payload,
   };
 }
 
-export type AiopsExplainLogRateSpikesApiAction = ApiActionAddFields;
+export type AiopsExplainLogRateSpikesApiAction =
+  | ApiActionAddChangePoints
+  | ApiActionUpdateLoadingState;

x-pack/plugins/aiops/common/api/stream_reducer.ts

@@ -5,14 +5,22 @@
  * 2.0.
  */
 
+import type { ChangePoint } from '../types';
+
 import { API_ACTION_NAME, AiopsExplainLogRateSpikesApiAction } from './explain_log_rate_spikes';
 
 interface StreamState {
-  fields: string[];
+  ccsWarning: boolean;
+  changePoints: ChangePoint[];
+  loaded: number;
+  loadingState: string;
 }
 
 export const initialState: StreamState = {
-  fields: [],
+  ccsWarning: false,
+  changePoints: [],
+  loaded: 0,
+  loadingState: '',
 };
 
 export function streamReducer(
@@ -24,10 +32,10 @@ export function streamReducer(
   }
 
   switch (action.type) {
-    case API_ACTION_NAME.ADD_FIELDS:
-      return {
-        fields: [...state.fields, ...action.payload],
-      };
+    case API_ACTION_NAME.ADD_CHANGE_POINTS:
+      return { ...state, changePoints: [...state.changePoints, ...action.payload] };
+    case API_ACTION_NAME.UPDATE_LOADING_STATE:
+      return { ...state, ...action.payload };
     default:
       return state;
   }

x-pack/plugins/aiops/common/constants.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const SPIKE_ANALYSIS_THRESHOLD = 0.02;

x-pack/plugins/aiops/common/index.ts

@@ -19,4 +19,4 @@ export const PLUGIN_NAME = 'AIOps';
  * This is an internal hard coded feature flag so we can easily turn on/off the
  * "Explain log rate spikes UI" during development until the first release.
  */
-export const AIOPS_ENABLED = false;
+export const AIOPS_ENABLED = true;

x-pack/plugins/aiops/common/types.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface FieldValuePair {
+  fieldName: string;
+  fieldValue: string;
+  isFallbackResult?: boolean;
+}
+
+export interface ChangePoint extends FieldValuePair {
+  doc_count: number;
+  bg_count: number;
+  score: number;
+  pValue: number | null;
+}