[ML] Transforms: Adds retention policy options to transform UI. (#91162)

Adds retention policy options to the transform UI.
elastic · Feb 16, 2021 · 073cd4d · 073cd4d
1 parent 3fe5228
commit 073cd4d
Show file tree

Hide file tree

Showing 14 changed files with 614 additions and 128 deletions.
diff --git a/x-pack/plugins/transform/common/api_schemas/transforms.ts b/x-pack/plugins/transform/common/api_schemas/transforms.ts
@@ -51,6 +51,13 @@ export type PivotConfig = TypeOf<typeof pivotSchema>;
 
 export type LatestFunctionConfig = TypeOf<typeof latestFunctionSchema>;
 
+export const retentionPolicySchema = schema.object({
+  time: schema.object({
+    field: schema.string(),
+    max_age: schema.string(),
+  }),
+});
+
 export const settingsSchema = schema.object({
   max_page_search_size: schema.maybe(schema.number()),
   // The default value is null, which disables throttling.
@@ -94,6 +101,7 @@ export const putTransformsRequestSchema = schema.object(
      * Latest and pivot are mutually exclusive, i.e. exactly one must be specified in the transform configuration
      */
     latest: schema.maybe(latestFunctionSchema),
+    retention_policy: schema.maybe(retentionPolicySchema),
     settings: schema.maybe(settingsSchema),
     source: sourceSchema,
     sync: schema.maybe(syncSchema),

diff --git a/x-pack/plugins/transform/common/api_schemas/update_transforms.ts b/x-pack/plugins/transform/common/api_schemas/update_transforms.ts
@@ -9,7 +9,7 @@ import { schema, TypeOf } from '@kbn/config-schema';
 
 import { TransformPivotConfig } from '../types/transform';
 
-import { settingsSchema, sourceSchema, syncSchema } from './transforms';
+import { retentionPolicySchema, settingsSchema, sourceSchema, syncSchema } from './transforms';
 
 // POST _transform/{transform_id}/_update
 export const postTransformsUpdateRequestSchema = schema.object({
@@ -22,6 +22,7 @@ export const postTransformsUpdateRequestSchema = schema.object({
     })
   ),
   frequency: schema.maybe(schema.string()),
+  retention_policy: schema.maybe(retentionPolicySchema),
   settings: schema.maybe(settingsSchema),
   source: schema.maybe(sourceSchema),
   sync: schema.maybe(syncSchema),

diff --git a/x-pack/plugins/transform/common/types/transform_stats.ts b/x-pack/plugins/transform/common/types/transform_stats.ts
@@ -33,6 +33,8 @@ export interface TransformStats {
     attributes: Record<string, any>;
   };
   stats: {
+    delete_time_in_ms: number;
+    documents_deleted: number;
     documents_indexed: number;
     documents_processed: number;
     index_failures: number;

diff --git a/x-pack/plugins/transform/public/app/common/request.test.ts b/x-pack/plugins/transform/public/app/common/request.test.ts
@@ -10,7 +10,7 @@ import { PIVOT_SUPPORTED_AGGS } from '../../../common/types/pivot_aggs';
 import { PivotGroupByConfig } from '../common';
 
 import { StepDefineExposedState } from '../sections/create_transform/components/step_define';
-import { StepDetailsExposedState } from '../sections/create_transform/components/step_details/step_details_form';
+import { StepDetailsExposedState } from '../sections/create_transform/components/step_details';
 
 import { PIVOT_SUPPORTED_GROUP_BY_AGGS } from './pivot_group_by';
 import { PivotAggsConfig } from './pivot_aggs';
@@ -174,6 +174,9 @@ describe('Transform: Common', () => {
       continuousModeDelay: 'the-continuous-mode-delay',
       createIndexPattern: false,
       isContinuousModeEnabled: false,
+      isRetentionPolicyEnabled: false,
+      retentionPolicyDateField: '',
+      retentionPolicyMaxAge: '',
       transformId: 'the-transform-id',
       transformDescription: 'the-transform-description',
       transformFrequency: '1m',

diff --git a/x-pack/plugins/transform/public/app/common/request.ts b/x-pack/plugins/transform/public/app/common/request.ts
@@ -19,7 +19,7 @@ import type {
 
 import type { SavedSearchQuery } from '../hooks/use_search_items';
 import type { StepDefineExposedState } from '../sections/create_transform/components/step_define';
-import type { StepDetailsExposedState } from '../sections/create_transform/components/step_details/step_details_form';
+import type { StepDetailsExposedState } from '../sections/create_transform/components/step_details';
 
 export interface SimpleQuery {
   query_string: {
@@ -119,6 +119,17 @@ export const getCreateTransformRequestBody = (
         },
       }
     : {}),
+  // conditionally add retention policy settings
+  ...(transformDetailsState.isRetentionPolicyEnabled
+    ? {
+        retention_policy: {
+          time: {
+            field: transformDetailsState.retentionPolicyDateField,
+            max_age: transformDetailsState.retentionPolicyMaxAge,
+          },
+        },
+      }
+    : {}),
   // conditionally add additional settings
   ...getCreateTransformSettingsRequestBody(transformDetailsState),
 });

diff --git a/x-pack/plugins/transform/public/app/common/validators.test.ts b/x-pack/plugins/transform/public/app/common/validators.test.ts
@@ -5,7 +5,12 @@
  * 2.0.
  */
 
-import { continuousModeDelayValidator, transformFrequencyValidator } from './validators';
+import {
+  continuousModeDelayValidator,
+  parseDuration,
+  retentionPolicyMaxAgeValidator,
+  transformFrequencyValidator,
+} from './validators';
 
 describe('continuousModeDelayValidator', () => {
   it('should allow 0 input without unit', () => {
@@ -29,6 +34,73 @@ describe('continuousModeDelayValidator', () => {
   });
 });
 
+describe('parseDuration', () => {
+  it('should return undefined when the input is not an integer and valid time unit.', () => {
+    expect(parseDuration('0')).toBe(undefined);
+    expect(parseDuration('0.1s')).toBe(undefined);
+    expect(parseDuration('1.1m')).toBe(undefined);
+    expect(parseDuration('10.1asdf')).toBe(undefined);
+  });
+
+  it('should return parsed data for valid time units nanos|micros|ms|s|m|h|d.', () => {
+    expect(parseDuration('1a')).toEqual(undefined);
+    expect(parseDuration('1nanos')).toEqual({
+      number: 1,
+      timeUnit: 'nanos',
+    });
+    expect(parseDuration('1micros')).toEqual({
+      number: 1,
+      timeUnit: 'micros',
+    });
+    expect(parseDuration('1ms')).toEqual({ number: 1, timeUnit: 'ms' });
+    expect(parseDuration('1s')).toEqual({ number: 1, timeUnit: 's' });
+    expect(parseDuration('1m')).toEqual({ number: 1, timeUnit: 'm' });
+    expect(parseDuration('1h')).toEqual({ number: 1, timeUnit: 'h' });
+    expect(parseDuration('1d')).toEqual({ number: 1, timeUnit: 'd' });
+  });
+});
+
+describe('retentionPolicyMaxAgeValidator', () => {
+  it('should fail when the input is not an integer and valid time unit.', () => {
+    expect(retentionPolicyMaxAgeValidator('0')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('0.1s')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('1.1m')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('10.1asdf')).toBe(false);
+  });
+
+  it('should only allow values equal or above 60s.', () => {
+    expect(retentionPolicyMaxAgeValidator('0nanos')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('59999999999nanos')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('60000000000nanos')).toBe(true);
+    expect(retentionPolicyMaxAgeValidator('60000000001nanos')).toBe(true);
+
+    expect(retentionPolicyMaxAgeValidator('0micros')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('59999999micros')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('60000000micros')).toBe(true);
+    expect(retentionPolicyMaxAgeValidator('60000001micros')).toBe(true);
+
+    expect(retentionPolicyMaxAgeValidator('0ms')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('59999ms')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('60000ms')).toBe(true);
+    expect(retentionPolicyMaxAgeValidator('60001ms')).toBe(true);
+
+    expect(retentionPolicyMaxAgeValidator('0s')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('1s')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('59s')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('60s')).toBe(true);
+    expect(retentionPolicyMaxAgeValidator('61s')).toBe(true);
+    expect(retentionPolicyMaxAgeValidator('10000s')).toBe(true);
+
+    expect(retentionPolicyMaxAgeValidator('0m')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('1m')).toBe(true);
+    expect(retentionPolicyMaxAgeValidator('100m')).toBe(true);
+
+    expect(retentionPolicyMaxAgeValidator('0h')).toBe(false);
+    expect(retentionPolicyMaxAgeValidator('1h')).toBe(true);
+    expect(retentionPolicyMaxAgeValidator('2h')).toBe(true);
+  });
+});
+
 describe('transformFrequencyValidator', () => {
   it('should fail when the input is not an integer and valid time unit.', () => {
     expect(transformFrequencyValidator('0')).toBe(false);

diff --git a/x-pack/plugins/transform/public/app/common/validators.ts b/x-pack/plugins/transform/public/app/common/validators.ts
@@ -5,6 +5,9 @@
  * 2.0.
  */
 
+const RETENTION_POLICY_MIN_AGE_SECONDS = 60;
+const TIME_UNITS = ['nanos', 'micros', 'ms', 's', 'm', 'h', 'd'];
+
 /**
  * Validates continuous mode time delay input.
  * Doesn't allow floating intervals.
@@ -14,6 +17,78 @@ export function continuousModeDelayValidator(value: string): boolean {
   return value.match(/^(0|\d*(nanos|micros|ms|s|m|h|d))$/) !== null;
 }
 
+/**
+ * Parses a duration uses a string format like `60s`.
+ * @param value User input value.
+ */
+export interface ParsedDuration {
+  number: number;
+  timeUnit: string;
+}
+export function parseDuration(value: string): ParsedDuration | undefined {
+  if (typeof value !== 'string' || value === null) {
+    return;
+  }
+
+  // split string by groups of numbers and letters
+  const regexStr = value.match(/[a-z]+|[^a-z]+/gi);
+
+  // only valid if one group of numbers and one group of letters
+  if (regexStr === null || (Array.isArray(regexStr) && regexStr.length !== 2)) {
+    return;
+  }
+
+  const number = +regexStr[0];
+  const timeUnit = regexStr[1];
+
+  // only valid if number is an integer
+  if (isNaN(number) || !Number.isInteger(number)) {
+    return;
+  }
+
+  if (!TIME_UNITS.includes(timeUnit)) {
+    return;
+  }
+
+  return { number, timeUnit };
+}
+
+export function isValidRetentionPolicyMaxAge({ number, timeUnit }: ParsedDuration): boolean {
+  // only valid if value is equal or more than 60s
+  // supported time units: https://www.elastic.co/guide/en/elasticsearch/reference/master/common-options.html#time-units
+  return (
+    (timeUnit === 'nanos' && number >= RETENTION_POLICY_MIN_AGE_SECONDS * 1000000000) ||
+    (timeUnit === 'micros' && number >= RETENTION_POLICY_MIN_AGE_SECONDS * 1000000) ||
+    (timeUnit === 'ms' && number >= RETENTION_POLICY_MIN_AGE_SECONDS * 1000) ||
+    (timeUnit === 's' && number >= RETENTION_POLICY_MIN_AGE_SECONDS) ||
+    ((timeUnit === 'm' || timeUnit === 'h' || timeUnit === 'd') && number >= 1)
+  );
+}
+
+/**
+ * Validates retention policy max age input.
+ * Doesn't allow floating intervals.
+ * @param value User input value. Minimum of 60s.
+ */
+export function retentionPolicyMaxAgeValidator(value: string): boolean {
+  const parsedValue = parseDuration(value);
+
+  if (parsedValue === undefined) {
+    return false;
+  }
+
+  return isValidRetentionPolicyMaxAge(parsedValue);
+}
+
+// only valid if value is up to 1 hour
+export function isValidFrequency({ number, timeUnit }: ParsedDuration): boolean {
+  return (
+    (timeUnit === 's' && number <= 3600) ||
+    (timeUnit === 'm' && number <= 60) ||
+    (timeUnit === 'h' && number === 1)
+  );
+}
+
 /**
  * Validates transform frequency input.
  * Allows time units of s/m/h only.
@@ -33,20 +108,15 @@ export const transformFrequencyValidator = (value: string): boolean => {
     return false;
   }
 
-  const valueNumber = +regexStr[0];
-  const valueTimeUnit = regexStr[1];
+  const number = +regexStr[0];
+  const timeUnit = regexStr[1];
 
   // only valid if number is an integer above 0
-  if (isNaN(valueNumber) || !Number.isInteger(valueNumber) || valueNumber === 0) {
+  if (isNaN(number) || !Number.isInteger(number) || number === 0) {
     return false;
   }
 
-  // only valid if value is up to 1 hour
-  return (
-    (valueTimeUnit === 's' && valueNumber <= 3600) ||
-    (valueTimeUnit === 'm' && valueNumber <= 60) ||
-    (valueTimeUnit === 'h' && valueNumber === 1)
-  );
+  return isValidFrequency({ number, timeUnit });
 };
 
 /**

diff --git a/.../plugins/transform/public/app/sections/create_transform/components/step_details/common.ts b/.../plugins/transform/public/app/sections/create_transform/components/step_details/common.ts
@@ -5,5 +5,95 @@
  * 2.0.
  */
 
+import type { TransformId, TransformPivotConfig } from '../../../../../../common/types/transform';
+
 export type EsIndexName = string;
 export type IndexPatternTitle = string;
+
+export interface StepDetailsExposedState {
+  continuousModeDateField: string;
+  continuousModeDelay: string;
+  createIndexPattern: boolean;
+  destinationIndex: EsIndexName;
+  isContinuousModeEnabled: boolean;
+  isRetentionPolicyEnabled: boolean;
+  retentionPolicyDateField: string;
+  retentionPolicyMaxAge: string;
+  touched: boolean;
+  transformId: TransformId;
+  transformDescription: string;
+  transformFrequency: string;
+  transformSettingsMaxPageSearchSize: number;
+  transformSettingsDocsPerSecond?: number;
+  valid: boolean;
+  indexPatternTimeField?: string | undefined;
+}
+
+const defaultContinuousModeDelay = '60s';
+const defaultTransformFrequency = '1m';
+const defaultTransformSettingsMaxPageSearchSize = 500;
+
+export function getDefaultStepDetailsState(): StepDetailsExposedState {
+  return {
+    continuousModeDateField: '',
+    continuousModeDelay: defaultContinuousModeDelay,
+    createIndexPattern: true,
+    isContinuousModeEnabled: false,
+    isRetentionPolicyEnabled: false,
+    retentionPolicyDateField: '',
+    retentionPolicyMaxAge: '',
+    transformId: '',
+    transformDescription: '',
+    transformFrequency: defaultTransformFrequency,
+    transformSettingsMaxPageSearchSize: defaultTransformSettingsMaxPageSearchSize,
+    destinationIndex: '',
+    touched: false,
+    valid: false,
+    indexPatternTimeField: undefined,
+  };
+}
+
+export function applyTransformConfigToDetailsState(
+  state: StepDetailsExposedState,
+  transformConfig?: TransformPivotConfig
+): StepDetailsExposedState {
+  // apply the transform configuration to wizard DETAILS state
+  if (transformConfig !== undefined) {
+    // Continuous mode
+    const continuousModeTime = transformConfig.sync?.time;
+    if (continuousModeTime !== undefined) {
+      state.continuousModeDateField = continuousModeTime.field;
+      state.continuousModeDelay = continuousModeTime?.delay ?? defaultContinuousModeDelay;
+      state.isContinuousModeEnabled = true;
+    }
+
+    // Description
+    if (transformConfig.description !== undefined) {
+      state.transformDescription = transformConfig.description;
+    }
+
+    // Frequency
+    if (transformConfig.frequency !== undefined) {
+      state.transformFrequency = transformConfig.frequency;
+    }
+
+    // Retention policy
+    const retentionPolicyTime = transformConfig.retention_policy?.time;
+    if (retentionPolicyTime !== undefined) {
+      state.retentionPolicyDateField = retentionPolicyTime.field;
+      state.retentionPolicyMaxAge = retentionPolicyTime.max_age;
+      state.isRetentionPolicyEnabled = true;
+    }
+
+    // Settings
+    if (transformConfig.settings) {
+      if (typeof transformConfig.settings?.max_page_search_size === 'number') {
+        state.transformSettingsMaxPageSearchSize = transformConfig.settings.max_page_search_size;
+      }
+      if (typeof transformConfig.settings?.docs_per_second === 'number') {
+        state.transformSettingsDocsPerSecond = transformConfig.settings.docs_per_second;
+      }
+    }
+  }
+  return state;
+}