diff --git a/x-pack/plugins/transform/common/api_schemas/transforms.ts b/x-pack/plugins/transform/common/api_schemas/transforms.ts index face319f141db..f9dedf0acb56a 100644 --- a/x-pack/plugins/transform/common/api_schemas/transforms.ts +++ b/x-pack/plugins/transform/common/api_schemas/transforms.ts @@ -51,6 +51,13 @@ export type PivotConfig = TypeOf; export type LatestFunctionConfig = TypeOf; +export const retentionPolicySchema = schema.object({ + time: schema.object({ + field: schema.string(), + max_age: schema.string(), + }), +}); + export const settingsSchema = schema.object({ max_page_search_size: schema.maybe(schema.number()), // The default value is null, which disables throttling. @@ -94,6 +101,7 @@ export const putTransformsRequestSchema = schema.object( * Latest and pivot are mutually exclusive, i.e. exactly one must be specified in the transform configuration */ latest: schema.maybe(latestFunctionSchema), + retention_policy: schema.maybe(retentionPolicySchema), settings: schema.maybe(settingsSchema), source: sourceSchema, sync: schema.maybe(syncSchema), diff --git a/x-pack/plugins/transform/common/api_schemas/update_transforms.ts b/x-pack/plugins/transform/common/api_schemas/update_transforms.ts index 4ff9780be1f5d..9bd4df5108049 100644 --- a/x-pack/plugins/transform/common/api_schemas/update_transforms.ts +++ b/x-pack/plugins/transform/common/api_schemas/update_transforms.ts @@ -9,7 +9,7 @@ import { schema, TypeOf } from '@kbn/config-schema'; import { TransformPivotConfig } from '../types/transform'; -import { settingsSchema, sourceSchema, syncSchema } from './transforms'; +import { retentionPolicySchema, settingsSchema, sourceSchema, syncSchema } from './transforms'; // POST _transform/{transform_id}/_update export const postTransformsUpdateRequestSchema = schema.object({ @@ -22,6 +22,7 @@ export const postTransformsUpdateRequestSchema = schema.object({ }) ), frequency: schema.maybe(schema.string()), + retention_policy: schema.maybe(retentionPolicySchema), settings: schema.maybe(settingsSchema), source: schema.maybe(sourceSchema), sync: schema.maybe(syncSchema), diff --git a/x-pack/plugins/transform/common/types/transform_stats.ts b/x-pack/plugins/transform/common/types/transform_stats.ts index d280f4ce3505c..f3b7000a424db 100644 --- a/x-pack/plugins/transform/common/types/transform_stats.ts +++ b/x-pack/plugins/transform/common/types/transform_stats.ts @@ -33,6 +33,8 @@ export interface TransformStats { attributes: Record; }; stats: { + delete_time_in_ms: number; + documents_deleted: number; documents_indexed: number; documents_processed: number; index_failures: number; diff --git a/x-pack/plugins/transform/public/app/common/request.test.ts b/x-pack/plugins/transform/public/app/common/request.test.ts index 778b2c24325f6..fa39419c254ba 100644 --- a/x-pack/plugins/transform/public/app/common/request.test.ts +++ b/x-pack/plugins/transform/public/app/common/request.test.ts @@ -10,7 +10,7 @@ import { PIVOT_SUPPORTED_AGGS } from '../../../common/types/pivot_aggs'; import { PivotGroupByConfig } from '../common'; import { StepDefineExposedState } from '../sections/create_transform/components/step_define'; -import { StepDetailsExposedState } from '../sections/create_transform/components/step_details/step_details_form'; +import { StepDetailsExposedState } from '../sections/create_transform/components/step_details'; import { PIVOT_SUPPORTED_GROUP_BY_AGGS } from './pivot_group_by'; import { PivotAggsConfig } from './pivot_aggs'; @@ -174,6 +174,9 @@ describe('Transform: Common', () => { continuousModeDelay: 'the-continuous-mode-delay', createIndexPattern: false, isContinuousModeEnabled: false, + isRetentionPolicyEnabled: false, + retentionPolicyDateField: '', + retentionPolicyMaxAge: '', transformId: 'the-transform-id', transformDescription: 'the-transform-description', transformFrequency: '1m', diff --git a/x-pack/plugins/transform/public/app/common/request.ts b/x-pack/plugins/transform/public/app/common/request.ts index 8e535e653a380..82faa80213816 100644 --- a/x-pack/plugins/transform/public/app/common/request.ts +++ b/x-pack/plugins/transform/public/app/common/request.ts @@ -19,7 +19,7 @@ import type { import type { SavedSearchQuery } from '../hooks/use_search_items'; import type { StepDefineExposedState } from '../sections/create_transform/components/step_define'; -import type { StepDetailsExposedState } from '../sections/create_transform/components/step_details/step_details_form'; +import type { StepDetailsExposedState } from '../sections/create_transform/components/step_details'; export interface SimpleQuery { query_string: { @@ -119,6 +119,17 @@ export const getCreateTransformRequestBody = ( }, } : {}), + // conditionally add retention policy settings + ...(transformDetailsState.isRetentionPolicyEnabled + ? { + retention_policy: { + time: { + field: transformDetailsState.retentionPolicyDateField, + max_age: transformDetailsState.retentionPolicyMaxAge, + }, + }, + } + : {}), // conditionally add additional settings ...getCreateTransformSettingsRequestBody(transformDetailsState), }); diff --git a/x-pack/plugins/transform/public/app/common/validators.test.ts b/x-pack/plugins/transform/public/app/common/validators.test.ts index 44126b8f3fa26..f48039052d203 100644 --- a/x-pack/plugins/transform/public/app/common/validators.test.ts +++ b/x-pack/plugins/transform/public/app/common/validators.test.ts @@ -5,7 +5,12 @@ * 2.0. */ -import { continuousModeDelayValidator, transformFrequencyValidator } from './validators'; +import { + continuousModeDelayValidator, + parseDuration, + retentionPolicyMaxAgeValidator, + transformFrequencyValidator, +} from './validators'; describe('continuousModeDelayValidator', () => { it('should allow 0 input without unit', () => { @@ -29,6 +34,73 @@ describe('continuousModeDelayValidator', () => { }); }); +describe('parseDuration', () => { + it('should return undefined when the input is not an integer and valid time unit.', () => { + expect(parseDuration('0')).toBe(undefined); + expect(parseDuration('0.1s')).toBe(undefined); + expect(parseDuration('1.1m')).toBe(undefined); + expect(parseDuration('10.1asdf')).toBe(undefined); + }); + + it('should return parsed data for valid time units nanos|micros|ms|s|m|h|d.', () => { + expect(parseDuration('1a')).toEqual(undefined); + expect(parseDuration('1nanos')).toEqual({ + number: 1, + timeUnit: 'nanos', + }); + expect(parseDuration('1micros')).toEqual({ + number: 1, + timeUnit: 'micros', + }); + expect(parseDuration('1ms')).toEqual({ number: 1, timeUnit: 'ms' }); + expect(parseDuration('1s')).toEqual({ number: 1, timeUnit: 's' }); + expect(parseDuration('1m')).toEqual({ number: 1, timeUnit: 'm' }); + expect(parseDuration('1h')).toEqual({ number: 1, timeUnit: 'h' }); + expect(parseDuration('1d')).toEqual({ number: 1, timeUnit: 'd' }); + }); +}); + +describe('retentionPolicyMaxAgeValidator', () => { + it('should fail when the input is not an integer and valid time unit.', () => { + expect(retentionPolicyMaxAgeValidator('0')).toBe(false); + expect(retentionPolicyMaxAgeValidator('0.1s')).toBe(false); + expect(retentionPolicyMaxAgeValidator('1.1m')).toBe(false); + expect(retentionPolicyMaxAgeValidator('10.1asdf')).toBe(false); + }); + + it('should only allow values equal or above 60s.', () => { + expect(retentionPolicyMaxAgeValidator('0nanos')).toBe(false); + expect(retentionPolicyMaxAgeValidator('59999999999nanos')).toBe(false); + expect(retentionPolicyMaxAgeValidator('60000000000nanos')).toBe(true); + expect(retentionPolicyMaxAgeValidator('60000000001nanos')).toBe(true); + + expect(retentionPolicyMaxAgeValidator('0micros')).toBe(false); + expect(retentionPolicyMaxAgeValidator('59999999micros')).toBe(false); + expect(retentionPolicyMaxAgeValidator('60000000micros')).toBe(true); + expect(retentionPolicyMaxAgeValidator('60000001micros')).toBe(true); + + expect(retentionPolicyMaxAgeValidator('0ms')).toBe(false); + expect(retentionPolicyMaxAgeValidator('59999ms')).toBe(false); + expect(retentionPolicyMaxAgeValidator('60000ms')).toBe(true); + expect(retentionPolicyMaxAgeValidator('60001ms')).toBe(true); + + expect(retentionPolicyMaxAgeValidator('0s')).toBe(false); + expect(retentionPolicyMaxAgeValidator('1s')).toBe(false); + expect(retentionPolicyMaxAgeValidator('59s')).toBe(false); + expect(retentionPolicyMaxAgeValidator('60s')).toBe(true); + expect(retentionPolicyMaxAgeValidator('61s')).toBe(true); + expect(retentionPolicyMaxAgeValidator('10000s')).toBe(true); + + expect(retentionPolicyMaxAgeValidator('0m')).toBe(false); + expect(retentionPolicyMaxAgeValidator('1m')).toBe(true); + expect(retentionPolicyMaxAgeValidator('100m')).toBe(true); + + expect(retentionPolicyMaxAgeValidator('0h')).toBe(false); + expect(retentionPolicyMaxAgeValidator('1h')).toBe(true); + expect(retentionPolicyMaxAgeValidator('2h')).toBe(true); + }); +}); + describe('transformFrequencyValidator', () => { it('should fail when the input is not an integer and valid time unit.', () => { expect(transformFrequencyValidator('0')).toBe(false); diff --git a/x-pack/plugins/transform/public/app/common/validators.ts b/x-pack/plugins/transform/public/app/common/validators.ts index 125a7cd714aa5..065a6b4d1c0ca 100644 --- a/x-pack/plugins/transform/public/app/common/validators.ts +++ b/x-pack/plugins/transform/public/app/common/validators.ts @@ -5,6 +5,9 @@ * 2.0. */ +const RETENTION_POLICY_MIN_AGE_SECONDS = 60; +const TIME_UNITS = ['nanos', 'micros', 'ms', 's', 'm', 'h', 'd']; + /** * Validates continuous mode time delay input. * Doesn't allow floating intervals. @@ -14,6 +17,78 @@ export function continuousModeDelayValidator(value: string): boolean { return value.match(/^(0|\d*(nanos|micros|ms|s|m|h|d))$/) !== null; } +/** + * Parses a duration uses a string format like `60s`. + * @param value User input value. + */ +export interface ParsedDuration { + number: number; + timeUnit: string; +} +export function parseDuration(value: string): ParsedDuration | undefined { + if (typeof value !== 'string' || value === null) { + return; + } + + // split string by groups of numbers and letters + const regexStr = value.match(/[a-z]+|[^a-z]+/gi); + + // only valid if one group of numbers and one group of letters + if (regexStr === null || (Array.isArray(regexStr) && regexStr.length !== 2)) { + return; + } + + const number = +regexStr[0]; + const timeUnit = regexStr[1]; + + // only valid if number is an integer + if (isNaN(number) || !Number.isInteger(number)) { + return; + } + + if (!TIME_UNITS.includes(timeUnit)) { + return; + } + + return { number, timeUnit }; +} + +export function isValidRetentionPolicyMaxAge({ number, timeUnit }: ParsedDuration): boolean { + // only valid if value is equal or more than 60s + // supported time units: https://www.elastic.co/guide/en/elasticsearch/reference/master/common-options.html#time-units + return ( + (timeUnit === 'nanos' && number >= RETENTION_POLICY_MIN_AGE_SECONDS * 1000000000) || + (timeUnit === 'micros' && number >= RETENTION_POLICY_MIN_AGE_SECONDS * 1000000) || + (timeUnit === 'ms' && number >= RETENTION_POLICY_MIN_AGE_SECONDS * 1000) || + (timeUnit === 's' && number >= RETENTION_POLICY_MIN_AGE_SECONDS) || + ((timeUnit === 'm' || timeUnit === 'h' || timeUnit === 'd') && number >= 1) + ); +} + +/** + * Validates retention policy max age input. + * Doesn't allow floating intervals. + * @param value User input value. Minimum of 60s. + */ +export function retentionPolicyMaxAgeValidator(value: string): boolean { + const parsedValue = parseDuration(value); + + if (parsedValue === undefined) { + return false; + } + + return isValidRetentionPolicyMaxAge(parsedValue); +} + +// only valid if value is up to 1 hour +export function isValidFrequency({ number, timeUnit }: ParsedDuration): boolean { + return ( + (timeUnit === 's' && number <= 3600) || + (timeUnit === 'm' && number <= 60) || + (timeUnit === 'h' && number === 1) + ); +} + /** * Validates transform frequency input. * Allows time units of s/m/h only. @@ -33,20 +108,15 @@ export const transformFrequencyValidator = (value: string): boolean => { return false; } - const valueNumber = +regexStr[0]; - const valueTimeUnit = regexStr[1]; + const number = +regexStr[0]; + const timeUnit = regexStr[1]; // only valid if number is an integer above 0 - if (isNaN(valueNumber) || !Number.isInteger(valueNumber) || valueNumber === 0) { + if (isNaN(number) || !Number.isInteger(number) || number === 0) { return false; } - // only valid if value is up to 1 hour - return ( - (valueTimeUnit === 's' && valueNumber <= 3600) || - (valueTimeUnit === 'm' && valueNumber <= 60) || - (valueTimeUnit === 'h' && valueNumber === 1) - ); + return isValidFrequency({ number, timeUnit }); }; /** diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/common.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/common.ts index 3b8df3b977fff..fbe32e9bea12f 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/common.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/common.ts @@ -5,5 +5,95 @@ * 2.0. */ +import type { TransformId, TransformPivotConfig } from '../../../../../../common/types/transform'; + export type EsIndexName = string; export type IndexPatternTitle = string; + +export interface StepDetailsExposedState { + continuousModeDateField: string; + continuousModeDelay: string; + createIndexPattern: boolean; + destinationIndex: EsIndexName; + isContinuousModeEnabled: boolean; + isRetentionPolicyEnabled: boolean; + retentionPolicyDateField: string; + retentionPolicyMaxAge: string; + touched: boolean; + transformId: TransformId; + transformDescription: string; + transformFrequency: string; + transformSettingsMaxPageSearchSize: number; + transformSettingsDocsPerSecond?: number; + valid: boolean; + indexPatternTimeField?: string | undefined; +} + +const defaultContinuousModeDelay = '60s'; +const defaultTransformFrequency = '1m'; +const defaultTransformSettingsMaxPageSearchSize = 500; + +export function getDefaultStepDetailsState(): StepDetailsExposedState { + return { + continuousModeDateField: '', + continuousModeDelay: defaultContinuousModeDelay, + createIndexPattern: true, + isContinuousModeEnabled: false, + isRetentionPolicyEnabled: false, + retentionPolicyDateField: '', + retentionPolicyMaxAge: '', + transformId: '', + transformDescription: '', + transformFrequency: defaultTransformFrequency, + transformSettingsMaxPageSearchSize: defaultTransformSettingsMaxPageSearchSize, + destinationIndex: '', + touched: false, + valid: false, + indexPatternTimeField: undefined, + }; +} + +export function applyTransformConfigToDetailsState( + state: StepDetailsExposedState, + transformConfig?: TransformPivotConfig +): StepDetailsExposedState { + // apply the transform configuration to wizard DETAILS state + if (transformConfig !== undefined) { + // Continuous mode + const continuousModeTime = transformConfig.sync?.time; + if (continuousModeTime !== undefined) { + state.continuousModeDateField = continuousModeTime.field; + state.continuousModeDelay = continuousModeTime?.delay ?? defaultContinuousModeDelay; + state.isContinuousModeEnabled = true; + } + + // Description + if (transformConfig.description !== undefined) { + state.transformDescription = transformConfig.description; + } + + // Frequency + if (transformConfig.frequency !== undefined) { + state.transformFrequency = transformConfig.frequency; + } + + // Retention policy + const retentionPolicyTime = transformConfig.retention_policy?.time; + if (retentionPolicyTime !== undefined) { + state.retentionPolicyDateField = retentionPolicyTime.field; + state.retentionPolicyMaxAge = retentionPolicyTime.max_age; + state.isRetentionPolicyEnabled = true; + } + + // Settings + if (transformConfig.settings) { + if (typeof transformConfig.settings?.max_page_search_size === 'number') { + state.transformSettingsMaxPageSearchSize = transformConfig.settings.max_page_search_size; + } + if (typeof transformConfig.settings?.docs_per_second === 'number') { + state.transformSettingsDocsPerSecond = transformConfig.settings.docs_per_second; + } + } + } + return state; +} diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/index.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/index.ts index 4b01e0c3746ec..bbc4b42e1b236 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/index.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/index.ts @@ -8,6 +8,7 @@ export { applyTransformConfigToDetailsState, getDefaultStepDetailsState, - StepDetailsForm, -} from './step_details_form'; + StepDetailsExposedState, +} from './common'; +export { StepDetailsForm } from './step_details_form'; export { StepDetailsSummary } from './step_details_summary'; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx index 100c37d911fa0..1fa16e26565b6 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { Fragment, FC, useEffect, useState } from 'react'; +import React, { FC, useEffect, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -49,87 +49,23 @@ import { import { EsIndexName, IndexPatternTitle } from './common'; import { continuousModeDelayValidator, + retentionPolicyMaxAgeValidator, transformFrequencyValidator, transformSettingsMaxPageSearchSizeValidator, } from '../../../../common/validators'; import { StepDefineExposedState } from '../step_define/common'; import { TRANSFORM_FUNCTION } from '../../../../../../common/constants'; -export interface StepDetailsExposedState { - continuousModeDateField: string; - continuousModeDelay: string; - createIndexPattern: boolean; - destinationIndex: EsIndexName; - isContinuousModeEnabled: boolean; - touched: boolean; - transformId: TransformId; - transformDescription: string; - transformFrequency: string; - transformSettingsMaxPageSearchSize: number; - transformSettingsDocsPerSecond?: number; - valid: boolean; - indexPatternTimeField?: string | undefined; -} - -const defaultContinuousModeDelay = '60s'; -const defaultTransformFrequency = '1m'; -const defaultTransformSettingsMaxPageSearchSize = 500; - -export function getDefaultStepDetailsState(): StepDetailsExposedState { - return { - continuousModeDateField: '', - continuousModeDelay: defaultContinuousModeDelay, - createIndexPattern: true, - isContinuousModeEnabled: false, - transformId: '', - transformDescription: '', - transformFrequency: defaultTransformFrequency, - transformSettingsMaxPageSearchSize: defaultTransformSettingsMaxPageSearchSize, - destinationIndex: '', - touched: false, - valid: false, - indexPatternTimeField: undefined, - }; -} +import { getDefaultStepDetailsState, StepDetailsExposedState } from './common'; -export function applyTransformConfigToDetailsState( - state: StepDetailsExposedState, - transformConfig?: TransformPivotConfig -): StepDetailsExposedState { - // apply the transform configuration to wizard DETAILS state - if (transformConfig !== undefined) { - const time = transformConfig.sync?.time; - if (time !== undefined) { - state.continuousModeDateField = time.field; - state.continuousModeDelay = time?.delay ?? defaultContinuousModeDelay; - state.isContinuousModeEnabled = true; - } - if (transformConfig.description !== undefined) { - state.transformDescription = transformConfig.description; - } - if (transformConfig.frequency !== undefined) { - state.transformFrequency = transformConfig.frequency; - } - if (transformConfig.settings) { - if (typeof transformConfig.settings?.max_page_search_size === 'number') { - state.transformSettingsMaxPageSearchSize = transformConfig.settings.max_page_search_size; - } - if (typeof transformConfig.settings?.docs_per_second === 'number') { - state.transformSettingsDocsPerSecond = transformConfig.settings.docs_per_second; - } - } - } - return state; -} - -interface Props { +interface StepDetailsFormProps { overrides?: StepDetailsExposedState; onChange(s: StepDetailsExposedState): void; searchItems: SearchItems; stepDefineState: StepDefineExposedState; } -export const StepDetailsForm: FC = React.memo( +export const StepDetailsForm: FC = React.memo( ({ overrides = {}, onChange, searchItems, stepDefineState }) => { const deps = useAppDependencies(); const toastNotifications = useToastNotifications(); @@ -171,11 +107,6 @@ export const StepDetailsForm: FC = React.memo( [setIndexPatternTimeField, indexPatternAvailableTimeFields] ); - // Continuous mode state - const [isContinuousModeEnabled, setContinuousModeEnabled] = useState( - defaults.isContinuousModeEnabled - ); - const api = useApi(); // fetch existing transform IDs and indices once for form validation @@ -268,13 +199,41 @@ export const StepDetailsForm: FC = React.memo( .filter((f) => f.type === KBN_FIELD_TYPES.DATE) .map((f) => f.name) .sort(); + + // Continuous Mode const isContinuousModeAvailable = dateFieldNames.length > 0; + const [isContinuousModeEnabled, setContinuousModeEnabled] = useState( + defaults.isContinuousModeEnabled + ); const [continuousModeDateField, setContinuousModeDateField] = useState( isContinuousModeAvailable ? dateFieldNames[0] : '' ); const [continuousModeDelay, setContinuousModeDelay] = useState(defaults.continuousModeDelay); const isContinuousModeDelayValid = continuousModeDelayValidator(continuousModeDelay); + // Retention Policy + const isRetentionPolicyAvailable = dateFieldNames.length > 0; + const [isRetentionPolicyEnabled, setRetentionPolicyEnabled] = useState( + defaults.isRetentionPolicyEnabled + ); + const [retentionPolicyDateField, setRetentionPolicyDateField] = useState( + isRetentionPolicyAvailable ? dateFieldNames[0] : '' + ); + const [retentionPolicyMaxAge, setRetentionPolicyMaxAge] = useState( + defaults.retentionPolicyMaxAge + ); + const retentionPolicyMaxAgeEmpty = retentionPolicyMaxAge === ''; + const isRetentionPolicyMaxAgeValid = retentionPolicyMaxAgeValidator(retentionPolicyMaxAge); + + // Reset retention policy settings when the user disables the whole option + useEffect(() => { + if (!isRetentionPolicyEnabled) { + setRetentionPolicyDateField(isRetentionPolicyAvailable ? dateFieldNames[0] : ''); + setRetentionPolicyMaxAge(''); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [isRetentionPolicyEnabled]); + const transformIdExists = transformIds.some((id) => transformId === id); const transformIdEmpty = transformId === ''; const transformIdValid = isTransformIdValid(transformId); @@ -305,7 +264,13 @@ export const StepDetailsForm: FC = React.memo( !indexNameEmpty && indexNameValid && (!indexPatternTitleExists || !createIndexPattern) && - (!isContinuousModeAvailable || (isContinuousModeAvailable && isContinuousModeDelayValid)); + (!isContinuousModeAvailable || (isContinuousModeAvailable && isContinuousModeDelayValid)) && + (!isRetentionPolicyAvailable || + !isRetentionPolicyEnabled || + (isRetentionPolicyAvailable && + isRetentionPolicyEnabled && + !retentionPolicyMaxAgeEmpty && + isRetentionPolicyMaxAgeValid)); // expose state to wizard useEffect(() => { @@ -314,6 +279,9 @@ export const StepDetailsForm: FC = React.memo( continuousModeDelay, createIndexPattern, isContinuousModeEnabled, + isRetentionPolicyEnabled, + retentionPolicyDateField, + retentionPolicyMaxAge, transformId, transformDescription, transformFrequency, @@ -331,6 +299,9 @@ export const StepDetailsForm: FC = React.memo( continuousModeDelay, createIndexPattern, isContinuousModeEnabled, + isRetentionPolicyEnabled, + retentionPolicyDateField, + retentionPolicyMaxAge, transformId, transformDescription, transformFrequency, @@ -417,7 +388,7 @@ export const StepDetailsForm: FC = React.memo( error={ !indexNameEmpty && !indexNameValid && [ - + <> {i18n.translate('xpack.transform.stepDetailsForm.destinationIndexInvalidError', { defaultMessage: 'Invalid destination index name.', })} @@ -430,7 +401,7 @@ export const StepDetailsForm: FC = React.memo( } )} - , + , ] } > @@ -502,6 +473,8 @@ export const StepDetailsForm: FC = React.memo( onTimeFieldChanged={onTimeFieldChanged} /> )} + + {/* Continuous mode */} = React.memo( /> {isContinuousModeEnabled && ( - + <> = React.memo( )} > setContinuousModeDelay(e.target.value)} aria-label={i18n.translate( @@ -580,7 +559,100 @@ export const StepDetailsForm: FC = React.memo( data-test-subj="transformContinuousDelayInput" /> - + + )} + + {/* Retention policy */} + + setRetentionPolicyEnabled(!isRetentionPolicyEnabled)} + disabled={isRetentionPolicyAvailable === false} + data-test-subj="transformRetentionPolicySwitch" + /> + + {isRetentionPolicyEnabled && ( + <> + + ({ text }))} + value={retentionPolicyDateField} + onChange={(e) => setRetentionPolicyDateField(e.target.value)} + data-test-subj="transformRetentionPolicyDateFieldSelect" + /> + + + setRetentionPolicyMaxAge(e.target.value)} + aria-label={i18n.translate( + 'xpack.transform.stepDetailsForm.retentionPolicyMaxAgeAriaLabel', + { + defaultMessage: 'Choose a max age.', + } + )} + isInvalid={!retentionPolicyMaxAgeEmpty && !isRetentionPolicyMaxAgeValid} + data-test-subj="transformRetentionPolicyMaxAgeInput" + /> + + )} diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_summary.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_summary.tsx index 7fb9f8ba06c05..f39132da81985 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_summary.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_summary.tsx @@ -11,13 +11,16 @@ import { i18n } from '@kbn/i18n'; import { EuiAccordion, EuiFormRow, EuiSpacer } from '@elastic/eui'; -import { StepDetailsExposedState } from './step_details_form'; +import { StepDetailsExposedState } from './common'; export const StepDetailsSummary: FC = React.memo((props) => { const { continuousModeDateField, createIndexPattern, isContinuousModeEnabled, + isRetentionPolicyEnabled, + retentionPolicyDateField, + retentionPolicyMaxAge, transformId, transformDescription, transformFrequency, @@ -85,6 +88,28 @@ export const StepDetailsSummary: FC = React.memo((props )} + {isRetentionPolicyEnabled && ( + <> + + {retentionPolicyDateField} + + + {retentionPolicyMaxAge} + + + )} + = ({ + + dispatch({ field: 'retentionPolicyField', value })} + value={formFields.retentionPolicyField.value} + /> + + dispatch({ field: 'retentionPolicyMaxAge', value })} + value={formFields.retentionPolicyMaxAge.value} + /> + + + + { }); }); -describe('Transfom: stringValidator()', () => { +describe('Transform: stringValidator()', () => { it('should allow an empty string for optional fields', () => { expect(stringValidator('')).toHaveLength(0); }); @@ -270,6 +271,43 @@ describe('Transform: frequencyValidator()', () => { }); }); +describe('Transform: retentionPolicyMaxAgeValidator()', () => { + const transformRetentionPolicyMaxAgeValidator = (arg: string) => + retentionPolicyMaxAgeValidator(arg).length === 0; + + it('should only allow values equal or above 60s.', () => { + expect(transformRetentionPolicyMaxAgeValidator('0nanos')).toBe(false); + expect(transformRetentionPolicyMaxAgeValidator('59999999999nanos')).toBe(false); + expect(transformRetentionPolicyMaxAgeValidator('60000000000nanos')).toBe(true); + expect(transformRetentionPolicyMaxAgeValidator('60000000001nanos')).toBe(true); + + expect(transformRetentionPolicyMaxAgeValidator('0micros')).toBe(false); + expect(transformRetentionPolicyMaxAgeValidator('59999999micros')).toBe(false); + expect(transformRetentionPolicyMaxAgeValidator('60000000micros')).toBe(true); + expect(transformRetentionPolicyMaxAgeValidator('60000001micros')).toBe(true); + + expect(transformRetentionPolicyMaxAgeValidator('0ms')).toBe(false); + expect(transformRetentionPolicyMaxAgeValidator('59999ms')).toBe(false); + expect(transformRetentionPolicyMaxAgeValidator('60000ms')).toBe(true); + expect(transformRetentionPolicyMaxAgeValidator('60001ms')).toBe(true); + + expect(transformRetentionPolicyMaxAgeValidator('0s')).toBe(false); + expect(transformRetentionPolicyMaxAgeValidator('1s')).toBe(false); + expect(transformRetentionPolicyMaxAgeValidator('59s')).toBe(false); + expect(transformRetentionPolicyMaxAgeValidator('60s')).toBe(true); + expect(transformRetentionPolicyMaxAgeValidator('61s')).toBe(true); + expect(transformRetentionPolicyMaxAgeValidator('10000s')).toBe(true); + + expect(transformRetentionPolicyMaxAgeValidator('0m')).toBe(false); + expect(transformRetentionPolicyMaxAgeValidator('1m')).toBe(true); + expect(transformRetentionPolicyMaxAgeValidator('100m')).toBe(true); + + expect(transformRetentionPolicyMaxAgeValidator('0h')).toBe(false); + expect(transformRetentionPolicyMaxAgeValidator('1h')).toBe(true); + expect(transformRetentionPolicyMaxAgeValidator('2h')).toBe(true); + }); +}); + describe('Transform: integerAboveZeroValidator()', () => { it('should only allow integers above zero', () => { // integerAboveZeroValidator() returns an array of error messages so diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.ts b/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.ts index a86a9cd801262..6680495bdab91 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.ts +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.ts @@ -16,6 +16,12 @@ import { PostTransformsUpdateRequestSchema } from '../../../../../../common/api_ import { TransformConfigUnion } from '../../../../../../common/types/transform'; import { getNestedProperty, setNestedProperty } from '../../../../../../common/utils/object_utils'; +import { + isValidFrequency, + isValidRetentionPolicyMaxAge, + ParsedDuration, +} from '../../../../common/validators'; + // This custom hook uses nested reducers to provide a generic framework to manage form state // and apply it to a final possibly nested configuration object suitable for passing on // directly to an API call. For now this is only used for the transform edit form. @@ -25,21 +31,23 @@ import { getNestedProperty, setNestedProperty } from '../../../../../../common/u // The outer most level reducer defines a flat structure of names for form fields. // This is a flat structure regardless of whether the final request object will be nested. // For example, `destinationIndex` and `destinationPipeline` will later be nested under `dest`. -interface EditTransformFlyoutFieldsState { - [key: string]: FormField; - description: FormField; - destinationIndex: FormField; - destinationPipeline: FormField; - frequency: FormField; - docsPerSecond: FormField; -} +type EditTransformFormFields = + | 'description' + | 'destinationIndex' + | 'destinationPipeline' + | 'frequency' + | 'docsPerSecond' + | 'maxPageSearchSize' + | 'retentionPolicyField' + | 'retentionPolicyMaxAge'; +type EditTransformFlyoutFieldsState = Record; // The inner reducers apply validation based on supplied attributes of each field. export interface FormField { formFieldName: string; configFieldName: string; defaultValue: string; - dependsOn: string[]; + dependsOn: EditTransformFormFields[]; errorMessages: string[]; isNullable: boolean; isOptional: boolean; @@ -122,14 +130,7 @@ export const stringValidator: Validator = (value, isOptional = true) => { return []; }; -// Only allow frequencies in the form of 1s/1h etc. -const frequencyNotValidErrorMessage = i18n.translate( - 'xpack.transform.transformList.editFlyoutFormFrequencyNotValidErrorMessage', - { - defaultMessage: 'The frequency value is not valid.', - } -); -export const frequencyValidator: Validator = (arg) => { +function parseDurationAboveZero(arg: any, errorMessage: string): ParsedDuration | string[] { if (typeof arg !== 'string' || arg === null) { return [stringNotValidErrorMessage]; } @@ -142,20 +143,49 @@ export const frequencyValidator: Validator = (arg) => { return [frequencyNotValidErrorMessage]; } - const valueNumber = +regexStr[0]; - const valueTimeUnit = regexStr[1]; + const number = +regexStr[0]; + const timeUnit = regexStr[1]; // only valid if number is an integer above 0 - if (isNaN(valueNumber) || !Number.isInteger(valueNumber) || valueNumber === 0) { + if (isNaN(number) || !Number.isInteger(number) || number === 0) { return [frequencyNotValidErrorMessage]; } - // only valid if value is up to 1 hour - return (valueTimeUnit === 's' && valueNumber <= 3600) || - (valueTimeUnit === 'm' && valueNumber <= 60) || - (valueTimeUnit === 'h' && valueNumber === 1) - ? [] - : [frequencyNotValidErrorMessage]; + return { number, timeUnit }; +} + +// Only allow frequencies in the form of 1s/1h etc. +const frequencyNotValidErrorMessage = i18n.translate( + 'xpack.transform.transformList.editFlyoutFormFrequencyNotValidErrorMessage', + { + defaultMessage: 'The frequency value is not valid.', + } +); +export const frequencyValidator: Validator = (arg) => { + const parsedArg = parseDurationAboveZero(arg, frequencyNotValidErrorMessage); + + if (Array.isArray(parsedArg)) { + return parsedArg; + } + + return isValidFrequency(parsedArg) ? [] : [frequencyNotValidErrorMessage]; +}; + +// Retention policy max age validator +const retentionPolicyMaxAgeNotValidErrorMessage = i18n.translate( + 'xpack.transform.transformList.editFlyoutFormRetentionPolicyMaxAgeNotValidErrorMessage', + { + defaultMessage: 'Invalid max age format. Minimum of 60s required.', + } +); +export const retentionPolicyMaxAgeValidator: Validator = (arg) => { + const parsedArg = parseDurationAboveZero(arg, retentionPolicyMaxAgeNotValidErrorMessage); + + if (Array.isArray(parsedArg)) { + return parsedArg; + } + + return isValidRetentionPolicyMaxAge(parsedArg) ? [] : [retentionPolicyMaxAgeNotValidErrorMessage]; }; const validate = { @@ -163,10 +193,11 @@ const validate = { frequency: frequencyValidator, integerAboveZero: integerAboveZeroValidator, integerRange10To10000: integerRange10To10000Validator, + retentionPolicyMaxAge: retentionPolicyMaxAgeValidator, } as const; export const initializeField = ( - formFieldName: string, + formFieldName: EditTransformFormFields, configFieldName: string, config: TransformConfigUnion, overloads?: Partial @@ -199,7 +230,7 @@ export interface EditTransformFlyoutState { // This is not a redux type action, // since for now we only have one action type. interface Action { - field: keyof EditTransformFlyoutFieldsState; + field: EditTransformFormFields; value: string; } @@ -207,7 +238,7 @@ interface Action { // of the expected final configuration request object. // Considers options like if a value is nullable or optional. const getUpdateValue = ( - attribute: keyof EditTransformFlyoutFieldsState, + attribute: EditTransformFormFields, config: TransformConfigUnion, formState: EditTransformFlyoutFieldsState, enforceFormValue = false @@ -251,7 +282,7 @@ export const applyFormFieldsToTransformConfig = ( ): PostTransformsUpdateRequestSchema => // Iterates over all form fields and only if necessary applies them to // the request object used for updating the transform. - Object.keys(formState).reduce( + (Object.keys(formState) as EditTransformFormFields[]).reduce( (updateConfig, field) => merge({ ...updateConfig }, getUpdateValue(field, config, formState)), {} ); @@ -292,6 +323,25 @@ export const getDefaultState = (config: TransformConfigUnion): EditTransformFlyo valueParser: (v) => +v, } ), + + // retention_policy.* + retentionPolicyField: initializeField( + 'retentionPolicyField', + 'retention_policy.time.field', + config, + { dependsOn: ['retentionPolicyMaxAge'], isNullable: false, isOptional: true } + ), + retentionPolicyMaxAge: initializeField( + 'retentionPolicyMaxAge', + 'retention_policy.time.max_age', + config, + { + dependsOn: ['retentionPolicyField'], + isNullable: false, + isOptional: true, + validator: 'retentionPolicyMaxAge', + } + ), }, isFormTouched: false, isFormValid: true, @@ -300,7 +350,10 @@ export const getDefaultState = (config: TransformConfigUnion): EditTransformFlyo // Checks each form field for error messages to return // if the overall form is valid or not. const isFormValid = (fieldsState: EditTransformFlyoutFieldsState) => - Object.keys(fieldsState).reduce((p, c) => p && fieldsState[c].errorMessages.length === 0, true); + (Object.keys(fieldsState) as EditTransformFormFields[]).reduce( + (p, c) => p && fieldsState[c].errorMessages.length === 0, + true + ); // Updates a form field with its new value, // runs validation and populates