Skip to content

Commit

Permalink
[ML] Move shared transform to common location.
Browse files Browse the repository at this point in the history
  • Loading branch information
walterra committed Sep 9, 2019
1 parent 17f74a7 commit 717d894
Show file tree
Hide file tree
Showing 26 changed files with 209 additions and 151 deletions.
9 changes: 9 additions & 0 deletions x-pack/legacy/plugins/ml/public/data_frame/common/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,15 @@ export {
IndexPattern,
REFRESH_TRANSFORM_LIST_STATE,
} from './transform';
export { DataFrameTransformListColumn, DataFrameTransformListRow } from './transform_list';
export {
getTransformProgress,
isCompletedBatchTransform,
isDataFrameTransformStats,
DataFrameTransformStats,
DATA_FRAME_MODE,
DATA_FRAME_TRANSFORM_STATE,
} from './transform_stats';
export { moveToDataFrameTransformList, moveToDataFrameWizard, moveToDiscover } from './navigation';
export {
getEsAggFromAggConfig,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { DataFrameTransformId, DataFrameTransformPivotConfig } from './transform';
import { DataFrameTransformStats } from './transform_stats';

// Used to pass on attribute names to table columns
export enum DataFrameTransformListColumn {
configDestIndex = 'config.dest.index',
configSourceIndex = 'config.source.index',
description = 'config.description',
id = 'id',
}

export interface DataFrameTransformListRow {
id: DataFrameTransformId;
config: DataFrameTransformPivotConfig;
mode?: string; // added property on client side to allow filtering by this field
stats: DataFrameTransformStats;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import mockDataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json';

import { DataFrameTransformListRow } from './transform_list';
import { isCompletedBatchTransform, DATA_FRAME_TRANSFORM_STATE } from './transform_stats';

describe('Data Frame: isCompletedBatchTransform()', () => {
test('isCompletedBatchTransform()', () => {
// check the transform config/state against the conditions
// that will be used by isCompletedBatchTransform()
// followed by a call to isCompletedBatchTransform() itself
const row = mockDataFrameTransformListRow as DataFrameTransformListRow;
expect(row.stats.checkpointing.last.checkpoint === 1).toBe(true);
expect(row.config.sync === undefined).toBe(true);
expect(row.stats.state === DATA_FRAME_TRANSFORM_STATE.STOPPED).toBe(true);
expect(isCompletedBatchTransform(mockDataFrameTransformListRow)).toBe(true);

// adapt the mock config to resemble a non-completed transform.
row.stats.checkpointing.last.checkpoint = 0;
expect(isCompletedBatchTransform(mockDataFrameTransformListRow)).toBe(false);
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { idx } from '@kbn/elastic-idx';

import { DataFrameTransformId } from './transform';
import { DataFrameTransformListRow } from './transform_list';

// reflects https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java#L243
export enum DATA_FRAME_TRANSFORM_STATE {
ABORTING = 'aborting',
FAILED = 'failed',
INDEXING = 'indexing',
STARTED = 'started',
STOPPED = 'stopped',
STOPPING = 'stopping',
}

export enum DATA_FRAME_MODE {
BATCH = 'batch',
CONTINUOUS = 'continuous',
}

export interface DataFrameTransformStats {
id: DataFrameTransformId;
checkpointing: {
last: {
checkpoint: number;
timestamp_millis?: number;
};
next?: {
checkpoint: number;
checkpoint_progress?: {
total_docs: number;
docs_remaining: number;
percent_complete: number;
};
};
operations_behind: number;
};
node?: {
id: string;
name: string;
ephemeral_id: string;
transport_address: string;
attributes: Record<string, any>;
};
stats: {
documents_indexed: number;
documents_processed: number;
index_failures: number;
index_time_in_ms: number;
index_total: number;
pages_processed: number;
search_failures: number;
search_time_in_ms: number;
search_total: number;
trigger_count: number;
};
reason?: string;
state: DATA_FRAME_TRANSFORM_STATE;
}

export function isDataFrameTransformStats(arg: any): arg is DataFrameTransformStats {
return (
typeof arg === 'object' &&
arg !== null &&
{}.hasOwnProperty.call(arg, 'state') &&
Object.values(DATA_FRAME_TRANSFORM_STATE).includes(arg.state)
);
}

export function getTransformProgress(item: DataFrameTransformListRow) {
if (isCompletedBatchTransform(item)) {
return 100;
}

const progress = idx(item, _ => _.stats.checkpointing.next.checkpoint_progress.percent_complete);

return progress !== undefined ? Math.round(progress) : undefined;
}

export function isCompletedBatchTransform(item: DataFrameTransformListRow) {
// If `checkpoint=1`, `sync` is missing from the config and state is stopped,
// then this is a completed batch data frame transform.
return (
item.stats.checkpointing.last.checkpoint === 1 &&
item.config.sync === undefined &&
item.stats.state === DATA_FRAME_TRANSFORM_STATE.STOPPED
);
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
import { shallow } from 'enzyme';
import React from 'react';

import { DataFrameTransformListRow } from './common';
import { DataFrameTransformListRow } from '../../../../common';
import { DeleteAction } from './action_delete';

import dataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json';
import dataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json';

describe('Data Frame: Transform List Actions <DeleteAction />', () => {
test('Minimal initialization', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import {
createPermissionFailureMessage,
} from '../../../../../privilege/check_privilege';

import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from './common';
import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from '../../../../common';

interface DeleteActionProps {
items: DataFrameTransformListRow[];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
import { shallow } from 'enzyme';
import React from 'react';

import { DataFrameTransformListRow } from './common';
import { DataFrameTransformListRow } from '../../../../common';
import { StartAction } from './action_start';

import dataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json';
import dataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json';

describe('Data Frame: Transform List Actions <StartAction />', () => {
test('Minimal initialization', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import {
DataFrameTransformListRow,
isCompletedBatchTransform,
DATA_FRAME_TRANSFORM_STATE,
} from './common';
} from '../../../../common';

interface StartActionProps {
items: DataFrameTransformListRow[];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
import { shallow } from 'enzyme';
import React from 'react';

import { DataFrameTransformListRow } from './common';
import { DataFrameTransformListRow } from '../../../../common';
import { StopAction } from './action_stop';

import dataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json';
import dataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json';

describe('Data Frame: Transform List Actions <StopAction />', () => {
test('Minimal initialization', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import React, { FC } from 'react';
import { i18n } from '@kbn/i18n';
import { EuiButtonEmpty, EuiToolTip } from '@elastic/eui';

import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from './common';
import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from '../../../../common';
import {
checkPermission,
createPermissionFailureMessage,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*/

import React from 'react';
import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from './common';
import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from '../../../../common';
import { StartAction } from './action_start';
import { StopAction } from './action_stop';
import { DeleteAction } from './action_delete';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,14 @@ import {
FieldDataColumnType,
} from '../../../../../../common/types/eui/in_memory_table';

import { DataFrameTransformId } from '../../../../common';
import {
getTransformProgress,
DATA_FRAME_TRANSFORM_STATE,
DataFrameTransformId,
DataFrameTransformListColumn,
DataFrameTransformListRow,
DataFrameTransformStats,
} from './common';
DATA_FRAME_TRANSFORM_STATE,
} from '../../../../common';
import { getActions } from './actions';

enum STATE_COLOR {
Expand Down Expand Up @@ -167,7 +167,7 @@ export const getColumns = (
},
{
name: i18n.translate('xpack.ml.dataframe.progress', { defaultMessage: 'Progress' }),
sortable: getTransformProgress || 0,
sortable: (item: DataFrameTransformListRow) => getTransformProgress(item) || 0,
truncateText: true,
render(item: DataFrameTransformListRow) {
const progress = getTransformProgress(item);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
* you may not use this file except in compliance with the Elastic License.
*/

import mockDataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json';
import mockDataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json';

import {
DataFrameTransformListRow,
isCompletedBatchTransform,
DATA_FRAME_TRANSFORM_STATE,
} from './common';
} from '../../../../common';

describe('Data Frame: isCompletedBatchTransform()', () => {
test('isCompletedBatchTransform()', () => {
Expand Down
Loading

0 comments on commit 717d894

Please sign in to comment.