= (
coreSetup: CoreSetup,
@@ -49,6 +50,7 @@ export type ServiceFromProvider = P extends CanvasServiceProvider ?
export const services = {
notify: new CanvasServiceProvider(notifyServiceFactory),
+ platform: new CanvasServiceProvider(platformServiceFactory),
};
export interface CanvasServices {
@@ -70,4 +72,4 @@ export const stopServices = () => {
Object.entries(services).forEach(([key, provider]) => provider.stop());
};
-export const { notify: notifyService } = services;
+export const { notify: notifyService, platform: platformService } = services;
diff --git a/x-pack/legacy/plugins/canvas/public/services/platform.ts b/x-pack/legacy/plugins/canvas/public/services/platform.ts
new file mode 100644
index 0000000000000..440e9523044c1
--- /dev/null
+++ b/x-pack/legacy/plugins/canvas/public/services/platform.ts
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { CanvasServiceFactory } from '.';
+import { CoreStart, CoreSetup, CanvasSetupDeps, CanvasStartDeps } from '../plugin';
+
+interface PlatformService {
+ coreSetup: CoreSetup;
+ coreStart: CoreStart;
+ setupPlugins: CanvasSetupDeps;
+ startPlugins: CanvasStartDeps;
+}
+
+export const platformServiceFactory: CanvasServiceFactory = (
+ coreSetup,
+ coreStart,
+ setupPlugins,
+ startPlugins
+) => {
+ return { coreSetup, coreStart, setupPlugins, startPlugins };
+};
diff --git a/x-pack/legacy/plugins/canvas/public/state/initial_state.js b/x-pack/legacy/plugins/canvas/public/state/initial_state.js
index 40c017543147f..bfa68b33908e0 100644
--- a/x-pack/legacy/plugins/canvas/public/state/initial_state.js
+++ b/x-pack/legacy/plugins/canvas/public/state/initial_state.js
@@ -5,7 +5,7 @@
*/
import { get } from 'lodash';
-import { getCoreStart } from '../legacy';
+import { platformService } from '../services';
import { getDefaultWorkpad } from './defaults';
export const getInitialState = path => {
@@ -13,7 +13,7 @@ export const getInitialState = path => {
app: {}, // Kibana stuff in here
assets: {}, // assets end up here
transient: {
- canUserWrite: getCoreStart().application.capabilities.canvas.save,
+ canUserWrite: platformService.getService().coreStart.application.capabilities.canvas.save,
zoomScale: 1,
elementStats: {
total: 0,
diff --git a/x-pack/legacy/plugins/canvas/public/state/reducers/workpad.js b/x-pack/legacy/plugins/canvas/public/state/reducers/workpad.js
index 12733680ed32d..30f9c638a054f 100644
--- a/x-pack/legacy/plugins/canvas/public/state/reducers/workpad.js
+++ b/x-pack/legacy/plugins/canvas/public/state/reducers/workpad.js
@@ -5,7 +5,7 @@
*/
import { handleActions } from 'redux-actions';
-import { getCoreStart } from '../../legacy';
+import { platformService } from '../../services';
import { getDefaultWorkpad } from '../defaults';
import {
setWorkpad,
@@ -22,11 +22,13 @@ import { APP_ROUTE_WORKPAD } from '../../../common/lib/constants';
export const workpadReducer = handleActions(
{
[setWorkpad]: (workpadState, { payload }) => {
- getCoreStart().chrome.recentlyAccessed.add(
- `${APP_ROUTE_WORKPAD}/${payload.id}`,
- payload.name,
- payload.id
- );
+ platformService
+ .getService()
+ .coreStart.chrome.recentlyAccessed.add(
+ `${APP_ROUTE_WORKPAD}/${payload.id}`,
+ payload.name,
+ payload.id
+ );
return payload;
},
@@ -39,11 +41,13 @@ export const workpadReducer = handleActions(
},
[setName]: (workpadState, { payload }) => {
- getCoreStart().chrome.recentlyAccessed.add(
- `${APP_ROUTE_WORKPAD}/${workpadState.id}`,
- payload,
- workpadState.id
- );
+ platformService
+ .getService()
+ .coreStart.chrome.recentlyAccessed.add(
+ `${APP_ROUTE_WORKPAD}/${workpadState.id}`,
+ payload,
+ workpadState.id
+ );
return { ...workpadState, name: payload };
},
diff --git a/x-pack/legacy/plugins/maps/index.js b/x-pack/legacy/plugins/maps/index.js
index d1e8892fa2c98..a1186e04ee27a 100644
--- a/x-pack/legacy/plugins/maps/index.js
+++ b/x-pack/legacy/plugins/maps/index.js
@@ -54,7 +54,7 @@ export function maps(kibana) {
emsLandingPageUrl: mapConfig.emsLandingPageUrl,
kbnPkgVersion: serverConfig.get('pkg.version'),
regionmapLayers: _.get(mapConfig, 'regionmap.layers', []),
- tilemap: _.get(mapConfig, 'tilemap', []),
+ tilemap: _.get(mapConfig, 'tilemap', {}),
};
},
styleSheetPaths: `${__dirname}/public/index.scss`,
diff --git a/x-pack/legacy/plugins/maps/public/angular/map_controller.js b/x-pack/legacy/plugins/maps/public/angular/map_controller.js
index 1b1fbf111fe04..bb1a6b74d43a7 100644
--- a/x-pack/legacy/plugins/maps/public/angular/map_controller.js
+++ b/x-pack/legacy/plugins/maps/public/angular/map_controller.js
@@ -336,7 +336,7 @@ app.controller(
function addFilters(newFilters) {
newFilters.forEach(filter => {
- filter.$state = esFilters.FilterStateStore.APP_STATE;
+ filter.$state = { store: esFilters.FilterStateStore.APP_STATE };
});
$scope.updateFiltersAndDispatch([...$scope.filters, ...newFilters]);
}
diff --git a/x-pack/legacy/plugins/monitoring/common/constants.ts b/x-pack/legacy/plugins/monitoring/common/constants.ts
index 3a4c7b71dcd03..36030e1fa7f2a 100644
--- a/x-pack/legacy/plugins/monitoring/common/constants.ts
+++ b/x-pack/legacy/plugins/monitoring/common/constants.ts
@@ -251,7 +251,7 @@ export const ALERT_TYPES = [ALERT_TYPE_LICENSE_EXPIRATION, ALERT_TYPE_CLUSTER_ST
/**
* Matches the id for the built-in in email action type
- * See x-pack/legacy/plugins/actions/server/builtin_action_types/email.ts
+ * See x-pack/plugins/actions/server/builtin_action_types/email.ts
*/
export const ALERT_ACTION_TYPE_EMAIL = '.email';
diff --git a/x-pack/legacy/plugins/monitoring/public/np_imports/angular/modules.ts b/x-pack/legacy/plugins/monitoring/public/np_imports/angular/modules.ts
index a047c25c2b1d7..c6031cb220334 100644
--- a/x-pack/legacy/plugins/monitoring/public/np_imports/angular/modules.ts
+++ b/x-pack/legacy/plugins/monitoring/public/np_imports/angular/modules.ts
@@ -5,6 +5,8 @@
*/
import angular, { IWindowService } from 'angular';
+// required for `ngSanitize` angular module
+import 'angular-sanitize';
import { i18nDirective, i18nFilter, I18nProvider } from '@kbn/i18n/angular';
import { AppMountContext } from 'kibana/public';
diff --git a/x-pack/legacy/plugins/security/index.ts b/x-pack/legacy/plugins/security/index.ts
index 5b2218af1fd52..b1dec2ce82c52 100644
--- a/x-pack/legacy/plugins/security/index.ts
+++ b/x-pack/legacy/plugins/security/index.ts
@@ -78,9 +78,7 @@ export const security = (kibana: Record) =>
// features are up to date.
xpackInfo
.feature(this.id)
- .registerLicenseCheckResultsGenerator(() =>
- securityPlugin.__legacyCompat.license.getFeatures()
- );
+ .registerLicenseCheckResultsGenerator(() => securityPlugin.license.getFeatures());
server.expose({
getUser: async (request: LegacyRequest) =>
diff --git a/x-pack/legacy/plugins/task_manager/server/index.ts b/x-pack/legacy/plugins/task_manager/server/index.ts
index 3ea687f7003f4..a3167920efa06 100644
--- a/x-pack/legacy/plugins/task_manager/server/index.ts
+++ b/x-pack/legacy/plugins/task_manager/server/index.ts
@@ -6,8 +6,6 @@
import { Root } from 'joi';
import { Legacy } from 'kibana';
-import mappings from './mappings.json';
-import { migrations } from './migrations';
import { createLegacyApi, getTaskManagerSetup } from './legacy';
export { LegacyTaskManagerApi, getTaskManagerSetup, getTaskManagerStart } from './legacy';
@@ -21,19 +19,6 @@ import {
ArrayOrItem,
} from '../../../../../src/legacy/plugin_discovery/types';
-const savedObjectSchemas = {
- task: {
- hidden: true,
- isNamespaceAgnostic: true,
- convertToAliasScript: `ctx._id = ctx._source.type + ':' + ctx._id`,
- // legacy config is marked as any in core, no choice here
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- indexPattern(config: any) {
- return config.get('xpack.task_manager.index');
- },
- },
-};
-
export function taskManager(kibana: LegacyPluginApi): ArrayOrItem {
return new kibana.Plugin({
id: 'task_manager',
@@ -58,7 +43,7 @@ export function taskManager(kibana: LegacyPluginApi): ArrayOrItem {
// we can't tell the Kibana Platform Task Manager plugin to
// to wait to `start` as that happens before legacy plugins
@@ -77,10 +62,5 @@ export function taskManager(kibana: LegacyPluginApi): ArrayOrItem
- new kibana.Plugin({
- configPrefix: 'xpack.uptime',
- id: PLUGIN.ID,
- publicDir: resolve(__dirname, 'public'),
- require: ['alerting', 'kibana', 'elasticsearch', 'xpack_main'],
- uiExports: {
- app: {
- description: i18n.translate('xpack.uptime.pluginDescription', {
- defaultMessage: 'Uptime monitoring',
- description: 'The description text that will be shown to users in Kibana',
- }),
- icon: 'plugins/uptime/icons/heartbeat_white.svg',
- euiIconType: 'uptimeApp',
- title: i18n.translate('xpack.uptime.uptimeFeatureCatalogueTitle', {
- defaultMessage: 'Uptime',
- }),
- main: 'plugins/uptime/app',
- order: 8900,
- url: '/app/uptime#/',
- category: DEFAULT_APP_CATEGORIES.observability,
- },
- home: ['plugins/uptime/register_feature'],
- },
- });
diff --git a/x-pack/legacy/plugins/uptime/public/apps/index.ts b/x-pack/legacy/plugins/uptime/public/apps/index.ts
deleted file mode 100644
index d58bf8398fcde..0000000000000
--- a/x-pack/legacy/plugins/uptime/public/apps/index.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { npSetup } from 'ui/new_platform';
-import { Plugin } from './plugin';
-import 'uiExports/embeddableFactories';
-
-const plugin = new Plugin({
- opaqueId: Symbol('uptime'),
- env: {} as any,
- config: { get: () => ({} as any) },
-});
-plugin.setup(npSetup);
diff --git a/x-pack/legacy/plugins/uptime/public/apps/plugin.ts b/x-pack/legacy/plugins/uptime/public/apps/plugin.ts
deleted file mode 100644
index e73598c44c9f0..0000000000000
--- a/x-pack/legacy/plugins/uptime/public/apps/plugin.ts
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { LegacyCoreSetup, PluginInitializerContext, AppMountParameters } from 'src/core/public';
-import { PluginsSetup } from 'ui/new_platform/new_platform';
-import { FeatureCatalogueCategory } from '../../../../../../src/plugins/home/public';
-import { UMFrontendLibs } from '../lib/lib';
-import { PLUGIN } from '../../common/constants';
-import { getKibanaFrameworkAdapter } from '../lib/adapters/framework/new_platform_adapter';
-
-export interface SetupObject {
- core: LegacyCoreSetup;
- plugins: PluginsSetup;
-}
-
-export class Plugin {
- constructor(
- // @ts-ignore this is added to satisfy the New Platform typing constraint,
- // but we're not leveraging any of its functionality yet.
- private readonly initializerContext: PluginInitializerContext
- ) {}
-
- public setup(setup: SetupObject) {
- const { core, plugins } = setup;
- const { home } = plugins;
-
- home.featureCatalogue.register({
- category: FeatureCatalogueCategory.DATA,
- description: PLUGIN.DESCRIPTION,
- icon: 'uptimeApp',
- id: PLUGIN.ID,
- path: '/app/uptime#/',
- showOnHomePage: true,
- title: PLUGIN.TITLE,
- });
-
- core.application.register({
- id: PLUGIN.ID,
- euiIconType: 'uptimeApp',
- order: 8900,
- title: 'Uptime',
- async mount(params: AppMountParameters) {
- const [coreStart] = await core.getStartServices();
- const { element } = params;
- const libs: UMFrontendLibs = {
- framework: getKibanaFrameworkAdapter(coreStart, plugins),
- };
- libs.framework.render(element);
- return () => {};
- },
- });
- }
-}
diff --git a/x-pack/legacy/plugins/uptime/public/register_feature.ts b/x-pack/legacy/plugins/uptime/public/register_feature.ts
deleted file mode 100644
index 2f83fa33ba4bc..0000000000000
--- a/x-pack/legacy/plugins/uptime/public/register_feature.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { i18n } from '@kbn/i18n';
-import { npSetup } from 'ui/new_platform';
-import { FeatureCatalogueCategory } from '../../../../../src/plugins/home/public';
-
-const {
- plugins: { home },
-} = npSetup;
-
-home.featureCatalogue.register({
- id: 'uptime',
- title: i18n.translate('xpack.uptime.uptimeFeatureCatalogueTitle', { defaultMessage: 'Uptime' }),
- description: i18n.translate('xpack.uptime.featureCatalogueDescription', {
- defaultMessage: 'Perform endpoint health checks and uptime monitoring.',
- }),
- icon: 'uptimeApp',
- path: `uptime#/`,
- showOnHomePage: true,
- category: FeatureCatalogueCategory.DATA,
-});
diff --git a/x-pack/legacy/plugins/uptime/tsconfig.json b/x-pack/legacy/plugins/uptime/tsconfig.json
deleted file mode 100644
index 53425909db3e8..0000000000000
--- a/x-pack/legacy/plugins/uptime/tsconfig.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
- "extends": "../../../tsconfig.json",
- "exclude": ["**/node_modules/**"],
- "paths": {
- "react": ["../../../node_modules/@types/react"]
- }
-}
\ No newline at end of file
diff --git a/x-pack/package.json b/x-pack/package.json
index 604889c6094b9..dcc9b8c61cb96 100644
--- a/x-pack/package.json
+++ b/x-pack/package.json
@@ -105,6 +105,7 @@
"@types/recompose": "^0.30.6",
"@types/reduce-reducers": "^1.0.0",
"@types/redux-actions": "^2.6.1",
+ "@types/set-value": "^2.0.0",
"@types/sinon": "^7.0.13",
"@types/styled-components": "^4.4.2",
"@types/supertest": "^2.0.5",
@@ -344,6 +345,7 @@
"rison-node": "0.3.1",
"rxjs": "^6.5.3",
"semver": "5.7.0",
+ "set-value": "^3.0.2",
"squel": "^5.13.0",
"stats-lite": "^2.2.0",
"style-it": "^2.1.3",
diff --git a/x-pack/plugins/actions/server/lib/action_executor.ts b/x-pack/plugins/actions/server/lib/action_executor.ts
index 101e18f2583e3..3e9262c05efac 100644
--- a/x-pack/plugins/actions/server/lib/action_executor.ts
+++ b/x-pack/plugins/actions/server/lib/action_executor.ts
@@ -17,7 +17,7 @@ import {
import { EncryptedSavedObjectsPluginStart } from '../../../encrypted_saved_objects/server';
import { SpacesServiceSetup } from '../../../spaces/server';
import { EVENT_LOG_ACTIONS } from '../plugin';
-import { IEvent, IEventLogger } from '../../../event_log/server';
+import { IEvent, IEventLogger, SAVED_OBJECT_REL_PRIMARY } from '../../../event_log/server';
export interface ActionExecutorContext {
logger: Logger;
@@ -110,7 +110,16 @@ export class ActionExecutor {
const actionLabel = `${actionTypeId}:${actionId}: ${name}`;
const event: IEvent = {
event: { action: EVENT_LOG_ACTIONS.execute },
- kibana: { saved_objects: [{ type: 'action', id: actionId, ...namespace }] },
+ kibana: {
+ saved_objects: [
+ {
+ rel: SAVED_OBJECT_REL_PRIMARY,
+ type: 'action',
+ id: actionId,
+ ...namespace,
+ },
+ ],
+ },
};
eventLogger.startTiming(event);
diff --git a/x-pack/plugins/actions/server/plugin.ts b/x-pack/plugins/actions/server/plugin.ts
index 10e4d64584340..a6cc1fb5463bb 100644
--- a/x-pack/plugins/actions/server/plugin.ts
+++ b/x-pack/plugins/actions/server/plugin.ts
@@ -53,6 +53,7 @@ import {
} from './routes';
import { IEventLogger, IEventLogService } from '../../event_log/server';
import { initializeActionsTelemetry, scheduleActionsTelemetry } from './usage/task';
+import { setupSavedObjects } from './saved_objects';
const EVENT_LOG_PROVIDER = 'actions';
export const EVENT_LOG_ACTIONS = {
@@ -133,19 +134,7 @@ export class ActionsPlugin implements Plugin, Plugi
);
}
- // Encrypted attributes
- // - `secrets` properties will be encrypted
- // - `config` will be included in AAD
- // - everything else excluded from AAD
- plugins.encryptedSavedObjects.registerType({
- type: 'action',
- attributesToEncrypt: new Set(['secrets']),
- attributesToExcludeFromAAD: new Set(['name']),
- });
- plugins.encryptedSavedObjects.registerType({
- type: 'action_task_params',
- attributesToEncrypt: new Set(['apiKey']),
- });
+ setupSavedObjects(core.savedObjects, plugins.encryptedSavedObjects);
plugins.eventLog.registerProviderActions(EVENT_LOG_PROVIDER, Object.values(EVENT_LOG_ACTIONS));
this.eventLogger = plugins.eventLog.getLogger({
diff --git a/x-pack/plugins/actions/server/saved_objects/index.ts b/x-pack/plugins/actions/server/saved_objects/index.ts
new file mode 100644
index 0000000000000..dbd7925f96871
--- /dev/null
+++ b/x-pack/plugins/actions/server/saved_objects/index.ts
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SavedObjectsServiceSetup } from 'kibana/server';
+import mappings from './mappings.json';
+import { EncryptedSavedObjectsPluginSetup } from '../../../encrypted_saved_objects/server';
+
+export function setupSavedObjects(
+ savedObjects: SavedObjectsServiceSetup,
+ encryptedSavedObjects: EncryptedSavedObjectsPluginSetup
+) {
+ savedObjects.registerType({
+ name: 'action',
+ hidden: false,
+ namespaceType: 'single',
+ mappings: mappings.action,
+ });
+
+ // Encrypted attributes
+ // - `secrets` properties will be encrypted
+ // - `config` will be included in AAD
+ // - everything else excluded from AAD
+ encryptedSavedObjects.registerType({
+ type: 'action',
+ attributesToEncrypt: new Set(['secrets']),
+ attributesToExcludeFromAAD: new Set(['name']),
+ });
+
+ savedObjects.registerType({
+ name: 'action_task_params',
+ hidden: false,
+ namespaceType: 'single',
+ mappings: mappings.action_task_params,
+ });
+ encryptedSavedObjects.registerType({
+ type: 'action_task_params',
+ attributesToEncrypt: new Set(['apiKey']),
+ });
+}
diff --git a/x-pack/legacy/plugins/actions/server/mappings.json b/x-pack/plugins/actions/server/saved_objects/mappings.json
similarity index 100%
rename from x-pack/legacy/plugins/actions/server/mappings.json
rename to x-pack/plugins/actions/server/saved_objects/mappings.json
diff --git a/x-pack/plugins/alerting/server/plugin.ts b/x-pack/plugins/alerting/server/plugin.ts
index c03d3506a051d..8cdde2eeb9877 100644
--- a/x-pack/plugins/alerting/server/plugin.ts
+++ b/x-pack/plugins/alerting/server/plugin.ts
@@ -58,6 +58,7 @@ import { Services } from './types';
import { registerAlertsUsageCollector } from './usage';
import { initializeAlertingTelemetry, scheduleAlertingTelemetry } from './usage/task';
import { IEventLogger, IEventLogService } from '../../event_log/server';
+import { setupSavedObjects } from './saved_objects';
const EVENT_LOG_PROVIDER = 'alerting';
export const EVENT_LOG_ACTIONS = {
@@ -134,17 +135,7 @@ export class AlertingPlugin {
);
}
- // Encrypted attributes
- plugins.encryptedSavedObjects.registerType({
- type: 'alert',
- attributesToEncrypt: new Set(['apiKey']),
- attributesToExcludeFromAAD: new Set([
- 'scheduledTaskId',
- 'muteAll',
- 'mutedInstanceIds',
- 'updatedBy',
- ]),
- });
+ setupSavedObjects(core.savedObjects, plugins.encryptedSavedObjects);
plugins.eventLog.registerProviderActions(EVENT_LOG_PROVIDER, Object.values(EVENT_LOG_ACTIONS));
this.eventLogger = plugins.eventLog.getLogger({
diff --git a/x-pack/plugins/alerting/server/saved_objects/index.ts b/x-pack/plugins/alerting/server/saved_objects/index.ts
new file mode 100644
index 0000000000000..4efec2fe55ef0
--- /dev/null
+++ b/x-pack/plugins/alerting/server/saved_objects/index.ts
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SavedObjectsServiceSetup } from 'kibana/server';
+import mappings from './mappings.json';
+import { EncryptedSavedObjectsPluginSetup } from '../../../encrypted_saved_objects/server';
+
+export function setupSavedObjects(
+ savedObjects: SavedObjectsServiceSetup,
+ encryptedSavedObjects: EncryptedSavedObjectsPluginSetup
+) {
+ savedObjects.registerType({
+ name: 'alert',
+ hidden: false,
+ namespaceType: 'single',
+ mappings: mappings.alert,
+ });
+
+ // Encrypted attributes
+ encryptedSavedObjects.registerType({
+ type: 'alert',
+ attributesToEncrypt: new Set(['apiKey']),
+ attributesToExcludeFromAAD: new Set([
+ 'scheduledTaskId',
+ 'muteAll',
+ 'mutedInstanceIds',
+ 'updatedBy',
+ ]),
+ });
+}
diff --git a/x-pack/legacy/plugins/alerting/server/mappings.json b/x-pack/plugins/alerting/server/saved_objects/mappings.json
similarity index 100%
rename from x-pack/legacy/plugins/alerting/server/mappings.json
rename to x-pack/plugins/alerting/server/saved_objects/mappings.json
diff --git a/x-pack/plugins/alerting/server/task_runner/create_execution_handler.test.ts b/x-pack/plugins/alerting/server/task_runner/create_execution_handler.test.ts
index 0e46ef4919626..a564b87f2ca50 100644
--- a/x-pack/plugins/alerting/server/task_runner/create_execution_handler.test.ts
+++ b/x-pack/plugins/alerting/server/task_runner/create_execution_handler.test.ts
@@ -95,6 +95,7 @@ test('calls actionsPlugin.execute per selected action', async () => {
"saved_objects": Array [
Object {
"id": "1",
+ "rel": "primary",
"type": "alert",
},
Object {
diff --git a/x-pack/plugins/alerting/server/task_runner/create_execution_handler.ts b/x-pack/plugins/alerting/server/task_runner/create_execution_handler.ts
index 5c3e36b88879d..16fadc8b06cd5 100644
--- a/x-pack/plugins/alerting/server/task_runner/create_execution_handler.ts
+++ b/x-pack/plugins/alerting/server/task_runner/create_execution_handler.ts
@@ -9,7 +9,7 @@ import { AlertAction, State, Context, AlertType } from '../types';
import { Logger } from '../../../../../src/core/server';
import { transformActionParams } from './transform_action_params';
import { PluginStartContract as ActionsPluginStartContract } from '../../../../plugins/actions/server';
-import { IEventLogger, IEvent } from '../../../event_log/server';
+import { IEventLogger, IEvent, SAVED_OBJECT_REL_PRIMARY } from '../../../event_log/server';
import { EVENT_LOG_ACTIONS } from '../plugin';
interface CreateExecutionHandlerOptions {
@@ -96,7 +96,7 @@ export function createExecutionHandler({
instance_id: alertInstanceId,
},
saved_objects: [
- { type: 'alert', id: alertId, ...namespace },
+ { rel: SAVED_OBJECT_REL_PRIMARY, type: 'alert', id: alertId, ...namespace },
{ type: 'action', id: action.id, ...namespace },
],
},
diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts
index 26d8a1d1777c0..35a0018049c33 100644
--- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts
+++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts
@@ -172,6 +172,7 @@ describe('Task Runner', () => {
Object {
"id": "1",
"namespace": undefined,
+ "rel": "primary",
"type": "alert",
},
],
@@ -234,6 +235,7 @@ describe('Task Runner', () => {
Object {
"id": "1",
"namespace": undefined,
+ "rel": "primary",
"type": "alert",
},
],
@@ -254,6 +256,7 @@ describe('Task Runner', () => {
Object {
"id": "1",
"namespace": undefined,
+ "rel": "primary",
"type": "alert",
},
],
@@ -274,6 +277,7 @@ describe('Task Runner', () => {
Object {
"id": "1",
"namespace": undefined,
+ "rel": "primary",
"type": "alert",
},
Object {
@@ -351,6 +355,7 @@ describe('Task Runner', () => {
Object {
"id": "1",
"namespace": undefined,
+ "rel": "primary",
"type": "alert",
},
],
@@ -371,6 +376,7 @@ describe('Task Runner', () => {
Object {
"id": "1",
"namespace": undefined,
+ "rel": "primary",
"type": "alert",
},
],
@@ -568,6 +574,7 @@ describe('Task Runner', () => {
Object {
"id": "1",
"namespace": undefined,
+ "rel": "primary",
"type": "alert",
},
],
diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts
index 26970dc6b2b0d..bf005301adc07 100644
--- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts
+++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts
@@ -25,7 +25,7 @@ import { promiseResult, map, Resultable, asOk, asErr, resolveErr } from '../lib/
import { taskInstanceToAlertTaskInstance } from './alert_task_instance';
import { AlertInstances } from '../alert_instance/alert_instance';
import { EVENT_LOG_ACTIONS } from '../plugin';
-import { IEvent, IEventLogger } from '../../../event_log/server';
+import { IEvent, IEventLogger, SAVED_OBJECT_REL_PRIMARY } from '../../../event_log/server';
import { isAlertSavedObjectNotFoundError } from '../lib/is_alert_not_found_error';
const FALLBACK_RETRY_INTERVAL: IntervalSchedule = { interval: '5m' };
@@ -174,7 +174,16 @@ export class TaskRunner {
const alertLabel = `${this.alertType.id}:${alertId}: '${name}'`;
const event: IEvent = {
event: { action: EVENT_LOG_ACTIONS.execute },
- kibana: { saved_objects: [{ type: 'alert', id: alertId, namespace }] },
+ kibana: {
+ saved_objects: [
+ {
+ rel: SAVED_OBJECT_REL_PRIMARY,
+ type: 'alert',
+ id: alertId,
+ namespace,
+ },
+ ],
+ },
};
eventLogger.startTiming(event);
@@ -393,7 +402,14 @@ function generateNewAndResolvedInstanceEvents(params: GenerateNewAndResolvedInst
alerting: {
instance_id: id,
},
- saved_objects: [{ type: 'alert', id: params.alertId, namespace: params.namespace }],
+ saved_objects: [
+ {
+ rel: SAVED_OBJECT_REL_PRIMARY,
+ type: 'alert',
+ id: params.alertId,
+ namespace: params.namespace,
+ },
+ ],
},
message,
};
diff --git a/x-pack/plugins/endpoint/common/generate_data.ts b/x-pack/plugins/endpoint/common/generate_data.ts
index 61fbb779986dc..e40fc3e386bc8 100644
--- a/x-pack/plugins/endpoint/common/generate_data.ts
+++ b/x-pack/plugins/endpoint/common/generate_data.ts
@@ -307,7 +307,7 @@ export class EndpointDocGenerator {
process: {
entity_id: options.entityID ? options.entityID : this.randomString(10),
parent: options.parentEntityID ? { entity_id: options.parentEntityID } : undefined,
- name: options.processName ? options.processName : 'powershell.exe',
+ name: options.processName ? options.processName : randomProcessName(),
},
};
}
@@ -645,3 +645,16 @@ export class EndpointDocGenerator {
return uuid.v4({ random: [...this.randomNGenerator(255, 16)] });
}
}
+
+const fakeProcessNames = [
+ 'lsass.exe',
+ 'notepad.exe',
+ 'mimikatz.exe',
+ 'powershell.exe',
+ 'iexlorer.exe',
+ 'explorer.exe',
+];
+/** Return a random fake process name */
+function randomProcessName(): string {
+ return fakeProcessNames[Math.floor(Math.random() * fakeProcessNames.length)];
+}
diff --git a/x-pack/plugins/endpoint/common/models/event.ts b/x-pack/plugins/endpoint/common/models/event.ts
index 650486f3c3858..47f39d2d11797 100644
--- a/x-pack/plugins/endpoint/common/models/event.ts
+++ b/x-pack/plugins/endpoint/common/models/event.ts
@@ -4,17 +4,13 @@
* you may not use this file except in compliance with the Elastic License.
*/
-import { EndpointEvent, LegacyEndpointEvent } from '../types';
+import { LegacyEndpointEvent, ResolverEvent } from '../types';
-export function isLegacyEvent(
- event: EndpointEvent | LegacyEndpointEvent
-): event is LegacyEndpointEvent {
+export function isLegacyEvent(event: ResolverEvent): event is LegacyEndpointEvent {
return (event as LegacyEndpointEvent).endgame !== undefined;
}
-export function eventTimestamp(
- event: EndpointEvent | LegacyEndpointEvent
-): string | undefined | number {
+export function eventTimestamp(event: ResolverEvent): string | undefined | number {
if (isLegacyEvent(event)) {
return event.endgame.timestamp_utc;
} else {
@@ -22,10 +18,31 @@ export function eventTimestamp(
}
}
-export function eventName(event: EndpointEvent | LegacyEndpointEvent): string {
+export function eventName(event: ResolverEvent): string {
if (isLegacyEvent(event)) {
return event.endgame.process_name ? event.endgame.process_name : '';
} else {
return event.process.name;
}
}
+
+export function eventId(event: ResolverEvent): string {
+ if (isLegacyEvent(event)) {
+ return event.endgame.serial_event_id ? String(event.endgame.serial_event_id) : '';
+ }
+ return event.event.id;
+}
+
+export function entityId(event: ResolverEvent): string {
+ if (isLegacyEvent(event)) {
+ return event.endgame.unique_pid ? String(event.endgame.unique_pid) : '';
+ }
+ return event.process.entity_id;
+}
+
+export function parentEntityId(event: ResolverEvent): string | undefined {
+ if (isLegacyEvent(event)) {
+ return event.endgame.unique_ppid ? String(event.endgame.unique_ppid) : undefined;
+ }
+ return event.process.parent?.entity_id;
+}
diff --git a/x-pack/plugins/endpoint/common/schema/resolver.ts b/x-pack/plugins/endpoint/common/schema/resolver.ts
new file mode 100644
index 0000000000000..f21307e407fd0
--- /dev/null
+++ b/x-pack/plugins/endpoint/common/schema/resolver.ts
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema } from '@kbn/config-schema';
+
+/**
+ * Used to validate GET requests for a complete resolver tree.
+ */
+export const validateTree = {
+ params: schema.object({ id: schema.string() }),
+ query: schema.object({
+ children: schema.number({ defaultValue: 10, min: 0, max: 100 }),
+ generations: schema.number({ defaultValue: 3, min: 0, max: 3 }),
+ ancestors: schema.number({ defaultValue: 3, min: 0, max: 5 }),
+ events: schema.number({ defaultValue: 100, min: 0, max: 1000 }),
+ afterEvent: schema.maybe(schema.string()),
+ afterChild: schema.maybe(schema.string()),
+ legacyEndpointID: schema.maybe(schema.string()),
+ }),
+};
+
+/**
+ * Used to validate GET requests for non process events for a specific event.
+ */
+export const validateEvents = {
+ params: schema.object({ id: schema.string() }),
+ query: schema.object({
+ events: schema.number({ defaultValue: 100, min: 1, max: 1000 }),
+ afterEvent: schema.maybe(schema.string()),
+ legacyEndpointID: schema.maybe(schema.string()),
+ }),
+};
+
+/**
+ * Used to validate GET requests for the ancestors of a process event.
+ */
+export const validateAncestry = {
+ params: schema.object({ id: schema.string() }),
+ query: schema.object({
+ ancestors: schema.number({ defaultValue: 0, min: 0, max: 10 }),
+ legacyEndpointID: schema.maybe(schema.string()),
+ }),
+};
+
+/**
+ * Used to validate GET requests for children of a specified process event.
+ */
+export const validateChildren = {
+ params: schema.object({ id: schema.string() }),
+ query: schema.object({
+ children: schema.number({ defaultValue: 10, min: 10, max: 100 }),
+ generations: schema.number({ defaultValue: 3, min: 0, max: 3 }),
+ afterChild: schema.maybe(schema.string()),
+ legacyEndpointID: schema.maybe(schema.string()),
+ }),
+};
diff --git a/x-pack/plugins/endpoint/common/types.ts b/x-pack/plugins/endpoint/common/types.ts
index 8773c5441860a..8fce15d1c794c 100644
--- a/x-pack/plugins/endpoint/common/types.ts
+++ b/x-pack/plugins/endpoint/common/types.ts
@@ -35,6 +35,31 @@ type ImmutableObject = { readonly [K in keyof T]: Immutable };
*/
export type AlertAPIOrdering = 'asc' | 'desc';
+export interface ResolverNodeStats {
+ totalEvents: number;
+ totalAlerts: number;
+}
+
+export interface ResolverNodePagination {
+ nextChild?: string | null;
+ nextEvent?: string | null;
+ nextAncestor?: string | null;
+ nextAlert?: string | null;
+}
+
+/**
+ * A node that contains pointers to other nodes, arrrays of resolver events, and any metadata associated with resolver specific data
+ */
+export interface ResolverNode {
+ id: string;
+ children: ResolverNode[];
+ events: ResolverEvent[];
+ lifecycle: ResolverEvent[];
+ ancestors?: ResolverNode[];
+ pagination: ResolverNodePagination;
+ stats?: ResolverNodeStats;
+}
+
/**
* Returned by 'api/endpoint/alerts'
*/
diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/store/hosts/middleware.ts b/x-pack/plugins/endpoint/public/applications/endpoint/store/hosts/middleware.ts
index d1b9a2cde4b31..bcfd6b96c9eb8 100644
--- a/x-pack/plugins/endpoint/public/applications/endpoint/store/hosts/middleware.ts
+++ b/x-pack/plugins/endpoint/public/applications/endpoint/store/hosts/middleware.ts
@@ -70,7 +70,6 @@ export const hostMiddlewareFactory: ImmutableMiddlewareFactory = core
type: 'serverReturnedHostDetails',
payload: response,
});
- // FIXME: once we have the API implementation in place, we should call it parallel with the above api call and then dispatch this with the results of the second call
dispatch({
type: 'serverReturnedHostPolicyResponse',
payload: {
diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/view/alerts/resolver.tsx b/x-pack/plugins/endpoint/public/applications/endpoint/view/alerts/resolver.tsx
index d18bc59a35f52..d32ad4dd9defc 100644
--- a/x-pack/plugins/endpoint/public/applications/endpoint/view/alerts/resolver.tsx
+++ b/x-pack/plugins/endpoint/public/applications/endpoint/view/alerts/resolver.tsx
@@ -33,5 +33,6 @@ export const AlertDetailResolver = styled(
width: 100%;
display: flex;
flex-grow: 1;
- min-height: 500px;
+ /* gross demo hack */
+ min-height: calc(100vh - 505px);
`;
diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/action.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/action.ts
index 373afa89921dc..3ec15f2f1985d 100644
--- a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/action.ts
+++ b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/action.ts
@@ -8,13 +8,11 @@ import { ResolverEvent } from '../../../../../common/types';
interface ServerReturnedResolverData {
readonly type: 'serverReturnedResolverData';
- readonly payload: {
- readonly data: {
- readonly result: {
- readonly search_results: readonly ResolverEvent[];
- };
- };
- };
+ readonly payload: ResolverEvent[];
}
-export type DataAction = ServerReturnedResolverData;
+interface ServerFailedToReturnResolverData {
+ readonly type: 'serverFailedToReturnResolverData';
+}
+
+export type DataAction = ServerReturnedResolverData | ServerFailedToReturnResolverData;
diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/graphing.test.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/graphing.test.ts
index f01136fe20ebf..f95ecc63d2a66 100644
--- a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/graphing.test.ts
+++ b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/graphing.test.ts
@@ -8,7 +8,7 @@ import { Store, createStore } from 'redux';
import { DataAction } from './action';
import { dataReducer } from './reducer';
import { DataState } from '../../types';
-import { LegacyEndpointEvent } from '../../../../../common/types';
+import { LegacyEndpointEvent, ResolverEvent } from '../../../../../common/types';
import { graphableProcesses, processNodePositionsAndEdgeLineSegments } from './selectors';
import { mockProcessEvent } from '../../models/process_event_test_helpers';
@@ -113,13 +113,7 @@ describe('resolver graph layout', () => {
});
describe('when rendering no nodes', () => {
beforeEach(() => {
- const payload = {
- data: {
- result: {
- search_results: [],
- },
- },
- };
+ const payload: ResolverEvent[] = [];
const action: DataAction = { type: 'serverReturnedResolverData', payload };
store.dispatch(action);
});
@@ -133,13 +127,7 @@ describe('resolver graph layout', () => {
});
describe('when rendering one node', () => {
beforeEach(() => {
- const payload = {
- data: {
- result: {
- search_results: [processA],
- },
- },
- };
+ const payload = [processA];
const action: DataAction = { type: 'serverReturnedResolverData', payload };
store.dispatch(action);
});
@@ -153,13 +141,7 @@ describe('resolver graph layout', () => {
});
describe('when rendering two nodes, one being the parent of the other', () => {
beforeEach(() => {
- const payload = {
- data: {
- result: {
- search_results: [processA, processB],
- },
- },
- };
+ const payload = [processA, processB];
const action: DataAction = { type: 'serverReturnedResolverData', payload };
store.dispatch(action);
});
@@ -173,23 +155,17 @@ describe('resolver graph layout', () => {
});
describe('when rendering two forks, and one fork has an extra long tine', () => {
beforeEach(() => {
- const payload = {
- data: {
- result: {
- search_results: [
- processA,
- processB,
- processC,
- processD,
- processE,
- processF,
- processG,
- processH,
- processI,
- ],
- },
- },
- };
+ const payload = [
+ processA,
+ processB,
+ processC,
+ processD,
+ processE,
+ processF,
+ processG,
+ processH,
+ processI,
+ ];
const action: DataAction = { type: 'serverReturnedResolverData', payload };
store.dispatch(action);
});
diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/reducer.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/reducer.ts
index a3184389a794e..fc307002819a9 100644
--- a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/reducer.ts
+++ b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/reducer.ts
@@ -11,25 +11,28 @@ function initialState(): DataState {
return {
results: [],
isLoading: false,
+ hasError: false,
};
}
export const dataReducer: Reducer = (state = initialState(), action) => {
if (action.type === 'serverReturnedResolverData') {
- const {
- data: {
- result: { search_results },
- },
- } = action.payload;
return {
...state,
- results: search_results,
+ results: action.payload,
isLoading: false,
+ hasError: false,
};
} else if (action.type === 'appRequestedResolverData') {
return {
...state,
isLoading: true,
+ hasError: false,
+ };
+ } else if (action.type === 'serverFailedToReturnResolverData') {
+ return {
+ ...state,
+ hasError: true,
};
} else {
return state;
diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/selectors.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/selectors.ts
index 5dda54d4ed029..59ee4b3b87505 100644
--- a/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/selectors.ts
+++ b/x-pack/plugins/endpoint/public/embeddables/resolver/store/data/selectors.ts
@@ -34,6 +34,10 @@ export function isLoading(state: DataState) {
return state.isLoading;
}
+export function hasError(state: DataState) {
+ return state.hasError;
+}
+
/**
* An isometric projection is a method for representing three dimensional objects in 2 dimensions.
* More information about isometric projections can be found here https://en.wikipedia.org/wiki/Isometric_projection.
@@ -293,7 +297,7 @@ function* levelOrderWithWidths(
metadata.firstChildWidth = width;
} else {
const firstChildWidth = widths.get(siblings[0]);
- const lastChildWidth = widths.get(siblings[0]);
+ const lastChildWidth = widths.get(siblings[siblings.length - 1]);
if (firstChildWidth === undefined || lastChildWidth === undefined) {
/**
* All widths have been precalcluated, so this will not happen.
diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/store/middleware.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/store/middleware.ts
index 4e57212e5c0c2..c7177c6387e7a 100644
--- a/x-pack/plugins/endpoint/public/embeddables/resolver/store/middleware.ts
+++ b/x-pack/plugins/endpoint/public/embeddables/resolver/store/middleware.ts
@@ -8,7 +8,7 @@ import { Dispatch, MiddlewareAPI } from 'redux';
import { KibanaReactContextValue } from '../../../../../../../src/plugins/kibana_react/public';
import { EndpointPluginServices } from '../../../plugin';
import { ResolverState, ResolverAction } from '../types';
-import { ResolverEvent } from '../../../../common/types';
+import { ResolverEvent, ResolverNode } from '../../../../common/types';
import * as event from '../../../../common/models/event';
type MiddlewareFactory = (
@@ -16,18 +16,18 @@ type MiddlewareFactory = (
) => (
api: MiddlewareAPI, S>
) => (next: Dispatch) => (action: ResolverAction) => unknown;
-interface Lifecycle {
- lifecycle: ResolverEvent[];
-}
-type ChildResponse = [Lifecycle];
-function flattenEvents(events: ChildResponse): ResolverEvent[] {
- return events
- .map((child: Lifecycle) => child.lifecycle)
- .reduce(
- (accumulator: ResolverEvent[], value: ResolverEvent[]) => accumulator.concat(value),
- []
- );
+function flattenEvents(children: ResolverNode[], events: ResolverEvent[] = []): ResolverEvent[] {
+ return children.reduce((flattenedEvents, currentNode) => {
+ if (currentNode.lifecycle && currentNode.lifecycle.length > 0) {
+ flattenedEvents.push(...currentNode.lifecycle);
+ }
+ if (currentNode.children && currentNode.children.length > 0) {
+ return flattenEvents(currentNode.children, events);
+ } else {
+ return flattenedEvents;
+ }
+ }, events);
}
export const resolverMiddlewareFactory: MiddlewareFactory = context => {
@@ -39,53 +39,43 @@ export const resolverMiddlewareFactory: MiddlewareFactory = context => {
*/
if (context?.services.http && action.payload.selectedEvent) {
api.dispatch({ type: 'appRequestedResolverData' });
- let response = [];
- let lifecycle: ResolverEvent[];
- let childEvents: ResolverEvent[];
- let relatedEvents: ResolverEvent[];
- let children = [];
- const ancestors: ResolverEvent[] = [];
- const maxAncestors = 5;
- if (event.isLegacyEvent(action.payload.selectedEvent)) {
- const uniquePid = action.payload.selectedEvent?.endgame?.unique_pid;
- const legacyEndpointID = action.payload.selectedEvent?.agent?.id;
- [{ lifecycle }, { children }, { events: relatedEvents }] = await Promise.all([
- context.services.http.get(`/api/endpoint/resolver/${uniquePid}`, {
- query: { legacyEndpointID },
- }),
- context.services.http.get(`/api/endpoint/resolver/${uniquePid}/children`, {
- query: { legacyEndpointID },
- }),
- context.services.http.get(`/api/endpoint/resolver/${uniquePid}/related`, {
- query: { legacyEndpointID },
- }),
- ]);
- childEvents = children.length > 0 ? flattenEvents(children) : [];
- } else {
- const uniquePid = action.payload.selectedEvent.process.entity_id;
- const ppid = action.payload.selectedEvent.process.parent?.entity_id;
- async function getAncestors(pid: string | undefined) {
- if (ancestors.length < maxAncestors && pid !== undefined) {
- const parent = await context?.services.http.get(`/api/endpoint/resolver/${pid}`);
- ancestors.push(parent.lifecycle[0]);
- if (parent.lifecycle[0].process?.parent?.entity_id) {
- await getAncestors(parent.lifecycle[0].process.parent.entity_id);
- }
- }
+ try {
+ let lifecycle: ResolverEvent[];
+ let children: ResolverNode[];
+ let ancestors: ResolverNode[];
+ if (event.isLegacyEvent(action.payload.selectedEvent)) {
+ const entityId = action.payload.selectedEvent?.endgame?.unique_pid;
+ const legacyEndpointID = action.payload.selectedEvent?.agent?.id;
+ [{ lifecycle, children, ancestors }] = await Promise.all([
+ context.services.http.get(`/api/endpoint/resolver/${entityId}`, {
+ query: { legacyEndpointID, children: 5, ancestors: 5 },
+ }),
+ ]);
+ } else {
+ const entityId = action.payload.selectedEvent.process.entity_id;
+ [{ lifecycle, children, ancestors }] = await Promise.all([
+ context.services.http.get(`/api/endpoint/resolver/${entityId}`, {
+ query: {
+ children: 5,
+ ancestors: 5,
+ },
+ }),
+ ]);
}
- [{ lifecycle }, { children }, { events: relatedEvents }] = await Promise.all([
- context.services.http.get(`/api/endpoint/resolver/${uniquePid}`),
- context.services.http.get(`/api/endpoint/resolver/${uniquePid}/children`),
- context.services.http.get(`/api/endpoint/resolver/${uniquePid}/related`),
- getAncestors(ppid),
- ]);
+ const response: ResolverEvent[] = [
+ ...lifecycle,
+ ...flattenEvents(children),
+ ...flattenEvents(ancestors),
+ ];
+ api.dispatch({
+ type: 'serverReturnedResolverData',
+ payload: response,
+ });
+ } catch (error) {
+ api.dispatch({
+ type: 'serverFailedToReturnResolverData',
+ });
}
- childEvents = children.length > 0 ? flattenEvents(children) : [];
- response = [...lifecycle, ...childEvents, ...relatedEvents, ...ancestors];
- api.dispatch({
- type: 'serverReturnedResolverData',
- payload: { data: { result: { search_results: response } } },
- });
}
}
};
diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/store/selectors.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/store/selectors.ts
index e8ae3d08e5cb6..7d09d90881da9 100644
--- a/x-pack/plugins/endpoint/public/embeddables/resolver/store/selectors.ts
+++ b/x-pack/plugins/endpoint/public/embeddables/resolver/store/selectors.ts
@@ -102,6 +102,11 @@ function uiStateSelector(state: ResolverState) {
*/
export const isLoading = composeSelectors(dataStateSelector, dataSelectors.isLoading);
+/**
+ * Whether or not the resolver encountered an error while fetching data
+ */
+export const hasError = composeSelectors(dataStateSelector, dataSelectors.hasError);
+
/**
* Calls the `secondSelector` with the result of the `selector`. Use this when re-exporting a
* concern-specific selector. `selector` should return the concern-specific state.
diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/types.ts b/x-pack/plugins/endpoint/public/embeddables/resolver/types.ts
index d370bda0d1842..17aa598720c59 100644
--- a/x-pack/plugins/endpoint/public/embeddables/resolver/types.ts
+++ b/x-pack/plugins/endpoint/public/embeddables/resolver/types.ts
@@ -136,6 +136,7 @@ export type CameraState = {
export interface DataState {
readonly results: readonly ResolverEvent[];
isLoading: boolean;
+ hasError: boolean;
}
export type Vector2 = readonly [number, number];
diff --git a/x-pack/plugins/endpoint/public/embeddables/resolver/view/index.tsx b/x-pack/plugins/endpoint/public/embeddables/resolver/view/index.tsx
index 36155ece57a9c..2e7ca65c92dc1 100644
--- a/x-pack/plugins/endpoint/public/embeddables/resolver/view/index.tsx
+++ b/x-pack/plugins/endpoint/public/embeddables/resolver/view/index.tsx
@@ -8,6 +8,7 @@ import React, { useLayoutEffect } from 'react';
import { useSelector, useDispatch } from 'react-redux';
import styled from 'styled-components';
import { EuiLoadingSpinner } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
import * as selectors from '../store/selectors';
import { EdgeLine } from './edge_line';
import { Panel } from './panel';
@@ -59,6 +60,7 @@ export const Resolver = styled(
const { projectionMatrix, ref, onMouseDown } = useCamera();
const isLoading = useSelector(selectors.isLoading);
+ const hasError = useSelector(selectors.hasError);
const activeDescendantId = useSelector(selectors.uiActiveDescendantId);
useLayoutEffect(() => {
@@ -74,6 +76,16 @@ export const Resolver = styled(
+ ) : hasError ? (
+
) : (
{
}
const serverResponseAction: ResolverAction = {
type: 'serverReturnedResolverData',
- payload: {
- data: {
- result: {
- search_results: events,
- },
- },
- },
+ payload: events,
};
act(() => {
store.dispatch(serverResponseAction);
diff --git a/x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts b/x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts
index 92bd07a813d26..114251820ce4b 100644
--- a/x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts
+++ b/x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts
@@ -60,8 +60,6 @@ export const getRequestData = async (
reqData.fromIndex = reqData.pageIndex * reqData.pageSize;
}
- // See: https://github.com/elastic/elasticsearch-js/issues/662
- // and https://github.com/elastic/endpoint-app-team/issues/221
if (
reqData.searchBefore !== undefined &&
reqData.searchBefore[0] === '' &&
diff --git a/x-pack/plugins/endpoint/server/routes/metadata/index.ts b/x-pack/plugins/endpoint/server/routes/metadata/index.ts
index 99dc4ac9f9e33..08950930441df 100644
--- a/x-pack/plugins/endpoint/server/routes/metadata/index.ts
+++ b/x-pack/plugins/endpoint/server/routes/metadata/index.ts
@@ -171,7 +171,6 @@ async function enrichHostMetadata(
try {
/**
* Get agent status by elastic agent id if available or use the host id.
- * https://github.com/elastic/endpoint-app-team/issues/354
*/
if (!elasticAgentId) {
diff --git a/x-pack/plugins/endpoint/server/routes/resolver.ts b/x-pack/plugins/endpoint/server/routes/resolver.ts
index a96d431225b15..3599acacb4f59 100644
--- a/x-pack/plugins/endpoint/server/routes/resolver.ts
+++ b/x-pack/plugins/endpoint/server/routes/resolver.ts
@@ -6,20 +6,27 @@
import { IRouter } from 'kibana/server';
import { EndpointAppContext } from '../types';
-import { handleRelatedEvents, validateRelatedEvents } from './resolver/related_events';
-import { handleChildren, validateChildren } from './resolver/children';
-import { handleLifecycle, validateLifecycle } from './resolver/lifecycle';
+import {
+ validateTree,
+ validateEvents,
+ validateChildren,
+ validateAncestry,
+} from '../../common/schema/resolver';
+import { handleEvents } from './resolver/events';
+import { handleChildren } from './resolver/children';
+import { handleAncestry } from './resolver/ancestry';
+import { handleTree } from './resolver/tree';
export function registerResolverRoutes(router: IRouter, endpointAppContext: EndpointAppContext) {
const log = endpointAppContext.logFactory.get('resolver');
router.get(
{
- path: '/api/endpoint/resolver/{id}/related',
- validate: validateRelatedEvents,
+ path: '/api/endpoint/resolver/{id}/events',
+ validate: validateEvents,
options: { authRequired: true },
},
- handleRelatedEvents(log, endpointAppContext)
+ handleEvents(log, endpointAppContext)
);
router.get(
@@ -31,12 +38,21 @@ export function registerResolverRoutes(router: IRouter, endpointAppContext: Endp
handleChildren(log, endpointAppContext)
);
+ router.get(
+ {
+ path: '/api/endpoint/resolver/{id}/ancestry',
+ validate: validateAncestry,
+ options: { authRequired: true },
+ },
+ handleAncestry(log, endpointAppContext)
+ );
+
router.get(
{
path: '/api/endpoint/resolver/{id}',
- validate: validateLifecycle,
+ validate: validateTree,
options: { authRequired: true },
},
- handleLifecycle(log, endpointAppContext)
+ handleTree(log, endpointAppContext)
);
}
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/ancestry.ts b/x-pack/plugins/endpoint/server/routes/resolver/ancestry.ts
new file mode 100644
index 0000000000000..6648dc5b9e493
--- /dev/null
+++ b/x-pack/plugins/endpoint/server/routes/resolver/ancestry.ts
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { RequestHandler, Logger } from 'kibana/server';
+import { TypeOf } from '@kbn/config-schema';
+import { validateAncestry } from '../../../common/schema/resolver';
+import { Fetcher } from './utils/fetch';
+import { EndpointAppContext } from '../../types';
+
+export function handleAncestry(
+ log: Logger,
+ endpointAppContext: EndpointAppContext
+): RequestHandler, TypeOf> {
+ return async (context, req, res) => {
+ const {
+ params: { id },
+ query: { ancestors, legacyEndpointID: endpointID },
+ } = req;
+ try {
+ const indexRetriever = endpointAppContext.service.getIndexPatternRetriever();
+
+ const client = context.core.elasticsearch.dataClient;
+ const indexPattern = await indexRetriever.getEventIndexPattern(context);
+
+ const fetcher = new Fetcher(client, id, indexPattern, endpointID);
+ const tree = await fetcher.ancestors(ancestors + 1);
+
+ return res.ok({
+ body: tree.render(),
+ });
+ } catch (err) {
+ log.warn(err);
+ return res.internalError({ body: err });
+ }
+ };
+}
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/children.ts b/x-pack/plugins/endpoint/server/routes/resolver/children.ts
index c3b19b6c912b6..bb18b29a4b947 100644
--- a/x-pack/plugins/endpoint/server/routes/resolver/children.ts
+++ b/x-pack/plugins/endpoint/server/routes/resolver/children.ts
@@ -4,87 +4,32 @@
* you may not use this file except in compliance with the Elastic License.
*/
-import _ from 'lodash';
-import { schema } from '@kbn/config-schema';
import { RequestHandler, Logger } from 'kibana/server';
-import { extractEntityID } from './utils/normalize';
-import { getPaginationParams } from './utils/pagination';
-import { LifecycleQuery } from './queries/lifecycle';
-import { ChildrenQuery } from './queries/children';
+import { TypeOf } from '@kbn/config-schema';
+import { validateChildren } from '../../../common/schema/resolver';
+import { Fetcher } from './utils/fetch';
import { EndpointAppContext } from '../../types';
-interface ChildrenQueryParams {
- after?: string;
- limit: number;
- /**
- * legacyEndpointID is optional because there are two different types of identifiers:
- *
- * Legacy
- * A legacy Entity ID is made up of the agent.id and unique_pid fields. The client will need to identify if
- * it's looking at a legacy event and use those fields when making requests to the backend. The
- * request would be /resolver/{id}?legacyEndpointID=and the {id} would be the unique_pid.
- *
- * Elastic Endpoint
- * When interacting with the new form of data the client doesn't need the legacyEndpointID because it's already a
- * part of the entityID in the new type of event. So for the same request the client would just hit resolver/{id}
- * and the {id} would be entityID stored in the event's process.entity_id field.
- */
- legacyEndpointID?: string;
-}
-
-interface ChildrenPathParams {
- id: string;
-}
-
-export const validateChildren = {
- params: schema.object({ id: schema.string() }),
- query: schema.object({
- after: schema.maybe(schema.string()),
- limit: schema.number({ defaultValue: 10, min: 1, max: 100 }),
- legacyEndpointID: schema.maybe(schema.string()),
- }),
-};
-
export function handleChildren(
log: Logger,
endpointAppContext: EndpointAppContext
-): RequestHandler {
+): RequestHandler, TypeOf> {
return async (context, req, res) => {
const {
params: { id },
- query: { limit, after, legacyEndpointID },
+ query: { children, generations, afterChild, legacyEndpointID: endpointID },
} = req;
try {
const indexRetriever = endpointAppContext.service.getIndexPatternRetriever();
- const pagination = getPaginationParams(limit, after);
const indexPattern = await indexRetriever.getEventIndexPattern(context);
const client = context.core.elasticsearch.dataClient;
- const childrenQuery = new ChildrenQuery(indexPattern, legacyEndpointID, pagination);
- const lifecycleQuery = new LifecycleQuery(indexPattern, legacyEndpointID);
-
- // Retrieve the related child process events for a given process
- const { total, results: events, nextCursor } = await childrenQuery.search(client, id);
- const childIDs = events.map(extractEntityID);
-
- // Retrieve the lifecycle events for the child processes (e.g. started, terminated etc)
- // this needs to fire after the above since we don't yet have the entity ids until we
- // run the first query
- const { results: lifecycleEvents } = await lifecycleQuery.search(client, ...childIDs);
- // group all of the lifecycle events by the child process id
- const lifecycleGroups = Object.values(_.groupBy(lifecycleEvents, extractEntityID));
- const children = lifecycleGroups.map(group => ({ lifecycle: group }));
+ const fetcher = new Fetcher(client, id, indexPattern, endpointID);
+ const tree = await fetcher.children(children, generations, afterChild);
return res.ok({
- body: {
- children,
- pagination: {
- total,
- next: nextCursor,
- limit,
- },
- },
+ body: tree.render(),
});
} catch (err) {
log.warn(err);
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/events.ts b/x-pack/plugins/endpoint/server/routes/resolver/events.ts
new file mode 100644
index 0000000000000..a70a6e8d097d0
--- /dev/null
+++ b/x-pack/plugins/endpoint/server/routes/resolver/events.ts
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { TypeOf } from '@kbn/config-schema';
+import { RequestHandler, Logger } from 'kibana/server';
+import { validateEvents } from '../../../common/schema/resolver';
+import { Fetcher } from './utils/fetch';
+import { EndpointAppContext } from '../../types';
+
+export function handleEvents(
+ log: Logger,
+ endpointAppContext: EndpointAppContext
+): RequestHandler, TypeOf> {
+ return async (context, req, res) => {
+ const {
+ params: { id },
+ query: { events, afterEvent, legacyEndpointID: endpointID },
+ } = req;
+ try {
+ const indexRetriever = endpointAppContext.service.getIndexPatternRetriever();
+ const client = context.core.elasticsearch.dataClient;
+ const indexPattern = await indexRetriever.getEventIndexPattern(context);
+
+ const fetcher = new Fetcher(client, id, indexPattern, endpointID);
+ const tree = await fetcher.events(events, afterEvent);
+
+ return res.ok({
+ body: tree.render(),
+ });
+ } catch (err) {
+ log.warn(err);
+ return res.internalError({ body: err });
+ }
+ };
+}
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts b/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts
deleted file mode 100644
index 91a4c5d49bc54..0000000000000
--- a/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { schema } from '@kbn/config-schema';
-import { RequestHandler, Logger } from 'kibana/server';
-import { extractParentEntityID } from './utils/normalize';
-import { LifecycleQuery } from './queries/lifecycle';
-import { ResolverEvent } from '../../../common/types';
-import { EndpointAppContext } from '../../types';
-
-interface LifecycleQueryParams {
- ancestors: number;
- /**
- * legacyEndpointID is optional because there are two different types of identifiers:
- *
- * Legacy
- * A legacy Entity ID is made up of the agent.id and unique_pid fields. The client will need to identify if
- * it's looking at a legacy event and use those fields when making requests to the backend. The
- * request would be /resolver/{id}?legacyEndpointID=and the {id} would be the unique_pid.
- *
- * Elastic Endpoint
- * When interacting with the new form of data the client doesn't need the legacyEndpointID because it's already a
- * part of the entityID in the new type of event. So for the same request the client would just hit resolver/{id}
- * and the {id} would be entityID stored in the event's process.entity_id field.
- */
- legacyEndpointID?: string;
-}
-
-interface LifecyclePathParams {
- id: string;
-}
-
-export const validateLifecycle = {
- params: schema.object({ id: schema.string() }),
- query: schema.object({
- ancestors: schema.number({ defaultValue: 0, min: 0, max: 10 }),
- legacyEndpointID: schema.maybe(schema.string()),
- }),
-};
-
-function getParentEntityID(results: ResolverEvent[]) {
- return results.length === 0 ? undefined : extractParentEntityID(results[0]);
-}
-
-export function handleLifecycle(
- log: Logger,
- endpointAppContext: EndpointAppContext
-): RequestHandler {
- return async (context, req, res) => {
- const {
- params: { id },
- query: { ancestors, legacyEndpointID },
- } = req;
- try {
- const indexRetriever = endpointAppContext.service.getIndexPatternRetriever();
- const ancestorLifecycles = [];
- const client = context.core.elasticsearch.dataClient;
- const indexPattern = await indexRetriever.getEventIndexPattern(context);
- const lifecycleQuery = new LifecycleQuery(indexPattern, legacyEndpointID);
- const { results: processLifecycle } = await lifecycleQuery.search(client, id);
- let nextParentID = getParentEntityID(processLifecycle);
-
- if (nextParentID) {
- for (let i = 0; i < ancestors; i++) {
- const { results: lifecycle } = await lifecycleQuery.search(client, nextParentID);
- nextParentID = getParentEntityID(lifecycle);
-
- if (!nextParentID) {
- break;
- }
-
- ancestorLifecycles.push({
- lifecycle,
- });
- }
- }
-
- return res.ok({
- body: {
- lifecycle: processLifecycle,
- ancestors: ancestorLifecycles,
- pagination: {
- next: nextParentID || null,
- ancestors,
- },
- },
- });
- } catch (err) {
- log.warn(err);
- return res.internalError({ body: err });
- }
- };
-}
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts
index 3b3b4b0c9e8ac..eba4e5581c136 100644
--- a/x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts
+++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts
@@ -4,8 +4,15 @@
* you may not use this file except in compliance with the Elastic License.
*/
+import { SearchResponse } from 'elasticsearch';
import { IScopedClusterClient } from 'kibana/server';
-import { paginate, paginatedResults, PaginationParams } from '../utils/pagination';
+import { ResolverEvent } from '../../../../common/types';
+import {
+ paginate,
+ paginatedResults,
+ PaginationParams,
+ PaginatedResults,
+} from '../utils/pagination';
import { JsonObject } from '../../../../../../../src/plugins/kibana_utils/public';
import { legacyEventIndexPattern } from './legacy_event_index_pattern';
@@ -16,11 +23,13 @@ export abstract class ResolverQuery {
private readonly pagination?: PaginationParams
) {}
- protected paginateBy(field: string, query: JsonObject) {
- if (!this.pagination) {
- return query;
- }
- return paginate(this.pagination, field, query);
+ protected paginateBy(tiebreaker: string, aggregator: string) {
+ return (query: JsonObject) => {
+ if (!this.pagination) {
+ return query;
+ }
+ return paginate(this.pagination, tiebreaker, aggregator, query);
+ };
}
build(...ids: string[]) {
@@ -31,7 +40,11 @@ export abstract class ResolverQuery {
}
async search(client: IScopedClusterClient, ...ids: string[]) {
- return paginatedResults(await client.callAsCurrentUser('search', this.build(...ids)));
+ return this.postSearch(await client.callAsCurrentUser('search', this.build(...ids)));
+ }
+
+ protected postSearch(response: SearchResponse): PaginatedResults {
+ return paginatedResults(response);
}
protected abstract legacyQuery(
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/children.test.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/children.test.ts
index d1d090579f0cd..2a097e87c38b2 100644
--- a/x-pack/plugins/endpoint/server/routes/resolver/queries/children.test.ts
+++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/children.test.ts
@@ -32,15 +32,28 @@ describe('children events query', () => {
term: { 'event.category': 'process' },
},
{
- term: { 'event.type': 'process_start' },
+ term: { 'event.kind': 'event' },
+ },
+ {
+ bool: {
+ should: [
+ {
+ term: { 'event.type': 'process_start' },
+ },
+ {
+ term: { 'event.action': 'fork_event' },
+ },
+ ],
+ },
},
],
},
},
aggs: {
- total: {
- value_count: {
- field: 'endgame.serial_event_id',
+ totals: {
+ terms: {
+ field: 'endgame.unique_ppid',
+ size: 1,
},
},
},
@@ -67,20 +80,14 @@ describe('children events query', () => {
bool: {
filter: [
{
- bool: {
- should: [
- {
- terms: { 'endpoint.process.parent.entity_id': ['baz'] },
- },
- {
- terms: { 'process.parent.entity_id': ['baz'] },
- },
- ],
- },
+ terms: { 'process.parent.entity_id': ['baz'] },
},
{
term: { 'event.category': 'process' },
},
+ {
+ term: { 'event.kind': 'event' },
+ },
{
term: { 'event.type': 'start' },
},
@@ -88,9 +95,10 @@ describe('children events query', () => {
},
},
aggs: {
- total: {
- value_count: {
- field: 'event.id',
+ totals: {
+ terms: {
+ field: 'process.parent.entity_id',
+ size: 1,
},
},
},
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/children.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/children.ts
index 6d084a0cf20e5..690c926d7e6d6 100644
--- a/x-pack/plugins/endpoint/server/routes/resolver/queries/children.ts
+++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/children.ts
@@ -7,8 +7,9 @@ import { ResolverQuery } from './base';
export class ChildrenQuery extends ResolverQuery {
protected legacyQuery(endpointID: string, uniquePIDs: string[], index: string) {
+ const paginator = this.paginateBy('endgame.serial_event_id', 'endgame.unique_ppid');
return {
- body: this.paginateBy('endgame.serial_event_id', {
+ body: paginator({
query: {
bool: {
filter: [
@@ -22,11 +23,19 @@ export class ChildrenQuery extends ResolverQuery {
term: { 'event.category': 'process' },
},
{
- // Corner case, we could only have a process_running or process_terminated
- // so to solve this we'll probably want to either search for all of them and only return one if that's
- // possible in elastic search or in memory pull out a single event to return
- // https://github.com/elastic/endpoint-app-team/issues/168
- term: { 'event.type': 'process_start' },
+ term: { 'event.kind': 'event' },
+ },
+ {
+ bool: {
+ should: [
+ {
+ term: { 'event.type': 'process_start' },
+ },
+ {
+ term: { 'event.action': 'fork_event' },
+ },
+ ],
+ },
},
],
},
@@ -37,31 +46,22 @@ export class ChildrenQuery extends ResolverQuery {
}
protected query(entityIDs: string[], index: string) {
+ const paginator = this.paginateBy('event.id', 'process.parent.entity_id');
return {
- body: this.paginateBy('event.id', {
+ body: paginator({
query: {
bool: {
filter: [
{
- bool: {
- should: [
- {
- terms: { 'endpoint.process.parent.entity_id': entityIDs },
- },
- {
- terms: { 'process.parent.entity_id': entityIDs },
- },
- ],
- },
+ terms: { 'process.parent.entity_id': entityIDs },
},
{
term: { 'event.category': 'process' },
},
{
- // Corner case, we could only have a process_running or process_terminated
- // so to solve this we'll probably want to either search for all of them and only return one if that's
- // possible in elastic search or in memory pull out a single event to return
- // https://github.com/elastic/endpoint-app-team/issues/168
+ term: { 'event.kind': 'event' },
+ },
+ {
term: { 'event.type': 'start' },
},
],
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.test.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/events.test.ts
similarity index 75%
rename from x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.test.ts
rename to x-pack/plugins/endpoint/server/routes/resolver/queries/events.test.ts
index 26205e9db6e45..78e5ee9226581 100644
--- a/x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.test.ts
+++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/events.test.ts
@@ -3,7 +3,7 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
-import { RelatedEventsQuery } from './related_events';
+import { EventsQuery } from './events';
import { fakeEventIndexPattern } from './children.test';
import { legacyEventIndexPattern } from './legacy_event_index_pattern';
@@ -11,7 +11,7 @@ describe('related events query', () => {
it('generates the correct legacy queries', () => {
const timestamp = new Date().getTime();
expect(
- new RelatedEventsQuery(legacyEventIndexPattern, 'awesome-id', {
+ new EventsQuery(legacyEventIndexPattern, 'awesome-id', {
size: 1,
timestamp,
eventID: 'foo',
@@ -27,6 +27,9 @@ describe('related events query', () => {
{
term: { 'agent.id': 'awesome-id' },
},
+ {
+ term: { 'event.kind': 'event' },
+ },
{
bool: {
must_not: {
@@ -38,9 +41,10 @@ describe('related events query', () => {
},
},
aggs: {
- total: {
- value_count: {
- field: 'endgame.serial_event_id',
+ totals: {
+ terms: {
+ field: 'endgame.unique_pid',
+ size: 1,
},
},
},
@@ -56,7 +60,7 @@ describe('related events query', () => {
const timestamp = new Date().getTime();
expect(
- new RelatedEventsQuery(fakeEventIndexPattern, undefined, {
+ new EventsQuery(fakeEventIndexPattern, undefined, {
size: 1,
timestamp,
eventID: 'bar',
@@ -67,16 +71,10 @@ describe('related events query', () => {
bool: {
filter: [
{
- bool: {
- should: [
- {
- terms: { 'endpoint.process.entity_id': ['baz'] },
- },
- {
- terms: { 'process.entity_id': ['baz'] },
- },
- ],
- },
+ terms: { 'process.entity_id': ['baz'] },
+ },
+ {
+ term: { 'event.kind': 'event' },
},
{
bool: {
@@ -89,9 +87,10 @@ describe('related events query', () => {
},
},
aggs: {
- total: {
- value_count: {
- field: 'event.id',
+ totals: {
+ terms: {
+ field: 'process.entity_id',
+ size: 1,
},
},
},
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/events.ts
similarity index 74%
rename from x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.ts
rename to x-pack/plugins/endpoint/server/routes/resolver/queries/events.ts
index cc5afe8face8d..b622cb8a21111 100644
--- a/x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.ts
+++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/events.ts
@@ -6,10 +6,11 @@
import { ResolverQuery } from './base';
import { JsonObject } from '../../../../../../../src/plugins/kibana_utils/public';
-export class RelatedEventsQuery extends ResolverQuery {
+export class EventsQuery extends ResolverQuery {
protected legacyQuery(endpointID: string, uniquePIDs: string[], index: string): JsonObject {
+ const paginator = this.paginateBy('endgame.serial_event_id', 'endgame.unique_pid');
return {
- body: this.paginateBy('endgame.serial_event_id', {
+ body: paginator({
query: {
bool: {
filter: [
@@ -19,6 +20,9 @@ export class RelatedEventsQuery extends ResolverQuery {
{
term: { 'agent.id': endpointID },
},
+ {
+ term: { 'event.kind': 'event' },
+ },
{
bool: {
must_not: {
@@ -35,22 +39,17 @@ export class RelatedEventsQuery extends ResolverQuery {
}
protected query(entityIDs: string[], index: string): JsonObject {
+ const paginator = this.paginateBy('event.id', 'process.entity_id');
return {
- body: this.paginateBy('event.id', {
+ body: paginator({
query: {
bool: {
filter: [
{
- bool: {
- should: [
- {
- terms: { 'endpoint.process.entity_id': entityIDs },
- },
- {
- terms: { 'process.entity_id': entityIDs },
- },
- ],
- },
+ terms: { 'process.entity_id': entityIDs },
+ },
+ {
+ term: { 'event.kind': 'event' },
},
{
bool: {
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.test.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.test.ts
index 9d3c675ecc54c..296135af83b72 100644
--- a/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.test.ts
+++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.test.ts
@@ -21,12 +21,16 @@ describe('lifecycle query', () => {
{
term: { 'agent.id': 'awesome-id' },
},
+ {
+ term: { 'event.kind': 'event' },
+ },
{
term: { 'event.category': 'process' },
},
],
},
},
+ size: 10000,
sort: [{ '@timestamp': 'asc' }],
},
index: legacyEventIndexPattern,
@@ -40,16 +44,10 @@ describe('lifecycle query', () => {
bool: {
filter: [
{
- bool: {
- should: [
- {
- terms: { 'endpoint.process.entity_id': ['baz'] },
- },
- {
- terms: { 'process.entity_id': ['baz'] },
- },
- ],
- },
+ terms: { 'process.entity_id': ['baz'] },
+ },
+ {
+ term: { 'event.kind': 'event' },
},
{
term: { 'event.category': 'process' },
@@ -57,6 +55,7 @@ describe('lifecycle query', () => {
],
},
},
+ size: 10000,
sort: [{ '@timestamp': 'asc' }],
},
index: fakeEventIndexPattern,
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.ts
index 290c601e0e9d8..e775b0cf9b6d2 100644
--- a/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.ts
+++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.ts
@@ -6,7 +6,6 @@
import { ResolverQuery } from './base';
import { JsonObject } from '../../../../../../../src/plugins/kibana_utils/public';
-// consider limiting the response size to a reasonable value in case we have a bunch of lifecycle events
export class LifecycleQuery extends ResolverQuery {
protected legacyQuery(endpointID: string, uniquePIDs: string[], index: string): JsonObject {
return {
@@ -20,12 +19,16 @@ export class LifecycleQuery extends ResolverQuery {
{
term: { 'agent.id': endpointID },
},
+ {
+ term: { 'event.kind': 'event' },
+ },
{
term: { 'event.category': 'process' },
},
],
},
},
+ size: 10000,
sort: [{ '@timestamp': 'asc' }],
},
index,
@@ -39,16 +42,10 @@ export class LifecycleQuery extends ResolverQuery {
bool: {
filter: [
{
- bool: {
- should: [
- {
- terms: { 'endpoint.process.entity_id': entityIDs },
- },
- {
- terms: { 'process.entity_id': entityIDs },
- },
- ],
- },
+ terms: { 'process.entity_id': entityIDs },
+ },
+ {
+ term: { 'event.kind': 'event' },
},
{
term: { 'event.category': 'process' },
@@ -56,6 +53,7 @@ export class LifecycleQuery extends ResolverQuery {
],
},
},
+ size: 10000,
sort: [{ '@timestamp': 'asc' }],
},
index,
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/stats.test.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/stats.test.ts
new file mode 100644
index 0000000000000..17a158aec7cf5
--- /dev/null
+++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/stats.test.ts
@@ -0,0 +1,188 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { legacyEventIndexPattern } from './legacy_event_index_pattern';
+import { StatsQuery } from './stats';
+import { fakeEventIndexPattern } from './children.test';
+
+describe('stats query', () => {
+ it('generates the correct legacy queries', () => {
+ expect(new StatsQuery(legacyEventIndexPattern, 'awesome-id').build('5')).toStrictEqual({
+ body: {
+ size: 0,
+ query: {
+ bool: {
+ filter: [
+ {
+ term: {
+ 'agent.id': 'awesome-id',
+ },
+ },
+ {
+ bool: {
+ should: [
+ {
+ bool: {
+ filter: [
+ {
+ term: {
+ 'event.kind': 'event',
+ },
+ },
+ {
+ terms: {
+ 'endgame.unique_pid': ['5'],
+ },
+ },
+ {
+ bool: {
+ must_not: {
+ term: {
+ 'event.category': 'process',
+ },
+ },
+ },
+ },
+ ],
+ },
+ },
+ {
+ bool: {
+ filter: [
+ {
+ term: {
+ 'event.kind': 'alert',
+ },
+ },
+ {
+ terms: {
+ 'endgame.data.alert_details.acting_process.unique_pid': ['5'],
+ },
+ },
+ ],
+ },
+ },
+ ],
+ },
+ },
+ ],
+ },
+ },
+ aggs: {
+ alerts: {
+ filter: {
+ term: {
+ 'event.kind': 'alert',
+ },
+ },
+ aggs: {
+ ids: {
+ terms: {
+ field: 'endgame.data.alert_details.acting_process.unique_pid',
+ },
+ },
+ },
+ },
+ events: {
+ filter: {
+ term: {
+ 'event.kind': 'event',
+ },
+ },
+ aggs: {
+ ids: {
+ terms: {
+ field: 'endgame.unique_pid',
+ },
+ },
+ },
+ },
+ },
+ },
+ index: legacyEventIndexPattern,
+ });
+ });
+
+ it('generates the correct non-legacy queries', () => {
+ expect(new StatsQuery(fakeEventIndexPattern).build('baz')).toStrictEqual({
+ body: {
+ size: 0,
+ query: {
+ bool: {
+ filter: [
+ {
+ terms: {
+ 'process.entity_id': ['baz'],
+ },
+ },
+ {
+ bool: {
+ should: [
+ {
+ bool: {
+ filter: [
+ {
+ term: {
+ 'event.kind': 'event',
+ },
+ },
+ {
+ bool: {
+ must_not: {
+ term: {
+ 'event.category': 'process',
+ },
+ },
+ },
+ },
+ ],
+ },
+ },
+ {
+ term: {
+ 'event.kind': 'alert',
+ },
+ },
+ ],
+ },
+ },
+ ],
+ },
+ },
+ aggs: {
+ alerts: {
+ filter: {
+ term: {
+ 'event.kind': 'alert',
+ },
+ },
+ aggs: {
+ ids: {
+ terms: {
+ field: 'process.entity_id',
+ },
+ },
+ },
+ },
+ events: {
+ filter: {
+ term: {
+ 'event.kind': 'event',
+ },
+ },
+ aggs: {
+ ids: {
+ terms: {
+ field: 'process.entity_id',
+ },
+ },
+ },
+ },
+ },
+ },
+ index: fakeEventIndexPattern,
+ });
+ });
+});
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/stats.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/stats.ts
new file mode 100644
index 0000000000000..7db3ab2b0cb1f
--- /dev/null
+++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/stats.ts
@@ -0,0 +1,147 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { SearchResponse } from 'elasticsearch';
+import { ResolverQuery } from './base';
+import { ResolverEvent } from '../../../../common/types';
+import { JsonObject } from '../../../../../../../src/plugins/kibana_utils/public';
+import { PaginatedResults } from '../utils/pagination';
+
+export class StatsQuery extends ResolverQuery {
+ protected postSearch(response: SearchResponse): PaginatedResults {
+ const alerts = response.aggregations.alerts.ids.buckets.reduce(
+ (cummulative: any, bucket: any) => ({ ...cummulative, [bucket.key]: bucket.doc_count }),
+ {}
+ );
+ const events = response.aggregations.events.ids.buckets.reduce(
+ (cummulative: any, bucket: any) => ({ ...cummulative, [bucket.key]: bucket.doc_count }),
+ {}
+ );
+ return {
+ totals: {},
+ results: [],
+ extras: {
+ alerts,
+ events,
+ },
+ };
+ }
+
+ protected legacyQuery(endpointID: string, uniquePIDs: string[], index: string): JsonObject {
+ return {
+ body: {
+ size: 0,
+ query: {
+ bool: {
+ filter: [
+ {
+ term: { 'agent.id': endpointID },
+ },
+ {
+ bool: {
+ should: [
+ {
+ bool: {
+ filter: [
+ { term: { 'event.kind': 'event' } },
+ { terms: { 'endgame.unique_pid': uniquePIDs } },
+ {
+ bool: {
+ must_not: {
+ term: { 'event.category': 'process' },
+ },
+ },
+ },
+ ],
+ },
+ },
+ {
+ bool: {
+ filter: [
+ { term: { 'event.kind': 'alert' } },
+ {
+ terms: {
+ 'endgame.data.alert_details.acting_process.unique_pid': uniquePIDs,
+ },
+ },
+ ],
+ },
+ },
+ ],
+ },
+ },
+ ],
+ },
+ },
+ aggs: {
+ alerts: {
+ filter: { term: { 'event.kind': 'alert' } },
+ aggs: {
+ ids: { terms: { field: 'endgame.data.alert_details.acting_process.unique_pid' } },
+ },
+ },
+ events: {
+ filter: { term: { 'event.kind': 'event' } },
+ aggs: {
+ ids: { terms: { field: 'endgame.unique_pid' } },
+ },
+ },
+ },
+ },
+ index,
+ };
+ }
+
+ protected query(entityIDs: string[], index: string): JsonObject {
+ return {
+ body: {
+ size: 0,
+ query: {
+ bool: {
+ filter: [
+ { terms: { 'process.entity_id': entityIDs } },
+ {
+ bool: {
+ should: [
+ {
+ bool: {
+ filter: [
+ { term: { 'event.kind': 'event' } },
+ {
+ bool: {
+ must_not: {
+ term: { 'event.category': 'process' },
+ },
+ },
+ },
+ ],
+ },
+ },
+ { term: { 'event.kind': 'alert' } },
+ ],
+ },
+ },
+ ],
+ },
+ },
+ aggs: {
+ alerts: {
+ filter: { term: { 'event.kind': 'alert' } },
+ aggs: {
+ ids: { terms: { field: 'process.entity_id' } },
+ },
+ },
+ events: {
+ filter: { term: { 'event.kind': 'event' } },
+ aggs: {
+ ids: { terms: { field: 'process.entity_id' } },
+ },
+ },
+ },
+ },
+ index,
+ };
+ }
+}
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts b/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts
deleted file mode 100644
index 83e111a1e62e6..0000000000000
--- a/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { schema } from '@kbn/config-schema';
-import { RequestHandler, Logger } from 'kibana/server';
-import { getPaginationParams } from './utils/pagination';
-import { RelatedEventsQuery } from './queries/related_events';
-import { EndpointAppContext } from '../../types';
-
-interface RelatedEventsQueryParams {
- after?: string;
- limit: number;
- /**
- * legacyEndpointID is optional because there are two different types of identifiers:
- *
- * Legacy
- * A legacy Entity ID is made up of the agent.id and unique_pid fields. The client will need to identify if
- * it's looking at a legacy event and use those fields when making requests to the backend. The
- * request would be /resolver/{id}?legacyEndpointID=and the {id} would be the unique_pid.
- *
- * Elastic Endpoint
- * When interacting with the new form of data the client doesn't need the legacyEndpointID because it's already a
- * part of the entityID in the new type of event. So for the same request the client would just hit resolver/{id}
- * and the {id} would be entityID stored in the event's process.entity_id field.
- */
- legacyEndpointID?: string;
-}
-
-interface RelatedEventsPathParams {
- id: string;
-}
-
-export const validateRelatedEvents = {
- params: schema.object({ id: schema.string() }),
- query: schema.object({
- after: schema.maybe(schema.string()),
- limit: schema.number({ defaultValue: 100, min: 1, max: 1000 }),
- legacyEndpointID: schema.maybe(schema.string()),
- }),
-};
-
-export function handleRelatedEvents(
- log: Logger,
- endpointAppContext: EndpointAppContext
-): RequestHandler {
- return async (context, req, res) => {
- const {
- params: { id },
- query: { limit, after, legacyEndpointID },
- } = req;
- try {
- const indexRetriever = endpointAppContext.service.getIndexPatternRetriever();
- const pagination = getPaginationParams(limit, after);
-
- const client = context.core.elasticsearch.dataClient;
- const indexPattern = await indexRetriever.getEventIndexPattern(context);
- // Retrieve the related non-process events for a given process
- const relatedEventsQuery = new RelatedEventsQuery(indexPattern, legacyEndpointID, pagination);
- const relatedEvents = await relatedEventsQuery.search(client, id);
-
- const { total, results: events, nextCursor } = relatedEvents;
-
- return res.ok({
- body: {
- events,
- pagination: { total, next: nextCursor, limit },
- },
- });
- } catch (err) {
- log.warn(err);
- return res.internalError({ body: err });
- }
- };
-}
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/tree.ts b/x-pack/plugins/endpoint/server/routes/resolver/tree.ts
new file mode 100644
index 0000000000000..25f15586341d5
--- /dev/null
+++ b/x-pack/plugins/endpoint/server/routes/resolver/tree.ts
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { RequestHandler, Logger } from 'kibana/server';
+import { TypeOf } from '@kbn/config-schema';
+import { validateTree } from '../../../common/schema/resolver';
+import { Fetcher } from './utils/fetch';
+import { Tree } from './utils/tree';
+import { EndpointAppContext } from '../../types';
+
+export function handleTree(
+ log: Logger,
+ endpointAppContext: EndpointAppContext
+): RequestHandler, TypeOf> {
+ return async (context, req, res) => {
+ const {
+ params: { id },
+ query: {
+ children,
+ generations,
+ ancestors,
+ events,
+ afterEvent,
+ afterChild,
+ legacyEndpointID: endpointID,
+ },
+ } = req;
+ try {
+ const client = context.core.elasticsearch.dataClient;
+ const indexRetriever = endpointAppContext.service.getIndexPatternRetriever();
+ const indexPattern = await indexRetriever.getEventIndexPattern(context);
+
+ const fetcher = new Fetcher(client, id, indexPattern, endpointID);
+ const tree = await Tree.merge(
+ fetcher.children(children, generations, afterChild),
+ fetcher.ancestors(ancestors + 1),
+ fetcher.events(events, afterEvent)
+ );
+
+ const enrichedTree = await fetcher.stats(tree);
+
+ return res.ok({
+ body: enrichedTree.render(),
+ });
+ } catch (err) {
+ log.warn(err);
+ return res.internalError({ body: 'Error retrieving tree.' });
+ }
+ };
+}
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/utils/fetch.ts b/x-pack/plugins/endpoint/server/routes/resolver/utils/fetch.ts
new file mode 100644
index 0000000000000..7315b4ee6c618
--- /dev/null
+++ b/x-pack/plugins/endpoint/server/routes/resolver/utils/fetch.ts
@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IScopedClusterClient } from 'kibana/server';
+import { entityId, parentEntityId } from '../../../../common/models/event';
+import { getPaginationParams } from './pagination';
+import { Tree } from './tree';
+import { LifecycleQuery } from '../queries/lifecycle';
+import { ChildrenQuery } from '../queries/children';
+import { EventsQuery } from '../queries/events';
+import { StatsQuery } from '../queries/stats';
+
+export class Fetcher {
+ constructor(
+ private readonly client: IScopedClusterClient,
+ private readonly id: string,
+ private readonly indexPattern: string,
+ private readonly endpointID?: string
+ ) {}
+
+ public async ancestors(limit: number): Promise {
+ const tree = new Tree(this.id);
+ await this.doAncestors(tree, this.id, this.id, limit);
+ return tree;
+ }
+
+ public async children(limit: number, generations: number, after?: string): Promise {
+ const tree = new Tree(this.id);
+ await this.doChildren(tree, [this.id], limit, generations, after);
+ return tree;
+ }
+
+ public async events(limit: number, after?: string): Promise {
+ const tree = new Tree(this.id);
+ await this.doEvents(tree, limit, after);
+ return tree;
+ }
+
+ public async stats(tree: Tree): Promise {
+ await this.doStats(tree);
+ return tree;
+ }
+
+ private async doAncestors(tree: Tree, curNode: string, previousNode: string, levels: number) {
+ if (levels === 0) {
+ tree.setNextAncestor(curNode);
+ return;
+ }
+
+ const query = new LifecycleQuery(this.indexPattern, this.endpointID);
+ const { results } = await query.search(this.client, curNode);
+
+ if (results.length === 0) {
+ tree.setNextAncestor(null);
+ return;
+ }
+ tree.addAncestor(previousNode, ...results);
+
+ const next = parentEntityId(results[0]);
+ if (next !== undefined) {
+ await this.doAncestors(tree, next, curNode, levels - 1);
+ }
+ }
+
+ private async doEvents(tree: Tree, limit: number, after?: string) {
+ const query = new EventsQuery(
+ this.indexPattern,
+ this.endpointID,
+ getPaginationParams(limit, after)
+ );
+
+ const { totals, results } = await query.search(this.client, this.id);
+ tree.addEvent(...results);
+ tree.paginateEvents(totals, results);
+ if (results.length === 0) tree.setNextEvent(null);
+ }
+
+ private async doChildren(
+ tree: Tree,
+ ids: string[],
+ limit: number,
+ levels: number,
+ after?: string
+ ) {
+ if (levels === 0 || ids.length === 0) return;
+
+ const childrenQuery = new ChildrenQuery(
+ this.indexPattern,
+ this.endpointID,
+ getPaginationParams(limit, after)
+ );
+ const lifecycleQuery = new LifecycleQuery(this.indexPattern, this.endpointID);
+
+ const { totals, results } = await childrenQuery.search(this.client, ...ids);
+ if (results.length === 0) {
+ tree.markLeafNode(...ids);
+ return;
+ }
+
+ const childIDs = results.map(entityId);
+ const children = (await lifecycleQuery.search(this.client, ...childIDs)).results;
+
+ tree.addChild(...children);
+ tree.paginateChildren(totals, results);
+ tree.markLeafNode(...childIDs);
+
+ await this.doChildren(tree, childIDs, limit * limit, levels - 1);
+ }
+
+ private async doStats(tree: Tree) {
+ const statsQuery = new StatsQuery(this.indexPattern, this.endpointID);
+ const ids = tree.ids();
+ const { extras } = await statsQuery.search(this.client, ...ids);
+ const alerts = extras?.alerts || {};
+ const events = extras?.events || {};
+ ids.forEach(id => {
+ tree.addStats(id, { totalAlerts: alerts[id] || 0, totalEvents: events[id] || 0 });
+ });
+ }
+}
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/utils/normalize.ts b/x-pack/plugins/endpoint/server/routes/resolver/utils/normalize.ts
deleted file mode 100644
index 6d5ac8efdc1da..0000000000000
--- a/x-pack/plugins/endpoint/server/routes/resolver/utils/normalize.ts
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { ResolverEvent } from '../../../../common/types';
-import { isLegacyEvent } from '../../../../common/models/event';
-
-export function extractEventID(event: ResolverEvent) {
- if (isLegacyEvent(event)) {
- return String(event.endgame.serial_event_id);
- }
- return event.event.id;
-}
-
-export function extractEntityID(event: ResolverEvent) {
- if (isLegacyEvent(event)) {
- return String(event.endgame.unique_pid);
- }
- return event.process.entity_id;
-}
-
-export function extractParentEntityID(event: ResolverEvent) {
- if (isLegacyEvent(event)) {
- const ppid = event.endgame.unique_ppid;
- return ppid && String(ppid); // if unique_ppid is undefined return undefined
- }
- return event.process.parent?.entity_id;
-}
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/utils/pagination.ts b/x-pack/plugins/endpoint/server/routes/resolver/utils/pagination.ts
index 5a64f3ff9ddb6..20249b81660bb 100644
--- a/x-pack/plugins/endpoint/server/routes/resolver/utils/pagination.ts
+++ b/x-pack/plugins/endpoint/server/routes/resolver/utils/pagination.ts
@@ -6,7 +6,7 @@
import { SearchResponse } from 'elasticsearch';
import { ResolverEvent } from '../../../../common/types';
-import { extractEventID } from './normalize';
+import { entityId } from '../../../../common/models/event';
import { JsonObject } from '../../../../../../../src/plugins/kibana_utils/public';
export interface PaginationParams {
@@ -15,12 +15,19 @@ export interface PaginationParams {
eventID?: string;
}
+export interface PaginatedResults {
+ totals: Record;
+ results: ResolverEvent[];
+ // content holder for any other extra aggregation counts
+ extras?: Record>;
+}
+
interface PaginationCursor {
timestamp: number;
eventID: string;
}
-function urlEncodeCursor(data: PaginationCursor) {
+function urlEncodeCursor(data: PaginationCursor): string {
const value = JSON.stringify(data);
return Buffer.from(value, 'utf8')
.toString('base64')
@@ -56,10 +63,16 @@ export function getPaginationParams(limit: number, after?: string): PaginationPa
return { size: limit };
}
-export function paginate(pagination: PaginationParams, field: string, query: JsonObject) {
+export function paginate(
+ pagination: PaginationParams,
+ tiebreaker: string,
+ aggregator: string,
+ query: JsonObject
+): JsonObject {
const { size, timestamp, eventID } = pagination;
- query.sort = [{ '@timestamp': 'asc' }, { [field]: 'asc' }];
- query.aggs = { total: { value_count: { field } } };
+ query.sort = [{ '@timestamp': 'asc' }, { [tiebreaker]: 'asc' }];
+ query.aggs = query.aggs || {};
+ query.aggs = Object.assign({}, query.aggs, { totals: { terms: { field: aggregator, size } } });
query.size = size;
if (timestamp && eventID) {
query.search_after = [timestamp, eventID] as Array;
@@ -67,25 +80,28 @@ export function paginate(pagination: PaginationParams, field: string, query: Jso
return query;
}
-export function paginatedResults(
- response: SearchResponse
-): { total: number; results: ResolverEvent[]; nextCursor: string | null } {
- const total = response.aggregations?.total?.value || 0;
- if (response.hits.hits.length === 0) {
- return { total, results: [], nextCursor: null };
+export function buildPaginationCursor(total: number, results: ResolverEvent[]): string | null {
+ if (total > results.length && results.length > 0) {
+ const lastResult = results[results.length - 1];
+ const cursor = {
+ timestamp: lastResult['@timestamp'],
+ eventID: entityId(lastResult),
+ };
+ return urlEncodeCursor(cursor);
}
+ return null;
+}
- const results: ResolverEvent[] = [];
- for (const hit of response.hits.hits) {
- results.push(hit._source);
+export function paginatedResults(response: SearchResponse): PaginatedResults {
+ if (response.hits.hits.length === 0) {
+ return { totals: {}, results: [] };
}
- // results will be at least 1 because of length check at the top of the function
- const next = results[results.length - 1];
- const cursor = {
- timestamp: next['@timestamp'],
- eventID: extractEventID(next),
- };
+ const totals = response.aggregations?.totals?.buckets?.reduce(
+ (cummulative: any, bucket: any) => ({ ...cummulative, [bucket.key]: bucket.doc_count }),
+ {}
+ );
- return { total, results, nextCursor: urlEncodeCursor(cursor) };
+ const results = response.hits.hits.map(hit => hit._source);
+ return { totals, results };
}
diff --git a/x-pack/plugins/endpoint/server/routes/resolver/utils/tree.ts b/x-pack/plugins/endpoint/server/routes/resolver/utils/tree.ts
new file mode 100644
index 0000000000000..5a55c23b90873
--- /dev/null
+++ b/x-pack/plugins/endpoint/server/routes/resolver/utils/tree.ts
@@ -0,0 +1,230 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import _ from 'lodash';
+import {
+ ResolverEvent,
+ ResolverNode,
+ ResolverNodeStats,
+ ResolverNodePagination,
+} from '../../../../common/types';
+import { entityId, parentEntityId } from '../../../../common/models/event';
+import { buildPaginationCursor } from './pagination';
+
+type ExtractFunction = (event: ResolverEvent) => string | undefined;
+
+function createNode(id: string): ResolverNode {
+ return { id, children: [], pagination: {}, events: [], lifecycle: [] };
+}
+/**
+ * This class aids in constructing a tree of process events. It works in the following way:
+ *
+ * 1. We construct a tree structure starting with the root node for the event we're requesting.
+ * 2. We leverage the ability to pass hashes and arrays by reference to construct a fast cache of
+ * process identifiers that updates the tree structure as we push values into the cache.
+ *
+ * When we query a single level of results for child process events we have a flattened, sorted result
+ * list that we need to add into a constructed tree. We also need to signal in an API response whether
+ * or not there are more child processes events that we have not yet retrieved, and, if so, for what parent
+ * process. So, at the end of our tree construction we have a relational layout of the events with no
+ * pagination information for the given parent nodes. In order to actually construct both the tree and
+ * insert the pagination information we basically do the following:
+ *
+ * 1. Using a terms aggregation query, we return an approximate roll-up of the number of child process
+ * "creation" events, this gives us an estimation of the number of associated children per parent
+ * 2. We feed these child process creation event "unique identifiers" (basically a process.entity_id)
+ * into a second query to get the current state of the process via its "lifecycle" events.
+ * 3. We construct the tree above with the "lifecycle" events.
+ * 4. Using the terms query results, we mark each non-leaf node with the number of expected children, if our
+ * tree has less children than expected, we create a pagination cursor to indicate "we have a truncated set
+ * of values".
+ * 5. We mark each leaf node (the last level of the tree we're constructing) with a "null" for the expected
+ * number of children to indicate "we have not yet attempted to get any children".
+ *
+ * Following this scheme, we use exactly 2 queries per level of children that we return--one for the pagination
+ * and one for the lifecycle events of the processes. The downside to this is that we need to dynamically expand
+ * the number of documents we can retrieve per level due to the exponential fanout of child processes,
+ * what this means is that noisy neighbors for a given level may hide other child process events that occur later
+ * temporally in the same level--so, while a heavily forking process might get shown, maybe the actually malicious
+ * event doesn't show up in the tree at the beginning.
+ */
+
+export class Tree {
+ protected cache: Map;
+ protected root: ResolverNode;
+ protected id: string;
+
+ constructor(id: string) {
+ const root = createNode(id);
+ this.id = id;
+ this.cache = new Map();
+ this.root = root;
+ this.cache.set(id, root);
+ }
+
+ public render(): ResolverNode {
+ return this.root;
+ }
+
+ public ids(): string[] {
+ return [...this.cache.keys()];
+ }
+
+ public static async merge(
+ childrenPromise: Promise,
+ ancestorsPromise: Promise,
+ eventsPromise: Promise
+ ): Promise {
+ const [children, ancestors, events] = await Promise.all([
+ childrenPromise,
+ ancestorsPromise,
+ eventsPromise,
+ ]);
+
+ /*
+ * we only allow for merging when we have partial trees that
+ * represent the same root node
+ */
+ const rootID = children.id;
+ if (rootID !== ancestors.id || rootID !== events.id) {
+ throw new Error('cannot merge trees with different roots');
+ }
+
+ Object.entries(ancestors.cache).forEach(([id, node]) => {
+ if (rootID !== id) {
+ children.cache.set(id, node);
+ }
+ });
+
+ children.root.lifecycle = ancestors.root.lifecycle;
+ children.root.ancestors = ancestors.root.ancestors;
+ children.root.events = events.root.events;
+
+ Object.assign(children.root.pagination, ancestors.root.pagination, events.root.pagination);
+
+ return children;
+ }
+
+ public addEvent(...events: ResolverEvent[]): void {
+ events.forEach(event => {
+ const id = entityId(event);
+
+ this.ensureCache(id);
+ const currentNode = this.cache.get(id);
+ if (currentNode !== undefined) {
+ currentNode.events.push(event);
+ }
+ });
+ }
+
+ public addAncestor(id: string, ...events: ResolverEvent[]): void {
+ events.forEach(event => {
+ const ancestorID = entityId(event);
+ if (this.cache.get(ancestorID) === undefined) {
+ const newParent = createNode(ancestorID);
+ this.cache.set(ancestorID, newParent);
+ if (!this.root.ancestors) {
+ this.root.ancestors = [];
+ }
+ this.root.ancestors.push(newParent);
+ }
+ const currentAncestor = this.cache.get(ancestorID);
+ if (currentAncestor !== undefined) {
+ currentAncestor.lifecycle.push(event);
+ }
+ });
+ }
+
+ public addStats(id: string, stats: ResolverNodeStats): void {
+ this.ensureCache(id);
+ const currentNode = this.cache.get(id);
+ if (currentNode !== undefined) {
+ currentNode.stats = stats;
+ }
+ }
+
+ public setNextAncestor(next: string | null): void {
+ this.root.pagination.nextAncestor = next;
+ }
+
+ public setNextEvent(next: string | null): void {
+ this.root.pagination.nextEvent = next;
+ }
+
+ public setNextAlert(next: string | null): void {
+ this.root.pagination.nextAlert = next;
+ }
+
+ public addChild(...events: ResolverEvent[]): void {
+ events.forEach(event => {
+ const id = entityId(event);
+ const parentID = parentEntityId(event);
+
+ this.ensureCache(parentID);
+ let currentNode = this.cache.get(id);
+
+ if (currentNode === undefined) {
+ currentNode = createNode(id);
+ this.cache.set(id, currentNode);
+ if (parentID !== undefined) {
+ const parentNode = this.cache.get(parentID);
+ if (parentNode !== undefined) {
+ parentNode.children.push(currentNode);
+ }
+ }
+ }
+ currentNode.lifecycle.push(event);
+ });
+ }
+
+ public markLeafNode(...ids: string[]): void {
+ ids.forEach(id => {
+ this.ensureCache(id);
+ const currentNode = this.cache.get(id);
+ if (currentNode !== undefined && !currentNode.pagination.nextChild) {
+ currentNode.pagination.nextChild = null;
+ }
+ });
+ }
+
+ public paginateEvents(totals: Record, events: ResolverEvent[]): void {
+ return this.paginate(entityId, 'nextEvent', totals, events);
+ }
+
+ public paginateChildren(totals: Record, children: ResolverEvent[]): void {
+ return this.paginate(parentEntityId, 'nextChild', totals, children);
+ }
+
+ private paginate(
+ grouper: ExtractFunction,
+ attribute: keyof ResolverNodePagination,
+ totals: Record,
+ records: ResolverEvent[]
+ ): void {
+ const grouped = _.groupBy(records, grouper);
+ Object.entries(totals).forEach(([id, total]) => {
+ if (this.cache.get(id) !== undefined) {
+ if (grouped[id]) {
+ /*
+ * if we have any results, attempt to build a pagination cursor, the function
+ * below hands back a null value if no cursor is necessary because we have
+ * all of the records.
+ */
+ const currentNode = this.cache.get(id);
+ if (currentNode !== undefined) {
+ currentNode.pagination[attribute] = buildPaginationCursor(total, grouped[id]);
+ }
+ }
+ }
+ });
+ }
+
+ private ensureCache(id: string | undefined): void {
+ if (id === undefined || this.cache.get(id) === undefined) {
+ throw new Error('dangling node');
+ }
+ }
+}
diff --git a/x-pack/plugins/event_log/README.md b/x-pack/plugins/event_log/README.md
index 38364033cb70b..941dedc3d1093 100644
--- a/x-pack/plugins/event_log/README.md
+++ b/x-pack/plugins/event_log/README.md
@@ -274,10 +274,16 @@ PUT _ilm/policy/event_log_policy
"hot": {
"actions": {
"rollover": {
- "max_size": "5GB",
+ "max_size": "50GB",
"max_age": "30d"
}
}
+ },
+ "delete": {
+ "min_age": "90d",
+ "actions": {
+ "delete": {}
+ }
}
}
}
@@ -285,10 +291,11 @@ PUT _ilm/policy/event_log_policy
```
This means that ILM would "rollover" the current index, say
-`.kibana-event-log-000001` by creating a new index `.kibana-event-log-000002`,
+`.kibana-event-log-8.0.0-000001` by creating a new index `.kibana-event-log-8.0.0-000002`,
which would "inherit" everything from the index template, and then ILM will
set the write index of the the alias to the new index. This would happen
-when the original index grew past 5 GB, or was created more than 30 days ago.
+when the original index grew past 50 GB, or was created more than 30 days ago.
+After rollover, the indices will be removed after 90 days to avoid disks to fill up.
For more relevant information on ILM, see:
[getting started with ILM doc][] and [write index alias behavior][]:
diff --git a/x-pack/plugins/event_log/generated/mappings.json b/x-pack/plugins/event_log/generated/mappings.json
index f487e9262e50e..0a858969c4f6a 100644
--- a/x-pack/plugins/event_log/generated/mappings.json
+++ b/x-pack/plugins/event_log/generated/mappings.json
@@ -86,6 +86,10 @@
},
"saved_objects": {
"properties": {
+ "rel": {
+ "type": "keyword",
+ "ignore_above": 1024
+ },
"namespace": {
"type": "keyword",
"ignore_above": 1024
diff --git a/x-pack/plugins/event_log/generated/schemas.ts b/x-pack/plugins/event_log/generated/schemas.ts
index 9c923fe77d035..57fe90a8e876e 100644
--- a/x-pack/plugins/event_log/generated/schemas.ts
+++ b/x-pack/plugins/event_log/generated/schemas.ts
@@ -65,6 +65,7 @@ export const EventSchema = schema.maybe(
saved_objects: schema.maybe(
schema.arrayOf(
schema.object({
+ rel: ecsString(),
namespace: ecsString(),
id: ecsString(),
type: ecsString(),
diff --git a/x-pack/plugins/event_log/scripts/mappings.js b/x-pack/plugins/event_log/scripts/mappings.js
index 8cc2c74b60e57..fd149d132031e 100644
--- a/x-pack/plugins/event_log/scripts/mappings.js
+++ b/x-pack/plugins/event_log/scripts/mappings.js
@@ -24,6 +24,11 @@ exports.EcsKibanaExtensionsMappings = {
saved_objects: {
type: 'nested',
properties: {
+ // relation; currently only supports "primary" or not set
+ rel: {
+ type: 'keyword',
+ ignore_above: 1024,
+ },
// relevant kibana space
namespace: {
type: 'keyword',
@@ -58,6 +63,7 @@ exports.EcsEventLogProperties = [
'user.name',
'kibana.server_uuid',
'kibana.alerting.instance_id',
+ 'kibana.saved_objects.rel',
'kibana.saved_objects.namespace',
'kibana.saved_objects.id',
'kibana.saved_objects.name',
diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts
index f79962a324131..66c16d0ddf383 100644
--- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts
+++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts
@@ -236,6 +236,13 @@ describe('queryEventsBySavedObject', () => {
query: {
bool: {
must: [
+ {
+ term: {
+ 'kibana.saved_objects.rel': {
+ value: 'primary',
+ },
+ },
+ },
{
term: {
'kibana.saved_objects.type': {
@@ -319,6 +326,13 @@ describe('queryEventsBySavedObject', () => {
query: {
bool: {
must: [
+ {
+ term: {
+ 'kibana.saved_objects.rel': {
+ value: 'primary',
+ },
+ },
+ },
{
term: {
'kibana.saved_objects.type': {
@@ -388,6 +402,13 @@ describe('queryEventsBySavedObject', () => {
query: {
bool: {
must: [
+ {
+ term: {
+ 'kibana.saved_objects.rel': {
+ value: 'primary',
+ },
+ },
+ },
{
term: {
'kibana.saved_objects.type': {
diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts
index 47d273b9981e3..c0ff87234c09d 100644
--- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts
+++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts
@@ -7,7 +7,7 @@
import { reject, isUndefined } from 'lodash';
import { SearchResponse, Client } from 'elasticsearch';
import { Logger, ClusterClient } from '../../../../../src/core/server';
-import { IEvent } from '../types';
+import { IEvent, SAVED_OBJECT_REL_PRIMARY } from '../types';
import { FindOptionsType } from '../event_log_client';
export type EsClusterClient = Pick;
@@ -155,6 +155,13 @@ export class ClusterClientAdapter {
query: {
bool: {
must: [
+ {
+ term: {
+ 'kibana.saved_objects.rel': {
+ value: SAVED_OBJECT_REL_PRIMARY,
+ },
+ },
+ },
{
term: {
'kibana.saved_objects.type': {
diff --git a/x-pack/plugins/event_log/server/es/documents.ts b/x-pack/plugins/event_log/server/es/documents.ts
index a6af209d6d3a0..91b3db554964f 100644
--- a/x-pack/plugins/event_log/server/es/documents.ts
+++ b/x-pack/plugins/event_log/server/es/documents.ts
@@ -31,12 +31,18 @@ export function getIlmPolicy() {
hot: {
actions: {
rollover: {
- max_size: '5GB',
+ max_size: '50GB',
max_age: '30d',
// max_docs: 1, // you know, for testing
},
},
},
+ delete: {
+ min_age: '90d',
+ actions: {
+ delete: {},
+ },
+ },
},
},
};
diff --git a/x-pack/plugins/event_log/server/event_logger.test.ts b/x-pack/plugins/event_log/server/event_logger.test.ts
index 6a745931420c0..2bda194a65d13 100644
--- a/x-pack/plugins/event_log/server/event_logger.test.ts
+++ b/x-pack/plugins/event_log/server/event_logger.test.ts
@@ -150,6 +150,35 @@ describe('EventLogger', () => {
message = await waitForLogMessage(systemLogger);
expect(message).toMatch(/invalid event logged.*action.*undefined.*/);
});
+
+ test('logs warnings when writing invalid events', async () => {
+ service.registerProviderActions('provider', ['action-a']);
+ eventLogger = service.getLogger({});
+
+ eventLogger.logEvent(({ event: { PROVIDER: 'provider' } } as unknown) as IEvent);
+ let message = await waitForLogMessage(systemLogger);
+ expect(message).toMatch(/invalid event logged.*provider.*undefined.*/);
+
+ const event: IEvent = {
+ event: {
+ provider: 'provider',
+ action: 'action-a',
+ },
+ kibana: {
+ saved_objects: [
+ {
+ rel: 'ZZZ-primary',
+ namespace: 'default',
+ type: 'event_log_test',
+ id: '123',
+ },
+ ],
+ },
+ };
+ eventLogger.logEvent(event);
+ message = await waitForLogMessage(systemLogger);
+ expect(message).toMatch(/invalid rel property.*ZZZ-primary.*/);
+ });
});
// return the next logged event; throw if not an event
diff --git a/x-pack/plugins/event_log/server/event_logger.ts b/x-pack/plugins/event_log/server/event_logger.ts
index bcfd7bd45a6f5..1a710a6fa4865 100644
--- a/x-pack/plugins/event_log/server/event_logger.ts
+++ b/x-pack/plugins/event_log/server/event_logger.ts
@@ -19,6 +19,7 @@ import {
ECS_VERSION,
EventSchema,
} from './types';
+import { SAVED_OBJECT_REL_PRIMARY } from './types';
type SystemLogger = Plugin['systemLogger'];
@@ -118,6 +119,8 @@ const RequiredEventSchema = schema.object({
action: schema.string({ minLength: 1 }),
});
+const ValidSavedObjectRels = new Set([undefined, SAVED_OBJECT_REL_PRIMARY]);
+
function validateEvent(eventLogService: IEventLogService, event: IEvent): IValidatedEvent {
if (event?.event == null) {
throw new Error(`no "event" property`);
@@ -137,7 +140,17 @@ function validateEvent(eventLogService: IEventLogService, event: IEvent): IValid
}
// could throw an error
- return EventSchema.validate(event);
+ const result = EventSchema.validate(event);
+
+ if (result?.kibana?.saved_objects?.length) {
+ for (const so of result?.kibana?.saved_objects) {
+ if (!ValidSavedObjectRels.has(so.rel)) {
+ throw new Error(`invalid rel property in saved_objects: "${so.rel}"`);
+ }
+ }
+ }
+
+ return result;
}
export const EVENT_LOGGED_PREFIX = `event logged: `;
diff --git a/x-pack/plugins/event_log/server/index.ts b/x-pack/plugins/event_log/server/index.ts
index b7fa25cb6eb9c..0612b5319c15b 100644
--- a/x-pack/plugins/event_log/server/index.ts
+++ b/x-pack/plugins/event_log/server/index.ts
@@ -8,6 +8,12 @@ import { PluginInitializerContext } from 'src/core/server';
import { ConfigSchema } from './types';
import { Plugin } from './plugin';
-export { IEventLogService, IEventLogger, IEventLogClientService, IEvent } from './types';
+export {
+ IEventLogService,
+ IEventLogger,
+ IEventLogClientService,
+ IEvent,
+ SAVED_OBJECT_REL_PRIMARY,
+} from './types';
export const config = { schema: ConfigSchema };
export const plugin = (context: PluginInitializerContext) => new Plugin(context);
diff --git a/x-pack/plugins/event_log/server/types.ts b/x-pack/plugins/event_log/server/types.ts
index baf53ef447914..58be6707b0373 100644
--- a/x-pack/plugins/event_log/server/types.ts
+++ b/x-pack/plugins/event_log/server/types.ts
@@ -13,6 +13,8 @@ import { IEvent } from '../generated/schemas';
import { FindOptionsType } from './event_log_client';
import { QueryEventsBySavedObjectResult } from './es/cluster_client_adapter';
+export const SAVED_OBJECT_REL_PRIMARY = 'primary';
+
export const ConfigSchema = schema.object({
enabled: schema.boolean({ defaultValue: true }),
logEntries: schema.boolean({ defaultValue: false }),
diff --git a/x-pack/plugins/graph/public/application.ts b/x-pack/plugins/graph/public/application.ts
index 35ec0bb2bf6ce..fa479b1b06d5e 100644
--- a/x-pack/plugins/graph/public/application.ts
+++ b/x-pack/plugins/graph/public/application.ts
@@ -10,7 +10,7 @@
import angular from 'angular';
import { i18nDirective, i18nFilter, I18nProvider } from '@kbn/i18n/angular';
import '../../../../webpackShims/ace';
-// required for i18nIdDirective
+// required for i18nIdDirective and `ngSanitize` angular module
import 'angular-sanitize';
// required for ngRoute
import 'angular-route';
diff --git a/x-pack/plugins/graph/server/saved_objects/migrations.ts b/x-pack/plugins/graph/server/saved_objects/migrations.ts
index e77d2ea0fb7c9..beb31d548c670 100644
--- a/x-pack/plugins/graph/server/saved_objects/migrations.ts
+++ b/x-pack/plugins/graph/server/saved_objects/migrations.ts
@@ -8,7 +8,7 @@ import { get } from 'lodash';
import { SavedObjectUnsanitizedDoc } from 'kibana/server';
export const graphMigrations = {
- '7.0.0': (doc: SavedObjectUnsanitizedDoc) => {
+ '7.0.0': (doc: SavedObjectUnsanitizedDoc) => {
// Set new "references" attribute
doc.references = doc.references || [];
// Migrate index pattern
diff --git a/x-pack/plugins/infra/public/components/alerting/metrics/expression.tsx b/x-pack/plugins/infra/public/components/alerting/metrics/expression.tsx
index d4d53b81109c6..f2ac34ccb752f 100644
--- a/x-pack/plugins/infra/public/components/alerting/metrics/expression.tsx
+++ b/x-pack/plugins/infra/public/components/alerting/metrics/expression.tsx
@@ -13,6 +13,9 @@ import {
EuiText,
EuiFormRow,
EuiButtonEmpty,
+ EuiCheckbox,
+ EuiToolTip,
+ EuiIcon,
EuiFieldSearch,
} from '@elastic/eui';
import { IFieldType } from 'src/plugins/data/public';
@@ -57,6 +60,7 @@ interface Props {
groupBy?: string;
filterQuery?: string;
sourceId?: string;
+ alertOnNoData?: boolean;
};
alertsContext: AlertsContextValue;
setAlertParams(key: string, value: any): void;
@@ -282,6 +286,28 @@ export const Expressions: React.FC = props => {
+
+
+ {i18n.translate('xpack.infra.metrics.alertFlyout.alertOnNoData', {
+ defaultMessage: "Alert me if there's no data",
+ })}{' '}
+
+
+
+
+ }
+ checked={alertParams.alertOnNoData}
+ onChange={e => setAlertParams('alertOnNoData', e.target.checked)}
+ />
+
{
+ const gtText = i18n.translate('xpack.infra.metrics.alerting.threshold.gtComparator', {
+ defaultMessage: 'greater than',
+ });
+ const ltText = i18n.translate('xpack.infra.metrics.alerting.threshold.ltComparator', {
+ defaultMessage: 'less than',
+ });
+ const eqText = i18n.translate('xpack.infra.metrics.alerting.threshold.eqComparator', {
+ defaultMessage: 'equal to',
+ });
+
+ switch (comparator) {
+ case Comparator.BETWEEN:
+ return i18n.translate('xpack.infra.metrics.alerting.threshold.betweenComparator', {
+ defaultMessage: 'between',
+ });
+ case Comparator.OUTSIDE_RANGE:
+ return i18n.translate('xpack.infra.metrics.alerting.threshold.outsideRangeComparator', {
+ defaultMessage: 'not between',
+ });
+ case Comparator.GT:
+ return gtText;
+ case Comparator.LT:
+ return ltText;
+ case Comparator.GT_OR_EQ:
+ case Comparator.LT_OR_EQ:
+ if (threshold[0] === currentValue) return eqText;
+ else if (threshold[0] < currentValue) return ltText;
+ return gtText;
+ }
+};
+
+const thresholdToI18n = ([a, b]: number[]) => {
+ if (typeof b === 'undefined') return a;
+ return i18n.translate('xpack.infra.metrics.alerting.threshold.thresholdRange', {
+ defaultMessage: '{a} and {b}',
+ values: { a, b },
+ });
+};
+
+export const buildFiredAlertReason: (alertResult: {
+ metric: string;
+ comparator: Comparator;
+ threshold: number[];
+ currentValue: number;
+}) => string = ({ metric, comparator, threshold, currentValue }) =>
+ i18n.translate('xpack.infra.metrics.alerting.threshold.firedAlertReason', {
+ defaultMessage:
+ '{metric} is {comparator} a threshold of {threshold} (current value is {currentValue})',
+ values: {
+ metric,
+ comparator: comparatorToI18n(comparator, threshold, currentValue),
+ threshold: thresholdToI18n(threshold),
+ currentValue,
+ },
+ });
+
+export const buildNoDataAlertReason: (alertResult: {
+ metric: string;
+ timeSize: number;
+ timeUnit: string;
+}) => string = ({ metric, timeSize, timeUnit }) =>
+ i18n.translate('xpack.infra.metrics.alerting.threshold.noDataAlertReason', {
+ defaultMessage: '{metric} has reported no data over the past {interval}',
+ values: {
+ metric,
+ interval: `${timeSize}${timeUnit}`,
+ },
+ });
+
+export const buildErrorAlertReason = (metric: string) =>
+ i18n.translate('xpack.infra.metrics.alerting.threshold.errorAlertReason', {
+ defaultMessage: 'Elasticsearch failed when attempting to query data for {metric}',
+ values: {
+ metric,
+ },
+ });
diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts
index 24b6ba2ec378b..0007b8bd719f4 100644
--- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts
+++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts
@@ -34,6 +34,8 @@ services.callCluster.mockImplementation(async (_: string, { body, index }: any)
}
if (metric === 'test.metric.2') {
return mocks.alternateMetricResponse;
+ } else if (metric === 'test.metric.3') {
+ return mocks.emptyMetricResponse;
}
return mocks.basicMetricResponse;
});
@@ -161,9 +163,9 @@ describe('The metric threshold alert type', () => {
await execute(Comparator.GT, [0.75]);
const { action } = mostRecentAction(instanceID);
expect(action.group).toBe('*');
- expect(action.valueOf.condition0).toBe(1);
- expect(action.thresholdOf.condition0).toStrictEqual([0.75]);
- expect(action.metricOf.condition0).toBe('test.metric.1');
+ expect(action.reason).toContain('current value is 1');
+ expect(action.reason).toContain('threshold of 0.75');
+ expect(action.reason).toContain('test.metric.1');
});
test('fetches the index pattern dynamically', async () => {
await execute(Comparator.LT, [17], 'alternate');
@@ -271,12 +273,14 @@ describe('The metric threshold alert type', () => {
const instanceID = 'test-*';
await execute(Comparator.GT_OR_EQ, [1.0], [3.0]);
const { action } = mostRecentAction(instanceID);
- expect(action.valueOf.condition0).toBe(1);
- expect(action.valueOf.condition1).toBe(3.5);
- expect(action.thresholdOf.condition0).toStrictEqual([1.0]);
- expect(action.thresholdOf.condition1).toStrictEqual([3.0]);
- expect(action.metricOf.condition0).toBe('test.metric.1');
- expect(action.metricOf.condition1).toBe('test.metric.2');
+ const reasons = action.reason.split('\n');
+ expect(reasons.length).toBe(2);
+ expect(reasons[0]).toContain('test.metric.1');
+ expect(reasons[1]).toContain('test.metric.2');
+ expect(reasons[0]).toContain('current value is 1');
+ expect(reasons[1]).toContain('current value is 3.5');
+ expect(reasons[0]).toContain('threshold of 1');
+ expect(reasons[1]).toContain('threshold of 3');
});
});
describe('querying with the count aggregator', () => {
@@ -305,4 +309,32 @@ describe('The metric threshold alert type', () => {
expect(getState(instanceID).alertState).toBe(AlertStates.OK);
});
});
+ describe("querying a metric that hasn't reported data", () => {
+ const instanceID = 'test-*';
+ const execute = (alertOnNoData: boolean) =>
+ executor({
+ services,
+ params: {
+ criteria: [
+ {
+ ...baseCriterion,
+ comparator: Comparator.GT,
+ threshold: 1,
+ metric: 'test.metric.3',
+ },
+ ],
+ alertOnNoData,
+ },
+ });
+ test('sends a No Data alert when configured to do so', async () => {
+ await execute(true);
+ expect(mostRecentAction(instanceID).id).toBe(FIRED_ACTIONS.id);
+ expect(getState(instanceID).alertState).toBe(AlertStates.NO_DATA);
+ });
+ test('does not send a No Data alert when not configured to do so', async () => {
+ await execute(false);
+ expect(mostRecentAction(instanceID)).toBe(undefined);
+ expect(getState(instanceID).alertState).toBe(AlertStates.NO_DATA);
+ });
+ });
});
diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.ts
index 946f1c14bf593..bd77e5e2daf42 100644
--- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.ts
+++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.ts
@@ -12,6 +12,13 @@ import { createAfterKeyHandler } from '../../../utils/create_afterkey_handler';
import { getAllCompositeData } from '../../../utils/get_all_composite_data';
import { networkTraffic } from '../../../../common/inventory_models/shared/metrics/snapshot/network_traffic';
import { MetricExpressionParams, Comparator, Aggregators, AlertStates } from './types';
+import {
+ buildErrorAlertReason,
+ buildFiredAlertReason,
+ buildNoDataAlertReason,
+ DOCUMENT_COUNT_I18N,
+ stateToAlertMessage,
+} from './messages';
import { AlertServices, AlertExecutorOptions } from '../../../../../alerting/server';
import { getIntervalInSeconds } from '../../../utils/get_interval_in_seconds';
import { getDateHistogramOffset } from '../../snapshot/query_helpers';
@@ -51,17 +58,19 @@ const getCurrentValueFromAggregations = (
const getParsedFilterQuery: (
filterQuery: string | undefined
-) => Record = filterQuery => {
+) => Record | Array> = filterQuery => {
if (!filterQuery) return {};
try {
return JSON.parse(filterQuery).bool;
} catch (e) {
- return {
- query_string: {
- query: filterQuery,
- analyze_wildcard: true,
+ return [
+ {
+ query_string: {
+ query: filterQuery,
+ analyze_wildcard: true,
+ },
},
- };
+ ];
}
};
@@ -159,8 +168,12 @@ export const getElasticsearchMetricQuery = (
return {
query: {
bool: {
- filter: [...rangeFilters, ...metricFieldFilters],
- ...parsedFilterQuery,
+ filter: [
+ ...rangeFilters,
+ ...metricFieldFilters,
+ ...(Array.isArray(parsedFilterQuery) ? parsedFilterQuery : []),
+ ],
+ ...(!Array.isArray(parsedFilterQuery) ? parsedFilterQuery : {}),
},
},
size: 0,
@@ -233,6 +246,7 @@ const getMetric: (
body: searchBody,
index,
});
+
return { '*': getCurrentValueFromAggregations(result.aggregations, aggType) };
} catch (e) {
return { '*': undefined }; // Trigger an Error state
@@ -251,24 +265,14 @@ const comparatorMap = {
[Comparator.LT_OR_EQ]: (a: number, [b]: number[]) => a <= b,
};
-const mapToConditionsLookup = (
- list: any[],
- mapFn: (value: any, index: number, array: any[]) => unknown
-) =>
- list
- .map(mapFn)
- .reduce(
- (result: Record, value, i) => ({ ...result, [`condition${i}`]: value }),
- {}
- );
-
export const createMetricThresholdExecutor = (alertUUID: string) =>
async function({ services, params }: AlertExecutorOptions) {
- const { criteria, groupBy, filterQuery, sourceId } = params as {
+ const { criteria, groupBy, filterQuery, sourceId, alertOnNoData } = params as {
criteria: MetricExpressionParams[];
groupBy: string | undefined;
filterQuery: string | undefined;
sourceId?: string;
+ alertOnNoData: boolean;
};
const alertResults = await Promise.all(
@@ -279,9 +283,11 @@ export const createMetricThresholdExecutor = (alertUUID: string) =>
const { threshold, comparator } = criterion;
const comparisonFunction = comparatorMap[comparator];
return mapValues(currentValues, value => ({
+ ...criterion,
+ metric: criterion.metric ?? DOCUMENT_COUNT_I18N,
+ currentValue: value,
shouldFire:
value !== undefined && value !== null && comparisonFunction(value, threshold),
- currentValue: value,
isNoData: value === null,
isError: value === undefined,
}));
@@ -299,23 +305,43 @@ export const createMetricThresholdExecutor = (alertUUID: string) =>
// whole alert is in a No Data/Error state
const isNoData = alertResults.some(result => result[group].isNoData);
const isError = alertResults.some(result => result[group].isError);
- if (shouldAlertFire) {
+
+ const nextState = isError
+ ? AlertStates.ERROR
+ : isNoData
+ ? AlertStates.NO_DATA
+ : shouldAlertFire
+ ? AlertStates.ALERT
+ : AlertStates.OK;
+
+ let reason;
+ if (nextState === AlertStates.ALERT) {
+ reason = alertResults.map(result => buildFiredAlertReason(result[group])).join('\n');
+ }
+ if (alertOnNoData) {
+ if (nextState === AlertStates.NO_DATA) {
+ reason = alertResults
+ .filter(result => result[group].isNoData)
+ .map(result => buildNoDataAlertReason(result[group]))
+ .join('\n');
+ } else if (nextState === AlertStates.ERROR) {
+ reason = alertResults
+ .filter(result => result[group].isError)
+ .map(result => buildErrorAlertReason(result[group].metric))
+ .join('\n');
+ }
+ }
+ if (reason) {
alertInstance.scheduleActions(FIRED_ACTIONS.id, {
group,
- valueOf: mapToConditionsLookup(alertResults, result => result[group].currentValue),
- thresholdOf: mapToConditionsLookup(criteria, criterion => criterion.threshold),
- metricOf: mapToConditionsLookup(criteria, criterion => criterion.metric),
+ alertState: stateToAlertMessage[nextState],
+ reason,
});
}
+
// Future use: ability to fetch display current alert state
alertInstance.replaceState({
- alertState: isError
- ? AlertStates.ERROR
- : isNoData
- ? AlertStates.NO_DATA
- : shouldAlertFire
- ? AlertStates.ALERT
- : AlertStates.OK,
+ alertState: nextState,
});
}
};
diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_alert_type.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_alert_type.ts
index 3415ae9873bfb..029491c1168cf 100644
--- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_alert_type.ts
+++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_alert_type.ts
@@ -55,27 +55,18 @@ export async function registerMetricThresholdAlertType(
}
);
- const valueOfActionVariableDescription = i18n.translate(
- 'xpack.infra.metrics.alerting.threshold.alerting.valueOfActionVariableDescription',
+ const alertStateActionVariableDescription = i18n.translate(
+ 'xpack.infra.metrics.alerting.threshold.alerting.alertStateActionVariableDescription',
{
- defaultMessage:
- 'Record of the current value of the watched metric; grouped by condition, i.e valueOf.condition0, valueOf.condition1, etc.',
- }
- );
-
- const thresholdOfActionVariableDescription = i18n.translate(
- 'xpack.infra.metrics.alerting.threshold.alerting.thresholdOfActionVariableDescription',
- {
- defaultMessage:
- 'Record of the alerting threshold; grouped by condition, i.e thresholdOf.condition0, thresholdOf.condition1, etc.',
+ defaultMessage: 'Current state of the alert',
}
);
- const metricOfActionVariableDescription = i18n.translate(
- 'xpack.infra.metrics.alerting.threshold.alerting.metricOfActionVariableDescription',
+ const reasonActionVariableDescription = i18n.translate(
+ 'xpack.infra.metrics.alerting.threshold.alerting.reasonActionVariableDescription',
{
defaultMessage:
- 'Record of the watched metric; grouped by condition, i.e metricOf.condition0, metricOf.condition1, etc.',
+ 'A description of why the alert is in this state, including which metrics have crossed which thresholds',
}
);
@@ -88,6 +79,7 @@ export async function registerMetricThresholdAlertType(
groupBy: schema.maybe(schema.string()),
filterQuery: schema.maybe(schema.string()),
sourceId: schema.string(),
+ alertOnNoData: schema.maybe(schema.boolean()),
}),
},
defaultActionGroupId: FIRED_ACTIONS.id,
@@ -96,9 +88,8 @@ export async function registerMetricThresholdAlertType(
actionVariables: {
context: [
{ name: 'group', description: groupActionVariableDescription },
- { name: 'valueOf', description: valueOfActionVariableDescription },
- { name: 'thresholdOf', description: thresholdOfActionVariableDescription },
- { name: 'metricOf', description: metricOfActionVariableDescription },
+ { name: 'alertState', description: alertStateActionVariableDescription },
+ { name: 'reason', description: reasonActionVariableDescription },
],
},
});
diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts
index 66e0a363c8983..fa55f80e472de 100644
--- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts
+++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts
@@ -53,6 +53,14 @@ export const alternateMetricResponse = {
},
};
+export const emptyMetricResponse = {
+ aggregations: {
+ aggregatedIntervals: {
+ buckets: [],
+ },
+ },
+};
+
export const basicCompositeResponse = {
aggregations: {
groupings: {
diff --git a/x-pack/plugins/ingest_manager/common/constants/agent_config.ts b/x-pack/plugins/ingest_manager/common/constants/agent_config.ts
index c5067480fb953..9bc1293799d3c 100644
--- a/x-pack/plugins/ingest_manager/common/constants/agent_config.ts
+++ b/x-pack/plugins/ingest_manager/common/constants/agent_config.ts
@@ -14,6 +14,7 @@ export const DEFAULT_AGENT_CONFIG = {
status: AgentConfigStatus.Active,
datasources: [],
is_default: true,
+ monitoring_enabled: ['logs', 'metrics'] as Array<'logs' | 'metrics'>,
};
export const DEFAULT_AGENT_CONFIGS_PACKAGES = [DefaultPackages.system];
diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts
index a7d4e36d16f2a..bff799798ff6e 100644
--- a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts
+++ b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts
@@ -3,11 +3,12 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
-import { Datasource, NewDatasource, DatasourceInput } from '../types';
+import { Datasource, DatasourceInput } from '../types';
import { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource';
describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
- const mockNewDatasource: NewDatasource = {
+ const mockDatasource: Datasource = {
+ id: 'some-uuid',
name: 'mock-datasource',
description: '',
config_id: '',
@@ -15,11 +16,6 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
output_id: '',
namespace: 'default',
inputs: [],
- };
-
- const mockDatasource: Datasource = {
- ...mockNewDatasource,
- id: 'some-uuid',
revision: 1,
};
@@ -107,17 +103,6 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
});
});
- it('uses name for id when id is not provided in case of new datasource', () => {
- expect(storedDatasourceToAgentDatasource(mockNewDatasource)).toEqual({
- id: 'mock-datasource',
- name: 'mock-datasource',
- namespace: 'default',
- enabled: true,
- use_output: 'default',
- inputs: [],
- });
- });
-
it('returns agent datasource config with flattened input and package stream', () => {
expect(storedDatasourceToAgentDatasource({ ...mockDatasource, inputs: [mockInput] })).toEqual({
id: 'some-uuid',
diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts
index 5deb33ccf10f1..620b663451ea3 100644
--- a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts
+++ b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts
@@ -3,16 +3,16 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
-import { Datasource, NewDatasource, FullAgentConfigDatasource } from '../types';
+import { Datasource, FullAgentConfigDatasource } from '../types';
import { DEFAULT_OUTPUT } from '../constants';
export const storedDatasourceToAgentDatasource = (
- datasource: Datasource | NewDatasource
+ datasource: Datasource
): FullAgentConfigDatasource => {
- const { name, namespace, enabled, package: pkg, inputs } = datasource;
+ const { id, name, namespace, enabled, package: pkg, inputs } = datasource;
const fullDatasource: FullAgentConfigDatasource = {
- id: 'id' in datasource ? datasource.id : name,
+ id: id || name,
name,
namespace,
enabled,
diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts b/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts
index cb290e61b17e5..a977a1a66e059 100644
--- a/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts
+++ b/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts
@@ -11,6 +11,7 @@ describe('Ingest Manager - packageToConfig', () => {
name: 'mock-package',
title: 'Mock package',
version: '0.0.0',
+ latestVersion: '0.0.0',
description: 'description',
type: 'mock',
categories: [],
diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent.ts b/x-pack/plugins/ingest_manager/common/types/models/agent.ts
index fcd3955f3a32f..e3ca7635fdb40 100644
--- a/x-pack/plugins/ingest_manager/common/types/models/agent.ts
+++ b/x-pack/plugins/ingest_manager/common/types/models/agent.ts
@@ -60,6 +60,11 @@ export interface AgentEvent {
export interface AgentEventSOAttributes extends AgentEvent, SavedObjectAttributes {}
+type MetadataValue = string | AgentMetadata;
+
+export interface AgentMetadata {
+ [x: string]: MetadataValue;
+}
interface AgentBase {
type: AgentType;
active: boolean;
@@ -72,19 +77,17 @@ interface AgentBase {
config_revision?: number | null;
config_newest_revision?: number;
last_checkin?: string;
+ user_provided_metadata: AgentMetadata;
+ local_metadata: AgentMetadata;
}
export interface Agent extends AgentBase {
id: string;
current_error_events: AgentEvent[];
- user_provided_metadata: Record;
- local_metadata: Record;
access_api_key?: string;
status?: string;
}
export interface AgentSOAttributes extends AgentBase, SavedObjectAttributes {
- user_provided_metadata: string;
- local_metadata: string;
current_error_events?: string;
}
diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts
index 2372caee512af..96121251b133e 100644
--- a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts
+++ b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts
@@ -3,8 +3,6 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
-
-import { SavedObjectAttributes } from 'src/core/public';
import {
Datasource,
DatasourcePackage,
@@ -23,9 +21,10 @@ export interface NewAgentConfig {
namespace?: string;
description?: string;
is_default?: boolean;
+ monitoring_enabled?: Array<'logs' | 'metrics'>;
}
-export interface AgentConfig extends NewAgentConfig, SavedObjectAttributes {
+export interface AgentConfig extends NewAgentConfig {
id: string;
status: AgentConfigStatus;
datasources: string[] | Datasource[];
@@ -60,4 +59,12 @@ export interface FullAgentConfig {
};
datasources: FullAgentConfigDatasource[];
revision?: number;
+ settings?: {
+ monitoring: {
+ use_output?: string;
+ enabled: boolean;
+ metrics: boolean;
+ logs: boolean;
+ };
+ };
}
diff --git a/x-pack/plugins/ingest_manager/common/types/models/datasource.ts b/x-pack/plugins/ingest_manager/common/types/models/datasource.ts
index 885e0a9316d79..ca61a93d9be93 100644
--- a/x-pack/plugins/ingest_manager/common/types/models/datasource.ts
+++ b/x-pack/plugins/ingest_manager/common/types/models/datasource.ts
@@ -17,22 +17,29 @@ export interface DatasourceConfigRecordEntry {
export type DatasourceConfigRecord = Record;
-export interface DatasourceInputStream {
+export interface NewDatasourceInputStream {
id: string;
enabled: boolean;
dataset: string;
processors?: string[];
config?: DatasourceConfigRecord;
vars?: DatasourceConfigRecord;
+}
+
+export interface DatasourceInputStream extends NewDatasourceInputStream {
agent_stream?: any;
}
-export interface DatasourceInput {
+export interface NewDatasourceInput {
type: string;
enabled: boolean;
processors?: string[];
config?: DatasourceConfigRecord;
vars?: DatasourceConfigRecord;
+ streams: NewDatasourceInputStream[];
+}
+
+export interface DatasourceInput extends Omit {
streams: DatasourceInputStream[];
}
@@ -44,10 +51,11 @@ export interface NewDatasource {
enabled: boolean;
package?: DatasourcePackage;
output_id: string;
- inputs: DatasourceInput[];
+ inputs: NewDatasourceInput[];
}
-export type Datasource = NewDatasource & {
+export interface Datasource extends Omit {
id: string;
+ inputs: DatasourceInput[];
revision: number;
-};
+}
diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts
index fb6bf235d1e26..f8779a879a049 100644
--- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts
+++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts
@@ -97,6 +97,7 @@ export interface RegistryStream {
description?: string;
enabled?: boolean;
vars?: RegistryVarsEntry[];
+ template?: string;
}
export type RequirementVersion = string;
@@ -203,6 +204,8 @@ export interface RegistryVarsEntry {
// internal until we need them
interface PackageAdditions {
title: string;
+ latestVersion: string;
+ installedVersion?: string;
assets: AssetsGroupedByServiceByType;
}
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/alpha_flyout.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/alpha_flyout.tsx
new file mode 100644
index 0000000000000..1e7a14e350229
--- /dev/null
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/alpha_flyout.tsx
@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import React from 'react';
+import {
+ EuiButtonEmpty,
+ EuiFlyout,
+ EuiFlyoutBody,
+ EuiFlyoutHeader,
+ EuiFlyoutFooter,
+ EuiLink,
+ EuiText,
+ EuiTitle,
+} from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
+
+interface Props {
+ onClose: () => void;
+}
+
+export const AlphaFlyout: React.FunctionComponent = ({ onClose }) => {
+ return (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ),
+ forumLink: (
+
+
+
+ ),
+ }}
+ />
+
+
+
+
+
+
+ ),
+ }}
+ />
+
+
+
+
+
+
+
+
+
+ );
+};
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/alpha_messaging.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/alpha_messaging.tsx
index 0f3ddee29fa44..5a06a9a879441 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/alpha_messaging.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/alpha_messaging.tsx
@@ -3,35 +3,45 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
-import React from 'react';
+import React, { useState } from 'react';
import styled from 'styled-components';
import { FormattedMessage } from '@kbn/i18n/react';
-import { EuiText } from '@elastic/eui';
+import { EuiText, EuiLink } from '@elastic/eui';
+import { AlphaFlyout } from './alpha_flyout';
const Message = styled(EuiText).attrs(props => ({
color: 'subdued',
textAlign: 'center',
+ size: 's',
}))`
padding: ${props => props.theme.eui.paddingSizes.m};
`;
-export const AlphaMessaging: React.FC<{}> = () => (
-
-
-
-
+export const AlphaMessaging: React.FC<{}> = () => {
+ const [isAlphaFlyoutOpen, setIsAlphaFlyoutOpen] = useState(false);
+
+ return (
+ <>
+
+
+
+
+
+ {' – '}
-
- {' – '}
-
-
-
-
-);
+ />{' '}
+ setIsAlphaFlyoutOpen(true)}>
+ View more details.
+
+
+
+ {isAlphaFlyoutOpen && setIsAlphaFlyoutOpen(false)} />}
+
+ );
+};
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/search_bar.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/search_bar.tsx
index 4429b9d8e6b82..1c9bd9107515d 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/search_bar.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/search_bar.tsx
@@ -92,7 +92,6 @@ function useSuggestions(fieldPrefix: string, search: string) {
const res = (await data.indexPatterns.getFieldsForWildcard({
pattern: INDEX_NAME,
})) as IFieldType[];
-
if (!data || !data.autocomplete) {
throw new Error('Missing data plugin');
}
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/settings_flyout.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/settings_flyout.tsx
index 92146e9ee5679..9863463e68a01 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/settings_flyout.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/settings_flyout.tsx
@@ -209,7 +209,7 @@ export const SettingFlyout: React.FunctionComponent = ({ onClose }) => {
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/datasource.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/datasource.ts
index 0d19ecd0cb735..e2fc190e158f9 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/datasource.ts
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/datasource.ts
@@ -5,12 +5,18 @@
*/
import { sendRequest, useRequest } from './use_request';
import { datasourceRouteService } from '../../services';
-import { CreateDatasourceRequest, CreateDatasourceResponse } from '../../types';
+import {
+ CreateDatasourceRequest,
+ CreateDatasourceResponse,
+ UpdateDatasourceRequest,
+ UpdateDatasourceResponse,
+} from '../../types';
import {
DeleteDatasourcesRequest,
DeleteDatasourcesResponse,
GetDatasourcesRequest,
GetDatasourcesResponse,
+ GetOneDatasourceResponse,
} from '../../../../../common/types/rest_spec';
export const sendCreateDatasource = (body: CreateDatasourceRequest['body']) => {
@@ -21,6 +27,17 @@ export const sendCreateDatasource = (body: CreateDatasourceRequest['body']) => {
});
};
+export const sendUpdateDatasource = (
+ datasourceId: string,
+ body: UpdateDatasourceRequest['body']
+) => {
+ return sendRequest({
+ path: datasourceRouteService.getUpdatePath(datasourceId),
+ method: 'put',
+ body: JSON.stringify(body),
+ });
+};
+
export const sendDeleteDatasource = (body: DeleteDatasourcesRequest['body']) => {
return sendRequest({
path: datasourceRouteService.getDeletePath(),
@@ -36,3 +53,10 @@ export function useGetDatasources(query: GetDatasourcesRequest['query']) {
query,
});
}
+
+export const sendGetOneDatasource = (datasourceId: string) => {
+ return sendRequest({
+ path: datasourceRouteService.getInfoPath(datasourceId),
+ method: 'get',
+ });
+};
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx
index 10245e73520f7..4a9cfe02b74ac 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx
@@ -96,12 +96,28 @@ export const DefaultLayout: React.FunctionComponent = ({ section, childre
- setIsSettingsFlyoutOpen(true)}>
-
-
+
+
+
+
+
+
+
+ setIsSettingsFlyoutOpen(true)}>
+
+
+
+
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx
index 0d53ca34a1fef..92c44d86e47c6 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx
@@ -18,6 +18,7 @@ import {
EuiText,
EuiComboBox,
EuiIconTip,
+ EuiCheckboxGroup,
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import { i18n } from '@kbn/i18n';
@@ -30,7 +31,7 @@ interface ValidationResults {
const StyledEuiAccordion = styled(EuiAccordion)`
.ingest-active-button {
- color: ${props => props.theme.eui.euiColorPrimary}};
+ color: ${props => props.theme.eui.euiColorPrimary};
}
`;
@@ -244,6 +245,68 @@ export const AgentConfigForm: React.FunctionComponent = ({
)}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {
+ acc[key] = true;
+ return acc;
+ },
+ { logs: false, metrics: false }
+ )}
+ onChange={id => {
+ if (id !== 'logs' && id !== 'metrics') {
+ return;
+ }
+
+ const hasLogs =
+ agentConfig.monitoring_enabled && agentConfig.monitoring_enabled.indexOf(id) >= 0;
+
+ const previousValues = agentConfig.monitoring_enabled || [];
+ updateAgentConfig({
+ monitoring_enabled: hasLogs
+ ? previousValues.filter(type => type !== id)
+ : [...previousValues, id],
+ });
+ }}
+ />
+
+
);
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/layout.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/layout.tsx
index 39d882f7fdf65..f1e3fea6a0742 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/layout.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/layout.tsx
@@ -39,17 +39,29 @@ export const CreateDatasourcePageLayout: React.FunctionComponent<{
-
+ {from === 'edit' ? (
+
+ ) : (
+
+ )}
- {from === 'config' ? (
+ {from === 'edit' ? (
+
+ ) : from === 'config' ? (
- {agentConfig && from === 'config' ? (
+ {agentConfig && (from === 'config' || from === 'edit') ? (
{
const updatePackageInfo = (updatedPackageInfo: PackageInfo | undefined) => {
if (updatedPackageInfo) {
setPackageInfo(updatedPackageInfo);
+ setFormState('VALID');
} else {
setFormState('INVALID');
setPackageInfo(undefined);
@@ -152,9 +153,7 @@ export const CreateDatasourcePage: React.FunctionComponent = () => {
const cancelUrl = from === 'config' ? CONFIG_URL : PACKAGE_URL;
// Save datasource
- const [formState, setFormState] = useState<
- 'VALID' | 'INVALID' | 'CONFIRM' | 'LOADING' | 'SUBMITTED'
- >('INVALID');
+ const [formState, setFormState] = useState('INVALID');
const saveDatasource = async () => {
setFormState('LOADING');
const result = await sendCreateDatasource(datasource);
@@ -174,6 +173,23 @@ export const CreateDatasourcePage: React.FunctionComponent = () => {
const { error } = await saveDatasource();
if (!error) {
history.push(`${AGENT_CONFIG_DETAILS_PATH}${agentConfig ? agentConfig.id : configId}`);
+ notifications.toasts.addSuccess({
+ title: i18n.translate('xpack.ingestManager.createDatasource.addedNotificationTitle', {
+ defaultMessage: `Successfully added '{datasourceName}'`,
+ values: {
+ datasourceName: datasource.name,
+ },
+ }),
+ text:
+ agentCount && agentConfig
+ ? i18n.translate('xpack.ingestManager.createDatasource.addedNotificationMessage', {
+ defaultMessage: `Fleet will deploy updates to all agents that use the '{agentConfigName}' configuration`,
+ values: {
+ agentConfigName: agentConfig.name,
+ },
+ })
+ : undefined,
+ });
} else {
notifications.toasts.addError(error, {
title: 'Error',
@@ -229,6 +245,7 @@ export const CreateDatasourcePage: React.FunctionComponent = () => {
packageInfo={packageInfo}
datasource={datasource}
updateDatasource={updateDatasource}
+ validationResults={validationResults!}
/>
) : null,
},
@@ -240,7 +257,6 @@ export const CreateDatasourcePage: React.FunctionComponent = () => {
children:
agentConfig && packageInfo ? (
) => void;
validationResults: DatasourceValidationResults;
submitAttempted: boolean;
-}> = ({
- agentConfig,
- packageInfo,
- datasource,
- updateDatasource,
- validationResults,
- submitAttempted,
-}) => {
- // Form show/hide states
-
+}> = ({ packageInfo, datasource, updateDatasource, validationResults, submitAttempted }) => {
const hasErrors = validationResults ? validationHasErrors(validationResults) : false;
- // Update datasource's package and config info
- useEffect(() => {
- const dsPackage = datasource.package;
- const currentPkgKey = dsPackage ? `${dsPackage.name}-${dsPackage.version}` : '';
- const pkgKey = `${packageInfo.name}-${packageInfo.version}`;
-
- // If package has changed, create shell datasource with input&stream values based on package info
- if (currentPkgKey !== pkgKey) {
- // Existing datasources on the agent config using the package name, retrieve highest number appended to datasource name
- const dsPackageNamePattern = new RegExp(`${packageInfo.name}-(\\d+)`);
- const dsWithMatchingNames = (agentConfig.datasources as Datasource[])
- .filter(ds => Boolean(ds.name.match(dsPackageNamePattern)))
- .map(ds => parseInt(ds.name.match(dsPackageNamePattern)![1], 10))
- .sort();
-
- updateDatasource({
- name: `${packageInfo.name}-${
- dsWithMatchingNames.length ? dsWithMatchingNames[dsWithMatchingNames.length - 1] + 1 : 1
- }`,
- package: {
- name: packageInfo.name,
- title: packageInfo.title,
- version: packageInfo.version,
- },
- inputs: packageToConfigDatasourceInputs(packageInfo),
- });
- }
-
- // If agent config has changed, update datasource's config ID and namespace
- if (datasource.config_id !== agentConfig.id) {
- updateDatasource({
- config_id: agentConfig.id,
- namespace: agentConfig.namespace,
- });
- }
- }, [datasource.package, datasource.config_id, agentConfig, packageInfo, updateDatasource]);
-
- // Step B, configure inputs (and their streams)
+ // Configure inputs (and their streams)
// Assume packages only export one datasource for now
const renderConfigureInputs = () =>
packageInfo.datasources &&
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/step_define_datasource.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/step_define_datasource.tsx
index 792389381eaf0..c4d602c2c2081 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/step_define_datasource.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/step_define_datasource.tsx
@@ -17,13 +17,16 @@ import {
} from '@elastic/eui';
import { AgentConfig, PackageInfo, Datasource, NewDatasource } from '../../../types';
import { packageToConfigDatasourceInputs } from '../../../services';
+import { Loading } from '../../../components';
+import { DatasourceValidationResults } from './services';
export const StepDefineDatasource: React.FunctionComponent<{
agentConfig: AgentConfig;
packageInfo: PackageInfo;
datasource: NewDatasource;
updateDatasource: (fields: Partial) => void;
-}> = ({ agentConfig, packageInfo, datasource, updateDatasource }) => {
+ validationResults: DatasourceValidationResults;
+}> = ({ agentConfig, packageInfo, datasource, updateDatasource, validationResults }) => {
// Form show/hide states
const [isShowingAdvancedDefine, setIsShowingAdvancedDefine] = useState(false);
@@ -64,11 +67,13 @@ export const StepDefineDatasource: React.FunctionComponent<{
}
}, [datasource.package, datasource.config_id, agentConfig, packageInfo, updateDatasource]);
- return (
+ return validationResults ? (
<>
}
+ isInvalid={!!validationResults.description}
+ error={validationResults.description}
>
) : null}
+ ) : (
+
);
};
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/types.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/types.ts
index 85cc758fc4c46..10b30a5696d83 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/types.ts
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/types.ts
@@ -4,4 +4,5 @@
* you may not use this file except in compliance with the Elastic License.
*/
-export type CreateDatasourceFrom = 'package' | 'config';
+export type CreateDatasourceFrom = 'package' | 'config' | 'edit';
+export type DatasourceFormState = 'VALID' | 'INVALID' | 'CONFIRM' | 'LOADING' | 'SUBMITTED';
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/details_page/components/datasources/datasources_table.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/details_page/components/datasources/datasources_table.tsx
index 1eee9f6b0c346..a0418c5f256c4 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/details_page/components/datasources/datasources_table.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/details_page/components/datasources/datasources_table.tsx
@@ -19,7 +19,7 @@ import {
import { AgentConfig, Datasource } from '../../../../../types';
import { TableRowActions } from '../../../components/table_row_actions';
import { DangerEuiContextMenuItem } from '../../../components/danger_eui_context_menu_item';
-import { useCapabilities } from '../../../../../hooks';
+import { useCapabilities, useLink } from '../../../../../hooks';
import { useAgentConfigLink } from '../../hooks/use_details_uri';
import { DatasourceDeleteProvider } from '../../../components/datasource_delete_provider';
import { useConfigRefresh } from '../../hooks/use_config';
@@ -56,6 +56,7 @@ export const DatasourcesTable: React.FunctionComponent = ({
}) => {
const hasWriteCapabilities = useCapabilities().write;
const addDatasourceLink = useAgentConfigLink('add-datasource', { configId: config.id });
+ const editDatasourceLink = useLink(`/configs/${config.id}/edit-datasource`);
const refreshConfig = useConfigRefresh();
// With the datasources provided on input, generate the list of datasources
@@ -201,22 +202,21 @@ export const DatasourcesTable: React.FunctionComponent = ({
{}}
+ // key="datasourceView"
+ // >
+ //
+ // ,
{}}
- key="datasourceView"
- >
-
- ,
- // FIXME: implement Edit datasource action
- {}}
+ href={`${editDatasourceLink}/${datasource.id}`}
key="datasourceEdit"
>
= ({
/>
,
// FIXME: implement Copy datasource action
- {}} key="datasourceCopy">
-
- ,
+ // {}} key="datasourceCopy">
+ //
+ // ,
{deleteDatasourcePrompt => {
return (
@@ -256,7 +256,7 @@ export const DatasourcesTable: React.FunctionComponent = ({
],
},
],
- [config, hasWriteCapabilities, refreshConfig]
+ [config, editDatasourceLink, hasWriteCapabilities, refreshConfig]
);
return (
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/edit_datasource_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/edit_datasource_page/index.tsx
new file mode 100644
index 0000000000000..d4c39f21a1ea6
--- /dev/null
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/edit_datasource_page/index.tsx
@@ -0,0 +1,323 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import React, { useState, useEffect } from 'react';
+import { useRouteMatch, useHistory } from 'react-router-dom';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+import {
+ EuiButtonEmpty,
+ EuiButton,
+ EuiSteps,
+ EuiBottomBar,
+ EuiFlexGroup,
+ EuiFlexItem,
+ EuiSpacer,
+} from '@elastic/eui';
+import { AGENT_CONFIG_DETAILS_PATH } from '../../../constants';
+import { AgentConfig, PackageInfo, NewDatasource } from '../../../types';
+import {
+ useLink,
+ useCore,
+ useConfig,
+ sendUpdateDatasource,
+ sendGetAgentStatus,
+ sendGetOneAgentConfig,
+ sendGetOneDatasource,
+ sendGetPackageInfoByKey,
+} from '../../../hooks';
+import { Loading, Error } from '../../../components';
+import {
+ CreateDatasourcePageLayout,
+ ConfirmCreateDatasourceModal,
+} from '../create_datasource_page/components';
+import {
+ DatasourceValidationResults,
+ validateDatasource,
+ validationHasErrors,
+} from '../create_datasource_page/services';
+import { DatasourceFormState, CreateDatasourceFrom } from '../create_datasource_page/types';
+import { StepConfigureDatasource } from '../create_datasource_page/step_configure_datasource';
+import { StepDefineDatasource } from '../create_datasource_page/step_define_datasource';
+
+export const EditDatasourcePage: React.FunctionComponent = () => {
+ const { notifications } = useCore();
+ const {
+ fleet: { enabled: isFleetEnabled },
+ } = useConfig();
+ const {
+ params: { configId, datasourceId },
+ } = useRouteMatch();
+ const history = useHistory();
+
+ // Agent config, package info, and datasource states
+ const [isLoadingData, setIsLoadingData] = useState(true);
+ const [loadingError, setLoadingError] = useState();
+ const [agentConfig, setAgentConfig] = useState();
+ const [packageInfo, setPackageInfo] = useState();
+ const [datasource, setDatasource] = useState({
+ name: '',
+ description: '',
+ config_id: '',
+ enabled: true,
+ output_id: '',
+ inputs: [],
+ });
+
+ // Retrieve agent config, package, and datasource info
+ useEffect(() => {
+ const getData = async () => {
+ setIsLoadingData(true);
+ setLoadingError(undefined);
+ try {
+ const [{ data: agentConfigData }, { data: datasourceData }] = await Promise.all([
+ sendGetOneAgentConfig(configId),
+ sendGetOneDatasource(datasourceId),
+ ]);
+ if (agentConfigData?.item) {
+ setAgentConfig(agentConfigData.item);
+ }
+ if (datasourceData?.item) {
+ const { id, revision, inputs, ...restOfDatasource } = datasourceData.item;
+ // Remove `agent_stream` from all stream info, we assign this after saving
+ const newDatasource = {
+ ...restOfDatasource,
+ inputs: inputs.map(input => {
+ const { streams, ...restOfInput } = input;
+ return {
+ ...restOfInput,
+ streams: streams.map(stream => {
+ const { agent_stream, ...restOfStream } = stream;
+ return restOfStream;
+ }),
+ };
+ }),
+ };
+ setDatasource(newDatasource);
+ if (datasourceData.item.package) {
+ const { data: packageData } = await sendGetPackageInfoByKey(
+ `${datasourceData.item.package.name}-${datasourceData.item.package.version}`
+ );
+ if (packageData?.response) {
+ setPackageInfo(packageData.response);
+ setValidationResults(validateDatasource(newDatasource, packageData.response));
+ setFormState('VALID');
+ }
+ }
+ }
+ } catch (e) {
+ setLoadingError(e);
+ }
+ setIsLoadingData(false);
+ };
+ getData();
+ }, [configId, datasourceId]);
+
+ // Retrieve agent count
+ const [agentCount, setAgentCount] = useState(0);
+ useEffect(() => {
+ const getAgentCount = async () => {
+ const { data } = await sendGetAgentStatus({ configId });
+ if (data?.results.total) {
+ setAgentCount(data.results.total);
+ }
+ };
+
+ if (isFleetEnabled) {
+ getAgentCount();
+ }
+ }, [configId, isFleetEnabled]);
+
+ // Datasource validation state
+ const [validationResults, setValidationResults] = useState();
+ const hasErrors = validationResults ? validationHasErrors(validationResults) : false;
+
+ // Update datasource method
+ const updateDatasource = (updatedFields: Partial) => {
+ const newDatasource = {
+ ...datasource,
+ ...updatedFields,
+ };
+ setDatasource(newDatasource);
+
+ // eslint-disable-next-line no-console
+ console.debug('Datasource updated', newDatasource);
+ const newValidationResults = updateDatasourceValidation(newDatasource);
+ const hasValidationErrors = newValidationResults
+ ? validationHasErrors(newValidationResults)
+ : false;
+ if (!hasValidationErrors) {
+ setFormState('VALID');
+ }
+ };
+
+ const updateDatasourceValidation = (newDatasource?: NewDatasource) => {
+ if (packageInfo) {
+ const newValidationResult = validateDatasource(newDatasource || datasource, packageInfo);
+ setValidationResults(newValidationResult);
+ // eslint-disable-next-line no-console
+ console.debug('Datasource validation results', newValidationResult);
+
+ return newValidationResult;
+ }
+ };
+
+ // Cancel url
+ const CONFIG_URL = useLink(`${AGENT_CONFIG_DETAILS_PATH}${configId}`);
+ const cancelUrl = CONFIG_URL;
+
+ // Save datasource
+ const [formState, setFormState] = useState('INVALID');
+ const saveDatasource = async () => {
+ setFormState('LOADING');
+ const result = await sendUpdateDatasource(datasourceId, datasource);
+ setFormState('SUBMITTED');
+ return result;
+ };
+
+ const onSubmit = async () => {
+ if (formState === 'VALID' && hasErrors) {
+ setFormState('INVALID');
+ return;
+ }
+ if (agentCount !== 0 && formState !== 'CONFIRM') {
+ setFormState('CONFIRM');
+ return;
+ }
+ const { error } = await saveDatasource();
+ if (!error) {
+ history.push(`${AGENT_CONFIG_DETAILS_PATH}${configId}`);
+ notifications.toasts.addSuccess({
+ title: i18n.translate('xpack.ingestManager.editDatasource.updatedNotificationTitle', {
+ defaultMessage: `Successfully updated '{datasourceName}'`,
+ values: {
+ datasourceName: datasource.name,
+ },
+ }),
+ text:
+ agentCount && agentConfig
+ ? i18n.translate('xpack.ingestManager.editDatasource.updatedNotificationMessage', {
+ defaultMessage: `Fleet will deploy updates to all agents that use the '{agentConfigName}' configuration`,
+ values: {
+ agentConfigName: agentConfig.name,
+ },
+ })
+ : undefined,
+ });
+ } else {
+ notifications.toasts.addError(error, {
+ title: 'Error',
+ });
+ setFormState('VALID');
+ }
+ };
+
+ const layoutProps = {
+ from: 'edit' as CreateDatasourceFrom,
+ cancelUrl,
+ agentConfig,
+ packageInfo,
+ };
+
+ return (
+
+ {isLoadingData ? (
+
+ ) : loadingError || !agentConfig || !packageInfo ? (
+
+ }
+ error={
+ loadingError ||
+ i18n.translate('xpack.ingestManager.editDatasource.errorLoadingDataMessage', {
+ defaultMessage: 'There was an error loading this data source information',
+ })
+ }
+ />
+ ) : (
+ <>
+ {formState === 'CONFIRM' && (
+ setFormState('VALID')}
+ />
+ )}
+
+ ),
+ },
+ {
+ title: i18n.translate(
+ 'xpack.ingestManager.editDatasource.stepConfgiureDatasourceTitle',
+ {
+ defaultMessage: 'Select the data you want to collect',
+ }
+ ),
+ children: (
+
+ ),
+ },
+ ]}
+ />
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ )}
+
+ );
+};
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/index.tsx
index 71ada155373bf..ef88aa5d17f1e 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/index.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/index.tsx
@@ -8,10 +8,14 @@ import { HashRouter as Router, Switch, Route } from 'react-router-dom';
import { AgentConfigListPage } from './list_page';
import { AgentConfigDetailsPage } from './details_page';
import { CreateDatasourcePage } from './create_datasource_page';
+import { EditDatasourcePage } from './edit_datasource_page';
export const AgentConfigApp: React.FunctionComponent = () => (
+
+
+
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/components/create_config.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/components/create_config.tsx
index 1fe116ef36090..9f582e7e2fbe6 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/components/create_config.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/components/create_config.tsx
@@ -34,6 +34,7 @@ export const CreateAgentConfigFlyout: React.FunctionComponent = ({ onClos
description: '',
namespace: '',
is_default: undefined,
+ monitoring_enabled: ['logs', 'metrics'],
});
const [isLoading, setIsLoading] = useState(false);
const [withSysMonitoring, setWithSysMonitoring] = useState(true);
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx
index 1ea162252c741..3dcc19bc4a5ae 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx
@@ -36,13 +36,11 @@ import {
useConfig,
useUrlParams,
} from '../../../hooks';
-import { AgentConfigDeleteProvider } from '../components';
import { CreateAgentConfigFlyout } from './components';
import { SearchBar } from '../../../components/search_bar';
import { LinkedAgentCount } from '../components';
import { useAgentConfigLink } from '../details_page/hooks/use_details_uri';
import { TableRowActions } from '../components/table_row_actions';
-import { DangerEuiContextMenuItem } from '../components/danger_eui_context_menu_item';
const NO_WRAP_TRUNCATE_STYLE: CSSProperties = Object.freeze({
overflow: 'hidden',
@@ -108,30 +106,12 @@ const ConfigRowActions = memo<{ config: AgentConfig; onDelete: () => void }>(
defaultMessage="Create data source"
/>
,
-
-
-
- ,
-
-
- {deleteAgentConfigsPrompt => {
- return (
- deleteAgentConfigsPrompt([config.id], onDelete)}
- >
-
-
- );
- }}
- ,
+ //
+ //
+ // ,
]}
/>
);
@@ -156,7 +136,6 @@ export const AgentConfigListPage: React.FunctionComponent<{}> = () => {
: urlParams.kuery ?? ''
);
const { pagination, pageSizeOptions, setPagination } = usePagination();
- const [selectedAgentConfigs, setSelectedAgentConfigs] = useState([]);
const history = useHistory();
const isCreateAgentConfigFlyoutOpen = 'create' in urlParams;
const setIsCreateAgentConfigFlyoutOpen = useCallback(
@@ -321,34 +300,6 @@ export const AgentConfigListPage: React.FunctionComponent<{}> = () => {
/>
) : null}
- {selectedAgentConfigs.length ? (
-
-
- {deleteAgentConfigsPrompt => (
- {
- deleteAgentConfigsPrompt(
- selectedAgentConfigs.map(agentConfig => agentConfig.id),
- () => {
- sendRequest();
- setSelectedAgentConfigs([]);
- }
- );
- }}
- >
-
-
- )}
-
-
- ) : null}
= () => {
items={agentConfigData ? agentConfigData.items : []}
itemId="id"
columns={columns}
- isSelectable={true}
- selection={{
- selectable: (agentConfig: AgentConfig) => !agentConfig.is_default,
- onSelectionChange: (newSelectedAgentConfigs: AgentConfig[]) => {
- setSelectedAgentConfigs(newSelectedAgentConfigs);
- },
- }}
+ isSelectable={false}
pagination={{
pageIndex: pagination.currentPage - 1,
pageSize: pagination.pageSize,
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/components/icons.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/components/icons.tsx
new file mode 100644
index 0000000000000..64223efefaab8
--- /dev/null
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/components/icons.tsx
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { EuiIcon } from '@elastic/eui';
+import React from 'react';
+import styled from 'styled-components';
+
+export const StyledAlert = styled(EuiIcon)`
+ color: ${props => props.theme.eui.euiColorWarning};
+ padding: 0 5px;
+`;
+
+export const UpdateIcon = () => ;
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/components/package_card.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/components/package_card.tsx
index 8ad081cbbabe4..ab7e87b3ad06c 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/components/package_card.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/components/package_card.tsx
@@ -30,9 +30,15 @@ export function PackageCard({
showInstalledBadge,
status,
icons,
+ ...restProps
}: PackageCardProps) {
const { toDetailView } = useLinks();
- const url = toDetailView({ name, version });
+ let urlVersion = version;
+ // if this is an installed package, link to the version installed
+ if ('savedObject' in restProps) {
+ urlVersion = restProps.savedObject.attributes.version || version;
+ }
+ const url = toDetailView({ name, version: urlVersion });
return (
;
+type InstallPackageProps = Pick & {
+ fromUpdate?: boolean;
+};
+type SetPackageInstallStatusProps = Pick & PackageInstallItem;
function usePackageInstall({ notifications }: { notifications: NotificationsStart }) {
+ const { toDetailView } = useLinks();
const [packages, setPackage] = useState({});
const setPackageInstallStatus = useCallback(
- ({ name, status }: { name: PackageInfo['name']; status: InstallStatus }) => {
+ ({ name, status, version }: SetPackageInstallStatusProps) => {
+ const packageProps: PackageInstallItem = {
+ status,
+ version,
+ };
setPackage((prev: PackagesInstall) => ({
...prev,
- [name]: { status },
+ [name]: packageProps,
}));
},
[]
);
+ const getPackageInstallStatus = useCallback(
+ (pkg: string): PackageInstallItem => {
+ return packages[pkg];
+ },
+ [packages]
+ );
+
const installPackage = useCallback(
- async ({ name, version, title }: InstallPackageProps) => {
- setPackageInstallStatus({ name, status: InstallStatus.installing });
+ async ({ name, version, title, fromUpdate = false }: InstallPackageProps) => {
+ const currStatus = getPackageInstallStatus(name);
+ const newStatus = { ...currStatus, name, status: InstallStatus.installing };
+ setPackageInstallStatus(newStatus);
const pkgkey = `${name}-${version}`;
const res = await sendInstallPackage(pkgkey);
if (res.error) {
- setPackageInstallStatus({ name, status: InstallStatus.notInstalled });
+ if (fromUpdate) {
+ // if there is an error during update, set it back to the previous version
+ // as handling of bad update is not implemented yet
+ setPackageInstallStatus({ ...currStatus, name });
+ } else {
+ setPackageInstallStatus({ name, status: InstallStatus.notInstalled, version });
+ }
notifications.toasts.addWarning({
title: toMountPoint(
{
- return packages[pkg].status;
- },
- [packages]
+ [getPackageInstallStatus, notifications.toasts, setPackageInstallStatus, toDetailView]
);
const uninstallPackage = useCallback(
async ({ name, version, title }: Pick) => {
- setPackageInstallStatus({ name, status: InstallStatus.uninstalling });
+ setPackageInstallStatus({ name, status: InstallStatus.uninstalling, version });
const pkgkey = `${name}-${version}`;
const res = await sendRemovePackage(pkgkey);
if (res.error) {
- setPackageInstallStatus({ name, status: InstallStatus.installed });
+ setPackageInstallStatus({ name, status: InstallStatus.installed, version });
notifications.toasts.addWarning({
title: toMountPoint(
;
export function ContentPanel(props: ContentPanelProps) {
- const { panel, name, version, assets, title, removable } = props;
+ const { panel, name, version, assets, title, removable, latestVersion } = props;
switch (panel) {
case 'settings':
return (
@@ -60,6 +60,7 @@ export function ContentPanel(props: ContentPanelProps) {
assets={assets}
title={title}
removable={removable}
+ latestVersion={latestVersion}
/>
);
case 'data-sources':
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/data_sources_panel.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/data_sources_panel.tsx
index fa3245aec02c5..c82b7ed2297a7 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/data_sources_panel.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/data_sources_panel.tsx
@@ -20,7 +20,7 @@ export const DataSourcesPanel = ({ name, version }: DataSourcesPanelProps) => {
const packageInstallStatus = getPackageInstallStatus(name);
// if they arrive at this page and the package is not installed, send them to overview
// this happens if they arrive with a direct url or they uninstall while on this tab
- if (packageInstallStatus !== InstallStatus.installed)
+ if (packageInstallStatus.status !== InstallStatus.installed)
return (
props.theme.eui.euiSizeM};
`;
-const StyledVersion = styled(Version)`
- font-size: ${props => props.theme.eui.euiFontSizeS};
- color: ${props => props.theme.eui.euiColorDarkShade};
-`;
-
type HeaderProps = PackageInfo & { iconType?: IconType };
export function Header(props: HeaderProps) {
- const { iconType, name, title, version } = props;
+ const { iconType, name, title, version, installedVersion, latestVersion } = props;
const hasWriteCapabilites = useCapabilities().write;
const { toListView } = useLinks();
const ADD_DATASOURCE_URI = useLink(`${EPM_PATH}/${name}-${version}/add-datasource`);
-
+ const updateAvailable = installedVersion && installedVersion < latestVersion ? true : false;
return (
@@ -59,7 +54,11 @@ export function Header(props: HeaderProps) {
{title}
-
+
+
+ {version} {updateAvailable && }
+
+
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/index.tsx
index 3239d7b90e3c3..1f3eb2cc9362e 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/index.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/index.tsx
@@ -32,11 +32,12 @@ export function Detail() {
const packageInfo = response.data?.response;
const title = packageInfo?.title;
const name = packageInfo?.name;
+ const installedVersion = packageInfo?.installedVersion;
const status: InstallStatus = packageInfo?.status as any;
// track install status state
if (name) {
- setPackageInstallStatus({ name, status });
+ setPackageInstallStatus({ name, status, version: installedVersion || null });
}
if (packageInfo) {
setInfo({ ...packageInfo, title: title || '' });
@@ -64,7 +65,6 @@ type LayoutProps = PackageInfo & Pick & Pick
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/installation_button.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/installation_button.tsx
index cbbf1ce53c4ea..cdad67fd87548 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/installation_button.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/installation_button.tsx
@@ -13,19 +13,21 @@ import { ConfirmPackageUninstall } from './confirm_package_uninstall';
import { ConfirmPackageInstall } from './confirm_package_install';
type InstallationButtonProps = Pick & {
- disabled: boolean;
+ disabled?: boolean;
+ isUpdate?: boolean;
};
export function InstallationButton(props: InstallationButtonProps) {
- const { assets, name, title, version, disabled = true } = props;
+ const { assets, name, title, version, disabled = true, isUpdate = false } = props;
const hasWriteCapabilites = useCapabilities().write;
const installPackage = useInstallPackage();
const uninstallPackage = useUninstallPackage();
const getPackageInstallStatus = useGetPackageInstallStatus();
- const installationStatus = getPackageInstallStatus(name);
+ const { status: installationStatus } = getPackageInstallStatus(name);
const isInstalling = installationStatus === InstallStatus.installing;
const isRemoving = installationStatus === InstallStatus.uninstalling;
const isInstalled = installationStatus === InstallStatus.installed;
+ const showUninstallButton = isInstalled || isRemoving;
const [isModalVisible, setModalVisible] = useState(false);
const toggleModal = useCallback(() => {
setModalVisible(!isModalVisible);
@@ -36,6 +38,10 @@ export function InstallationButton(props: InstallationButtonProps) {
toggleModal();
}, [installPackage, name, title, toggleModal, version]);
+ const handleClickUpdate = useCallback(() => {
+ installPackage({ name, version, title, fromUpdate: true });
+ }, [installPackage, name, title, version]);
+
const handleClickUninstall = useCallback(() => {
uninstallPackage({ name, version, title });
toggleModal();
@@ -78,6 +84,15 @@ export function InstallationButton(props: InstallationButtonProps) {
);
+ const updateButton = (
+
+
+
+ );
+
const uninstallButton = (
- {isInstalled || isRemoving ? uninstallButton : installButton}
+ {isUpdate ? updateButton : showUninstallButton ? uninstallButton : installButton}
{isModalVisible && (isInstalled ? uninstallModal : installModal)}
) : null;
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/settings_panel.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/settings_panel.tsx
index 3589a1a9444e1..4d4dba2a64e5a 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/settings_panel.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/settings_panel.tsx
@@ -8,11 +8,22 @@ import React from 'react';
import { FormattedMessage } from '@kbn/i18n/react';
import { EuiTitle, EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui';
import { EuiSpacer } from '@elastic/eui';
+import styled from 'styled-components';
import { InstallStatus, PackageInfo } from '../../../../types';
import { useGetDatasources } from '../../../../hooks';
import { DATASOURCE_SAVED_OBJECT_TYPE } from '../../../../constants';
import { useGetPackageInstallStatus } from '../../hooks';
import { InstallationButton } from './installation_button';
+import { UpdateIcon } from '../../components/icons';
+
+const SettingsTitleCell = styled.td`
+ padding-right: ${props => props.theme.eui.spacerSizes.xl};
+ padding-bottom: ${props => props.theme.eui.spacerSizes.m};
+`;
+
+const UpdatesAvailableMsgContainer = styled.span`
+ padding-left: ${props => props.theme.eui.spacerSizes.s};
+`;
const NoteLabel = () => (
(
defaultMessage="Note:"
/>
);
+const UpdatesAvailableMsg = () => (
+
+
+
+
+);
+
export const SettingsPanel = (
- props: Pick
+ props: Pick
) => {
+ const { name, title, removable, latestVersion, version } = props;
const getPackageInstallStatus = useGetPackageInstallStatus();
const { data: datasourcesData } = useGetDatasources({
perPage: 0,
page: 1,
kuery: `${DATASOURCE_SAVED_OBJECT_TYPE}.package.name:${props.name}`,
});
- const { name, title, removable } = props;
- const packageInstallStatus = getPackageInstallStatus(name);
+ const { status: installationStatus, version: installedVersion } = getPackageInstallStatus(name);
const packageHasDatasources = !!datasourcesData?.total;
+ const updateAvailable = installedVersion && installedVersion < latestVersion ? true : false;
+ const isViewingOldPackage = version < latestVersion;
+ // hide install/remove options if the user has version of the package is installed
+ // and this package is out of date or if they do have a version installed but it's not this one
+ const hideInstallOptions =
+ (installationStatus === InstallStatus.notInstalled && isViewingOldPackage) ||
+ (installationStatus === InstallStatus.installed && installedVersion !== version);
+
+ const isUpdating = installationStatus === InstallStatus.installing && installedVersion;
return (
@@ -43,14 +73,13 @@ export const SettingsPanel = (
- {packageInstallStatus === InstallStatus.notInstalled ||
- packageInstallStatus === InstallStatus.installing ? (
+ {installedVersion !== null && (
-
-
-
+
+
+
+
+
+
+ |
+
+ {installedVersion}
+
+ {updateAvailable && }
+ |
+
+
+
+
+
+ |
+
+ {latestVersion}
+
+ |
+
+
+
+ {updateAvailable && (
+
+
+
+ )}
- ) : (
+ )}
+ {!hideInstallOptions && !isUpdating && (
-
-
+
+ {installationStatus === InstallStatus.notInstalled ||
+ installationStatus === InstallStatus.installing ? (
+
+ ) : (
+
+ )}
+
+
+
+
+
+
+
+ {packageHasDatasources && removable === true && (
+
+
+
+ ),
}}
/>
-
-
-
-
-
-
+
+ )}
+ {removable === false && (
+
+
+
+
+ ),
+ }}
+ />
+
+ )}
-
-
-
-
- {packageHasDatasources && removable === true && (
-
-
-
-
- ),
- }}
- />
-
- )}
- {removable === false && (
-
-
-
-
- ),
- }}
- />
-
- )}
);
};
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/side_nav_links.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/side_nav_links.tsx
index 05729ccfc1af4..ab168ef1530bd 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/side_nav_links.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/side_nav_links.tsx
@@ -37,7 +37,7 @@ export function SideNavLinks({ name, version, active }: NavLinkProps) {
: p.theme.eui.euiFontWeightRegular};
`;
// don't display Data Sources tab if the package is not installed
- if (packageInstallStatus !== InstallStatus.installed && panel === 'data-sources')
+ if (packageInstallStatus.status !== InstallStatus.installed && panel === 'data-sources')
return null;
return (
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/helper.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/helper.ts
new file mode 100644
index 0000000000000..508190fef0fc2
--- /dev/null
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/helper.ts
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { AgentMetadata } from '../../../../types';
+
+export function flattenMetadata(metadata: AgentMetadata) {
+ return Object.entries(metadata).reduce((acc, [key, value]) => {
+ if (typeof value === 'string') {
+ acc[key] = value;
+
+ return acc;
+ }
+
+ Object.entries(flattenMetadata(value)).forEach(([flattenedKey, flattenedValue]) => {
+ acc[`${key}.${flattenedKey}`] = flattenedValue;
+ });
+
+ return acc;
+ }, {} as { [k: string]: string });
+}
+export function unflattenMetadata(flattened: { [k: string]: string }) {
+ const metadata: AgentMetadata = {};
+
+ Object.entries(flattened).forEach(([flattenedKey, flattenedValue]) => {
+ const keyParts = flattenedKey.split('.');
+ const lastKey = keyParts.pop();
+
+ if (!lastKey) {
+ throw new Error('Invalid metadata');
+ }
+
+ let metadataPart = metadata;
+ keyParts.forEach(keyPart => {
+ if (!metadataPart[keyPart]) {
+ metadataPart[keyPart] = {};
+ }
+
+ metadataPart = metadataPart[keyPart] as AgentMetadata;
+ });
+ metadataPart[lastKey] = flattenedValue;
+ });
+
+ return metadata;
+}
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_flyout.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_flyout.tsx
index ee43385e601c2..aa6da36f8fb6c 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_flyout.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_flyout.tsx
@@ -17,11 +17,13 @@ import {
} from '@elastic/eui';
import { MetadataForm } from './metadata_form';
import { Agent } from '../../../../types';
+import { flattenMetadata } from './helper';
interface Props {
agent: Agent;
flyout: { hide: () => void };
}
+
export const AgentMetadataFlyout: React.FunctionComponent = ({ agent, flyout }) => {
const mapMetadata = (obj: { [key: string]: string } | undefined) => {
return Object.keys(obj || {}).map(key => ({
@@ -30,8 +32,8 @@ export const AgentMetadataFlyout: React.FunctionComponent = ({ agent, fly
}));
};
- const localItems = mapMetadata(agent.local_metadata);
- const userProvidedItems = mapMetadata(agent.user_provided_metadata);
+ const localItems = mapMetadata(flattenMetadata(agent.local_metadata));
+ const userProvidedItems = mapMetadata(flattenMetadata(agent.user_provided_metadata));
return (
flyout.hide()} size="s" aria-labelledby="flyoutTitle">
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_form.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_form.tsx
index ce28bbdc590b0..af7e8c674db4c 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_form.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_form.tsx
@@ -22,6 +22,7 @@ import { useAgentRefresh } from '../hooks';
import { useInput, sendRequest } from '../../../../hooks';
import { Agent } from '../../../../types';
import { agentRouteService } from '../../../../services';
+import { flattenMetadata, unflattenMetadata } from './helper';
function useAddMetadataForm(agent: Agent, done: () => void) {
const refreshAgent = useAgentRefresh();
@@ -66,15 +67,17 @@ function useAddMetadataForm(agent: Agent, done: () => void) {
isLoading: true,
});
+ const metadata = unflattenMetadata({
+ ...flattenMetadata(agent.user_provided_metadata),
+ [keyInput.value]: valueInput.value,
+ });
+
try {
const { error } = await sendRequest({
path: agentRouteService.getUpdatePath(agent.id),
method: 'put',
body: JSON.stringify({
- user_provided_metadata: {
- ...agent.user_provided_metadata,
- [keyInput.value]: valueInput.value,
- },
+ user_provided_metadata: metadata,
}),
});
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx
index 23fe18b82468c..05264c157434e 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx
@@ -238,7 +238,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => {
const columns = [
{
- field: 'local_metadata.host',
+ field: 'local_metadata.host.hostname',
name: i18n.translate('xpack.ingestManager.agentList.hostColumnTitle', {
defaultMessage: 'Host',
}),
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/overview/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/overview/index.tsx
index 05d150fd9ae23..70d8e7d6882f8 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/overview/index.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/overview/index.tsx
@@ -8,6 +8,7 @@ import styled from 'styled-components';
import {
EuiButton,
EuiButtonEmpty,
+ EuiBetaBadge,
EuiPanel,
EuiText,
EuiTitle,
@@ -19,10 +20,11 @@ import {
EuiFlexItem,
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
+import { i18n } from '@kbn/i18n';
import { WithHeaderLayout } from '../../layouts';
import { useLink, useGetAgentConfigs } from '../../hooks';
import { AgentEnrollmentFlyout } from '../fleet/agent_list_page/components';
-import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH } from '../../constants';
+import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH, DATA_STREAM_PATH } from '../../constants';
const OverviewPanel = styled(EuiPanel).attrs(props => ({
paddingSize: 'm',
@@ -57,6 +59,11 @@ const OverviewStats = styled(EuiDescriptionList).attrs(props => ({
}
`;
+const AlphaBadge = styled(EuiBetaBadge)`
+ vertical-align: top;
+ margin-left: ${props => props.theme.eui.paddingSizes.s};
+`;
+
export const IngestManagerOverview: React.FunctionComponent = () => {
// Agent enrollment flyout state
const [isEnrollmentFlyoutOpen, setIsEnrollmentFlyoutOpen] = useState(false);
@@ -79,6 +86,19 @@ export const IngestManagerOverview: React.FunctionComponent = () => {
id="xpack.ingestManager.overviewPageTitle"
defaultMessage="Ingest Manager"
/>
+
@@ -213,7 +233,7 @@ export const IngestManagerOverview: React.FunctionComponent = () => {
/>
-
+
+ (agentConfig: GetAgentConfigsResponseItem) =>
listAgents(soClient, {
showInactive: true,
perPage: 0,
diff --git a/x-pack/plugins/ingest_manager/server/routes/datasource/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/datasource/handlers.ts
index 56d6053a1451b..8f07e3ed1de02 100644
--- a/x-pack/plugins/ingest_manager/server/routes/datasource/handlers.ts
+++ b/x-pack/plugins/ingest_manager/server/routes/datasource/handlers.ts
@@ -7,7 +7,7 @@ import { TypeOf } from '@kbn/config-schema';
import Boom from 'boom';
import { RequestHandler } from 'src/core/server';
import { appContextService, datasourceService } from '../../services';
-import { ensureInstalledPackage } from '../../services/epm/packages';
+import { ensureInstalledPackage, getPackageInfo } from '../../services/epm/packages';
import {
GetDatasourcesRequestSchema,
GetOneDatasourceRequestSchema,
@@ -85,12 +85,13 @@ export const createDatasourceHandler: RequestHandler<
pkgName: request.body.package.name,
callCluster,
});
-
+ const pkgInfo = await getPackageInfo({
+ savedObjectsClient: soClient,
+ pkgName: request.body.package.name,
+ pkgVersion: request.body.package.version,
+ });
newData.inputs = (await datasourceService.assignPackageStream(
- {
- pkgName: request.body.package.name,
- pkgVersion: request.body.package.version,
- },
+ pkgInfo,
request.body.inputs
)) as TypeOf['inputs'];
}
@@ -127,13 +128,14 @@ export const updateDatasourceHandler: RequestHandler<
const pkg = newData.package || datasource.package;
const inputs = newData.inputs || datasource.inputs;
if (pkg && (newData.inputs || newData.package)) {
- newData.inputs = (await datasourceService.assignPackageStream(
- {
- pkgName: pkg.name,
- pkgVersion: pkg.version,
- },
- inputs
- )) as TypeOf['inputs'];
+ const pkgInfo = await getPackageInfo({
+ savedObjectsClient: soClient,
+ pkgName: pkg.name,
+ pkgVersion: pkg.version,
+ });
+ newData.inputs = (await datasourceService.assignPackageStream(pkgInfo, inputs)) as TypeOf<
+ typeof CreateDatasourceRequestSchema.body
+ >['inputs'];
}
const updatedDatasource = await datasourceService.update(
diff --git a/x-pack/plugins/ingest_manager/server/routes/setup/index.ts b/x-pack/plugins/ingest_manager/server/routes/setup/index.ts
index edc9a0a268161..5ee7ee7733220 100644
--- a/x-pack/plugins/ingest_manager/server/routes/setup/index.ts
+++ b/x-pack/plugins/ingest_manager/server/routes/setup/index.ts
@@ -5,13 +5,14 @@
*/
import { IRouter } from 'src/core/server';
import { PLUGIN_ID, FLEET_SETUP_API_ROUTES, SETUP_API_ROUTE } from '../../constants';
+import { IngestManagerConfigType } from '../../../common';
import {
getFleetSetupHandler,
createFleetSetupHandler,
ingestManagerSetupHandler,
} from './handlers';
-export const registerRoutes = (router: IRouter) => {
+export const registerRoutes = (router: IRouter, config: IngestManagerConfigType) => {
// Ingest manager setup
router.post(
{
@@ -23,6 +24,11 @@ export const registerRoutes = (router: IRouter) => {
},
ingestManagerSetupHandler
);
+
+ if (!config.fleet.enabled) {
+ return;
+ }
+
// Get Fleet setup
router.get(
{
diff --git a/x-pack/plugins/ingest_manager/server/saved_objects.ts b/x-pack/plugins/ingest_manager/server/saved_objects.ts
index 882258e859555..89d8b9e173ffe 100644
--- a/x-pack/plugins/ingest_manager/server/saved_objects.ts
+++ b/x-pack/plugins/ingest_manager/server/saved_objects.ts
@@ -56,8 +56,8 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = {
enrolled_at: { type: 'date' },
access_api_key_id: { type: 'keyword' },
version: { type: 'keyword' },
- user_provided_metadata: { type: 'text' },
- local_metadata: { type: 'text' },
+ user_provided_metadata: { type: 'flattened' },
+ local_metadata: { type: 'flattened' },
config_id: { type: 'keyword' },
last_updated: { type: 'date' },
last_checkin: { type: 'date' },
@@ -128,6 +128,7 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = {
updated_on: { type: 'keyword' },
updated_by: { type: 'keyword' },
revision: { type: 'integer' },
+ monitoring_enabled: { type: 'keyword' },
},
},
},
diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts
new file mode 100644
index 0000000000000..17758f6e3d7f1
--- /dev/null
+++ b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts
@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { savedObjectsClientMock } from 'src/core/server/mocks';
+import { agentConfigService } from './agent_config';
+import { Output } from '../types';
+
+function getSavedObjectMock(configAttributes: any) {
+ const mock = savedObjectsClientMock.create();
+
+ mock.get.mockImplementation(async (type: string, id: string) => {
+ return {
+ type,
+ id,
+ references: [],
+ attributes: configAttributes,
+ };
+ });
+
+ return mock;
+}
+
+jest.mock('./output', () => {
+ return {
+ outputService: {
+ getDefaultOutputId: () => 'test-id',
+ get: (): Output => {
+ return {
+ id: 'test-id',
+ is_default: true,
+ name: 'default',
+ // @ts-ignore
+ type: 'elasticsearch',
+ hosts: ['http://127.0.0.1:9201'],
+ };
+ },
+ },
+ };
+});
+
+describe('agent config', () => {
+ describe('getFullConfig', () => {
+ it('should return a config without monitoring if not monitoring is not enabled', async () => {
+ const soClient = getSavedObjectMock({
+ revision: 1,
+ });
+ const config = await agentConfigService.getFullConfig(soClient, 'config');
+
+ expect(config).toMatchObject({
+ id: 'config',
+ outputs: {
+ default: {
+ type: 'elasticsearch',
+ hosts: ['http://127.0.0.1:9201'],
+ ca_sha256: undefined,
+ api_key: undefined,
+ },
+ },
+ datasources: [],
+ revision: 1,
+ settings: {
+ monitoring: {
+ enabled: false,
+ logs: false,
+ metrics: false,
+ },
+ },
+ });
+ });
+
+ it('should return a config with monitoring if monitoring is enabled for logs', async () => {
+ const soClient = getSavedObjectMock({
+ revision: 1,
+ monitoring_enabled: ['logs'],
+ });
+ const config = await agentConfigService.getFullConfig(soClient, 'config');
+
+ expect(config).toMatchObject({
+ id: 'config',
+ outputs: {
+ default: {
+ type: 'elasticsearch',
+ hosts: ['http://127.0.0.1:9201'],
+ ca_sha256: undefined,
+ api_key: undefined,
+ },
+ },
+ datasources: [],
+ revision: 1,
+ settings: {
+ monitoring: {
+ use_output: 'default',
+ enabled: true,
+ logs: true,
+ metrics: false,
+ },
+ },
+ });
+ });
+
+ it('should return a config with monitoring if monitoring is enabled for metrics', async () => {
+ const soClient = getSavedObjectMock({
+ revision: 1,
+ monitoring_enabled: ['metrics'],
+ });
+ const config = await agentConfigService.getFullConfig(soClient, 'config');
+
+ expect(config).toMatchObject({
+ id: 'config',
+ outputs: {
+ default: {
+ type: 'elasticsearch',
+ hosts: ['http://127.0.0.1:9201'],
+ ca_sha256: undefined,
+ api_key: undefined,
+ },
+ },
+ datasources: [],
+ revision: 1,
+ settings: {
+ monitoring: {
+ use_output: 'default',
+ enabled: true,
+ logs: false,
+ metrics: true,
+ },
+ },
+ });
+ });
+ });
+});
diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.ts
index 75bbfc21293c2..7ab6ef1920c18 100644
--- a/x-pack/plugins/ingest_manager/server/services/agent_config.ts
+++ b/x-pack/plugins/ingest_manager/server/services/agent_config.ts
@@ -301,28 +301,49 @@ class AgentConfigService {
if (!config) {
return null;
}
+ const defaultOutput = await outputService.get(
+ soClient,
+ await outputService.getDefaultOutputId(soClient)
+ );
const agentConfig: FullAgentConfig = {
id: config.id,
outputs: {
// TEMPORARY as we only support a default output
- ...[
- await outputService.get(soClient, await outputService.getDefaultOutputId(soClient)),
- ].reduce((outputs, { config: outputConfig, name, type, hosts, ca_sha256, api_key }) => {
- outputs[name] = {
- type,
- hosts,
- ca_sha256,
- api_key,
- ...outputConfig,
- };
- return outputs;
- }, {} as FullAgentConfig['outputs']),
+ ...[defaultOutput].reduce(
+ (outputs, { config: outputConfig, name, type, hosts, ca_sha256, api_key }) => {
+ outputs[name] = {
+ type,
+ hosts,
+ ca_sha256,
+ api_key,
+ ...outputConfig,
+ };
+ return outputs;
+ },
+ {} as FullAgentConfig['outputs']
+ ),
},
datasources: (config.datasources as Datasource[])
.filter(datasource => datasource.enabled)
.map(ds => storedDatasourceToAgentDatasource(ds)),
revision: config.revision,
+ ...(config.monitoring_enabled && config.monitoring_enabled.length > 0
+ ? {
+ settings: {
+ monitoring: {
+ use_output: defaultOutput.name,
+ enabled: true,
+ logs: config.monitoring_enabled.indexOf('logs') >= 0,
+ metrics: config.monitoring_enabled.indexOf('metrics') >= 0,
+ },
+ },
+ }
+ : {
+ settings: {
+ monitoring: { enabled: false, logs: false, metrics: false },
+ },
+ }),
};
return agentConfig;
diff --git a/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts b/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts
index c96a81ed9b758..9b1565e7d74aa 100644
--- a/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts
+++ b/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts
@@ -11,6 +11,7 @@ import {
AgentAction,
AgentSOAttributes,
AgentEventSOAttributes,
+ AgentMetadata,
} from '../../types';
import { agentConfigService } from '../agent_config';
@@ -28,7 +29,7 @@ export async function agentCheckin(
const updateData: {
last_checkin: string;
default_api_key?: string;
- local_metadata?: string;
+ local_metadata?: AgentMetadata;
current_error_events?: string;
} = {
last_checkin: new Date().toISOString(),
diff --git a/x-pack/plugins/ingest_manager/server/services/agents/crud.ts b/x-pack/plugins/ingest_manager/server/services/agents/crud.ts
index 175b92b75aca0..43fd5a3ce0ac9 100644
--- a/x-pack/plugins/ingest_manager/server/services/agents/crud.ts
+++ b/x-pack/plugins/ingest_manager/server/services/agents/crud.ts
@@ -103,7 +103,7 @@ export async function updateAgent(
}
) {
await soClient.update(AGENT_SAVED_OBJECT_TYPE, agentId, {
- user_provided_metadata: JSON.stringify(data.userProvidedMetatada),
+ user_provided_metadata: data.userProvidedMetatada,
});
}
diff --git a/x-pack/plugins/ingest_manager/server/services/agents/enroll.ts b/x-pack/plugins/ingest_manager/server/services/agents/enroll.ts
index a34d2e03e9b3d..81afa70ecb818 100644
--- a/x-pack/plugins/ingest_manager/server/services/agents/enroll.ts
+++ b/x-pack/plugins/ingest_manager/server/services/agents/enroll.ts
@@ -32,8 +32,8 @@ export async function enroll(
config_id: configId,
type,
enrolled_at: enrolledAt,
- user_provided_metadata: JSON.stringify(metadata?.userProvided ?? {}),
- local_metadata: JSON.stringify(metadata?.local ?? {}),
+ user_provided_metadata: metadata?.userProvided ?? {},
+ local_metadata: metadata?.local ?? {},
current_error_events: undefined,
access_api_key_id: undefined,
last_checkin: undefined,
diff --git a/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts b/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts
index b182662e0fb4e..11beba1cd7e43 100644
--- a/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts
+++ b/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts
@@ -19,8 +19,8 @@ export function savedObjectToAgent(so: SavedObject): Agent {
current_error_events: so.attributes.current_error_events
? JSON.parse(so.attributes.current_error_events)
: [],
- local_metadata: JSON.parse(so.attributes.local_metadata),
- user_provided_metadata: JSON.parse(so.attributes.user_provided_metadata),
+ local_metadata: so.attributes.local_metadata,
+ user_provided_metadata: so.attributes.user_provided_metadata,
access_api_key: undefined,
status: undefined,
};
diff --git a/x-pack/plugins/ingest_manager/server/services/agents/status.test.ts b/x-pack/plugins/ingest_manager/server/services/agents/status.test.ts
index b6de083cbe0cb..8140b1e6de470 100644
--- a/x-pack/plugins/ingest_manager/server/services/agents/status.test.ts
+++ b/x-pack/plugins/ingest_manager/server/services/agents/status.test.ts
@@ -18,8 +18,8 @@ describe('Agent status service', () => {
type: AGENT_TYPE_PERMANENT,
attributes: {
active: false,
- local_metadata: '{}',
- user_provided_metadata: '{}',
+ local_metadata: {},
+ user_provided_metadata: {},
},
} as SavedObject);
const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
@@ -33,8 +33,8 @@ describe('Agent status service', () => {
type: AGENT_TYPE_PERMANENT,
attributes: {
active: true,
- local_metadata: '{}',
- user_provided_metadata: '{}',
+ local_metadata: {},
+ user_provided_metadata: {},
},
} as SavedObject);
const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
diff --git a/x-pack/plugins/ingest_manager/server/services/datasource.test.ts b/x-pack/plugins/ingest_manager/server/services/datasource.test.ts
index 4cbbadce7f5bb..3682ae6d1167b 100644
--- a/x-pack/plugins/ingest_manager/server/services/datasource.test.ts
+++ b/x-pack/plugins/ingest_manager/server/services/datasource.test.ts
@@ -5,39 +5,38 @@
*/
import { datasourceService } from './datasource';
+import { PackageInfo } from '../types';
-async function mockedGetAssetsData(_a: any, _b: any, dataset: string) {
- if (dataset === 'dataset1') {
- return [
- {
- buffer: Buffer.from(`
+const TEMPLATE = `
type: log
metricset: ["dataset1"]
paths:
{{#each paths}}
- {{this}}
{{/each}}
-`),
- },
- ];
- }
- return [];
-}
-
-jest.mock('./epm/packages/assets', () => {
- return {
- getAssetsDataForPackageKey: mockedGetAssetsData,
- };
-});
+`;
describe('Datasource service', () => {
describe('assignPackageStream', () => {
- it('should work with cofig variables from the stream', async () => {
+ it('should work with config variables from the stream', async () => {
const inputs = await datasourceService.assignPackageStream(
- {
- pkgName: 'package',
- pkgVersion: '1.0.0',
- },
+ ({
+ datasources: [
+ {
+ inputs: [
+ {
+ type: 'log',
+ streams: [
+ {
+ dataset: 'package.dataset1',
+ template: TEMPLATE,
+ },
+ ],
+ },
+ ],
+ },
+ ],
+ } as unknown) as PackageInfo,
[
{
type: 'log',
@@ -85,10 +84,23 @@ describe('Datasource service', () => {
it('should work with config variables at the input level', async () => {
const inputs = await datasourceService.assignPackageStream(
- {
- pkgName: 'package',
- pkgVersion: '1.0.0',
- },
+ ({
+ datasources: [
+ {
+ inputs: [
+ {
+ type: 'log',
+ streams: [
+ {
+ dataset: 'package.dataset1',
+ template: TEMPLATE,
+ },
+ ],
+ },
+ ],
+ },
+ ],
+ } as unknown) as PackageInfo,
[
{
type: 'log',
diff --git a/x-pack/plugins/ingest_manager/server/services/datasource.ts b/x-pack/plugins/ingest_manager/server/services/datasource.ts
index 804039cf508ba..0a5ba43e40fba 100644
--- a/x-pack/plugins/ingest_manager/server/services/datasource.ts
+++ b/x-pack/plugins/ingest_manager/server/services/datasource.ts
@@ -4,20 +4,19 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { SavedObjectsClientContract } from 'src/core/server';
-import { safeLoad } from 'js-yaml';
import { AuthenticatedUser } from '../../../security/server';
import {
DeleteDatasourcesResponse,
packageToConfigDatasource,
DatasourceInput,
DatasourceInputStream,
+ PackageInfo,
} from '../../common';
import { DATASOURCE_SAVED_OBJECT_TYPE } from '../constants';
import { NewDatasource, Datasource, ListWithKuery } from '../types';
import { agentConfigService } from './agent_config';
import { getPackageInfo, getInstallation } from './epm/packages';
import { outputService } from './output';
-import { getAssetsDataForPackageKey } from './epm/packages/assets';
import { createStream } from './epm/agent/agent';
const SAVED_OBJECT_TYPE = DATASOURCE_SAVED_OBJECT_TYPE;
@@ -201,20 +200,16 @@ class DatasourceService {
}
public async assignPackageStream(
- pkgInfo: { pkgName: string; pkgVersion: string },
+ pkgInfo: PackageInfo,
inputs: DatasourceInput[]
): Promise {
const inputsPromises = inputs.map(input => _assignPackageStreamToInput(pkgInfo, input));
+
return Promise.all(inputsPromises);
}
}
-const _isAgentStream = (p: string) => !!p.match(/agent\/stream\/stream\.yml/);
-
-async function _assignPackageStreamToInput(
- pkgInfo: { pkgName: string; pkgVersion: string },
- input: DatasourceInput
-) {
+async function _assignPackageStreamToInput(pkgInfo: PackageInfo, input: DatasourceInput) {
const streamsPromises = input.streams.map(stream =>
_assignPackageStreamToStream(pkgInfo, input, stream)
);
@@ -224,7 +219,7 @@ async function _assignPackageStreamToInput(
}
async function _assignPackageStreamToStream(
- pkgInfo: { pkgName: string; pkgVersion: string },
+ pkgInfo: PackageInfo,
input: DatasourceInput,
stream: DatasourceInputStream
) {
@@ -232,27 +227,35 @@ async function _assignPackageStreamToStream(
return { ...stream, agent_stream: undefined };
}
const dataset = getDataset(stream.dataset);
- const assetsData = await getAssetsDataForPackageKey(pkgInfo, _isAgentStream, dataset);
+ const datasource = pkgInfo.datasources?.[0];
+ if (!datasource) {
+ throw new Error('Stream template not found, no datasource');
+ }
- const [pkgStream] = assetsData;
- if (!pkgStream || !pkgStream.buffer) {
- throw new Error(`Stream template not found for dataset ${dataset}`);
+ const inputFromPkg = datasource.inputs.find(pkgInput => pkgInput.type === input.type);
+ if (!inputFromPkg) {
+ throw new Error(`Stream template not found, unable to found input ${input.type}`);
}
- // Populate template variables from input config and stream config
- const data: { [k: string]: string | string[] } = {};
- if (input.vars) {
- for (const key of Object.keys(input.vars)) {
- data[key] = input.vars[key].value;
- }
+ const streamFromPkg = inputFromPkg.streams.find(
+ pkgStream => pkgStream.dataset === stream.dataset
+ );
+ if (!streamFromPkg) {
+ throw new Error(`Stream template not found, unable to found stream ${stream.dataset}`);
}
- if (stream.vars) {
- for (const key of Object.keys(stream.vars)) {
- data[key] = stream.vars[key].value;
- }
+
+ if (!streamFromPkg.template) {
+ throw new Error(`Stream template not found for dataset ${dataset}`);
}
- const yaml = safeLoad(createStream(data, pkgStream.buffer.toString()));
+
+ const yaml = createStream(
+ // Populate template variables from input vars and stream vars
+ Object.assign({}, input.vars, stream.vars),
+ streamFromPkg.template
+ );
+
stream.agent_stream = yaml;
+
return { ...stream };
}
diff --git a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.test.ts
index 21de625532f03..db2e4fe474640 100644
--- a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.test.ts
+++ b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.test.ts
@@ -6,29 +6,61 @@
import { createStream } from './agent';
-test('Test creating a stream from template', () => {
- const streamTemplate = `
-input: log
-paths:
-{{#each paths}}
- - {{this}}
-{{/each}}
-exclude_files: [".gz$"]
-processors:
- - add_locale: ~
- `;
- const vars = {
- paths: ['/usr/local/var/log/nginx/access.log'],
- };
+describe('createStream', () => {
+ it('should work', () => {
+ const streamTemplate = `
+ input: log
+ paths:
+ {{#each paths}}
+ - {{this}}
+ {{/each}}
+ exclude_files: [".gz$"]
+ processors:
+ - add_locale: ~
+ `;
+ const vars = {
+ paths: { value: ['/usr/local/var/log/nginx/access.log'] },
+ };
- const output = createStream(vars, streamTemplate);
+ const output = createStream(vars, streamTemplate);
+ expect(output).toEqual({
+ input: 'log',
+ paths: ['/usr/local/var/log/nginx/access.log'],
+ exclude_files: ['.gz$'],
+ processors: [{ add_locale: null }],
+ });
+ });
- expect(output).toBe(`
-input: log
-paths:
- - /usr/local/var/log/nginx/access.log
-exclude_files: [".gz$"]
-processors:
- - add_locale: ~
- `);
+ it('should support yaml values', () => {
+ const streamTemplate = `
+ input: redis/metrics
+ metricsets: ["key"]
+ test: null
+ {{#if key.patterns}}
+ key.patterns: {{key.patterns}}
+ {{/if}}
+ `;
+ const vars = {
+ 'key.patterns': {
+ type: 'yaml',
+ value: `
+ - limit: 20
+ pattern: '*'
+ `,
+ },
+ };
+
+ const output = createStream(vars, streamTemplate);
+ expect(output).toEqual({
+ input: 'redis/metrics',
+ metricsets: ['key'],
+ test: null,
+ 'key.patterns': [
+ {
+ limit: 20,
+ pattern: '*',
+ },
+ ],
+ });
+ });
});
diff --git a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts
index 5d9a6d409aa1a..8254c0d8aaa37 100644
--- a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts
+++ b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts
@@ -5,12 +5,71 @@
*/
import Handlebars from 'handlebars';
+import { safeLoad } from 'js-yaml';
+import { DatasourceConfigRecord } from '../../../../common';
-interface StreamVars {
- [k: string]: string | string[];
+function isValidKey(key: string) {
+ return key !== '__proto__' && key !== 'constructor' && key !== 'prototype';
}
-export function createStream(vars: StreamVars, streamTemplate: string) {
- const template = Handlebars.compile(streamTemplate);
- return template(vars);
+function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any) {
+ if (Object.keys(yamlVariables).length === 0 || !yaml) {
+ return yaml;
+ }
+
+ Object.entries(yaml).forEach(([key, value]: [string, any]) => {
+ if (typeof value === 'object') {
+ yaml[key] = replaceVariablesInYaml(yamlVariables, value);
+ }
+ if (typeof value === 'string' && value in yamlVariables) {
+ yaml[key] = yamlVariables[value];
+ }
+ });
+
+ return yaml;
+}
+
+function buildTemplateVariables(variables: DatasourceConfigRecord) {
+ const yamlValues: { [k: string]: any } = {};
+ const vars = Object.entries(variables).reduce((acc, [key, recordEntry]) => {
+ // support variables with . like key.patterns
+ const keyParts = key.split('.');
+ const lastKeyPart = keyParts.pop();
+
+ if (!lastKeyPart || !isValidKey(lastKeyPart)) {
+ throw new Error('Invalid key');
+ }
+
+ let varPart = acc;
+ for (const keyPart of keyParts) {
+ if (!isValidKey(keyPart)) {
+ throw new Error('Invalid key');
+ }
+ if (!varPart[keyPart]) {
+ varPart[keyPart] = {};
+ }
+ varPart = varPart[keyPart];
+ }
+
+ if (recordEntry.type && recordEntry.type === 'yaml') {
+ const yamlKeyPlaceholder = `##${key}##`;
+ varPart[lastKeyPart] = `"${yamlKeyPlaceholder}"`;
+ yamlValues[yamlKeyPlaceholder] = recordEntry.value ? safeLoad(recordEntry.value) : null;
+ } else {
+ varPart[lastKeyPart] = recordEntry.value;
+ }
+ return acc;
+ }, {} as { [k: string]: any });
+
+ return { vars, yamlValues };
+}
+
+export function createStream(variables: DatasourceConfigRecord, streamTemplate: string) {
+ const { vars, yamlValues } = buildTemplateVariables(variables);
+
+ const template = Handlebars.compile(streamTemplate, { noEscape: true });
+ const stream = template(vars);
+ const yamlFromStream = safeLoad(stream, {});
+
+ return replaceVariablesInYaml(yamlValues, yamlFromStream);
}
diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts
index 1a73c9581a2de..cacf84381dd88 100644
--- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts
+++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts
@@ -93,7 +93,7 @@ test('tests processing text field with multi fields', () => {
const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
- expect(JSON.stringify(mappings)).toEqual(JSON.stringify(textWithMultiFieldsMapping));
+ expect(mappings).toEqual(textWithMultiFieldsMapping);
});
test('tests processing keyword field with multi fields', () => {
@@ -127,7 +127,7 @@ test('tests processing keyword field with multi fields', () => {
const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
- expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithMultiFieldsMapping));
+ expect(mappings).toEqual(keywordWithMultiFieldsMapping);
});
test('tests processing keyword field with multi fields with analyzed text field', () => {
@@ -159,7 +159,7 @@ test('tests processing keyword field with multi fields with analyzed text field'
const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
- expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithAnalyzedMultiFieldsMapping));
+ expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping);
});
test('tests processing object field with no other attributes', () => {
@@ -177,7 +177,7 @@ test('tests processing object field with no other attributes', () => {
const fields: Field[] = safeLoad(objectFieldLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
- expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldMapping));
+ expect(mappings).toEqual(objectFieldMapping);
});
test('tests processing object field with enabled set to false', () => {
@@ -197,7 +197,7 @@ test('tests processing object field with enabled set to false', () => {
const fields: Field[] = safeLoad(objectFieldEnabledFalseLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
- expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldEnabledFalseMapping));
+ expect(mappings).toEqual(objectFieldEnabledFalseMapping);
});
test('tests processing object field with dynamic set to false', () => {
@@ -217,7 +217,7 @@ test('tests processing object field with dynamic set to false', () => {
const fields: Field[] = safeLoad(objectFieldDynamicFalseLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
- expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicFalseMapping));
+ expect(mappings).toEqual(objectFieldDynamicFalseMapping);
});
test('tests processing object field with dynamic set to true', () => {
@@ -237,7 +237,7 @@ test('tests processing object field with dynamic set to true', () => {
const fields: Field[] = safeLoad(objectFieldDynamicTrueLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
- expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicTrueMapping));
+ expect(mappings).toEqual(objectFieldDynamicTrueMapping);
});
test('tests processing object field with dynamic set to strict', () => {
@@ -257,5 +257,159 @@ test('tests processing object field with dynamic set to strict', () => {
const fields: Field[] = safeLoad(objectFieldDynamicStrictLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
- expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicStrictMapping));
+ expect(mappings).toEqual(objectFieldDynamicStrictMapping);
+});
+
+test('tests processing object field with property', () => {
+ const objectFieldWithPropertyLiteralYml = `
+- name: a
+ type: object
+- name: a.b
+ type: keyword
+ `;
+ const objectFieldWithPropertyMapping = {
+ properties: {
+ a: {
+ properties: {
+ b: {
+ ignore_above: 1024,
+ type: 'keyword',
+ },
+ },
+ },
+ },
+ };
+ const fields: Field[] = safeLoad(objectFieldWithPropertyLiteralYml);
+ const processedFields = processFields(fields);
+ const mappings = generateMappings(processedFields);
+ expect(mappings).toEqual(objectFieldWithPropertyMapping);
+});
+
+test('tests processing object field with property, reverse order', () => {
+ const objectFieldWithPropertyReversedLiteralYml = `
+- name: a.b
+ type: keyword
+- name: a
+ type: object
+ dynamic: false
+ `;
+ const objectFieldWithPropertyReversedMapping = {
+ properties: {
+ a: {
+ dynamic: false,
+ properties: {
+ b: {
+ ignore_above: 1024,
+ type: 'keyword',
+ },
+ },
+ },
+ },
+ };
+ const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml);
+ const processedFields = processFields(fields);
+ const mappings = generateMappings(processedFields);
+ expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
+});
+
+test('tests processing nested field with property', () => {
+ const nestedYaml = `
+ - name: a.b
+ type: keyword
+ - name: a
+ type: nested
+ dynamic: false
+ `;
+ const expectedMapping = {
+ properties: {
+ a: {
+ dynamic: false,
+ type: 'nested',
+ properties: {
+ b: {
+ ignore_above: 1024,
+ type: 'keyword',
+ },
+ },
+ },
+ },
+ };
+ const fields: Field[] = safeLoad(nestedYaml);
+ const processedFields = processFields(fields);
+ const mappings = generateMappings(processedFields);
+ expect(mappings).toEqual(expectedMapping);
+});
+
+test('tests processing nested field with property, nested field first', () => {
+ const nestedYaml = `
+ - name: a
+ type: nested
+ include_in_parent: true
+ - name: a.b
+ type: keyword
+ `;
+ const expectedMapping = {
+ properties: {
+ a: {
+ include_in_parent: true,
+ type: 'nested',
+ properties: {
+ b: {
+ ignore_above: 1024,
+ type: 'keyword',
+ },
+ },
+ },
+ },
+ };
+ const fields: Field[] = safeLoad(nestedYaml);
+ const processedFields = processFields(fields);
+ const mappings = generateMappings(processedFields);
+ expect(mappings).toEqual(expectedMapping);
+});
+
+test('tests processing nested leaf field with properties', () => {
+ const nestedYaml = `
+ - name: a
+ type: object
+ dynamic: false
+ - name: a.b
+ type: nested
+ enabled: false
+ `;
+ const expectedMapping = {
+ properties: {
+ a: {
+ dynamic: false,
+ properties: {
+ b: {
+ enabled: false,
+ type: 'nested',
+ },
+ },
+ },
+ },
+ };
+ const fields: Field[] = safeLoad(nestedYaml);
+ const processedFields = processFields(fields);
+ const mappings = generateMappings(processedFields);
+ expect(mappings).toEqual(expectedMapping);
+});
+
+test('tests constant_keyword field type handling', () => {
+ const constantKeywordLiteralYaml = `
+- name: constantKeyword
+ type: constant_keyword
+ `;
+ const constantKeywordMapping = {
+ properties: {
+ constantKeyword: {
+ type: 'constant_keyword',
+ },
+ },
+ };
+ const fields: Field[] = safeLoad(constantKeywordLiteralYaml);
+ const processedFields = processFields(fields);
+ const mappings = generateMappings(processedFields);
+ expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping));
});
diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts
index 9736f6d1cbd3c..c45c7e706be58 100644
--- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts
+++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts
@@ -71,7 +71,14 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
switch (type) {
case 'group':
- fieldProps = generateMappings(field.fields!);
+ fieldProps = { ...generateMappings(field.fields!), ...generateDynamicAndEnabled(field) };
+ break;
+ case 'group-nested':
+ fieldProps = {
+ ...generateMappings(field.fields!),
+ ...generateNestedProps(field),
+ type: 'nested',
+ };
break;
case 'integer':
fieldProps.type = 'long';
@@ -95,13 +102,10 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
}
break;
case 'object':
- fieldProps.type = 'object';
- if (field.hasOwnProperty('enabled')) {
- fieldProps.enabled = field.enabled;
- }
- if (field.hasOwnProperty('dynamic')) {
- fieldProps.dynamic = field.dynamic;
- }
+ fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' };
+ break;
+ case 'nested':
+ fieldProps = { ...fieldProps, ...generateNestedProps(field), type: 'nested' };
break;
case 'array':
// this assumes array fields were validated in an earlier step
@@ -128,6 +132,29 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
return { properties: props };
}
+function generateDynamicAndEnabled(field: Field) {
+ const props: Properties = {};
+ if (field.hasOwnProperty('enabled')) {
+ props.enabled = field.enabled;
+ }
+ if (field.hasOwnProperty('dynamic')) {
+ props.dynamic = field.dynamic;
+ }
+ return props;
+}
+
+function generateNestedProps(field: Field) {
+ const props = generateDynamicAndEnabled(field);
+
+ if (field.hasOwnProperty('include_in_parent')) {
+ props.include_in_parent = field.include_in_parent;
+ }
+ if (field.hasOwnProperty('include_in_root')) {
+ props.include_in_root = field.include_in_root;
+ }
+ return props;
+}
+
function generateMultiFields(fields: Fields): MultiFields {
const multiFields: MultiFields = {};
if (fields) {
diff --git a/x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts
index e3aef6077dbc3..f0ff4c6125452 100644
--- a/x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts
+++ b/x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts
@@ -179,4 +179,197 @@ describe('processFields', () => {
JSON.stringify(mixedFieldsExpanded)
);
});
+
+ const objectFieldWithProperty = [
+ {
+ name: 'a',
+ type: 'object',
+ dynamic: true,
+ },
+ {
+ name: 'a.b',
+ type: 'keyword',
+ },
+ ];
+
+ const objectFieldWithPropertyExpanded = [
+ {
+ name: 'a',
+ type: 'group',
+ dynamic: true,
+ fields: [
+ {
+ name: 'b',
+ type: 'keyword',
+ },
+ ],
+ },
+ ];
+ test('correctly handles properties of object type fields', () => {
+ expect(JSON.stringify(processFields(objectFieldWithProperty))).toEqual(
+ JSON.stringify(objectFieldWithPropertyExpanded)
+ );
+ });
+
+ test('correctly handles properties of object type fields where object comes second', () => {
+ const nested = [
+ {
+ name: 'a.b',
+ type: 'keyword',
+ },
+ {
+ name: 'a',
+ type: 'object',
+ dynamic: true,
+ },
+ ];
+
+ const nestedExpanded = [
+ {
+ name: 'a',
+ type: 'group',
+ dynamic: true,
+ fields: [
+ {
+ name: 'b',
+ type: 'keyword',
+ },
+ ],
+ },
+ ];
+ expect(processFields(nested)).toEqual(nestedExpanded);
+ });
+
+ test('correctly handles properties of nested type fields', () => {
+ const nested = [
+ {
+ name: 'a',
+ type: 'nested',
+ dynamic: true,
+ },
+ {
+ name: 'a.b',
+ type: 'keyword',
+ },
+ ];
+
+ const nestedExpanded = [
+ {
+ name: 'a',
+ type: 'group-nested',
+ dynamic: true,
+ fields: [
+ {
+ name: 'b',
+ type: 'keyword',
+ },
+ ],
+ },
+ ];
+ expect(processFields(nested)).toEqual(nestedExpanded);
+ });
+
+ test('correctly handles properties of nested type where nested top level comes second', () => {
+ const nested = [
+ {
+ name: 'a.b',
+ type: 'keyword',
+ },
+ {
+ name: 'a',
+ type: 'nested',
+ dynamic: true,
+ },
+ ];
+
+ const nestedExpanded = [
+ {
+ name: 'a',
+ type: 'group-nested',
+ dynamic: true,
+ fields: [
+ {
+ name: 'b',
+ type: 'keyword',
+ },
+ ],
+ },
+ ];
+ expect(processFields(nested)).toEqual(nestedExpanded);
+ });
+
+ test('ignores redefinitions of an object field', () => {
+ const object = [
+ {
+ name: 'a',
+ type: 'object',
+ dynamic: true,
+ },
+ {
+ name: 'a',
+ type: 'object',
+ dynamic: false,
+ },
+ ];
+
+ const objectExpected = [
+ {
+ name: 'a',
+ type: 'object',
+ // should preserve the field that was parsed first which had dynamic: true
+ dynamic: true,
+ },
+ ];
+ expect(processFields(object)).toEqual(objectExpected);
+ });
+
+ test('ignores redefinitions of a nested field', () => {
+ const nested = [
+ {
+ name: 'a',
+ type: 'nested',
+ dynamic: true,
+ },
+ {
+ name: 'a',
+ type: 'nested',
+ dynamic: false,
+ },
+ ];
+
+ const nestedExpected = [
+ {
+ name: 'a',
+ type: 'nested',
+ // should preserve the field that was parsed first which had dynamic: true
+ dynamic: true,
+ },
+ ];
+ expect(processFields(nested)).toEqual(nestedExpected);
+ });
+
+ test('ignores redefinitions of a nested and object field', () => {
+ const nested = [
+ {
+ name: 'a',
+ type: 'nested',
+ dynamic: true,
+ },
+ {
+ name: 'a',
+ type: 'object',
+ dynamic: false,
+ },
+ ];
+
+ const nestedExpected = [
+ {
+ name: 'a',
+ type: 'nested',
+ // should preserve the field that was parsed first which had dynamic: true
+ dynamic: true,
+ },
+ ];
+ expect(processFields(nested)).toEqual(nestedExpected);
+ });
});
diff --git a/x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts b/x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts
index 9c9843e0454ab..abaf7ab5b0dfc 100644
--- a/x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts
+++ b/x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts
@@ -28,6 +28,8 @@ export interface Field {
object_type?: string;
scaling_factor?: number;
dynamic?: 'strict' | boolean;
+ include_in_parent?: boolean;
+ include_in_root?: boolean;
// Kibana specific
analyzed?: boolean;
@@ -108,10 +110,54 @@ function dedupFields(fields: Fields): Fields {
return f.name === field.name;
});
if (found) {
- if (found.type === 'group' && field.type === 'group' && found.fields && field.fields) {
- found.fields = dedupFields(found.fields.concat(field.fields));
+ // remove name, type, and fields from `field` variable so we avoid merging them into `found`
+ const { name, type, fields: nestedFields, ...importantFieldProps } = field;
+ /**
+ * There are a couple scenarios this if is trying to account for:
+ * Example 1
+ * - name: a.b
+ * - name: a
+ * In this scenario found will be `group` and field could be either `object` or `nested`
+ * Example 2
+ * - name: a
+ * - name: a.b
+ * In this scenario found could be `object` or `nested` and field will be group
+ */
+ if (
+ // only merge if found is a group and field is object, nested, or group.
+ // Or if found is object, or nested, and field is a group.
+ // This is to avoid merging two objects, or nested, or object with a nested.
+ (found.type === 'group' &&
+ (field.type === 'object' || field.type === 'nested' || field.type === 'group')) ||
+ ((found.type === 'object' || found.type === 'nested') && field.type === 'group')
+ ) {
+ // if the new field has properties let's dedup and concat them with the already existing found variable in
+ // the array
+ if (field.fields) {
+ // if the found type was object or nested it won't have a fields array so let's initialize it
+ if (!found.fields) {
+ found.fields = [];
+ }
+ found.fields = dedupFields(found.fields.concat(field.fields));
+ }
+
+ // if found already had fields or got new ones from the new field coming in we need to assign the right
+ // type to it
+ if (found.fields) {
+ // If this field is supposed to be `nested` and we have fields, we need to preserve the fact that it is
+ // supposed to be `nested` for when the template is actually generated
+ if (found.type === 'nested' || field.type === 'nested') {
+ found.type = 'group-nested';
+ } else {
+ // found was either `group` already or `object` so just set it to `group`
+ found.type = 'group';
+ }
+ }
+ // we need to merge in other properties (like `dynamic`) that might exist
+ Object.assign(found, importantFieldProps);
+ // if `field.type` wasn't group object or nested, then there's a conflict in types, so lets ignore it
} else {
- // only 'group' fields can be merged in this way
+ // only `group`, `object`, and `nested` fields can be merged in this way
// XXX: don't abort on error for now
// see discussion in https://github.com/elastic/kibana/pull/59894
// throw new Error(
diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.test.ts
index bc1694348b4c2..f1660fbc01591 100644
--- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.test.ts
+++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.test.ts
@@ -150,6 +150,7 @@ describe('creating index patterns from yaml fields', () => {
{ fields: [{ name: 'testField', type: 'text' }], expect: 'string' },
{ fields: [{ name: 'testField', type: 'date' }], expect: 'date' },
{ fields: [{ name: 'testField', type: 'geo_point' }], expect: 'geo_point' },
+ { fields: [{ name: 'testField', type: 'constant_keyword' }], expect: 'string' },
];
tests.forEach(test => {
@@ -191,6 +192,7 @@ describe('creating index patterns from yaml fields', () => {
attr: 'aggregatable',
},
{ fields: [{ name, type: 'keyword' }], expect: true, attr: 'aggregatable' },
+ { fields: [{ name, type: 'constant_keyword' }], expect: true, attr: 'aggregatable' },
{ fields: [{ name, type: 'text', aggregatable: true }], expect: false, attr: 'aggregatable' },
{ fields: [{ name, type: 'text' }], expect: false, attr: 'aggregatable' },
{
diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts
index 05e64c6565dc6..ec657820a2225 100644
--- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts
+++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts
@@ -47,6 +47,7 @@ const typeMap: TypeMap = {
date: 'date',
ip: 'ip',
boolean: 'boolean',
+ constant_keyword: 'string',
};
export interface IndexPatternField {
diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts
index d76584225877c..da8d79a04b97c 100644
--- a/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts
+++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts
@@ -67,9 +67,10 @@ export async function getPackageInfo(options: {
pkgVersion: string;
}): Promise {
const { savedObjectsClient, pkgName, pkgVersion } = options;
- const [item, savedObject, assets] = await Promise.all([
+ const [item, savedObject, latestPackage, assets] = await Promise.all([
Registry.fetchInfo(pkgName, pkgVersion),
getInstallationObject({ savedObjectsClient, pkgName }),
+ Registry.fetchFindLatestPackage(pkgName),
Registry.getArchiveInfo(pkgName, pkgVersion),
] as const);
// adding `as const` due to regression in TS 3.7.2
@@ -79,6 +80,7 @@ export async function getPackageInfo(options: {
// add properties that aren't (or aren't yet) on Registry response
const updated = {
...item,
+ latestVersion: latestPackage.version,
title: item.title || nameAsTitle(item.name),
assets: Registry.groupPathsByService(assets || []),
};
diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts
index d49e0e661440f..c67cccd044bf5 100644
--- a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts
+++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts
@@ -43,6 +43,7 @@ export function createInstallableFrom(
? {
...from,
status: InstallationStatus.installed,
+ installedVersion: savedObject.attributes.version,
savedObject,
}
: {
diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts
index 06f3decdbbe6f..8f51c4d78305c 100644
--- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts
+++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts
@@ -106,7 +106,7 @@ export async function installPackage(options: {
try {
await deleteKibanaSavedObjectsAssets(savedObjectsClient, installedPkg.attributes.installed);
} catch (err) {
- // some assets may not exist if deleting during a failed update
+ // log these errors, some assets may not exist if deleted during a failed update
}
}
diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts
index ac2c869f3b9e9..befb4722b6504 100644
--- a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts
+++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts
@@ -121,8 +121,12 @@ export async function deleteKibanaSavedObjectsAssets(
const deletePromises = installedObjects.map(({ id, type }) => {
const assetType = type as AssetType;
if (savedObjectTypes.includes(assetType)) {
- savedObjectsClient.delete(assetType, id);
+ return savedObjectsClient.delete(assetType, id);
}
});
- await Promise.all(deletePromises);
+ try {
+ await Promise.all(deletePromises);
+ } catch (err) {
+ throw new Error('error deleting saved object asset');
+ }
}
diff --git a/x-pack/plugins/ingest_manager/server/services/setup.ts b/x-pack/plugins/ingest_manager/server/services/setup.ts
index 206ad76703cf5..390e240841611 100644
--- a/x-pack/plugins/ingest_manager/server/services/setup.ts
+++ b/x-pack/plugins/ingest_manager/server/services/setup.ts
@@ -145,7 +145,7 @@ async function addPackageToConfig(
config.namespace
);
newDatasource.inputs = await datasourceService.assignPackageStream(
- { pkgName: packageToInstall.name, pkgVersion: packageToInstall.version },
+ packageInfo,
newDatasource.inputs
);
diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx
index a7019ebc0a271..27ed1de849987 100644
--- a/x-pack/plugins/ingest_manager/server/types/index.tsx
+++ b/x-pack/plugins/ingest_manager/server/types/index.tsx
@@ -8,6 +8,7 @@ import { ScopedClusterClient } from 'src/core/server';
export {
// Object types
Agent,
+ AgentMetadata,
AgentSOAttributes,
AgentStatus,
AgentType,
diff --git a/x-pack/plugins/ingest_manager/server/types/models/agent_config.ts b/x-pack/plugins/ingest_manager/server/types/models/agent_config.ts
index 040b2eb16289a..59cadf3bd7f74 100644
--- a/x-pack/plugins/ingest_manager/server/types/models/agent_config.ts
+++ b/x-pack/plugins/ingest_manager/server/types/models/agent_config.ts
@@ -11,6 +11,9 @@ const AgentConfigBaseSchema = {
name: schema.string(),
namespace: schema.maybe(schema.string()),
description: schema.maybe(schema.string()),
+ monitoring_enabled: schema.maybe(
+ schema.arrayOf(schema.oneOf([schema.literal('logs'), schema.literal('metrics')]))
+ ),
};
export const NewAgentConfigSchema = schema.object({
diff --git a/x-pack/plugins/lens/public/datatable_visualization/visualization.tsx b/x-pack/plugins/lens/public/datatable_visualization/visualization.tsx
index 359c06a6a9ebc..21bbcce68bf36 100644
--- a/x-pack/plugins/lens/public/datatable_visualization/visualization.tsx
+++ b/x-pack/plugins/lens/public/datatable_visualization/visualization.tsx
@@ -41,6 +41,10 @@ export const datatableVisualization: Visualization<
},
],
+ getVisualizationTypeId() {
+ return 'lnsDatatable';
+ },
+
getLayerIds(state) {
return state.layers.map(l => l.layerId);
},
@@ -122,8 +126,8 @@ export const datatableVisualization: Visualization<
],
},
previewIcon: chartTableSVG,
- // dont show suggestions for reduced versions or single-line tables
- hide: table.changeType === 'reduced' || !table.isMultiRow,
+ // tables are hidden from suggestion bar, but used for drag & drop and chart switching
+ hide: true,
},
];
},
diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/chart_switch.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/chart_switch.test.tsx
index 3c61d270b1bcf..c8d8064e60e38 100644
--- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/chart_switch.test.tsx
+++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/chart_switch.test.tsx
@@ -62,7 +62,25 @@ describe('chart_switch', () => {
id: 'subvisC2',
label: 'C2',
},
+ {
+ icon: 'empty',
+ id: 'subvisC3',
+ label: 'C3',
+ },
],
+ getSuggestions: jest.fn(options => {
+ if (options.subVisualizationId === 'subvisC2') {
+ return [];
+ }
+ return [
+ {
+ score: 1,
+ title: '',
+ state: `suggestion`,
+ previewIcon: 'empty',
+ },
+ ];
+ }),
},
};
}
@@ -313,10 +331,11 @@ describe('chart_switch', () => {
expect(getMenuItem('subvisB', component).prop('betaBadgeIconType')).toBeUndefined();
});
- it('should not indicate data loss if visualization is not changed', () => {
+ it('should not show a warning when the subvisualization is the same', () => {
const dispatch = jest.fn();
const frame = mockFrame(['a', 'b', 'c']);
const visualizations = mockVisualizations();
+ visualizations.visC.getVisualizationTypeId.mockReturnValue('subvisC2');
const switchVisualizationType = jest.fn(() => 'therebedragons');
visualizations.visC.switchVisualizationType = switchVisualizationType;
@@ -333,10 +352,10 @@ describe('chart_switch', () => {
/>
);
- expect(getMenuItem('subvisC2', component).prop('betaBadgeIconType')).toBeUndefined();
+ expect(getMenuItem('subvisC2', component).prop('betaBadgeIconType')).not.toBeDefined();
});
- it('should remove all layers if there is no suggestion', () => {
+ it('should get suggestions when switching subvisualization', () => {
const dispatch = jest.fn();
const visualizations = mockVisualizations();
visualizations.visB.getSuggestions.mockReturnValueOnce([]);
@@ -377,7 +396,7 @@ describe('chart_switch', () => {
const dispatch = jest.fn();
const frame = mockFrame(['a', 'b', 'c']);
const visualizations = mockVisualizations();
- const switchVisualizationType = jest.fn(() => 'therebedragons');
+ const switchVisualizationType = jest.fn(() => 'switched');
visualizations.visC.switchVisualizationType = switchVisualizationType;
@@ -393,12 +412,12 @@ describe('chart_switch', () => {
/>
);
- switchTo('subvisC2', component);
- expect(switchVisualizationType).toHaveBeenCalledWith('subvisC2', 'therebegriffins');
+ switchTo('subvisC3', component);
+ expect(switchVisualizationType).toHaveBeenCalledWith('subvisC3', 'suggestion');
expect(dispatch).toHaveBeenCalledWith(
expect.objectContaining({
type: 'SWITCH_VISUALIZATION',
- initialState: 'therebedragons',
+ initialState: 'switched',
})
);
expect(frame.removeLayers).not.toHaveBeenCalled();
diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/chart_switch.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/chart_switch.tsx
index 1461449f3c1c8..d73f83e75c0e4 100644
--- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/chart_switch.tsx
+++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/chart_switch.tsx
@@ -105,7 +105,16 @@ export function ChartSwitch(props: Props) {
const switchVisType =
props.visualizationMap[visualizationId].switchVisualizationType ||
((_type: string, initialState: unknown) => initialState);
- if (props.visualizationId === visualizationId) {
+ const layers = Object.entries(props.framePublicAPI.datasourceLayers);
+ const containsData = layers.some(
+ ([_layerId, datasource]) => datasource.getTableSpec().length > 0
+ );
+ // Always show the active visualization as a valid selection
+ if (
+ props.visualizationId === visualizationId &&
+ props.visualizationState &&
+ newVisualization.getVisualizationTypeId(props.visualizationState) === subVisualizationId
+ ) {
return {
visualizationId,
subVisualizationId,
@@ -116,13 +125,13 @@ export function ChartSwitch(props: Props) {
};
}
- const layers = Object.entries(props.framePublicAPI.datasourceLayers);
- const containsData = layers.some(
- ([_layerId, datasource]) => datasource.getTableSpec().length > 0
+ const topSuggestion = getTopSuggestion(
+ props,
+ visualizationId,
+ newVisualization,
+ subVisualizationId
);
- const topSuggestion = getTopSuggestion(props, visualizationId, newVisualization);
-
let dataLoss: VisualizationSelection['dataLoss'];
if (!containsData) {
@@ -250,7 +259,8 @@ export function ChartSwitch(props: Props) {
function getTopSuggestion(
props: Props,
visualizationId: string,
- newVisualization: Visualization
+ newVisualization: Visualization,
+ subVisualizationId?: string
): Suggestion | undefined {
const suggestions = getSuggestions({
datasourceMap: props.datasourceMap,
@@ -258,6 +268,7 @@ function getTopSuggestion(
visualizationMap: { [visualizationId]: newVisualization },
activeVisualizationId: props.visualizationId,
visualizationState: props.visualizationState,
+ subVisualizationId,
}).filter(suggestion => {
// don't use extended versions of current data table on switching between visualizations
// to avoid confusing the user.
diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/suggestion_helpers.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/suggestion_helpers.ts
index eabcdfa7a24ab..949ae1f43448e 100644
--- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/suggestion_helpers.ts
+++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/suggestion_helpers.ts
@@ -44,6 +44,7 @@ export function getSuggestions({
datasourceStates,
visualizationMap,
activeVisualizationId,
+ subVisualizationId,
visualizationState,
field,
}: {
@@ -57,6 +58,7 @@ export function getSuggestions({
>;
visualizationMap: Record;
activeVisualizationId: string | null;
+ subVisualizationId?: string;
visualizationState: unknown;
field?: unknown;
}): Suggestion[] {
@@ -89,7 +91,8 @@ export function getSuggestions({
table,
visualizationId,
datasourceSuggestion,
- currentVisualizationState
+ currentVisualizationState,
+ subVisualizationId
);
})
)
@@ -108,13 +111,15 @@ function getVisualizationSuggestions(
table: TableSuggestion,
visualizationId: string,
datasourceSuggestion: DatasourceSuggestion & { datasourceId: string },
- currentVisualizationState: unknown
+ currentVisualizationState: unknown,
+ subVisualizationId?: string
) {
return visualization
.getSuggestions({
table,
state: currentVisualizationState,
keptLayerIds: datasourceSuggestion.keptLayerIds,
+ subVisualizationId,
})
.map(({ state, ...visualizationSuggestion }) => ({
...visualizationSuggestion,
diff --git a/x-pack/plugins/lens/public/editor_frame_service/mocks.tsx b/x-pack/plugins/lens/public/editor_frame_service/mocks.tsx
index 50cd1ad8bd53a..e684fe8b3b5d6 100644
--- a/x-pack/plugins/lens/public/editor_frame_service/mocks.tsx
+++ b/x-pack/plugins/lens/public/editor_frame_service/mocks.tsx
@@ -28,6 +28,7 @@ export function createMockVisualization(): jest.Mocked {
label: 'TEST',
},
],
+ getVisualizationTypeId: jest.fn(_state => 'empty'),
getDescription: jest.fn(_state => ({ label: '' })),
switchVisualizationType: jest.fn((_, x) => x),
getPersistableState: jest.fn(_state => _state),
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/auto_date.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/auto_date.test.ts
deleted file mode 100644
index 5f35ef650a08c..0000000000000
--- a/x-pack/plugins/lens/public/indexpattern_datasource/auto_date.test.ts
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks';
-import { getAutoDate } from './auto_date';
-
-describe('auto_date', () => {
- let autoDate: ReturnType;
-
- beforeEach(() => {
- autoDate = getAutoDate({ data: dataPluginMock.createSetupContract() });
- });
-
- it('should do nothing if no time range is provided', () => {
- const result = autoDate.fn(
- {
- type: 'kibana_context',
- },
- {
- aggConfigs: 'canttouchthis',
- },
- // eslint-disable-next-line
- {} as any
- );
-
- expect(result).toEqual('canttouchthis');
- });
-
- it('should not change anything if there are no auto date histograms', () => {
- const aggConfigs = JSON.stringify([
- { type: 'date_histogram', params: { interval: '35h' } },
- { type: 'count' },
- ]);
- const result = autoDate.fn(
- {
- timeRange: {
- from: 'now-10d',
- to: 'now',
- },
- type: 'kibana_context',
- },
- {
- aggConfigs,
- },
- // eslint-disable-next-line
- {} as any
- );
-
- expect(result).toEqual(aggConfigs);
- });
-
- it('should change auto date histograms', () => {
- const aggConfigs = JSON.stringify([
- { type: 'date_histogram', params: { interval: 'auto' } },
- { type: 'count' },
- ]);
- const result = autoDate.fn(
- {
- timeRange: {
- from: 'now-10d',
- to: 'now',
- },
- type: 'kibana_context',
- },
- {
- aggConfigs,
- },
- // eslint-disable-next-line
- {} as any
- );
-
- const interval = JSON.parse(result).find(
- (agg: { type: string }) => agg.type === 'date_histogram'
- ).params.interval;
-
- expect(interval).toBeTruthy();
- expect(typeof interval).toEqual('string');
- expect(interval).not.toEqual('auto');
- });
-});
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/auto_date.ts b/x-pack/plugins/lens/public/indexpattern_datasource/auto_date.ts
deleted file mode 100644
index 97a46f4a3e176..0000000000000
--- a/x-pack/plugins/lens/public/indexpattern_datasource/auto_date.ts
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { DataPublicPluginSetup } from '../../../../../src/plugins/data/public';
-import {
- ExpressionFunctionDefinition,
- KibanaContext,
-} from '../../../../../src/plugins/expressions/public';
-
-interface LensAutoDateProps {
- aggConfigs: string;
-}
-
-export function getAutoDate(deps: {
- data: DataPublicPluginSetup;
-}): ExpressionFunctionDefinition<
- 'lens_auto_date',
- KibanaContext | null,
- LensAutoDateProps,
- string
-> {
- function autoIntervalFromContext(ctx?: KibanaContext | null) {
- if (!ctx || !ctx.timeRange) {
- return;
- }
-
- return deps.data.search.aggs.calculateAutoTimeExpression(ctx.timeRange);
- }
-
- /**
- * Convert all 'auto' date histograms into a concrete value (e.g. 2h).
- * This allows us to support 'auto' on all date fields, and opens the
- * door to future customizations (e.g. adjusting the level of detail, etc).
- */
- return {
- name: 'lens_auto_date',
- aliases: [],
- help: '',
- inputTypes: ['kibana_context', 'null'],
- args: {
- aggConfigs: {
- types: ['string'],
- default: '""',
- help: '',
- },
- },
- fn(input, args) {
- const interval = autoIntervalFromContext(input);
-
- if (!interval) {
- return args.aggConfigs;
- }
-
- const configs = JSON.parse(args.aggConfigs) as Array<{
- type: string;
- params: { interval: string };
- }>;
-
- const updatedConfigs = configs.map(c => {
- if (c.type !== 'date_histogram' || !c.params || c.params.interval !== 'auto') {
- return c;
- }
-
- return {
- ...c,
- params: {
- ...c.params,
- interval,
- },
- };
- });
-
- return JSON.stringify(updatedConfigs);
- },
- };
-}
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx
index 074c40759f8d8..9df79aa9e0908 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx
@@ -1380,5 +1380,43 @@ describe('IndexPatternDimensionEditorPanel', () => {
},
});
});
+
+ it('does not set the size of the terms aggregation', () => {
+ const dragging = {
+ field: { type: 'string', name: 'mystring', aggregatable: true },
+ indexPatternId: 'foo',
+ };
+ const testState = dragDropState();
+ onDrop({
+ ...defaultProps,
+ dragDropContext: {
+ ...dragDropContext,
+ dragging,
+ },
+ droppedItem: dragging,
+ state: testState,
+ columnId: 'col2',
+ filterOperations: (op: OperationMetadata) => op.isBucketed,
+ layerId: 'myLayer',
+ });
+
+ expect(setState).toBeCalledTimes(1);
+ expect(setState).toHaveBeenCalledWith({
+ ...testState,
+ layers: {
+ myLayer: {
+ ...testState.layers.myLayer,
+ columnOrder: ['col1', 'col2'],
+ columns: {
+ ...testState.layers.myLayer.columns,
+ col2: expect.objectContaining({
+ operationType: 'terms',
+ params: expect.objectContaining({ size: 3 }),
+ }),
+ },
+ },
+ },
+ });
+ });
});
});
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts
index fe14f472341af..73fd144b9c7f8 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts
@@ -8,7 +8,6 @@ import { CoreSetup } from 'kibana/public';
import { Storage } from '../../../../../src/plugins/kibana_utils/public';
import { getIndexPatternDatasource } from './indexpattern';
import { renameColumns } from './rename_columns';
-import { getAutoDate } from './auto_date';
import { ExpressionsSetup } from '../../../../../src/plugins/expressions/public';
import {
DataPublicPluginSetup,
@@ -31,10 +30,9 @@ export class IndexPatternDatasource {
setup(
core: CoreSetup,
- { data: dataSetup, expressions, editorFrame }: IndexPatternDatasourceSetupPlugins
+ { expressions, editorFrame }: IndexPatternDatasourceSetupPlugins
) {
expressions.registerFunction(renameColumns);
- expressions.registerFunction(getAutoDate({ data: dataSetup }));
editorFrame.registerDatasource(
core.getStartServices().then(([coreStart, { data }]) =>
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts
index e4f3677d0fe88..06635e663361d 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts
@@ -10,6 +10,7 @@ import { DatasourcePublicAPI, Operation, Datasource } from '../types';
import { coreMock } from 'src/core/public/mocks';
import { IndexPatternPersistedState, IndexPatternPrivateState } from './types';
import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks';
+import { Ast } from '@kbn/interpreter/common';
jest.mock('./loader');
jest.mock('../id_generator');
@@ -262,20 +263,7 @@ describe('IndexPattern Data Source', () => {
Object {
"arguments": Object {
"aggConfigs": Array [
- Object {
- "chain": Array [
- Object {
- "arguments": Object {
- "aggConfigs": Array [
- "[{\\"id\\":\\"col1\\",\\"enabled\\":true,\\"type\\":\\"count\\",\\"schema\\":\\"metric\\",\\"params\\":{}},{\\"id\\":\\"col2\\",\\"enabled\\":true,\\"type\\":\\"date_histogram\\",\\"schema\\":\\"segment\\",\\"params\\":{\\"field\\":\\"timestamp\\",\\"useNormalizedEsInterval\\":true,\\"interval\\":\\"1d\\",\\"drop_partials\\":false,\\"min_doc_count\\":0,\\"extended_bounds\\":{}}}]",
- ],
- },
- "function": "lens_auto_date",
- "type": "function",
- },
- ],
- "type": "expression",
- },
+ "[{\\"id\\":\\"col1\\",\\"enabled\\":true,\\"type\\":\\"count\\",\\"schema\\":\\"metric\\",\\"params\\":{}},{\\"id\\":\\"col2\\",\\"enabled\\":true,\\"type\\":\\"date_histogram\\",\\"schema\\":\\"segment\\",\\"params\\":{\\"field\\":\\"timestamp\\",\\"useNormalizedEsInterval\\":true,\\"interval\\":\\"1d\\",\\"drop_partials\\":false,\\"min_doc_count\\":0,\\"extended_bounds\\":{}}}]",
],
"includeFormatHints": Array [
true,
@@ -289,6 +277,9 @@ describe('IndexPattern Data Source', () => {
"partialRows": Array [
false,
],
+ "timeFields": Array [
+ "timestamp",
+ ],
},
"function": "esaggs",
"type": "function",
@@ -307,6 +298,89 @@ describe('IndexPattern Data Source', () => {
}
`);
});
+
+ it('should put all time fields used in date_histograms to the esaggs timeFields parameter', async () => {
+ const queryPersistedState: IndexPatternPersistedState = {
+ currentIndexPatternId: '1',
+ layers: {
+ first: {
+ indexPatternId: '1',
+ columnOrder: ['col1', 'col2', 'col3'],
+ columns: {
+ col1: {
+ label: 'Count of records',
+ dataType: 'number',
+ isBucketed: false,
+ sourceField: 'Records',
+ operationType: 'count',
+ },
+ col2: {
+ label: 'Date',
+ dataType: 'date',
+ isBucketed: true,
+ operationType: 'date_histogram',
+ sourceField: 'timestamp',
+ params: {
+ interval: 'auto',
+ },
+ },
+ col3: {
+ label: 'Date 2',
+ dataType: 'date',
+ isBucketed: true,
+ operationType: 'date_histogram',
+ sourceField: 'another_datefield',
+ params: {
+ interval: 'auto',
+ },
+ },
+ },
+ },
+ },
+ };
+
+ const state = stateFromPersistedState(queryPersistedState);
+
+ const ast = indexPatternDatasource.toExpression(state, 'first') as Ast;
+ expect(ast.chain[0].arguments.timeFields).toEqual(['timestamp', 'another_datefield']);
+ });
+
+ it('should not put date fields used outside date_histograms to the esaggs timeFields parameter', async () => {
+ const queryPersistedState: IndexPatternPersistedState = {
+ currentIndexPatternId: '1',
+ layers: {
+ first: {
+ indexPatternId: '1',
+ columnOrder: ['col1', 'col2'],
+ columns: {
+ col1: {
+ label: 'Count of records',
+ dataType: 'date',
+ isBucketed: false,
+ sourceField: 'timefield',
+ operationType: 'cardinality',
+ },
+ col2: {
+ label: 'Date',
+ dataType: 'date',
+ isBucketed: true,
+ operationType: 'date_histogram',
+ sourceField: 'timestamp',
+ params: {
+ interval: 'auto',
+ },
+ },
+ },
+ },
+ },
+ };
+
+ const state = stateFromPersistedState(queryPersistedState);
+
+ const ast = indexPatternDatasource.toExpression(state, 'first') as Ast;
+ expect(ast.chain[0].arguments.timeFields).toEqual(['timestamp']);
+ expect(ast.chain[0].arguments.timeFields).not.toContain('timefield');
+ });
});
describe('#insertLayer', () => {
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx
index 2008b326a539c..f26fd39a60c0e 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx
@@ -184,6 +184,7 @@ describe('IndexPattern Data Source suggestions', () => {
id2: expect.objectContaining({
operationType: 'terms',
sourceField: 'source',
+ params: expect.objectContaining({ size: 5 }),
}),
id3: expect.objectContaining({
operationType: 'count',
@@ -388,6 +389,7 @@ describe('IndexPattern Data Source suggestions', () => {
id1: expect.objectContaining({
operationType: 'terms',
sourceField: 'source',
+ params: expect.objectContaining({ size: 5 }),
}),
id2: expect.objectContaining({
operationType: 'count',
@@ -779,7 +781,7 @@ describe('IndexPattern Data Source suggestions', () => {
expect(suggestions[0].table.columns[0].operation.isBucketed).toBeFalsy();
});
- it('appends a terms column on string field', () => {
+ it('appends a terms column with default size on string field', () => {
const initialState = stateWithNonEmptyTables();
const suggestions = getDatasourceSuggestionsForField(initialState, '1', {
name: 'dest',
@@ -800,6 +802,7 @@ describe('IndexPattern Data Source suggestions', () => {
id1: expect.objectContaining({
operationType: 'terms',
sourceField: 'dest',
+ params: expect.objectContaining({ size: 3 }),
}),
},
}),
@@ -1549,6 +1552,62 @@ describe('IndexPattern Data Source suggestions', () => {
expect(suggestions[0].table.columns.length).toBe(1);
expect(suggestions[0].table.columns[0].operation.label).toBe('Sum of field1');
});
+
+ it('contains a reordering suggestion when there are exactly 2 buckets', () => {
+ const initialState = testInitialState();
+ const state: IndexPatternPrivateState = {
+ indexPatternRefs: [],
+ existingFields: {},
+ currentIndexPatternId: '1',
+ indexPatterns: expectedIndexPatterns,
+ showEmptyFields: true,
+ layers: {
+ first: {
+ ...initialState.layers.first,
+ columns: {
+ id1: {
+ label: 'Date histogram',
+ dataType: 'date',
+ isBucketed: true,
+
+ operationType: 'date_histogram',
+ sourceField: 'field2',
+ params: {
+ interval: 'd',
+ },
+ },
+ id2: {
+ label: 'Top 5',
+ dataType: 'string',
+ isBucketed: true,
+
+ operationType: 'terms',
+ sourceField: 'field1',
+ params: { size: 5, orderBy: { type: 'alphabetical' }, orderDirection: 'asc' },
+ },
+ id3: {
+ label: 'Average of field1',
+ dataType: 'number',
+ isBucketed: false,
+
+ operationType: 'avg',
+ sourceField: 'field1',
+ },
+ },
+ columnOrder: ['id1', 'id2', 'id3'],
+ },
+ },
+ };
+
+ const suggestions = getDatasourceSuggestionsFromCurrentState(state);
+ expect(suggestions).toContainEqual(
+ expect.objectContaining({
+ table: expect.objectContaining({
+ changeType: 'reorder',
+ }),
+ })
+ );
+ });
});
});
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts
index 2b3e976a77ea7..487c1bf759fc2 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts
@@ -17,6 +17,7 @@ import {
OperationType,
} from './operations';
import { operationDefinitions } from './operations/definitions';
+import { TermsIndexPatternColumn } from './operations/definitions/terms';
import { hasField } from './utils';
import {
IndexPattern,
@@ -232,6 +233,10 @@ function addFieldAsBucketOperation(
[newColumnId]: newColumn,
};
+ if (buckets.length === 0 && operation === 'terms') {
+ (newColumn as TermsIndexPatternColumn).params.size = 5;
+ }
+
const oldDateHistogramIndex = layer.columnOrder.findIndex(
columnId => layer.columns[columnId].operationType === 'date_histogram'
);
@@ -327,6 +332,9 @@ function createNewLayerWithBucketAggregation(
field,
suggestedPriority: undefined,
});
+ if (operation === 'terms') {
+ (column as TermsIndexPatternColumn).params.size = 5;
+ }
return {
indexPatternId: indexPattern.id,
@@ -478,7 +486,7 @@ function createChangedNestingSuggestion(state: IndexPatternPrivateState, layerId
layerId,
updatedLayer,
label: getNestedTitle([layer.columns[secondBucket], layer.columns[firstBucket]]),
- changeType: 'extended',
+ changeType: 'reorder',
});
}
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts
index 3ab51b5fa3f2b..1308fa3b7ca60 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts
@@ -10,6 +10,7 @@ import { IndexPatternColumn } from './indexpattern';
import { operationDefinitionMap } from './operations';
import { IndexPattern, IndexPatternPrivateState } from './types';
import { OriginalColumn } from './rename_columns';
+import { dateHistogramOperation } from './operations/definitions';
function getExpressionForLayer(
indexPattern: IndexPattern,
@@ -68,6 +69,12 @@ function getExpressionForLayer(
return base;
});
+ const allDateHistogramFields = Object.values(columns)
+ .map(column =>
+ column.operationType === dateHistogramOperation.type ? column.sourceField : null
+ )
+ .filter((field): field is string => Boolean(field));
+
return {
type: 'expression',
chain: [
@@ -79,20 +86,8 @@ function getExpressionForLayer(
metricsAtAllLevels: [false],
partialRows: [false],
includeFormatHints: [true],
- aggConfigs: [
- {
- type: 'expression',
- chain: [
- {
- type: 'function',
- function: 'lens_auto_date',
- arguments: {
- aggConfigs: [JSON.stringify(aggs)],
- },
- },
- ],
- },
- ],
+ timeFields: allDateHistogramFields,
+ aggConfigs: [JSON.stringify(aggs)],
},
},
{
diff --git a/x-pack/plugins/lens/public/metric_visualization/metric_visualization.tsx b/x-pack/plugins/lens/public/metric_visualization/metric_visualization.tsx
index 73b8019a31eaa..04a1c3865f22d 100644
--- a/x-pack/plugins/lens/public/metric_visualization/metric_visualization.tsx
+++ b/x-pack/plugins/lens/public/metric_visualization/metric_visualization.tsx
@@ -53,6 +53,10 @@ export const metricVisualization: Visualization = {
},
],
+ getVisualizationTypeId() {
+ return 'lnsMetric';
+ },
+
clearLayer(state) {
return {
...state,
diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts
index 181f192520d0d..04efc642793b0 100644
--- a/x-pack/plugins/lens/public/types.ts
+++ b/x-pack/plugins/lens/public/types.ts
@@ -103,9 +103,16 @@ export interface TableSuggestion {
* * `unchanged` means the table is the same in the currently active configuration
* * `reduced` means the table is a reduced version of the currently active table (some columns dropped, but not all of them)
* * `extended` means the table is an extended version of the currently active table (added one or multiple additional columns)
+ * * `reorder` means the table columns have changed order, which change the data as well
* * `layers` means the change is a change to the layer structure, not to the table
*/
-export type TableChangeType = 'initial' | 'unchanged' | 'reduced' | 'extended' | 'layers';
+export type TableChangeType =
+ | 'initial'
+ | 'unchanged'
+ | 'reduced'
+ | 'extended'
+ | 'reorder'
+ | 'layers';
export interface DatasourceSuggestion {
state: T;
@@ -312,6 +319,10 @@ export interface SuggestionRequest {
* The visualization needs to know which table is being suggested
*/
keptLayerIds: string[];
+ /**
+ * Different suggestions can be generated for each subtype of the visualization
+ */
+ subVisualizationId?: string;
}
/**
@@ -388,6 +399,11 @@ export interface Visualization {
* but can register multiple subtypes
*/
visualizationTypes: VisualizationType[];
+ /**
+ * Return the ID of the current visualization. Used to highlight
+ * the active subtype of the visualization.
+ */
+ getVisualizationTypeId: (state: T) => string;
/**
* If the visualization has subtypes, update the subtype in state.
*/
diff --git a/x-pack/plugins/lens/public/xy_visualization/index.ts b/x-pack/plugins/lens/public/xy_visualization/index.ts
index 5dfae097be834..3a280fb045b06 100644
--- a/x-pack/plugins/lens/public/xy_visualization/index.ts
+++ b/x-pack/plugins/lens/public/xy_visualization/index.ts
@@ -53,6 +53,7 @@ export class XyVisualization {
? EUI_CHARTS_THEME_DARK.theme
: EUI_CHARTS_THEME_LIGHT.theme,
timeZone: getTimeZone(core.uiSettings),
+ histogramBarTarget: core.uiSettings.get('histogram:barTarget'),
})
);
diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_expression.test.tsx b/x-pack/plugins/lens/public/xy_visualization/xy_expression.test.tsx
index e75e5fe763d6a..8db00aba0e36d 100644
--- a/x-pack/plugins/lens/public/xy_visualization/xy_expression.test.tsx
+++ b/x-pack/plugins/lens/public/xy_visualization/xy_expression.test.tsx
@@ -40,7 +40,7 @@ const createSampleDatatableWithRows = (rows: KibanaDatatableRow[]): KibanaDatata
id: 'c',
name: 'c',
formatHint: { id: 'string' },
- meta: { type: 'date-histogram', aggConfigParams: { interval: '10s' } },
+ meta: { type: 'date-histogram', aggConfigParams: { interval: 'auto' } },
},
{ id: 'd', name: 'ColD', formatHint: { id: 'string' } },
],
@@ -156,6 +156,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -203,6 +204,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -237,15 +239,17 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
+ // real auto interval is 30mins = 1800000
expect(component.find(Settings).prop('xDomain')).toMatchInlineSnapshot(`
Object {
"max": 1546491600000,
"min": 1546405200000,
- "minInterval": 10000,
+ "minInterval": 1728000,
}
`);
});
@@ -271,6 +275,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -279,7 +284,7 @@ describe('xy_expression', () => {
Object {
"max": 1546491600000,
"min": 1546405200000,
- "minInterval": 10000,
+ "minInterval": 1728000,
}
`);
});
@@ -307,6 +312,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -350,6 +356,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -383,6 +390,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -398,6 +406,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -414,6 +423,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -430,6 +440,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -472,6 +483,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -510,6 +522,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -527,6 +540,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -547,6 +561,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -565,6 +580,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="CEST"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -582,6 +598,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -606,6 +623,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -624,6 +642,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -684,6 +703,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -878,6 +898,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -894,6 +915,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -910,6 +932,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -927,6 +950,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
@@ -943,6 +967,7 @@ describe('xy_expression', () => {
args={{ ...args, layers: [{ ...args.layers[0], accessors: ['a'] }] }}
formatFactory={getFormatSpy}
chartTheme={{}}
+ histogramBarTarget={50}
timeZone="UTC"
executeTriggerActions={executeTriggerActions}
/>
@@ -963,6 +988,7 @@ describe('xy_expression', () => {
formatFactory={getFormatSpy}
timeZone="UTC"
chartTheme={{}}
+ histogramBarTarget={50}
executeTriggerActions={executeTriggerActions}
/>
);
diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_expression.tsx b/x-pack/plugins/lens/public/xy_visualization/xy_expression.tsx
index d6b6de479acfb..85cf5753befd7 100644
--- a/x-pack/plugins/lens/public/xy_visualization/xy_expression.tsx
+++ b/x-pack/plugins/lens/public/xy_visualization/xy_expression.tsx
@@ -6,6 +6,7 @@
import React, { useState, useEffect } from 'react';
import ReactDOM from 'react-dom';
+import moment from 'moment';
import {
Chart,
Settings,
@@ -35,8 +36,8 @@ import { XYArgs, SeriesType, visualizationTypes } from './types';
import { VisualizationContainer } from '../visualization_container';
import { isHorizontalChart } from './state_helpers';
import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public';
-import { parseInterval } from '../../../../../src/plugins/data/common';
import { getExecuteTriggerActions } from './services';
+import { parseInterval } from '../../../../../src/plugins/data/common';
type InferPropType = T extends React.FunctionComponent ? P : T;
type SeriesSpec = InferPropType &
@@ -58,6 +59,7 @@ type XYChartRenderProps = XYChartProps & {
chartTheme: PartialTheme;
formatFactory: FormatFactory;
timeZone: string;
+ histogramBarTarget: number;
executeTriggerActions: UiActionsStart['executeTriggerActions'];
};
@@ -110,6 +112,7 @@ export const xyChart: ExpressionFunctionDefinition<
export const getXyChartRenderer = (dependencies: {
formatFactory: Promise;
chartTheme: PartialTheme;
+ histogramBarTarget: number;
timeZone: string;
}): ExpressionRenderDefinition => ({
name: 'lens_xy_chart_renderer',
@@ -130,6 +133,7 @@ export const getXyChartRenderer = (dependencies: {
formatFactory={formatFactory}
chartTheme={dependencies.chartTheme}
timeZone={dependencies.timeZone}
+ histogramBarTarget={dependencies.histogramBarTarget}
executeTriggerActions={executeTriggerActions}
/>
,
@@ -169,6 +173,7 @@ export function XYChart({
formatFactory,
timeZone,
chartTheme,
+ histogramBarTarget,
executeTriggerActions,
}: XYChartRenderProps) {
const { legend, layers } = args;
@@ -212,18 +217,26 @@ export function XYChart({
const xTitle = (xAxisColumn && xAxisColumn.name) || args.xTitle;
- // add minInterval only for single row value as it cannot be determined from dataset
+ function calculateMinInterval() {
+ // add minInterval only for single row value as it cannot be determined from dataset
+ if (data.dateRange && layers.every(layer => data.tables[layer.layerId].rows.length <= 1)) {
+ if (xAxisColumn?.meta?.aggConfigParams?.interval !== 'auto')
+ return parseInterval(xAxisColumn?.meta?.aggConfigParams?.interval)?.asMilliseconds();
- const minInterval = layers.every(layer => data.tables[layer.layerId].rows.length <= 1)
- ? parseInterval(xAxisColumn?.meta?.aggConfigParams?.interval)?.asMilliseconds()
- : undefined;
+ const { fromDate, toDate } = data.dateRange;
+ const duration = moment(toDate).diff(moment(fromDate));
+ const targetMs = duration / histogramBarTarget;
+ return isNaN(targetMs) ? 0 : Math.max(Math.floor(targetMs), 1);
+ }
+ return undefined;
+ }
const xDomain =
data.dateRange && layers.every(l => l.xScaleType === 'time')
? {
min: data.dateRange.fromDate.getTime(),
max: data.dateRange.toDate.getTime(),
- minInterval,
+ minInterval: calculateMinInterval(),
}
: undefined;
return (
diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.test.ts b/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.test.ts
index ddbd9d11b5fad..73ff88e97f479 100644
--- a/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.test.ts
+++ b/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.test.ts
@@ -186,7 +186,7 @@ describe('xy_suggestions', () => {
isMultiRow: true,
columns: [numCol('price'), numCol('quantity'), dateCol('date'), strCol('product')],
layerId: 'first',
- changeType: 'unchanged',
+ changeType: 'extended',
label: 'Datasource title',
},
keptLayerIds: [],
@@ -196,6 +196,34 @@ describe('xy_suggestions', () => {
expect(suggestion.title).toEqual('Datasource title');
});
+ test('suggests only stacked bar chart when xy chart is inactive', () => {
+ const [suggestion, ...rest] = getSuggestions({
+ table: {
+ isMultiRow: true,
+ columns: [dateCol('date'), numCol('price')],
+ layerId: 'first',
+ changeType: 'unchanged',
+ label: 'Datasource title',
+ },
+ keptLayerIds: [],
+ });
+
+ expect(rest).toHaveLength(0);
+ expect(suggestion.title).toEqual('Bar chart');
+ expect(suggestion.state).toEqual(
+ expect.objectContaining({
+ layers: [
+ expect.objectContaining({
+ seriesType: 'bar_stacked',
+ xAccessor: 'date',
+ accessors: ['price'],
+ splitAccessor: undefined,
+ }),
+ ],
+ })
+ );
+ });
+
test('hides reduced suggestions if there is a current state', () => {
const [suggestion, ...rest] = getSuggestions({
table: {
@@ -224,7 +252,7 @@ describe('xy_suggestions', () => {
expect(suggestion.hide).toBeTruthy();
});
- test('does not hide reduced suggestions if xy visualization is not active', () => {
+ test('hides reduced suggestions if xy visualization is not active', () => {
const [suggestion, ...rest] = getSuggestions({
table: {
isMultiRow: true,
@@ -236,7 +264,7 @@ describe('xy_suggestions', () => {
});
expect(rest).toHaveLength(0);
- expect(suggestion.hide).toBeFalsy();
+ expect(suggestion.hide).toBeTruthy();
});
test('only makes a seriesType suggestion for unchanged table without split', () => {
@@ -419,6 +447,44 @@ describe('xy_suggestions', () => {
});
});
+ test('changes column mappings when suggestion is reorder', () => {
+ const currentState: XYState = {
+ legend: { isVisible: true, position: 'bottom' },
+ preferredSeriesType: 'bar',
+ layers: [
+ {
+ accessors: ['price'],
+ layerId: 'first',
+ seriesType: 'bar',
+ splitAccessor: 'category',
+ xAccessor: 'product',
+ },
+ ],
+ };
+ const [suggestion, ...rest] = getSuggestions({
+ table: {
+ isMultiRow: true,
+ columns: [strCol('category'), strCol('product'), numCol('price')],
+ layerId: 'first',
+ changeType: 'reorder',
+ },
+ state: currentState,
+ keptLayerIds: [],
+ });
+
+ expect(rest).toHaveLength(0);
+ expect(suggestion.state).toEqual({
+ ...currentState,
+ layers: [
+ {
+ ...currentState.layers[0],
+ xAccessor: 'category',
+ splitAccessor: 'product',
+ },
+ ],
+ });
+ });
+
test('overwrites column to dimension mappings if a date dimension is added', () => {
(generateId as jest.Mock).mockReturnValueOnce('dummyCol');
const currentState: XYState = {
diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts b/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts
index 5e9311bb1e928..abd7640344064 100644
--- a/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts
+++ b/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts
@@ -99,11 +99,14 @@ function getBucketMappings(table: TableSuggestion, currentState?: State) {
// reverse the buckets before prioritization to always use the most inner
// bucket of the highest-prioritized group as x value (don't use nested
// buckets as split series)
- const prioritizedBuckets = prioritizeColumns(buckets.reverse());
+ const prioritizedBuckets = prioritizeColumns([...buckets].reverse());
if (!currentLayer || table.changeType === 'initial') {
return prioritizedBuckets;
}
+ if (table.changeType === 'reorder') {
+ return buckets;
+ }
// if existing table is just modified, try to map buckets to the current dimensions
const currentXColumnIndex = prioritizedBuckets.findIndex(
@@ -175,12 +178,24 @@ function getSuggestionsForLayer({
keptLayerIds,
};
- const isSameState = currentState && changeType === 'unchanged';
+ // handles the simplest cases, acting as a chart switcher
+ if (!currentState && changeType === 'unchanged') {
+ return [
+ {
+ ...buildSuggestion(options),
+ title: i18n.translate('xpack.lens.xySuggestions.barChartTitle', {
+ defaultMessage: 'Bar chart',
+ }),
+ },
+ ];
+ }
+ const isSameState = currentState && changeType === 'unchanged';
if (!isSameState) {
return buildSuggestion(options);
}
+ // Suggestions are either changing the data, or changing the way the data is used
const sameStateSuggestions: Array> = [];
// if current state is using the same data, suggest same chart with different presentational configuration
@@ -374,8 +389,11 @@ function buildSuggestion({
return {
title,
score: getScore(yValues, splitBy, changeType),
- // don't advertise chart of same type but with less data
- hide: currentState && changeType === 'reduced',
+ hide:
+ // Only advertise very clear changes when XY chart is not active
+ (!currentState && changeType !== 'unchanged' && changeType !== 'extended') ||
+ // Don't advertise removing dimensions
+ (currentState && changeType === 'reduced'),
state,
previewIcon: getIconForSeries(seriesType),
};
diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_visualization.test.ts b/x-pack/plugins/lens/public/xy_visualization/xy_visualization.test.ts
index beccf0dc46eb4..d176905c65120 100644
--- a/x-pack/plugins/lens/public/xy_visualization/xy_visualization.test.ts
+++ b/x-pack/plugins/lens/public/xy_visualization/xy_visualization.test.ts
@@ -27,7 +27,7 @@ function exampleState(): State {
}
describe('xy_visualization', () => {
- describe('getDescription', () => {
+ describe('#getDescription', () => {
function mixedState(...types: SeriesType[]) {
const state = exampleState();
return {
@@ -81,6 +81,45 @@ describe('xy_visualization', () => {
});
});
+ describe('#getVisualizationTypeId', () => {
+ function mixedState(...types: SeriesType[]) {
+ const state = exampleState();
+ return {
+ ...state,
+ layers: types.map((t, i) => ({
+ ...state.layers[0],
+ layerId: `layer_${i}`,
+ seriesType: t,
+ })),
+ };
+ }
+
+ it('should show mixed when each layer is different', () => {
+ expect(xyVisualization.getVisualizationTypeId(mixedState('bar', 'line'))).toEqual('mixed');
+ });
+
+ it('should show the preferredSeriesType if there are no layers', () => {
+ expect(xyVisualization.getVisualizationTypeId(mixedState())).toEqual('bar');
+ });
+
+ it('should combine multiple layers into one type', () => {
+ expect(
+ xyVisualization.getVisualizationTypeId(mixedState('bar_horizontal', 'bar_horizontal'))
+ ).toEqual('bar_horizontal');
+ });
+
+ it('should return the subtype for single layers', () => {
+ expect(xyVisualization.getVisualizationTypeId(mixedState('area'))).toEqual('area');
+ expect(xyVisualization.getVisualizationTypeId(mixedState('line'))).toEqual('line');
+ expect(xyVisualization.getVisualizationTypeId(mixedState('area_stacked'))).toEqual(
+ 'area_stacked'
+ );
+ expect(xyVisualization.getVisualizationTypeId(mixedState('bar_horizontal_stacked'))).toEqual(
+ 'bar_horizontal_stacked'
+ );
+ });
+ });
+
describe('#initialize', () => {
it('loads default state', () => {
const mockFrame = createMockFramePublicAPI();
diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_visualization.tsx b/x-pack/plugins/lens/public/xy_visualization/xy_visualization.tsx
index c72fa0fec24d7..e91edf9cc0183 100644
--- a/x-pack/plugins/lens/public/xy_visualization/xy_visualization.tsx
+++ b/x-pack/plugins/lens/public/xy_visualization/xy_visualization.tsx
@@ -12,7 +12,7 @@ import { I18nProvider } from '@kbn/i18n/react';
import { i18n } from '@kbn/i18n';
import { getSuggestions } from './xy_suggestions';
import { LayerContextMenu } from './xy_config_panel';
-import { Visualization, OperationMetadata } from '../types';
+import { Visualization, OperationMetadata, VisualizationType } from '../types';
import { State, PersistableState, SeriesType, visualizationTypes, LayerConfig } from './types';
import { toExpression, toPreviewExpression } from './to_expression';
import chartBarStackedSVG from '../assets/chart_bar_stacked.svg';
@@ -24,6 +24,18 @@ const defaultSeriesType = 'bar_stacked';
const isNumericMetric = (op: OperationMetadata) => !op.isBucketed && op.dataType === 'number';
const isBucketed = (op: OperationMetadata) => op.isBucketed;
+function getVisualizationType(state: State): VisualizationType | 'mixed' {
+ if (!state.layers.length) {
+ return (
+ visualizationTypes.find(t => t.id === state.preferredSeriesType) ?? visualizationTypes[0]
+ );
+ }
+ const visualizationType = visualizationTypes.find(t => t.id === state.layers[0].seriesType);
+ const seriesTypes = _.unique(state.layers.map(l => l.seriesType));
+
+ return visualizationType && seriesTypes.length === 1 ? visualizationType : 'mixed';
+}
+
function getDescription(state?: State) {
if (!state) {
return {
@@ -34,32 +46,31 @@ function getDescription(state?: State) {
};
}
+ const visualizationType = getVisualizationType(state);
+
if (!state.layers.length) {
- const visualizationType = visualizationTypes.find(v => v.id === state.preferredSeriesType)!;
+ const preferredType = visualizationType as VisualizationType;
return {
- icon: visualizationType.largeIcon || visualizationType.icon,
- label: visualizationType.label,
+ icon: preferredType.largeIcon || preferredType.icon,
+ label: preferredType.label,
};
}
- const visualizationType = visualizationTypes.find(t => t.id === state.layers[0].seriesType)!;
- const seriesTypes = _.unique(state.layers.map(l => l.seriesType));
-
return {
icon:
- seriesTypes.length === 1
- ? visualizationType.largeIcon || visualizationType.icon
- : chartMixedSVG,
+ visualizationType === 'mixed'
+ ? chartMixedSVG
+ : visualizationType.largeIcon || visualizationType.icon,
label:
- seriesTypes.length === 1
- ? visualizationType.label
- : isHorizontalChart(state.layers)
- ? i18n.translate('xpack.lens.xyVisualization.mixedBarHorizontalLabel', {
- defaultMessage: 'Mixed horizontal bar',
- })
- : i18n.translate('xpack.lens.xyVisualization.mixedLabel', {
- defaultMessage: 'Mixed XY',
- }),
+ visualizationType === 'mixed'
+ ? isHorizontalChart(state.layers)
+ ? i18n.translate('xpack.lens.xyVisualization.mixedBarHorizontalLabel', {
+ defaultMessage: 'Mixed horizontal bar',
+ })
+ : i18n.translate('xpack.lens.xyVisualization.mixedLabel', {
+ defaultMessage: 'Mixed XY',
+ })
+ : visualizationType.label,
};
}
@@ -67,6 +78,10 @@ export const xyVisualization: Visualization = {
id: 'lnsXY',
visualizationTypes,
+ getVisualizationTypeId(state) {
+ const type = getVisualizationType(state);
+ return type === 'mixed' ? type : type.id;
+ },
getLayerIds(state) {
return state.layers.map(l => l.layerId);
diff --git a/x-pack/plugins/lens/server/migrations.test.ts b/x-pack/plugins/lens/server/migrations.test.ts
index e80308cc9acdb..4cc330d40efd7 100644
--- a/x-pack/plugins/lens/server/migrations.test.ts
+++ b/x-pack/plugins/lens/server/migrations.test.ts
@@ -158,4 +158,124 @@ describe('Lens migrations', () => {
]);
});
});
+
+ describe('7.8.0 auto timestamp', () => {
+ const context = {} as SavedObjectMigrationContext;
+
+ const example = {
+ type: 'lens',
+ attributes: {
+ expression: `kibana
+ | kibana_context query="{\\"query\\":\\"\\",\\"language\\":\\"kuery\\"}" filters="[]"
+ | lens_merge_tables layerIds="bd09dc71-a7e2-42d0-83bd-85df8291f03c"
+ tables={esaggs
+ index="ff959d40-b880-11e8-a6d9-e546fe2bba5f"
+ metricsAtAllLevels=false
+ partialRows=false
+ includeFormatHints=true
+ aggConfigs={
+ lens_auto_date
+ aggConfigs="[{\\"id\\":\\"1d9cc16c-1460-41de-88f8-471932ecbc97\\",\\"enabled\\":true,\\"type\\":\\"date_histogram\\",\\"schema\\":\\"segment\\",\\"params\\":{\\"field\\":\\"products.created_on\\",\\"useNormalizedEsInterval\\":true,\\"interval\\":\\"auto\\",\\"drop_partials\\":false,\\"min_doc_count\\":0,\\"extended_bounds\\":{}}},{\\"id\\":\\"66115819-8481-4917-a6dc-8ffb10dd02df\\",\\"enabled\\":true,\\"type\\":\\"count\\",\\"schema\\":\\"metric\\",\\"params\\":{}}]"
+ }
+ | lens_rename_columns idMap="{\\"col-0-1d9cc16c-1460-41de-88f8-471932ecbc97\\":{\\"label\\":\\"products.created_on\\",\\"dataType\\":\\"date\\",\\"operationType\\":\\"date_histogram\\",\\"sourceField\\":\\"products.created_on\\",\\"isBucketed\\":true,\\"scale\\":\\"interval\\",\\"params\\":{\\"interval\\":\\"auto\\"},\\"id\\":\\"1d9cc16c-1460-41de-88f8-471932ecbc97\\"},\\"col-1-66115819-8481-4917-a6dc-8ffb10dd02df\\":{\\"label\\":\\"Count of records\\",\\"dataType\\":\\"number\\",\\"operationType\\":\\"count\\",\\"suggestedPriority\\":0,\\"isBucketed\\":false,\\"scale\\":\\"ratio\\",\\"sourceField\\":\\"Records\\",\\"id\\":\\"66115819-8481-4917-a6dc-8ffb10dd02df\\"}}"
+ }
+ | lens_xy_chart
+ xTitle="products.created_on"
+ yTitle="Count of records"
+ legend={lens_xy_legendConfig isVisible=true position="right"}
+ layers={lens_xy_layer
+ layerId="bd09dc71-a7e2-42d0-83bd-85df8291f03c"
+ hide=false
+ xAccessor="1d9cc16c-1460-41de-88f8-471932ecbc97"
+ yScaleType="linear"
+ xScaleType="time"
+ isHistogram=true
+ seriesType="bar_stacked"
+ accessors="66115819-8481-4917-a6dc-8ffb10dd02df"
+ columnToLabel="{\\"66115819-8481-4917-a6dc-8ffb10dd02df\\":\\"Count of records\\"}"
+ }
+ `,
+ state: {
+ datasourceStates: {
+ indexpattern: {
+ currentIndexPatternId: 'ff959d40-b880-11e8-a6d9-e546fe2bba5f',
+ layers: {
+ 'bd09dc71-a7e2-42d0-83bd-85df8291f03c': {
+ indexPatternId: 'ff959d40-b880-11e8-a6d9-e546fe2bba5f',
+ columns: {
+ '1d9cc16c-1460-41de-88f8-471932ecbc97': {
+ label: 'products.created_on',
+ dataType: 'date',
+ operationType: 'date_histogram',
+ sourceField: 'products.created_on',
+ isBucketed: true,
+ scale: 'interval',
+ params: { interval: 'auto' },
+ },
+ '66115819-8481-4917-a6dc-8ffb10dd02df': {
+ label: 'Count of records',
+ dataType: 'number',
+ operationType: 'count',
+ suggestedPriority: 0,
+ isBucketed: false,
+ scale: 'ratio',
+ sourceField: 'Records',
+ },
+ },
+ columnOrder: [
+ '1d9cc16c-1460-41de-88f8-471932ecbc97',
+ '66115819-8481-4917-a6dc-8ffb10dd02df',
+ ],
+ },
+ },
+ },
+ },
+ datasourceMetaData: {
+ filterableIndexPatterns: [
+ { id: 'ff959d40-b880-11e8-a6d9-e546fe2bba5f', title: 'kibana_sample_data_ecommerce' },
+ ],
+ },
+ visualization: {
+ legend: { isVisible: true, position: 'right' },
+ preferredSeriesType: 'bar_stacked',
+ layers: [
+ {
+ layerId: 'bd09dc71-a7e2-42d0-83bd-85df8291f03c',
+ accessors: ['66115819-8481-4917-a6dc-8ffb10dd02df'],
+ position: 'top',
+ seriesType: 'bar_stacked',
+ showGridlines: false,
+ xAccessor: '1d9cc16c-1460-41de-88f8-471932ecbc97',
+ },
+ ],
+ },
+ query: { query: '', language: 'kuery' },
+ filters: [],
+ },
+ title: 'Bar chart',
+ visualizationType: 'lnsXY',
+ },
+ };
+
+ it('should remove the lens_auto_date expression', () => {
+ const result = migrations['7.8.0'](example, context);
+ expect(result.attributes.expression).toContain(`timeFields=\"products.created_on\"`);
+ });
+
+ it('should handle pre-migrated expression', () => {
+ const input = {
+ type: 'lens',
+ attributes: {
+ ...example.attributes,
+ expression: `kibana
+| kibana_context query="{\\"query\\":\\"\\",\\"language\\":\\"kuery\\"}" filters="[]"
+| lens_merge_tables layerIds="bd09dc71-a7e2-42d0-83bd-85df8291f03c"
+ tables={esaggs index="ff959d40-b880-11e8-a6d9-e546fe2bba5f" metricsAtAllLevels=false partialRows=false includeFormatHints=true aggConfigs="[{\\"id\\":\\"1d9cc16c-1460-41de-88f8-471932ecbc97\\",\\"enabled\\":true,\\"type\\":\\"date_histogram\\",\\"schema\\":\\"segment\\",\\"params\\":{\\"field\\":\\"products.created_on\\",\\"useNormalizedEsInterval\\":true,\\"interval\\":\\"auto\\",\\"drop_partials\\":false,\\"min_doc_count\\":0,\\"extended_bounds\\":{}}},{\\"id\\":\\"66115819-8481-4917-a6dc-8ffb10dd02df\\",\\"enabled\\":true,\\"type\\":\\"count\\",\\"schema\\":\\"metric\\",\\"params\\":{}}]" timeFields=\"products.created_on\"}
+| lens_xy_chart xTitle="products.created_on" yTitle="Count of records" legend={lens_xy_legendConfig isVisible=true position="right"} layers={}`,
+ },
+ };
+ const result = migrations['7.8.0'](input, context);
+ expect(result).toEqual(input);
+ });
+ });
});
diff --git a/x-pack/plugins/lens/server/migrations.ts b/x-pack/plugins/lens/server/migrations.ts
index 3d238723b7438..583fba1a4a999 100644
--- a/x-pack/plugins/lens/server/migrations.ts
+++ b/x-pack/plugins/lens/server/migrations.ts
@@ -4,7 +4,8 @@
* you may not use this file except in compliance with the Elastic License.
*/
-import { cloneDeep } from 'lodash';
+import { cloneDeep, flow } from 'lodash';
+import { fromExpression, toExpression, Ast, ExpressionFunctionAST } from '@kbn/interpreter/common';
import { SavedObjectMigrationFn } from 'src/core/server';
interface XYLayerPre77 {
@@ -14,7 +15,126 @@ interface XYLayerPre77 {
accessors: string[];
}
-export const migrations: Record = {
+/**
+ * Removes the `lens_auto_date` subexpression from a stored expression
+ * string. For example: aggConfigs={lens_auto_date aggConfigs="JSON string"}
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const removeLensAutoDate: SavedObjectMigrationFn = (doc, context) => {
+ const expression: string = doc.attributes?.expression;
+ try {
+ const ast = fromExpression(expression);
+ const newChain: ExpressionFunctionAST[] = ast.chain.map(topNode => {
+ if (topNode.function !== 'lens_merge_tables') {
+ return topNode;
+ }
+ return {
+ ...topNode,
+ arguments: {
+ ...topNode.arguments,
+ tables: (topNode.arguments.tables as Ast[]).map(middleNode => {
+ return {
+ type: 'expression',
+ chain: middleNode.chain.map(node => {
+ // Check for sub-expression in aggConfigs
+ if (
+ node.function === 'esaggs' &&
+ typeof node.arguments.aggConfigs[0] !== 'string'
+ ) {
+ return {
+ ...node,
+ arguments: {
+ ...node.arguments,
+ aggConfigs: (node.arguments.aggConfigs[0] as Ast).chain[0].arguments
+ .aggConfigs,
+ },
+ };
+ }
+ return node;
+ }),
+ };
+ }),
+ },
+ };
+ });
+
+ return {
+ ...doc,
+ attributes: {
+ ...doc.attributes,
+ expression: toExpression({ ...ast, chain: newChain }),
+ },
+ };
+ } catch (e) {
+ context.log.warning(e.message);
+ return { ...doc };
+ }
+};
+
+/**
+ * Adds missing timeField arguments to esaggs in the Lens expression
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const addTimeFieldToEsaggs: SavedObjectMigrationFn = (doc, context) => {
+ const expression: string = doc.attributes?.expression;
+
+ try {
+ const ast = fromExpression(expression);
+ const newChain: ExpressionFunctionAST[] = ast.chain.map(topNode => {
+ if (topNode.function !== 'lens_merge_tables') {
+ return topNode;
+ }
+ return {
+ ...topNode,
+ arguments: {
+ ...topNode.arguments,
+ tables: (topNode.arguments.tables as Ast[]).map(middleNode => {
+ return {
+ type: 'expression',
+ chain: middleNode.chain.map(node => {
+ // Skip if there are any timeField arguments already, because that indicates
+ // the fix is already applied
+ if (node.function !== 'esaggs' || node.arguments.timeFields) {
+ return node;
+ }
+ const timeFields: string[] = [];
+ JSON.parse(node.arguments.aggConfigs[0] as string).forEach(
+ (agg: { type: string; params: { field: string } }) => {
+ if (agg.type !== 'date_histogram') {
+ return;
+ }
+ timeFields.push(agg.params.field);
+ }
+ );
+ return {
+ ...node,
+ arguments: {
+ ...node.arguments,
+ timeFields,
+ },
+ };
+ }),
+ };
+ }),
+ },
+ };
+ });
+
+ return {
+ ...doc,
+ attributes: {
+ ...doc.attributes,
+ expression: toExpression({ ...ast, chain: newChain }),
+ },
+ };
+ } catch (e) {
+ context.log.warning(e.message);
+ return { ...doc };
+ }
+};
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const migrations: Record> = {
'7.7.0': doc => {
const newDoc = cloneDeep(doc);
if (newDoc.attributes?.visualizationType === 'lnsXY') {
@@ -34,4 +154,7 @@ export const migrations: Record = {
}
return newDoc;
},
+ // The order of these migrations matter, since the timefield migration relies on the aggConfigs
+ // sitting directly on the esaggs as an argument and not a nested function (which lens_auto_date was).
+ '7.8.0': flow(removeLensAutoDate, addTimeFieldToEsaggs),
};
diff --git a/x-pack/plugins/lists/common/constants.ts b/x-pack/plugins/lists/common/constants.ts
new file mode 100644
index 0000000000000..dbe31fed66413
--- /dev/null
+++ b/x-pack/plugins/lists/common/constants.ts
@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/**
+ * Lists routes
+ */
+export const LIST_URL = `/api/lists`;
+export const LIST_INDEX = `${LIST_URL}/index`;
+export const LIST_ITEM_URL = `${LIST_URL}/items`;
diff --git a/x-pack/legacy/plugins/actions/index.ts b/x-pack/plugins/lists/common/schemas/common/index.ts
similarity index 90%
rename from x-pack/legacy/plugins/actions/index.ts
rename to x-pack/plugins/lists/common/schemas/common/index.ts
index 276d1ea3accea..a05e97ded38ee 100644
--- a/x-pack/legacy/plugins/actions/index.ts
+++ b/x-pack/plugins/lists/common/schemas/common/index.ts
@@ -4,4 +4,4 @@
* you may not use this file except in compliance with the Elastic License.
*/
-export * from './server/';
+export * from './schemas';
diff --git a/x-pack/plugins/lists/common/schemas/common/schemas.ts b/x-pack/plugins/lists/common/schemas/common/schemas.ts
new file mode 100644
index 0000000000000..edc037ed7a0b1
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/common/schemas.ts
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { NonEmptyString } from '../types/non_empty_string';
+
+export const name = t.string;
+export type Name = t.TypeOf;
+export const nameOrUndefined = t.union([name, t.undefined]);
+export type NameOrUndefined = t.TypeOf;
+
+export const description = t.string;
+export type Description = t.TypeOf;
+export const descriptionOrUndefined = t.union([description, t.undefined]);
+export type DescriptionOrUndefined = t.TypeOf;
+
+export const list_id = NonEmptyString;
+export const list_idOrUndefined = t.union([list_id, t.undefined]);
+export type List_idOrUndefined = t.TypeOf;
+
+export const item = t.string;
+export const created_at = t.string; // TODO: Make this into an ISO Date string check
+export const updated_at = t.string; // TODO: Make this into an ISO Date string check
+export const updated_by = t.string;
+export const created_by = t.string;
+export const file = t.object;
+
+export const id = NonEmptyString;
+export type Id = t.TypeOf;
+export const idOrUndefined = t.union([id, t.undefined]);
+export type IdOrUndefined = t.TypeOf;
+
+export const ip = t.string;
+export const ipOrUndefined = t.union([ip, t.undefined]);
+
+export const keyword = t.string;
+export const keywordOrUndefined = t.union([keyword, t.undefined]);
+
+export const value = t.string;
+export const valueOrUndefined = t.union([value, t.undefined]);
+
+export const tie_breaker_id = t.string; // TODO: Use UUID for this instead of a string for validation
+export const _index = t.string;
+
+export const type = t.keyof({ ip: null, keyword: null }); // TODO: Add the other data types here
+
+export const typeOrUndefined = t.union([type, t.undefined]);
+export type Type = t.TypeOf;
+
+export const meta = t.object;
+export type Meta = t.TypeOf;
+export const metaOrUndefined = t.union([meta, t.undefined]);
+export type MetaOrUndefined = t.TypeOf;
+
+export const esDataTypeUnion = t.union([t.type({ ip }), t.type({ keyword })]);
+export type EsDataTypeUnion = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/create_es_bulk_type.ts b/x-pack/plugins/lists/common/schemas/elastic_query/create_es_bulk_type.ts
new file mode 100644
index 0000000000000..4a825382c06e4
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/elastic_query/create_es_bulk_type.ts
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+import { _index } from '../common/schemas';
+
+export const createEsBulkTypeSchema = t.exact(
+ t.type({
+ create: t.exact(t.type({ _index })),
+ })
+);
+
+export type CreateEsBulkTypeSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/index.ts b/x-pack/plugins/lists/common/schemas/elastic_query/index.ts
new file mode 100644
index 0000000000000..d70dd09849fa6
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/elastic_query/index.ts
@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+export * from './update_es_list_schema';
+export * from './index_es_list_schema';
+export * from './update_es_list_item_schema';
+export * from './index_es_list_item_schema';
+export * from './create_es_bulk_type';
diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.ts
new file mode 100644
index 0000000000000..596498b64b771
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.ts
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+ created_at,
+ created_by,
+ esDataTypeUnion,
+ list_id,
+ metaOrUndefined,
+ tie_breaker_id,
+ updated_at,
+ updated_by,
+} from '../common/schemas';
+
+export const indexEsListItemSchema = t.intersection([
+ t.exact(
+ t.type({
+ created_at,
+ created_by,
+ list_id,
+ meta: metaOrUndefined,
+ tie_breaker_id,
+ updated_at,
+ updated_by,
+ })
+ ),
+ esDataTypeUnion,
+]);
+
+export type IndexEsListItemSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts
new file mode 100644
index 0000000000000..e0924392628a9
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+ created_at,
+ created_by,
+ description,
+ metaOrUndefined,
+ name,
+ tie_breaker_id,
+ type,
+ updated_at,
+ updated_by,
+} from '../common/schemas';
+
+export const indexEsListSchema = t.exact(
+ t.type({
+ created_at,
+ created_by,
+ description,
+ meta: metaOrUndefined,
+ name,
+ tie_breaker_id,
+ type,
+ updated_at,
+ updated_by,
+ })
+);
+
+export type IndexEsListSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_item_schema.ts
new file mode 100644
index 0000000000000..e4cf46bc39429
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_item_schema.ts
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { esDataTypeUnion, metaOrUndefined, updated_at, updated_by } from '../common/schemas';
+
+export const updateEsListItemSchema = t.intersection([
+ t.exact(
+ t.type({
+ meta: metaOrUndefined,
+ updated_at,
+ updated_by,
+ })
+ ),
+ esDataTypeUnion,
+]);
+
+export type UpdateEsListItemSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_schema.ts
new file mode 100644
index 0000000000000..8f23f3744e563
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_schema.ts
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+ descriptionOrUndefined,
+ metaOrUndefined,
+ nameOrUndefined,
+ updated_at,
+ updated_by,
+} from '../common/schemas';
+
+export const updateEsListSchema = t.exact(
+ t.type({
+ description: descriptionOrUndefined,
+ meta: metaOrUndefined,
+ name: nameOrUndefined,
+ updated_at,
+ updated_by,
+ })
+);
+
+export type UpdateEsListSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/index.ts b/x-pack/plugins/lists/common/schemas/elastic_response/index.ts
new file mode 100644
index 0000000000000..6fbc6ef293064
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/elastic_response/index.ts
@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './search_es_list_item_schema';
+export * from './search_es_list_schema';
diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.ts
new file mode 100644
index 0000000000000..902d3e6a9896e
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.ts
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+ created_at,
+ created_by,
+ ipOrUndefined,
+ keywordOrUndefined,
+ list_id,
+ metaOrUndefined,
+ tie_breaker_id,
+ updated_at,
+ updated_by,
+} from '../common/schemas';
+
+export const searchEsListItemSchema = t.exact(
+ t.type({
+ created_at,
+ created_by,
+ ip: ipOrUndefined,
+ keyword: keywordOrUndefined,
+ list_id,
+ meta: metaOrUndefined,
+ tie_breaker_id,
+ updated_at,
+ updated_by,
+ })
+);
+
+export type SearchEsListItemSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts
new file mode 100644
index 0000000000000..00a7c6f321d38
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+ created_at,
+ created_by,
+ description,
+ metaOrUndefined,
+ name,
+ tie_breaker_id,
+ type,
+ updated_at,
+ updated_by,
+} from '../common/schemas';
+
+export const searchEsListSchema = t.exact(
+ t.type({
+ created_at,
+ created_by,
+ description,
+ meta: metaOrUndefined,
+ name,
+ tie_breaker_id,
+ type,
+ updated_at,
+ updated_by,
+ })
+);
+
+export type SearchEsListSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/index.ts b/x-pack/plugins/lists/common/schemas/index.ts
new file mode 100644
index 0000000000000..6a60a6df55691
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/index.ts
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './common';
+export * from './request';
+export * from './response';
+export * from './elastic_query';
+export * from './elastic_response';
diff --git a/x-pack/plugins/lists/common/schemas/request/create_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/request/create_list_item_schema.ts
new file mode 100644
index 0000000000000..8168e5a9838f2
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/create_list_item_schema.ts
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { idOrUndefined, list_id, metaOrUndefined, value } from '../common/schemas';
+
+export const createListItemSchema = t.exact(
+ t.type({
+ id: idOrUndefined,
+ list_id,
+ meta: metaOrUndefined,
+ value,
+ })
+);
+
+export type CreateListItemSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/request/create_list_schema.test.ts b/x-pack/plugins/lists/common/schemas/request/create_list_schema.test.ts
new file mode 100644
index 0000000000000..ba791a55d17eb
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/create_list_schema.test.ts
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { left } from 'fp-ts/lib/Either';
+import { pipe } from 'fp-ts/lib/pipeable';
+
+import { exactCheck, foldLeftRight, getPaths } from '../../siem_common_deps';
+
+import { getListRequest } from './mocks/utils';
+import { createListSchema } from './create_list_schema';
+
+describe('create_list_schema', () => {
+ // TODO: Finish the tests for this
+ test('it should validate a typical lists request', () => {
+ const payload = getListRequest();
+ const decoded = createListSchema.decode(payload);
+ const checked = exactCheck(payload, decoded);
+ const message = pipe(checked, foldLeftRight);
+
+ expect(getPaths(left(message.errors))).toEqual([]);
+ expect(message.schema).toEqual({
+ description: 'Description of a list item',
+ id: 'some-list-id',
+ name: 'Name of a list item',
+ type: 'ip',
+ });
+ });
+});
diff --git a/x-pack/plugins/lists/common/schemas/request/create_list_schema.ts b/x-pack/plugins/lists/common/schemas/request/create_list_schema.ts
new file mode 100644
index 0000000000000..353a4ecdafa0c
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/create_list_schema.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { description, idOrUndefined, metaOrUndefined, name, type } from '../common/schemas';
+
+export const createListSchema = t.exact(
+ t.type({
+ description,
+ id: idOrUndefined,
+ meta: metaOrUndefined,
+ name,
+ type,
+ })
+);
+
+export type CreateListSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/request/delete_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/request/delete_list_item_schema.ts
new file mode 100644
index 0000000000000..f4c1fb5c43eb0
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/delete_list_item_schema.ts
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { idOrUndefined, list_idOrUndefined, valueOrUndefined } from '../common/schemas';
+
+export const deleteListItemSchema = t.exact(
+ t.type({
+ id: idOrUndefined,
+ list_id: list_idOrUndefined,
+ value: valueOrUndefined,
+ })
+);
+
+export type DeleteListItemSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/request/delete_list_schema.ts b/x-pack/plugins/lists/common/schemas/request/delete_list_schema.ts
new file mode 100644
index 0000000000000..fd6aa5b85f81a
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/delete_list_schema.ts
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { id } from '../common/schemas';
+
+export const deleteListSchema = t.exact(
+ t.type({
+ id,
+ })
+);
+
+export type DeleteListSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/request/export_list_item_query_schema.ts b/x-pack/plugins/lists/common/schemas/request/export_list_item_query_schema.ts
new file mode 100644
index 0000000000000..14b201bf8089d
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/export_list_item_query_schema.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { list_id } from '../common/schemas';
+
+export const exportListItemQuerySchema = t.exact(
+ t.type({
+ list_id,
+ // TODO: Add file_name here with a default value
+ })
+);
+
+export type ExportListItemQuerySchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.ts b/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.ts
new file mode 100644
index 0000000000000..b8467d141bdd8
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.ts
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { list_idOrUndefined, typeOrUndefined } from '../common/schemas';
+
+export const importListItemQuerySchema = t.exact(
+ t.type({ list_id: list_idOrUndefined, type: typeOrUndefined })
+);
+
+export type ImportListItemQuerySchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/request/import_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/request/import_list_item_schema.ts
new file mode 100644
index 0000000000000..0cf01db8617f0
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/import_list_item_schema.ts
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import { Readable } from 'stream';
+
+import * as t from 'io-ts';
+
+import { file } from '../common/schemas';
+
+export const importListItemSchema = t.exact(
+ t.type({
+ file,
+ })
+);
+
+export interface HapiReadableStream extends Readable {
+ hapi: {
+ filename: string;
+ };
+}
+
+/**
+ * Special interface since we are streaming in a file through a reader
+ */
+export interface ImportListItemSchema {
+ file: HapiReadableStream;
+}
diff --git a/x-pack/plugins/lists/common/schemas/request/index.ts b/x-pack/plugins/lists/common/schemas/request/index.ts
new file mode 100644
index 0000000000000..d332ab1eb1bab
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/index.ts
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './create_list_item_schema';
+export * from './create_list_schema';
+export * from './delete_list_item_schema';
+export * from './delete_list_schema';
+export * from './export_list_item_query_schema';
+export * from './import_list_item_schema';
+export * from './patch_list_item_schema';
+export * from './patch_list_schema';
+export * from './read_list_item_schema';
+export * from './read_list_schema';
+export * from './import_list_item_query_schema';
+export * from './update_list_schema';
+export * from './update_list_item_schema';
diff --git a/x-pack/plugins/lists/common/schemas/request/mocks/utils.ts b/x-pack/plugins/lists/common/schemas/request/mocks/utils.ts
new file mode 100644
index 0000000000000..e5d189db8490b
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/mocks/utils.ts
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { CreateListSchema } from '../create_list_schema';
+
+export const getListRequest = (): CreateListSchema => ({
+ description: 'Description of a list item',
+ id: 'some-list-id',
+ meta: undefined,
+ name: 'Name of a list item',
+ type: 'ip',
+});
diff --git a/x-pack/plugins/lists/common/schemas/request/patch_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/request/patch_list_item_schema.ts
new file mode 100644
index 0000000000000..3e8198a5109b3
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/patch_list_item_schema.ts
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { id, metaOrUndefined, valueOrUndefined } from '../common/schemas';
+
+export const patchListItemSchema = t.exact(
+ t.type({
+ id,
+ meta: metaOrUndefined,
+ value: valueOrUndefined,
+ })
+);
+
+export type PatchListItemSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/request/patch_list_schema.ts b/x-pack/plugins/lists/common/schemas/request/patch_list_schema.ts
new file mode 100644
index 0000000000000..efcb81fc8be2a
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/patch_list_schema.ts
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { descriptionOrUndefined, id, metaOrUndefined, nameOrUndefined } from '../common/schemas';
+
+export const patchListSchema = t.exact(
+ t.type({
+ description: descriptionOrUndefined,
+ id,
+ meta: metaOrUndefined,
+ name: nameOrUndefined,
+ })
+);
+
+export type PatchListSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/request/read_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/request/read_list_item_schema.ts
new file mode 100644
index 0000000000000..9ea14a2a21ed8
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/read_list_item_schema.ts
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { idOrUndefined, list_idOrUndefined, valueOrUndefined } from '../common/schemas';
+
+export const readListItemSchema = t.exact(
+ t.type({ id: idOrUndefined, list_id: list_idOrUndefined, value: valueOrUndefined })
+);
+
+export type ReadListItemSchema = t.TypeOf;
diff --git a/x-pack/plugins/lists/common/schemas/request/read_list_schema.ts b/x-pack/plugins/lists/common/schemas/request/read_list_schema.ts
new file mode 100644
index 0000000000000..8803346709c31
--- /dev/null
+++ b/x-pack/plugins/lists/common/schemas/request/read_list_schema.ts
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { id } from '../common/schemas';
+
+export const readListSchema = t.exact(
+ t.type({
+ id,
+ })
+);
+
+export type ReadListSchema = t.TypeOf