From 693255c92d855ff33f9650bdbd1cb1ea71c544c5 Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Fri, 7 Feb 2020 12:27:18 -0500 Subject: [PATCH 1/5] [ML] DF Analytics creation: update schema definition for create route (#56979) * fix df analytics create route schema definition * add all optional params to schema definiton --- .../ml/server/new_platform/data_analytics_schema.ts | 10 +++++++++- .../plugins/ml/server/routes/data_frame_analytics.ts | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/ml/server/new_platform/data_analytics_schema.ts b/x-pack/legacy/plugins/ml/server/new_platform/data_analytics_schema.ts index f5d72c51dc070..21454fa884b82 100644 --- a/x-pack/legacy/plugins/ml/server/new_platform/data_analytics_schema.ts +++ b/x-pack/legacy/plugins/ml/server/new_platform/data_analytics_schema.ts @@ -13,8 +13,16 @@ export const dataAnalyticsJobConfigSchema = { results_field: schema.maybe(schema.string()), }), source: schema.object({ - index: schema.string(), + index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]), + query: schema.maybe(schema.any()), + _source: schema.maybe( + schema.object({ + includes: schema.maybe(schema.arrayOf(schema.maybe(schema.string()))), + excludes: schema.maybe(schema.arrayOf(schema.maybe(schema.string()))), + }) + ), }), + allow_lazy_start: schema.maybe(schema.boolean()), analysis: schema.any(), analyzed_fields: schema.any(), model_memory_limit: schema.string(), diff --git a/x-pack/legacy/plugins/ml/server/routes/data_frame_analytics.ts b/x-pack/legacy/plugins/ml/server/routes/data_frame_analytics.ts index 67fa2fba46f1a..f134820adbb48 100644 --- a/x-pack/legacy/plugins/ml/server/routes/data_frame_analytics.ts +++ b/x-pack/legacy/plugins/ml/server/routes/data_frame_analytics.ts @@ -156,7 +156,7 @@ export function dataFrameAnalyticsRoutes({ xpackMainPlugin, router }: RouteIniti params: schema.object({ analyticsId: schema.string(), }), - body: schema.object({ ...dataAnalyticsJobConfigSchema }), + body: schema.object(dataAnalyticsJobConfigSchema), }, }, licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { From b2c9beb00f25a0ef9d84eb8597416c7134d19d1e Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Fri, 7 Feb 2020 13:15:25 -0500 Subject: [PATCH 2/5] [APM][docs] Add troubleshooting for non-indexed fields (#54948) --- docs/apm/troubleshooting.asciidoc | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/apm/troubleshooting.asciidoc b/docs/apm/troubleshooting.asciidoc index 22279b69b70fe..c4611f3b41e55 100644 --- a/docs/apm/troubleshooting.asciidoc +++ b/docs/apm/troubleshooting.asciidoc @@ -6,6 +6,7 @@ your proposed changes at https://github.com/elastic/kibana. Also check out the https://discuss.elastic.co/c/apm[APM discussion forum]. +[[no-apm-data-found]] ==== No APM data found This section can help with any of the following: @@ -69,3 +70,41 @@ or because something is happening to the request that the Agent doesn't understa To resolve this, you'll need to head over to the relevant {apm-agents-ref}[Agent documentation]. Specifically, view the Agent's supported technologies page. You can also use the Agent's public API to manually set a name for the transaction. + +==== Fields are not searchable + +In Elasticsearch, index patterns are used to define settings and mappings that determine how fields should be analyzed. +The recommended index template file for APM Server is installed when Kibana starts. +This template defines which fields are available in Kibana for features like the Kuery bar, +or for linking to other plugins like Logs, Uptime, and Discover. + +As an example, some agents store cookie values in `http.request.cookies`. +Since `http.request` has disabled dynamic indexing, and `http.request.cookies` is not declared in a custom mapping, +the values in `http.request.cookies` are not indexed and thus not searchable. + +*Ensure an index pattern exists* +As a first step, you should ensure the correct index pattern exists. +In Kibana, navigate to *Management > Kibana > Index Patterns*. +In the pattern list, you should see an apm index pattern; The default is `apm-*`. +If you don't, the index pattern doesn't exist. See <> for information on how to fix this problem. + +Selecting the `apm-*` index pattern shows a listing of every field defined in the pattern. + +*Ensure a field is searchable* +There are two things you can do to if you'd like to ensure a field is searchable: + +1. Index your additional data as {apm-overview-ref}/metadata.html[labels] instead. +These are dynamic by default, which means they will be indexed and become searchable and aggregatable. + +2. Use the {apm-server-ref}/configuration-template.html[`append_fields`] feature. As an example, +adding the following to `apm-server.yml` will enable dynamic indexing for `http.request.cookies`: + +[source,yml] +---- +setup.template.enabled: true +setup.template.overwrite: true +setup.template.append_fields: + - name: http.request.cookies + type: object + dynamic: true +---- From 02f309c20631a9826c46a78bc9b08776791653e7 Mon Sep 17 00:00:00 2001 From: Brandon Kobel Date: Fri, 7 Feb 2020 10:27:55 -0800 Subject: [PATCH 3/5] Specifying valid licenses for the Graph feature (#55911) * Specifying valid licenses for the Graph feature * Adding option to /api/features to ignore valid licenses This allow us to take advantage of the /api/featues endpoint within our tests to disable all features, including those which are disabled by the current license. The ui capabilities don't take into considerating the license at the moment, so they're separate entirely separeate mechanisms at this point in time. * Addressing PR comments Co-authored-by: Elastic Machine --- x-pack/legacy/plugins/graph/index.ts | 1 + x-pack/plugins/features/common/feature.ts | 2 +- .../plugins/features/server/feature_schema.ts | 2 +- .../features/server/routes/index.test.ts | 65 ++++++++++++++++++- .../plugins/features/server/routes/index.ts | 10 ++- .../common/services/features.ts | 6 +- .../spaces_only/tests/index.ts | 3 +- 7 files changed, 80 insertions(+), 9 deletions(-) diff --git a/x-pack/legacy/plugins/graph/index.ts b/x-pack/legacy/plugins/graph/index.ts index f798fa5e9f39d..143d07cfdbd57 100644 --- a/x-pack/legacy/plugins/graph/index.ts +++ b/x-pack/legacy/plugins/graph/index.ts @@ -61,6 +61,7 @@ export const graph: LegacyPluginInitializer = kibana => { navLinkId: 'graph', app: ['graph', 'kibana'], catalogue: ['graph'], + validLicenses: ['platinum', 'enterprise', 'trial'], privileges: { all: { savedObject: { diff --git a/x-pack/plugins/features/common/feature.ts b/x-pack/plugins/features/common/feature.ts index 423fe1eb99704..748076b95ad77 100644 --- a/x-pack/plugins/features/common/feature.ts +++ b/x-pack/plugins/features/common/feature.ts @@ -43,7 +43,7 @@ export interface Feature< * This does not restrict access to your feature based on license. * Its only purpose is to inform the space and roles UIs on which features to display. */ - validLicenses?: Array<'basic' | 'standard' | 'gold' | 'platinum' | 'enterprise'>; + validLicenses?: Array<'basic' | 'standard' | 'gold' | 'platinum' | 'enterprise' | 'trial'>; /** * An optional EUI Icon to be used when displaying your feature. diff --git a/x-pack/plugins/features/server/feature_schema.ts b/x-pack/plugins/features/server/feature_schema.ts index 7732686db5ee1..cc12ea1b78dce 100644 --- a/x-pack/plugins/features/server/feature_schema.ts +++ b/x-pack/plugins/features/server/feature_schema.ts @@ -51,7 +51,7 @@ const schema = Joi.object({ name: Joi.string().required(), excludeFromBasePrivileges: Joi.boolean(), validLicenses: Joi.array().items( - Joi.string().valid('basic', 'standard', 'gold', 'platinum', 'enterprise') + Joi.string().valid('basic', 'standard', 'gold', 'platinum', 'enterprise', 'trial') ), icon: Joi.string(), description: Joi.string(), diff --git a/x-pack/plugins/features/server/routes/index.test.ts b/x-pack/plugins/features/server/routes/index.test.ts index 98a23a61d542c..b0f8417b7175d 100644 --- a/x-pack/plugins/features/server/routes/index.test.ts +++ b/x-pack/plugins/features/server/routes/index.test.ts @@ -53,7 +53,7 @@ describe('GET /api/features', () => { it('returns a list of available features', async () => { const mockResponse = httpServerMock.createResponseFactory(); - routeHandler(undefined as any, undefined as any, mockResponse); + routeHandler(undefined as any, { query: {} } as any, mockResponse); expect(mockResponse.ok.mock.calls).toMatchInlineSnapshot(` Array [ @@ -84,11 +84,11 @@ describe('GET /api/features', () => { `); }); - it(`does not return features that arent allowed by current license`, async () => { + it(`by default does not return features that arent allowed by current license`, async () => { currentLicenseLevel = 'basic'; const mockResponse = httpServerMock.createResponseFactory(); - routeHandler(undefined as any, undefined as any, mockResponse); + routeHandler(undefined as any, { query: {} } as any, mockResponse); expect(mockResponse.ok.mock.calls).toMatchInlineSnapshot(` Array [ @@ -107,4 +107,63 @@ describe('GET /api/features', () => { ] `); }); + + it(`ignoreValidLicenses=false does not return features that arent allowed by current license`, async () => { + currentLicenseLevel = 'basic'; + + const mockResponse = httpServerMock.createResponseFactory(); + routeHandler(undefined as any, { query: { ignoreValidLicenses: false } } as any, mockResponse); + + expect(mockResponse.ok.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "body": Array [ + Object { + "app": Array [], + "id": "feature_1", + "name": "Feature 1", + "privileges": Object {}, + }, + ], + }, + ], + ] + `); + }); + + it(`ignoreValidLicenses=true returns features that arent allowed by current license`, async () => { + currentLicenseLevel = 'basic'; + + const mockResponse = httpServerMock.createResponseFactory(); + routeHandler(undefined as any, { query: { ignoreValidLicenses: true } } as any, mockResponse); + + expect(mockResponse.ok.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "body": Array [ + Object { + "app": Array [], + "id": "feature_1", + "name": "Feature 1", + "privileges": Object {}, + }, + Object { + "app": Array [ + "bar-app", + ], + "id": "licensed_feature", + "name": "Licensed Feature", + "privileges": Object {}, + "validLicenses": Array [ + "gold", + ], + }, + ], + }, + ], + ] + `); + }); }); diff --git a/x-pack/plugins/features/server/routes/index.ts b/x-pack/plugins/features/server/routes/index.ts index 51869c39cf83c..cf4d61ccac88b 100644 --- a/x-pack/plugins/features/server/routes/index.ts +++ b/x-pack/plugins/features/server/routes/index.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { schema } from '@kbn/config-schema'; import { IRouter } from '../../../../../src/core/server'; import { LegacyAPI } from '../plugin'; import { FeatureRegistry } from '../feature_registry'; @@ -19,13 +20,20 @@ export interface RouteDefinitionParams { export function defineRoutes({ router, featureRegistry, getLegacyAPI }: RouteDefinitionParams) { router.get( - { path: '/api/features', options: { tags: ['access:features'] }, validate: false }, + { + path: '/api/features', + options: { tags: ['access:features'] }, + validate: { + query: schema.object({ ignoreValidLicenses: schema.boolean({ defaultValue: false }) }), + }, + }, (context, request, response) => { const allFeatures = featureRegistry.getAll(); return response.ok({ body: allFeatures.filter( feature => + request.query.ignoreValidLicenses || !feature.validLicenses || !feature.validLicenses.length || getLegacyAPI().xpackInfo.license.isOneOf(feature.validLicenses) diff --git a/x-pack/test/ui_capabilities/common/services/features.ts b/x-pack/test/ui_capabilities/common/services/features.ts index 9f644fd6d0f6e..0f796c1d0a0cc 100644 --- a/x-pack/test/ui_capabilities/common/services/features.ts +++ b/x-pack/test/ui_capabilities/common/services/features.ts @@ -22,9 +22,11 @@ export class FeaturesService { }); } - public async get(): Promise { + public async get({ ignoreValidLicenses } = { ignoreValidLicenses: false }): Promise { this.log.debug('requesting /api/features to get the features'); - const response = await this.axios.get('/api/features'); + const response = await this.axios.get( + `/api/features?ignoreValidLicenses=${ignoreValidLicenses}` + ); if (response.status !== 200) { throw new Error( diff --git a/x-pack/test/ui_capabilities/spaces_only/tests/index.ts b/x-pack/test/ui_capabilities/spaces_only/tests/index.ts index 0b40f9716dcb4..a25838ac4f76d 100644 --- a/x-pack/test/ui_capabilities/spaces_only/tests/index.ts +++ b/x-pack/test/ui_capabilities/spaces_only/tests/index.ts @@ -16,7 +16,8 @@ export default function uiCapabilitesTests({ loadTestFile, getService }: FtrProv this.tags('ciGroup9'); before(async () => { - const features = await featuresService.get(); + // we're using a basic license, so if we want to disable all features, we have to ignore the valid licenses + const features = await featuresService.get({ ignoreValidLicenses: true }); for (const space of SpaceScenarios) { const disabledFeatures = space.disabledFeatures === '*' ? Object.keys(features) : space.disabledFeatures; From 4776cd939c1e8e52ccd4d4b6bccd4e8f218a3c46 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Fri, 7 Feb 2020 20:06:56 +0100 Subject: [PATCH 4/5] [ML] New Platform server shim: update file data visualizer routes to use new platform router (#56972) * [ML] change import endpoint call to fileupload plugin, update file analyzer endpoint * [ML] add apiDoc annotation * [ML] AnalysisResult interface, remove url from apidoc.json * [ML] delete import_data.js * [ML] remove caching code, address PR comments * [ML] file import * [ML] apidoc * [ML] schema validation Co-authored-by: Elastic Machine --- .../file_data_visualizer.js | 100 ----------- .../file_data_visualizer.ts | 103 ++++++++++++ .../{import_data.js => import_data.ts} | 59 +++++-- .../{index.js => index.ts} | 10 +- .../plugins/ml/server/routes/apidoc.json | 4 +- .../ml/server/routes/file_data_visualizer.js | 69 -------- .../ml/server/routes/file_data_visualizer.ts | 159 ++++++++++++++++++ 7 files changed, 319 insertions(+), 185 deletions(-) delete mode 100644 x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.js create mode 100644 x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts rename x-pack/legacy/plugins/ml/server/models/file_data_visualizer/{import_data.js => import_data.ts} (71%) rename x-pack/legacy/plugins/ml/server/models/file_data_visualizer/{index.js => index.ts} (53%) delete mode 100644 x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.js create mode 100644 x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.ts diff --git a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.js b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.js deleted file mode 100644 index 28bb7c24cf12e..0000000000000 --- a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.js +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import Boom from 'boom'; -import fs from 'fs'; -import os from 'os'; -const util = require('util'); -// const readFile = util.promisify(fs.readFile); -const readdir = util.promisify(fs.readdir); -const writeFile = util.promisify(fs.writeFile); - -export function fileDataVisualizerProvider(callWithRequest) { - async function analyzeFile(data, overrides) { - let cached = false; - let results = []; - - try { - results = await callWithRequest('ml.fileStructure', { body: data, ...overrides }); - if (false) { - // disabling caching for now - cached = await cacheData(data); - } - } catch (error) { - const err = error.message !== undefined ? error.message : error; - throw Boom.badRequest(err); - } - - const { hasOverrides, reducedOverrides } = formatOverrides(overrides); - - return { - ...(hasOverrides && { overrides: reducedOverrides }), - cached, - results, - }; - } - - async function cacheData(data) { - const outputPath = `${os.tmpdir()}/kibana-ml`; - const tempFile = 'es-ml-tempFile'; - const tempFilePath = `${outputPath}/${tempFile}`; - - try { - createOutputDir(outputPath); - await deleteOutputFiles(outputPath); - await writeFile(tempFilePath, data); - return true; - } catch (error) { - return false; - } - } - - function createOutputDir(dir) { - if (fs.existsSync(dir) === false) { - fs.mkdirSync(dir); - } - } - - async function deleteOutputFiles(outputPath) { - const files = await readdir(outputPath); - files.forEach(f => { - fs.unlinkSync(`${outputPath}/${f}`); - }); - } - - return { - analyzeFile, - }; -} - -function formatOverrides(overrides) { - let hasOverrides = false; - - const reducedOverrides = Object.keys(overrides).reduce((p, c) => { - if (overrides[c] !== '') { - p[c] = overrides[c]; - hasOverrides = true; - } - return p; - }, {}); - - if (reducedOverrides.column_names !== undefined) { - reducedOverrides.column_names = reducedOverrides.column_names.split(','); - } - - if (reducedOverrides.has_header_row !== undefined) { - reducedOverrides.has_header_row = reducedOverrides.has_header_row === 'true'; - } - - if (reducedOverrides.should_trim_fields !== undefined) { - reducedOverrides.should_trim_fields = reducedOverrides.should_trim_fields === 'true'; - } - - return { - reducedOverrides, - hasOverrides, - }; -} diff --git a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts new file mode 100644 index 0000000000000..fd5b5221393fc --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Boom from 'boom'; +import { RequestHandlerContext } from 'kibana/server'; + +export type InputData = any[]; + +export interface InputOverrides { + [key: string]: string; +} + +export type FormattedOverrides = InputOverrides & { + column_names: string[]; + has_header_row: boolean; + should_trim_fields: boolean; +}; + +export interface AnalysisResult { + results: { + charset: string; + has_header_row: boolean; + has_byte_order_marker: boolean; + format: string; + field_stats: { + [fieldName: string]: { + count: number; + cardinality: number; + top_hits: Array<{ count: number; value: any }>; + }; + }; + sample_start: string; + num_messages_analyzed: number; + mappings: { + [fieldName: string]: { + type: string; + }; + }; + quote: string; + delimiter: string; + need_client_timezone: boolean; + num_lines_analyzed: number; + column_names: string[]; + }; + overrides?: FormattedOverrides; +} + +export function fileDataVisualizerProvider(context: RequestHandlerContext) { + async function analyzeFile(data: any, overrides: any): Promise { + let results = []; + + try { + results = await context.ml!.mlClient.callAsCurrentUser('ml.fileStructure', { + body: data, + ...overrides, + }); + } catch (error) { + const err = error.message !== undefined ? error.message : error; + throw Boom.badRequest(err); + } + + const { hasOverrides, reducedOverrides } = formatOverrides(overrides); + + return { + ...(hasOverrides && { overrides: reducedOverrides }), + results, + }; + } + + return { + analyzeFile, + }; +} + +function formatOverrides(overrides: InputOverrides) { + let hasOverrides = false; + + const reducedOverrides: FormattedOverrides = Object.keys(overrides).reduce((acc, overrideKey) => { + const overrideValue: string = overrides[overrideKey]; + if (overrideValue !== '') { + if (overrideKey === 'column_names') { + acc.column_names = overrideValue.split(','); + } else if (overrideKey === 'has_header_row') { + acc.has_header_row = overrideValue === 'true'; + } else if (overrideKey === 'should_trim_fields') { + acc.should_trim_fields = overrideValue === 'true'; + } else { + acc[overrideKey] = overrideValue; + } + + hasOverrides = true; + } + return acc; + }, {} as FormattedOverrides); + + return { + reducedOverrides, + hasOverrides, + }; +} diff --git a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/import_data.js b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/import_data.ts similarity index 71% rename from x-pack/legacy/plugins/ml/server/models/file_data_visualizer/import_data.js rename to x-pack/legacy/plugins/ml/server/models/file_data_visualizer/import_data.ts index 644a137fbc092..008efb43a6c07 100644 --- a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/import_data.js +++ b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/import_data.ts @@ -4,10 +4,43 @@ * you may not use this file except in compliance with the Elastic License. */ +import { RequestHandlerContext } from 'kibana/server'; import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer'; +import { InputData } from './file_data_visualizer'; -export function importDataProvider(callWithRequest) { - async function importData(id, index, settings, mappings, ingestPipeline, data) { +export interface Settings { + pipeline?: string; + index: string; + body: any[]; + [key: string]: any; +} + +export interface Mappings { + [key: string]: any; +} + +export interface InjectPipeline { + id: string; + pipeline: any; +} + +interface Failure { + item: number; + reason: string; + doc: any; +} + +export function importDataProvider(context: RequestHandlerContext) { + const callAsCurrentUser = context.ml!.mlClient.callAsCurrentUser; + + async function importData( + id: string, + index: string, + settings: Settings, + mappings: Mappings, + ingestPipeline: InjectPipeline, + data: InputData + ) { let createdIndex; let createdPipelineId; const docCount = data.length; @@ -35,7 +68,7 @@ export function importDataProvider(callWithRequest) { createdPipelineId = pipelineId; } - let failures = []; + let failures: Failure[] = []; if (data.length) { const resp = await indexData(index, createdPipelineId, data); if (resp.success === false) { @@ -72,8 +105,8 @@ export function importDataProvider(callWithRequest) { } } - async function createIndex(index, settings, mappings) { - const body = { + async function createIndex(index: string, settings: Settings, mappings: Mappings) { + const body: { mappings: Mappings; settings?: Settings } = { mappings: { _meta: { created_by: INDEX_META_DATA_CREATED_BY, @@ -86,10 +119,10 @@ export function importDataProvider(callWithRequest) { body.settings = settings; } - await callWithRequest('indices.create', { index, body }); + await callAsCurrentUser('indices.create', { index, body }); } - async function indexData(index, pipelineId, data) { + async function indexData(index: string, pipelineId: string, data: InputData) { try { const body = []; for (let i = 0; i < data.length; i++) { @@ -97,12 +130,12 @@ export function importDataProvider(callWithRequest) { body.push(data[i]); } - const settings = { index, body }; + const settings: Settings = { index, body }; if (pipelineId !== undefined) { settings.pipeline = pipelineId; } - const resp = await callWithRequest('bulk', settings); + const resp = await callAsCurrentUser('bulk', settings); if (resp.errors) { throw resp; } else { @@ -113,7 +146,7 @@ export function importDataProvider(callWithRequest) { }; } } catch (error) { - let failures = []; + let failures: Failure[] = []; let ingestError = false; if (error.errors !== undefined && Array.isArray(error.items)) { // an expected error where some or all of the bulk request @@ -134,11 +167,11 @@ export function importDataProvider(callWithRequest) { } } - async function createPipeline(id, pipeline) { - return await callWithRequest('ingest.putPipeline', { id, body: pipeline }); + async function createPipeline(id: string, pipeline: any) { + return await callAsCurrentUser('ingest.putPipeline', { id, body: pipeline }); } - function getFailures(items, data) { + function getFailures(items: any[], data: InputData): Failure[] { const failures = []; for (let i = 0; i < items.length; i++) { const item = items[i]; diff --git a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.js b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.ts similarity index 53% rename from x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.js rename to x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.ts index 3bda5599e7181..94529dc111696 100644 --- a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.js +++ b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.ts @@ -4,5 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -export { fileDataVisualizerProvider } from './file_data_visualizer'; -export { importDataProvider } from './import_data'; +export { + fileDataVisualizerProvider, + InputOverrides, + InputData, + AnalysisResult, +} from './file_data_visualizer'; + +export { importDataProvider, Settings, InjectPipeline, Mappings } from './import_data'; diff --git a/x-pack/legacy/plugins/ml/server/routes/apidoc.json b/x-pack/legacy/plugins/ml/server/routes/apidoc.json index 574065446827d..1be31e2316228 100644 --- a/x-pack/legacy/plugins/ml/server/routes/apidoc.json +++ b/x-pack/legacy/plugins/ml/server/routes/apidoc.json @@ -3,7 +3,6 @@ "version": "0.1.0", "description": "ML Kibana API", "title": "ML Kibana API", - "url" : "/api/ml/", "order": [ "DataFrameAnalytics", "GetDataFrameAnalytics", @@ -34,6 +33,9 @@ "ForecastAnomalyDetector", "GetOverallBuckets", "GetCategories", + "FileDataVisualizer", + "AnalyzeFile", + "ImportFile" "ResultsService", "GetAnomaliesTableData", "GetCategoryDefinition", diff --git a/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.js b/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.js deleted file mode 100644 index fc6a0ff756928..0000000000000 --- a/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.js +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { callWithRequestFactory } from '../client/call_with_request_factory'; -import { wrapError } from '../client/errors'; -import { fileDataVisualizerProvider, importDataProvider } from '../models/file_data_visualizer'; -import { MAX_BYTES } from '../../common/constants/file_datavisualizer'; - -import { incrementFileDataVisualizerIndexCreationCount } from '../lib/ml_telemetry/ml_telemetry'; - -function analyzeFiles(callWithRequest, data, overrides) { - const { analyzeFile } = fileDataVisualizerProvider(callWithRequest); - return analyzeFile(data, overrides); -} - -function importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data) { - const { importData: importDataFunc } = importDataProvider(callWithRequest); - return importDataFunc(id, index, settings, mappings, ingestPipeline, data); -} - -export function fileDataVisualizerRoutes({ - commonRouteConfig, - elasticsearchPlugin, - route, - savedObjects, -}) { - route({ - method: 'POST', - path: '/api/ml/file_data_visualizer/analyze_file', - handler(request) { - const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); - const data = request.payload; - - return analyzeFiles(callWithRequest, data, request.query).catch(wrapError); - }, - config: { - ...commonRouteConfig, - payload: { maxBytes: MAX_BYTES }, - }, - }); - - route({ - method: 'POST', - path: '/api/ml/file_data_visualizer/import', - handler(request) { - const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); - const { id } = request.query; - const { index, data, settings, mappings, ingestPipeline } = request.payload; - - // `id` being `undefined` tells us that this is a new import due to create a new index. - // follow-up import calls to just add additional data will include the `id` of the created - // index, we'll ignore those and don't increment the counter. - if (id === undefined) { - incrementFileDataVisualizerIndexCreationCount(elasticsearchPlugin, savedObjects); - } - - return importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data).catch( - wrapError - ); - }, - config: { - ...commonRouteConfig, - payload: { maxBytes: MAX_BYTES }, - }, - }); -} diff --git a/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.ts b/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.ts new file mode 100644 index 0000000000000..95f2a9fe7298f --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.ts @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema } from '@kbn/config-schema'; +import { RequestHandlerContext } from 'kibana/server'; +import { MAX_BYTES } from '../../common/constants/file_datavisualizer'; +import { wrapError } from '../client/error_wrapper'; +import { + InputOverrides, + InputData, + fileDataVisualizerProvider, + importDataProvider, + Settings, + InjectPipeline, + Mappings, +} from '../models/file_data_visualizer'; + +import { licensePreRoutingFactory } from '../new_platform/licence_check_pre_routing_factory'; +import { RouteInitialization } from '../new_platform/plugin'; +import { incrementFileDataVisualizerIndexCreationCount } from '../lib/ml_telemetry'; + +function analyzeFiles(context: RequestHandlerContext, data: InputData, overrides: InputOverrides) { + const { analyzeFile } = fileDataVisualizerProvider(context); + return analyzeFile(data, overrides); +} + +function importData( + context: RequestHandlerContext, + id: string, + index: string, + settings: Settings, + mappings: Mappings, + ingestPipeline: InjectPipeline, + data: InputData +) { + const { importData: importDataFunc } = importDataProvider(context); + return importDataFunc(id, index, settings, mappings, ingestPipeline, data); +} + +/** + * Routes for the file data visualizer. + */ +export function fileDataVisualizerRoutes({ + router, + xpackMainPlugin, + savedObjects, + elasticsearchPlugin, +}: RouteInitialization) { + /** + * @apiGroup FileDataVisualizer + * + * @api {post} /api/ml/file_data_visualizer/analyze_file Analyze file data + * @apiName AnalyzeFile + * @apiDescription Performs analysis of the file data. + */ + router.post( + { + path: '/api/ml/file_data_visualizer/analyze_file', + validate: { + body: schema.any(), + query: schema.maybe( + schema.object({ + charset: schema.maybe(schema.string()), + column_names: schema.maybe(schema.string()), + delimiter: schema.maybe(schema.string()), + explain: schema.maybe(schema.string()), + format: schema.maybe(schema.string()), + grok_pattern: schema.maybe(schema.string()), + has_header_row: schema.maybe(schema.string()), + line_merge_size_limit: schema.maybe(schema.string()), + lines_to_sample: schema.maybe(schema.string()), + quote: schema.maybe(schema.string()), + should_trim_fields: schema.maybe(schema.string()), + timeout: schema.maybe(schema.string()), + timestamp_field: schema.maybe(schema.string()), + timestamp_format: schema.maybe(schema.string()), + }) + ), + }, + options: { + body: { + accepts: ['text/*', 'application/json'], + maxBytes: MAX_BYTES, + }, + }, + }, + licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { + try { + const result = await analyzeFiles(context, request.body, request.query); + return response.ok({ body: result }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + + /** + * @apiGroup FileDataVisualizer + * + * @api {post} /api/ml/file_data_visualizer/import Import file data + * @apiName ImportFile + * @apiDescription Imports file data into elasticsearch index. + */ + router.post( + { + path: '/api/ml/file_data_visualizer/import', + validate: { + query: schema.object({ + id: schema.maybe(schema.string()), + }), + body: schema.object({ + index: schema.maybe(schema.string()), + data: schema.arrayOf(schema.any()), + settings: schema.maybe(schema.any()), + mappings: schema.any(), + ingestPipeline: schema.object({ + id: schema.maybe(schema.string()), + pipeline: schema.maybe(schema.any()), + }), + }), + }, + options: { + body: { + accepts: ['application/json'], + maxBytes: MAX_BYTES, + }, + }, + }, + licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { + try { + const { id } = request.query; + const { index, data, settings, mappings, ingestPipeline } = request.body; + + // `id` being `undefined` tells us that this is a new import due to create a new index. + // follow-up import calls to just add additional data will include the `id` of the created + // index, we'll ignore those and don't increment the counter. + if (id === undefined) { + await incrementFileDataVisualizerIndexCreationCount(elasticsearchPlugin, savedObjects!); + } + + const result = await importData( + context, + id, + index, + settings, + mappings, + ingestPipeline, + data + ); + return response.ok({ body: result }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); +} From a98ad7a55e4f73922a18320c8c3f9e831cd79d1e Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Fri, 7 Feb 2020 19:22:36 +0000 Subject: [PATCH 5/5] chore(NA): removes use of parallel option in the terser minimizer (#57077) * chore(NA): removes use of parallel option in the terser minimizer * docs(NA): update note --- src/optimize/base_optimizer.js | 2 +- .../dynamic_dll_plugin/dll_config_model.js | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/optimize/base_optimizer.js b/src/optimize/base_optimizer.js index d9df2a1955df3..539c55c969653 100644 --- a/src/optimize/base_optimizer.js +++ b/src/optimize/base_optimizer.js @@ -459,7 +459,7 @@ export default class BaseOptimizer { optimization: { minimizer: [ new TerserPlugin({ - parallel: this.getThreadLoaderPoolConfig().workers, + parallel: false, sourceMap: false, cache: false, extractComments: false, diff --git a/src/optimize/dynamic_dll_plugin/dll_config_model.js b/src/optimize/dynamic_dll_plugin/dll_config_model.js index 2e74cb6af86d4..9ca6071b8f515 100644 --- a/src/optimize/dynamic_dll_plugin/dll_config_model.js +++ b/src/optimize/dynamic_dll_plugin/dll_config_model.js @@ -214,16 +214,20 @@ function common(config) { return webpackMerge(generateDLL(config)); } -function optimized(config) { +function optimized() { return webpackMerge({ mode: 'production', optimization: { minimizer: [ new TerserPlugin({ - // Apply the same logic used to calculate the - // threadLoaderPool workers number to spawn - // the parallel processes on terser - parallel: config.threadLoaderPoolConfig.workers, + // NOTE: we should not enable that option for now + // Since 2.0.0 terser-webpack-plugin is using jest-worker + // to run tasks in a pool of workers. Currently it looks like + // is requiring too much memory and break on large entry points + // compilations (like this) one. Also the gain we have enabling + // that option was barely noticed. + // https://github.com/webpack-contrib/terser-webpack-plugin/issues/143 + parallel: false, sourceMap: false, cache: false, extractComments: false, @@ -250,5 +254,5 @@ export function configModel(rawConfig = {}) { return webpackMerge(common(config), unoptimized()); } - return webpackMerge(common(config), optimized(config)); + return webpackMerge(common(config), optimized()); }