diff --git a/docs/apm/troubleshooting.asciidoc b/docs/apm/troubleshooting.asciidoc index 22279b69b70fe..c4611f3b41e55 100644 --- a/docs/apm/troubleshooting.asciidoc +++ b/docs/apm/troubleshooting.asciidoc @@ -6,6 +6,7 @@ your proposed changes at https://github.com/elastic/kibana. Also check out the https://discuss.elastic.co/c/apm[APM discussion forum]. +[[no-apm-data-found]] ==== No APM data found This section can help with any of the following: @@ -69,3 +70,41 @@ or because something is happening to the request that the Agent doesn't understa To resolve this, you'll need to head over to the relevant {apm-agents-ref}[Agent documentation]. Specifically, view the Agent's supported technologies page. You can also use the Agent's public API to manually set a name for the transaction. + +==== Fields are not searchable + +In Elasticsearch, index patterns are used to define settings and mappings that determine how fields should be analyzed. +The recommended index template file for APM Server is installed when Kibana starts. +This template defines which fields are available in Kibana for features like the Kuery bar, +or for linking to other plugins like Logs, Uptime, and Discover. + +As an example, some agents store cookie values in `http.request.cookies`. +Since `http.request` has disabled dynamic indexing, and `http.request.cookies` is not declared in a custom mapping, +the values in `http.request.cookies` are not indexed and thus not searchable. + +*Ensure an index pattern exists* +As a first step, you should ensure the correct index pattern exists. +In Kibana, navigate to *Management > Kibana > Index Patterns*. +In the pattern list, you should see an apm index pattern; The default is `apm-*`. +If you don't, the index pattern doesn't exist. See <> for information on how to fix this problem. + +Selecting the `apm-*` index pattern shows a listing of every field defined in the pattern. + +*Ensure a field is searchable* +There are two things you can do to if you'd like to ensure a field is searchable: + +1. Index your additional data as {apm-overview-ref}/metadata.html[labels] instead. +These are dynamic by default, which means they will be indexed and become searchable and aggregatable. + +2. Use the {apm-server-ref}/configuration-template.html[`append_fields`] feature. As an example, +adding the following to `apm-server.yml` will enable dynamic indexing for `http.request.cookies`: + +[source,yml] +---- +setup.template.enabled: true +setup.template.overwrite: true +setup.template.append_fields: + - name: http.request.cookies + type: object + dynamic: true +---- diff --git a/src/optimize/base_optimizer.js b/src/optimize/base_optimizer.js index a29774b7e1582..4002e4a6db95c 100644 --- a/src/optimize/base_optimizer.js +++ b/src/optimize/base_optimizer.js @@ -461,7 +461,7 @@ export default class BaseOptimizer { optimization: { minimizer: [ new TerserPlugin({ - parallel: this.getThreadLoaderPoolConfig().workers, + parallel: false, sourceMap: false, cache: false, extractComments: false, diff --git a/src/optimize/dynamic_dll_plugin/dll_config_model.js b/src/optimize/dynamic_dll_plugin/dll_config_model.js index 2e74cb6af86d4..9ca6071b8f515 100644 --- a/src/optimize/dynamic_dll_plugin/dll_config_model.js +++ b/src/optimize/dynamic_dll_plugin/dll_config_model.js @@ -214,16 +214,20 @@ function common(config) { return webpackMerge(generateDLL(config)); } -function optimized(config) { +function optimized() { return webpackMerge({ mode: 'production', optimization: { minimizer: [ new TerserPlugin({ - // Apply the same logic used to calculate the - // threadLoaderPool workers number to spawn - // the parallel processes on terser - parallel: config.threadLoaderPoolConfig.workers, + // NOTE: we should not enable that option for now + // Since 2.0.0 terser-webpack-plugin is using jest-worker + // to run tasks in a pool of workers. Currently it looks like + // is requiring too much memory and break on large entry points + // compilations (like this) one. Also the gain we have enabling + // that option was barely noticed. + // https://github.com/webpack-contrib/terser-webpack-plugin/issues/143 + parallel: false, sourceMap: false, cache: false, extractComments: false, @@ -250,5 +254,5 @@ export function configModel(rawConfig = {}) { return webpackMerge(common(config), unoptimized()); } - return webpackMerge(common(config), optimized(config)); + return webpackMerge(common(config), optimized()); } diff --git a/x-pack/legacy/plugins/graph/index.ts b/x-pack/legacy/plugins/graph/index.ts index f798fa5e9f39d..143d07cfdbd57 100644 --- a/x-pack/legacy/plugins/graph/index.ts +++ b/x-pack/legacy/plugins/graph/index.ts @@ -61,6 +61,7 @@ export const graph: LegacyPluginInitializer = kibana => { navLinkId: 'graph', app: ['graph', 'kibana'], catalogue: ['graph'], + validLicenses: ['platinum', 'enterprise', 'trial'], privileges: { all: { savedObject: { diff --git a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.js b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.js deleted file mode 100644 index 28bb7c24cf12e..0000000000000 --- a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.js +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import Boom from 'boom'; -import fs from 'fs'; -import os from 'os'; -const util = require('util'); -// const readFile = util.promisify(fs.readFile); -const readdir = util.promisify(fs.readdir); -const writeFile = util.promisify(fs.writeFile); - -export function fileDataVisualizerProvider(callWithRequest) { - async function analyzeFile(data, overrides) { - let cached = false; - let results = []; - - try { - results = await callWithRequest('ml.fileStructure', { body: data, ...overrides }); - if (false) { - // disabling caching for now - cached = await cacheData(data); - } - } catch (error) { - const err = error.message !== undefined ? error.message : error; - throw Boom.badRequest(err); - } - - const { hasOverrides, reducedOverrides } = formatOverrides(overrides); - - return { - ...(hasOverrides && { overrides: reducedOverrides }), - cached, - results, - }; - } - - async function cacheData(data) { - const outputPath = `${os.tmpdir()}/kibana-ml`; - const tempFile = 'es-ml-tempFile'; - const tempFilePath = `${outputPath}/${tempFile}`; - - try { - createOutputDir(outputPath); - await deleteOutputFiles(outputPath); - await writeFile(tempFilePath, data); - return true; - } catch (error) { - return false; - } - } - - function createOutputDir(dir) { - if (fs.existsSync(dir) === false) { - fs.mkdirSync(dir); - } - } - - async function deleteOutputFiles(outputPath) { - const files = await readdir(outputPath); - files.forEach(f => { - fs.unlinkSync(`${outputPath}/${f}`); - }); - } - - return { - analyzeFile, - }; -} - -function formatOverrides(overrides) { - let hasOverrides = false; - - const reducedOverrides = Object.keys(overrides).reduce((p, c) => { - if (overrides[c] !== '') { - p[c] = overrides[c]; - hasOverrides = true; - } - return p; - }, {}); - - if (reducedOverrides.column_names !== undefined) { - reducedOverrides.column_names = reducedOverrides.column_names.split(','); - } - - if (reducedOverrides.has_header_row !== undefined) { - reducedOverrides.has_header_row = reducedOverrides.has_header_row === 'true'; - } - - if (reducedOverrides.should_trim_fields !== undefined) { - reducedOverrides.should_trim_fields = reducedOverrides.should_trim_fields === 'true'; - } - - return { - reducedOverrides, - hasOverrides, - }; -} diff --git a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts new file mode 100644 index 0000000000000..fd5b5221393fc --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Boom from 'boom'; +import { RequestHandlerContext } from 'kibana/server'; + +export type InputData = any[]; + +export interface InputOverrides { + [key: string]: string; +} + +export type FormattedOverrides = InputOverrides & { + column_names: string[]; + has_header_row: boolean; + should_trim_fields: boolean; +}; + +export interface AnalysisResult { + results: { + charset: string; + has_header_row: boolean; + has_byte_order_marker: boolean; + format: string; + field_stats: { + [fieldName: string]: { + count: number; + cardinality: number; + top_hits: Array<{ count: number; value: any }>; + }; + }; + sample_start: string; + num_messages_analyzed: number; + mappings: { + [fieldName: string]: { + type: string; + }; + }; + quote: string; + delimiter: string; + need_client_timezone: boolean; + num_lines_analyzed: number; + column_names: string[]; + }; + overrides?: FormattedOverrides; +} + +export function fileDataVisualizerProvider(context: RequestHandlerContext) { + async function analyzeFile(data: any, overrides: any): Promise { + let results = []; + + try { + results = await context.ml!.mlClient.callAsCurrentUser('ml.fileStructure', { + body: data, + ...overrides, + }); + } catch (error) { + const err = error.message !== undefined ? error.message : error; + throw Boom.badRequest(err); + } + + const { hasOverrides, reducedOverrides } = formatOverrides(overrides); + + return { + ...(hasOverrides && { overrides: reducedOverrides }), + results, + }; + } + + return { + analyzeFile, + }; +} + +function formatOverrides(overrides: InputOverrides) { + let hasOverrides = false; + + const reducedOverrides: FormattedOverrides = Object.keys(overrides).reduce((acc, overrideKey) => { + const overrideValue: string = overrides[overrideKey]; + if (overrideValue !== '') { + if (overrideKey === 'column_names') { + acc.column_names = overrideValue.split(','); + } else if (overrideKey === 'has_header_row') { + acc.has_header_row = overrideValue === 'true'; + } else if (overrideKey === 'should_trim_fields') { + acc.should_trim_fields = overrideValue === 'true'; + } else { + acc[overrideKey] = overrideValue; + } + + hasOverrides = true; + } + return acc; + }, {} as FormattedOverrides); + + return { + reducedOverrides, + hasOverrides, + }; +} diff --git a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/import_data.js b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/import_data.ts similarity index 71% rename from x-pack/legacy/plugins/ml/server/models/file_data_visualizer/import_data.js rename to x-pack/legacy/plugins/ml/server/models/file_data_visualizer/import_data.ts index 644a137fbc092..008efb43a6c07 100644 --- a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/import_data.js +++ b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/import_data.ts @@ -4,10 +4,43 @@ * you may not use this file except in compliance with the Elastic License. */ +import { RequestHandlerContext } from 'kibana/server'; import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer'; +import { InputData } from './file_data_visualizer'; -export function importDataProvider(callWithRequest) { - async function importData(id, index, settings, mappings, ingestPipeline, data) { +export interface Settings { + pipeline?: string; + index: string; + body: any[]; + [key: string]: any; +} + +export interface Mappings { + [key: string]: any; +} + +export interface InjectPipeline { + id: string; + pipeline: any; +} + +interface Failure { + item: number; + reason: string; + doc: any; +} + +export function importDataProvider(context: RequestHandlerContext) { + const callAsCurrentUser = context.ml!.mlClient.callAsCurrentUser; + + async function importData( + id: string, + index: string, + settings: Settings, + mappings: Mappings, + ingestPipeline: InjectPipeline, + data: InputData + ) { let createdIndex; let createdPipelineId; const docCount = data.length; @@ -35,7 +68,7 @@ export function importDataProvider(callWithRequest) { createdPipelineId = pipelineId; } - let failures = []; + let failures: Failure[] = []; if (data.length) { const resp = await indexData(index, createdPipelineId, data); if (resp.success === false) { @@ -72,8 +105,8 @@ export function importDataProvider(callWithRequest) { } } - async function createIndex(index, settings, mappings) { - const body = { + async function createIndex(index: string, settings: Settings, mappings: Mappings) { + const body: { mappings: Mappings; settings?: Settings } = { mappings: { _meta: { created_by: INDEX_META_DATA_CREATED_BY, @@ -86,10 +119,10 @@ export function importDataProvider(callWithRequest) { body.settings = settings; } - await callWithRequest('indices.create', { index, body }); + await callAsCurrentUser('indices.create', { index, body }); } - async function indexData(index, pipelineId, data) { + async function indexData(index: string, pipelineId: string, data: InputData) { try { const body = []; for (let i = 0; i < data.length; i++) { @@ -97,12 +130,12 @@ export function importDataProvider(callWithRequest) { body.push(data[i]); } - const settings = { index, body }; + const settings: Settings = { index, body }; if (pipelineId !== undefined) { settings.pipeline = pipelineId; } - const resp = await callWithRequest('bulk', settings); + const resp = await callAsCurrentUser('bulk', settings); if (resp.errors) { throw resp; } else { @@ -113,7 +146,7 @@ export function importDataProvider(callWithRequest) { }; } } catch (error) { - let failures = []; + let failures: Failure[] = []; let ingestError = false; if (error.errors !== undefined && Array.isArray(error.items)) { // an expected error where some or all of the bulk request @@ -134,11 +167,11 @@ export function importDataProvider(callWithRequest) { } } - async function createPipeline(id, pipeline) { - return await callWithRequest('ingest.putPipeline', { id, body: pipeline }); + async function createPipeline(id: string, pipeline: any) { + return await callAsCurrentUser('ingest.putPipeline', { id, body: pipeline }); } - function getFailures(items, data) { + function getFailures(items: any[], data: InputData): Failure[] { const failures = []; for (let i = 0; i < items.length; i++) { const item = items[i]; diff --git a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.js b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.ts similarity index 53% rename from x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.js rename to x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.ts index 3bda5599e7181..94529dc111696 100644 --- a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.js +++ b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.ts @@ -4,5 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -export { fileDataVisualizerProvider } from './file_data_visualizer'; -export { importDataProvider } from './import_data'; +export { + fileDataVisualizerProvider, + InputOverrides, + InputData, + AnalysisResult, +} from './file_data_visualizer'; + +export { importDataProvider, Settings, InjectPipeline, Mappings } from './import_data'; diff --git a/x-pack/legacy/plugins/ml/server/new_platform/data_analytics_schema.ts b/x-pack/legacy/plugins/ml/server/new_platform/data_analytics_schema.ts index f5d72c51dc070..21454fa884b82 100644 --- a/x-pack/legacy/plugins/ml/server/new_platform/data_analytics_schema.ts +++ b/x-pack/legacy/plugins/ml/server/new_platform/data_analytics_schema.ts @@ -13,8 +13,16 @@ export const dataAnalyticsJobConfigSchema = { results_field: schema.maybe(schema.string()), }), source: schema.object({ - index: schema.string(), + index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]), + query: schema.maybe(schema.any()), + _source: schema.maybe( + schema.object({ + includes: schema.maybe(schema.arrayOf(schema.maybe(schema.string()))), + excludes: schema.maybe(schema.arrayOf(schema.maybe(schema.string()))), + }) + ), }), + allow_lazy_start: schema.maybe(schema.boolean()), analysis: schema.any(), analyzed_fields: schema.any(), model_memory_limit: schema.string(), diff --git a/x-pack/legacy/plugins/ml/server/routes/apidoc.json b/x-pack/legacy/plugins/ml/server/routes/apidoc.json index 574065446827d..1be31e2316228 100644 --- a/x-pack/legacy/plugins/ml/server/routes/apidoc.json +++ b/x-pack/legacy/plugins/ml/server/routes/apidoc.json @@ -3,7 +3,6 @@ "version": "0.1.0", "description": "ML Kibana API", "title": "ML Kibana API", - "url" : "/api/ml/", "order": [ "DataFrameAnalytics", "GetDataFrameAnalytics", @@ -34,6 +33,9 @@ "ForecastAnomalyDetector", "GetOverallBuckets", "GetCategories", + "FileDataVisualizer", + "AnalyzeFile", + "ImportFile" "ResultsService", "GetAnomaliesTableData", "GetCategoryDefinition", diff --git a/x-pack/legacy/plugins/ml/server/routes/data_frame_analytics.ts b/x-pack/legacy/plugins/ml/server/routes/data_frame_analytics.ts index 67fa2fba46f1a..f134820adbb48 100644 --- a/x-pack/legacy/plugins/ml/server/routes/data_frame_analytics.ts +++ b/x-pack/legacy/plugins/ml/server/routes/data_frame_analytics.ts @@ -156,7 +156,7 @@ export function dataFrameAnalyticsRoutes({ xpackMainPlugin, router }: RouteIniti params: schema.object({ analyticsId: schema.string(), }), - body: schema.object({ ...dataAnalyticsJobConfigSchema }), + body: schema.object(dataAnalyticsJobConfigSchema), }, }, licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { diff --git a/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.js b/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.js deleted file mode 100644 index fc6a0ff756928..0000000000000 --- a/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.js +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { callWithRequestFactory } from '../client/call_with_request_factory'; -import { wrapError } from '../client/errors'; -import { fileDataVisualizerProvider, importDataProvider } from '../models/file_data_visualizer'; -import { MAX_BYTES } from '../../common/constants/file_datavisualizer'; - -import { incrementFileDataVisualizerIndexCreationCount } from '../lib/ml_telemetry/ml_telemetry'; - -function analyzeFiles(callWithRequest, data, overrides) { - const { analyzeFile } = fileDataVisualizerProvider(callWithRequest); - return analyzeFile(data, overrides); -} - -function importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data) { - const { importData: importDataFunc } = importDataProvider(callWithRequest); - return importDataFunc(id, index, settings, mappings, ingestPipeline, data); -} - -export function fileDataVisualizerRoutes({ - commonRouteConfig, - elasticsearchPlugin, - route, - savedObjects, -}) { - route({ - method: 'POST', - path: '/api/ml/file_data_visualizer/analyze_file', - handler(request) { - const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); - const data = request.payload; - - return analyzeFiles(callWithRequest, data, request.query).catch(wrapError); - }, - config: { - ...commonRouteConfig, - payload: { maxBytes: MAX_BYTES }, - }, - }); - - route({ - method: 'POST', - path: '/api/ml/file_data_visualizer/import', - handler(request) { - const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); - const { id } = request.query; - const { index, data, settings, mappings, ingestPipeline } = request.payload; - - // `id` being `undefined` tells us that this is a new import due to create a new index. - // follow-up import calls to just add additional data will include the `id` of the created - // index, we'll ignore those and don't increment the counter. - if (id === undefined) { - incrementFileDataVisualizerIndexCreationCount(elasticsearchPlugin, savedObjects); - } - - return importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data).catch( - wrapError - ); - }, - config: { - ...commonRouteConfig, - payload: { maxBytes: MAX_BYTES }, - }, - }); -} diff --git a/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.ts b/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.ts new file mode 100644 index 0000000000000..95f2a9fe7298f --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.ts @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema } from '@kbn/config-schema'; +import { RequestHandlerContext } from 'kibana/server'; +import { MAX_BYTES } from '../../common/constants/file_datavisualizer'; +import { wrapError } from '../client/error_wrapper'; +import { + InputOverrides, + InputData, + fileDataVisualizerProvider, + importDataProvider, + Settings, + InjectPipeline, + Mappings, +} from '../models/file_data_visualizer'; + +import { licensePreRoutingFactory } from '../new_platform/licence_check_pre_routing_factory'; +import { RouteInitialization } from '../new_platform/plugin'; +import { incrementFileDataVisualizerIndexCreationCount } from '../lib/ml_telemetry'; + +function analyzeFiles(context: RequestHandlerContext, data: InputData, overrides: InputOverrides) { + const { analyzeFile } = fileDataVisualizerProvider(context); + return analyzeFile(data, overrides); +} + +function importData( + context: RequestHandlerContext, + id: string, + index: string, + settings: Settings, + mappings: Mappings, + ingestPipeline: InjectPipeline, + data: InputData +) { + const { importData: importDataFunc } = importDataProvider(context); + return importDataFunc(id, index, settings, mappings, ingestPipeline, data); +} + +/** + * Routes for the file data visualizer. + */ +export function fileDataVisualizerRoutes({ + router, + xpackMainPlugin, + savedObjects, + elasticsearchPlugin, +}: RouteInitialization) { + /** + * @apiGroup FileDataVisualizer + * + * @api {post} /api/ml/file_data_visualizer/analyze_file Analyze file data + * @apiName AnalyzeFile + * @apiDescription Performs analysis of the file data. + */ + router.post( + { + path: '/api/ml/file_data_visualizer/analyze_file', + validate: { + body: schema.any(), + query: schema.maybe( + schema.object({ + charset: schema.maybe(schema.string()), + column_names: schema.maybe(schema.string()), + delimiter: schema.maybe(schema.string()), + explain: schema.maybe(schema.string()), + format: schema.maybe(schema.string()), + grok_pattern: schema.maybe(schema.string()), + has_header_row: schema.maybe(schema.string()), + line_merge_size_limit: schema.maybe(schema.string()), + lines_to_sample: schema.maybe(schema.string()), + quote: schema.maybe(schema.string()), + should_trim_fields: schema.maybe(schema.string()), + timeout: schema.maybe(schema.string()), + timestamp_field: schema.maybe(schema.string()), + timestamp_format: schema.maybe(schema.string()), + }) + ), + }, + options: { + body: { + accepts: ['text/*', 'application/json'], + maxBytes: MAX_BYTES, + }, + }, + }, + licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { + try { + const result = await analyzeFiles(context, request.body, request.query); + return response.ok({ body: result }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + + /** + * @apiGroup FileDataVisualizer + * + * @api {post} /api/ml/file_data_visualizer/import Import file data + * @apiName ImportFile + * @apiDescription Imports file data into elasticsearch index. + */ + router.post( + { + path: '/api/ml/file_data_visualizer/import', + validate: { + query: schema.object({ + id: schema.maybe(schema.string()), + }), + body: schema.object({ + index: schema.maybe(schema.string()), + data: schema.arrayOf(schema.any()), + settings: schema.maybe(schema.any()), + mappings: schema.any(), + ingestPipeline: schema.object({ + id: schema.maybe(schema.string()), + pipeline: schema.maybe(schema.any()), + }), + }), + }, + options: { + body: { + accepts: ['application/json'], + maxBytes: MAX_BYTES, + }, + }, + }, + licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { + try { + const { id } = request.query; + const { index, data, settings, mappings, ingestPipeline } = request.body; + + // `id` being `undefined` tells us that this is a new import due to create a new index. + // follow-up import calls to just add additional data will include the `id` of the created + // index, we'll ignore those and don't increment the counter. + if (id === undefined) { + await incrementFileDataVisualizerIndexCreationCount(elasticsearchPlugin, savedObjects!); + } + + const result = await importData( + context, + id, + index, + settings, + mappings, + ingestPipeline, + data + ); + return response.ok({ body: result }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); +} diff --git a/x-pack/plugins/features/common/feature.ts b/x-pack/plugins/features/common/feature.ts index 423fe1eb99704..748076b95ad77 100644 --- a/x-pack/plugins/features/common/feature.ts +++ b/x-pack/plugins/features/common/feature.ts @@ -43,7 +43,7 @@ export interface Feature< * This does not restrict access to your feature based on license. * Its only purpose is to inform the space and roles UIs on which features to display. */ - validLicenses?: Array<'basic' | 'standard' | 'gold' | 'platinum' | 'enterprise'>; + validLicenses?: Array<'basic' | 'standard' | 'gold' | 'platinum' | 'enterprise' | 'trial'>; /** * An optional EUI Icon to be used when displaying your feature. diff --git a/x-pack/plugins/features/server/feature_schema.ts b/x-pack/plugins/features/server/feature_schema.ts index 7732686db5ee1..cc12ea1b78dce 100644 --- a/x-pack/plugins/features/server/feature_schema.ts +++ b/x-pack/plugins/features/server/feature_schema.ts @@ -51,7 +51,7 @@ const schema = Joi.object({ name: Joi.string().required(), excludeFromBasePrivileges: Joi.boolean(), validLicenses: Joi.array().items( - Joi.string().valid('basic', 'standard', 'gold', 'platinum', 'enterprise') + Joi.string().valid('basic', 'standard', 'gold', 'platinum', 'enterprise', 'trial') ), icon: Joi.string(), description: Joi.string(), diff --git a/x-pack/plugins/features/server/routes/index.test.ts b/x-pack/plugins/features/server/routes/index.test.ts index 98a23a61d542c..b0f8417b7175d 100644 --- a/x-pack/plugins/features/server/routes/index.test.ts +++ b/x-pack/plugins/features/server/routes/index.test.ts @@ -53,7 +53,7 @@ describe('GET /api/features', () => { it('returns a list of available features', async () => { const mockResponse = httpServerMock.createResponseFactory(); - routeHandler(undefined as any, undefined as any, mockResponse); + routeHandler(undefined as any, { query: {} } as any, mockResponse); expect(mockResponse.ok.mock.calls).toMatchInlineSnapshot(` Array [ @@ -84,11 +84,11 @@ describe('GET /api/features', () => { `); }); - it(`does not return features that arent allowed by current license`, async () => { + it(`by default does not return features that arent allowed by current license`, async () => { currentLicenseLevel = 'basic'; const mockResponse = httpServerMock.createResponseFactory(); - routeHandler(undefined as any, undefined as any, mockResponse); + routeHandler(undefined as any, { query: {} } as any, mockResponse); expect(mockResponse.ok.mock.calls).toMatchInlineSnapshot(` Array [ @@ -107,4 +107,63 @@ describe('GET /api/features', () => { ] `); }); + + it(`ignoreValidLicenses=false does not return features that arent allowed by current license`, async () => { + currentLicenseLevel = 'basic'; + + const mockResponse = httpServerMock.createResponseFactory(); + routeHandler(undefined as any, { query: { ignoreValidLicenses: false } } as any, mockResponse); + + expect(mockResponse.ok.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "body": Array [ + Object { + "app": Array [], + "id": "feature_1", + "name": "Feature 1", + "privileges": Object {}, + }, + ], + }, + ], + ] + `); + }); + + it(`ignoreValidLicenses=true returns features that arent allowed by current license`, async () => { + currentLicenseLevel = 'basic'; + + const mockResponse = httpServerMock.createResponseFactory(); + routeHandler(undefined as any, { query: { ignoreValidLicenses: true } } as any, mockResponse); + + expect(mockResponse.ok.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "body": Array [ + Object { + "app": Array [], + "id": "feature_1", + "name": "Feature 1", + "privileges": Object {}, + }, + Object { + "app": Array [ + "bar-app", + ], + "id": "licensed_feature", + "name": "Licensed Feature", + "privileges": Object {}, + "validLicenses": Array [ + "gold", + ], + }, + ], + }, + ], + ] + `); + }); }); diff --git a/x-pack/plugins/features/server/routes/index.ts b/x-pack/plugins/features/server/routes/index.ts index 51869c39cf83c..cf4d61ccac88b 100644 --- a/x-pack/plugins/features/server/routes/index.ts +++ b/x-pack/plugins/features/server/routes/index.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { schema } from '@kbn/config-schema'; import { IRouter } from '../../../../../src/core/server'; import { LegacyAPI } from '../plugin'; import { FeatureRegistry } from '../feature_registry'; @@ -19,13 +20,20 @@ export interface RouteDefinitionParams { export function defineRoutes({ router, featureRegistry, getLegacyAPI }: RouteDefinitionParams) { router.get( - { path: '/api/features', options: { tags: ['access:features'] }, validate: false }, + { + path: '/api/features', + options: { tags: ['access:features'] }, + validate: { + query: schema.object({ ignoreValidLicenses: schema.boolean({ defaultValue: false }) }), + }, + }, (context, request, response) => { const allFeatures = featureRegistry.getAll(); return response.ok({ body: allFeatures.filter( feature => + request.query.ignoreValidLicenses || !feature.validLicenses || !feature.validLicenses.length || getLegacyAPI().xpackInfo.license.isOneOf(feature.validLicenses) diff --git a/x-pack/test/ui_capabilities/common/services/features.ts b/x-pack/test/ui_capabilities/common/services/features.ts index 9f644fd6d0f6e..0f796c1d0a0cc 100644 --- a/x-pack/test/ui_capabilities/common/services/features.ts +++ b/x-pack/test/ui_capabilities/common/services/features.ts @@ -22,9 +22,11 @@ export class FeaturesService { }); } - public async get(): Promise { + public async get({ ignoreValidLicenses } = { ignoreValidLicenses: false }): Promise { this.log.debug('requesting /api/features to get the features'); - const response = await this.axios.get('/api/features'); + const response = await this.axios.get( + `/api/features?ignoreValidLicenses=${ignoreValidLicenses}` + ); if (response.status !== 200) { throw new Error( diff --git a/x-pack/test/ui_capabilities/spaces_only/tests/index.ts b/x-pack/test/ui_capabilities/spaces_only/tests/index.ts index 0b40f9716dcb4..a25838ac4f76d 100644 --- a/x-pack/test/ui_capabilities/spaces_only/tests/index.ts +++ b/x-pack/test/ui_capabilities/spaces_only/tests/index.ts @@ -16,7 +16,8 @@ export default function uiCapabilitesTests({ loadTestFile, getService }: FtrProv this.tags('ciGroup9'); before(async () => { - const features = await featuresService.get(); + // we're using a basic license, so if we want to disable all features, we have to ignore the valid licenses + const features = await featuresService.get({ ignoreValidLicenses: true }); for (const space of SpaceScenarios) { const disabledFeatures = space.disabledFeatures === '*' ? Object.keys(features) : space.disabledFeatures;