Merge branch 'master' of github.com:elastic/kibana into implement/kbn…

…-optimizer

docs/apm/troubleshooting.asciidoc

@@ -6,6 +6,7 @@ your proposed changes at https://github.com/elastic/kibana.
 
 Also check out the https://discuss.elastic.co/c/apm[APM discussion forum].
 
+[[no-apm-data-found]]
 ==== No APM data found
 
 This section can help with any of the following:
@@ -69,3 +70,41 @@ or because something is happening to the request that the Agent doesn't understa
 To resolve this, you'll need to head over to the relevant {apm-agents-ref}[Agent documentation].
 Specifically, view the Agent's supported technologies page.
 You can also use the Agent's public API to manually set a name for the transaction.
+
+==== Fields are not searchable
+
+In Elasticsearch, index patterns are used to define settings and mappings that determine how fields should be analyzed.
+The recommended index template file for APM Server is installed when Kibana starts.
+This template defines which fields are available in Kibana for features like the Kuery bar,
+or for linking to other plugins like Logs, Uptime, and Discover.
+
+As an example, some agents store cookie values in `http.request.cookies`.
+Since `http.request` has disabled dynamic indexing, and `http.request.cookies` is not declared in a custom mapping,
+the values in `http.request.cookies` are not indexed and thus not searchable.
+
+*Ensure an index pattern exists*
+As a first step, you should ensure the correct index pattern exists.
+In Kibana, navigate to *Management > Kibana > Index Patterns*.
+In the pattern list, you should see an apm index pattern; The default is `apm-*`.
+If you don't, the index pattern doesn't exist. See <<no-apm-data-found>> for information on how to fix this problem.
+
+Selecting the `apm-*` index pattern shows a listing of every field defined in the pattern.
+
+*Ensure a field is searchable*
+There are two things you can do to if you'd like to ensure a field is searchable:
+
+1. Index your additional data as {apm-overview-ref}/metadata.html[labels] instead.
+These are dynamic by default, which means they will be indexed and become searchable and aggregatable.
+
+2. Use the {apm-server-ref}/configuration-template.html[`append_fields`] feature. As an example,
+adding the following to `apm-server.yml` will enable dynamic indexing for `http.request.cookies`:
+
+[source,yml]
+----
+setup.template.enabled: true
+setup.template.overwrite: true
+setup.template.append_fields:
+  - name: http.request.cookies
+    type: object
+    dynamic: true
+----

src/optimize/base_optimizer.js

@@ -461,7 +461,7 @@ export default class BaseOptimizer {
       optimization: {
         minimizer: [
           new TerserPlugin({
-            parallel: this.getThreadLoaderPoolConfig().workers,
+            parallel: false,
             sourceMap: false,
             cache: false,
             extractComments: false,

src/optimize/dynamic_dll_plugin/dll_config_model.js

@@ -214,16 +214,20 @@ function common(config) {
   return webpackMerge(generateDLL(config));
 }
 
-function optimized(config) {
+function optimized() {
   return webpackMerge({
     mode: 'production',
     optimization: {
       minimizer: [
         new TerserPlugin({
-          // Apply the same logic used to calculate the
-          // threadLoaderPool workers number to spawn
-          // the parallel processes on terser
-          parallel: config.threadLoaderPoolConfig.workers,
+          // NOTE: we should not enable that option for now
+          // Since 2.0.0 terser-webpack-plugin is using jest-worker
+          // to run tasks in a pool of workers. Currently it looks like
+          // is requiring too much memory and break on large entry points
+          // compilations (like this) one. Also the gain we have enabling
+          // that option was barely noticed.
+          // https://github.com/webpack-contrib/terser-webpack-plugin/issues/143
+          parallel: false,
           sourceMap: false,
           cache: false,
           extractComments: false,
@@ -250,5 +254,5 @@ export function configModel(rawConfig = {}) {
     return webpackMerge(common(config), unoptimized());
   }
 
-  return webpackMerge(common(config), optimized(config));
+  return webpackMerge(common(config), optimized());
 }

x-pack/legacy/plugins/graph/index.ts

@@ -61,6 +61,7 @@ export const graph: LegacyPluginInitializer = kibana => {
         navLinkId: 'graph',
         app: ['graph', 'kibana'],
         catalogue: ['graph'],
+        validLicenses: ['platinum', 'enterprise', 'trial'],
         privileges: {
           all: {
             savedObject: {

x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import Boom from 'boom';
+import { RequestHandlerContext } from 'kibana/server';
+
+export type InputData = any[];
+
+export interface InputOverrides {
+  [key: string]: string;
+}
+
+export type FormattedOverrides = InputOverrides & {
+  column_names: string[];
+  has_header_row: boolean;
+  should_trim_fields: boolean;
+};
+
+export interface AnalysisResult {
+  results: {
+    charset: string;
+    has_header_row: boolean;
+    has_byte_order_marker: boolean;
+    format: string;
+    field_stats: {
+      [fieldName: string]: {
+        count: number;
+        cardinality: number;
+        top_hits: Array<{ count: number; value: any }>;
+      };
+    };
+    sample_start: string;
+    num_messages_analyzed: number;
+    mappings: {
+      [fieldName: string]: {
+        type: string;
+      };
+    };
+    quote: string;
+    delimiter: string;
+    need_client_timezone: boolean;
+    num_lines_analyzed: number;
+    column_names: string[];
+  };
+  overrides?: FormattedOverrides;
+}
+
+export function fileDataVisualizerProvider(context: RequestHandlerContext) {
+  async function analyzeFile(data: any, overrides: any): Promise<AnalysisResult> {
+    let results = [];
+
+    try {
+      results = await context.ml!.mlClient.callAsCurrentUser('ml.fileStructure', {
+        body: data,
+        ...overrides,
+      });
+    } catch (error) {
+      const err = error.message !== undefined ? error.message : error;
+      throw Boom.badRequest(err);
+    }
+
+    const { hasOverrides, reducedOverrides } = formatOverrides(overrides);
+
+    return {
+      ...(hasOverrides && { overrides: reducedOverrides }),
+      results,
+    };
+  }
+
+  return {
+    analyzeFile,
+  };
+}
+
+function formatOverrides(overrides: InputOverrides) {
+  let hasOverrides = false;
+
+  const reducedOverrides: FormattedOverrides = Object.keys(overrides).reduce((acc, overrideKey) => {
+    const overrideValue: string = overrides[overrideKey];
+    if (overrideValue !== '') {
+      if (overrideKey === 'column_names') {
+        acc.column_names = overrideValue.split(',');
+      } else if (overrideKey === 'has_header_row') {
+        acc.has_header_row = overrideValue === 'true';
+      } else if (overrideKey === 'should_trim_fields') {
+        acc.should_trim_fields = overrideValue === 'true';
+      } else {
+        acc[overrideKey] = overrideValue;
+      }
+
+      hasOverrides = true;
+    }
+    return acc;
+  }, {} as FormattedOverrides);
+
+  return {
+    reducedOverrides,
+    hasOverrides,
+  };
+}