Merge branch 'master' into poc/filtering

# Conflicts:
#	src/legacy/core_plugins/visualizations/public/embeddable/visualize_embeddable.ts
#	src/legacy/core_plugins/visualizations/public/np_ready/public/types/base_vis_type.js
#	src/legacy/ui/public/agg_types/index.ts

.ci/Jenkinsfile_coverage

@@ -40,16 +40,11 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
                 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
                 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
               ]),
-              'kibana-xpack-agent-1': kibanaPipeline.withWorkers('kibana-xpack-tests-1', { kibanaPipeline.buildXpack() }, [
+              'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
                 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
                 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2),
-              ]),
-              'kibana-xpack-agent-2': kibanaPipeline.withWorkers('kibana-xpack-tests-2', { kibanaPipeline.buildXpack() }, [
                 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3),
                 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4),
-              ]),
-
-              'kibana-xpack-agent-3': kibanaPipeline.withWorkers('kibana-xpack-tests-3', { kibanaPipeline.buildXpack() }, [
                 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5),
                 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6),
                 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7),
@@ -75,9 +70,7 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
                   echo extracting kibana-oss-tests
                   tar -xzf /tmp/downloaded_coverage/coverage/kibana-oss-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage
                   echo extracting kibana-xpack-tests
-                  for i in {1..3}; do
-                    tar -xzf /tmp/downloaded_coverage/coverage/kibana-xpack-tests-${i}/kibana-coverage.tar.gz -C /tmp/extracted_coverage
-                  done
+                  tar -xzf /tmp/downloaded_coverage/coverage/kibana-xpack-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage
                   # replace path in json files to have valid html report
                   pwd=$(pwd)
                   du -sh /tmp/extracted_coverage/target/kibana-coverage/

.eslintrc.js

@@ -88,6 +88,14 @@ module.exports = {
         'react-hooks/exhaustive-deps': 'off',
       },
     },
+    {
+      files: [
+        'src/legacy/core_plugins/vis_default_editor/public/components/controls/**/*.{ts,tsx}',
+      ],
+      rules: {
+        'react-hooks/exhaustive-deps': 'off',
+      },
+    },
     {
       files: ['src/legacy/ui/public/vis/**/*.{js,ts,tsx}'],
       rules: {

.github/CODEOWNERS

@@ -87,6 +87,7 @@
 /src/dev/ @elastic/kibana-operations
 /src/setup_node_env/ @elastic/kibana-operations
 /src/optimize/ @elastic/kibana-operations
+/src/es_archiver/ @elastic/kibana-operations
 /packages/*eslint*/ @elastic/kibana-operations
 /packages/*babel*/ @elastic/kibana-operations
 /packages/kbn-dev-utils*/ @elastic/kibana-operations
@@ -112,6 +113,7 @@
 /src/legacy/server/logging/ @elastic/kibana-platform
 /src/legacy/server/saved_objects/  @elastic/kibana-platform
 /src/legacy/server/status/ @elastic/kibana-platform
+/src/dev/run_check_core_api_changes.ts @elastic/kibana-platform
 
 # Security
 /src/core/server/csp/  @elastic/kibana-security @elastic/kibana-platform

.i18nrc.json

@@ -35,6 +35,7 @@
     "tileMap": "src/legacy/core_plugins/tile_map",
     "timelion": ["src/legacy/core_plugins/timelion", "src/legacy/core_plugins/vis_type_timelion", "src/plugins/timelion"],
     "uiActions": "src/plugins/ui_actions",
+    "visDefaultEditor": "src/legacy/core_plugins/vis_default_editor",
     "visTypeMarkdown": "src/legacy/core_plugins/vis_type_markdown",
     "visTypeMetric": "src/legacy/core_plugins/vis_type_metric",
     "visTypeTable": "src/legacy/core_plugins/vis_type_table",

NOTICE.txt

@@ -153,6 +153,40 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
+---
+This product bundles rules based on https://github.com/BlueTeamLabs/sentinel-attack
+which is available under a "MIT" license. The files based on this license are:
+
+- windows_defense_evasion_via_filter_manager.json
+- windows_process_discovery_via_tasklist_command.json
+- windows_priv_escalation_via_accessibility_features.json
+- windows_persistence_via_application_shimming.json
+- windows_execution_via_trusted_developer_utilities.json
+- windows_execution_via_net_com_assemblies.json
+- windows_execution_via_connection_manager.json
+
+MIT License
+
+Copyright (c) 2019 Edoardo Gerosa, Olaf Hartong
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 ---
 This product includes code that is adapted from mapbox-gl-js, which is
 available under a "BSD-3-Clause" license.

docs/migration/migrate_8_0.asciidoc

@@ -80,4 +80,15 @@ specified explicitly.
 
 *Impact:* Any workflow that involved manually clearing generated bundles will have to be updated with the new path.
 
+
+[float]
+[[breaking_80_reporting_changes]]
+=== Reporting changes
+
+[float]
+==== Legacy job parameters are no longer supported
+*Details:* POST URL snippets that were copied in Kibana 6.2 or below are no longer supported. These logs have
+been deprecated with warnings that have been logged throughout 7.x. Please use Kibana UI to re-generate the
+POST URL snippets if you depend on these for automated PDF reports.
+
 // end::notable-breaking-changes[]

src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts

@@ -108,8 +108,6 @@ describe('getSortedObjectsForExport()', () => {
               "namespace": undefined,
               "perPage": 500,
               "search": undefined,
-              "sortField": "_id",
-              "sortOrder": "asc",
               "type": Array [
                 "index-pattern",
                 "search",
@@ -256,8 +254,6 @@ describe('getSortedObjectsForExport()', () => {
               "namespace": undefined,
               "perPage": 500,
               "search": "foo",
-              "sortField": "_id",
-              "sortOrder": "asc",
               "type": Array [
                 "index-pattern",
                 "search",
@@ -345,8 +341,6 @@ describe('getSortedObjectsForExport()', () => {
               "namespace": "foo",
               "perPage": 500,
               "search": undefined,
-              "sortField": "_id",
-              "sortOrder": "asc",
               "type": Array [
                 "index-pattern",
                 "search",
@@ -399,6 +393,79 @@ describe('getSortedObjectsForExport()', () => {
     ).rejects.toThrowErrorMatchingInlineSnapshot(`"Can't export more than 1 objects"`);
   });
 
+  test('sorts objects within type', async () => {
+    savedObjectsClient.find.mockResolvedValueOnce({
+      total: 3,
+      per_page: 10000,
+      page: 1,
+      saved_objects: [
+        {
+          id: '3',
+          type: 'index-pattern',
+          attributes: {
+            name: 'baz',
+          },
+          references: [],
+        },
+        {
+          id: '1',
+          type: 'index-pattern',
+          attributes: {
+            name: 'foo',
+          },
+          references: [],
+        },
+        {
+          id: '2',
+          type: 'index-pattern',
+          attributes: {
+            name: 'bar',
+          },
+          references: [],
+        },
+      ],
+    });
+    const exportStream = await getSortedObjectsForExport({
+      exportSizeLimit: 10000,
+      savedObjectsClient,
+      types: ['index-pattern'],
+    });
+    const response = await readStreamToCompletion(exportStream);
+    expect(response).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "attributes": Object {
+            "name": "foo",
+          },
+          "id": "1",
+          "references": Array [],
+          "type": "index-pattern",
+        },
+        Object {
+          "attributes": Object {
+            "name": "bar",
+          },
+          "id": "2",
+          "references": Array [],
+          "type": "index-pattern",
+        },
+        Object {
+          "attributes": Object {
+            "name": "baz",
+          },
+          "id": "3",
+          "references": Array [],
+          "type": "index-pattern",
+        },
+        Object {
+          "exportedCount": 3,
+          "missingRefCount": 0,
+          "missingReferences": Array [],
+        },
+      ]
+    `);
+  });
+
   test('exports selected objects and sorts them', async () => {
     savedObjectsClient.bulkGet.mockResolvedValueOnce({
       saved_objects: [

src/core/server/saved_objects/export/get_sorted_objects_for_export.ts

@@ -19,7 +19,7 @@
 
 import Boom from 'boom';
 import { createListStream } from '../../../../legacy/utils/streams';
-import { SavedObjectsClientContract } from '../types';
+import { SavedObjectsClientContract, SavedObject } from '../types';
 import { fetchNestedDependencies } from './inject_nested_depdendencies';
 import { sortObjects } from './sort_objects';
 
@@ -105,15 +105,17 @@ async function fetchObjectsToExport({
     const findResponse = await savedObjectsClient.find({
       type: types,
       search,
-      sortField: '_id',
-      sortOrder: 'asc',
       perPage: exportSizeLimit,
       namespace,
     });
     if (findResponse.total > exportSizeLimit) {
       throw Boom.badRequest(`Can't export more than ${exportSizeLimit} objects`);
     }
-    return findResponse.saved_objects;
+
+    // sorts server-side by _id, since it's only available in fielddata
+    return findResponse.saved_objects.sort((a: SavedObject, b: SavedObject) =>
+      a.id > b.id ? 1 : -1
+    );
   } else {
     throw Boom.badRequest('Either `type` or `objects` are required.');
   }
@@ -137,14 +139,17 @@ export async function getSortedObjectsForExport({
     exportSizeLimit,
     namespace,
   });
-  let exportedObjects = [...rootObjects];
+  let exportedObjects = [];
   let missingReferences: SavedObjectsExportResultDetails['missingReferences'] = [];
+
   if (includeReferencesDeep) {
     const fetchResult = await fetchNestedDependencies(rootObjects, savedObjectsClient, namespace);
-    exportedObjects = fetchResult.objects;
+    exportedObjects = sortObjects(fetchResult.objects);
     missingReferences = fetchResult.missingRefs;
+  } else {
+    exportedObjects = sortObjects(rootObjects);
   }
-  exportedObjects = sortObjects(exportedObjects);
+
   const exportDetails: SavedObjectsExportResultDetails = {
     exportedCount: exportedObjects.length,
     missingRefCount: missingReferences.length,

src/dev/run_check_lockfile_symlinks.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { existsSync, lstatSync, readFileSync } from 'fs';
+import { existsSync, lstatSync, readFileSync, readlinkSync } from 'fs';
 import globby from 'globby';
 import { dirname } from 'path';
 
@@ -63,6 +63,7 @@ async function checkLockfileSymlinks(log, files) {
   await checkOnlyLockfileAtProjectRoot(filtered);
   await checkSuperfluousSymlinks(log, filtered);
   await checkMissingSymlinks(log, filtered);
+  await checkIncorrectSymlinks(log, filtered);
 }
 
 async function checkOnlyLockfileAtProjectRoot(files) {
@@ -157,8 +158,9 @@ async function checkMissingSymlinks(log, files) {
         try {
           const json = JSON.parse(manifest);
           if (json.dependencies && Object.keys(json.dependencies).length) {
+            const correctSymlink = getCorrectSymlink(lockfilePath);
             log.warning(
-              `Manifest at '${path}' has dependencies, but did not find an adjacent 'yarn.lock' symlink.`
+              `Manifest at '${path}' has dependencies, but did not find an adjacent 'yarn.lock' symlink to '${correctSymlink}'.`
             );
             errorPaths.push(`${parent}/yarn.lock`);
           }
@@ -177,6 +179,42 @@ async function checkMissingSymlinks(log, files) {
   }
 }
 
+async function checkIncorrectSymlinks(log, files) {
+  const errorPaths = [];
+
+  files
+    .filter(file => matchesAnyGlob(file.getRelativePath(), LOCKFILE_GLOBS))
+    .forEach(file => {
+      const path = file.getRelativePath();
+      const stats = lstatSync(path);
+      if (!stats.isSymbolicLink()) {
+        return;
+      }
+
+      const symlink = readlinkSync(path);
+      const correctSymlink = getCorrectSymlink(path);
+      if (symlink !== correctSymlink) {
+        log.warning(
+          `Symlink at '${path}' points to '${symlink}', but it should point to '${correctSymlink}'.`
+        );
+        errorPaths.push(path);
+      }
+    });
+
+  if (errorPaths.length) {
+    throw createFailError(
+      `These symlinks do NOT point to the 'yarn.lock' file in the project root:\n${listPaths(
+        errorPaths
+      )}`
+    );
+  }
+}
+
+function getCorrectSymlink(path) {
+  const count = path.split('/').length - 1;
+  return `${'../'.repeat(count)}yarn.lock`;
+}
+
 function listPaths(paths) {
   return paths.map(path => ` - ${path}`).join('\n');
 }

src/legacy/core_plugins/input_control_vis/public/components/editor/controls_tab.tsx

@@ -30,6 +30,8 @@ import {
   EuiSelect,
 } from '@elastic/eui';
 
+import { VisOptionsProps } from 'src/legacy/core_plugins/vis_default_editor/public';
+import { IIndexPattern } from 'src/plugins/data/public';
 import { ControlEditor } from './control_editor';
 import {
   addControl,
@@ -42,8 +44,6 @@ import {
   ControlParamsOptions,
 } from '../../editor_utils';
 import { getLineageMap, getParentCandidates } from '../../lineage';
-import { IIndexPattern } from '../../../../../../plugins/data/public';
-import { VisOptionsProps } from '../../legacy_imports';
 import { InputControlVisDependencies } from '../../plugin';
 
 interface ControlsTabUiState {

src/legacy/core_plugins/input_control_vis/public/components/editor/options_tab.test.tsx

@@ -21,8 +21,8 @@ import React from 'react';
 import { shallow } from 'enzyme';
 import { mountWithIntl } from 'test_utils/enzyme_helpers';
 
+import { Vis } from 'src/legacy/core_plugins/visualizations/public';
 import { OptionsTab, OptionsTabProps } from './options_tab';
-import { Vis } from '../../legacy_imports';
 
 describe('OptionsTab', () => {
   let props: OptionsTabProps;

src/legacy/core_plugins/input_control_vis/public/components/editor/options_tab.tsx

@@ -23,7 +23,7 @@ import { EuiForm, EuiFormRow, EuiSwitch } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { EuiSwitchEvent } from '@elastic/eui';
 
-import { VisOptionsProps } from '../../legacy_imports';
+import { VisOptionsProps } from 'src/legacy/core_plugins/vis_default_editor/public';
 
 interface OptionsTabParams {
   updateFiltersOnChange: boolean;

src/legacy/core_plugins/input_control_vis/public/legacy_imports.ts

@@ -22,8 +22,6 @@ import { SearchSource as SearchSourceClass, ISearchSource } from '../../../../pl
 
 export { SearchSourceFields } from '../../../../plugins/data/public';
 
-export { Vis, VisParams } from 'ui/vis';
-export { VisOptionsProps } from 'ui/vis/editors/default';
 export { ValidatedDualRange } from 'ui/validated_range';
 
 export type SearchSource = Class<ISearchSource>;