Merge branch '7.x' into feat/disableable-plugins-7.x

.buildkite/pipelines/hourly.yml

@@ -1,3 +1,5 @@
+env:
+  REPORT_FAILED_TESTS_TO_GITHUB: 'true'
 steps:
   - command: .buildkite/scripts/lifecycle/pre_build.sh
     label: Pre-Build

docs/settings/fleet-settings.asciidoc

@@ -87,6 +87,8 @@ Optional properties are:
   be changed by updating the {kib} config.
   `is_default`:: If `true`, this policy is the default agent policy.
   `is_default_fleet_server`:: If `true`, this policy is the default {fleet-server} agent policy.
+  `data_output_id`:: ID of the output to send data (Need to be identical to `monitoring_output_id`)
+  `monitoring_output_id`:: ID of the output to send monitoring data. (Need to be identical to `data_output_id`)
   `package_policies`:: List of integration policies to add to this policy.
     `name`::: (required) Name of the integration policy.
     `package`::: (required) Integration that this policy configures
@@ -97,6 +99,20 @@ Optional properties are:
     integration. Follows the same schema as integration inputs, with the
     exception that any object in `vars` can be passed `frozen: true` in order to
     prevent that specific `var` from being edited by the user.
+
+| `xpack.fleet.outputs`
+  | List of ouputs that are configured when the {fleet} app starts.
+Required properties are:
+
+  `id`:: Unique ID for this output. The ID should be a string.
+  `name`:: Output name.
+  `type`:: Type of Output. Currently we only support "elasticsearch".
+  `hosts`:: Array that contains the list of host for that output.
+  `config`:: Extra config for that output.
+
+Optional properties are:
+
+  `is_default`:: If `true`, this output is the default output.
 |===
 
 Example configuration:

packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts

@@ -37,30 +37,31 @@ export function runFailedTestsReporterCli() {
       }
 
       if (updateGithub) {
-        let branch: string | undefined = '';
+        let branch: string = '';
         let isPr = false;
 
         if (process.env.BUILDKITE === 'true') {
-          branch = process.env.BUILDKITE_BRANCH;
+          branch = process.env.BUILDKITE_BRANCH || '';
           isPr = process.env.BUILDKITE_PULL_REQUEST === 'true';
+          updateGithub = process.env.REPORT_FAILED_TESTS_TO_GITHUB === 'true';
         } else {
           // JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others
           const jobNameSplit = (process.env.JOB_NAME || '').split(/\+|\//);
-          branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH;
+          branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH || '';
           isPr = !!process.env.ghprbPullId;
+
+          const isMasterOrVersion = branch === 'master' || branch.match(/^\d+\.(x|\d+)$/);
+          if (!isMasterOrVersion || isPr) {
+            log.info('Failure issues only created on master/version branch jobs');
+            updateGithub = false;
+          }
         }
 
         if (!branch) {
           throw createFailError(
             'Unable to determine originating branch from job name or other environment variables'
           );
         }
-
-        const isMasterOrVersion = branch === 'master' || branch.match(/^\d+\.(x|\d+)$/);
-        if (!isMasterOrVersion || isPr) {
-          log.info('Failure issues only created on master/version branch jobs');
-          updateGithub = false;
-        }
       }
 
       const githubApi = new GithubApi({

src/core/server/saved_objects/migrations/core/migration_context.test.ts

@@ -74,4 +74,18 @@ describe('disableUnknownTypeMappingFields', () => {
       },
     });
   });
+
+  it('does not fail if the source mapping does not have `properties` defined', () => {
+    const missingPropertiesMappings = {
+      ...sourceMappings,
+      properties: undefined,
+    };
+    const result = disableUnknownTypeMappingFields(
+      activeMappings,
+      // @ts-expect-error `properties` should not be undefined
+      missingPropertiesMappings
+    );
+
+    expect(Object.keys(result.properties)).toEqual(['known_type']);
+  });
 });

src/core/server/saved_objects/migrations/core/migration_context.ts

@@ -154,7 +154,7 @@ export function disableUnknownTypeMappingFields(
 ): IndexMapping {
   const targetTypes = Object.keys(activeMappings.properties);
 
-  const disabledTypesProperties = Object.keys(sourceMappings.properties)
+  const disabledTypesProperties = Object.keys(sourceMappings.properties ?? {})
     .filter((sourceType) => {
       const isObjectType = 'properties' in sourceMappings.properties[sourceType];
       // Only Object/Nested datatypes can be excluded from the field count by

src/plugins/dashboard/public/application/hooks/use_dashboard_app_state.ts

@@ -237,27 +237,25 @@ export const useDashboardAppState = ({
         .pipe(debounceTime(DashboardConstants.CHANGE_CHECK_DEBOUNCE))
         .subscribe((states) => {
           const [lastSaved, current] = states;
-          const unsavedChanges =
-            current.viewMode === ViewMode.EDIT ? diffDashboardState(lastSaved, current) : {};
-
-          let savedTimeChanged = false;
+          const unsavedChanges = diffDashboardState(lastSaved, current);
+
+          const savedTimeChanged =
+            lastSaved.timeRestore &&
+            !areTimeRangesEqual(
+              {
+                from: savedDashboard?.timeFrom,
+                to: savedDashboard?.timeTo,
+              },
+              timefilter.getTime()
+            );
 
           /**
-           * changes to the time filter should only be considered 'unsaved changes' when
+           * changes to the dashboard should only be considered 'unsaved changes' when
            * editing the dashboard
            */
-          if (current.viewMode === ViewMode.EDIT) {
-            savedTimeChanged =
-              lastSaved.timeRestore &&
-              !areTimeRangesEqual(
-                {
-                  from: savedDashboard?.timeFrom,
-                  to: savedDashboard?.timeTo,
-                },
-                timefilter.getTime()
-              );
-          }
-          const hasUnsavedChanges = Object.keys(unsavedChanges).length > 0 || savedTimeChanged;
+          const hasUnsavedChanges =
+            current.viewMode === ViewMode.EDIT &&
+            (Object.keys(unsavedChanges).length > 0 || savedTimeChanged);
           setDashboardAppState((s) => ({ ...s, hasUnsavedChanges }));
 
           unsavedChanges.viewMode = current.viewMode; // always push view mode into session store.

src/plugins/dashboard/public/application/lib/dashboard_session_storage.ts

@@ -11,6 +11,7 @@ import { Storage } from '../../services/kibana_utils';
 import { NotificationsStart } from '../../services/core';
 import { panelStorageErrorStrings } from '../../dashboard_strings';
 import { DashboardState } from '../../types';
+import { ViewMode } from '../../services/embeddable';
 
 export const DASHBOARD_PANELS_UNSAVED_ID = 'unsavedDashboard';
 const DASHBOARD_PANELS_SESSION_KEY = 'dashboardStateManagerPanels';
@@ -69,6 +70,7 @@ export class DashboardSessionStorage {
       const dashboardsWithUnsavedChanges: string[] = [];
       Object.keys(dashboardStatesInSpace).map((dashboardId) => {
         if (
+          dashboardStatesInSpace[dashboardId].viewMode === ViewMode.EDIT &&
           Object.keys(dashboardStatesInSpace[dashboardId]).some(
             (stateKey) => stateKey !== 'viewMode'
           )

src/plugins/dashboard/public/application/listing/dashboard_no_match.tsx

@@ -23,9 +23,8 @@ export const DashboardNoMatch = ({ history }: { history: RouteComponentProps['hi
 
   useEffect(() => {
     services.restorePreviousUrl();
-
     const { navigated } = services.urlForwarding.navigateToLegacyKibanaUrl(
-      history.location.pathname
+      history.location.pathname + history.location.search
     );
 
     if (!navigated) {

src/plugins/data/common/data_views/data_views/_pattern_cache.ts

@@ -16,12 +16,12 @@ export interface DataViewCache {
 }
 
 export function createDataViewCache(): DataViewCache {
-  const vals: Record<string, any> = {};
+  const vals: Record<string, Promise<DataView>> = {};
   const cache: DataViewCache = {
     get: (id: string) => {
       return vals[id];
     },
-    set: (id: string, prom: any) => {
+    set: (id: string, prom: Promise<DataView>) => {
       vals[id] = prom;
       return prom;
     },

src/plugins/data/common/data_views/data_views/data_view.ts

@@ -158,7 +158,7 @@ export class DataView implements IIndexPattern {
   };
 
   getComputedFields() {
-    const scriptFields: any = {};
+    const scriptFields: Record<string, { script: { source: string; lang: string } }> = {};
     if (!this.fields) {
       return {
         storedFields: ['*'],
@@ -171,23 +171,21 @@ export class DataView implements IIndexPattern {
     // Date value returned in "_source" could be in any number of formats
     // Use a docvalue for each date field to ensure standardized formats when working with date fields
     // indexPattern.flattenHit will override "_source" values when the same field is also defined in "fields"
-    const docvalueFields = reject(this.fields.getByType('date'), 'scripted').map(
-      (dateField: any) => {
-        return {
-          field: dateField.name,
-          format:
-            dateField.esTypes && dateField.esTypes.indexOf('date_nanos') !== -1
-              ? 'strict_date_time'
-              : 'date_time',
-        };
-      }
-    );
+    const docvalueFields = reject(this.fields.getByType('date'), 'scripted').map((dateField) => {
+      return {
+        field: dateField.name,
+        format:
+          dateField.esTypes && dateField.esTypes.indexOf('date_nanos') !== -1
+            ? 'strict_date_time'
+            : 'date_time',
+      };
+    });
 
     each(this.getScriptedFields(), function (field) {
       scriptFields[field.name] = {
         script: {
-          source: field.script,
-          lang: field.lang,
+          source: field.script as string,
+          lang: field.lang as string,
         },
       };
     });
@@ -227,7 +225,7 @@ export class DataView implements IIndexPattern {
    */
   getSourceFiltering() {
     return {
-      excludes: (this.sourceFilters && this.sourceFilters.map((filter: any) => filter.value)) || [],
+      excludes: (this.sourceFilters && this.sourceFilters.map((filter) => filter.value)) || [],
     };
   }
 
@@ -322,8 +320,8 @@ export class DataView implements IIndexPattern {
   }
 
   isTimeNanosBased(): boolean {
-    const timeField: any = this.getTimeField();
-    return timeField && timeField.esTypes && timeField.esTypes.indexOf('date_nanos') !== -1;
+    const timeField = this.getTimeField();
+    return !!(timeField && timeField.esTypes && timeField.esTypes.indexOf('date_nanos') !== -1);
   }
 
   getTimeField() {

src/plugins/data/common/data_views/data_views/data_views.ts

@@ -293,7 +293,7 @@ export class DataViewsService {
       indexPattern.fields.replaceAll(fieldsWithSavedAttrs);
     } catch (err) {
       if (err instanceof DataViewMissingIndices) {
-        this.onNotification({ title: (err as any).message, color: 'danger', iconType: 'alert' });
+        this.onNotification({ title: err.message, color: 'danger', iconType: 'alert' });
       }
 
       this.onError(err, {
@@ -338,7 +338,7 @@ export class DataViewsService {
       return this.fieldArrayToMap(updatedFieldList, fieldAttrs);
     } catch (err) {
       if (err instanceof DataViewMissingIndices) {
-        this.onNotification({ title: (err as any).message, color: 'danger', iconType: 'alert' });
+        this.onNotification({ title: err.message, color: 'danger', iconType: 'alert' });
         return {};
       }
 
@@ -479,7 +479,7 @@ export class DataViewsService {
     } catch (err) {
       if (err instanceof DataViewMissingIndices) {
         this.onNotification({
-          title: (err as any).message,
+          title: err.message,
           color: 'danger',
           iconType: 'alert',
         });

src/plugins/data/common/data_views/lib/get_title.ts

@@ -6,17 +6,18 @@
  * Side Public License, v 1.
  */
 
-import { SavedObjectsClientContract, SimpleSavedObject } from '../../../../../core/public';
+import { SavedObjectsClientContract } from '../../../../../core/public';
 import { DATA_VIEW_SAVED_OBJECT_TYPE } from '../../constants';
+import { DataViewAttributes } from '../types';
 
 export async function getTitle(
   client: SavedObjectsClientContract,
   indexPatternId: string
-): Promise<SimpleSavedObject<any>> {
-  const savedObject = (await client.get(
+): Promise<string> {
+  const savedObject = await client.get<DataViewAttributes>(
     DATA_VIEW_SAVED_OBJECT_TYPE,
     indexPatternId
-  )) as SimpleSavedObject<any>;
+  );
 
   if (savedObject.error) {
     throw new Error(`Unable to get index-pattern title: ${savedObject.error.message}`);

src/plugins/data/common/data_views/lib/validate_index_pattern.ts

@@ -24,7 +24,7 @@ function findIllegalCharacters(indexPattern: string): string[] {
 }
 
 export function validateDataView(indexPattern: string) {
-  const errors: Record<string, any> = {};
+  const errors: { [ILLEGAL_CHARACTERS_KEY]?: string[]; [CONTAINS_SPACES_KEY]?: boolean } = {};
 
   const illegalCharacters = findIllegalCharacters(indexPattern);
 

src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts

@@ -377,6 +377,86 @@ describe('Terms Agg Other bucket helper', () => {
       }
     });
 
+    test('correctly builds query for nested terms agg with one disabled', () => {
+      const oneDisabledNestedTerms = {
+        aggs: [
+          {
+            id: '2',
+            type: BUCKET_TYPES.TERMS,
+            enabled: false,
+            params: {
+              field: {
+                name: 'machine.os.raw',
+                indexPattern,
+                filterable: true,
+              },
+              size: 2,
+              otherBucket: false,
+              missingBucket: true,
+            },
+          },
+          {
+            id: '1',
+            type: BUCKET_TYPES.TERMS,
+            params: {
+              field: {
+                name: 'geo.src',
+                indexPattern,
+                filterable: true,
+              },
+              size: 2,
+              otherBucket: true,
+              missingBucket: false,
+            },
+          },
+        ],
+      };
+      const aggConfigs = getAggConfigs(oneDisabledNestedTerms.aggs);
+      const agg = buildOtherBucketAgg(
+        aggConfigs,
+        aggConfigs.aggs[1] as IBucketAggConfig,
+        singleTermResponse
+      );
+      const expectedResponse = {
+        'other-filter': {
+          aggs: undefined,
+          filters: {
+            filters: {
+              '': {
+                bool: {
+                  filter: [
+                    {
+                      exists: {
+                        field: 'geo.src',
+                      },
+                    },
+                  ],
+                  must: [],
+                  must_not: [
+                    {
+                      match_phrase: {
+                        'geo.src': 'ios',
+                      },
+                    },
+                    {
+                      match_phrase: {
+                        'geo.src': 'win xp',
+                      },
+                    },
+                  ],
+                  should: [],
+                },
+              },
+            },
+          },
+        },
+      };
+      expect(agg).toBeDefined();
+      if (agg) {
+        expect(agg()).toEqual(expectedResponse);
+      }
+    });
+
     test('does not build query if sum_other_doc_count is 0 (exhaustive terms)', () => {
       const aggConfigs = getAggConfigs(nestedTerm.aggs);
       expect(