Merge branch '8.10' into backport/8.10/pr-164122

docs/user/ml/index.asciidoc

@@ -31,6 +31,34 @@ privileges to use {ml-features}. Refer to
 NOTE: There are limitations in {ml-features} that affect {kib}. For more 
 information, refer to {ml-docs}/ml-limitations.html[{ml-cap}].
 
+[discrete]
+[[data-comparison-view]]
+== Data comparison
+
+preview::[]
+
+You can find the data comparison view in **{ml-app}** > *{data-viz}* in {kib}. 
+The data comparison view shows you the differences in each field for two 
+different time ranges in a given {data-source}. The view helps you to visualize 
+the changes in your data over time and enables you to understand its behavior 
+better.
+
+[role="screenshot"]
+image::user/ml/images/ml-data-comparison.png[Data comparison view for {kib} Sample Data Flights ]
+
+Select a {data-source} that you want to analyze, then select a time range for 
+the reference and the comparison data in the appearing histogram chart. You can 
+adjust the time range for both the reference and the comparison data by moving 
+the respective brushes. When you finished setting the time ranges, click 
+*Run analysis*. 
+
+You can decide whether you want to see all the fields in the {data-source} or 
+only the ones that contains drifted data. The analysis results table displays 
+the fields, their types, if drift is detected, the p-value that indicates how 
+significant the detected change is, the reference and comparison distribution, 
+and the comparison chart. You can expand the results for a particular field by 
+clicking the arrow icon at the beginning of the field's row. 
+
 --
 
 [[xpack-ml-anomalies]]
@@ -123,11 +151,11 @@ advanced statistical methods to help you interpret your data and its behavior.
 preview::[]
 
 Log rate analysis is a feature that uses advanced statistical methods to 
-identify reasons for increases or decreases in log rates. It makes it easy to find and 
-investigate causes of unusual spikes or drops by using the analysis workflow view. 
-Examine the histogram chart of the log rates for a given {data-source}, and find 
-the reason behind a particular change possibly in millions of log events across 
-multiple fields and values.
+identify reasons for increases or decreases in log rates. It makes it easy to 
+find and investigate causes of unusual spikes or drops by using the analysis 
+workflow view. Examine the histogram chart of the log rates for a given 
+{data-source}, and find the reason behind a particular change possibly in 
+millions of log events across multiple fields and values.
 
 You can find log rate analysis under **{ml-app}** > **AIOps Labs** where 
 you can select the {data-source} or saved search that you want to analyze.

docs/user/production-considerations/alerting-production-considerations.asciidoc

@@ -56,10 +56,14 @@ Predicting the buffer required to account for actions depends heavily on the rul
 
 experimental[]
 
-Alerts and actions log activity in a set of "event log" data streams, one per Kibana version, named `.kibana-event-log-{VERSION}`.  These data streams are configured with a lifecycle data retention of 90 days. This can be updated to other values via the standard data stream lifecycle APIs.  Note that the event log data contains the data shown in the alerting pages in {kib}, so reducing the data retention period will result in less data being available to view.
+Alerts and actions log activity in a set of "event log" indices.  These indices are configured with an index lifecycle management (ILM) policy, which you can customize.  The default policy rolls over the index when it reaches 50GB, or after 30 days.  Indices over 90 days old are deleted.
 
-For more information on data stream lifecycle management, see:
-{ref}/data-stream-lifecycle.html[Data stream lifecycle].
+The name of the index policy is `kibana-event-log-policy`.  {kib} creates the index policy on startup, if it doesn't already exist.  The index policy can be customized for your environment, but {kib} never modifies the index policy after creating it.
+
+Because {kib} uses the documents to display historic data, you should set the delete phase longer than you would like the historic data to be shown. For example, if you would like to see one month's worth of historic data, you should set the delete phase to at least one month.
+
+For more information on index lifecycle management, see:
+{ref}/index-lifecycle-management.html[Index Lifecycle Policies].
 
 [float]
 [[alerting-circuit-breakers]]

fleet_packages.json

@@ -58,6 +58,6 @@
   },
   {
     "name": "security_detection_engine",
-    "version": "8.9.3"
+    "version": "8.9.4"
   }
 ]

packages/core/fatal-errors/core-fatal-errors-browser-internal/src/fatal_errors_screen.tsx

@@ -75,7 +75,10 @@ export class FatalErrorsScreen extends React.Component<Props, State> {
 
   public render() {
     return (
-      <EuiPage style={{ minHeight: '100vh', alignItems: 'center' }}>
+      <EuiPage
+        style={{ minHeight: '100vh', alignItems: 'center' }}
+        data-test-subj="fatalErrorScreen"
+      >
         <EuiPageBody>
           <EuiPageSection alignment="center">
             <EuiEmptyPrompt

packages/core/fatal-errors/core-fatal-errors-browser-internal/src/fatal_errors_service.tsx

@@ -109,7 +109,11 @@ export class FatalErrorsService {
   private setupGlobalErrorHandlers() {
     if (window.addEventListener) {
       window.addEventListener('unhandledrejection', (e) => {
-        console.log(`Detected an unhandled Promise rejection.\n${e.reason}`); // eslint-disable-line no-console
+        const { message, stack } = getErrorInfo(e.reason);
+        // eslint-disable-next-line no-console
+        console.log(`Detected an unhandled Promise rejection.\n
+        Message: ${message}\n
+        Stack: ${stack}`);
       });
     }
   }

packages/core/logging/core-logging-server-internal/src/appenders/rolling_file/strategies/numeric/rolling_tasks.ts

@@ -7,8 +7,9 @@
  */
 
 import { join } from 'path';
-import { readdir, rename, unlink, access } from 'fs/promises';
+import { readdir, unlink, access } from 'fs/promises';
 import { getFileNameMatcher, getRollingFileName } from './pattern_matcher';
+import { moveFile } from './utils';
 
 export const shouldSkipRollout = async ({ logFilePath }: { logFilePath: string }) => {
   // in case of time-interval triggering policy, we can have an entire
@@ -70,7 +71,7 @@ export const rollPreviousFilesInOrder = async ({
   for (let i = filesToRoll.length - 1; i >= 0; i--) {
     const oldFileName = filesToRoll[i];
     const newFileName = getRollingFileName(logFileBaseName, pattern, i + 2);
-    await rename(join(logFileFolder, oldFileName), join(logFileFolder, newFileName));
+    await moveFile(join(logFileFolder, oldFileName), join(logFileFolder, newFileName));
   }
 };
 
@@ -84,5 +85,5 @@ export const rollCurrentFile = async ({
   pattern: string;
 }) => {
   const rolledBaseName = getRollingFileName(logFileBaseName, pattern, 1);
-  await rename(join(logFileFolder, logFileBaseName), join(logFileFolder, rolledBaseName));
+  await moveFile(join(logFileFolder, logFileBaseName), join(logFileFolder, rolledBaseName));
 };

packages/core/logging/core-logging-server-internal/src/appenders/rolling_file/strategies/numeric/utils.test.mocks.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const copyFileMock = jest.fn();
+export const renameMock = jest.fn();
+export const unlinkMock = jest.fn();
+
+jest.doMock('fs/promises', () => {
+  const actual = jest.requireActual('fs/promises');
+  return {
+    ...actual,
+    copyFile: copyFileMock,
+    rename: renameMock,
+    unlink: unlinkMock,
+  };
+});

packages/core/logging/core-logging-server-internal/src/appenders/rolling_file/strategies/numeric/utils.test.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { unlinkMock, renameMock, copyFileMock } from './utils.test.mocks';
+import { moveFile } from './utils';
+
+describe('moveFile', () => {
+  beforeEach(() => {
+    unlinkMock.mockReset();
+    renameMock.mockReset();
+    copyFileMock.mockReset();
+  });
+
+  it('only calls `rename` when call succeeds', async () => {
+    await moveFile('from', 'to');
+
+    expect(renameMock).toHaveBeenCalledTimes(1);
+    expect(renameMock).toHaveBeenCalledWith('from', 'to');
+
+    expect(copyFileMock).not.toHaveBeenCalled();
+    expect(unlinkMock).not.toHaveBeenCalled();
+  });
+
+  const createError = (code: string) => {
+    const err = new Error(code);
+    (err as any).code = code;
+    return err;
+  };
+
+  it('throws error if `rename` throws a non-EXDEV error', async () => {
+    renameMock.mockRejectedValue(createError('something'));
+
+    await expect(moveFile('from', 'to')).rejects.toThrowError('something');
+
+    expect(renameMock).toHaveBeenCalledTimes(1);
+    expect(copyFileMock).not.toHaveBeenCalled();
+    expect(unlinkMock).not.toHaveBeenCalled();
+  });
+
+  it('fallback to copy+unlink when `rename` throws a EXDEV error', async () => {
+    renameMock.mockRejectedValue(createError('EXDEV'));
+
+    await moveFile('from', 'to');
+
+    expect(renameMock).toHaveBeenCalledTimes(1);
+    expect(renameMock).toHaveBeenCalledWith('from', 'to');
+
+    expect(copyFileMock).toHaveBeenCalledTimes(1);
+    expect(copyFileMock).toHaveBeenCalledWith('from', 'to');
+
+    expect(unlinkMock).toHaveBeenCalledTimes(1);
+    expect(unlinkMock).toHaveBeenCalledWith('from');
+  });
+
+  it('throws if copyFile call throws', async () => {
+    renameMock.mockRejectedValue(createError('EXDEV'));
+    copyFileMock.mockRejectedValue(createError('anything'));
+
+    await expect(moveFile('from', 'to')).rejects.toThrowError('anything');
+
+    expect(renameMock).toHaveBeenCalledTimes(1);
+
+    expect(copyFileMock).toHaveBeenCalledTimes(1);
+
+    expect(unlinkMock).not.toHaveBeenCalled();
+  });
+
+  it('throws if unlink call throws', async () => {
+    renameMock.mockRejectedValue(createError('EXDEV'));
+    unlinkMock.mockRejectedValue(createError('something-else'));
+
+    await expect(moveFile('from', 'to')).rejects.toThrowError('something-else');
+
+    expect(renameMock).toHaveBeenCalledTimes(1);
+
+    expect(copyFileMock).toHaveBeenCalledTimes(1);
+
+    expect(unlinkMock).toHaveBeenCalledTimes(1);
+  });
+});

packages/core/logging/core-logging-server-internal/src/appenders/rolling_file/strategies/numeric/utils.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { copyFile, rename, unlink } from 'fs/promises';
+
+export const moveFile = async (oldPath: string, newPath: string): Promise<void> => {
+  try {
+    await rename(oldPath, newPath);
+  } catch (err) {
+    // rename isn't supported on some file systems / volumes
+    // so we fallback to copy+delete
+    if (err.code === 'EXDEV') {
+      await copyFile(oldPath, newPath);
+      await unlink(oldPath);
+    } else {
+      throw err;
+    }
+  }
+};

src/plugins/telemetry/server/routes/telemetry_usage_stats.ts

@@ -92,6 +92,7 @@ export function registerTelemetryUsageStatsRoutes(
     .post({
       access: 'internal',
       path: FetchSnapshotTelemetry,
+      enableQueryVersion: true, // Allow specifying the version through querystring so that we can use it in Dev Console
     })
     // Just because it used to be /v2/, we are creating identical v1 and v2.
     .addVersion({ version: '1', validate: v2Validations }, v2Handler)

test/functional/page_objects/common_page.ts

@@ -227,6 +227,7 @@ export class CommonPageObject extends FtrService {
       search = '',
       disableWelcomePrompt = true,
       insertTimestamp = true,
+      retryOnFatalError = true,
     } = {}
   ) {
     let appUrl: string;
@@ -293,6 +294,13 @@ export class CommonPageObject extends FtrService {
           this.log.debug(msg);
           throw new Error(msg);
         }
+
+        if (retryOnFatalError && (await this.isFatalErrorScreen())) {
+          const msg = `Fatal error screen shown. Let's try refreshing the page once more.`;
+          this.log.debug(msg);
+          throw new Error(msg);
+        }
+
         if (appName === 'discover') {
           await this.browser.setLocalStorageItem('data.autocompleteFtuePopover', 'true');
         }
@@ -402,6 +410,10 @@ export class CommonPageObject extends FtrService {
     return await this.testSubjects.exists('kbnAppWrapper hiddenChrome');
   }
 
+  async isFatalErrorScreen() {
+    return await this.testSubjects.exists('fatalErrorScreen');
+  }
+
   async waitForTopNavToBeVisible() {
     await this.retry.try(async () => {
       const isNavVisible = await this.testSubjects.exists('top-nav');

x-pack/packages/kbn-slo-schema/src/rest_specs/slo.ts

@@ -170,6 +170,24 @@ const fetchHistoricalSummaryResponseSchema = t.array(
   })
 );
 
+/**
+ * The query params schema for /internal/observability/slo/_definitions
+ *
+ * @private
+ */
+const findSloDefinitionsParamsSchema = t.type({
+  query: t.type({
+    search: t.string,
+  }),
+});
+
+/**
+ * The response schema for /internal/observability/slo/_definitions
+ *
+ * @private
+ */
+const findSloDefinitionsResponseSchema = t.array(sloResponseSchema);
+
 const getSLODiagnosisParamsSchema = t.type({
   path: t.type({ id: t.string }),
 });
@@ -229,6 +247,13 @@ type FetchHistoricalSummaryParams = t.TypeOf<typeof fetchHistoricalSummaryParams
 type FetchHistoricalSummaryResponse = t.OutputOf<typeof fetchHistoricalSummaryResponseSchema>;
 type HistoricalSummaryResponse = t.OutputOf<typeof historicalSummarySchema>;
 
+/**
+ * The response type for /internal/observability/slo/_definitions
+ *
+ * @private
+ */
+type FindSloDefinitionsResponse = t.OutputOf<typeof findSloDefinitionsResponseSchema>;
+
 type GetPreviewDataParams = t.TypeOf<typeof getPreviewDataParamsSchema.props.body>;
 type GetPreviewDataResponse = t.OutputOf<typeof getPreviewDataResponseSchema>;
 
@@ -257,6 +282,8 @@ export {
   getSLOResponseSchema,
   fetchHistoricalSummaryParamsSchema,
   fetchHistoricalSummaryResponseSchema,
+  findSloDefinitionsParamsSchema,
+  findSloDefinitionsResponseSchema,
   manageSLOParamsSchema,
   sloResponseSchema,
   sloWithSummaryResponseSchema,
@@ -281,6 +308,7 @@ export type {
   FetchHistoricalSummaryParams,
   FetchHistoricalSummaryResponse,
   HistoricalSummaryResponse,
+  FindSloDefinitionsResponse,
   ManageSLOParams,
   SLOResponse,
   SLOWithSummaryResponse,