Merge branch 'main' into 202255-poc-migration-to-emotion

.buildkite/pipelines/on_merge.yml

@@ -179,6 +179,9 @@ steps:
     timeout_in_minutes: 50
     parallelism: 1
     agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     retry:

packages/kbn-monaco/src/console/lexer_rules/nested_painless.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { lexerRules as painlessLexerRules } from '../../painless/lexer_rules';
+
+/*
+ * This rule is used inside json root to start a painless highlighting sequence
+ */
+export const buildPainlessStartRule = (painlessRoot: string = 'painless_root') => {
+  return [
+    /("(?:[^"]*_)?script"|"inline"|"source")(\s*?)(:)(\s*?)(""")/,
+    [
+      'variable',
+      'whitespace',
+      'punctuation.colon',
+      'whitespace',
+      {
+        token: 'punctuation',
+        next: `@${painlessRoot}`,
+      },
+    ],
+  ];
+};
+
+/*
+ * This function creates a group of rules needed for painless highlighting in console.
+ * It reuses the lexer rules from the "painless" language, but since not all rules are referenced in the root
+ * tokenizer and to avoid conflicts with existing console rules, only selected rules are used.
+ */
+export const buildPainlessRules = (painlessRoot: string = 'painless_root') => {
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  const { root, comment, string_dq, string_sq } = painlessLexerRules.tokenizer;
+  return {
+    [painlessRoot]: [
+      // the rule to end painless highlighting and get back to the previous tokenizer state
+      [
+        /"""/,
+        {
+          token: 'punctuation',
+          next: '@pop',
+        },
+      ],
+      ...root,
+    ],
+    comment,
+    string_dq,
+    string_sq,
+  };
+};
+
+/*
+ * These language attributes need to be added to the console language definition for painless tokenizer
+ * to work correctly.
+ */
+export const painlessLanguageAttributes = {
+  keywords: painlessLexerRules.keywords,
+  primitives: painlessLexerRules.primitives,
+  constants: painlessLexerRules.constants,
+  operators: painlessLexerRules.operators,
+  symbols: painlessLexerRules.symbols,
+  digits: painlessLexerRules.digits,
+  octaldigits: painlessLexerRules.octaldigits,
+  binarydigits: painlessLexerRules.binarydigits,
+  hexdigits: painlessLexerRules.hexdigits,
+};

packages/kbn-monaco/src/console/lexer_rules/shared.ts

@@ -8,6 +8,11 @@
  */
 
 import { buildSqlRules, buildSqlStartRule, sqlLanguageAttributes } from './nested_sql';
+import {
+  buildPainlessRules,
+  buildPainlessStartRule,
+  painlessLanguageAttributes,
+} from './nested_painless';
 import { monaco } from '../../..';
 import { globals } from '../../common/lexer_rules';
 import { buildXjsonRules } from '../../xjson/lexer_rules/xjson';
@@ -16,11 +21,13 @@ export const consoleSharedLanguageConfiguration: monaco.languages.LanguageConfig
   brackets: [
     ['{', '}'],
     ['[', ']'],
+    ['(', ')'],
     ['"""', '"""\n'],
   ],
   autoClosingPairs: [
     { open: '{', close: '}' },
     { open: '[', close: ']' },
+    { open: '(', close: ')' },
     { open: '"', close: '"' },
     { open: '"""', close: '"""' },
   ],
@@ -100,17 +107,22 @@ xjsonRules.json_root = [
   matchToken('variable.template', /("\${\w+}")/),
   // @ts-expect-error include a rule to start sql highlighting
   buildSqlStartRule(),
+  // @ts-expect-error include a rule to start painless highlighting
+  buildPainlessStartRule(),
   ...xjsonRules.json_root,
 ];
 
 const sqlRules = buildSqlRules();
+const painlessRules = buildPainlessRules();
 /*
  Lexer rules that are shared between the Console editor and the Console output panel.
  */
 export const consoleSharedLexerRules: monaco.languages.IMonarchLanguage = {
   ...(globals as any),
   defaultToken: 'invalid',
   ...sqlLanguageAttributes,
+  ...painlessLanguageAttributes,
+  keywords: [...sqlLanguageAttributes.keywords, ...painlessLanguageAttributes.keywords],
   tokenizer: {
     root: [
       // warning comment
@@ -138,5 +150,7 @@ export const consoleSharedLexerRules: monaco.languages.IMonarchLanguage = {
     ...xjsonRules,
     // include sql rules
     ...sqlRules,
+    // include painless rules
+    ...painlessRules,
   },
 };

x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.test.ts

@@ -1148,6 +1148,68 @@ describe('AlertingEventLogger', () => {
       expect(eventLogger.logEvent).toHaveBeenCalledWith(loggedEvent);
     });
 
+    test('should set fields from backfill even when no rule data is provided', () => {
+      alertingEventLogger.initialize({
+        context: backfillContext,
+        runDate,
+        type: executionType.BACKFILL,
+      });
+
+      alertingEventLogger.done({
+        backfill: {
+          id: 'abc',
+          start: '2024-03-13T00:00:00.000Z',
+          interval: '1h',
+        },
+        status: {
+          lastExecutionDate: new Date('2022-05-05T15:59:54.480Z'),
+          status: 'error',
+          error: {
+            reason: RuleExecutionStatusErrorReasons.Execute,
+            message: 'something went wrong',
+          },
+        },
+      });
+
+      const event = initializeExecuteBackfillRecord(backfillContextWithScheduleDelay, [adHocRunSO]);
+      const loggedEvent = {
+        ...event,
+        event: {
+          ...event?.event,
+          outcome: 'failure',
+          reason: RuleExecutionStatusErrorReasons.Execute,
+        },
+        error: {
+          message: 'something went wrong',
+        },
+        message: 'def: execution failed',
+        kibana: {
+          ...event.kibana,
+          alert: {
+            ...event.kibana?.alert,
+            rule: {
+              ...event.kibana?.alert?.rule,
+              execution: {
+                ...event.kibana?.alert?.rule?.execution,
+                backfill: {
+                  id: 'abc',
+                  start: '2024-03-13T00:00:00.000Z',
+                  interval: '1h',
+                },
+              },
+            },
+          },
+          alerting: {
+            outcome: 'failure',
+            status: 'error',
+          },
+        },
+      };
+
+      expect(alertingEventLogger.getEvent()).toEqual(loggedEvent);
+      expect(eventLogger.logEvent).toHaveBeenCalledWith(loggedEvent);
+    });
+
     test('should set fields from execution metrics if provided', () => {
       alertingEventLogger.initialize({ context: ruleContext, runDate, ruleData });
       alertingEventLogger.done({

x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts

@@ -346,7 +346,7 @@ export class AlertingEventLogger {
   }
 
   public done({ status, metrics, timings, backfill }: DoneOpts) {
-    if (!this.isInitialized || !this.event || !this.context || !this.ruleData) {
+    if (!this.isInitialized || !this.event || !this.context) {
       throw new Error('AlertingEventLogger not initialized');
     }
 
@@ -360,16 +360,15 @@ export class AlertingEventLogger {
       updateEvent(this.event, { status: status.status });
 
       if (status.error) {
+        const message = this.ruleData
+          ? `${this.ruleData.type?.id}:${this.context.savedObjectId}: execution failed`
+          : `${this.context.savedObjectId}: execution failed`;
         updateEvent(this.event, {
           outcome: 'failure',
           alertingOutcome: 'failure',
           reason: status.error?.reason || 'unknown',
           error: this.event?.error?.message || status.error.message,
-          ...(this.event.message && this.event.event?.outcome === 'failure'
-            ? {}
-            : {
-                message: `${this.ruleData.type?.id}:${this.context.savedObjectId}: execution failed`,
-              }),
+          ...(this.event.message && this.event.event?.outcome === 'failure' ? {} : { message }),
         });
       } else {
         if (status.warning) {

x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/events_timeline/events_timeline.tsx

@@ -11,9 +11,6 @@ import { Chart, Axis, AreaSeries, Position, ScaleType, Settings } from '@elastic
 import { useActiveCursor } from '@kbn/charts-plugin/public';
 import { EuiSkeletonText } from '@elastic/eui';
 import { getBrushData } from '@kbn/observability-utils-browser/chart/utils';
-import { Group } from '@kbn/observability-alerting-rule-utils';
-import { ALERT_GROUP } from '@kbn/rule-data-utils';
-import { SERVICE_NAME } from '@kbn/observability-shared-plugin/common';
 import { AnnotationEvent } from './annotation_event';
 import { TIME_LINE_THEME } from './timeline_theme';
 import { useFetchEvents } from '../../../../hooks/use_fetch_events';
@@ -27,19 +24,10 @@ export const EventsTimeLine = () => {
   const baseTheme = dependencies.start.charts.theme.useChartsBaseTheme();
 
   const { globalParams, updateInvestigationParams } = useInvestigation();
-  const { alert } = useInvestigation();
-
-  const filter = useMemo(() => {
-    const group = (alert?.[ALERT_GROUP] as unknown as Group[])?.find(
-      ({ field }) => field === SERVICE_NAME
-    );
-    return group ? `{"${SERVICE_NAME}":"${alert?.[SERVICE_NAME]}"}` : '';
-  }, [alert]);
 
   const { data: events, isLoading } = useFetchEvents({
     rangeFrom: globalParams.timeRange.from,
     rangeTo: globalParams.timeRange.to,
-    filter,
   });
 
   const chartRef = useRef(null);

x-pack/plugins/observability_solution/investigate_app/tsconfig.json

@@ -66,7 +66,6 @@
     "@kbn/calculate-auto",
     "@kbn/ml-random-sampler-utils",
     "@kbn/charts-plugin",
-    "@kbn/observability-alerting-rule-utils",
     "@kbn/observability-utils-browser",
     "@kbn/usage-collection-plugin",
     "@kbn/inference-common",

x-pack/test/api_integration/deployment_agnostic/README.md

@@ -9,6 +9,12 @@ A deployment-agnostic API integration test is a test suite that fulfills the fol
 
 A deployment-agnostic test should be loaded in stateful and at least 1 serverless FTR config files.
 
+## Tests, that are not deployment-agnostic:
+- tests verifying Kibana behavior under a basic license.
+- tests dependent on ES/Kibana server arguments, that are not set in Elastic Cloud
+- tests requiring a custom plugin to be loaded specifically for testing purposes.
+- tests dependent on varying user privileges between serverless and stateful environments.
+
 ## Tests Design Requirements
 A deployment-agnostic test is contained within a single test file and always utilizes the [DeploymentAgnosticFtrProviderContext](https://github.com/elastic/kibana/blob/main/x-pack/test/api_integration/deployment_agnostic/ftr_provider_context.d.ts) to load compatible FTR services. A compatible FTR service must support:
 
@@ -243,3 +249,15 @@ node scripts/functional_test_runner --config x-pack/test/api_integration/deploym
 Since deployment-agnostic tests are designed to run both locally and on MKI/Cloud, we believe no extra tagging is required. If a test is not working on MKI/Cloud or both, there is most likely an issue with the FTR service or the configuration file it uses.
 
 When a test fails on CI, automation will apply `.skip` to the top-level describe block. This means the test will be skipped in **both serverless and stateful environments**. If a test is unstable in a specific environment only, it is probably a sign that the test is not truly deployment-agnostic.
+
+## Migrating existing tests
+If your tests align with the outlined criteria and requirements, you can migrate them to deployment-agnostic by following these steps:
+
+1. Move your tests to the `x-pack/test/api_integration/deployment_agnostic/apis/<plugin>` directory.
+2. Update each test file to use the `DeploymentAgnosticFtrProviderContext` context and load the required FTR services it provides.
+3. Ensure the `roleScopedSupertest` or `samlAuth` service is used instead for `supertest` for authentication and test API calls.
+4. Remove all usage of the `supertest` service. It is authenticated as system index superuser and often causes test failures on Cloud, where priveleges are more strict.
+5. Avoid modifying `config` files, as this could disrupt test runs in Cloud environments.
+6. Include your tests in both the platform and at least one solution index file.
+7. Execute your tests locally against a local environment.
+8. Verify your tests by running them locally against a real MKI project and an ESS deployment to ensure full compatibility.