Merge branch 'main' into security/feature/EQL-query-validation

.github/CODEOWNERS

@@ -862,6 +862,7 @@ packages/kbn-stdio-dev-helpers @elastic/kibana-operations
 packages/kbn-storybook @elastic/kibana-operations
 x-pack/plugins/observability_solution/synthetics/e2e @elastic/obs-ux-management-team
 x-pack/plugins/observability_solution/synthetics @elastic/obs-ux-management-team
+x-pack/packages/kbn-synthetics-private-location @elastic/obs-ux-management-team
 x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture @elastic/response-ops
 x-pack/test/plugin_api_perf/plugins/task_manager_performance @elastic/response-ops
 x-pack/plugins/task_manager @elastic/response-ops

package.json

@@ -870,6 +870,7 @@
     "@kbn/status-plugin-b-plugin": "link:test/server_integration/plugins/status_plugin_b",
     "@kbn/std": "link:packages/kbn-std",
     "@kbn/synthetics-plugin": "link:x-pack/plugins/observability_solution/synthetics",
+    "@kbn/synthetics-private-location": "link:x-pack/packages/kbn-synthetics-private-location",
     "@kbn/task-manager-fixture-plugin": "link:x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture",
     "@kbn/task-manager-performance-plugin": "link:x-pack/test/plugin_api_perf/plugins/task_manager_performance",
     "@kbn/task-manager-plugin": "link:x-pack/plugins/task_manager",

packages/core/metrics/core-metrics-server-internal/src/routes/elu_history.ts

@@ -9,6 +9,7 @@
 import type { IRouter } from '@kbn/core-http-server';
 import type { OpsMetrics } from '@kbn/core-metrics-server';
 import type { Observable } from 'rxjs';
+import apm from 'elastic-apm-node';
 import { HistoryWindow } from './history_window';
 
 interface ELUHistoryResponse {
@@ -42,6 +43,17 @@ export function registerEluHistoryRoute(router: IRouter, metrics$: Observable<Op
     eluHistoryWindow.addObservation(metrics.process.event_loop_utilization.utilization);
   });
 
+  // Report the same metrics to APM
+  apm.registerMetric('elu.history.short', () =>
+    eluHistoryWindow.getAverage(HISTORY_WINDOW_SIZE_SHORT)
+  );
+  apm.registerMetric('elu.history.medium', () =>
+    eluHistoryWindow.getAverage(HISTORY_WINDOW_SIZE_MED)
+  );
+  apm.registerMetric('elu.history.long', () =>
+    eluHistoryWindow.getAverage(HISTORY_WINDOW_SIZE_LONG)
+  );
+
   router.versioned
     .get({
       access: 'internal',

packages/kbn-esql-ast/src/walker/walker.test.ts

@@ -791,8 +791,7 @@ describe('Walker.params()', () => {
   });
 
   test('can collect all params from grouping functions', () => {
-    const query =
-      'ROW x=1, time=2024-07-10 | stats z = avg(x) by bucket(time, 20, ?t_start,?t_end)';
+    const query = 'ROW x=1, time=2024-07-10 | stats z = avg(x) by bucket(time, 20, ?start,?end)';
     const { ast } = getAstAndSyntaxErrors(query);
     const params = Walker.params(ast);
 
@@ -801,13 +800,13 @@ describe('Walker.params()', () => {
         type: 'literal',
         literalType: 'param',
         paramType: 'named',
-        value: 't_start',
+        value: 'start',
       },
       {
         type: 'literal',
         literalType: 'param',
         paramType: 'named',
-        value: 't_end',
+        value: 'end',
       },
     ]);
   });

packages/kbn-esql-utils/src/utils/get_initial_esql_query.test.ts

@@ -96,7 +96,7 @@ describe('getInitialESQLQuery', () => {
     ] as DataView['fields'];
     const dataView = getDataView('logs*', fields, '@custom_timestamp');
     expect(getInitialESQLQuery(dataView)).toBe(
-      'FROM logs* | WHERE @custom_timestamp >= ?t_start AND @custom_timestamp <= ?t_end | LIMIT 10'
+      'FROM logs* | WHERE @custom_timestamp >= ?start AND @custom_timestamp <= ?end | LIMIT 10'
     );
   });
 });

packages/kbn-esql-utils/src/utils/get_initial_esql_query.ts

@@ -18,7 +18,7 @@ export function getInitialESQLQuery(dataView: DataView): string {
   const timeFieldName = dataView?.timeFieldName;
   const filterByTimeParams =
     !hasAtTimestampField && timeFieldName
-      ? ` | WHERE ${timeFieldName} >= ?t_start AND ${timeFieldName} <= ?t_end`
+      ? ` | WHERE ${timeFieldName} >= ?start AND ${timeFieldName} <= ?end`
       : '';
   return `FROM ${dataView.getIndexPattern()}${filterByTimeParams} | LIMIT 10`;
 }

packages/kbn-esql-utils/src/utils/query_parsing_helpers.test.ts

@@ -150,27 +150,25 @@ describe('esql query helpers', () => {
     });
 
     it('should return the time field if there is at least one time param', () => {
-      expect(getTimeFieldFromESQLQuery('from a | eval b = 1 | where time >= ?t_start')).toBe(
-        'time'
-      );
+      expect(getTimeFieldFromESQLQuery('from a | eval b = 1 | where time >= ?start')).toBe('time');
     });
 
-    it('should return undefined if there is one named param but is not ?t_start or ?t_end', () => {
+    it('should return undefined if there is one named param but is not ?start or ?end', () => {
       expect(
         getTimeFieldFromESQLQuery('from a | eval b = 1 | where time >= ?late')
       ).toBeUndefined();
     });
 
     it('should return undefined if there is one named param but is used without a time field', () => {
       expect(
-        getTimeFieldFromESQLQuery('from a | eval b = DATE_TRUNC(1 day, ?t_start)')
+        getTimeFieldFromESQLQuery('from a | eval b = DATE_TRUNC(1 day, ?start)')
       ).toBeUndefined();
     });
 
     it('should return the time field if there is at least one time param in the bucket function', () => {
       expect(
         getTimeFieldFromESQLQuery(
-          'from a | stats meow = avg(bytes) by bucket(event.timefield, 200, ?t_start, ?t_end)'
+          'from a | stats meow = avg(bytes) by bucket(event.timefield, 200, ?start, ?end)'
         )
       ).toBe('event.timefield');
     });

packages/kbn-esql-utils/src/utils/query_parsing_helpers.ts

@@ -56,7 +56,7 @@ export function removeDropCommandsFromESQLQuery(esql?: string): string {
 }
 
 /**
- * When the ?t_start and ?t_end params are used, we want to retrieve the timefield from the query.
+ * When the ?start and ?end params are used, we want to retrieve the timefield from the query.
  * @param esql:string
  * @returns string
  */
@@ -69,9 +69,7 @@ export const getTimeFieldFromESQLQuery = (esql: string) => {
   });
 
   const params = Walker.params(ast);
-  const timeNamedParam = params.find(
-    (param) => param.value === 't_start' || param.value === 't_end'
-  );
+  const timeNamedParam = params.find((param) => param.value === 'start' || param.value === 'end');
   if (!timeNamedParam || !functions.length) {
     return undefined;
   }

packages/kbn-esql-utils/src/utils/run_query.test.ts

@@ -17,26 +17,26 @@ describe('getStartEndParams', () => {
 
   it('should return an array with the start param if exists at the query', () => {
     const time = { from: 'Jul 5, 2024 @ 08:03:56.849', to: 'Jul 5, 2024 @ 10:03:56.849' };
-    const query = 'FROM foo | where time > ?t_start';
+    const query = 'FROM foo | where time > ?start';
     const params = getStartEndParams(query, time);
     expect(params).toHaveLength(1);
-    expect(params[0]).toHaveProperty('t_start');
+    expect(params[0]).toHaveProperty('start');
   });
 
   it('should return an array with the end param if exists at the query', () => {
     const time = { from: 'Jul 5, 2024 @ 08:03:56.849', to: 'Jul 5, 2024 @ 10:03:56.849' };
-    const query = 'FROM foo | where time < ?t_end';
+    const query = 'FROM foo | where time < ?end';
     const params = getStartEndParams(query, time);
     expect(params).toHaveLength(1);
-    expect(params[0]).toHaveProperty('t_end');
+    expect(params[0]).toHaveProperty('end');
   });
 
   it('should return an array with the end and start params if exist at the query', () => {
     const time = { from: 'Jul 5, 2024 @ 08:03:56.849', to: 'Jul 5, 2024 @ 10:03:56.849' };
-    const query = 'FROM foo | where time < ?t_end amd time > ?t_start';
+    const query = 'FROM foo | where time < ?end amd time > ?start';
     const params = getStartEndParams(query, time);
     expect(params).toHaveLength(2);
-    expect(params[0]).toHaveProperty('t_start');
-    expect(params[1]).toHaveProperty('t_end');
+    expect(params[0]).toHaveProperty('start');
+    expect(params[1]).toHaveProperty('end');
   });
 });

packages/kbn-esql-utils/src/utils/run_query.ts

@@ -14,22 +14,22 @@ import { esFieldTypeToKibanaFieldType } from '@kbn/field-types';
 import type { ESQLColumn, ESQLSearchResponse, ESQLSearchParams } from '@kbn/es-types';
 import { lastValueFrom } from 'rxjs';
 
-export const hasStartEndParams = (query: string) => /\?t_start|\?t_end/i.test(query);
+export const hasStartEndParams = (query: string) => /\?start|\?end/i.test(query);
 
 export const getStartEndParams = (query: string, time?: TimeRange) => {
-  const startNamedParams = /\?t_start/i.test(query);
-  const endNamedParams = /\?t_end/i.test(query);
+  const startNamedParams = /\?start/i.test(query);
+  const endNamedParams = /\?end/i.test(query);
   if (time && (startNamedParams || endNamedParams)) {
     const timeParams = {
       start: startNamedParams ? dateMath.parse(time.from)?.toISOString() : undefined,
       end: endNamedParams ? dateMath.parse(time.to)?.toISOString() : undefined,
     };
     const namedParams = [];
     if (timeParams?.start) {
-      namedParams.push({ t_start: timeParams.start });
+      namedParams.push({ start: timeParams.start });
     }
     if (timeParams?.end) {
-      namedParams.push({ t_end: timeParams.end });
+      namedParams.push({ end: timeParams.end });
     }
     return namedParams;
   }

packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts

@@ -116,7 +116,8 @@ describe('autocomplete', () => {
     },
   });
 
-  const sourceCommands = ['row', 'from', 'show', 'metrics'];
+  // const sourceCommands = ['row', 'from', 'show', 'metrics']; Uncomment when metrics is being released
+  const sourceCommands = ['row', 'from', 'show'];
 
   describe('New command', () => {
     testSuggestions(
@@ -1278,7 +1279,7 @@ describe('autocomplete', () => {
     // Source command
     testSuggestions(
       'F',
-      ['FROM $0', 'ROW $0', 'SHOW $0', 'METRICS $0'].map(attachTriggerCommand).map(attachAsSnippet),
+      ['FROM $0', 'ROW $0', 'SHOW $0'].map(attachTriggerCommand).map(attachAsSnippet),
       undefined,
       1
     );

packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts

@@ -29,7 +29,7 @@ const allFunctions = statsAggregationFunctionDefinitions
   .concat(evalFunctionDefinitions)
   .concat(groupingFunctionDefinitions);
 
-export const TIME_SYSTEM_PARAMS = ['?t_start', '?t_end'];
+export const TIME_SYSTEM_PARAMS = ['?start', '?end'];
 
 export const TRIGGER_SUGGESTION_COMMAND = {
   title: 'Trigger Suggestion Dialog',

packages/kbn-esql-validation-autocomplete/src/definitions/commands.ts

@@ -192,6 +192,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'metrics',
+    hidden: true,
     description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.metricsDoc', {
       defaultMessage:
         'A metrics-specific source command, use this command to load data from TSDB indices. ' +

packages/kbn-esql-validation-autocomplete/src/validation/__tests__/validation.params.test.ts

@@ -23,16 +23,16 @@ test('should allow param inside agg function argument', async () => {
 test('allow params in WHERE command expressions', async () => {
   const { validate } = await setup();
 
-  const res1 = await validate('FROM index | WHERE textField >= ?t_start');
+  const res1 = await validate('FROM index | WHERE textField >= ?start');
   const res2 = await validate(`
     FROM index
-      | WHERE textField >= ?t_start
+      | WHERE textField >= ?start
       | WHERE textField <= ?0
       | WHERE textField == ?
   `);
   const res3 = await validate(`
     FROM index
-      | WHERE textField >= ?t_start
+      | WHERE textField >= ?start
         AND textField <= ?0
         AND textField == ?
   `);

src/plugins/discover/public/application/main/state_management/utils/get_esql_data_view.test.ts

@@ -32,7 +32,7 @@ describe('getEsqlDataView', () => {
   });
 
   it('returns an adhoc dataview if it is adhoc with named params and query index pattern is the same as the dataview index pattern', async () => {
-    const query = { esql: 'from data-view-ad-hoc-title | where time >= ?t_start' };
+    const query = { esql: 'from data-view-ad-hoc-title | where time >= ?start' };
     const dataView = await getEsqlDataView(query, dataViewAdHocNoAtTimestamp, services);
     expect(dataView.timeFieldName).toBe('time');
   });

test/functional/apps/discover/esql/_esql_view.ts

@@ -113,11 +113,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         expect(await testSubjects.exists('unifiedHistogramChart')).to.be(false);
       });
 
-      it('should render the histogram for indices with no @timestamp field when the ?t_start, ?t_end params are in the query', async function () {
+      it('should render the histogram for indices with no @timestamp field when the ?start, ?end params are in the query', async function () {
         await PageObjects.discover.selectTextBaseLang();
         await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
 
-        const testQuery = `from kibana_sample_data_flights | limit 10 | where timestamp >= ?t_start and timestamp <= ?t_end`;
+        const testQuery = `from kibana_sample_data_flights | limit 10 | where timestamp >= ?start and timestamp <= ?end`;
 
         await monacoEditor.setCodeEditorValue(testQuery);
         await testSubjects.click('querySubmitButton');

tsconfig.base.json

@@ -1718,6 +1718,8 @@
       "@kbn/synthetics-e2e/*": ["x-pack/plugins/observability_solution/synthetics/e2e/*"],
       "@kbn/synthetics-plugin": ["x-pack/plugins/observability_solution/synthetics"],
       "@kbn/synthetics-plugin/*": ["x-pack/plugins/observability_solution/synthetics/*"],
+      "@kbn/synthetics-private-location": ["x-pack/packages/kbn-synthetics-private-location"],
+      "@kbn/synthetics-private-location/*": ["x-pack/packages/kbn-synthetics-private-location/*"],
       "@kbn/task-manager-fixture-plugin": ["x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture"],
       "@kbn/task-manager-fixture-plugin/*": ["x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture/*"],
       "@kbn/task-manager-performance-plugin": ["x-pack/test/plugin_api_perf/plugins/task_manager_performance"],

x-pack/packages/kbn-synthetics-private-location/README.md

@@ -0,0 +1,56 @@
+# @kbn/synthetics-private-location
+
+Quickily start Fleet, enroll Elastic Agent, and create a private location.
+
+## Usage
+
+```
+node x-pack/scripts/synthetics_private_location.js
+```
+
+For available options, run `--help`.
+
+## Prerequistes
+
+This script requires `docker` and the following `kibama.yml` configuration.
+
+```
+# Create an agent policy for Fleet Server.
+xpack.fleet.agentPolicies:
+  - name: Fleet Server policy
+    id: fleet-server-policy
+    is_default_fleet_server: true
+    # is_managed: true # Useful to mimic cloud environment
+    description: Fleet server policy
+    namespace: default
+    package_policies:
+      - name: Fleet Server
+        package:
+          name: fleet_server
+        inputs:
+          - type: fleet-server
+            keep_enabled: true
+            vars:
+              - name: host
+                value: 0.0.0.0
+                frozen: true
+              - name: port
+                value: 8220
+                frozen: true
+
+# Set a default Fleet Server host.
+xpack.fleet.fleetServerHosts:
+  - id: default-fleet-server
+    name: Default Fleet server
+    is_default: true
+    host_urls: ['https://host.docker.internal:8220'] # For running a Fleet Server Docker container
+
+# Set a default Elasticsearch output.
+xpack.fleet.outputs:
+  - id: es-default-output
+    name: Default output
+    type: elasticsearch
+    is_default: true
+    is_default_monitoring: true
+    hosts: ['http://host.docker.internal:9200'] # For enrolling dockerized agents
+```

x-pack/packages/kbn-synthetics-private-location/index.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export type { CliOptions } from './src/types';
+export { run } from './src/run';
+export { cli } from './src/cli';
+// export { cleanup } from './src/cleanup';
+export { DEFAULTS } from './src/constants';

x-pack/packages/kbn-synthetics-private-location/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/packages/kbn-synthetics-private-location'],
+};

x-pack/packages/kbn-synthetics-private-location/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/synthetics-private-location",
+  "owner": "@elastic/obs-ux-management-team"
+}

x-pack/packages/kbn-synthetics-private-location/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/synthetics-private-location",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}