Merge branch 'main' into 190343-failing-test-serverless-observability…

…-functional-testsx-packtest_serverlessfunctionaltest_suitesobservabilityobservability_logs_explorerdata_source_selectorts

.buildkite/scripts/steps/functional/osquery_cypress.sh

@@ -4,11 +4,7 @@ set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
 
-# TODO: remove the line below to use build artifacts for tests.
-# in addition to remove the line, we will have to expose the kibana install dir into the downloaded build location
-# by exporting a var like:
-# export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
-node scripts/build_kibana_platform_plugins.js
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
 export JOB=kibana-osquery-cypress
 

.buildkite/scripts/steps/functional/osquery_cypress_burn.sh

@@ -2,10 +2,9 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/functional/common.sh
 
-.buildkite/scripts/bootstrap.sh
-node scripts/build_kibana_platform_plugins.js
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
 export JOB=kibana-osquery-cypress
 

.buildkite/scripts/steps/functional/security_serverless_osquery.sh

@@ -4,11 +4,7 @@ set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
 
-# TODO: remove the line below to use build artifacts for tests.
-# in addition to remove the line, we will have to expose the kibana install dir into the downloaded build location
-# by exporting a var like:
-# export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
-node scripts/build_kibana_platform_plugins.js
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
 export JOB=kibana-osquery-cypress-serverless
 

docs/setup/settings.asciidoc

@@ -14,7 +14,7 @@ KBN_PATH_CONF=/home/kibana/config ./bin/kibana
 --
 
 The default host and port settings configure {kib} to run on `localhost:5601`. To change this behavior and allow remote users to connect, you'll need to update your `kibana.yml` file. You can also enable SSL and set a
-variety of other options. 
+variety of other options.
 
 Environment variables can be injected into configuration using `${MY_ENV_VAR}` syntax. By default, configuration validation
 will fail if an environment variable used in the config file is not present when Kibana starts. This behavior can be changed by using a default value
@@ -93,7 +93,7 @@ The maximum number of sockets that can be used for communications with {es}.
 *Default: `Infinity`*
 
 [[elasticsearch-maxResponseSize]] `elasticsearch.maxResponseSize`::
-Either `false` or a `byteSize` value. When set, responses from {es} with a size higher than the defined limit will be rejected. 
+Either `false` or a `byteSize` value. When set, responses from {es} with a size higher than the defined limit will be rejected.
 This is intended to be used as a circuit-breaker mechanism to avoid memory errors in case of unexpectedly high responses coming from {es}.
 *Default: `false`*
 
@@ -117,6 +117,9 @@ listed here must be on the same cluster. *Default: `[ "http://localhost:9200" ]`
 To enable SSL/TLS for outbound connections to {es}, use the `https` protocol
 in this setting.
 
+[[elasticsearch-publicBaseUrl]] `elasticsearch.publicBaseUrl:`::
+The URL through which Elasticsearch is publicly accessible, if any. This will be shown to users in Kibana when they need connection details for your Elasticsearch cluster.
+
 [[elasticsearch-pingTimeout]] `elasticsearch.pingTimeout`::
 Time in milliseconds to wait for {es} to respond to pings.
 *Default: the value of the <<elasticsearch-requestTimeout, `elasticsearch.requestTimeout`>> setting*
@@ -470,7 +473,7 @@ identifies this {kib} instance. *Default: `"your-hostname"`*
 setting specifies the port to use. *Default: `5601`*
 
 `server.protocol`::
-experimental[] The http protocol to use, either `http1` or `http2`. Set to `http2` to enable `HTTP/2` support for the {kib} server. 
+experimental[] The http protocol to use, either `http1` or `http2`. Set to `http2` to enable `HTTP/2` support for the {kib} server.
 *Default: `http1`*
 +
 NOTE: By default, enabling `http2` requires a valid `h2c` configuration, meaning that TLS must be enabled via <<server-ssl-enabled, `server.ssl.enabled`>>

packages/core/elasticsearch/core-elasticsearch-server-internal/src/elasticsearch_config.ts

@@ -190,6 +190,7 @@ export const configSchema = schema.object({
     { defaultValue: [] }
   ),
   dnsCacheTtl: schema.duration({ defaultValue: 0, min: 0 }),
+  publicBaseUrl: schema.maybe(hostURISchema),
 });
 
 const deprecations: ConfigDeprecationProvider = () => [

packages/core/elasticsearch/core-elasticsearch-server/src/contracts.ts

@@ -90,6 +90,12 @@ export interface ElasticsearchServiceSetup {
      */
     readonly config$: Observable<IElasticsearchConfig>;
   };
+
+  /**
+   * The public base URL (if any) that should be used by end users to access the Elasticsearch cluster.
+   */
+
+  readonly publicBaseUrl?: string;
 }
 
 /**

packages/kbn-apm-synthtrace-client/index.ts

@@ -35,3 +35,4 @@ export { appendHash, hashKeysOf } from './src/lib/utils/hash';
 export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator } from './src/types';
 export { log, type LogDocument, LONG_FIELD_NAME } from './src/lib/logs';
 export { type AssetDocument } from './src/lib/assets';
+export { syntheticsMonitor, type SyntheticsMonitorDocument } from './src/lib/synthetics';

packages/kbn-apm-synthtrace-client/src/lib/synthetics/index.ts

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Fields } from '../entity';
+import { Serializable } from '../serializable';
+
+export type SyntheticsMonitorDocument = Fields &
+  Partial<{
+    'data_stream.namespace': string;
+    'data_stream.type': string;
+    'data_stream.dataset': string;
+    'monitor.id': string;
+    'monitor.origin': string;
+    'monitor.name': string;
+    'monitor.type': string;
+    'monitor.check_group': string;
+    'monitor.timespan.lt': string;
+    'monitor.timespan.gte': string;
+    'monitor.duration.us'?: number;
+    'monitor.ip'?: string;
+    'monitor.project.name'?: string;
+    'monitor.project.id'?: string;
+    'monitor.fleet_managed'?: boolean;
+    'monitor.status'?: string;
+    'synthetics.type'?: string;
+    'synthetics.step.index'?: number;
+    'observer.os.name'?: string;
+    'observer.product'?: string;
+  }>;
+
+type MonitorDataStream =
+  | 'http'
+  | 'tcp'
+  | 'icmp'
+  | 'browser'
+  | 'browser.screenshot'
+  | 'browser.network';
+
+class SyntheticsMonitor extends Serializable<SyntheticsMonitorDocument> {
+  constructor(fields: SyntheticsMonitorDocument) {
+    super({
+      ...fields,
+    });
+  }
+
+  namespace(value: string) {
+    this.fields['data_stream.namespace'] = value;
+    return this;
+  }
+
+  dataset(value: MonitorDataStream) {
+    this.fields['data_stream.dataset'] = value;
+
+    if (value === 'browser.screenshot' || value === 'browser.network') {
+      this.fields['monitor.type'] = 'browser';
+      return this;
+    }
+
+    this.fields['monitor.type'] = value;
+    return this;
+  }
+
+  name(value: string) {
+    this.fields['monitor.name'] = value;
+    return this;
+  }
+
+  origin(value: string) {
+    this.fields['monitor.origin'] = value;
+    return this;
+  }
+
+  ip(value: string) {
+    this.fields['monitor.ip'] = value;
+    return this;
+  }
+
+  status(value: string) {
+    this.fields['monitor.status'] = value;
+    return this;
+  }
+
+  timestamp(time: number) {
+    super.timestamp(time);
+    return this;
+  }
+}
+
+function create(): SyntheticsMonitor {
+  return new SyntheticsMonitor({
+    'data_stream.namespace': 'default',
+    'data_stream.type': 'synthetics',
+  }).dataset('http');
+}
+
+export const syntheticsMonitor = {
+  create,
+};

packages/kbn-apm-synthtrace/README.md

@@ -22,6 +22,7 @@ This library can currently be used in two ways:
 - `Timerange`: an object that will return an array of timestamps based on an interval and a rate. These timestamps can be used to generate events/metricsets.
 - `Transaction`, `Span`, `APMError` and `Metricset`: events/metricsets that occur on an instance. For more background, see the [explanation of the APM data model](https://www.elastic.co/guide/en/apm/get-started/7.15/apm-data-model.html)
 - `Log`: An instance of Log generating Service which supports additional helpers to customise fields like `messages`, `logLevel`
+- `SyntheticsMonitor`: An instance of Synthetic monitor. For more information see [Synthetic monitoring](https://www.elastic.co/guide/en/observability/current/monitor-uptime-synthetics.html).
 
 #### Example
 

packages/kbn-apm-synthtrace/index.ts

@@ -15,6 +15,7 @@ export { InfraSynthtraceKibanaClient } from './src/lib/infra/infra_synthtrace_ki
 export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_synthtrace_es_client';
 export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client';
 export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_client';
+export { SyntheticsSynthtraceEsClient } from './src/lib/synthetics/synthetics_synthtrace_es_client';
 export {
   addObserverVersionTransform,
   deleteSummaryFieldTransform,

packages/kbn-apm-synthtrace/src/cli/scenario.ts

@@ -7,7 +7,12 @@
  */
 
 import { Timerange } from '@kbn/apm-synthtrace-client';
-import { ApmSynthtraceEsClient, InfraSynthtraceEsClient, LogsSynthtraceEsClient } from '../..';
+import {
+  ApmSynthtraceEsClient,
+  InfraSynthtraceEsClient,
+  LogsSynthtraceEsClient,
+  SyntheticsSynthtraceEsClient,
+} from '../..';
 import { AssetsSynthtraceEsClient } from '../lib/assets/assets_synthtrace_es_client';
 import { Logger } from '../lib/utils/create_logger';
 import { ScenarioReturnType } from '../lib/utils/with_client';
@@ -18,6 +23,7 @@ interface EsClients {
   logsEsClient: LogsSynthtraceEsClient;
   infraEsClient: InfraSynthtraceEsClient;
   assetsEsClient: AssetsSynthtraceEsClient;
+  syntheticsEsClient: SyntheticsSynthtraceEsClient;
 }
 
 type Generate<TFields> = (options: {

packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts

@@ -14,6 +14,7 @@ import { getKibanaClient } from './get_kibana_client';
 import { getServiceUrls } from './get_service_urls';
 import { RunOptions } from './parse_run_cli_flags';
 import { getAssetsEsClient } from './get_assets_es_client';
+import { getSyntheticsEsClient } from './get_synthetics_es_client';
 
 export async function bootstrap(runOptions: RunOptions) {
   const logger = createLogger(runOptions.logLevel);
@@ -61,11 +62,18 @@ export async function bootstrap(runOptions: RunOptions) {
     concurrency: runOptions.concurrency,
   });
 
+  const syntheticsEsClient = getSyntheticsEsClient({
+    target: esUrl,
+    logger,
+    concurrency: runOptions.concurrency,
+  });
+
   if (runOptions.clean) {
     await apmEsClient.clean();
     await logsEsClient.clean();
     await infraEsClient.clean();
     await assetsEsClient.clean();
+    await syntheticsEsClient.clean();
   }
 
   return {
@@ -74,6 +82,7 @@ export async function bootstrap(runOptions: RunOptions) {
     logsEsClient,
     infraEsClient,
     assetsEsClient,
+    syntheticsEsClient,
     version,
     kibanaUrl,
     esUrl,

packages/kbn-apm-synthtrace/src/cli/utils/get_synthetics_es_client.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { Logger } from '../../lib/utils/create_logger';
+import { RunOptions } from './parse_run_cli_flags';
+import { getEsClientTlsSettings } from './ssl';
+import { SyntheticsSynthtraceEsClient } from '../../lib/synthetics/synthetics_synthtrace_es_client';
+
+export function getSyntheticsEsClient({
+  target,
+  logger,
+  concurrency,
+}: Pick<RunOptions, 'concurrency'> & {
+  target: string;
+  logger: Logger;
+}) {
+  const client = new Client({
+    node: target,
+    tls: getEsClientTlsSettings(target),
+  });
+
+  return new SyntheticsSynthtraceEsClient({
+    client,
+    logger,
+    concurrency,
+  });
+}

packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts

@@ -25,9 +25,8 @@ export async function startLiveDataUpload({
 }) {
   const file = runOptions.file;
 
-  const { logger, apmEsClient, logsEsClient, infraEsClient, assetsEsClient } = await bootstrap(
-    runOptions
-  );
+  const { logger, apmEsClient, logsEsClient, infraEsClient, assetsEsClient, syntheticsEsClient } =
+    await bootstrap(runOptions);
 
   const scenario = await getScenario({ file, logger });
   const { generate } = await scenario({ ...runOptions, logger });
@@ -65,7 +64,7 @@ export async function startLiveDataUpload({
 
       const generatorsAndClients = generate({
         range: timerange(bucketFrom.getTime(), bucketTo.getTime()),
-        clients: { logsEsClient, apmEsClient, infraEsClient, assetsEsClient },
+        clients: { logsEsClient, apmEsClient, infraEsClient, assetsEsClient, syntheticsEsClient },
       });
 
       const generatorsAndClientsArray = castArray(generatorsAndClients);

packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts

@@ -17,6 +17,7 @@ import { RunOptions } from './parse_run_cli_flags';
 import { getLogsEsClient } from './get_logs_es_client';
 import { getInfraEsClient } from './get_infra_es_client';
 import { getAssetsEsClient } from './get_assets_es_client';
+import { getSyntheticsEsClient } from './get_synthetics_es_client';
 
 export interface WorkerData {
   bucketFrom: Date;
@@ -56,6 +57,12 @@ async function start() {
     logger,
   });
 
+  const syntheticsEsClient = getSyntheticsEsClient({
+    concurrency: runOptions.concurrency,
+    target: esUrl,
+    logger,
+  });
+
   const file = runOptions.file;
 
   const scenario = await logger.perf('get_scenario', () => getScenario({ file, logger }));
@@ -70,6 +77,7 @@ async function start() {
       logsEsClient,
       infraEsClient,
       assetsEsClient,
+      syntheticsEsClient,
     });
   }
 
@@ -78,7 +86,7 @@ async function start() {
   const generatorsAndClients = logger.perf('generate_scenario', () =>
     generate({
       range: timerange(bucketFrom, bucketTo),
-      clients: { logsEsClient, apmEsClient, infraEsClient, assetsEsClient },
+      clients: { logsEsClient, apmEsClient, infraEsClient, assetsEsClient, syntheticsEsClient },
     })
   );