[Synthetics] add synthetics-private-location command (#189531)

# Summary

Quickily start Fleet, enroll Elastic Agent, and create a private
location.

## Usage

```
node x-pack/scripts/synthetics_private_location.js
```

For available options, run `--help`.

## Prerequistes

This script requires `docker` and the following `kibama.yml`
configuration.

```
# Create an agent policy for Fleet Server.
xpack.fleet.agentPolicies:
  - name: Fleet Server policy
    id: fleet-server-policy
    is_default_fleet_server: true
    # is_managed: true # Useful to mimic cloud environment
    description: Fleet server policy
    namespace: default
    package_policies:
      - name: Fleet Server
        package:
          name: fleet_server
        inputs:
          - type: fleet-server
            keep_enabled: true
            vars:
              - name: host
                value: 0.0.0.0
                frozen: true
              - name: port
                value: 8220
                frozen: true

# Set a default Fleet Server host.
xpack.fleet.fleetServerHosts:
  - id: default-fleet-server
    name: Default Fleet server
    is_default: true
    host_urls: ['https://host.docker.internal:8220'] # For running a Fleet Server Docker container

# Set a default Elasticsearch output.
xpack.fleet.outputs:
  - id: es-default-output
    name: Default output
    type: elasticsearch
    is_default: true
    is_default_monitoring: true
    hosts: ['http://host.docker.internal:9200'] # For enrolling dockerized agents
```

---------

Co-authored-by: kibanamachine <[email protected]>
Co-authored-by: Shahzad <[email protected]>

.github/CODEOWNERS

@@ -862,6 +862,7 @@ packages/kbn-stdio-dev-helpers @elastic/kibana-operations
 packages/kbn-storybook @elastic/kibana-operations
 x-pack/plugins/observability_solution/synthetics/e2e @elastic/obs-ux-management-team
 x-pack/plugins/observability_solution/synthetics @elastic/obs-ux-management-team
+x-pack/packages/kbn-synthetics-private-location @elastic/obs-ux-management-team
 x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture @elastic/response-ops
 x-pack/test/plugin_api_perf/plugins/task_manager_performance @elastic/response-ops
 x-pack/plugins/task_manager @elastic/response-ops

package.json

@@ -870,6 +870,7 @@
     "@kbn/status-plugin-b-plugin": "link:test/server_integration/plugins/status_plugin_b",
     "@kbn/std": "link:packages/kbn-std",
     "@kbn/synthetics-plugin": "link:x-pack/plugins/observability_solution/synthetics",
+    "@kbn/synthetics-private-location": "link:x-pack/packages/kbn-synthetics-private-location",
     "@kbn/task-manager-fixture-plugin": "link:x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture",
     "@kbn/task-manager-performance-plugin": "link:x-pack/test/plugin_api_perf/plugins/task_manager_performance",
     "@kbn/task-manager-plugin": "link:x-pack/plugins/task_manager",

tsconfig.base.json

@@ -1718,6 +1718,8 @@
       "@kbn/synthetics-e2e/*": ["x-pack/plugins/observability_solution/synthetics/e2e/*"],
       "@kbn/synthetics-plugin": ["x-pack/plugins/observability_solution/synthetics"],
       "@kbn/synthetics-plugin/*": ["x-pack/plugins/observability_solution/synthetics/*"],
+      "@kbn/synthetics-private-location": ["x-pack/packages/kbn-synthetics-private-location"],
+      "@kbn/synthetics-private-location/*": ["x-pack/packages/kbn-synthetics-private-location/*"],
       "@kbn/task-manager-fixture-plugin": ["x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture"],
       "@kbn/task-manager-fixture-plugin/*": ["x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture/*"],
       "@kbn/task-manager-performance-plugin": ["x-pack/test/plugin_api_perf/plugins/task_manager_performance"],

x-pack/packages/kbn-synthetics-private-location/README.md

@@ -0,0 +1,56 @@
+# @kbn/synthetics-private-location
+
+Quickily start Fleet, enroll Elastic Agent, and create a private location.
+
+## Usage
+
+```
+node x-pack/scripts/synthetics_private_location.js
+```
+
+For available options, run `--help`.
+
+## Prerequistes
+
+This script requires `docker` and the following `kibama.yml` configuration.
+
+```
+# Create an agent policy for Fleet Server.
+xpack.fleet.agentPolicies:
+  - name: Fleet Server policy
+    id: fleet-server-policy
+    is_default_fleet_server: true
+    # is_managed: true # Useful to mimic cloud environment
+    description: Fleet server policy
+    namespace: default
+    package_policies:
+      - name: Fleet Server
+        package:
+          name: fleet_server
+        inputs:
+          - type: fleet-server
+            keep_enabled: true
+            vars:
+              - name: host
+                value: 0.0.0.0
+                frozen: true
+              - name: port
+                value: 8220
+                frozen: true
+
+# Set a default Fleet Server host.
+xpack.fleet.fleetServerHosts:
+  - id: default-fleet-server
+    name: Default Fleet server
+    is_default: true
+    host_urls: ['https://host.docker.internal:8220'] # For running a Fleet Server Docker container
+
+# Set a default Elasticsearch output.
+xpack.fleet.outputs:
+  - id: es-default-output
+    name: Default output
+    type: elasticsearch
+    is_default: true
+    is_default_monitoring: true
+    hosts: ['http://host.docker.internal:9200'] # For enrolling dockerized agents
+```

x-pack/packages/kbn-synthetics-private-location/index.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export type { CliOptions } from './src/types';
+export { run } from './src/run';
+export { cli } from './src/cli';
+// export { cleanup } from './src/cleanup';
+export { DEFAULTS } from './src/constants';

x-pack/packages/kbn-synthetics-private-location/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/packages/kbn-synthetics-private-location'],
+};

x-pack/packages/kbn-synthetics-private-location/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/synthetics-private-location",
+  "owner": "@elastic/obs-ux-management-team"
+}

x-pack/packages/kbn-synthetics-private-location/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/synthetics-private-location",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/packages/kbn-synthetics-private-location/src/cli.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ToolingLog } from '@kbn/tooling-log';
+import { parseCliOptions } from './lib/parse_cli_options';
+import { CliOptions } from './types';
+import { run } from './run';
+
+export async function cli(cliOptions?: CliOptions) {
+  const options = cliOptions ?? parseCliOptions();
+  const logger = new ToolingLog({ level: 'info', writeTo: process.stdout });
+  return run(options, logger);
+}

x-pack/packages/kbn-synthetics-private-location/src/constants.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+
+export const DEFAULTS = {
+  LOCATION_NAME: `Default location ${uuidv4()}`,
+  AGENT_POLICY_NAME: `Synthetics agent policy ${uuidv4()}`,
+  ELASTICSEARCH_HOST: 'http://localhost:9200',
+  KIBANA_URL: 'http://localhost:5601',
+  KIBANA_USERNAME: 'elastic',
+  KIBANA_PASSWORD: 'changeme',
+};

x-pack/packages/kbn-synthetics-private-location/src/lib/create_agent_policy.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isError } from 'lodash';
+import { ToolingLog } from '@kbn/tooling-log';
+import { CliOptions } from '../types';
+import type { KibanaAPIClient } from './kibana_api_client';
+
+export async function createElasticAgentPolicy(
+  { agentPolicyName }: CliOptions,
+  logger: ToolingLog,
+  kibanaApiClient: KibanaAPIClient
+) {
+  try {
+    const response = await kibanaApiClient.sendRequest({
+      method: 'post',
+      url: 'api/fleet/agent_policies',
+      data: {
+        name: agentPolicyName,
+        description: '',
+        namespace: 'default',
+        monitoring_enabled: ['logs', 'metrics'],
+        inactivity_timeout: 1209600,
+        is_protected: false,
+      },
+    });
+
+    logger.info(`Generated elastic agent policy`);
+    return response.data;
+  } catch (error) {
+    if (isError(error)) {
+      logger.error(`Error generating elastic agent policy: ${error.message} ${error.stack}`);
+    }
+    throw error;
+  }
+}

x-pack/packages/kbn-synthetics-private-location/src/lib/create_private_location.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isError } from 'lodash';
+import { ToolingLog } from '@kbn/tooling-log';
+import { CliOptions } from '../types';
+import { KibanaAPIClient } from './kibana_api_client';
+
+export async function createPrivateLocation(
+  { kibanaUrl, kibanaPassword, kibanaUsername, locationName }: CliOptions,
+  logger: ToolingLog,
+  kibanaApiClient: KibanaAPIClient,
+  agentPolicyId: string
+) {
+  try {
+    const response = await kibanaApiClient.sendRequest({
+      method: 'post',
+      url: 'api/synthetics/private_locations',
+      data: {
+        label: locationName,
+        agentPolicyId,
+      },
+    });
+
+    logger.info(`Synthetics private location created successfully`);
+    return response.data;
+  } catch (error) {
+    if (isError(error)) {
+      logger.error(`Error creating synthetics private location: ${error.message} ${error.stack}`);
+    }
+    throw error;
+  }
+}

x-pack/packages/kbn-synthetics-private-location/src/lib/enroll_agent.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { spawn, spawnSync } from 'child_process';
+import * as path from 'path';
+import { CliOptions } from '../types';
+import { KibanaAPIClient } from './kibana_api_client';
+
+export async function enrollAgent(
+  { kibanaUrl, elasticsearchHost }: CliOptions,
+  enrollmentToken: string,
+  kibanaApiClient: KibanaAPIClient
+) {
+  const formattedKibanaURL = new URL(kibanaUrl);
+  const formattedElasticsearchHost = new URL(elasticsearchHost);
+  if (formattedKibanaURL.hostname === 'localhost') {
+    formattedKibanaURL.hostname = 'host.docker.internal';
+  }
+  if (formattedElasticsearchHost.hostname === 'localhost') {
+    formattedElasticsearchHost.hostname = 'host.docker.internal';
+  }
+  const version = `${await kibanaApiClient.getKibanaVersion()}-SNAPSHOT`;
+  await new Promise((res, rej) => {
+    try {
+      const fleetProcess = spawn(
+        'docker',
+        [
+          'run',
+          '-e',
+          'FLEET_SERVER_ENABLE=1',
+          '-e',
+          `FLEET_SERVER_ELASTICSEARCH_HOST=${formattedElasticsearchHost.origin}`,
+          '-e',
+          'FLEET_SERVER_POLICY_ID=fleet-server-policy',
+          '-e',
+          'FLEET_INSECURE=1',
+          '-e',
+          `KIBANA_HOST=${formattedKibanaURL.origin}`,
+          '-e',
+          'KIBANA_USERNAME=elastic',
+          '-e',
+          'KIBANA_PASSWORD=changeme',
+          '-e',
+          'KIBANA_FLEET_SETUP=1',
+          '-p',
+          '8220:8220',
+          '--rm',
+          `docker.elastic.co/beats/elastic-agent:${version}`,
+        ],
+        {
+          shell: true,
+          cwd: path.join(__dirname, '../'),
+          timeout: 120000,
+        }
+      );
+      setTimeout(res, 10_000);
+      fleetProcess.on('error', rej);
+    } catch (error) {
+      rej(error);
+    }
+  });
+
+  spawnSync(
+    'docker',
+    [
+      'run',
+      '-e',
+      'FLEET_URL=https://host.docker.internal:8220',
+      '-e',
+      'FLEET_ENROLL=1',
+      '-e',
+      `FLEET_ENROLLMENT_TOKEN=${enrollmentToken}`,
+      '-e',
+      'FLEET_INSECURE=1',
+      '--rm',
+      `docker.elastic.co/beats/elastic-agent-complete:${version}`,
+    ],
+    {
+      stdio: 'inherit',
+    }
+  );
+}

x-pack/packages/kbn-synthetics-private-location/src/lib/fetch_agent_policy_enrollment_token.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isError } from 'lodash';
+import { ToolingLog } from '@kbn/tooling-log';
+import { KibanaAPIClient } from './kibana_api_client';
+import { CliOptions } from '../types';
+
+export async function fetchAgentPolicyEnrollmentToken(
+  { kibanaUrl, kibanaPassword, kibanaUsername }: CliOptions,
+  logger: ToolingLog,
+  kibanaApiClient: KibanaAPIClient,
+  agentPolicyId: string
+) {
+  try {
+    const response = await kibanaApiClient.sendRequest({
+      method: 'get',
+      url: `api/fleet/enrollment_api_keys?kuery=policy_id:${agentPolicyId}`,
+    });
+
+    logger.info(`Fetching agent policy enrollment token`);
+    return response.data;
+  } catch (error) {
+    if (isError(error)) {
+      logger.error(`Error fetching agent enrollment token: ${error.message} ${error.stack}`);
+    }
+    throw error;
+  }
+}

x-pack/packages/kbn-synthetics-private-location/src/lib/generate_fleet_service_token.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isError } from 'lodash';
+import { ToolingLog } from '@kbn/tooling-log';
+import { CliOptions } from '../types';
+import { KibanaAPIClient } from './kibana_api_client';
+
+export async function generateFleetServiceToken(
+  { kibanaUrl, kibanaPassword, kibanaUsername }: CliOptions,
+  logger: ToolingLog,
+  kibanaApiClient: KibanaAPIClient
+) {
+  try {
+    const response = await kibanaApiClient.sendRequest({
+      method: 'post',
+      url: 'api/fleet/service_tokens',
+    });
+
+    logger.info(`Generated fleet server service token saved`);
+    return response.data;
+  } catch (error) {
+    if (isError(error)) {
+      logger.error(`Error generating fleet server service token: ${error.message} ${error.stack}`);
+    }
+    throw error;
+  }
+}