Merge branch 'master' into eui/40

docs/management/connectors/action-types/servicenow-sir.asciidoc

@@ -37,6 +37,7 @@ Use the <<action-settings, Action configuration settings>> to customize connecto
    actionTypeId: .servicenow-sir
    config:
      apiUrl: https://example.service-now.com/
+     isLegacy: false
    secrets:
      username: testuser
      password: passwordkeystorevalue
@@ -45,6 +46,9 @@ Use the <<action-settings, Action configuration settings>> to customize connecto
 Config defines information for the connector type.
 
 `apiUrl`:: An address that corresponds to *URL*.
+`isLegacy`:: A boolean that indicates if the connector should use the Table API (legacy) or the Import Set API.
+
+Note: If `isLegacy` is set to false the Elastic application should be installed in ServiceNow.
 
 Secrets defines sensitive information for the connector type.
 

docs/management/connectors/action-types/servicenow.asciidoc

@@ -37,6 +37,7 @@ Use the <<action-settings, Action configuration settings>> to customize connecto
    actionTypeId: .servicenow
    config:
      apiUrl: https://example.service-now.com/
+     isLegacy: false
    secrets:
      username: testuser
      password: passwordkeystorevalue
@@ -45,6 +46,9 @@ Use the <<action-settings, Action configuration settings>> to customize connecto
 Config defines information for the connector type.
 
 `apiUrl`:: An address that corresponds to *URL*.
+`isLegacy`:: A boolean that indicates if the connector should use the Table API (legacy) or the Import Set API.
+
+Note: If `isLegacy` is set to false the Elastic application should be installed in ServiceNow.
 
 Secrets defines sensitive information for the connector type.
 

package.json

@@ -98,7 +98,7 @@
     "@elastic/apm-generator": "link:bazel-bin/packages/elastic-apm-generator",
     "@elastic/apm-rum": "^5.9.1",
     "@elastic/apm-rum-react": "^1.3.1",
-    "@elastic/charts": "37.0.0",
+    "@elastic/charts": "38.0.1",
     "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath",
     "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.21",
     "@elastic/ems-client": "7.16.0",
@@ -289,7 +289,6 @@
     "moment-timezone": "^0.5.27",
     "monaco-editor": "^0.22.3",
     "mustache": "^2.3.2",
-    "ngreact": "^0.5.1",
     "nock": "12.0.3",
     "node-fetch": "^2.6.1",
     "node-forge": "^0.10.0",
@@ -744,7 +743,7 @@
     "jsondiffpatch": "0.4.1",
     "license-checker": "^16.0.0",
     "listr": "^0.14.1",
-    "lmdb-store": "^1.6.8",
+    "lmdb-store": "^1.6.11",
     "marge": "^1.0.1",
     "micromatch": "3.1.10",
     "minimist": "^1.2.5",
@@ -765,6 +764,7 @@
     "oboe": "^2.1.4",
     "parse-link-header": "^1.0.1",
     "pbf": "3.2.1",
+    "pdf-to-img": "^1.1.1",
     "pirates": "^4.0.1",
     "pixelmatch": "^5.1.0",
     "postcss": "^7.0.32",

packages/elastic-apm-generator/BUILD.bazel

@@ -25,6 +25,7 @@ NPM_MODULE_EXTRA_FILES = [
 ]
 
 RUNTIME_DEPS = [
+  "//packages/elastic-datemath",
   "@npm//@elastic/elasticsearch",
   "@npm//lodash",
   "@npm//moment",
@@ -36,6 +37,7 @@ RUNTIME_DEPS = [
 ]
 
 TYPES_DEPS = [
+  "//packages/elastic-datemath",
   "@npm//@elastic/elasticsearch",
   "@npm//moment",
   "@npm//p-limit",

packages/elastic-apm-generator/README.md

@@ -11,7 +11,7 @@ This section assumes that you've installed Kibana's dependencies by running `yar
 This library can currently be used in two ways:
 
 - Imported as a Node.js module, for instance to be used in Kibana's functional test suite.
-- With a command line interface, to index data based on some example scenarios.
+- With a command line interface, to index data based on a specified scenario.
 
 ### Using the Node.js module
 
@@ -32,7 +32,7 @@ const instance = service('synth-go', 'production', 'go')
   .instance('instance-a');
 
 const from = new Date('2021-01-01T12:00:00.000Z').getTime();
-const to = new Date('2021-01-01T12:00:00.000Z').getTime() - 1;
+const to = new Date('2021-01-01T12:00:00.000Z').getTime();
 
 const traceEvents = timerange(from, to)
   .interval('1m')
@@ -82,12 +82,26 @@ const esEvents = toElasticsearchOutput([
 
 ### CLI
 
-Via the CLI, you can upload examples. The supported examples are listed in `src/lib/es.ts`. A `--target` option that specifies the Elasticsearch URL should be defined when running the `example` command. Here's an example:
+Via the CLI, you can upload scenarios, either using a fixed time range or continuously generating data. Some examples are available in in `src/scripts/examples`. Here's an example for live data:
 
-`$ node packages/elastic-apm-generator/src/scripts/es.js example simple-trace --target=http://admin:changeme@localhost:9200`
+`$ node packages/elastic-apm-generator/src/scripts/run packages/elastic-apm-generator/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --live`
+
+For a fixed time window:
+`$ node packages/elastic-apm-generator/src/scripts/run packages/elastic-apm-generator/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now`
+
+The script will try to automatically find bootstrapped APM indices. __If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.__
 
 The following options are supported:
-- `to`: the end of the time range, in ISO format. By default, the current time will be used.
-- `from`: the start of the time range, in ISO format. By default, `to` minus 15 minutes will be used.
-- `apm-server-version`: the version used in the index names bootstrapped by APM Server, e.g. `7.16.0`. __If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.__
+| Option         | Description                                             | Default      |
+| -------------- | ------------------------------------------------------- | ------------ |
+| `--from`       | The start of the time window.                           | `now - 15m`  |
+| `--to`         | The end of the time window.                             | `now`        |
+| `--live`       | Continously ingest data                                 | `false`      |
+| `--bucketSize` | Size of bucket for which to generate data.              | `15m`        |
+| `--clean`      | Clean APM indices before indexing new data.             | `false`      |
+| `--interval`   | The interval at which to index data.                    | `10s`        |
+| `--logLevel`   | Log level.                                              | `info`       |
+| `--lookback`   | The lookback window for which data should be generated. | `15m`        |
+| `--target`     | Elasticsearch target, including username/password.      | **Required** |
+| `--workers`    | Amount of simultaneously connected ES clients.          | `1`          |
 

packages/elastic-apm-generator/src/.eslintrc.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  rules: {
+    'import/no-default-export': 'off',
+  },
+};

packages/elastic-apm-generator/src/lib/interval.ts

@@ -21,7 +21,7 @@ export class Interval {
       throw new Error('Failed to parse interval');
     }
     const timestamps: number[] = [];
-    while (now <= this.to) {
+    while (now < this.to) {
       timestamps.push(...new Array<number>(rate).fill(now));
       now = moment(now)
         .add(Number(args[1]), args[2] as any)

packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts

@@ -10,7 +10,25 @@ import { set } from 'lodash';
 import { getObserverDefaults } from '../..';
 import { Fields } from '../entity';
 
-export function toElasticsearchOutput(events: Fields[], versionOverride?: string) {
+export interface ElasticsearchOutput {
+  _index: string;
+  _source: unknown;
+}
+
+export interface ElasticsearchOutputWriteTargets {
+  transaction: string;
+  span: string;
+  error: string;
+  metric: string;
+}
+
+export function toElasticsearchOutput({
+  events,
+  writeTargets,
+}: {
+  events: Fields[];
+  writeTargets: ElasticsearchOutputWriteTargets;
+}): ElasticsearchOutput[] {
   return events.map((event) => {
     const values = {
       ...event,
@@ -29,7 +47,7 @@ export function toElasticsearchOutput(events: Fields[], versionOverride?: string
       set(document, key, val);
     }
     return {
-      _index: `apm-${versionOverride || values['observer.version']}-${values['processor.event']}`,
+      _index: writeTargets[event['processor.event'] as keyof ElasticsearchOutputWriteTargets],
       _source: document,
     };
   });

packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts

@@ -9,12 +9,12 @@
 import { service, timerange, getTransactionMetrics, getSpanDestinationMetrics } from '../..';
 import { getBreakdownMetrics } from '../../lib/utils/get_breakdown_metrics';
 
-export function simpleTrace(from: number, to: number) {
+export default function ({ from, to }: { from: number; to: number }) {
   const instance = service('opbeans-go', 'production', 'go').instance('instance');
 
   const range = timerange(from, to);
 
-  const transactionName = '240rpm/60% 1000ms';
+  const transactionName = '240rpm/75% 1000ms';
 
   const successfulTraceEvents = range
     .interval('1s')

packages/elastic-apm-generator/src/scripts/es.js → packages/elastic-apm-generator/src/scripts/run.js

@@ -12,4 +12,4 @@ require('@babel/register')({
   presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'],
 });
 
-require('./es.ts');
+require('./run.ts');