Skip to content

Commit

Permalink
Merge branch 'main' into bfilar.ml-refactor-2
Browse files Browse the repository at this point in the history
  • Loading branch information
randomuserid authored May 16, 2022
2 parents 55acc7c + 8a21895 commit f579bc7
Show file tree
Hide file tree
Showing 110 changed files with 4,384 additions and 1,455 deletions.
48 changes: 24 additions & 24 deletions .buildkite/scripts/steps/artifacts/cloud.sh
Original file line number Diff line number Diff line change
Expand Up @@ -43,39 +43,39 @@ jq '
' .buildkite/scripts/steps/cloud/deploy.json > "$DEPLOYMENT_SPEC"

ecctl deployment create --track --output json --file "$DEPLOYMENT_SPEC" &> "$LOGS"
CLOUD_DEPLOYMENT_USERNAME=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$LOGS")
CLOUD_DEPLOYMENT_PASSWORD=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$LOGS")
CLOUD_DEPLOYMENT_USERNAME=$(jq -r --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$LOGS")
CLOUD_DEPLOYMENT_PASSWORD=$(jq -r --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$LOGS")
CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$LOGS")
CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$LOGS")

CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')

# NOTE: disabled pending log sanitization
# echo "--- Setup FTR"
# export TEST_KIBANA_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol)")
# export TEST_KIBANA_HOSTNAME=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)")
# export TEST_KIBANA_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)")
# export TEST_KIBANA_USERNAME=$CLOUD_DEPLOYMENT_USERNAME"
# export TEST_KIBANA_PASS=$CLOUD_DEPLOYMENT_PASSWORD"
echo "Kibana: $CLOUD_DEPLOYMENT_KIBANA_URL"
echo "ES: $CLOUD_DEPLOYMENT_ELASTICSEARCH_URL"

# export TEST_ES_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol)")
# export TEST_ES_HOSTNAME==$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)")
# export TEST_ES_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)")
# export TEST_ES_USER="$CLOUD_DEPLOYMENT_USERNAME"
# export TEST_ES_PASS="$CLOUD_DEPLOYMENT_PASSWORD"
function shutdown {
echo "--- Shutdown deployment"
ecctl deployment shutdown "$CLOUD_DEPLOYMENT_ID" --force --track --output json &> "$LOGS"
}
trap "shutdown" EXIT

# export TEST_BROWSER_HEADLESS=1
export TEST_KIBANA_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol.replace(':', ''))")
export TEST_KIBANA_HOSTNAME=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)")
export TEST_KIBANA_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)")
export TEST_KIBANA_USERNAME="$CLOUD_DEPLOYMENT_USERNAME"
export TEST_KIBANA_PASSWORD="$CLOUD_DEPLOYMENT_PASSWORD"

# Error: attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ConnectionError: self signed certificate in certificate chain
# export NODE_TLS_REJECT_UNAUTHORIZED=0
export TEST_ES_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_ELASTICSEARCH_URL').protocol.replace(':', ''))")
export TEST_ES_HOSTNAME=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_ELASTICSEARCH_URL').hostname)")
export TEST_ES_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_ELASTICSEARCH_URL').port)")
export TEST_ES_USERNAME="$CLOUD_DEPLOYMENT_USERNAME"
export TEST_ES_PASSWORD="$CLOUD_DEPLOYMENT_PASSWORD"

# echo "--- Run default functional tests"
# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud -exclude-tag=skipCloud
export TEST_BROWSER_HEADLESS=1

# echo "--- Run x-pack functional tests"
# cd x-pack
# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud -exclude-tag=skipCloud
# Error: attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ConnectionError: self signed certificate in certificate chain
export NODE_TLS_REJECT_UNAUTHORIZED=0

echo "--- Shutdown deployment"
ecctl deployment shutdown "$CLOUD_DEPLOYMENT_ID" --force --track --output json &> "$LOGS"
echo "--- FTR - Reporting"
node --no-warnings scripts/functional_test_runner.js --config x-pack/test/functional/apps/visualize/config.ts --include-tag=smoke --quiet
2 changes: 1 addition & 1 deletion .eslintignore
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ snapshots.js
/x-pack/plugins/reporting/server/export_types/printable_pdf_v2/server/lib/pdf/assets/**

# package overrides
/packages/elastic-eslint-config-kibana
/packages/kbn-eslint-config
/packages/kbn-plugin-generator/template
/packages/kbn-generate/templates
/packages/kbn-pm/dist
Expand Down
4 changes: 2 additions & 2 deletions .eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ const RESTRICTED_IMPORTS = [
module.exports = {
root: true,

extends: ['@elastic/eslint-config-kibana', 'plugin:@elastic/eui/recommended'],
extends: ['plugin:@elastic/eui/recommended', '@kbn/eslint-config'],

overrides: [
/**
Expand Down Expand Up @@ -304,7 +304,7 @@ module.exports = {
*/
{
files: [
'packages/elastic-eslint-config-kibana/**/*.{js,mjs,ts,tsx}',
'packages/kbn-eslint-config/**/*.{js,mjs,ts,tsx}',
'packages/kbn-datemath/**/*.{js,mjs,ts,tsx}',
],
rules: {
Expand Down
1 change: 1 addition & 0 deletions docs/api/cases/cases-api-add-comment.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ The API returns details about the case and its comments. For example:
"syncAlerts":false
},
"owner": "cases",
"duration": null,
"closed_at": null,
"closed_by": null,
"created_at": "2022-03-24T00:37:03.906Z",
Expand Down
1 change: 1 addition & 0 deletions docs/api/cases/cases-api-create.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ the case identifier, version, and creation time. For example:
},
"owner": "securitySolution",
"description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants. Operation bubblegum is active. Repeat - operation bubblegum is now active",
"duration": null,
"closed_at": null,
"closed_by": null,
"created_at": "2022-05-13T09:16:17.416Z",
Expand Down
2 changes: 2 additions & 0 deletions docs/api/cases/cases-api-find-cases.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ The API returns a JSON object listing the retrieved cases. For example:
"syncAlerts": true
},
"owner": "securitySolution",
"duration": null,
"closed_at": null,
"closed_by": null,
"created_at": "2022-03-29T13:03:23.533Z",
Expand Down Expand Up @@ -164,6 +165,7 @@ The API returns a JSON object listing the retrieved cases. For example:
"syncAlerts": false
},
"owner": "cases",
"duration": null,
"closed_at": null,
"closed_by": null,
"created_at": "2022-03-29T11:30:02.658Z",
Expand Down
2 changes: 2 additions & 0 deletions docs/api/cases/cases-api-get-case.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ The API returns a JSON object with the retrieved case. For example:
"version": "Wzk4LDFd",
"comments": [],
"totalComment": 0,
"totalAlerts": 0,
"closed_at": null,
"closed_by": null,
"created_at": "2020-03-29T11:30:02.658Z",
Expand Down Expand Up @@ -90,6 +91,7 @@ The API returns a JSON object with the retrieved case. For example:
"syncAlerts": true
},
"owner": "securitySolution",
"duration": null,
"tags": [
"phishing",
"social engineering",
Expand Down
1 change: 1 addition & 0 deletions docs/api/cases/cases-api-push.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ The API returns a JSON object representing the pushed case. For example:
"syncAlerts": true
},
"owner": "securitySolution",
"duration": null,
"closed_at": null,
"closed_by": null,
"created_at": "2022-03-29T11:30:02.658Z",
Expand Down
23 changes: 12 additions & 11 deletions docs/api/cases/cases-api-update-comment.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -106,35 +106,36 @@ The API returns details about the case and its comments. For example:
"comments":[{
"id": "8af6ac20-74f6-11ea-b83a-553aecdb28b6",
"version": "WzIwNjM3LDFd",
"comment":"That is nothing - Ethan Hunt answered a targeted social media campaign promoting phishy pension schemes to IMF operatives. Even worse, he likes baked beans.",
"type":"user",
"owner":"cases",
"created_at":"2022-03-24T00:37:10.832Z",
"comment": "That is nothing - Ethan Hunt answered a targeted social media campaign promoting phishy pension schemes to IMF operatives. Even worse, he likes baked beans.",
"type": "user",
"owner": "cases",
"created_at": "2022-03-24T00:37:10.832Z",
"created_by": {
"email": "[email protected]",
"full_name": "Ms Moneypenny",
"username": "moneypenny"
},
"pushed_at":null,
"pushed_by":null,
"updated_at":"2022-03-24T01:27:06.210Z",
"pushed_at": null,
"pushed_by": null,
"updated_at": "2022-03-24T01:27:06.210Z",
"updated_by": {
"email": "[email protected]",
"full_name": "James Bond",
"username": "_007"
}
}
],
"totalAlerts":0,
"totalAlerts": 0,
"id": "293f1bc0-74f6-11ea-b83a-553aecdb28b6",
"version": "WzIwNjM2LDFd",
"totalComment": 1,
"title": "This case will self-destruct in 5 seconds",
"tags": ["phishing","social engineering"],
"tags": ["phishing","social engineering"],
"description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants.",
"settings": {"syncAlerts":false},
"owner": "cases","
closed_at": null,
"owner": "cases",
"duration": null,
"closed_at": null,
"closed_by": null,
"created_at": "2022-03-24T00:37:03.906Z",
"created_by": {
Expand Down
1 change: 1 addition & 0 deletions docs/api/cases/cases-api-update.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ The API returns the updated case with a new `version` value. For example:
},
"owner": "securitySolution",
"description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants. Operation bubblegum is active. Repeat - operation bubblegum is now active!",
"duration": null,
"closed_at": null,
"closed_by": null,
"created_at": "2022-05-13T09:16:17.416Z",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ If you’re installing dependencies and seeing an error that looks
something like

....
Unsupported URL Type: link:packages/elastic-eslint-config-kibana
Unsupported URL Type: link:packages/kbn-eslint-config
....

you’re likely running `npm`. To install dependencies in {kib} you
Expand Down
7 changes: 7 additions & 0 deletions docs/setup/upgrade/resolving-migration-failures.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -187,3 +187,10 @@ PUT /_cluster/settings
}
}
--------------------------------------------

[float]
[[cluster-shard-limit-exceeded]]
==== {es} cluster shard limit exceeded
When upgrading, {kib} creates new indices requiring a small number of new shards. If the amount of open {es} shards approaches or exceeds the {es} `cluster.max_shards_per_node` setting, {kib} is unable to complete the upgrade. Ensure that {kib} is able to add at least 10 more shards by removing indices to clear up resources, or by increasing the `cluster.max_shards_per_node` setting.

For more information, refer to the documentation on {ref}/allocation-total-shards.html[total shards per node].
6 changes: 3 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,8 @@
"@dnd-kit/core": "^3.1.1",
"@dnd-kit/sortable": "^4.0.0",
"@dnd-kit/utilities": "^2.0.0",
"@elastic/apm-rum": "^5.11.0",
"@elastic/apm-rum-react": "^1.4.0",
"@elastic/apm-rum": "^5.11.1",
"@elastic/apm-rum-react": "^1.4.1",
"@elastic/apm-synthtrace": "link:bazel-bin/packages/elastic-apm-synthtrace",
"@elastic/charts": "46.0.1",
"@elastic/datemath": "5.0.3",
Expand Down Expand Up @@ -470,7 +470,6 @@
"@cypress/code-coverage": "^3.9.12",
"@cypress/snapshot": "^2.1.7",
"@cypress/webpack-preprocessor": "^5.6.0",
"@elastic/eslint-config-kibana": "link:bazel-bin/packages/elastic-eslint-config-kibana",
"@elastic/eslint-plugin-eui": "0.0.2",
"@elastic/github-checks-reporter": "0.0.20b3",
"@elastic/makelogs": "^6.0.0",
Expand All @@ -493,6 +492,7 @@
"@kbn/docs-utils": "link:bazel-bin/packages/kbn-docs-utils",
"@kbn/es": "link:bazel-bin/packages/kbn-es",
"@kbn/es-archiver": "link:bazel-bin/packages/kbn-es-archiver",
"@kbn/eslint-config": "link:bazel-bin/packages/kbn-eslint-config",
"@kbn/eslint-plugin-eslint": "link:bazel-bin/packages/kbn-eslint-plugin-eslint",
"@kbn/expect": "link:bazel-bin/packages/kbn-expect",
"@kbn/find-used-node-modules": "link:bazel-bin/packages/kbn-find-used-node-modules",
Expand Down
2 changes: 1 addition & 1 deletion packages/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ filegroup(
"//packages/analytics/shippers/elastic_v3/server:build",
"//packages/analytics/shippers/fullstory:build",
"//packages/elastic-apm-synthtrace:build",
"//packages/elastic-eslint-config-kibana:build",
"//packages/elastic-safer-lodash-set:build",
"//packages/kbn-ace:build",
"//packages/kbn-alerts:build",
Expand Down Expand Up @@ -43,6 +42,7 @@ filegroup(
"//packages/kbn-es-archiver:build",
"//packages/kbn-es-query:build",
"//packages/kbn-es:build",
"//packages/kbn-eslint-config:build",
"//packages/kbn-eslint-plugin-eslint:build",
"//packages/kbn-eslint-plugin-imports:build",
"//packages/kbn-expect:build",
Expand Down
4 changes: 4 additions & 0 deletions packages/elastic-apm-synthtrace/src/lib/apm/apm_fields.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,10 @@ export type ApmFields = Fields &
'span.destination.service.response_time.count': number;
'span.self_time.count': number;
'span.self_time.sum.us': number;
'span.links': Array<{
trace: { id: string };
span: { id: string };
}>;
'cloud.provider': string;
'cloud.project.name': string;
'cloud.service.name': string;
Expand Down
114 changes: 114 additions & 0 deletions packages/elastic-apm-synthtrace/src/scripts/examples/04_span_links.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import { compact, shuffle } from 'lodash';
import { apm, ApmFields, EntityArrayIterable, timerange } from '../..';
import { generateLongId, generateShortId } from '../../lib/utils/generate_id';
import { Scenario } from '../scenario';

function generateExternalSpanLinks() {
// randomly creates external span links 0 - 10
return Array(Math.floor(Math.random() * 11))
.fill(0)
.map(() => ({ span: { id: generateLongId() }, trace: { id: generateShortId() } }));
}

function getSpanLinksFromEvents(events: ApmFields[]) {
return compact(
events.map((event) => {
const spanId = event['span.id'];
return spanId ? { span: { id: spanId }, trace: { id: event['trace.id']! } } : undefined;
})
);
}

const scenario: Scenario<ApmFields> = async () => {
return {
generate: ({ from, to }) => {
const producerInternalOnlyInstance = apm
.service('producer-internal-only', 'production', 'go')
.instance('instance-a');
const producerInternalOnlyEvents = timerange(
new Date('2022-04-25T19:00:00.000Z'),
new Date('2022-04-25T19:01:00.000Z')
)
.interval('1m')
.rate(1)
.generator((timestamp) => {
return producerInternalOnlyInstance
.transaction('Transaction A')
.timestamp(timestamp)
.duration(1000)
.success()
.children(
producerInternalOnlyInstance
.span('Span A', 'custom')
.timestamp(timestamp + 50)
.duration(100)
.success()
);
});

const producerInternalOnlyApmFields = producerInternalOnlyEvents.toArray();
const spanASpanLink = getSpanLinksFromEvents(producerInternalOnlyApmFields);

const producerConsumerInstance = apm
.service('producer-consumer', 'production', 'java')
.instance('instance-b');
const producerConsumerEvents = timerange(from, to)
.interval('1m')
.rate(1)
.generator((timestamp) => {
return producerConsumerInstance
.transaction('Transaction B')
.timestamp(timestamp)
.duration(1000)
.success()
.children(
producerConsumerInstance
.span('Span B', 'external')
.defaults({
'span.links': shuffle([...generateExternalSpanLinks(), ...spanASpanLink]),
})
.timestamp(timestamp + 50)
.duration(900)
.success()
);
});

const producerConsumerApmFields = producerConsumerEvents.toArray();
const spanBSpanLink = getSpanLinksFromEvents(producerConsumerApmFields);

const consumerInstance = apm.service('consumer', 'production', 'ruby').instance('instance-c');
const consumerEvents = timerange(from, to)
.interval('1m')
.rate(1)
.generator((timestamp) => {
return consumerInstance
.transaction('Transaction C')
.timestamp(timestamp)
.duration(1000)
.success()
.children(
consumerInstance
.span('Span C', 'external')
.defaults({ 'span.links': spanBSpanLink })
.timestamp(timestamp + 50)
.duration(900)
.success()
);
});

return new EntityArrayIterable(producerInternalOnlyApmFields)
.merge(consumerEvents)
.merge(new EntityArrayIterable(producerConsumerApmFields));
},
};
};

export default scenario;
2 changes: 0 additions & 2 deletions packages/elastic-eslint-config-kibana/.npmignore

This file was deleted.

Loading

0 comments on commit f579bc7

Please sign in to comment.