From 64cc1f8441e3657175ad0eae720daae403d45a24 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Thu, 6 Apr 2023 10:21:28 +0930 Subject: [PATCH] network_traffic: remove beta tags on data streams and add flow tests (#5778) * remove beta tags on data streams * add flow is final filter for flows dashboard and flow tests --- .../network_traffic/_dev/build/docs/README.md | 2 + packages/network_traffic/changelog.yml | 8 + .../data_stream/amqp/manifest.yml | 1 - .../data_stream/cassandra/manifest.yml | 1 - .../data_stream/dhcpv4/manifest.yml | 1 - .../data_stream/dns/manifest.yml | 1 - .../system/test-http-get-2k-file-config.yml | 6 + .../test/system/test-icmp-2-pings-config.yml | 6 + .../test/system/test-icmp4-ping-config.yml | 6 + .../test/system/test-icmp6-ping-config.yml | 6 + .../elasticsearch/ingest_pipeline/default.yml | 28 +- .../data_stream/flow/fields/ecs.yml | 10 + .../data_stream/flow/manifest.yml | 1 - .../data_stream/flow/sample_event.json | 85 ++++ .../data_stream/http/manifest.yml | 1 - .../data_stream/icmp/manifest.yml | 1 - .../data_stream/memcached/manifest.yml | 1 - .../data_stream/mongodb/manifest.yml | 1 - .../data_stream/mysql/manifest.yml | 1 - .../data_stream/nfs/manifest.yml | 1 - .../data_stream/pgsql/manifest.yml | 1 - .../data_stream/redis/manifest.yml | 1 - .../data_stream/sip/manifest.yml | 1 - .../data_stream/thrift/manifest.yml | 1 - .../data_stream/tls/manifest.yml | 1 - packages/network_traffic/docs/README.md | 95 ++++ .../dashboard/network_traffic-flows.json | 426 +++++++++--------- packages/network_traffic/manifest.yml | 2 +- 28 files changed, 466 insertions(+), 230 deletions(-) create mode 100644 packages/network_traffic/data_stream/flow/_dev/test/system/test-http-get-2k-file-config.yml create mode 100644 packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp-2-pings-config.yml create mode 100644 packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp4-ping-config.yml create mode 100644 packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp6-ping-config.yml create mode 100644 packages/network_traffic/data_stream/flow/sample_event.json diff --git a/packages/network_traffic/_dev/build/docs/README.md b/packages/network_traffic/_dev/build/docs/README.md index 1f6397e3273..e5bd6260639 100644 --- a/packages/network_traffic/_dev/build/docs/README.md +++ b/packages/network_traffic/_dev/build/docs/README.md @@ -141,6 +141,8 @@ The default value is 10s. {{fields "flow"}} +{{event "flow"}} + ## Protocols ### AMQP diff --git a/packages/network_traffic/changelog.yml b/packages/network_traffic/changelog.yml index e1e852fd3cc..ad4bcf195fb 100644 --- a/packages/network_traffic/changelog.yml +++ b/packages/network_traffic/changelog.yml @@ -1,4 +1,12 @@ # newer versions go on top +- version: "1.11.0" + changes: + - description: Add flow is final filter to Network flow dashboard. + type: enhancement + link: https://github.com/elastic/integrations/pull/5778 + - description: GA datastreams. + type: enhancement + link: https://github.com/elastic/integrations/pull/5778 - version: "1.10.1" changes: - description: Fix documentation for flows period. diff --git a/packages/network_traffic/data_stream/amqp/manifest.yml b/packages/network_traffic/data_stream/amqp/manifest.yml index 11f8aef8f00..4ec5b218e42 100644 --- a/packages/network_traffic/data_stream/amqp/manifest.yml +++ b/packages/network_traffic/data_stream/amqp/manifest.yml @@ -1,5 +1,4 @@ title: AMQP -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/cassandra/manifest.yml b/packages/network_traffic/data_stream/cassandra/manifest.yml index db98c83afa3..c8fbb394588 100644 --- a/packages/network_traffic/data_stream/cassandra/manifest.yml +++ b/packages/network_traffic/data_stream/cassandra/manifest.yml @@ -1,5 +1,4 @@ title: Cassandra -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/dhcpv4/manifest.yml b/packages/network_traffic/data_stream/dhcpv4/manifest.yml index ab531bd35e8..76145b9d5ae 100644 --- a/packages/network_traffic/data_stream/dhcpv4/manifest.yml +++ b/packages/network_traffic/data_stream/dhcpv4/manifest.yml @@ -1,5 +1,4 @@ title: DHCP -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/dns/manifest.yml b/packages/network_traffic/data_stream/dns/manifest.yml index fc1d91b39c8..09663525cdf 100644 --- a/packages/network_traffic/data_stream/dns/manifest.yml +++ b/packages/network_traffic/data_stream/dns/manifest.yml @@ -1,5 +1,4 @@ title: DNS -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/flow/_dev/test/system/test-http-get-2k-file-config.yml b/packages/network_traffic/data_stream/flow/_dev/test/system/test-http-get-2k-file-config.yml new file mode 100644 index 00000000000..82fb324e767 --- /dev/null +++ b/packages/network_traffic/data_stream/flow/_dev/test/system/test-http-get-2k-file-config.yml @@ -0,0 +1,6 @@ +vars: + interface: "{{SERVICE_LOGS_DIR}}/http_get_2k_file.pcap" +input: packet +data_stream: + vars: + flows.period: '-1s' diff --git a/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp-2-pings-config.yml b/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp-2-pings-config.yml new file mode 100644 index 00000000000..7aaee5f2e8f --- /dev/null +++ b/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp-2-pings-config.yml @@ -0,0 +1,6 @@ +vars: + interface: "{{SERVICE_LOGS_DIR}}/icmp_2_pings.pcap" +input: packet +data_stream: + vars: + flows.period: '-1s' diff --git a/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp4-ping-config.yml b/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp4-ping-config.yml new file mode 100644 index 00000000000..391541555b7 --- /dev/null +++ b/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp4-ping-config.yml @@ -0,0 +1,6 @@ +vars: + interface: "{{SERVICE_LOGS_DIR}}/icmp4_ping.pcap" +input: packet +data_stream: + vars: + flows.period: '1s' diff --git a/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp6-ping-config.yml b/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp6-ping-config.yml new file mode 100644 index 00000000000..5f94bcabb70 --- /dev/null +++ b/packages/network_traffic/data_stream/flow/_dev/test/system/test-icmp6-ping-config.yml @@ -0,0 +1,6 @@ +vars: + interface: "{{SERVICE_LOGS_DIR}}/icmp6_ping.pcap" +input: packet +data_stream: + vars: + flows.period: '1s' diff --git a/packages/network_traffic/data_stream/flow/elasticsearch/ingest_pipeline/default.yml b/packages/network_traffic/data_stream/flow/elasticsearch/ingest_pipeline/default.yml index 39ce858eb1a..e60c9efda24 100644 --- a/packages/network_traffic/data_stream/flow/elasticsearch/ingest_pipeline/default.yml +++ b/packages/network_traffic/data_stream/flow/elasticsearch/ingest_pipeline/default.yml @@ -5,7 +5,7 @@ processors: field: ecs.version value: '8.7.0' ## -# Set host.mac to dash separated upper case value +# Set {host,source,destination}.mac to dash separated upper case value # as per ECS recommendation ## - gsub: @@ -21,6 +21,32 @@ processors: - uppercase: field: host.mac ignore_missing: true +- gsub: + field: source.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true +- gsub: + field: source.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true +- uppercase: + field: source.mac + ignore_missing: true +- gsub: + field: destination.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true +- gsub: + field: destination.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true +- uppercase: + field: destination.mac + ignore_missing: true - pipeline: if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich diff --git a/packages/network_traffic/data_stream/flow/fields/ecs.yml b/packages/network_traffic/data_stream/flow/fields/ecs.yml index 339687c0c4c..9b9b41f5209 100644 --- a/packages/network_traffic/data_stream/flow/fields/ecs.yml +++ b/packages/network_traffic/data_stream/flow/fields/ecs.yml @@ -8,6 +8,10 @@ name: destination.bytes - external: ecs name: destination.ip +- external: ecs + name: destination.mac +- external: ecs + name: destination.packets - external: ecs name: destination.port - external: ecs @@ -36,6 +40,8 @@ name: network.forwarded_ip - external: ecs name: network.protocol +- external: ecs + name: network.packets - external: ecs name: network.transport - external: ecs @@ -62,8 +68,12 @@ name: source.bytes - external: ecs name: source.ip +- external: ecs + name: source.mac - external: ecs name: source.port +- external: ecs + name: source.packets - external: ecs name: client.geo.city_name - external: ecs diff --git a/packages/network_traffic/data_stream/flow/manifest.yml b/packages/network_traffic/data_stream/flow/manifest.yml index d0385032dce..8e98d048871 100644 --- a/packages/network_traffic/data_stream/flow/manifest.yml +++ b/packages/network_traffic/data_stream/flow/manifest.yml @@ -1,5 +1,4 @@ title: Flows -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/flow/sample_event.json b/packages/network_traffic/data_stream/flow/sample_event.json new file mode 100644 index 00000000000..dda72483269 --- /dev/null +++ b/packages/network_traffic/data_stream/flow/sample_event.json @@ -0,0 +1,85 @@ +{ + "@timestamp": "2023-04-04T23:12:40.755Z", + "agent": { + "ephemeral_id": "c368e835-b038-4610-b000-bc9fb23b35ab", + "id": "d35fb84a-73fb-4bc7-99b4-ac4df8c6ebb5", + "name": "docker-fleet-agent", + "type": "packetbeat", + "version": "8.6.2" + }, + "data_stream": { + "dataset": "network_traffic.flow", + "namespace": "ep", + "type": "logs" + }, + "destination": { + "bytes": 64, + "ip": "::1", + "packets": 1, + "port": 8000 + }, + "ecs": { + "version": "8.6.0" + }, + "elastic_agent": { + "id": "d35fb84a-73fb-4bc7-99b4-ac4df8c6ebb5", + "snapshot": false, + "version": "8.6.2" + }, + "event": { + "action": "network_flow", + "agent_id_status": "verified", + "category": [ + "network" + ], + "dataset": "network_traffic.flow", + "duration": 70523, + "end": "2023-04-04T23:12:30.760Z", + "ingested": "2023-04-04T23:12:41Z", + "kind": "event", + "start": "2023-04-04T23:12:30.759Z", + "type": [ + "connection" + ] + }, + "flow": { + "final": false, + "id": "QAT///////8A////IP8AAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAUAfeMg" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "f91b175388d443fca5c155815dfc2279", + "ip": [ + "192.168.208.7" + ], + "mac": [ + "02-42-C0-A8-D0-07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.49-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + } + }, + "network": { + "bytes": 152, + "community_id": "1:5y9AkdbV9U8xqD9dhlj6obkubHg=", + "packets": 2, + "transport": "tcp", + "type": "ipv6" + }, + "source": { + "bytes": 88, + "ip": "::1", + "packets": 1, + "port": 51320 + }, + "type": "flow" +} \ No newline at end of file diff --git a/packages/network_traffic/data_stream/http/manifest.yml b/packages/network_traffic/data_stream/http/manifest.yml index c4dceba060b..7d8d20f0d5a 100644 --- a/packages/network_traffic/data_stream/http/manifest.yml +++ b/packages/network_traffic/data_stream/http/manifest.yml @@ -1,5 +1,4 @@ title: HTTP -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/icmp/manifest.yml b/packages/network_traffic/data_stream/icmp/manifest.yml index 1a0acc4288f..4a8367be084 100644 --- a/packages/network_traffic/data_stream/icmp/manifest.yml +++ b/packages/network_traffic/data_stream/icmp/manifest.yml @@ -1,5 +1,4 @@ title: ICMP -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/memcached/manifest.yml b/packages/network_traffic/data_stream/memcached/manifest.yml index 52b300e8dbc..2398e78ca7d 100644 --- a/packages/network_traffic/data_stream/memcached/manifest.yml +++ b/packages/network_traffic/data_stream/memcached/manifest.yml @@ -1,5 +1,4 @@ title: Memcached -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/mongodb/manifest.yml b/packages/network_traffic/data_stream/mongodb/manifest.yml index 29c6144b6a5..c876d333b52 100644 --- a/packages/network_traffic/data_stream/mongodb/manifest.yml +++ b/packages/network_traffic/data_stream/mongodb/manifest.yml @@ -1,5 +1,4 @@ title: MongoDB -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/mysql/manifest.yml b/packages/network_traffic/data_stream/mysql/manifest.yml index f4f8c1209f4..a9e5e2226ca 100644 --- a/packages/network_traffic/data_stream/mysql/manifest.yml +++ b/packages/network_traffic/data_stream/mysql/manifest.yml @@ -1,5 +1,4 @@ title: MySQL -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/nfs/manifest.yml b/packages/network_traffic/data_stream/nfs/manifest.yml index 0695ef0d60c..55a6e3dfea8 100644 --- a/packages/network_traffic/data_stream/nfs/manifest.yml +++ b/packages/network_traffic/data_stream/nfs/manifest.yml @@ -1,5 +1,4 @@ title: NFS -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/pgsql/manifest.yml b/packages/network_traffic/data_stream/pgsql/manifest.yml index 8dafd5d3cfc..d74fc26094e 100644 --- a/packages/network_traffic/data_stream/pgsql/manifest.yml +++ b/packages/network_traffic/data_stream/pgsql/manifest.yml @@ -1,5 +1,4 @@ title: PostgreSQL -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/redis/manifest.yml b/packages/network_traffic/data_stream/redis/manifest.yml index e3d9025e9d5..f315c160b98 100644 --- a/packages/network_traffic/data_stream/redis/manifest.yml +++ b/packages/network_traffic/data_stream/redis/manifest.yml @@ -1,5 +1,4 @@ title: Redis -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/sip/manifest.yml b/packages/network_traffic/data_stream/sip/manifest.yml index 93e78e4e0a9..fb908ef0d80 100644 --- a/packages/network_traffic/data_stream/sip/manifest.yml +++ b/packages/network_traffic/data_stream/sip/manifest.yml @@ -1,5 +1,4 @@ title: SIP -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/thrift/manifest.yml b/packages/network_traffic/data_stream/thrift/manifest.yml index ba45e67fd7a..98d6ee0eb9e 100644 --- a/packages/network_traffic/data_stream/thrift/manifest.yml +++ b/packages/network_traffic/data_stream/thrift/manifest.yml @@ -1,5 +1,4 @@ title: Thrift -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/data_stream/tls/manifest.yml b/packages/network_traffic/data_stream/tls/manifest.yml index 2ef3a07f689..5e370363cfd 100644 --- a/packages/network_traffic/data_stream/tls/manifest.yml +++ b/packages/network_traffic/data_stream/tls/manifest.yml @@ -1,5 +1,4 @@ title: TLS -release: beta type: logs streams: - input: packet diff --git a/packages/network_traffic/docs/README.md b/packages/network_traffic/docs/README.md index 7890c5e98bf..265874bfd1b 100644 --- a/packages/network_traffic/docs/README.md +++ b/packages/network_traffic/docs/README.md @@ -184,6 +184,8 @@ The default value is 10s. | destination.geo.region_iso_code | Region ISO code. | keyword | | destination.geo.region_name | Region name. | keyword | | destination.ip | IP address of the destination (IPv4 or IPv6). | ip | +| destination.mac | MAC address of the destination. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | +| destination.packets | Packets sent from the destination to the source. | long | | destination.port | Port of the destination. | long | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | @@ -218,6 +220,7 @@ The default value is 10s. | network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | | network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | | network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip | +| network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long | | network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | | network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | | network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | @@ -260,11 +263,103 @@ The default value is 10s. | source.geo.region_iso_code | Region ISO code. | keyword | | source.geo.region_name | Region name. | keyword | | source.ip | IP address of the source (IPv4 or IPv6). | ip | +| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | +| source.packets | Packets sent from the source to the destination. | long | | source.port | Port of the source. | long | | status | The high level status of the transaction. The way to compute this value depends on the protocol, but the result has a meaning independent of the protocol. | keyword | | type | The type of the transaction (for example, HTTP, MySQL, Redis, or RUM) or "flow" in case of flows. | keyword | +An example event for `flow` looks as following: + +```json +{ + "@timestamp": "2023-04-04T23:12:40.755Z", + "agent": { + "ephemeral_id": "c368e835-b038-4610-b000-bc9fb23b35ab", + "id": "d35fb84a-73fb-4bc7-99b4-ac4df8c6ebb5", + "name": "docker-fleet-agent", + "type": "packetbeat", + "version": "8.6.2" + }, + "data_stream": { + "dataset": "network_traffic.flow", + "namespace": "ep", + "type": "logs" + }, + "destination": { + "bytes": 64, + "ip": "::1", + "packets": 1, + "port": 8000 + }, + "ecs": { + "version": "8.6.0" + }, + "elastic_agent": { + "id": "d35fb84a-73fb-4bc7-99b4-ac4df8c6ebb5", + "snapshot": false, + "version": "8.6.2" + }, + "event": { + "action": "network_flow", + "agent_id_status": "verified", + "category": [ + "network" + ], + "dataset": "network_traffic.flow", + "duration": 70523, + "end": "2023-04-04T23:12:30.760Z", + "ingested": "2023-04-04T23:12:41Z", + "kind": "event", + "start": "2023-04-04T23:12:30.759Z", + "type": [ + "connection" + ] + }, + "flow": { + "final": false, + "id": "QAT///////8A////IP8AAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAUAfeMg" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "f91b175388d443fca5c155815dfc2279", + "ip": [ + "192.168.208.7" + ], + "mac": [ + "02-42-C0-A8-D0-07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.49-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + } + }, + "network": { + "bytes": 152, + "community_id": "1:5y9AkdbV9U8xqD9dhlj6obkubHg=", + "packets": 2, + "transport": "tcp", + "type": "ipv6" + }, + "source": { + "bytes": 88, + "ip": "::1", + "packets": 1, + "port": 51320 + }, + "type": "flow" +} +``` + ## Protocols ### AMQP diff --git a/packages/network_traffic/kibana/dashboard/network_traffic-flows.json b/packages/network_traffic/kibana/dashboard/network_traffic-flows.json index 147d70ab6a7..73f4fcd8a68 100644 --- a/packages/network_traffic/kibana/dashboard/network_traffic-flows.json +++ b/packages/network_traffic/kibana/dashboard/network_traffic-flows.json @@ -1,17 +1,32 @@ { - "id": "network_traffic-flows", - "type": "dashboard", - "namespaces": [ - "default" - ], - "updated_at": "2022-10-27T20:54:04.747Z", - "version": "WzcyMSwxXQ==", "attributes": { "description": "", "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": { - "filter": [], + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": "Flow is final", + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "flow.final", + "negate": false, + "params": { + "query": true + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "flow.final": true + } + } + } + ], "highlightAll": true, "query": { "language": "kuery", @@ -28,9 +43,59 @@ "embeddableConfig": { "enhancements": {}, "savedVis": { - "title": "[Network Packet Capture] Top Hosts Creating Traffic", + "data": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Source Bytes", + "field": "source.bytes" + }, + "schema": "metric", + "type": "sum" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Source IP", + "field": "source.ip", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, "description": "", - "uiState": {}, "params": { "addLegend": true, "addTimeMarker": false, @@ -116,59 +181,9 @@ ], "yAxis": {} }, + "title": "[Network Packet Capture] Top Hosts Creating Traffic", "type": "area", - "data": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "customLabel": "Source Bytes", - "field": "source.bytes" - }, - "schema": "metric", - "type": "sum" - }, - { - "enabled": true, - "id": "2", - "params": { - "drop_partials": false, - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1, - "useNormalizedEsInterval": true - }, - "schema": "segment", - "type": "date_histogram" - }, - { - "enabled": true, - "id": "3", - "params": { - "customLabel": "Source IP", - "field": "source.ip", - "missingBucket": false, - "missingBucketLabel": "Missing", - "order": "desc", - "orderBy": "1", - "otherBucket": false, - "otherBucketLabel": "Other", - "size": 5 - }, - "schema": "group", - "type": "terms" - } - ], - "searchSource": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } - } + "uiState": {} } }, "gridData": { @@ -179,22 +194,13 @@ "y": 25 }, "panelIndex": "1", - "version": "7.17.0", - "type": "visualization" + "type": "visualization", + "version": "7.17.0" }, { "embeddableConfig": { "enhancements": {}, "savedVis": { - "title": "[Network Packet Capture] Navigation", - "description": "", - "uiState": {}, - "params": { - "fontSize": 10, - "markdown": "### Network Packet Capture:\n\n[Overview](#/dashboard/network_traffic-dashboard)\n\n[Network Flows](#/dashboard/network_traffic-flows)\n\n[DNS Overview](#/dashboard/network_traffic-65120940-1454-11e9-9de0-f98d1808db8e) | [Tunneling](#/dashboard/network_traffic-dns-unique-domains)\n\n[DHCPv4 Transactions](#/dashboard/network_traffic-a7b35890-8baa-11e8-9676-ef67484126fb)\n\n[TLS Overview](#/dashboard/network_traffic-tls-sessions)\n\n[HTTP transactions](#/dashboard/network_traffic-http)\n\nDatabases: [MySQL](#/dashboard/network_traffic-mysql-performance) | [PostgreSQL](#/dashboard/network_traffic-pgsql-performance) | [MongoDB](#/dashboard/network_traffic-mongodb-performance) | [Cassandra](#/dashboard/network_traffic-cassandra)\n\nRPC: [Thrift](#/dashboard/network_traffic-thrift-performance)\n\nStorage: [NFS](#/dashboard/network_traffic-nfs)", - "openLinksInNewTab": false - }, - "type": "markdown", "data": { "aggs": [], "searchSource": { @@ -204,7 +210,16 @@ "query": "" } } - } + }, + "description": "", + "params": { + "fontSize": 10, + "markdown": "### Network Packet Capture:\n\n[Overview](#/dashboard/network_traffic-dashboard)\n\n[Network Flows](#/dashboard/network_traffic-flows)\n\n[DNS Overview](#/dashboard/network_traffic-65120940-1454-11e9-9de0-f98d1808db8e) | [Tunneling](#/dashboard/network_traffic-dns-unique-domains)\n\n[DHCPv4 Transactions](#/dashboard/network_traffic-a7b35890-8baa-11e8-9676-ef67484126fb)\n\n[TLS Overview](#/dashboard/network_traffic-tls-sessions)\n\n[HTTP transactions](#/dashboard/network_traffic-http)\n\nDatabases: [MySQL](#/dashboard/network_traffic-mysql-performance) | [PostgreSQL](#/dashboard/network_traffic-pgsql-performance) | [MongoDB](#/dashboard/network_traffic-mongodb-performance) | [Cassandra](#/dashboard/network_traffic-cassandra)\n\nRPC: [Thrift](#/dashboard/network_traffic-thrift-performance)\n\nStorage: [NFS](#/dashboard/network_traffic-nfs)", + "openLinksInNewTab": false + }, + "title": "[Network Packet Capture] Navigation", + "type": "markdown", + "uiState": {} } }, "gridData": { @@ -215,16 +230,49 @@ "y": 0 }, "panelIndex": "2", - "version": "7.17.0", - "type": "visualization" + "type": "visualization", + "version": "7.17.0" }, { "embeddableConfig": { "enhancements": {}, "savedVis": { - "title": "[Network Packet Capture] Connections over time", + "data": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Unique Flows", + "field": "flow.id" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, "description": "", - "uiState": {}, "params": { "addLegend": true, "addTimeMarker": false, @@ -310,18 +358,37 @@ ], "yAxis": {} }, + "title": "[Network Packet Capture] Connections over time", "type": "area", + "uiState": {} + } + }, + "gridData": { + "h": 25, + "i": "3", + "w": 36, + "x": 12, + "y": 0 + }, + "panelIndex": "3", + "type": "visualization", + "version": "7.17.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "savedVis": { "data": { "aggs": [ { "enabled": true, "id": "1", "params": { - "customLabel": "Unique Flows", - "field": "flow.id" + "customLabel": "Destination Bytes", + "field": "destination.bytes" }, "schema": "metric", - "type": "cardinality" + "type": "sum" }, { "enabled": true, @@ -336,6 +403,23 @@ }, "schema": "segment", "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Destination IP", + "field": "destination.ip", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "group", + "type": "terms" } ], "searchSource": { @@ -345,27 +429,8 @@ "query": "" } } - } - } - }, - "gridData": { - "h": 25, - "i": "3", - "w": 36, - "x": 12, - "y": 0 - }, - "panelIndex": "3", - "version": "7.17.0", - "type": "visualization" - }, - { - "embeddableConfig": { - "enhancements": {}, - "savedVis": { - "title": "[Network Packet Capture] Top Hosts Receiving Traffic", + }, "description": "", - "uiState": {}, "params": { "addLegend": true, "addTimeMarker": false, @@ -451,59 +516,9 @@ ], "yAxis": {} }, + "title": "[Network Packet Capture] Top Hosts Receiving Traffic", "type": "area", - "data": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "customLabel": "Destination Bytes", - "field": "destination.bytes" - }, - "schema": "metric", - "type": "sum" - }, - { - "enabled": true, - "id": "2", - "params": { - "drop_partials": false, - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1, - "useNormalizedEsInterval": true - }, - "schema": "segment", - "type": "date_histogram" - }, - { - "enabled": true, - "id": "3", - "params": { - "customLabel": "Destination IP", - "field": "destination.ip", - "missingBucket": false, - "missingBucketLabel": "Missing", - "order": "desc", - "orderBy": "1", - "otherBucket": false, - "otherBucketLabel": "Other", - "size": 5 - }, - "schema": "group", - "type": "terms" - } - ], - "searchSource": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } - } + "uiState": {} } }, "gridData": { @@ -514,46 +529,13 @@ "y": 25 }, "panelIndex": "4", - "version": "7.17.0", - "type": "visualization" + "type": "visualization", + "version": "7.17.0" }, { "embeddableConfig": { "enhancements": {}, - "vis": { - "params": { - "sort": { - "columnIndex": null, - "direction": null - } - } - }, "savedVis": { - "title": "[Network Packet Capture] Traffic Between Hosts", - "description": "", - "uiState": { - "vis": { - "params": { - "sort": { - "columnIndex": null, - "direction": null - } - } - } - }, - "params": { - "perPage": 10, - "showMetricsAtAllLevels": false, - "showPartialRows": false, - "showToolbar": true, - "showTotal": false, - "sort": { - "columnIndex": null, - "direction": null - }, - "totalFunc": "sum" - }, - "type": "table", "data": { "aggs": [ { @@ -618,6 +600,39 @@ "query": "" } } + }, + "description": "", + "params": { + "perPage": 10, + "showMetricsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "[Network Packet Capture] Traffic Between Hosts", + "type": "table", + "uiState": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + } + }, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } } } }, @@ -629,38 +644,25 @@ "y": 60 }, "panelIndex": "5", - "version": "7.17.0", - "type": "visualization" + "type": "visualization", + "version": "7.17.0" } ], "timeRestore": false, "title": "[Network Packet Capture] Network Flows", "version": 1 }, + "coreMigrationVersion": "7.17.0", + "id": "network_traffic-flows", + "migrationVersion": { + "dashboard": "7.17.0" + }, "references": [ { - "type": "search", - "name": "1:search_0", - "id": "network_traffic-flows-search" - }, - { - "type": "search", - "name": "3:search_0", - "id": "network_traffic-flows-search" - }, - { - "type": "search", - "name": "4:search_0", - "id": "network_traffic-flows-search" - }, - { - "type": "search", - "name": "5:search_0", - "id": "network_traffic-flows-search" + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" } ], - "migrationVersion": { - "dashboard": "7.17.0" - }, - "coreMigrationVersion": "7.17.0" + "type": "dashboard" } \ No newline at end of file diff --git a/packages/network_traffic/manifest.yml b/packages/network_traffic/manifest.yml index 67f6b0a3d82..08fc3d3d0d8 100644 --- a/packages/network_traffic/manifest.yml +++ b/packages/network_traffic/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: network_traffic title: Network Packet Capture -version: "1.10.1" +version: "1.11.0" license: basic description: Capture and analyze network traffic from a host with Elastic Agent. type: integration