From a67d1fa415e60a297072b8e7e47b4a098128e7a8 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 26 Jul 2022 20:23:27 +0000 Subject: [PATCH 01/17] init commit --- packages/ti_cif3/_dev/build/build.yml | 3 + packages/ti_cif3/_dev/build/docs/README.md | 13 + .../_dev/deploy/docker/docker-compose.yml | 14 + .../_dev/deploy/docker/files/config.yml | 161 ++++ packages/ti_cif3/changelog.yml | 6 + .../test/pipeline/test-cif3-sample.ndjson.log | 1 + .../test-cif3-sample.ndjson.log-expected.json | 48 ++ .../_dev/test/pipeline/test-common-config.yml | 5 + .../_dev/test/system/test-default-config.yml | 11 + .../feed/agent/stream/httpjson.yml.hbs | 87 +++ .../elasticsearch/ingest_pipeline/default.yml | 341 ++++++++ .../data_stream/feed/fields/base-fields.yml | 24 + .../ti_cif3/data_stream/feed/fields/beats.yml | 12 + .../ti_cif3/data_stream/feed/fields/ecs.yml | 104 +++ .../data_stream/feed/fields/fields.yml | 129 ++++ .../ti_cif3/data_stream/feed/manifest.yml | 125 +++ .../data_stream/feed/sample_event.json | 87 +++ packages/ti_cif3/docs/README.md | 202 +++++ packages/ti_cif3/img/csg_logo_big.svg | 23 + ...-6005a190-0aba-11ed-bcc0-01c79f2670f3.json | 320 ++++++++ ...-63a0e470-0a30-11ed-bcc0-01c79f2670f3.json | 730 ++++++++++++++++++ ...-aedada10-0ab5-11ed-bcc0-01c79f2670f3.json | 699 +++++++++++++++++ ...-b4d9d9b0-0a2f-11ed-bcc0-01c79f2670f3.json | 719 +++++++++++++++++ ...-bda23600-0abb-11ed-bcc0-01c79f2670f3.json | 406 ++++++++++ ...-fef149c0-0a2f-11ed-bcc0-01c79f2670f3.json | 710 +++++++++++++++++ ...3e30-0c59-11ed-9b65-435777f1d8a1.json.json | 14 + packages/ti_cif3/manifest.yml | 43 ++ 27 files changed, 5037 insertions(+) create mode 100644 packages/ti_cif3/_dev/build/build.yml create mode 100644 packages/ti_cif3/_dev/build/docs/README.md create mode 100644 packages/ti_cif3/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/ti_cif3/_dev/deploy/docker/files/config.yml create mode 100644 packages/ti_cif3/changelog.yml create mode 100644 packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log create mode 100644 packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log-expected.json create mode 100644 packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-common-config.yml create mode 100644 packages/ti_cif3/data_stream/feed/_dev/test/system/test-default-config.yml create mode 100644 packages/ti_cif3/data_stream/feed/agent/stream/httpjson.yml.hbs create mode 100644 packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/ti_cif3/data_stream/feed/fields/base-fields.yml create mode 100644 packages/ti_cif3/data_stream/feed/fields/beats.yml create mode 100644 packages/ti_cif3/data_stream/feed/fields/ecs.yml create mode 100644 packages/ti_cif3/data_stream/feed/fields/fields.yml create mode 100644 packages/ti_cif3/data_stream/feed/manifest.yml create mode 100755 packages/ti_cif3/data_stream/feed/sample_event.json create mode 100644 packages/ti_cif3/docs/README.md create mode 100755 packages/ti_cif3/img/csg_logo_big.svg create mode 100644 packages/ti_cif3/kibana/dashboard/ti_cif3-6005a190-0aba-11ed-bcc0-01c79f2670f3.json create mode 100644 packages/ti_cif3/kibana/dashboard/ti_cif3-63a0e470-0a30-11ed-bcc0-01c79f2670f3.json create mode 100644 packages/ti_cif3/kibana/dashboard/ti_cif3-aedada10-0ab5-11ed-bcc0-01c79f2670f3.json create mode 100644 packages/ti_cif3/kibana/dashboard/ti_cif3-b4d9d9b0-0a2f-11ed-bcc0-01c79f2670f3.json create mode 100644 packages/ti_cif3/kibana/dashboard/ti_cif3-bda23600-0abb-11ed-bcc0-01c79f2670f3.json create mode 100644 packages/ti_cif3/kibana/dashboard/ti_cif3-fef149c0-0a2f-11ed-bcc0-01c79f2670f3.json create mode 100644 packages/ti_cif3/kibana/tag/ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1.json.json create mode 100644 packages/ti_cif3/manifest.yml diff --git a/packages/ti_cif3/_dev/build/build.yml b/packages/ti_cif3/_dev/build/build.yml new file mode 100644 index 00000000000..5661d603a89 --- /dev/null +++ b/packages/ti_cif3/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@v8.3.0 diff --git a/packages/ti_cif3/_dev/build/docs/README.md b/packages/ti_cif3/_dev/build/docs/README.md new file mode 100644 index 00000000000..ae4dc395d04 --- /dev/null +++ b/packages/ti_cif3/_dev/build/docs/README.md @@ -0,0 +1,13 @@ +# Collective Intelligence Framework v3 Integration + +This integration connects with the [REST API from the running CIFv3 instance](https://github.com/csirtgadgets/bearded-avenger/blob/master/cif/httpd/views/feed/__init__.py) to retrieve indicators. + +## Data Streams + +### Feed + +The CIFv3 integration collects threat indicators based on user-defined configuration including a polling interval, how far back in time it should look, and other filters like indicator type and tags. + +{{fields "feed"}} + +{{event "feed"}} \ No newline at end of file diff --git a/packages/ti_cif3/_dev/deploy/docker/docker-compose.yml b/packages/ti_cif3/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..af7a984b239 --- /dev/null +++ b/packages/ti_cif3/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,14 @@ +version: "2.3" +services: + cif3: + image: docker.elastic.co/observability/stream:v0.7.0 + ports: + - 8080 + volumes: + - ./files:/files:ro + environment: + PORT: 8080 + command: + - http-server + - --addr=:8080 + - --config=/files/config.yml \ No newline at end of file diff --git a/packages/ti_cif3/_dev/deploy/docker/files/config.yml b/packages/ti_cif3/_dev/deploy/docker/files/config.yml new file mode 100644 index 00000000000..aaf3a6360c3 --- /dev/null +++ b/packages/ti_cif3/_dev/deploy/docker/files/config.yml @@ -0,0 +1,161 @@ +rules: + - path: /feed + methods: ["GET"] + request_headers: + Authorization: "Token token=testing" + query_params: + itype: "ipv4" + confidence: "8" + tags: "botnet,exploit,malware,phishing" + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + { + "message": "success", + "data": [ + { + "indicator": "20.206.75.106", + "itype": "ipv4", + "tlp": "white", + "provider": "sslbl.abuse.ch", + "group": [ + "everyone" + ], + "count": 1, + "tags": [ + "botnet" + ], + "confidence": 10, + "uuid": "ac240898-1443-4d7e-a98a-1daed220c162", + "cc": "br", + "latitude": -22.9035, + "timezone": "america/sao_paulo", + "longitude": -47.0565, + "city": "campinas", + "region": "sao paulo", + "location": [ + -47.0565, + -22.9035 + ], + "application": "https", + "reference": "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv", + "portlist": "443", + "protocol": "tcp", + "asn": 8075, + "asn_desc": "microsoft-corp-msn-as-block", + "firsttime": "2022-07-20T20:25:53.000000Z", + "reporttime": "2022-07-21T20:33:26.585967Z", + "lasttime": "2022-07-20T20:25:53.000000Z", + "indicator_ipv4": "20.206.75.106" + }, + { + "indicator": "160.20.147.52", + "itype": "ipv4", + "tlp": "white", + "provider": "sslbl.abuse.ch", + "group": [ + "everyone" + ], + "count": 1, + "tags": [ + "botnet" + ], + "confidence": 10, + "uuid": "cb5e953d-f3f7-4a94-88f6-dc553fc30445", + "cc": "de", + "latitude": 50.1103, + "timezone": "europe/berlin", + "longitude": 8.7147, + "city": "frankfurt am main", + "region": "hesse", + "location": [ + 8.7147, + 50.1103 + ], + "application": "https", + "reference": "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv", + "portlist": "8848", + "protocol": "tcp", + "asn": 30823, + "asn_desc": "combahton gmbh", + "firsttime": "2022-07-20T20:00:30.000000Z", + "reporttime": "2022-07-21T09:32:44.946024Z", + "lasttime": "2022-07-20T20:00:30.000000Z", + "indicator_ipv4": "160.20.147.52" + }, + { + "indicator": "207.32.218.12", + "itype": "ipv4", + "tlp": "white", + "provider": "sslbl.abuse.ch", + "group": [ + "everyone" + ], + "count": 1, + "tags": [ + "botnet" + ], + "confidence": 10, + "uuid": "e0596a59-1139-42d0-8c3a-4b505405602c", + "cc": "us", + "latitude": 33.4413, + "timezone": "america/phoenix", + "longitude": -112.0421, + "city": "phoenix", + "region": "arizona", + "location": [ + -112.0421, + 33.4413 + ], + "application": "https", + "reference": "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv", + "portlist": "6606", + "protocol": "tcp", + "asn": 14315, + "asn_desc": "1gservers", + "firsttime": "2022-07-20T21:41:13.000000Z", + "reporttime": "2022-07-21T09:32:44.696140Z", + "lasttime": "2022-07-20T21:41:13.000000Z", + "indicator_ipv4": "207.32.218.12" + }, + { + "indicator": "103.133.105.50", + "itype": "ipv4", + "tlp": "white", + "provider": "sslbl.abuse.ch", + "group": [ + "everyone" + ], + "count": 1, + "tags": [ + "botnet", + "malware" + ], + "confidence": 10, + "uuid": "1aa35d5f-59ee-4364-8ad3-dd9d78cd2140", + "cc": "vn", + "latitude": 10.8326, + "timezone": "asia/ho_chi_minh", + "longitude": 106.6581, + "city": "ho chi minh city", + "region": "ho chi minh", + "location": [ + 106.6581, + 10.8326 + ], + "application": "https", + "reference": "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv", + "portlist": "1234", + "protocol": "tcp", + "asn": 135905, + "asn_desc": "vietnam posts and telecommunications group", + "firsttime": "2022-07-19T09:30:19.000000Z", + "reporttime": "2022-07-20T00:19:11.521288Z", + "lasttime": "2022-07-19T09:30:19.000000Z", + "indicator_ipv4": "103.133.105.50" + } + ] + } \ No newline at end of file diff --git a/packages/ti_cif3/changelog.yml b/packages/ti_cif3/changelog.yml new file mode 100644 index 00000000000..0da913124e3 --- /dev/null +++ b/packages/ti_cif3/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.1.0" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link diff --git a/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log b/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log new file mode 100644 index 00000000000..2980ede9051 --- /dev/null +++ b/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log @@ -0,0 +1 @@ +{"indicator":"89.160.20.156","itype":"ipv4","tlp":"white","provider":"threatfox.abuse.ch","group":["everyone"],"count":1,"tags":["agenttesla","botnet","hunter"],"confidence":8.0,"description":"agent tesla","uuid":"3fbdd654-b2b0-498c-8e20-ef87bce73672","reference":"https://threatfox.abuse.ch/ioc/838651/","rdata":"http://208.67.106.111/theme/inc/e26dbe0dcc481e.php","firsttime":"2022-07-19T07:40:41.000000Z","lasttime":"2022-07-19T08:35:05.971696Z","reporttime":"2022-07-19T08:35:05.971696Z","indicator_ipv4":"89.160.20.156"} \ No newline at end of file diff --git a/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log-expected.json b/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log-expected.json new file mode 100644 index 00000000000..15126e4c978 --- /dev/null +++ b/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log-expected.json @@ -0,0 +1,48 @@ +{ + "expected": [ + { + "cif3": { + "itype": "ipv4", + "rdata": "http://208.67.106.111/theme/inc/e26dbe0dcc481e.php", + "uuid": "3fbdd654-b2b0-498c-8e20-ef87bce73672" + }, + "ecs": { + "version": "8.3.0" + }, + "event": { + "category": "threat", + "kind": "enrichment", + "original": "{\"indicator\":\"89.160.20.156\",\"itype\":\"ipv4\",\"tlp\":\"white\",\"provider\":\"threatfox.abuse.ch\",\"group\":[\"everyone\"],\"count\":1,\"tags\":[\"agenttesla\",\"botnet\",\"hunter\"],\"confidence\":8.0,\"description\":\"agent tesla\",\"uuid\":\"3fbdd654-b2b0-498c-8e20-ef87bce73672\",\"reference\":\"https://threatfox.abuse.ch/ioc/838651/\",\"rdata\":\"http://208.67.106.111/theme/inc/e26dbe0dcc481e.php\",\"firsttime\":\"2022-07-19T07:40:41.000000Z\",\"lasttime\":\"2022-07-19T08:35:05.971696Z\",\"reporttime\":\"2022-07-19T08:35:05.971696Z\",\"indicator_ipv4\":\"89.160.20.156\"}", + "type": "indicator" + }, + "related": { + "ip": [ + "89.160.20.156" + ] + }, + "tags": [ + "preserve_original_event", + "agenttesla", + "botnet", + "hunter" + ], + "threat": { + "indicator": { + "confidence": "High", + "description": "agent tesla", + "first_seen": "2022-07-19T07:40:41.000000Z", + "ip": "89.160.20.156", + "last_seen": "2022-07-19T08:35:05.971696Z", + "marking": { + "tlp": "WHITE" + }, + "modified_at": "2022-07-19T08:35:05.971696Z", + "provider": "threatfox.abuse.ch", + "reference": "https://threatfox.abuse.ch/ioc/838651/", + "sightings": 1, + "type": "ipv4-addr" + } + } + } + ] +} \ No newline at end of file diff --git a/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-common-config.yml b/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..11e8ffa1d1e --- /dev/null +++ b/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +fields: + tags: + - preserve_original_event +dynamic_fields: + event.ingested: "^.*$" \ No newline at end of file diff --git a/packages/ti_cif3/data_stream/feed/_dev/test/system/test-default-config.yml b/packages/ti_cif3/data_stream/feed/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..9d3d19c7690 --- /dev/null +++ b/packages/ti_cif3/data_stream/feed/_dev/test/system/test-default-config.yml @@ -0,0 +1,11 @@ +input: httpjson +service: cif3 +vars: + url: http://{{Hostname}}:{{Port}} + api_token: testing +data_stream: + vars: + preserve_original_event: true + confidence: '8' + type: ipv4 + cif_tags: 'botnet,exploit,malware,phishing' \ No newline at end of file diff --git a/packages/ti_cif3/data_stream/feed/agent/stream/httpjson.yml.hbs b/packages/ti_cif3/data_stream/feed/agent/stream/httpjson.yml.hbs new file mode 100644 index 00000000000..42f0dcb645e --- /dev/null +++ b/packages/ti_cif3/data_stream/feed/agent/stream/httpjson.yml.hbs @@ -0,0 +1,87 @@ +config_version: "2" +interval: {{interval}} +request.method: "GET" + +{{#if url}} +request.url: {{url}}/feed +{{/if}} +{{#if proxy_url }} +request.proxy_url: {{proxy_url}} +{{/if}} +{{#if ssl}} +request.ssl: {{ssl}} +{{/if}} +{{#if http_client_timeout}} +request.timeout: {{http_client_timeout}} +{{/if}} +request.transforms: +- set: + target: header.Accept + value: 'application/vnd.cif.v3+json' +- delete: + target: header.User-Agent +- set: + target: header.User-Agent + value: elastic-integration/0.1.0 +{{#if api_token }} +- set: + target: header.Authorization + value: Token token={{ api_token }} +{{/if}} +{{#if type}} +- set: + target: url.params.itype + value: {{ type }} +{{/if}} +{{#if confidence}} +- set: + target: url.params.confidence + value: {{ confidence }} +{{/if}} +{{#if limit}} +- set: + target: url.params.limit + value: {{ limit }} +{{/if}} +{{#if cif_tags}} +- set: + target: url.params.tags + value: {{ cif_tags }} +{{/if}} +{{#if lookback_hours}} +- set: + target: url.params.hours + value: {{ lookback_hours }} +{{/if}} +- set: + target: url.params.reporttime + value: '[[.cursor.last_requested_at]]' + default: '[[ formatDate (now (parseDuration "-{{initial_lookback}}")) "RFC3339" ]]' + +{{#each filters}} +- set: + target: "url.params.{{{ @key }}}" + value: {{ this }} +{{/each}} + +response.split: + target: body.data + +cursor: + last_requested_at: + value: '[[ formatDate (now) "RFC3339" ]]' + +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml b/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..936385b4e07 --- /dev/null +++ b/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,341 @@ +--- +description: Pipeline for processing CIFv3 threat indicators +processors: + #################### + # Event ECS fields # + #################### + - set: + field: ecs.version + value: "8.3.0" + - set: + field: event.kind + value: enrichment + - set: + field: event.category + value: threat + - set: + field: event.type + value: indicator + + ###################### + # General ECS fields # + ###################### + - rename: + field: message + target_field: event.original + ignore_missing: true + - json: + field: event.original + target_field: cif3 + + ##################### + # Threat ECS Fields # + ##################### + - rename: + field: cif3.firsttime + target_field: threat.indicator.first_seen + ignore_missing: true + - rename: + field: cif3.lasttime + target_field: threat.indicator.last_seen + ignore_missing: true + - rename: + field: cif3.reporttime + target_field: threat.indicator.modified_at + ignore_missing: true + - rename: + field: cif3.provider + target_field: threat.indicator.provider + ignore_missing: true + - rename: + field: cif3.reference + target_field: threat.indicator.reference + ignore_missing: true + - rename: + field: cif3.count + target_field: threat.indicator.sightings + ignore_missing: true + - rename: + field: cif3.description + target_field: threat.indicator.description + ignore_missing: true + if: "ctx.cif3?.description != ''" + - uppercase: + field: cif3.tlp + target_field: threat.indicator.marking.tlp + ignore_missing: true + if: ctx.cif3?.tlp != null + ## File indicator operations + - set: + field: threat.indicator.type + value: file + if: "['md5', 'sha1', 'sha256', 'sha512', 'ssdeep'].contains(ctx.cif3?.itype) && !ctx.cif3?.tags.contains('ja3')" + - rename: + field: cif3.indicator + target_field: tls.client.ja3 + ignore_missing: true + if: "ctx.cif3?.itype == 'md5' && ctx.cif3?.tags.contains('ja3')" + - rename: + field: cif3.indicator + target_field: threat.indicator.file.pe.imphash + ignore_missing: true + if: "ctx.cif3?.itype == 'md5' && ctx.cif3?.tags.contains('imphash')" + - append: + field: related.hash + value: "{{{ threat.indicator.file.hash.pe.imphash }}}" + if: ctx?.threat?.indicator?.file?.pe?.imphash != null + - rename: + field: cif3.indicator + target_field: _tmp.hashvalue + ignore_missing: true + if: "ctx.threat?.indicator?.type == 'file'" + - set: + field: threat.indicator.file.hash.{{cif3.itype}} + value: "{{{ _tmp.hashvalue }}}" + if: "ctx.threat?.indicator?.type == 'file'" + - append: + field: related.hash + value: "{{{ _tmp.hashvalue }}}" + ignore_failure: true + if: "ctx.threat?.indicator?.type == 'file' && ctx?.threat?.indicator?.file?.pe?.imphash == null" + + ## ASN indicator operations + - set: + field: threat.indicator.type + value: autonomous-system + if: "ctx.cif3?.itype == 'asn'" + - grok: + field: cif3.indicator + patterns: + - "as(?:%{INT:threat.indicator.as.number})" + ignore_failure: true + if: "ctx.cif3?.itype == 'asn'" + + ## IP indicator operations + - set: + field: threat.indicator.type + value: ipv4-addr + if: "ctx.cif3?.itype == 'ipv4'" + - set: + field: threat.indicator.type + value: ipv6-addr + if: "ctx.cif3?.itype == 'ipv6'" + - rename: + field: cif3.indicator + target_field: threat.indicator.network.cidr + ignore_missing: true + if: "ctx.threat?.indicator?.type != null && ['ipv4-addr', 'ipv6-addr'].contains(ctx.threat.indicator.type) && (ctx.cif3?.indicator_ipv4_mask != null || ctx.cif3?.indicator_ipv6_mask != null)" + - convert: + field: cif3.indicator + type: ip + target_field: threat.indicator.ip + ignore_missing: true + if: "ctx.threat?.indicator?.type != null && ['ipv4-addr', 'ipv6-addr'].contains(ctx.threat.indicator.type) && ctx.cif3?.indicator_ipv4_mask == null && ctx.cif3?.indicator_ipv6_mask == null" + - append: + field: related.ip + value: "{{{ threat.indicator.ip }}}" + if: ctx?.threat?.indicator?.ip != null + - rename: + field: cif3.cc + target_field: threat.indicator.geo.country_iso_code + ignore_missing: true + if: "ctx.threat?.indicator?.type != null && ['ipv4-addr', 'ipv6-addr'].contains(ctx.threat.indicator.type) && ctx.cif3?.cc != null" + - rename: + field: cif3.asn + target_field: threat.indicator.as.number + ignore_missing: true + if: "ctx.threat?.indicator?.type != null && ['ipv4-addr', 'ipv6-addr'].contains(ctx.threat.indicator.type) && ctx.cif3?.asn != null" + - rename: + field: cif3.asn_desc + target_field: threat.indicator.as.organization.name + ignore_missing: true + if: "ctx.threat?.indicator?.type != null && ['ipv4-addr', 'ipv6-addr'].contains(ctx.threat.indicator.type) && ctx.cif3?.asn_desc != null" + - rename: + field: cif3.latitude + target_field: threat.indicator.geo.location.lat + ignore_missing: true + if: "ctx.threat?.indicator?.type != null && ['ipv4-addr', 'ipv6-addr'].contains(ctx.threat.indicator.type) && ctx.cif3?.latitude != null" + - rename: + field: cif3.longitude + target_field: threat.indicator.geo.location.lon + ignore_missing: true + if: "ctx.threat?.indicator?.type != null && ['ipv4-addr', 'ipv6-addr'].contains(ctx.threat.indicator.type) && ctx.cif3?.longitude != null" + - rename: + field: cif3.region + target_field: threat.indicator.geo.region_name + ignore_missing: true + if: "ctx.threat?.indicator?.type != null && ['ipv4-addr', 'ipv6-addr'].contains(ctx.threat.indicator.type) && ctx.cif3?.region != null" + - rename: + field: cif3.timezone + target_field: threat.indicator.geo.timezone + ignore_missing: true + if: "ctx.threat?.indicator?.type != null && ['ipv4-addr', 'ipv6-addr'].contains(ctx.threat.indicator.type) && ctx.cif3?.timezone != null" + + ## URL indicator operations + - set: + field: threat.indicator.type + value: url + if: "ctx.cif3?.itype == 'url'" + - uri_parts: + field: cif3.indicator + target_field: threat.indicator.url + keep_original: true + remove_if_successful: true + if: "ctx.threat?.indicator?.type == 'url'" + - set: + field: threat.indicator.url.full + value: "{{{threat.indicator.url.original}}}" + ignore_empty_value: true + if: "ctx.cif3?.itype == 'url'" + # Host could be either IP address or hostname + - grok: + field: cif3.indicator + patterns: + - "%{URIPROTO:threat.indicator.url.scheme}://(?:%{IP:threat.indicator.ip}|%{HOSTNAME:threat.indicator.url.domain})(?::%{POSINT:threat.indicator.url.port})?(?:%{URIPATH:threat.indicator.url.path})?.*" + ignore_failure: true + if: "ctx.cif3?.itype == 'url'" + + ## Email indicator operations + - set: + field: threat.indicator.type + value: email-addr + if: "ctx.cif3?.itype == 'email'" + - rename: + field: cif3.indicator + target_field: threat.indicator.email.address + ignore_missing: true + if: "ctx.threat?.indicator?.type == 'email-addr'" + - grok: + field: threat.indicator.email.address + patterns: + - "%{USERNAME}@%{GREEDYDATA:threat.indicator.url.domain}" + ignore_failure: true + if: "ctx.threat?.indicator?.type == 'email-addr'" + + ## Domain indicator operations + - set: + field: threat.indicator.type + value: domain-name + if: "ctx.cif3?.itype == 'fqdn'" + - rename: + field: cif3.indicator + target_field: threat.indicator.url.domain + ignore_missing: true + if: "ctx.threat?.indicator?.type == 'domain-name' && ctx.threat?.indicator?.url?.domain == null" + - append: + field: related.hosts + value: "{{{ threat.indicator.url.domain }}}" + if: ctx?.threat?.indicator?.url?.domain != null + + ###################### + # Confidence # + ###################### + - script: + lang: painless + if: ctx.cif3?.confidence != null + description: Normalize confidence level. + source: > + def value = ctx.cif3.confidence; + if (value < 0.0 || value > 10.0) { + ctx.threat.indicator.confidence = "None"; + return; + } + if (value >= 0.0 && value < 3.0) { + ctx.threat.indicator.confidence = "Low"; + return; + } + if (value >= 3.0 && value < 7.0) { + ctx.threat.indicator.confidence = "Med"; + return; + } + if (value >= 7.0 && value <= 10.0) { + ctx.threat.indicator.confidence = "High"; + return; + } + + ################### + # Tags ECS fields # + ################### + - foreach: + field: cif3.tags + ignore_missing: true + processor: + append: + field: tags + value: "{{_ingest._value}}" + allow_duplicates: false + if: ctx.cif3?.tags != null + + ## Misc + - rename: + field: cif3.protocol + target_field: network.transport + if: ctx.cif3?.protocol != null + - rename: + field: cif3.application + target_field: network.protocol + if: ctx.cif3?.application != null + - rename: + field: cif3.port + target_field: threat.indicator.port + # sometimes contains a range like 1000-1002 or CSVs like 10,22,52 + ignore_failure: true + if: ctx.cif3?.port != null + + ###################### + # Cleanup processors # + ###################### + - set: + field: threat.indicator.type + value: unknown + if: ctx.threat?.indicator?.type == null + - script: + lang: painless + if: ctx.cif3 != null + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + - remove: + field: cif3.rdata + ignore_missing: true + if: "ctx.cif3?.rdata == ''" + - remove: + field: + - cif3.indicator + - cif3.confidence + - cif3.indicator_ipv4 + - cif3.indicator_ipv6 + - cif3.group + - cif3.latitude + - cif3.longitude + - cif3.location + - cif3.city + - cif3.region + - cif3.tags + - cif3.tlp + - message + - _tmp + ignore_missing: true + if: ctx.threat?.indicator?.type != null +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/ti_cif3/data_stream/feed/fields/base-fields.yml b/packages/ti_cif3/data_stream/feed/fields/base-fields.yml new file mode 100644 index 00000000000..94818182d49 --- /dev/null +++ b/packages/ti_cif3/data_stream/feed/fields/base-fields.yml @@ -0,0 +1,24 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: ti_cif3 +- name: threat.feed.name + type: constant_keyword + description: Display friendly feed name + value: cif3 +- name: event.dataset + type: constant_keyword + description: Event dataset + value: ti_cif3.feed diff --git a/packages/ti_cif3/data_stream/feed/fields/beats.yml b/packages/ti_cif3/data_stream/feed/fields/beats.yml new file mode 100644 index 00000000000..b34ff711538 --- /dev/null +++ b/packages/ti_cif3/data_stream/feed/fields/beats.yml @@ -0,0 +1,12 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.flags + type: keyword + description: Flags for the log file. +- name: log.offset + type: long + description: Offset of the entry in the log file. +- name: log.file.path + type: keyword + description: Path to the log file. \ No newline at end of file diff --git a/packages/ti_cif3/data_stream/feed/fields/ecs.yml b/packages/ti_cif3/data_stream/feed/fields/ecs.yml new file mode 100644 index 00000000000..c7cdae942bd --- /dev/null +++ b/packages/ti_cif3/data_stream/feed/fields/ecs.yml @@ -0,0 +1,104 @@ +- external: ecs + name: ecs.version +- external: ecs + name: message +- external: ecs + name: error.message +- external: ecs + name: tags +- external: ecs + name: related.hash +- external: ecs + name: related.ip +- external: ecs + name: event.created +- external: ecs + name: event.ingested +- external: ecs + name: event.kind +- external: ecs + name: event.category +- external: ecs + name: event.module +- external: ecs + name: event.provider +- external: ecs + name: event.type +- external: ecs + name: event.original +- external: ecs + name: network.protocol +- external: ecs + name: network.transport +- external: ecs + name: threat.indicator.type +- external: ecs + name: threat.indicator.first_seen +- external: ecs + name: threat.indicator.last_seen +- external: ecs + name: threat.indicator.modified_at +- external: ecs + name: threat.indicator.reference +- external: ecs + name: threat.indicator.description +- external: ecs + name: threat.indicator.sightings +- external: ecs + name: threat.indicator.file.type +- external: ecs + name: threat.indicator.file.hash.md5 +- external: ecs + name: threat.indicator.file.hash.sha1 +- external: ecs + name: threat.indicator.file.hash.sha256 +- external: ecs + name: threat.indicator.file.hash.sha512 +- external: ecs + name: threat.indicator.file.pe.imphash +- external: ecs + name: threat.indicator.file.hash.ssdeep +- external: ecs + name: tls.client.ja3 +- external: ecs + name: threat.indicator.email.address +- external: ecs + name: threat.indicator.ip +- external: ecs + name: threat.indicator.url.domain +- external: ecs + name: threat.indicator.url.full +- external: ecs + name: threat.indicator.url.extension +- external: ecs + name: threat.indicator.url.original +- external: ecs + name: threat.indicator.url.path +- external: ecs + name: threat.indicator.url.port +- external: ecs + name: threat.indicator.url.scheme +- external: ecs + name: threat.indicator.url.query +- external: ecs + name: threat.indicator.provider +- external: ecs + name: threat.indicator.as.number +- external: ecs + name: threat.indicator.as.organization.name +- external: ecs + name: threat.indicator.marking.tlp +- external: ecs + name: threat.indicator.confidence +- external: ecs + name: threat.indicator.geo.location +- external: ecs + name: threat.indicator.geo.country_iso_code +- external: ecs + name: threat.indicator.geo.location.lat +- external: ecs + name: threat.indicator.geo.location.lon +- external: ecs + name: threat.indicator.geo.region_name +- external: ecs + name: threat.indicator.geo.timezone diff --git a/packages/ti_cif3/data_stream/feed/fields/fields.yml b/packages/ti_cif3/data_stream/feed/fields/fields.yml new file mode 100644 index 00000000000..27e4887a593 --- /dev/null +++ b/packages/ti_cif3/data_stream/feed/fields/fields.yml @@ -0,0 +1,129 @@ +- name: cif3 + type: group + description: Fields for CIFv3 Threat Indicators + fields: + - name: uuid + type: keyword + description: The ID of the indicator. + + - name: indicator + type: keyword + description: > + The value of the indicator, for example if the type is fqdn, this would be the value. + + - name: description + type: keyword + description: A description of the indicator. + + - name: rdata + type: keyword + description: > + Extra text or descriptive content related to the indicator such as OS, reverse lookup, etc. + + - name: reference + type: keyword + description: A reference URL with further info related to the indicator. + + - name: itype + type: keyword + description: > + The indicator type, can for example be "ipv4, fqdn, email, url, sha256". + + - name: tags + type: keyword + description: > + Comma-separated list of words describing the indicator such as "malware,exploit". + + - name: confidence + type: float + description: > + The confidence on a scale of 0-10 that the tags appropriately contextualize the indicator. + + - name: provider + type: keyword + description: The source of the indicator information. + + - name: application + type: keyword + description: The application used by the indicator, such as telnet or ssh. + + - name: protocol + type: text + description: The protocol used by the indicator. + + - name: portlist + type: text + description: The port or range of ports used by the indicator. + + - name: city + type: keyword + description: GeoIP city information. + + - name: region + type: keyword + description: GeoIP region information. + + - name: count + type: integer + description: > + The number of times the same indicator has been reported with the same metadata by the same provider. + + - name: cc + type: keyword + description: Country code of GeoIP. + + - name: location + type: geo_point + description: Lat/Long of GeoIP. + + - name: latitude + type: keyword + description: Latitude of GeoIP. + + - name: longitude + type: keyword + description: Longitude of GeoIP. + + - name: timezone + type: text + description: Timezone of GeoIP. + + - name: asn + type: integer + description: AS Number of IP. + + - name: asn_desc + type: keyword + description: AS Number org name. + + - name: indicator_ipv4 + type: ip + description: IPv4 address. + + - name: indicator_ipv4_mask + type: integer + description: subnet mask of IPv4 CIDR. + + - name: indicator_ipv6 + type: keyword + description: singleton IPv6 address. + + - name: indicator_ipv6_mask + type: integer + description: subnet mask of IPv6 CIDR. + + - name: indicator_iprange + type: ip_range + description: IPv4 or IPv6 IP Range. + + - name: indicator_ssdeep_chunksize + type: integer + description: SSDEEP hash chunk size. + + - name: indicator_ssdeep_chunk + type: text + description: SSDEEP hash chunk. + + - name: indicator_ssdeep_double_chunk + type: text + description: SSDEEP hash double chunk. diff --git a/packages/ti_cif3/data_stream/feed/manifest.yml b/packages/ti_cif3/data_stream/feed/manifest.yml new file mode 100644 index 00000000000..14fa0b86357 --- /dev/null +++ b/packages/ti_cif3/data_stream/feed/manifest.yml @@ -0,0 +1,125 @@ +title: "CIFv3 Feed" +type: logs +streams: + - input: httpjson + template_path: httpjson.yml.hbs + title: CIFv3 feed indicators + description: Collect CIFv3 feed indicators + vars: + - name: confidence + type: text + title: Confidence + multi: false + required: true + show_user: true + default: 8 + description: "Minimum confidence (0-10) to return indicator in feed" + + - name: cif_tags + type: text + title: Filter on indicator tags + multi: false + required: true + show_user: true + description: "A comma separated list of indicator tags to retrieve, e.g.: 'botnet,exploit,malware,phishing'" + + - name: type + type: text + title: Filter on indicator type + multi: false + required: true + show_user: true + description: "An indicator type (fqdn|ipv4|url|ssdeep) to retrieve, example: 'md5'" + + - name: limit + type: text + title: Result size limit + multi: false + required: true + show_user: true + default: 100000 + description: "Maximum result set size, capped at 250000" + + - name: initial_lookback + type: text + title: Initial lookback period + multi: false + required: true + show_user: true + default: 120h + description: How far back to look for indicators the first time the agent is started. + + - name: interval + type: text + title: Interval + multi: false + required: true + show_user: true + default: 60m + description: How frequently to pull the feed. + + # this doesn't currently work + #- name: filters + # type: yaml + # title: Optional REST API filters + # multi: false + # required: false + # show_user: false + # default: |- + # #tlp: white + # description: "Optional REST API Feed filters supported by [CIFv3](https://github.com/csirtgadgets/bearded-avenger/blob/master/cif/httpd/common.py#L7-L9)." + + - name: ssl + type: yaml + title: SSL + multi: false + required: false + show_user: false + description: "Default example enables https verification. Change to 'none' to disable. https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html" + default: |- + verification_mode: full + + - name: http_client_timeout + type: text + title: HTTP Client Timeout + multi: false + required: false + show_user: false + default: 120s + + - name: proxy_url + type: url + title: Proxy URL + multi: false + required: false + show_user: false + description: URL to proxy connections in the form of http[s]://:@: + + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + description: Tags to add to each event once ingested into Elastic. Ingested indicators' tags will be appended dynamically to this list. + default: + - forwarded + - cif3-indicator + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. \ No newline at end of file diff --git a/packages/ti_cif3/data_stream/feed/sample_event.json b/packages/ti_cif3/data_stream/feed/sample_event.json new file mode 100755 index 00000000000..9dfdd027cc3 --- /dev/null +++ b/packages/ti_cif3/data_stream/feed/sample_event.json @@ -0,0 +1,87 @@ +{ + "@timestamp": "2022-07-25T02:59:05.404Z", + "agent": { + "ephemeral_id": "6d30ac65-9d55-4014-9a2a-2fbcf8816fff", + "id": "f599fd51-b36d-45b4-a90f-4d63240b8477", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.3.2" + }, + "cif3": { + "itype": "ipv4", + "portlist": "443", + "uuid": "ac240898-1443-4d7e-a98a-1daed220c162" + }, + "data_stream": { + "dataset": "ti_cif3.feed", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "8.3.0" + }, + "elastic_agent": { + "id": "f599fd51-b36d-45b4-a90f-4d63240b8477", + "snapshot": false, + "version": "8.3.2" + }, + "event": { + "agent_id_status": "verified", + "category": "threat", + "created": "2022-07-25T02:59:05.404Z", + "dataset": "ti_cif3.feed", + "ingested": "2022-07-25T02:59:08Z", + "kind": "enrichment", + "original": "{\"application\":\"https\",\"asn\":8075,\"asn_desc\":\"microsoft-corp-msn-as-block\",\"cc\":\"br\",\"city\":\"campinas\",\"confidence\":10,\"count\":1,\"firsttime\":\"2022-07-20T20:25:53.000000Z\",\"group\":[\"everyone\"],\"indicator\":\"20.206.75.106\",\"indicator_ipv4\":\"20.206.75.106\",\"itype\":\"ipv4\",\"lasttime\":\"2022-07-20T20:25:53.000000Z\",\"latitude\":-22.9035,\"location\":[-47.0565,-22.9035],\"longitude\":-47.0565,\"portlist\":\"443\",\"protocol\":\"tcp\",\"provider\":\"sslbl.abuse.ch\",\"reference\":\"https://sslbl.abuse.ch/blacklist/sslipblacklist.csv\",\"region\":\"sao paulo\",\"reporttime\":\"2022-07-21T20:33:26.585967Z\",\"tags\":[\"botnet\"],\"timezone\":\"america/sao_paulo\",\"tlp\":\"white\",\"uuid\":\"ac240898-1443-4d7e-a98a-1daed220c162\"}", + "type": "indicator" + }, + "input": { + "type": "httpjson" + }, + "network": { + "protocol": "https", + "transport": "tcp" + }, + "related": { + "ip": [ + "20.206.75.106" + ] + }, + "tags": [ + "preserve_original_event", + "forwarded", + "cif3-indicator", + "botnet" + ], + "threat": { + "indicator": { + "as": { + "number": 8075, + "organization": { + "name": "microsoft-corp-msn-as-block" + } + }, + "confidence": "High", + "first_seen": "2022-07-20T20:25:53.000000Z", + "geo": { + "country_iso_code": "br", + "location": { + "lat": -22.9035, + "lon": -47.0565 + }, + "region_name": "sao paulo", + "timezone": "america/sao_paulo" + }, + "ip": "20.206.75.106", + "last_seen": "2022-07-20T20:25:53.000000Z", + "marking": { + "tlp": "WHITE" + }, + "modified_at": "2022-07-21T20:33:26.585967Z", + "provider": "sslbl.abuse.ch", + "reference": "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv", + "sightings": 1, + "type": "ipv4-addr" + } + } +} \ No newline at end of file diff --git a/packages/ti_cif3/docs/README.md b/packages/ti_cif3/docs/README.md new file mode 100644 index 00000000000..e5e3ebfcc23 --- /dev/null +++ b/packages/ti_cif3/docs/README.md @@ -0,0 +1,202 @@ +# Collective Intelligence Framework v3 Integration + +This integration connects with the [REST API from the running CIFv3 instance](https://github.com/csirtgadgets/bearded-avenger/blob/master/cif/httpd/views/feed/__init__.py) to retrieve indicators. + +## Data Streams + +### Feed + +The CIFv3 integration collects threat indicators based on user-defined configuration including a polling interval, how far back in time it should look, and other filters like indicator type and tags. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cif3.application | The application used by the indicator, such as telnet or ssh. | keyword | +| cif3.asn | AS Number of IP. | integer | +| cif3.asn_desc | AS Number org name. | keyword | +| cif3.cc | Country code of GeoIP. | keyword | +| cif3.city | GeoIP city information. | keyword | +| cif3.confidence | The confidence on a scale of 0-10 that the tags appropriately contextualize the indicator. | float | +| cif3.count | The number of times the same indicator has been reported with the same metadata by the same provider. | integer | +| cif3.description | A description of the indicator. | keyword | +| cif3.indicator | The value of the indicator, for example if the type is fqdn, this would be the value. | keyword | +| cif3.indicator_iprange | IPv4 or IPv6 IP Range. | ip_range | +| cif3.indicator_ipv4 | IPv4 address. | ip | +| cif3.indicator_ipv4_mask | subnet mask of IPv4 CIDR. | integer | +| cif3.indicator_ipv6 | singleton IPv6 address. | keyword | +| cif3.indicator_ipv6_mask | subnet mask of IPv6 CIDR. | integer | +| cif3.indicator_ssdeep_chunk | SSDEEP hash chunk. | text | +| cif3.indicator_ssdeep_chunksize | SSDEEP hash chunk size. | integer | +| cif3.indicator_ssdeep_double_chunk | SSDEEP hash double chunk. | text | +| cif3.itype | The indicator type, can for example be "ipv4, fqdn, email, url, sha256". | keyword | +| cif3.latitude | Latitude of GeoIP. | keyword | +| cif3.location | Lat/Long of GeoIP. | geo_point | +| cif3.longitude | Longitude of GeoIP. | keyword | +| cif3.portlist | The port or range of ports used by the indicator. | text | +| cif3.protocol | The protocol used by the indicator. | text | +| cif3.provider | The source of the indicator information. | keyword | +| cif3.rdata | Extra text or descriptive content related to the indicator such as OS, reverse lookup, etc. | keyword | +| cif3.reference | A reference URL with further info related to the indicator. | keyword | +| cif3.region | GeoIP region information. | keyword | +| cif3.tags | Comma-separated list of words describing the indicator such as "malware,exploit". | keyword | +| cif3.timezone | Timezone of GeoIP. | text | +| cif3.uuid | The ID of the indicator. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | +| event.dataset | Event dataset | constant_keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Path to the log file. | keyword | +| log.flags | Flags for the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | +| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | +| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| tags | List of keywords used to tag each event. | keyword | +| threat.feed.name | Display friendly feed name | constant_keyword | +| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| threat.indicator.as.organization.name | Organization name. | keyword | +| threat.indicator.as.organization.name.text | Multi-field of `threat.indicator.as.organization.name`. | match_only_text | +| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. Expected values are: \* Not Specified \* None \* Low \* Medium \* High | keyword | +| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | +| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | +| threat.indicator.file.hash.md5 | MD5 hash. | keyword | +| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | +| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | +| threat.indicator.file.hash.sha512 | SHA512 hash. | keyword | +| threat.indicator.file.hash.ssdeep | SSDEEP hash. | keyword | +| threat.indicator.file.pe.imphash | A hash of the imports in a PE file. An imphash -- or import hash -- can be used to fingerprint binaries even after recompilation or other code-level transformations have occurred, which would change more traditional hash values. Learn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html. | keyword | +| threat.indicator.file.type | File type (file, dir, or symlink). | keyword | +| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | +| threat.indicator.geo.country_iso_code | Country ISO code. | keyword | +| threat.indicator.geo.location | Longitude and latitude. | geo_point | +| threat.indicator.geo.location.lat | Longitude and latitude. | geo_point | +| threat.indicator.geo.location.lon | Longitude and latitude. | geo_point | +| threat.indicator.geo.region_name | Region name. | keyword | +| threat.indicator.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | +| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | +| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | +| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. Recommended values are: \* WHITE \* GREEN \* AMBER \* RED | keyword | +| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | +| threat.indicator.provider | The name of the indicator's provider. | keyword | +| threat.indicator.reference | Reference URL linking to additional information about this indicator. | keyword | +| threat.indicator.sightings | Number of times this indicator was observed conducting threat activity. | long | +| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. Recommended values: \* autonomous-system \* artifact \* directory \* domain-name \* email-addr \* file \* ipv4-addr \* ipv6-addr \* mac-addr \* mutex \* port \* process \* software \* url \* user-account \* windows-registry-key \* x509-certificate | keyword | +| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | +| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | +| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | +| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | +| threat.indicator.url.port | Port of the request, such as 443. | long | +| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | +| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | +| tls.client.ja3 | A hash that identifies clients based on how they perform an SSL/TLS handshake. | keyword | + + +An example event for `feed` looks as following: + +```json +{ + "@timestamp": "2022-07-25T02:59:05.404Z", + "agent": { + "ephemeral_id": "6d30ac65-9d55-4014-9a2a-2fbcf8816fff", + "id": "f599fd51-b36d-45b4-a90f-4d63240b8477", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.3.2" + }, + "cif3": { + "itype": "ipv4", + "portlist": "443", + "uuid": "ac240898-1443-4d7e-a98a-1daed220c162" + }, + "data_stream": { + "dataset": "ti_cif3.feed", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "8.3.0" + }, + "elastic_agent": { + "id": "f599fd51-b36d-45b4-a90f-4d63240b8477", + "snapshot": false, + "version": "8.3.2" + }, + "event": { + "agent_id_status": "verified", + "category": "threat", + "created": "2022-07-25T02:59:05.404Z", + "dataset": "ti_cif3.feed", + "ingested": "2022-07-25T02:59:08Z", + "kind": "enrichment", + "original": "{\"application\":\"https\",\"asn\":8075,\"asn_desc\":\"microsoft-corp-msn-as-block\",\"cc\":\"br\",\"city\":\"campinas\",\"confidence\":10,\"count\":1,\"firsttime\":\"2022-07-20T20:25:53.000000Z\",\"group\":[\"everyone\"],\"indicator\":\"20.206.75.106\",\"indicator_ipv4\":\"20.206.75.106\",\"itype\":\"ipv4\",\"lasttime\":\"2022-07-20T20:25:53.000000Z\",\"latitude\":-22.9035,\"location\":[-47.0565,-22.9035],\"longitude\":-47.0565,\"portlist\":\"443\",\"protocol\":\"tcp\",\"provider\":\"sslbl.abuse.ch\",\"reference\":\"https://sslbl.abuse.ch/blacklist/sslipblacklist.csv\",\"region\":\"sao paulo\",\"reporttime\":\"2022-07-21T20:33:26.585967Z\",\"tags\":[\"botnet\"],\"timezone\":\"america/sao_paulo\",\"tlp\":\"white\",\"uuid\":\"ac240898-1443-4d7e-a98a-1daed220c162\"}", + "type": "indicator" + }, + "input": { + "type": "httpjson" + }, + "network": { + "protocol": "https", + "transport": "tcp" + }, + "related": { + "ip": [ + "20.206.75.106" + ] + }, + "tags": [ + "preserve_original_event", + "forwarded", + "cif3-indicator", + "botnet" + ], + "threat": { + "indicator": { + "as": { + "number": 8075, + "organization": { + "name": "microsoft-corp-msn-as-block" + } + }, + "confidence": "High", + "first_seen": "2022-07-20T20:25:53.000000Z", + "geo": { + "country_iso_code": "br", + "location": { + "lat": -22.9035, + "lon": -47.0565 + }, + "region_name": "sao paulo", + "timezone": "america/sao_paulo" + }, + "ip": "20.206.75.106", + "last_seen": "2022-07-20T20:25:53.000000Z", + "marking": { + "tlp": "WHITE" + }, + "modified_at": "2022-07-21T20:33:26.585967Z", + "provider": "sslbl.abuse.ch", + "reference": "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv", + "sightings": 1, + "type": "ipv4-addr" + } + } +} +``` \ No newline at end of file diff --git a/packages/ti_cif3/img/csg_logo_big.svg b/packages/ti_cif3/img/csg_logo_big.svg new file mode 100755 index 00000000000..a794457dc29 --- /dev/null +++ b/packages/ti_cif3/img/csg_logo_big.svg @@ -0,0 +1,23 @@ + + + + + + + + + diff --git a/packages/ti_cif3/kibana/dashboard/ti_cif3-6005a190-0aba-11ed-bcc0-01c79f2670f3.json b/packages/ti_cif3/kibana/dashboard/ti_cif3-6005a190-0aba-11ed-bcc0-01c79f2670f3.json new file mode 100644 index 00000000000..0c579cc276a --- /dev/null +++ b/packages/ti_cif3/kibana/dashboard/ti_cif3-6005a190-0aba-11ed-bcc0-01c79f2670f3.json @@ -0,0 +1,320 @@ +{ + "attributes": { + "description": "Dashboard providing statistics about FQDN type indicators from the Collective Intelligence Framework v3 integration", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "threat.indicator.type", + "negate": false, + "params": { + "query": "domain-name" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "threat.indicator.type": "domain-name" + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "ti_cif3.feed" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "ti_cif3.feed" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "id": "", + "params": { + "fontSize": 12, + "markdown": "**Navigation**\n\n[CIFv3 Overview](/app/dashboards#/view/ti_cif3-b4d9d9b0-0a2f-11ed-bcc0-01c79f2670f3) \n[CIFv3 Emails](/app/dashboards#/view/ti_cif3-bda23600-0abb-11ed-bcc0-01c79f2670f3) \n[CIFv3 Files](/app/dashboards#/view/ti_cif3-63a0e470-0a30-11ed-bcc0-01c79f2670f3) \n**[CIFv3 FQDNs (This Page)](/app/dashboards#/view/ti_cif3-6005a190-0aba-11ed-bcc0-01c79f2670f3)** \n[CIFv3 IPs](/app/dashboards#/view/ti_cif3-aedada10-0ab5-11ed-bcc0-01c79f2670f3) \n[CIFv3 URLs](/app/dashboards#/view/ti_cif3-fef149c0-0a2f-11ed-bcc0-01c79f2670f3) \n\n[Integrations Page](/app/integrations/detail/ti_cif3/overview)\n\n\n**Overview**\n\nThis dashboard is an overview of the different threat intelligence indicators with a **threat.indicator.type: domain-name**. \n\nThe dashboard is made to provide general statistics and show the health of your indicators like popular domains and statistics about how many unique indicators are ingested and other relevant information.", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 39, + "i": "4c3ed6e1-8b4e-4eab-8d84-70ed4f506216", + "w": 7, + "x": 0, + "y": 0 + }, + "panelIndex": "4c3ed6e1-8b4e-4eab-8d84-70ed4f506216", + "title": "Files Navigation Textbox [Logs CIFv3]", + "type": "visualization", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-c94400ee-a135-4a99-9693-5879d29f7aad", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "c94400ee-a135-4a99-9693-5879d29f7aad": { + "columnOrder": [ + "2934249f-fce5-4637-87ff-d2596d1b6ec5" + ], + "columns": { + "2934249f-fce5-4637-87ff-d2596d1b6ec5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique Domains", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.url.domain" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "2934249f-fce5-4637-87ff-d2596d1b6ec5", + "layerId": "c94400ee-a135-4a99-9693-5879d29f7aad", + "layerType": "data", + "size": "xl", + "textAlign": "center", + "titlePosition": "bottom" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "02f1732b-a981-4fba-8b27-b944f2f3c98c", + "w": 6, + "x": 7, + "y": 0 + }, + "panelIndex": "02f1732b-a981-4fba-8b27-b944f2f3c98c", + "title": "Unique Domains [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-09bca2c1-c599-4575-be8a-a416589c7082", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "09bca2c1-c599-4575-be8a-a416589c7082": { + "columnOrder": [ + "87d9346d-c199-44ef-b58c-2c0c7625a523", + "40a4b01a-1e63-4cd8-ab62-da960940d757" + ], + "columns": { + "40a4b01a-1e63-4cd8-ab62-da960940d757": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.url.domain" + }, + "87d9346d-c199-44ef-b58c-2c0c7625a523": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "FQDN", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "40a4b01a-1e63-4cd8-ab62-da960940d757", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "size": 15 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.url.domain" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "87d9346d-c199-44ef-b58c-2c0c7625a523", + "isTransposed": false + }, + { + "columnId": "40a4b01a-1e63-4cd8-ab62-da960940d757", + "isTransposed": false + } + ], + "layerId": "09bca2c1-c599-4575-be8a-a416589c7082", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "c2db10e8-0e7e-4199-b787-48e14bd2e2fe", + "w": 18, + "x": 13, + "y": 0 + }, + "panelIndex": "c2db10e8-0e7e-4199-b787-48e14bd2e2fe", + "title": "Sample of Domains [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "[Logs CIFv3] FQDNs", + "version": 1 + }, + "coreMigrationVersion": "8.0.0", + "id": "ti_cif3-6005a190-0aba-11ed-bcc0-01c79f2670f3", + "migrationVersion": { + "dashboard": "8.0.0" + }, + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "02f1732b-a981-4fba-8b27-b944f2f3c98c:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "02f1732b-a981-4fba-8b27-b944f2f3c98c:indexpattern-datasource-layer-c94400ee-a135-4a99-9693-5879d29f7aad", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "c2db10e8-0e7e-4199-b787-48e14bd2e2fe:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "c2db10e8-0e7e-4199-b787-48e14bd2e2fe:indexpattern-datasource-layer-09bca2c1-c599-4575-be8a-a416589c7082", + "type": "index-pattern" + }, + { + "id": "ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "name": "tag-ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "type": "tag" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/ti_cif3/kibana/dashboard/ti_cif3-63a0e470-0a30-11ed-bcc0-01c79f2670f3.json b/packages/ti_cif3/kibana/dashboard/ti_cif3-63a0e470-0a30-11ed-bcc0-01c79f2670f3.json new file mode 100644 index 00000000000..bb03146dc6d --- /dev/null +++ b/packages/ti_cif3/kibana/dashboard/ti_cif3-63a0e470-0a30-11ed-bcc0-01c79f2670f3.json @@ -0,0 +1,730 @@ +{ + "attributes": { + "description": "Dashboard providing statistics about File type indicators from the Collective Intelligence Framework v3 integration", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "threat.indicator.type", + "negate": false, + "params": { + "query": "file" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "threat.indicator.type": "file" + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "ti_cif3.feed" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "ti_cif3.feed" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "**Navigation**\n\n[CIFv3 Overview](/app/dashboards#/view/ti_cif3-b4d9d9b0-0a2f-11ed-bcc0-01c79f2670f3) \n[CIFv3 Emails](/app/dashboards#/view/ti_cif3-bda23600-0abb-11ed-bcc0-01c79f2670f3) \n**[CIFv3 Files (This Page)](/app/dashboards#/view/ti_cif3-63a0e470-0a30-11ed-bcc0-01c79f2670f3)** \n[CIFv3 FQDNs](/app/dashboards#/view/ti_cif3-6005a190-0aba-11ed-bcc0-01c79f2670f3) \n[CIFv3 IPs](/app/dashboards#/view/ti_cif3-aedada10-0ab5-11ed-bcc0-01c79f2670f3) \n[CIFv3 URLs](/app/dashboards#/view/ti_cif3-fef149c0-0a2f-11ed-bcc0-01c79f2670f3) \n\n[Integrations Page](/app/integrations/detail/ti_cif3/overview)\n\n\n**Overview**\n\nThis dashboard is an overview of the different threat intelligence indicators with a **threat.indicator.type: file**.\n\nThe dashboard is made to provide general statistics and show the health of your indicators like hash type counters, popular domains, statistics about how many unique indicators are ingested and other relevant information.", + "openLinksInNewTab": false + }, + "title": "Files Navigation Textbox [Logs CIFv3]", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 35, + "i": "09ba3dc0-e2e2-4799-b47f-bb919bf290a1", + "w": 7, + "x": 0, + "y": 0 + }, + "panelIndex": "09ba3dc0-e2e2-4799-b47f-bb919bf290a1", + "title": "Files Navigation Textbox [Logs CIFv3]", + "type": "visualization", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-b83c382d-fab9-4e60-a632-475e221cc20c", + "type": "index-pattern" + } + ], + "sharingSavedObjectProps": { + "outcome": "exactMatch", + "sourceId": "ti_cif3-d888e3e0-3b38-11ec-ae50-2fdf1e96c6a6" + }, + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "b83c382d-fab9-4e60-a632-475e221cc20c": { + "columnOrder": [ + "eda3c6d9-dacb-4e5e-b977-50104f76e91a" + ], + "columns": { + "eda3c6d9-dacb-4e5e-b977-50104f76e91a": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique MD5", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.file.hash.md5" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "eda3c6d9-dacb-4e5e-b977-50104f76e91a", + "layerId": "b83c382d-fab9-4e60-a632-475e221cc20c", + "layerType": "data", + "size": "xl", + "textAlign": "center", + "titlePosition": "bottom" + } + }, + "title": "Unique MD5 [CIFv3]", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "4d3e11dc-c4cc-4373-bb83-3d39fe6ffa98", + "w": 6, + "x": 7, + "y": 0 + }, + "panelIndex": "4d3e11dc-c4cc-4373-bb83-3d39fe6ffa98", + "title": "Unique MD5 [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-49b7070a-f1d3-46e1-a980-2f6d6d130167", + "type": "index-pattern" + } + ], + "sharingSavedObjectProps": { + "outcome": "exactMatch", + "sourceId": "ti_cif3-5d6111a0-3b39-11ec-ae50-2fdf1e96c6a6" + }, + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "49b7070a-f1d3-46e1-a980-2f6d6d130167": { + "columnOrder": [ + "b6c5e221-88ff-490e-bd3e-188b3e0dd1f4" + ], + "columns": { + "b6c5e221-88ff-490e-bd3e-188b3e0dd1f4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique SHA256", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.file.hash.sha256" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "b6c5e221-88ff-490e-bd3e-188b3e0dd1f4", + "layerId": "49b7070a-f1d3-46e1-a980-2f6d6d130167", + "layerType": "data", + "size": "xl", + "textAlign": "center", + "titlePosition": "bottom" + } + }, + "title": "Unique SHA256 [CIFv3]", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "93e32abe-87e3-469e-b7e9-a7ef7dfa2cce", + "w": 6, + "x": 13, + "y": 0 + }, + "panelIndex": "93e32abe-87e3-469e-b7e9-a7ef7dfa2cce", + "title": "Unique SHA256 [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-2825d170-daeb-4a6d-9d8f-8fda4dccffcc", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "2825d170-daeb-4a6d-9d8f-8fda4dccffcc": { + "columnOrder": [ + "cb37ded7-9f40-418f-bfb9-6250652373d7" + ], + "columns": { + "cb37ded7-9f40-418f-bfb9-6250652373d7": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique SSDEEP", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.file.hash.ssdeep" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "cb37ded7-9f40-418f-bfb9-6250652373d7", + "layerId": "2825d170-daeb-4a6d-9d8f-8fda4dccffcc", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "703fd39c-9642-4c7d-93c8-056f019acf42", + "w": 6, + "x": 19, + "y": 0 + }, + "panelIndex": "703fd39c-9642-4c7d-93c8-056f019acf42", + "title": "Unique SSDEEP [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-ace6c894-6dac-441d-b0db-3e246db99579", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "ace6c894-6dac-441d-b0db-3e246db99579": { + "columnOrder": [ + "4c6f7061-d5e9-4c04-b9b2-39b984b06393", + "e00a1b25-655b-4541-8ce0-1f84bdb16b1e" + ], + "columns": { + "4c6f7061-d5e9-4c04-b9b2-39b984b06393": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of threat.indicator.description", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "e00a1b25-655b-4541-8ce0-1f84bdb16b1e", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.description" + }, + "e00a1b25-655b-4541-8ce0-1f84bdb16b1e": { + "dataType": "number", + "isBucketed": false, + "label": "Unique count of threat.indicator.description", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.description" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "groups": [ + "4c6f7061-d5e9-4c04-b9b2-39b984b06393" + ], + "layerId": "ace6c894-6dac-441d-b0db-3e246db99579", + "layerType": "data", + "legendDisplay": "default", + "metric": "e00a1b25-655b-4541-8ce0-1f84bdb16b1e", + "nestedLegend": false, + "numberDisplay": "percent" + } + ], + "shape": "treemap" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsPie" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 16, + "i": "9717eae1-9937-41e7-bad1-e9ce43d06723", + "w": 22, + "x": 25, + "y": 0 + }, + "panelIndex": "9717eae1-9937-41e7-bad1-e9ce43d06723", + "title": "File Descriptions [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-85ad73b3-3b76-49f1-ad20-6256b58918f8", + "type": "index-pattern" + } + ], + "sharingSavedObjectProps": { + "outcome": "exactMatch", + "sourceId": "ti_cif3-28549810-3b39-11ec-ae50-2fdf1e96c6a6" + }, + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "85ad73b3-3b76-49f1-ad20-6256b58918f8": { + "columnOrder": [ + "289bd005-bdd2-4f3b-83b9-ad6ae52a9ed3" + ], + "columns": { + "289bd005-bdd2-4f3b-83b9-ad6ae52a9ed3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique SHA1", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.file.hash.sha1" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "289bd005-bdd2-4f3b-83b9-ad6ae52a9ed3", + "layerId": "85ad73b3-3b76-49f1-ad20-6256b58918f8", + "layerType": "data", + "size": "xl", + "textAlign": "center", + "titlePosition": "bottom" + } + }, + "title": "Unique SHA1 [CIFv3]", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "e9b6f0ad-5e6b-44da-923e-dc0d5ccfdfea", + "w": 6, + "x": 7, + "y": 8 + }, + "panelIndex": "e9b6f0ad-5e6b-44da-923e-dc0d5ccfdfea", + "title": "Unique SHA1 [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-331e77de-53be-48a4-8793-3fe9a23b22b1", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "331e77de-53be-48a4-8793-3fe9a23b22b1": { + "columnOrder": [ + "428df405-7955-4c10-94c1-0791e75aed8f" + ], + "columns": { + "428df405-7955-4c10-94c1-0791e75aed8f": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique SHA512", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.file.hash.sha512" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "428df405-7955-4c10-94c1-0791e75aed8f", + "layerId": "331e77de-53be-48a4-8793-3fe9a23b22b1", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "cb4ca769-08b2-4570-8a30-27cff9b77093", + "w": 6, + "x": 13, + "y": 8 + }, + "panelIndex": "cb4ca769-08b2-4570-8a30-27cff9b77093", + "title": "Unique SHA512 [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-4c3ad4e3-46af-447e-a4ce-dab516c52797", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "4c3ad4e3-46af-447e-a4ce-dab516c52797": { + "columnOrder": [ + "181798f7-2b90-44e1-b76a-2f17b7210690" + ], + "columns": { + "181798f7-2b90-44e1-b76a-2f17b7210690": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique IMPHASH", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.file.pe.imphash" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "181798f7-2b90-44e1-b76a-2f17b7210690", + "layerId": "4c3ad4e3-46af-447e-a4ce-dab516c52797", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "823f92b7-a2ff-4883-aad1-28d3652371fe", + "w": 6, + "x": 19, + "y": 8 + }, + "panelIndex": "823f92b7-a2ff-4883-aad1-28d3652371fe", + "title": "Unique IMPHASH [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "[Logs CIFv3] Files", + "version": 1 + }, + "coreMigrationVersion": "8.0.0", + "id": "ti_cif3-63a0e470-0a30-11ed-bcc0-01c79f2670f3", + "migrationVersion": { + "dashboard": "8.0.0" + }, + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "4d3e11dc-c4cc-4373-bb83-3d39fe6ffa98:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "4d3e11dc-c4cc-4373-bb83-3d39fe6ffa98:indexpattern-datasource-layer-b83c382d-fab9-4e60-a632-475e221cc20c", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "93e32abe-87e3-469e-b7e9-a7ef7dfa2cce:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "93e32abe-87e3-469e-b7e9-a7ef7dfa2cce:indexpattern-datasource-layer-49b7070a-f1d3-46e1-a980-2f6d6d130167", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "703fd39c-9642-4c7d-93c8-056f019acf42:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "703fd39c-9642-4c7d-93c8-056f019acf42:indexpattern-datasource-layer-2825d170-daeb-4a6d-9d8f-8fda4dccffcc", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "9717eae1-9937-41e7-bad1-e9ce43d06723:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "9717eae1-9937-41e7-bad1-e9ce43d06723:indexpattern-datasource-layer-ace6c894-6dac-441d-b0db-3e246db99579", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "e9b6f0ad-5e6b-44da-923e-dc0d5ccfdfea:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "e9b6f0ad-5e6b-44da-923e-dc0d5ccfdfea:indexpattern-datasource-layer-85ad73b3-3b76-49f1-ad20-6256b58918f8", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "cb4ca769-08b2-4570-8a30-27cff9b77093:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "cb4ca769-08b2-4570-8a30-27cff9b77093:indexpattern-datasource-layer-331e77de-53be-48a4-8793-3fe9a23b22b1", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "823f92b7-a2ff-4883-aad1-28d3652371fe:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "823f92b7-a2ff-4883-aad1-28d3652371fe:indexpattern-datasource-layer-4c3ad4e3-46af-447e-a4ce-dab516c52797", + "type": "index-pattern" + }, + { + "id": "ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "name": "tag-ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "type": "tag" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/ti_cif3/kibana/dashboard/ti_cif3-aedada10-0ab5-11ed-bcc0-01c79f2670f3.json b/packages/ti_cif3/kibana/dashboard/ti_cif3-aedada10-0ab5-11ed-bcc0-01c79f2670f3.json new file mode 100644 index 00000000000..364b0bb6569 --- /dev/null +++ b/packages/ti_cif3/kibana/dashboard/ti_cif3-aedada10-0ab5-11ed-bcc0-01c79f2670f3.json @@ -0,0 +1,699 @@ +{ + "attributes": { + "description": "Dashboard providing statistics about IP type indicators from the Collective Intelligence Framework v3 integration", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "threat.indicator.type", + "negate": false, + "params": [ + "ipv6-addr", + "ipv4-addr" + ], + "type": "phrases" + }, + "query": { + "bool": { + "minimum_should_match": 1, + "should": [ + { + "match_phrase": { + "threat.indicator.type": "ipv6-addr" + } + }, + { + "match_phrase": { + "threat.indicator.type": "ipv4-addr" + } + } + ] + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "ti_cif3.feed" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "ti_cif3.feed" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "id": "", + "params": { + "fontSize": 12, + "markdown": "**Navigation**\n\n[CIFv3 Overview](/app/dashboards#/view/ti_cif3-b4d9d9b0-0a2f-11ed-bcc0-01c79f2670f3) \n[CIFv3 Emails](/app/dashboards#/view/ti_cif3-bda23600-0abb-11ed-bcc0-01c79f2670f3) \n[CIFv3 Files](/app/dashboards#/view/ti_cif3-63a0e470-0a30-11ed-bcc0-01c79f2670f3) \n[CIFv3 FQDNs](/app/dashboards#/view/ti_cif3-6005a190-0aba-11ed-bcc0-01c79f2670f3) \n**[CIFv3 IPs(This Page)](/app/dashboards#/view/ti_cif3-aedada10-0ab5-11ed-bcc0-01c79f2670f3)** \n[CIFv3 URLs](/app/dashboards#/view/ti_cif3-fef149c0-0a2f-11ed-bcc0-01c79f2670f3) \n\n[Integrations Page](/app/integrations/detail/ti_cif3/overview)\n\n\n**Overview**\n\nThis dashboard is an overview of the different threat intelligence indicators with a **threat.indicator.type: ipv4-addr OR ipv6-addr**. \n\nThe dashboard is made to provide general statistics and show the health of your indicators like prevalent ASNs, GeoIP regions, statistics about how many unique indicators are ingested, and other relevant information.", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 39, + "i": "4c3ed6e1-8b4e-4eab-8d84-70ed4f506216", + "w": 7, + "x": 0, + "y": 0 + }, + "panelIndex": "4c3ed6e1-8b4e-4eab-8d84-70ed4f506216", + "title": "Files Navigation Textbox [Logs CIFv3]", + "type": "visualization", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-79edd9a4-1178-4294-94df-5d4b145d0e40", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "79edd9a4-1178-4294-94df-5d4b145d0e40": { + "columnOrder": [ + "d1ce22a5-8010-4830-8c61-e8da8c2b2d11" + ], + "columns": { + "d1ce22a5-8010-4830-8c61-e8da8c2b2d11": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique IPs", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.ip" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "d1ce22a5-8010-4830-8c61-e8da8c2b2d11", + "layerId": "79edd9a4-1178-4294-94df-5d4b145d0e40", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "7725b9bd-df8d-491c-a518-fe00a4538ebc", + "w": 5, + "x": 7, + "y": 0 + }, + "panelIndex": "7725b9bd-df8d-491c-a518-fe00a4538ebc", + "title": "Unique IPs [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-e8210fab-252e-4357-82f5-c8fc55fe2057", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "e8210fab-252e-4357-82f5-c8fc55fe2057": { + "columnOrder": [ + "937cc845-c2e1-412a-b419-97c9d8076bee" + ], + "columns": { + "937cc845-c2e1-412a-b419-97c9d8076bee": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique ASNs", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.as.number" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "937cc845-c2e1-412a-b419-97c9d8076bee", + "layerId": "e8210fab-252e-4357-82f5-c8fc55fe2057", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "329518f4-c5f9-42b0-b396-85ffcbb8cda3", + "w": 5, + "x": 12, + "y": 0 + }, + "panelIndex": "329518f4-c5f9-42b0-b396-85ffcbb8cda3", + "title": "Unique ASNs [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-864ef66d-9195-45a5-9dcd-916bcac76fd1", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "864ef66d-9195-45a5-9dcd-916bcac76fd1": { + "columnOrder": [ + "d8bba7bc-4a82-40c3-a858-e92244ef476c", + "1c86e415-dcb9-49ae-aa85-e4c7c0ddffd7" + ], + "columns": { + "1c86e415-dcb9-49ae-aa85-e4c7c0ddffd7": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "d8bba7bc-4a82-40c3-a858-e92244ef476c": { + "dataType": "number", + "isBucketed": true, + "label": "Top values of threat.indicator.as.number", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "1c86e415-dcb9-49ae-aa85-e4c7c0ddffd7", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.as.number" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "groups": [ + "d8bba7bc-4a82-40c3-a858-e92244ef476c" + ], + "layerId": "864ef66d-9195-45a5-9dcd-916bcac76fd1", + "layerType": "data", + "legendDisplay": "default", + "metric": "1c86e415-dcb9-49ae-aa85-e4c7c0ddffd7", + "nestedLegend": false, + "numberDisplay": "percent" + } + ], + "shape": "treemap" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsPie" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "c651f85b-26e4-481e-91ff-39267e540183", + "w": 21, + "x": 17, + "y": 0 + }, + "panelIndex": "c651f85b-26e4-481e-91ff-39267e540183", + "title": "Most Prevalent ASNs [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-b3600118-bbef-4f41-b472-c08e802518c3", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "b3600118-bbef-4f41-b472-c08e802518c3": { + "columnOrder": [ + "deabebaa-8bfa-4b99-8996-5dd59ecd37ca", + "a9e4b58d-6503-4645-bc9b-69aede4b3a4c" + ], + "columns": { + "a9e4b58d-6503-4645-bc9b-69aede4b3a4c": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "deabebaa-8bfa-4b99-8996-5dd59ecd37ca": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Country Code", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "a9e4b58d-6503-4645-bc9b-69aede4b3a4c", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 15 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.geo.country_iso_code" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "deabebaa-8bfa-4b99-8996-5dd59ecd37ca", + "isTransposed": false + }, + { + "columnId": "a9e4b58d-6503-4645-bc9b-69aede4b3a4c", + "isTransposed": false + } + ], + "layerId": "b3600118-bbef-4f41-b472-c08e802518c3", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "aea51b8a-0962-4b21-aa7e-7c599f0f45a4", + "w": 10, + "x": 38, + "y": 0 + }, + "panelIndex": "aea51b8a-0962-4b21-aa7e-7c599f0f45a4", + "title": "Most Common Countries [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-da912e35-7510-42a6-b546-8d10a33b6546", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "da912e35-7510-42a6-b546-8d10a33b6546": { + "columnOrder": [ + "989df1d6-f18f-4874-8601-9e7741935cc8", + "f60fc28d-e739-46a2-a0ce-1340df8f7249" + ], + "columns": { + "989df1d6-f18f-4874-8601-9e7741935cc8": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of threat.indicator.type", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "f60fc28d-e739-46a2-a0ce-1340df8f7249", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "size": 2 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.type" + }, + "f60fc28d-e739-46a2-a0ce-1340df8f7249": { + "dataType": "number", + "isBucketed": false, + "label": "Unique count of threat.indicator.ip", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.ip" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "groups": [ + "989df1d6-f18f-4874-8601-9e7741935cc8" + ], + "layerId": "da912e35-7510-42a6-b546-8d10a33b6546", + "layerType": "data", + "legendDisplay": "default", + "metric": "f60fc28d-e739-46a2-a0ce-1340df8f7249", + "nestedLegend": false, + "numberDisplay": "percent" + } + ], + "shape": "donut" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsPie" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 7, + "i": "1536f4f2-41d6-4fd0-b6c4-3650a2b5f92d", + "w": 10, + "x": 7, + "y": 8 + }, + "panelIndex": "1536f4f2-41d6-4fd0-b6c4-3650a2b5f92d", + "title": "Percentage of IP Type [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true,\"lightModeDefault\":\"road_map_desaturated\"},\"id\":\"3df0f38b-db9e-451e-bb01-5a27226075df\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{\"type\":\"TILE\"},\"includeInFitToBounds\":true,\"type\":\"VECTOR_TILE\"},{\"sourceDescriptor\":{\"indexPatternId\":\"logs-*\",\"geoField\":\"threat.indicator.geo.location\",\"filterByMapBounds\":true,\"scalingType\":\"MVT\",\"id\":\"13a0c980-6195-4e3e-8506-b383ab8866c2\",\"type\":\"ES_SEARCH\",\"applyGlobalQuery\":true,\"applyGlobalTime\":true,\"applyForceRefresh\":true,\"tooltipProperties\":[],\"sortField\":\"\",\"sortOrder\":\"desc\",\"topHitsSplitField\":\"\",\"topHitsSize\":1},\"id\":\"0a0a1a3e-d002-47b0-a99a-03eb965b8bc4\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#ea7861\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#e05235\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"includeInFitToBounds\":true,\"type\":\"TILED_VECTOR\",\"joins\":[]}]", + "mapStateJSON": "{\"zoom\":1.14,\"center\":{\"lon\":0,\"lat\":19.94277},\"timeFilters\":{\"from\":\"now-75m\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"settings\":{\"autoFitToDataBounds\":false,\"backgroundColor\":\"#ffffff\",\"disableInteractive\":false,\"disableTooltipControl\":false,\"hideToolbarOverlay\":false,\"hideLayerControl\":false,\"hideViewControl\":false,\"initialLocation\":\"LAST_SAVED_LOCATION\",\"fixedLocation\":{\"lat\":0,\"lon\":0,\"zoom\":2},\"browserLocation\":{\"zoom\":2},\"maxZoom\":24,\"minZoom\":0,\"showScaleControl\":false,\"showSpatialFilters\":true,\"showTimesliderToggleButton\":true,\"spatialFiltersAlpa\":0.3,\"spatialFiltersFillColor\":\"#DA8B45\",\"spatialFiltersLineColor\":\"#DA8B45\"}}", + "title": "", + "uiStateJSON": "{\"isLayerTOCOpen\":false,\"openTOCDetails\":[]}" + }, + "enhancements": {}, + "hiddenLayers": [], + "hidePanelTitles": false, + "isLayerTOCOpen": false, + "mapBuffer": { + "maxLat": 85.05113, + "maxLon": 360, + "minLat": -85.05113, + "minLon": -360 + }, + "mapCenter": { + "lat": 26.16939, + "lon": 14.00125, + "zoom": 0.49 + }, + "openTOCDetails": [] + }, + "gridData": { + "h": 14, + "i": "ad624736-f1dd-4d77-8517-680e7bc4b882", + "w": 23, + "x": 7, + "y": 15 + }, + "panelIndex": "ad624736-f1dd-4d77-8517-680e7bc4b882", + "title": "IP Source Location [Logs CIFv3]", + "type": "map", + "version": "8.0.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "[Logs CIFv3] IPs", + "version": 1 + }, + "coreMigrationVersion": "8.0.0", + "id": "ti_cif3-aedada10-0ab5-11ed-bcc0-01c79f2670f3", + "migrationVersion": { + "dashboard": "8.0.0" + }, + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "7725b9bd-df8d-491c-a518-fe00a4538ebc:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "7725b9bd-df8d-491c-a518-fe00a4538ebc:indexpattern-datasource-layer-79edd9a4-1178-4294-94df-5d4b145d0e40", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "329518f4-c5f9-42b0-b396-85ffcbb8cda3:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "329518f4-c5f9-42b0-b396-85ffcbb8cda3:indexpattern-datasource-layer-e8210fab-252e-4357-82f5-c8fc55fe2057", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "c651f85b-26e4-481e-91ff-39267e540183:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "c651f85b-26e4-481e-91ff-39267e540183:indexpattern-datasource-layer-864ef66d-9195-45a5-9dcd-916bcac76fd1", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "aea51b8a-0962-4b21-aa7e-7c599f0f45a4:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "aea51b8a-0962-4b21-aa7e-7c599f0f45a4:indexpattern-datasource-layer-b3600118-bbef-4f41-b472-c08e802518c3", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "1536f4f2-41d6-4fd0-b6c4-3650a2b5f92d:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "1536f4f2-41d6-4fd0-b6c4-3650a2b5f92d:indexpattern-datasource-layer-da912e35-7510-42a6-b546-8d10a33b6546", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "ad624736-f1dd-4d77-8517-680e7bc4b882:layer_1_source_index_pattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "ab7ab31c-e76f-4613-b17d-fdd909f17e0d:indexpattern-datasource-layer-0f63318a-a857-4d83-89ce-a94e2242b79e", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "02f1732b-a981-4fba-8b27-b944f2f3c98c:indexpattern-datasource-layer-c94400ee-a135-4a99-9693-5879d29f7aad", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "fda93ed1-72f0-4489-80b7-9e69d14f30aa:indexpattern-datasource-layer-9fa49c4c-5544-472d-afce-e51d6a5687fe", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "73a752f9-bde5-4396-8ede-e9e77a37182d:indexpattern-datasource-layer-a6fa56f8-32fa-405d-8771-dade4fe75d62", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "8994501a-1550-4cf2-857f-d6b6491ffb62:indexpattern-datasource-layer-db89074c-e1fe-4091-bdb1-e42a36e82bac", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "c7c6e8dc-b649-434c-9650-8a1564d4d676:indexpattern-datasource-layer-88a112e1-6da1-49d3-9177-19f98280c200", + "type": "index-pattern" + }, + { + "id": "ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "name": "tag-ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "type": "tag" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/ti_cif3/kibana/dashboard/ti_cif3-b4d9d9b0-0a2f-11ed-bcc0-01c79f2670f3.json b/packages/ti_cif3/kibana/dashboard/ti_cif3-b4d9d9b0-0a2f-11ed-bcc0-01c79f2670f3.json new file mode 100644 index 00000000000..a4f913d02f5 --- /dev/null +++ b/packages/ti_cif3/kibana/dashboard/ti_cif3-b4d9d9b0-0a2f-11ed-bcc0-01c79f2670f3.json @@ -0,0 +1,719 @@ +{ + "attributes": { + "description": "Dashboard providing statistics about indicators ingested from the Collective Intelligence Framework v3 integration", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "event.kind", + "negate": false, + "params": { + "query": "enrichment" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.kind": "enrichment" + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "ti_cif3.feed" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "ti_cif3.feed" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "**Navigation**\n\n**[CIFv3 (This Page)](/app/dashboards#/view/ti_cif3-b4d9d9b0-0a2f-11ed-bcc0-01c79f2670f3)** \n[CIFv3 Emails](/app/dashboards#/view/ti_cif3-bda23600-0abb-11ed-bcc0-01c79f2670f3) \n[CIFv3 Files](/app/dashboards#/view/ti_cif3-63a0e470-0a30-11ed-bcc0-01c79f2670f3) \n[CIFv3 FQDNs](/app/dashboards#/view/ti_cif3-6005a190-0aba-11ed-bcc0-01c79f2670f3) \n[CIFv3 IPs](/app/dashboards#/view/ti_cif3-aedada10-0ab5-11ed-bcc0-01c79f2670f3) \n[CIFv3 URLs](/app/dashboards#/view/ti_cif3-fef149c0-0a2f-11ed-bcc0-01c79f2670f3) \n\n[Integrations Page](/app/integrations/detail/ti_cif3/overview)\n\n\n**Overview**\n\nThis dashboard is a health overview related to the Collective Intelligence Framework v3 integration.\n\nThe dashboard is made to provide general statistics and show the health of the ingestion of indicators from a CIFv3 instance. \n\nThe ingestion rates (by default it fetches new updates every 60 minutes) and provides a few filters for drilling down to specific indicator types retrieved from the CIFv3 instance.", + "openLinksInNewTab": false + }, + "title": "Overview Textbox [CIFv3]", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 31, + "i": "555e9e6c-04e9-4022-b6df-bda07dde30c4", + "w": 7, + "x": 0, + "y": 0 + }, + "panelIndex": "555e9e6c-04e9-4022-b6df-bda07dde30c4", + "title": "Overview Textbox [Logs CIFv3]", + "type": "visualization", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "event.dataset", + "negate": false, + "params": [ + "ti_cif3.feed" + ], + "type": "phrases" + }, + "query": { + "bool": { + "minimum_should_match": 1, + "should": [ + { + "match_phrase": { + "event.dataset": "ti_cif3.feed" + } + } + ] + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "key": "event.kind", + "negate": false, + "params": { + "query": "enrichment" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.kind": "enrichment" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "controls": [ + { + "fieldName": "threat.indicator.provider", + "id": "1635779603363", + "indexPatternRefName": "control_e971fedd-6afd-4d03-93ac-d0c751acc254_0_index_pattern", + "label": "Indicator Provider", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + }, + { + "fieldName": "threat.indicator.type", + "id": "1635779625911", + "indexPatternRefName": "control_e971fedd-6afd-4d03-93ac-d0c751acc254_1_index_pattern", + "label": "Indicator Type", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + }, + { + "fieldName": "tags", + "id": "1658691004225", + "indexPatternRefName": "control_e971fedd-6afd-4d03-93ac-d0c751acc254_2_index_pattern", + "label": "Indicator Tag", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": false, + "useTimeFilter": false + }, + "title": "Feed and Indicator Selector [CIFv3]", + "type": "input_control_vis", + "uiState": {} + } + }, + "gridData": { + "h": 6, + "i": "e971fedd-6afd-4d03-93ac-d0c751acc254", + "w": 41, + "x": 7, + "y": 0 + }, + "panelIndex": "e971fedd-6afd-4d03-93ac-d0c751acc254", + "title": "Feed and Indicator Selector [Logs CIFv3]", + "type": "visualization", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-c1cee622-e3dd-4d6b-a28a-0fb19dc2c7b7", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "c1cee622-e3dd-4d6b-a28a-0fb19dc2c7b7": { + "columnOrder": [ + "4d7ca99c-8a53-4a7f-96db-409251c0e391", + "b7f07f7c-1477-4f83-95f5-ad5cdc3a314b", + "0726d151-9edf-41cb-ab52-473ab27cf8b7" + ], + "columns": { + "0726d151-9edf-41cb-ab52-473ab27cf8b7": { + "dataType": "number", + "isBucketed": false, + "label": "Records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "4d7ca99c-8a53-4a7f-96db-409251c0e391": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of event.dataset", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "0726d151-9edf-41cb-ab52-473ab27cf8b7", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "size": 3 + }, + "scale": "ordinal", + "sourceField": "event.dataset" + }, + "b7f07f7c-1477-4f83-95f5-ad5cdc3a314b": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "includeEmptyRows": true, + "interval": "30s" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Zero", + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "0726d151-9edf-41cb-ab52-473ab27cf8b7" + ], + "layerId": "c1cee622-e3dd-4d6b-a28a-0fb19dc2c7b7", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "4d7ca99c-8a53-4a7f-96db-409251c0e391", + "xAccessor": "b7f07f7c-1477-4f83-95f5-ad5cdc3a314b" + } + ], + "legend": { + "isInside": false, + "isVisible": true, + "legendSize": "auto", + "position": "bottom", + "shouldTruncate": false, + "showSingleSeries": true + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide", + "valuesInLegend": false, + "xTitle": "Date", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + }, + "yTitle": "Total Indicators" + } + }, + "title": "Indicators ingested per Datastream [Logs CIFv3]", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 16, + "i": "aab4fac0-d39c-4521-aa9b-0a49d5938e9e", + "w": 29, + "x": 7, + "y": 6 + }, + "panelIndex": "aab4fac0-d39c-4521-aa9b-0a49d5938e9e", + "title": "Indicators ingested per Datastream [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-2c2ce8ee-a793-4242-aad4-06f3a8707b02", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "2c2ce8ee-a793-4242-aad4-06f3a8707b02": { + "columnOrder": [ + "1d9b6fbf-58e3-427e-a453-edec40466320", + "b6cbd44b-6f5d-4e1e-b1ab-0c09b3a67111" + ], + "columns": { + "1d9b6fbf-58e3-427e-a453-edec40466320": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of threat.indicator.type", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "b6cbd44b-6f5d-4e1e-b1ab-0c09b3a67111", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.type" + }, + "b6cbd44b-6f5d-4e1e-b1ab-0c09b3a67111": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "groups": [ + "1d9b6fbf-58e3-427e-a453-edec40466320" + ], + "layerId": "2c2ce8ee-a793-4242-aad4-06f3a8707b02", + "layerType": "data", + "legendDisplay": "default", + "metric": "b6cbd44b-6f5d-4e1e-b1ab-0c09b3a67111", + "nestedLegend": false, + "numberDisplay": "percent" + } + ], + "shape": "donut" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsPie" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "c446ea70-8a63-418e-8997-e43a5f7c5b5d", + "w": 12, + "x": 36, + "y": 6 + }, + "panelIndex": "c446ea70-8a63-418e-8997-e43a5f7c5b5d", + "title": "Total Percentage by Type [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-070f5dbc-7687-4e97-9a57-5542b401c13f", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "070f5dbc-7687-4e97-9a57-5542b401c13f": { + "columnOrder": [ + "1e352b49-3b83-44a6-98fe-8703d30f2517" + ], + "columns": { + "1e352b49-3b83-44a6-98fe-8703d30f2517": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Indicators", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "1e352b49-3b83-44a6-98fe-8703d30f2517", + "layerId": "070f5dbc-7687-4e97-9a57-5542b401c13f", + "layerType": "data", + "size": "xl", + "textAlign": "center", + "titlePosition": "bottom" + } + }, + "title": "Total Indicators [Logs CIFv3]", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "d37eb797-f273-43c2-9004-b947891cce55", + "w": 6, + "x": 36, + "y": 14 + }, + "panelIndex": "d37eb797-f273-43c2-9004-b947891cce55", + "title": "Total Indicators [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-df8e3a91-700b-428a-a763-525076e4d3c8", + "type": "index-pattern" + } + ], + "sharingSavedObjectProps": { + "outcome": "exactMatch", + "sourceId": "ti_cif3-49830790-3b27-11ec-ae50-2fdf1e96c6a6" + }, + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "df8e3a91-700b-428a-a763-525076e4d3c8": { + "columnOrder": [ + "e4f78e2f-f0a7-4cc6-96d0-af607ffbf326" + ], + "columns": { + "e4f78e2f-f0a7-4cc6-96d0-af607ffbf326": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Datastreams", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "event.dataset" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "e4f78e2f-f0a7-4cc6-96d0-af607ffbf326", + "layerId": "df8e3a91-700b-428a-a763-525076e4d3c8", + "layerType": "data", + "size": "xl", + "textAlign": "center", + "titlePosition": "bottom" + } + }, + "title": "Total Datastreams [Logs CIFv3]", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "6509dcc9-bb9c-4c1f-80e9-612f67ada340", + "w": 6, + "x": 42, + "y": 14 + }, + "panelIndex": "6509dcc9-bb9c-4c1f-80e9-612f67ada340", + "title": "Total Datastreams [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "[Logs CIFv3] Overview", + "version": 1 + }, + "coreMigrationVersion": "8.0.0", + "id": "ti_cif3-b4d9d9b0-0a2f-11ed-bcc0-01c79f2670f3", + "migrationVersion": { + "dashboard": "8.0.0" + }, + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "e971fedd-6afd-4d03-93ac-d0c751acc254:kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "e971fedd-6afd-4d03-93ac-d0c751acc254:kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "e971fedd-6afd-4d03-93ac-d0c751acc254:control_e971fedd-6afd-4d03-93ac-d0c751acc254_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "e971fedd-6afd-4d03-93ac-d0c751acc254:control_e971fedd-6afd-4d03-93ac-d0c751acc254_1_index_pattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "e971fedd-6afd-4d03-93ac-d0c751acc254:control_e971fedd-6afd-4d03-93ac-d0c751acc254_2_index_pattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "aab4fac0-d39c-4521-aa9b-0a49d5938e9e:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "aab4fac0-d39c-4521-aa9b-0a49d5938e9e:indexpattern-datasource-layer-c1cee622-e3dd-4d6b-a28a-0fb19dc2c7b7", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "c446ea70-8a63-418e-8997-e43a5f7c5b5d:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "c446ea70-8a63-418e-8997-e43a5f7c5b5d:indexpattern-datasource-layer-2c2ce8ee-a793-4242-aad4-06f3a8707b02", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "d37eb797-f273-43c2-9004-b947891cce55:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "d37eb797-f273-43c2-9004-b947891cce55:indexpattern-datasource-layer-070f5dbc-7687-4e97-9a57-5542b401c13f", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "6509dcc9-bb9c-4c1f-80e9-612f67ada340:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "6509dcc9-bb9c-4c1f-80e9-612f67ada340:indexpattern-datasource-layer-df8e3a91-700b-428a-a763-525076e4d3c8", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "f654c447-12d2-41a4-9091-06169af11ba5:indexpattern-datasource-layer-682732d8-8691-4c5a-bf89-de8e30d71dfb", + "type": "index-pattern" + }, + { + "id": "ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "name": "tag-ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "type": "tag" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/ti_cif3/kibana/dashboard/ti_cif3-bda23600-0abb-11ed-bcc0-01c79f2670f3.json b/packages/ti_cif3/kibana/dashboard/ti_cif3-bda23600-0abb-11ed-bcc0-01c79f2670f3.json new file mode 100644 index 00000000000..3b5a8713c09 --- /dev/null +++ b/packages/ti_cif3/kibana/dashboard/ti_cif3-bda23600-0abb-11ed-bcc0-01c79f2670f3.json @@ -0,0 +1,406 @@ +{ + "attributes": { + "description": "Dashboard providing statistics about Email type indicators from the Collective Intelligence Framework v3 integration", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "threat.indicator.type", + "negate": false, + "params": { + "query": "email-addr" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "threat.indicator.type": "email-addr" + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "ti_cif3.feed" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "ti_cif3.feed" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "id": "", + "params": { + "fontSize": 12, + "markdown": "**Navigation**\n\n[CIFv3 Overview](/app/dashboards#/view/ti_cif3-b4d9d9b0-0a2f-11ed-bcc0-01c79f2670f3) \n**[CIFv3 Emails (This Page)](/app/dashboards#/view/ti_cif3-bda23600-0abb-11ed-bcc0-01c79f2670f3)** \n[CIFv3 Files](/app/dashboards#/view/ti_cif3-63a0e470-0a30-11ed-bcc0-01c79f2670f3) \n[CIFv3 FQDNs](/app/dashboards#/view/ti_cif3-6005a190-0aba-11ed-bcc0-01c79f2670f3) \n[CIFv3 IPs](/app/dashboards#/view/ti_cif3-aedada10-0ab5-11ed-bcc0-01c79f2670f3) \n[CIFv3 URLs](/app/dashboards#/view/ti_cif3-fef149c0-0a2f-11ed-bcc0-01c79f2670f3) \n\n[Integrations Page](/app/integrations/detail/ti_cif3/overview)\n\n\n**Overview**\n\nThis dashboard is an overview of the different threat intelligence indicators with a **threat.indicator.type: email-addr**. \n\nThe dashboard is made to provide general statistics and show the health of your indicators like popular domains, and statistics about how many unique indicators are ingested and other relevant information.", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 39, + "i": "4c3ed6e1-8b4e-4eab-8d84-70ed4f506216", + "w": 7, + "x": 0, + "y": 0 + }, + "panelIndex": "4c3ed6e1-8b4e-4eab-8d84-70ed4f506216", + "title": "Files Navigation Textbox [Logs CIFv3]", + "type": "visualization", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-cd81a60b-2661-48b3-a40f-ba8451e802a6", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "cd81a60b-2661-48b3-a40f-ba8451e802a6": { + "columnOrder": [ + "4f96463f-c5f9-448b-ab9e-7e17a2bd5969" + ], + "columns": { + "4f96463f-c5f9-448b-ab9e-7e17a2bd5969": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique Addresses", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.email.address" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "4f96463f-c5f9-448b-ab9e-7e17a2bd5969", + "layerId": "cd81a60b-2661-48b3-a40f-ba8451e802a6", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "3a6a2852-0fb8-45df-9a79-e7729691fe5f", + "w": 6, + "x": 7, + "y": 0 + }, + "panelIndex": "3a6a2852-0fb8-45df-9a79-e7729691fe5f", + "title": "Unique Addresses [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-c94400ee-a135-4a99-9693-5879d29f7aad", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "c94400ee-a135-4a99-9693-5879d29f7aad": { + "columnOrder": [ + "2934249f-fce5-4637-87ff-d2596d1b6ec5" + ], + "columns": { + "2934249f-fce5-4637-87ff-d2596d1b6ec5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique Domains", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.url.domain" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "2934249f-fce5-4637-87ff-d2596d1b6ec5", + "layerId": "c94400ee-a135-4a99-9693-5879d29f7aad", + "layerType": "data", + "size": "xl", + "textAlign": "center", + "titlePosition": "bottom" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "02f1732b-a981-4fba-8b27-b944f2f3c98c", + "w": 6, + "x": 13, + "y": 0 + }, + "panelIndex": "02f1732b-a981-4fba-8b27-b944f2f3c98c", + "title": "Unique Domains [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-db89074c-e1fe-4091-bdb1-e42a36e82bac", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "db89074c-e1fe-4091-bdb1-e42a36e82bac": { + "columnOrder": [ + "b284ea2a-a2cd-4d08-bf44-fc73c08b5694", + "7ca1ac0b-2060-4431-a4b9-ec470af4448c" + ], + "columns": { + "7ca1ac0b-2060-4431-a4b9-ec470af4448c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "b284ea2a-a2cd-4d08-bf44-fc73c08b5694": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Domains", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "7ca1ac0b-2060-4431-a4b9-ec470af4448c", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.url.domain" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "7ca1ac0b-2060-4431-a4b9-ec470af4448c", + "isTransposed": false + }, + { + "columnId": "b284ea2a-a2cd-4d08-bf44-fc73c08b5694", + "isTransposed": false + } + ], + "layerId": "db89074c-e1fe-4091-bdb1-e42a36e82bac", + "layerType": "data", + "rowHeight": "single", + "rowHeightLines": 1 + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 16, + "i": "8994501a-1550-4cf2-857f-d6b6491ffb62", + "w": 18, + "x": 19, + "y": 0 + }, + "panelIndex": "8994501a-1550-4cf2-857f-d6b6491ffb62", + "title": "Most Popular Domains [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "[Logs CIFv3] Emails", + "version": 1 + }, + "coreMigrationVersion": "8.0.0", + "id": "ti_cif3-bda23600-0abb-11ed-bcc0-01c79f2670f3", + "migrationVersion": { + "dashboard": "8.0.0" + }, + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "3a6a2852-0fb8-45df-9a79-e7729691fe5f:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "3a6a2852-0fb8-45df-9a79-e7729691fe5f:indexpattern-datasource-layer-cd81a60b-2661-48b3-a40f-ba8451e802a6", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "02f1732b-a981-4fba-8b27-b944f2f3c98c:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "02f1732b-a981-4fba-8b27-b944f2f3c98c:indexpattern-datasource-layer-c94400ee-a135-4a99-9693-5879d29f7aad", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "8994501a-1550-4cf2-857f-d6b6491ffb62:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "8994501a-1550-4cf2-857f-d6b6491ffb62:indexpattern-datasource-layer-db89074c-e1fe-4091-bdb1-e42a36e82bac", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "c7c6e8dc-b649-434c-9650-8a1564d4d676:indexpattern-datasource-layer-88a112e1-6da1-49d3-9177-19f98280c200", + "type": "index-pattern" + }, + { + "id": "ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "name": "tag-ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "type": "tag" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/ti_cif3/kibana/dashboard/ti_cif3-fef149c0-0a2f-11ed-bcc0-01c79f2670f3.json b/packages/ti_cif3/kibana/dashboard/ti_cif3-fef149c0-0a2f-11ed-bcc0-01c79f2670f3.json new file mode 100644 index 00000000000..31b5359b56b --- /dev/null +++ b/packages/ti_cif3/kibana/dashboard/ti_cif3-fef149c0-0a2f-11ed-bcc0-01c79f2670f3.json @@ -0,0 +1,710 @@ +{ + "attributes": { + "description": "Dashboard providing statistics about URL type indicators from the Collective Intelligence Framework v3 integration", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "threat.indicator.type", + "negate": false, + "params": { + "query": "url" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "threat.indicator.type": "url" + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "ti_cif3.feed" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "ti_cif3.feed" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "id": "", + "params": { + "fontSize": 12, + "markdown": "**Navigation**\n\n[CIFv3 Overview](/app/dashboards#/view/ti_cif3-b4d9d9b0-0a2f-11ed-bcc0-01c79f2670f3) \n[CIFv3 Emails](/app/dashboards#/view/ti_cif3-bda23600-0abb-11ed-bcc0-01c79f2670f3) \n[CIFv3 Files](/app/dashboards#/view/ti_cif3-63a0e470-0a30-11ed-bcc0-01c79f2670f3) \n[CIFv3 FQDNs](/app/dashboards#/view/ti_cif3-6005a190-0aba-11ed-bcc0-01c79f2670f3) \n[CIFv3 IPs](/app/dashboards#/view/ti_cif3-aedada10-0ab5-11ed-bcc0-01c79f2670f3) \n**[CIFv3 URLs (This Page)](/app/dashboards#/view/ti_cif3-fef149c0-0a2f-11ed-bcc0-01c79f2670f3)** \n\n[Integrations Page](/app/integrations/detail/ti_cif3/overview)\n\n\n**Overview**\n\nThis dashboard is an overview of the different threat intelligence indicators with a **threat.indicator.type: url**. \n\nThe dashboard is made to provide general statistics and show the health of your indicators like popular domains, file extensions, statistics about how many unique indicators are ingested and other relevant information.", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 39, + "i": "4c3ed6e1-8b4e-4eab-8d84-70ed4f506216", + "w": 7, + "x": 0, + "y": 0 + }, + "panelIndex": "4c3ed6e1-8b4e-4eab-8d84-70ed4f506216", + "title": "Files Navigation Textbox [Logs CIFv3]", + "type": "visualization", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-88a112e1-6da1-49d3-9177-19f98280c200", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "88a112e1-6da1-49d3-9177-19f98280c200": { + "columnOrder": [ + "604f1693-15a6-437d-af69-03588db8e471" + ], + "columns": { + "604f1693-15a6-437d-af69-03588db8e471": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique Ports", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.url.port" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "604f1693-15a6-437d-af69-03588db8e471", + "layerId": "88a112e1-6da1-49d3-9177-19f98280c200", + "layerType": "data", + "size": "xl", + "textAlign": "center", + "titlePosition": "bottom" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "c7c6e8dc-b649-434c-9650-8a1564d4d676", + "w": 6, + "x": 7, + "y": 0 + }, + "panelIndex": "c7c6e8dc-b649-434c-9650-8a1564d4d676", + "title": "Unique Ports [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-a6fa56f8-32fa-405d-8771-dade4fe75d62", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "a6fa56f8-32fa-405d-8771-dade4fe75d62": { + "columnOrder": [ + "848c463b-bbc1-4b6a-af3e-76d844eb3cc5" + ], + "columns": { + "848c463b-bbc1-4b6a-af3e-76d844eb3cc5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique Extensions", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.url.extension" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "848c463b-bbc1-4b6a-af3e-76d844eb3cc5", + "layerId": "a6fa56f8-32fa-405d-8771-dade4fe75d62", + "layerType": "data", + "size": "xl", + "textAlign": "center", + "titlePosition": "bottom" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "73a752f9-bde5-4396-8ede-e9e77a37182d", + "w": 6, + "x": 13, + "y": 0 + }, + "panelIndex": "73a752f9-bde5-4396-8ede-e9e77a37182d", + "title": "Unique File Extensions [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-c94400ee-a135-4a99-9693-5879d29f7aad", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "c94400ee-a135-4a99-9693-5879d29f7aad": { + "columnOrder": [ + "2934249f-fce5-4637-87ff-d2596d1b6ec5" + ], + "columns": { + "2934249f-fce5-4637-87ff-d2596d1b6ec5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique Domains", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "threat.indicator.url.domain" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "2934249f-fce5-4637-87ff-d2596d1b6ec5", + "layerId": "c94400ee-a135-4a99-9693-5879d29f7aad", + "layerType": "data", + "size": "xl", + "textAlign": "center", + "titlePosition": "bottom" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "02f1732b-a981-4fba-8b27-b944f2f3c98c", + "w": 6, + "x": 19, + "y": 0 + }, + "panelIndex": "02f1732b-a981-4fba-8b27-b944f2f3c98c", + "title": "Unique Domains [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-0f63318a-a857-4d83-89ce-a94e2242b79e", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "0f63318a-a857-4d83-89ce-a94e2242b79e": { + "columnOrder": [ + "df0791a6-247c-4434-a43a-fdea7577ca34", + "77a48096-02aa-4b7a-8a7b-131fc38988bd" + ], + "columns": { + "77a48096-02aa-4b7a-8a7b-131fc38988bd": { + "dataType": "number", + "isBucketed": false, + "label": "Records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "df0791a6-247c-4434-a43a-fdea7577ca34": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of threat.indicator.url.scheme", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "77a48096-02aa-4b7a-8a7b-131fc38988bd", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.url.scheme" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "groups": [ + "df0791a6-247c-4434-a43a-fdea7577ca34" + ], + "layerId": "0f63318a-a857-4d83-89ce-a94e2242b79e", + "layerType": "data", + "legendDisplay": "show", + "legendSize": "auto", + "metric": "77a48096-02aa-4b7a-8a7b-131fc38988bd", + "nestedLegend": false, + "numberDisplay": "percent" + } + ], + "shape": "donut" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsPie" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "ab7ab31c-e76f-4613-b17d-fdd909f17e0d", + "w": 10, + "x": 25, + "y": 0 + }, + "panelIndex": "ab7ab31c-e76f-4613-b17d-fdd909f17e0d", + "title": "Percentage of URL Schema used [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-db89074c-e1fe-4091-bdb1-e42a36e82bac", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "db89074c-e1fe-4091-bdb1-e42a36e82bac": { + "columnOrder": [ + "b284ea2a-a2cd-4d08-bf44-fc73c08b5694", + "7ca1ac0b-2060-4431-a4b9-ec470af4448c" + ], + "columns": { + "7ca1ac0b-2060-4431-a4b9-ec470af4448c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "b284ea2a-a2cd-4d08-bf44-fc73c08b5694": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Domains", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "7ca1ac0b-2060-4431-a4b9-ec470af4448c", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.url.domain" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "7ca1ac0b-2060-4431-a4b9-ec470af4448c", + "isTransposed": false + }, + { + "columnId": "b284ea2a-a2cd-4d08-bf44-fc73c08b5694", + "isTransposed": false + } + ], + "layerId": "db89074c-e1fe-4091-bdb1-e42a36e82bac", + "layerType": "data", + "rowHeight": "single", + "rowHeightLines": 1 + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 16, + "i": "8994501a-1550-4cf2-857f-d6b6491ffb62", + "w": 13, + "x": 35, + "y": 0 + }, + "panelIndex": "8994501a-1550-4cf2-857f-d6b6491ffb62", + "title": "Most Popular Domains [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-dfaa5b71-ed27-4602-9dbe-d263fd33aa05", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "dfaa5b71-ed27-4602-9dbe-d263fd33aa05": { + "columnOrder": [ + "c00d8a88-7047-4fa4-b99f-7e8be1370b6f", + "14f7e661-8382-4e25-a998-10c6c576255e" + ], + "columns": { + "14f7e661-8382-4e25-a998-10c6c576255e": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "scale": "ratio", + "sourceField": "Records" + }, + "c00d8a88-7047-4fa4-b99f-7e8be1370b6f": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of threat.indicator.url.extension", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "14f7e661-8382-4e25-a998-10c6c576255e", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "threat.indicator.url.extension" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "groups": [ + "c00d8a88-7047-4fa4-b99f-7e8be1370b6f" + ], + "layerId": "dfaa5b71-ed27-4602-9dbe-d263fd33aa05", + "layerType": "data", + "legendDisplay": "default", + "metric": "14f7e661-8382-4e25-a998-10c6c576255e", + "nestedLegend": false, + "numberDisplay": "percent" + } + ], + "shape": "treemap" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsPie" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "353bb92f-8375-4dc6-b961-9ed7f7509627", + "w": 28, + "x": 7, + "y": 8 + }, + "panelIndex": "353bb92f-8375-4dc6-b961-9ed7f7509627", + "title": "Most Popular File Extensions [Logs CIFv3]", + "type": "lens", + "version": "8.0.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "[Logs CIFv3] URLs", + "version": 1 + }, + "coreMigrationVersion": "8.0.0", + "id": "ti_cif3-fef149c0-0a2f-11ed-bcc0-01c79f2670f3", + "migrationVersion": { + "dashboard": "8.0.0" + }, + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "c7c6e8dc-b649-434c-9650-8a1564d4d676:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "c7c6e8dc-b649-434c-9650-8a1564d4d676:indexpattern-datasource-layer-88a112e1-6da1-49d3-9177-19f98280c200", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "73a752f9-bde5-4396-8ede-e9e77a37182d:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "73a752f9-bde5-4396-8ede-e9e77a37182d:indexpattern-datasource-layer-a6fa56f8-32fa-405d-8771-dade4fe75d62", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "02f1732b-a981-4fba-8b27-b944f2f3c98c:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "02f1732b-a981-4fba-8b27-b944f2f3c98c:indexpattern-datasource-layer-c94400ee-a135-4a99-9693-5879d29f7aad", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "ab7ab31c-e76f-4613-b17d-fdd909f17e0d:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "ab7ab31c-e76f-4613-b17d-fdd909f17e0d:indexpattern-datasource-layer-0f63318a-a857-4d83-89ce-a94e2242b79e", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "8994501a-1550-4cf2-857f-d6b6491ffb62:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "8994501a-1550-4cf2-857f-d6b6491ffb62:indexpattern-datasource-layer-db89074c-e1fe-4091-bdb1-e42a36e82bac", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "353bb92f-8375-4dc6-b961-9ed7f7509627:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "353bb92f-8375-4dc6-b961-9ed7f7509627:indexpattern-datasource-layer-dfaa5b71-ed27-4602-9dbe-d263fd33aa05", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "fda93ed1-72f0-4489-80b7-9e69d14f30aa:indexpattern-datasource-layer-9fa49c4c-5544-472d-afce-e51d6a5687fe", + "type": "index-pattern" + }, + { + "id": "ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "name": "tag-ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "type": "tag" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/ti_cif3/kibana/tag/ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1.json.json b/packages/ti_cif3/kibana/tag/ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1.json.json new file mode 100644 index 00000000000..5d464afed90 --- /dev/null +++ b/packages/ti_cif3/kibana/tag/ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1.json.json @@ -0,0 +1,14 @@ +{ + "attributes": { + "color": "#01426A", + "description": "", + "name": "CIFv3" + }, + "coreMigrationVersion": "8.0.0", + "id": "ti_cif3-ec8c3e30-0c59-11ed-9b65-435777f1d8a1", + "migrationVersion": { + "tag": "8.0.0" + }, + "references": [], + "type": "tag" +} \ No newline at end of file diff --git a/packages/ti_cif3/manifest.yml b/packages/ti_cif3/manifest.yml new file mode 100644 index 00000000000..95813d8d37f --- /dev/null +++ b/packages/ti_cif3/manifest.yml @@ -0,0 +1,43 @@ +format_version: 1.0.0 +name: ti_cif3 +title: "Collective Intelligence Framework v3" +version: 0.1.0 +release: beta +license: basic +description: "Ingest threat indicators from a Collective Intelligence Framework v3 instance with Elastic Agent." +type: integration +categories: + - security +conditions: + kibana.version: "^8.0.0" +icons: + - src: /img/csg_logo_big.svg + title: csirtgadgets logo + size: 1047x748 + type: image/svg+xml +policy_templates: + - name: ti_cif3 + title: Collective Intelligence Framework v3 + description: Ingest threat indicators from a Collective Intelligence Framework v3 instance with Elastic Agent. + inputs: + - type: httpjson + title: Collect threat indicators via API + description: Ingest threat indicators from a Collective Intelligence Framework v3 instance with Elastic Agent. + vars: + - name: url + type: url + title: CIFv3 API base URL + multi: false + required: true + show_user: true + description: "Base URL for CIFv3 instance, e.g.: https://cif.yourdomain.tld" + + - name: api_token + type: password + title: API Token + multi: false + required: true + show_user: true + description: The CIFv3 API read token +owner: + github: elastic/security-external-integrations From 7a6d91667d4743930dea6318743260860c7bd99d Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 08:25:02 -0500 Subject: [PATCH 02/17] Update packages/ti_cif3/changelog.yml updated real PR link Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- packages/ti_cif3/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_cif3/changelog.yml b/packages/ti_cif3/changelog.yml index 0da913124e3..0c45ad26395 100644 --- a/packages/ti_cif3/changelog.yml +++ b/packages/ti_cif3/changelog.yml @@ -3,4 +3,4 @@ changes: - description: Initial draft of the package type: enhancement - link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link + link: https://github.com/elastic/integrations/pull/3839 From 5c12a46b3f5c24cdd7fc244937e5dc7b7600f7ae Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 08:29:21 -0500 Subject: [PATCH 03/17] update ecs reference --- packages/ti_cif3/_dev/build/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_cif3/_dev/build/build.yml b/packages/ti_cif3/_dev/build/build.yml index 5661d603a89..8d9e4bf7ac8 100644 --- a/packages/ti_cif3/_dev/build/build.yml +++ b/packages/ti_cif3/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@v8.3.0 + reference: git@v8.4.0 From d6c4fcb083966e5246e54200252c1fc995b0e40a Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 08:34:18 -0500 Subject: [PATCH 04/17] Update link used for REST API docs --- packages/ti_cif3/_dev/build/docs/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ti_cif3/_dev/build/docs/README.md b/packages/ti_cif3/_dev/build/docs/README.md index ae4dc395d04..03c359d87f7 100644 --- a/packages/ti_cif3/_dev/build/docs/README.md +++ b/packages/ti_cif3/_dev/build/docs/README.md @@ -1,6 +1,6 @@ # Collective Intelligence Framework v3 Integration -This integration connects with the [REST API from the running CIFv3 instance](https://github.com/csirtgadgets/bearded-avenger/blob/master/cif/httpd/views/feed/__init__.py) to retrieve indicators. +This integration connects with the [REST API from the running CIFv3 instance](https://github.com/csirtgadgets/bearded-avenger-deploymentkit/wiki/REST-API) to retrieve indicators. ## Data Streams @@ -10,4 +10,4 @@ The CIFv3 integration collects threat indicators based on user-defined configura {{fields "feed"}} -{{event "feed"}} \ No newline at end of file +{{event "feed"}} From b0bf2eb04414fdc5cfd7bdfdbe4997f6c8d354c8 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 08:36:02 -0500 Subject: [PATCH 05/17] Update to ecs 8.4.0 --- .../test/pipeline/test-cif3-sample.ndjson.log-expected.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log-expected.json b/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log-expected.json index 15126e4c978..2e0b2ab3724 100644 --- a/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log-expected.json +++ b/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log-expected.json @@ -7,7 +7,7 @@ "uuid": "3fbdd654-b2b0-498c-8e20-ef87bce73672" }, "ecs": { - "version": "8.3.0" + "version": "8.4.0" }, "event": { "category": "threat", @@ -45,4 +45,4 @@ } } ] -} \ No newline at end of file +} From 70b27c8f7cff30c22ff7706ec74ddaf35d68fc6c Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 08:36:52 -0500 Subject: [PATCH 06/17] Update to ecs 8.4.0 --- packages/ti_cif3/data_stream/feed/sample_event.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ti_cif3/data_stream/feed/sample_event.json b/packages/ti_cif3/data_stream/feed/sample_event.json index 9dfdd027cc3..b44e61828b8 100755 --- a/packages/ti_cif3/data_stream/feed/sample_event.json +++ b/packages/ti_cif3/data_stream/feed/sample_event.json @@ -18,7 +18,7 @@ "type": "logs" }, "ecs": { - "version": "8.3.0" + "version": "8.4.0" }, "elastic_agent": { "id": "f599fd51-b36d-45b4-a90f-4d63240b8477", @@ -84,4 +84,4 @@ "type": "ipv4-addr" } } -} \ No newline at end of file +} From 77099c570b144cc33481854b7eb8e6f9955c0a37 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 08:42:20 -0500 Subject: [PATCH 07/17] Add security-external-integrations for ti_cif3 pkg --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dd57826e92a..b130718c926 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -153,6 +153,7 @@ /packages/tenable_sc @elastic/security-external-integrations /packages/ti_abusech @elastic/security-external-integrations /packages/ti_anomali @elastic/security-external-integrations +/packages/ti_cif3 @elastic/security-external-integrations /packages/ti_cybersixgill @elastic/security-external-integrations /packages/ti_misp @elastic/security-external-integrations /packages/ti_otx @elastic/security-external-integrations From 4e6fe712f87e5c303e1b305d27a78b1d8a52ce6d Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 08:44:18 -0500 Subject: [PATCH 08/17] Remove unnecessary yml --- .../data_stream/feed/_dev/test/pipeline/test-common-config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-common-config.yml b/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-common-config.yml index 11e8ffa1d1e..4da22641654 100644 --- a/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-common-config.yml +++ b/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-common-config.yml @@ -1,5 +1,3 @@ fields: tags: - preserve_original_event -dynamic_fields: - event.ingested: "^.*$" \ No newline at end of file From 5cfc95bcf8a8150bad940e711492ab36024bed31 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 09:17:59 -0500 Subject: [PATCH 09/17] Update to ECS 8.4.0 --- .../data_stream/feed/elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml b/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml index 936385b4e07..bcdb72441ce 100644 --- a/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml @@ -6,7 +6,7 @@ processors: #################### - set: field: ecs.version - value: "8.3.0" + value: "8.4.0" - set: field: event.kind value: enrichment From 478bb34d4c2063800391bb24d3f7bd53ce219298 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 09:21:55 -0500 Subject: [PATCH 10/17] Update URL grok pattern Add support for IPV6 addresses with ports by looking for surrounding brackets, e.g., `http://[2001:d34d:b33f:0000:0000:0000:d34d:b33f]:8080` --- .../data_stream/feed/elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml b/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml index bcdb72441ce..b8315d1eb97 100644 --- a/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml @@ -191,7 +191,7 @@ processors: - grok: field: cif3.indicator patterns: - - "%{URIPROTO:threat.indicator.url.scheme}://(?:%{IP:threat.indicator.ip}|%{HOSTNAME:threat.indicator.url.domain})(?::%{POSINT:threat.indicator.url.port})?(?:%{URIPATH:threat.indicator.url.path})?.*" + - "%{URIPROTO:threat.indicator.url.scheme}://(?:%{IPV4:threat.indicator.ip}|\[?%{IPV6:threat.indicator.ip}\]?|%{HOSTNAME:threat.indicator.url.domain})(?::%{POSINT:threat.indicator.url.port})?(?:%{URIPATH:threat.indicator.url.path})?.*" ignore_failure: true if: "ctx.cif3?.itype == 'url'" From 696c2a05c24d0868dd5e6e41207dae7961ec0123 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 09:34:29 -0500 Subject: [PATCH 11/17] Document `confidence` conversion --- packages/ti_cif3/_dev/build/docs/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/ti_cif3/_dev/build/docs/README.md b/packages/ti_cif3/_dev/build/docs/README.md index 03c359d87f7..bbb1141b57b 100644 --- a/packages/ti_cif3/_dev/build/docs/README.md +++ b/packages/ti_cif3/_dev/build/docs/README.md @@ -8,6 +8,15 @@ This integration connects with the [REST API from the running CIFv3 instance](ht The CIFv3 integration collects threat indicators based on user-defined configuration including a polling interval, how far back in time it should look, and other filters like indicator type and tags. +CIFv3 `confidence` field values (0..10) are converted to ECS confidence (None, Low, Medium, High) in the following way: + +| CIFv3 Confidence | ECS Conversion | +| ---------------- | -------------- | +| Beyond Range | None | +| 0 - <3 | Low | +| 3 - <7 | Medium | +| 7 - 10 | High | + {{fields "feed"}} {{event "feed"}} From c66d43bc9bd6f242e1c89792a2236dac0bfe4eb6 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 10:02:08 -0500 Subject: [PATCH 12/17] Update logo to fix missing G --- packages/ti_cif3/img/csg_logo_big.svg | 291 ++++++++++++++++++++++++-- 1 file changed, 269 insertions(+), 22 deletions(-) diff --git a/packages/ti_cif3/img/csg_logo_big.svg b/packages/ti_cif3/img/csg_logo_big.svg index a794457dc29..5ee2369a853 100755 --- a/packages/ti_cif3/img/csg_logo_big.svg +++ b/packages/ti_cif3/img/csg_logo_big.svg @@ -1,23 +1,270 @@ - - - - - - - - + + + + From 57a110a5fa2338f4e89c3bb336b9a6f3925eeb92 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 19:21:05 -0500 Subject: [PATCH 13/17] Rebuild README from new template --- packages/ti_cif3/docs/README.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/ti_cif3/docs/README.md b/packages/ti_cif3/docs/README.md index e5e3ebfcc23..8a970c31d3c 100644 --- a/packages/ti_cif3/docs/README.md +++ b/packages/ti_cif3/docs/README.md @@ -1,6 +1,6 @@ # Collective Intelligence Framework v3 Integration -This integration connects with the [REST API from the running CIFv3 instance](https://github.com/csirtgadgets/bearded-avenger/blob/master/cif/httpd/views/feed/__init__.py) to retrieve indicators. +This integration connects with the [REST API from the running CIFv3 instance](https://github.com/csirtgadgets/bearded-avenger-deploymentkit/wiki/REST-API) to retrieve indicators. ## Data Streams @@ -8,6 +8,15 @@ This integration connects with the [REST API from the running CIFv3 instance](ht The CIFv3 integration collects threat indicators based on user-defined configuration including a polling interval, how far back in time it should look, and other filters like indicator type and tags. +CIFv3 `confidence` field values (0..10) are converted to ECS confidence (None, Low, Medium, High) in the following way: + +| CIFv3 Confidence | ECS Conversion | +| ---------------- | -------------- | +| Beyond Range | None | +| 0 - <3 | Low | +| 3 - <7 | Medium | +| 7 - 10 | High | + **Exported fields** | Field | Description | Type | @@ -71,7 +80,7 @@ The CIFv3 integration collects threat indicators based on user-defined configura | threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | threat.indicator.as.organization.name | Organization name. | keyword | | threat.indicator.as.organization.name.text | Multi-field of `threat.indicator.as.organization.name`. | match_only_text | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. Expected values are: \* Not Specified \* None \* Low \* Medium \* High | keyword | +| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | | threat.indicator.description | Describes the type of action conducted by the threat. | keyword | | threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | | threat.indicator.file.hash.md5 | MD5 hash. | keyword | @@ -90,12 +99,12 @@ The CIFv3 integration collects threat indicators based on user-defined configura | threat.indicator.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | | threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | | threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. Recommended values are: \* WHITE \* GREEN \* AMBER \* RED | keyword | +| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | | threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | | threat.indicator.provider | The name of the indicator's provider. | keyword | | threat.indicator.reference | Reference URL linking to additional information about this indicator. | keyword | | threat.indicator.sightings | Number of times this indicator was observed conducting threat activity. | long | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. Recommended values: \* autonomous-system \* artifact \* directory \* domain-name \* email-addr \* file \* ipv4-addr \* ipv6-addr \* mac-addr \* mutex \* port \* process \* software \* url \* user-account \* windows-registry-key \* x509-certificate | keyword | +| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | | threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | | threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | | threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | @@ -132,7 +141,7 @@ An example event for `feed` looks as following: "type": "logs" }, "ecs": { - "version": "8.3.0" + "version": "8.4.0" }, "elastic_agent": { "id": "f599fd51-b36d-45b4-a90f-4d63240b8477", @@ -199,4 +208,4 @@ An example event for `feed` looks as following: } } } -``` \ No newline at end of file +``` From 7944a065f189d72e7bf2f6b30c653deb77ce5052 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 6 Sep 2022 21:26:52 -0500 Subject: [PATCH 14/17] Update grok pattern with additional escaping Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- .../data_stream/feed/elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml b/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml index b8315d1eb97..2ac0cd9cd58 100644 --- a/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml @@ -191,7 +191,7 @@ processors: - grok: field: cif3.indicator patterns: - - "%{URIPROTO:threat.indicator.url.scheme}://(?:%{IPV4:threat.indicator.ip}|\[?%{IPV6:threat.indicator.ip}\]?|%{HOSTNAME:threat.indicator.url.domain})(?::%{POSINT:threat.indicator.url.port})?(?:%{URIPATH:threat.indicator.url.path})?.*" + - "%{URIPROTO:threat.indicator.url.scheme}://(?:%{IPV4:threat.indicator.ip}|\\[?%{IPV6:threat.indicator.ip}\\]?|%{HOSTNAME:threat.indicator.url.domain})(?::%{POSINT:threat.indicator.url.port})?(?:%{URIPATH:threat.indicator.url.path})?.*" ignore_failure: true if: "ctx.cif3?.itype == 'url'" From d81c94694acf2fa18d9820525e040844932f6332 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Wed, 7 Sep 2022 09:02:03 -0500 Subject: [PATCH 15/17] rename test files --- .../{test-cif3-sample.ndjson.log => test-cif3-sample-ndjson.log} | 0 ...og-expected.json => test-cif3-sample-ndjson.log-expected.json} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename packages/ti_cif3/data_stream/feed/_dev/test/pipeline/{test-cif3-sample.ndjson.log => test-cif3-sample-ndjson.log} (100%) rename packages/ti_cif3/data_stream/feed/_dev/test/pipeline/{test-cif3-sample.ndjson.log-expected.json => test-cif3-sample-ndjson.log-expected.json} (100%) diff --git a/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log b/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample-ndjson.log similarity index 100% rename from packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log rename to packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample-ndjson.log diff --git a/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log-expected.json b/packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample-ndjson.log-expected.json similarity index 100% rename from packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample.ndjson.log-expected.json rename to packages/ti_cif3/data_stream/feed/_dev/test/pipeline/test-cif3-sample-ndjson.log-expected.json From b6e54db63ad55ae8841547f2c1bb8587a99274d2 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Wed, 14 Sep 2022 18:02:09 -0500 Subject: [PATCH 16/17] Explicit definition for ja3 under `threat.indicator` --- .../feed/elasticsearch/ingest_pipeline/default.yml | 2 +- packages/ti_cif3/data_stream/feed/fields/ecs.yml | 6 ++++-- packages/ti_cif3/docs/README.md | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml b/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml index 2ac0cd9cd58..710037d1b14 100644 --- a/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml @@ -72,7 +72,7 @@ processors: if: "['md5', 'sha1', 'sha256', 'sha512', 'ssdeep'].contains(ctx.cif3?.itype) && !ctx.cif3?.tags.contains('ja3')" - rename: field: cif3.indicator - target_field: tls.client.ja3 + target_field: threat.indicator.tls.client.ja3 ignore_missing: true if: "ctx.cif3?.itype == 'md5' && ctx.cif3?.tags.contains('ja3')" - rename: diff --git a/packages/ti_cif3/data_stream/feed/fields/ecs.yml b/packages/ti_cif3/data_stream/feed/fields/ecs.yml index c7cdae942bd..432dad19afd 100644 --- a/packages/ti_cif3/data_stream/feed/fields/ecs.yml +++ b/packages/ti_cif3/data_stream/feed/fields/ecs.yml @@ -58,8 +58,10 @@ name: threat.indicator.file.pe.imphash - external: ecs name: threat.indicator.file.hash.ssdeep -- external: ecs - name: tls.client.ja3 +- name: threat.indicator.tls.client.ja3 + level: core + type: keyword + description: An md5 hash that identifies clients based on their TLS handshake. - external: ecs name: threat.indicator.email.address - external: ecs diff --git a/packages/ti_cif3/docs/README.md b/packages/ti_cif3/docs/README.md index 8a970c31d3c..88c66c8a26e 100644 --- a/packages/ti_cif3/docs/README.md +++ b/packages/ti_cif3/docs/README.md @@ -104,6 +104,7 @@ CIFv3 `confidence` field values (0..10) are converted to ECS confidence (None, L | threat.indicator.provider | The name of the indicator's provider. | keyword | | threat.indicator.reference | Reference URL linking to additional information about this indicator. | keyword | | threat.indicator.sightings | Number of times this indicator was observed conducting threat activity. | long | +| threat.indicator.tls.client.ja3 | An md5 hash that identifies clients based on their TLS handshake. | keyword | | threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | | threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | | threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | @@ -115,7 +116,6 @@ CIFv3 `confidence` field values (0..10) are converted to ECS confidence (None, L | threat.indicator.url.port | Port of the request, such as 443. | long | | threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | | threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| tls.client.ja3 | A hash that identifies clients based on how they perform an SSL/TLS handshake. | keyword | An example event for `feed` looks as following: From b557362f5023d6e2126e619fa1cf8c4d2ddbb184 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Wed, 14 Sep 2022 20:21:03 -0500 Subject: [PATCH 17/17] Adjust ja3 field from core to extended --- packages/ti_cif3/data_stream/feed/fields/ecs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_cif3/data_stream/feed/fields/ecs.yml b/packages/ti_cif3/data_stream/feed/fields/ecs.yml index 432dad19afd..8b7c1f619d2 100644 --- a/packages/ti_cif3/data_stream/feed/fields/ecs.yml +++ b/packages/ti_cif3/data_stream/feed/fields/ecs.yml @@ -59,7 +59,7 @@ - external: ecs name: threat.indicator.file.hash.ssdeep - name: threat.indicator.tls.client.ja3 - level: core + level: extended type: keyword description: An md5 hash that identifies clients based on their TLS handshake. - external: ecs