elastic · efd6 · Sep 20, 2022 · Jul 26, 2022 · Sep 6, 2022 · Sep 6, 2022
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -153,6 +153,7 @@
 /packages/tenable_sc @elastic/security-external-integrations
 /packages/ti_abusech @elastic/security-external-integrations
 /packages/ti_anomali @elastic/security-external-integrations
+/packages/ti_cif3 @elastic/security-external-integrations
 /packages/ti_cybersixgill @elastic/security-external-integrations
 /packages/ti_misp @elastic/security-external-integrations
 /packages/ti_otx @elastic/security-external-integrations

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]
@@ -0,0 +1,22 @@
+# Collective Intelligence Framework v3 Integration
+
+This integration connects with the [REST API from the running CIFv3 instance](https://github.com/csirtgadgets/bearded-avenger-deploymentkit/wiki/REST-API) to retrieve indicators.
+
+## Data Streams
+
+### Feed
+
+The CIFv3 integration collects threat indicators based on user-defined configuration including a polling interval, how far back in time it should look, and other filters like indicator type and tags.
+
+CIFv3 `confidence` field values (0..10) are converted to ECS confidence (None, Low, Medium, High) in the following way:
+
+| CIFv3 Confidence | ECS Conversion |
+| ---------------- | -------------- |
+| Beyond Range     | None           |
+| 0 - <3           | Low            |
+| 3 - <7           | Medium         |
+| 7 - 10           | High           |
+
+{{fields "feed"}}
+
+{{event "feed"}}
@@ -0,0 +1,14 @@
+version: "2.3"
+services:
+  cif3:
+    image: docker.elastic.co/observability/stream:v0.7.0
+    ports:
+      - 8080
+    volumes:
+      - ./files:/files:ro
+    environment:
+      PORT: 8080
+    command:
+      - http-server
+      - --addr=:8080
+      - --config=/files/config.yml
@@ -0,0 +1,161 @@
+rules:
+  - path: /feed
+    methods: ["GET"]
+    request_headers:
+      Authorization: "Token token=testing"
+    query_params:
+      itype: "ipv4"
+      confidence: "8"
+      tags: "botnet,exploit,malware,phishing"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {
+            "message": "success",
+            "data": [
+              {
+                "indicator": "20.206.75.106",
+                "itype": "ipv4",
+                "tlp": "white",
+                "provider": "sslbl.abuse.ch",
+                "group": [
+                  "everyone"
+                ],
+                "count": 1,
+                "tags": [
+                  "botnet"
+                ],
+                "confidence": 10,
+                "uuid": "ac240898-1443-4d7e-a98a-1daed220c162",
+                "cc": "br",
+                "latitude": -22.9035,
+                "timezone": "america/sao_paulo",
+                "longitude": -47.0565,
+                "city": "campinas",
+                "region": "sao paulo",
+                "location": [
+                  -47.0565,
+                  -22.9035
+                ],
+                "application": "https",
+                "reference": "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv",
+                "portlist": "443",
+                "protocol": "tcp",
+                "asn": 8075,
+                "asn_desc": "microsoft-corp-msn-as-block",
+                "firsttime": "2022-07-20T20:25:53.000000Z",
+                "reporttime": "2022-07-21T20:33:26.585967Z",
+                "lasttime": "2022-07-20T20:25:53.000000Z",
+                "indicator_ipv4": "20.206.75.106"
+              },
+              {
+                "indicator": "160.20.147.52",
+                "itype": "ipv4",
+                "tlp": "white",
+                "provider": "sslbl.abuse.ch",
+                "group": [
+                  "everyone"
+                ],
+                "count": 1,
+                "tags": [
+                  "botnet"
+                ],
+                "confidence": 10,
+                "uuid": "cb5e953d-f3f7-4a94-88f6-dc553fc30445",
+                "cc": "de",
+                "latitude": 50.1103,
+                "timezone": "europe/berlin",
+                "longitude": 8.7147,
+                "city": "frankfurt am main",
+                "region": "hesse",
+                "location": [
+                  8.7147,
+                  50.1103
+                ],
+                "application": "https",
+                "reference": "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv",
+                "portlist": "8848",
+                "protocol": "tcp",
+                "asn": 30823,
+                "asn_desc": "combahton gmbh",
+                "firsttime": "2022-07-20T20:00:30.000000Z",
+                "reporttime": "2022-07-21T09:32:44.946024Z",
+                "lasttime": "2022-07-20T20:00:30.000000Z",
+                "indicator_ipv4": "160.20.147.52"
+              },
+              {
+                "indicator": "207.32.218.12",
+                "itype": "ipv4",
+                "tlp": "white",
+                "provider": "sslbl.abuse.ch",
+                "group": [
+                  "everyone"
+                ],
+                "count": 1,
+                "tags": [
+                  "botnet"
+                ],
+                "confidence": 10,
+                "uuid": "e0596a59-1139-42d0-8c3a-4b505405602c",
+                "cc": "us",
+                "latitude": 33.4413,
+                "timezone": "america/phoenix",
+                "longitude": -112.0421,
+                "city": "phoenix",
+                "region": "arizona",
+                "location": [
+                  -112.0421,
+                  33.4413
+                ],
+                "application": "https",
+                "reference": "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv",
+                "portlist": "6606",
+                "protocol": "tcp",
+                "asn": 14315,
+                "asn_desc": "1gservers",
+                "firsttime": "2022-07-20T21:41:13.000000Z",
+                "reporttime": "2022-07-21T09:32:44.696140Z",
+                "lasttime": "2022-07-20T21:41:13.000000Z",
+                "indicator_ipv4": "207.32.218.12"
+              },
+              {
+                "indicator": "103.133.105.50",
+                "itype": "ipv4",
+                "tlp": "white",
+                "provider": "sslbl.abuse.ch",
+                "group": [
+                  "everyone"
+                ],
+                "count": 1,
+                "tags": [
+                  "botnet",
+                  "malware"
+                ],
+                "confidence": 10,
+                "uuid": "1aa35d5f-59ee-4364-8ad3-dd9d78cd2140",
+                "cc": "vn",
+                "latitude": 10.8326,
+                "timezone": "asia/ho_chi_minh",
+                "longitude": 106.6581,
+                "city": "ho chi minh city",
+                "region": "ho chi minh",
+                "location": [
+                  106.6581,
+                  10.8326
+                ],
+                "application": "https",
+                "reference": "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv",
+                "portlist": "1234",
+                "protocol": "tcp",
+                "asn": 135905,
+                "asn_desc": "vietnam posts and telecommunications group",
+                "firsttime": "2022-07-19T09:30:19.000000Z",
+                "reporttime": "2022-07-20T00:19:11.521288Z",
+                "lasttime": "2022-07-19T09:30:19.000000Z",
+                "indicator_ipv4": "103.133.105.50"
+              }
+            ]
+          }
@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.1.0"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3839
@@ -0,0 +1 @@
+{"indicator":"89.160.20.156","itype":"ipv4","tlp":"white","provider":"threatfox.abuse.ch","group":["everyone"],"count":1,"tags":["agenttesla","botnet","hunter"],"confidence":8.0,"description":"agent tesla","uuid":"3fbdd654-b2b0-498c-8e20-ef87bce73672","reference":"https://threatfox.abuse.ch/ioc/838651/","rdata":"http://208.67.106.111/theme/inc/e26dbe0dcc481e.php","firsttime":"2022-07-19T07:40:41.000000Z","lasttime":"2022-07-19T08:35:05.971696Z","reporttime":"2022-07-19T08:35:05.971696Z","indicator_ipv4":"89.160.20.156"}
@@ -0,0 +1,48 @@
+{
+    "expected": [
+        {
+            "cif3": {
+                "itype": "ipv4",
+                "rdata": "http://208.67.106.111/theme/inc/e26dbe0dcc481e.php",
+                "uuid": "3fbdd654-b2b0-498c-8e20-ef87bce73672"
+            },
+            "ecs": {
+                "version": "8.4.0"
+            },
+            "event": {
+                "category": "threat",
+                "kind": "enrichment",
+                "original": "{\"indicator\":\"89.160.20.156\",\"itype\":\"ipv4\",\"tlp\":\"white\",\"provider\":\"threatfox.abuse.ch\",\"group\":[\"everyone\"],\"count\":1,\"tags\":[\"agenttesla\",\"botnet\",\"hunter\"],\"confidence\":8.0,\"description\":\"agent tesla\",\"uuid\":\"3fbdd654-b2b0-498c-8e20-ef87bce73672\",\"reference\":\"https://threatfox.abuse.ch/ioc/838651/\",\"rdata\":\"http://208.67.106.111/theme/inc/e26dbe0dcc481e.php\",\"firsttime\":\"2022-07-19T07:40:41.000000Z\",\"lasttime\":\"2022-07-19T08:35:05.971696Z\",\"reporttime\":\"2022-07-19T08:35:05.971696Z\",\"indicator_ipv4\":\"89.160.20.156\"}",
+                "type": "indicator"
+            },
+            "related": {
+                "ip": [
+                    "89.160.20.156"
+                ]
+            },
+            "tags": [
+                "preserve_original_event",
+                "agenttesla",
+                "botnet",
+                "hunter"
+            ],
+            "threat": {
+                "indicator": {
+                    "confidence": "High",
+                    "description": "agent tesla",
+                    "first_seen": "2022-07-19T07:40:41.000000Z",
+                    "ip": "89.160.20.156",
+                    "last_seen": "2022-07-19T08:35:05.971696Z",
+                    "marking": {
+                        "tlp": "WHITE"
+                    },
+                    "modified_at": "2022-07-19T08:35:05.971696Z",
+                    "provider": "threatfox.abuse.ch",
+                    "reference": "https://threatfox.abuse.ch/ioc/838651/",
+                    "sightings": 1,
+                    "type": "ipv4-addr"
+                }
+            }
+        }
+    ]
+}
@@ -0,0 +1,3 @@
+fields:
+  tags:
+    - preserve_original_event
@@ -0,0 +1,11 @@
+input: httpjson
+service: cif3
+vars:
+  url: http://{{Hostname}}:{{Port}}
+  api_token: testing
+data_stream:
+  vars:
+    preserve_original_event: true
+    confidence: '8'
+    type: ipv4
+    cif_tags: 'botnet,exploit,malware,phishing'
@@ -0,0 +1,87 @@
+config_version: "2"
+interval: {{interval}}
+request.method: "GET"
+
+{{#if url}}
+request.url: {{url}}/feed
+{{/if}}
+{{#if proxy_url }}
+request.proxy_url: {{proxy_url}}
+{{/if}}
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+{{#if http_client_timeout}}
+request.timeout: {{http_client_timeout}}
+{{/if}}
+request.transforms:
+- set:
+    target: header.Accept
+    value: 'application/vnd.cif.v3+json'
+- delete:
+    target: header.User-Agent
+- set:
+    target: header.User-Agent
+    value: elastic-integration/0.1.0
+{{#if api_token }}
+- set:
+    target: header.Authorization
+    value: Token token={{ api_token }}
+{{/if}}
+{{#if type}}
+- set:
+    target: url.params.itype
+    value: {{ type }}
+{{/if}}
+{{#if confidence}}
+- set:
+    target: url.params.confidence
+    value: {{ confidence }}
+{{/if}}
+{{#if limit}}
+- set:
+    target: url.params.limit
+    value: {{ limit }}
+{{/if}}
+{{#if cif_tags}}
+- set:
+    target: url.params.tags
+    value: {{ cif_tags }}
+{{/if}}
+{{#if lookback_hours}}
+- set:
+    target: url.params.hours
+    value: {{ lookback_hours }}
+{{/if}}
+- set:
+    target: url.params.reporttime
+    value: '[[.cursor.last_requested_at]]'
+    default: '[[ formatDate (now (parseDuration "-{{initial_lookback}}")) "RFC3339" ]]'
+
+{{#each filters}}
+- set:
+    target: "url.params.{{{ @key }}}"
+    value: {{ this }}
+{{/each}}
+
+response.split:
+  target: body.data
+
+cursor:
+  last_requested_at:
+    value: '[[ formatDate (now) "RFC3339" ]]'
+
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"indicator":"89.160.20.156","itype":"ipv4","tlp":"white","provider":"threatfox.abuse.ch","group":["everyone"],"count":1,"tags":["agenttesla","botnet","hunter"],"confidence":8.0,"description":"agent tesla","uuid":"3fbdd654-b2b0-498c-8e20-ef87bce73672","reference":"https://threatfox.abuse.ch/ioc/838651/","rdata":"http://208.67.106.111/theme/inc/e26dbe0dcc481e.php","firsttime":"2022-07-19T07:40:41.000000Z","lasttime":"2022-07-19T08:35:05.971696Z","reporttime":"2022-07-19T08:35:05.971696Z","indicator_ipv4":"89.160.20.156"}