Skip to content

Commit

Permalink
Merge branch 'main' of https://github.com/aliabbas-elastic/integrations…
Browse files Browse the repository at this point in the history 
… into aws_benchmark_cloudtrail
  • Loading branch information
@ali786XI
ali786XI committed Apr 4, 2024
2 parents 84b5c35 + 45558de commit 1af78f3
Show file tree
Hide file tree
Showing 404 changed files with 37,597 additions and 968 deletions.
10 changes: 10 additions & 0 deletions packages/akamai/changelog.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,14 @@
# newer versions go on top
- version: "2.23.2"
changes:
- description: Handle HTTP headers without values.
type: bugfix
link: https://github.com/elastic/integrations/pull/9486
- version: "2.23.1"
changes:
- description: Fix errors processing empty userRiskData.{risk,trust,general} values.
type: bugfix
link: https://github.com/elastic/integrations/pull/9483
- version: "2.23.0"
changes:
- description: Set sensitive values as secret and add missing mappings.
Expand Down
1 change: 1 addition & 0 deletions packages/akamai/data_stream/siem/_dev/test/pipeline/test-http-json.log
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
{"format":"json","type":"akamai_siem","version":"1.0","attackData":{"clientIP":"89.160.20.156","configId":"14227","policyId":"qik1_26545","ruleActions":"YWxlcnQ%3d%3bYWxlcnQ%3d%3bZGVueQ%3d%3d","ruleData":"dGVsbmV0LmV4ZQ%3d%3d%3bdGVsbmV0LmV4ZQ%3d%3d%3bVmVjdG9yIFNjb3JlOiAxMCwgREVOWSB0aHJlc2hvbGQ6IDksIEFsZX ","ruleMessages":"U3lzdGVtIENvbW1hbmQgQWNjZXNz%3bU3lzdGVtIENvbW1hbmQgSW5qZWN0aW9u%3bQW5vbWFseSBTY29yZSBFeGNlZWRlZCBmb3 ","ruleSelectors":"QVJHUzpvcHRpb24%3d%3bQVJHUzpvcHRpb24%3d%3b","ruleTags":"T1dBU1BfQ1JTL1dFQl9BVFRBQ0svRklMRV9JTkpFQ1RJT04%3d%3bT1dBU1BfQ1JTL1dFQl9BVFRBQ0svQ09NTUFORF9JTkpFQ1R ","ruleVersions":"NA%3d%3d%3bNA%3d%3d%3bMQ%3d%3d","rules":"OTUwMDAy%3bOTUwMDA2%3bQ01ELUlOSkVDVElPTi1BTk9NQUxZ"},"geo":{"asn":"14618","city":"ASHBURN","continent":"288","country":"US","regionCode":"VA"},"httpMessage":{"bytes":"266","host":"www.hmapi.com","method":"GET","path":"/","port":"80","protocol":"HTTP/1.1","query":"option=com_jce%20telnet.exe","requestHeaders":"User-Agent%3a%20BOT%2f0.1%20(BOT%20for%20JCE)%0d%0aAccept%3a%20text%2fhtml,application%2fxhtml+xml","requestId":"1158db1758e37bfe67b7c09","responseHeaders":"Server%3a%20AkamaiGHost%0d%0aMime-Version%3a%201.0%0d%0aContent-Type%3a%20text%2fhtml%0d%0aContent-Length%3a%20150","start":"1491303422","status":"200"},"userRiskData":{"uuid":"964d54b7-0821-413a-a4d6-8131770ec8d5","status":"0","score":"75","risk":"udfp:1325gdg4g4343g/M|unp:74256/H","trust":"ugp:US","general":"duc_1h:10|duc_1d:30","allow":"0"},"clientData":{"appBundleId":"com.mydomain.myapp","appVersion":"1.23","sdkVersion":"4.7.1","telemetryType":"2"},"botData":{"botScore":"100","responseSegment":"3"}}
{"format":"json","type":"akamai_siem","version":"1.0","attackData":{"clientIP":"89.160.20.156","configId":"14227","policyId":"qik1_26545","ruleActions":"YWxlcnQ%3d%3bYWxlcnQ%3d%3bZGVueQ%3d%3d","ruleData":"dGVsbmV0LmV4ZQ%3d%3d%3bdGVsbmV0LmV4ZQ%3d%3d%3bVmVjdG9yIFNjb3JlOiAxMCwgREVOWSB0aHJlc2hvbGQ6IDksIEFsZX ","ruleMessages":"U3lzdGVtIENvbW1hbmQgQWNjZXNz%3bU3lzdGVtIENvbW1hbmQgSW5qZWN0aW9u%3bQW5vbWFseSBTY29yZSBFeGNlZWRlZCBmb3 ","ruleSelectors":"QVJHUzpvcHRpb24%3d%3bQVJHUzpvcHRpb24%3d%3b","ruleTags":"T1dBU1BfQ1JTL1dFQl9BVFRBQ0svRklMRV9JTkpFQ1RJT04%3d%3bT1dBU1BfQ1JTL1dFQl9BVFRBQ0svQ09NTUFORF9JTkpFQ1R ","ruleVersions":"NA%3d%3d%3bNA%3d%3d%3bMQ%3d%3d","rules":"OTUwMDAy%3bOTUwMDA2%3bQ01ELUlOSkVDVElPTi1BTk9NQUxZ"},"geo":{"asn":"14618","city":"ASHBURN","continent":"288","country":"US","regionCode":"VA"},"httpMessage":{"bytes":"266","host":"www.hmapi.com","method":"GET","path":"/","port":"80","protocol":"HTTP/1.1","query":"option=com_jce%20telnet.exe","requestHeaders":"Host:%20example.com%0D%0Ajsessionid:%0D%0AAccept-Encoding:%20gzip%0D%0A","requestId":"1158db1758e37bfe67b7c09","responseHeaders":"Server:%20AkamaiGHost%0D%0AMime-Version:%201.0%0D%0AContent-Type:%20text/html%0D%0AContent-Length:%20150%0D%0AX-NoValueHeader:%0D%0A","start":"1491303422","status":"200"},"userRiskData":{"uuid":"964d54b7-0821-413a-a4d6-8131770ec8d5","status":"0","score":"75","risk":"","trust":"","general":"","allow":"0"},"clientData":{"appBundleId":"com.mydomain.myapp","appVersion":"1.23","sdkVersion":"4.7.1","telemetryType":"2"},"botData":{"botScore":"100","responseSegment":"3"}}
{"format":"json","type":"akamai_siem","version":"1.0","attackData":{"clientIP":"89.160.20.156","configId":"6724","policyId":"scoe_5426","ruleActions":"QUxFUlQ;REVOWQ==","ruleData":"YWxlcnQo;Y3VybA==","ruleMessages":"Q3Jvc3Mtc2l0ZSBTY3 JpcHRpbmcgKFhTUykgQXR0YWNr; UmVxdWVzdCBJbmRpY2F0ZXMgYW4 gYXV0b21hdGVkIHByb2 dyYW0gZXhwbG9yZWQgdGhlIHNpdGU=","ruleSelectors":"QVJHUzph;UkVRVUVTVF9IRU FERVJTOlVzZXItQWdlbnQ=","ruleTags":"V0VCX0FUVEFDSy9YU1M=;QV VUT01BVElPTi9NSVND","ruleVersions":";","rules":"OTUwMDA0;OTkwMDEx"},"geo":{"asn":"12271","city":"NEWYORK","continent":"NA","country":"US","regionCode":"NY"},"httpMessage":{"bytes":"34523","host":"www.example.com","method":"POST","path":"/examples/1/","port":"80","protocol":"http/2","query":"a%3D..%2F..%2F..%2Fetc%2Fpasswd","requestHeaders":"User-Agent%3a%20BOT%2f0.1%20(BOT%20for%20JCE)%0d%0aAccept%3a%20text%2fhtml,application%2fxhtml+xml","requestId":"2ab418ac8515f33","responseHeaders":"Server%3a%20AkamaiGHost%0d%0aMime-Version%3a%201.0%0d%0aContent-Type%3a%20text%2fhtml","start":"1470923133.026","status":"301","tls": "TLSv1.2"},"userRiskData":{"uuid":"964d54b7-0821-413a-a4d6-8131770ec8d5","status":"0","score":"75","risk":"udfp:1325gdg4g4343g/M|unp:74256/H","trust":"ugp:US","general":"duc_1h:10|duc_1d:30","allow":"0"},"clientData":{"appBundleId":"com.mydomain.myapp","appVersion":"1.23","sdkVersion":"4.7.1","telemetryType":"2"},"botData":{"botScore":"100","responseSegment":"3"}}
{"total":10000,"offset":"71cca;3phZmEdPj6YEqml0rvbdWDZGW3mCiJIwjyhkJfsLFM2gVYPgE8-N_0CiLI9gwH0_4OJ87xDQ3b-gIsx_kEBdf7aaC_AvDpG9fMxypeaCma10FKrY9VKE","limit":10000}
164 changes: 164 additions & 0 deletions packages/akamai/data_stream/siem/_dev/test/pipeline/test-http-json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,170 @@
"query": "option=com_jce telnet.exe"
}
},
{
"@timestamp": "2017-04-04T10:57:02.000Z",
"akamai": {
"siem": {
"bot": {
"response_segment": 3,
"score": 100
},
"client_data": {
"app_bundle_id": "com.mydomain.myapp",
"app_version": "1.23",
"sdk_version": "4.7.1",
"telemetry_type": 2
},
"config_id": "14227",
"policy_id": "qik1_26545",
"request": {
"headers": {
"Accept-Encoding": "gzip",
"Host": "example.com"
}
},
"response": {
"headers": {
"Content-Length": "150",
"Content-Type": "text/html",
"Mime-Version": "1.0",
"Server": "AkamaiGHost"
}
},
"rule_actions": [
"alert",
"deny"
],
"rule_tags": [
"owasp_crs/web_attack/file_injection",
"owasp_crs/web_attack/command_inject"
],
"rules": [
{
"ruleActions": "alert",
"ruleData": "telnet.exe",
"ruleMessages": "System Command Access",
"ruleSelectors": "ARGS:option",
"ruleTags": "OWASP_CRS/WEB_ATTACK/FILE_INJECTION",
"ruleVersions": "4",
"rules": "950002"
},
{
"ruleActions": "alert",
"ruleData": "telnet.exe",
"ruleMessages": "System Command Injection",
"ruleSelectors": "ARGS:option",
"ruleTags": "OWASP_CRS/WEB_ATTACK/COMMAND_INJECT",
"ruleVersions": "4",
"rules": "950006"
},
{
"ruleActions": "deny",
"ruleData": "Vector Score: 10, DENY threshold: 9, Ale",
"ruleMessages": "Anomaly Score Exceeded fo",
"ruleVersions": "1",
"rules": "CMD-INJECTION-ANOMALY"
}
],
"user_risk": {
"allow": 0,
"score": 75,
"status": 0,
"uuid": "964d54b7-0821-413a-a4d6-8131770ec8d5"
}
}
},
"client": {
"address": "89.160.20.156",
"as": {
"number": 29518,
"organization": {
"name": "Bredband2 AB"
}
},
"geo": {
"city_name": "Linköping",
"continent_name": "Europe",
"country_iso_code": "SE",
"country_name": "Sweden",
"location": {
"lat": 58.4167,
"lon": 15.6167
},
"region_iso_code": "SE-E",
"region_name": "Östergötland County"
},
"ip": "89.160.20.156"
},
"ecs": {
"version": "8.11.0"
},
"event": {
"category": [
"network"
],
"id": "1158db1758e37bfe67b7c09",
"kind": "event",
"original": "{\"format\":\"json\",\"type\":\"akamai_siem\",\"version\":\"1.0\",\"attackData\":{\"clientIP\":\"89.160.20.156\",\"configId\":\"14227\",\"policyId\":\"qik1_26545\",\"ruleActions\":\"YWxlcnQ%3d%3bYWxlcnQ%3d%3bZGVueQ%3d%3d\",\"ruleData\":\"dGVsbmV0LmV4ZQ%3d%3d%3bdGVsbmV0LmV4ZQ%3d%3d%3bVmVjdG9yIFNjb3JlOiAxMCwgREVOWSB0aHJlc2hvbGQ6IDksIEFsZX \",\"ruleMessages\":\"U3lzdGVtIENvbW1hbmQgQWNjZXNz%3bU3lzdGVtIENvbW1hbmQgSW5qZWN0aW9u%3bQW5vbWFseSBTY29yZSBFeGNlZWRlZCBmb3 \",\"ruleSelectors\":\"QVJHUzpvcHRpb24%3d%3bQVJHUzpvcHRpb24%3d%3b\",\"ruleTags\":\"T1dBU1BfQ1JTL1dFQl9BVFRBQ0svRklMRV9JTkpFQ1RJT04%3d%3bT1dBU1BfQ1JTL1dFQl9BVFRBQ0svQ09NTUFORF9JTkpFQ1R \",\"ruleVersions\":\"NA%3d%3d%3bNA%3d%3d%3bMQ%3d%3d\",\"rules\":\"OTUwMDAy%3bOTUwMDA2%3bQ01ELUlOSkVDVElPTi1BTk9NQUxZ\"},\"geo\":{\"asn\":\"14618\",\"city\":\"ASHBURN\",\"continent\":\"288\",\"country\":\"US\",\"regionCode\":\"VA\"},\"httpMessage\":{\"bytes\":\"266\",\"host\":\"www.hmapi.com\",\"method\":\"GET\",\"path\":\"/\",\"port\":\"80\",\"protocol\":\"HTTP/1.1\",\"query\":\"option=com_jce%20telnet.exe\",\"requestHeaders\":\"Host:%20example.com%0D%0Ajsessionid:%0D%0AAccept-Encoding:%20gzip%0D%0A\",\"requestId\":\"1158db1758e37bfe67b7c09\",\"responseHeaders\":\"Server:%20AkamaiGHost%0D%0AMime-Version:%201.0%0D%0AContent-Type:%20text/html%0D%0AContent-Length:%20150%0D%0AX-NoValueHeader:%0D%0A\",\"start\":\"1491303422\",\"status\":\"200\"},\"userRiskData\":{\"uuid\":\"964d54b7-0821-413a-a4d6-8131770ec8d5\",\"status\":\"0\",\"score\":\"75\",\"risk\":\"\",\"trust\":\"\",\"general\":\"\",\"allow\":\"0\"},\"clientData\":{\"appBundleId\":\"com.mydomain.myapp\",\"appVersion\":\"1.23\",\"sdkVersion\":\"4.7.1\",\"telemetryType\":\"2\"},\"botData\":{\"botScore\":\"100\",\"responseSegment\":\"3\"}}",
"start": "2017-04-04T10:57:02.000Z"
},
"http": {
"request": {
"id": "1158db1758e37bfe67b7c09",
"method": "GET"
},
"response": {
"bytes": 266,
"status_code": 200
},
"version": "1.1"
},
"network": {
"protocol": "http",
"transport": "tcp"
},
"observer": {
"type": "proxy",
"vendor": "akamai"
},
"related": {
"ip": [
"89.160.20.156"
]
},
"source": {
"address": "89.160.20.156",
"as": {
"number": 29518,
"organization": {
"name": "Bredband2 AB"
}
},
"geo": {
"city_name": "Linköping",
"continent_name": "Europe",
"country_iso_code": "SE",
"country_name": "Sweden",
"location": {
"lat": 58.4167,
"lon": 15.6167
},
"region_iso_code": "SE-E",
"region_name": "Östergötland County"
},
"ip": "89.160.20.156"
},
"tags": [
"preserve_original_event"
],
"url": {
"domain": "www.hmapi.com",
"full": "www.hmapi.com/?option=com_jce%20telnet.exe",
"path": "/",
"port": 80,
"query": "option=com_jce telnet.exe"
}
},
{
"@timestamp": "2016-08-11T13:45:33.026Z",
"akamai": {
Expand Down
37 changes: 30 additions & 7 deletions packages/akamai/data_stream/siem/elasticsearch/ingest_pipeline/default.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,12 @@ processors:
target_field: url.domain
ignore_missing: true
- urldecode:
tag: urldecode_httpMessage_path
field: json.httpMessage.path
target_field: url.path
ignore_missing: true
- urldecode:
tag: urldecode_httpMessage_query
field: json.httpMessage.query
target_field: url.query
ignore_missing: true
Expand All @@ -83,24 +85,28 @@ processors:
type: long
ignore_missing: true
- urldecode:
tag: urldecode_httpMessage_responseHeaders
field: json.httpMessage.responseHeaders
target_field: _tmp.response.headers
ignore_missing: true
- kv:
field: _tmp.response.headers
if: ctx.json?.httpMessage?.responseHeaders != ""
tag: kv_httpMessage_responseHeaders
field: json.httpMessage.responseHeaders
target_field: akamai.siem.response.headers
field_split: '\r\n'
value_split: ': '
value_split: ':\s*'
ignore_missing: true
- urldecode:
tag: urldecode_httpMessage_requestHeaders
field: json.httpMessage.requestHeaders
target_field: _tmp.request.headers
ignore_missing: true
- kv:
field: _tmp.request.headers
if: ctx.json?.httpMessage?.requestHeaders != ""
tag: kv_httpMessage_requestHeaders
field: json.httpMessage.requestHeaders
target_field: akamai.siem.request.headers
field_split: '\r\n'
value_split: ': '
value_split: ':\s*'
ignore_missing: true
- script:
lang: painless
Expand Down Expand Up @@ -194,6 +200,7 @@ processors:
ignore_missing: true
## Attack Data
- urldecode:
tag: urldecode_attackData_ruleActions
field: json.attackData.ruleActions
target_field: json.attackData.ruleActions
ignore_missing: true
Expand All @@ -203,6 +210,7 @@ processors:
separator: ';'
preserve_trailing: true
- urldecode:
tag: urldecode_attackData_ruleData
field: json.attackData.ruleData
target_field: json.attackData.ruleData
ignore_missing: true
Expand All @@ -212,6 +220,7 @@ processors:
separator: ';'
preserve_trailing: true
- urldecode:
tag: urldecode_attackData_ruleMessages
field: json.attackData.ruleMessages
target_field: json.attackData.ruleMessages
ignore_missing: true
Expand All @@ -221,6 +230,7 @@ processors:
separator: ';'
preserve_trailing: true
- urldecode:
tag: urldecode_attackData_ruleSelectors
field: json.attackData.ruleSelectors
target_field: json.attackData.ruleSelectors
ignore_missing: true
Expand All @@ -230,6 +240,7 @@ processors:
separator: ';'
preserve_trailing: true
- urldecode:
tag: urldecode_attackData_ruleTags
field: json.attackData.ruleTags
target_field: json.attackData.ruleTags
ignore_missing: true
Expand All @@ -239,6 +250,7 @@ processors:
separator: ';'
preserve_trailing: true
- urldecode:
tag: urldecode_attackData_ruleVersions
field: json.attackData.ruleVersions
target_field: json.attackData.ruleVersions
ignore_missing: true
Expand All @@ -248,6 +260,7 @@ processors:
separator: ';'
preserve_trailing: true
- urldecode:
tag: urldecode_attackData_rules
field: json.attackData.rules
target_field: json.attackData.rules
ignore_missing: true
Expand All @@ -259,6 +272,7 @@ processors:
- script:
lang: painless
description: Base64 Decode the json.attackData.rule* fields
tag: script_base64_decode_attackData_rule
source: |
ArrayList items = new ArrayList(["rules", "ruleActions", "ruleData", "ruleMessages", "ruleTags", "ruleSelectors", "ruleVersions"]);
ArrayList rules_array = new ArrayList();
Expand Down Expand Up @@ -386,18 +400,24 @@ processors:
type: long
ignore_missing: true
- kv:
if: ctx.json?.userRiskData?.risk != ""
tag: kv_userRiskData_risk
field: json.userRiskData.risk
target_field: akamai.siem.user_risk.risk
field_split: '\|'
value_split: ':'
ignore_missing: true
- kv:
if: ctx.json?.userRiskData?.trust != ""
tag: kv_userRiskData_trust
field: json.userRiskData.trust
target_field: akamai.siem.user_risk.trust
field_split: '\|'
value_split: ':'
ignore_missing: true
- kv:
if: ctx.json?.userRiskData?.general != ""
tag: kv_userRiskData_general
field: json.userRiskData.general
target_field: akamai.siem.user_risk.general
field_split: '\|'
Expand Down Expand Up @@ -458,4 +478,7 @@ on_failure:
value: pipeline_error
- append:
field: error.message
value: '{{{ _ingest.on_failure_message }}}'
value: >-
Processor '{{ _ingest.on_failure_processor_type }}'
{{#_ingest.on_failure_processor_tag}}with tag '{{ _ingest.on_failure_processor_tag }}'
{{/_ingest.on_failure_processor_tag}}failed with message '{{ _ingest.on_failure_message }}'
2 changes: 1 addition & 1 deletion packages/akamai/manifest.yml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name: akamai
title: Akamai
version: "2.23.0"
version: "2.23.2"
description: Collect logs from Akamai with Elastic Agent.
type: integration
format_version: "3.0.2"
Expand Down
14 changes: 14 additions & 0 deletions packages/aws/_dev/benchmark/rally/waf-benchmark.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
---
description: Benchmark of 20000 aws.waf events ingested
data_stream:
name: waf
corpora:
generator:
total_events: 20000
template:
type: gotext
path: ./waf-benchmark/template.ndjson
config:
path: ./waf-benchmark/config.yml
fields:
path: ./waf-benchmark/fields.yml
Loading

0 comments on commit 1af78f3

Please sign in to comment.