From 269f3b472e26bbf9c5a59fc78f752f75aa32fd69 Mon Sep 17 00:00:00 2001 From: EijiSugiura Date: Tue, 15 Dec 2020 17:51:36 +0900 Subject: [PATCH 1/6] DeepSecurity support --- .../es_loader/README_deepsecurity_ja.md | 165 +++++++++++++++ source/lambda/es_loader/aws.ini | 27 +++ .../lambda/es_loader/siem/sf_deepsecurity.py | 191 ++++++++++++++++++ 3 files changed, 383 insertions(+) create mode 100644 source/lambda/es_loader/README_deepsecurity_ja.md create mode 100644 source/lambda/es_loader/siem/sf_deepsecurity.py diff --git a/source/lambda/es_loader/README_deepsecurity_ja.md b/source/lambda/es_loader/README_deepsecurity_ja.md new file mode 100644 index 00000000..724d0d16 --- /dev/null +++ b/source/lambda/es_loader/README_deepsecurity_ja.md @@ -0,0 +1,165 @@ +# es_loaderでDeepSecurityのログを取り込む + +以下の仕組みで、DeepSecurityのログをSIEMに取り込んでいきます。 + +1. ec2 instance上で動作しているDeepSecurity → syslogで/var/log/dsa.logにログを出力 +2. td-agent/fluentdで/var/log/dsa.log → s3 bucketに転送 +3. lambda functionで動作するes loaderでs3を読み取って、Elasticsearchにloadする + +## DeepSecurityでのlocalhostへのsyslog転送 + +DeepSecurity SaaSの管理画面にloginし、Adminitration -> System Settings -> Event Forwardingで、SIEMに127.0.0.1 514/udp Local1 falicityにCommonEventFormatでログを直接転送する設定を行っておきます。 +Agent should forward logs: Directory to the Syslog Server + +## rsyslogで/var/log/dsa.logにDeepSecurityのログを保存 +CEF:やLEEF:を含むログをDeepSecurityのログとして、/var/log/dsa.logに保存します。 + +/etc/rsyslog.d/ds_agent.conf +``` +$FileCreateMode 0644 + +:syslogtag, contains, "CEF:" /var/log/dsa.log +& stop + +:syslogtag, contains, "LEEF:" /var/log/dsa.log +& stop +``` + +## td-agent/fluendからS3へのlog転送 + +td-agentを用いて、S3にログを転送します。 + +/etc/td-agent/conf.d/ds_agent.conf +``` + + @type tail + format none + path /var/log/dsa.log + pos_file /var/log/td-agent/.dsa.pos + tag ds_agent.* + + + + @type record_transformer + @id ds_agent_record_modifier + enable_ruby true + + hostname "#{Socket.gethostname}" + timestamp ${time.strftime('%FT%T%:z')} + tag ${tag} + + + + + @type s3 + @id ds_agent_s3 + s3_bucket ${BUCKET_NAME} + s3_region ${REGION} + s3_object_key_format %{path}%{time_slice}_${hostname}_%{index}.%{file_extension} + path ds_agent/ + time_slice_format %Y/%m/%d/%H + timezone Asia/Tokyo + output_time false + output_tag false + + @type file + path /var/log/td-agent/buffer/s3_ds_agent + flush_mode interval + flush_interval 1m + flush_at_shutdown true + + +``` + +ec2 instanceからのs3への書き込みは、instance profileで許可をしてあげると良いです。 +https://aws.amazon.com/jp/premiumsupport/knowledge-center/ec2-instance-access-s3-bucket/ + +## elasticsearchでのlog-deepsecurity templateの定義 + +``` +PUT _template/log-deepsecurity +{ + "log-deepsecurity" : { + "index_patterns" : [ + "log-deepsecurity*" + ], + "mappings" : { + "properties" : { + "cloud.account" : { + "type" : "object" + }, + "cloud.region" : { + "type" : "keyword" + }, + "event.action" : { + "type" : "keyword" + }, + "event.severity" : { + "type" : "integer" + }, + "event.original" : { + "type" : "text" + }, + "event.count" : { + "type" : "integer" + }, + "file.path" : { + "type" : "keyword" + }, + "destination.port" : { + "type" : "integer" + }, + "destination.ip" : { + "type" : "ip" + }, + "geo.country_iso_code" : { + "type" : "keyword" + }, + "host.id" : { + "type" : "keyword" + }, + "network.transport" : { + "type" : "keyword" + }, + "rule.category" : { + "type" : "keyword" + }, + "rule.name" : { + "type" : "keyword" + }, + "rule.ruleset" : { + "type" : "keyword" + }, + "service.name" : { + "type" : "keyword" + }, + "source.ip" : { + "type" : "ip" + }, + "source.port" : { + "type" : "integer" + }, + "timestamp" : { + "type" : "date" + } + } + } + } +} +``` + +## es_loader側の設定 + +aws.ini/user.iniに以下を定義します。 +``` +[deepsecurity] +index = log-deepsecurity +s3_key = ds_agent +format = json +scripted_fields = True +``` + +lambda functionに、deepsecurityのlogを解釈する siem/sf_deepsecurity.py が存在していることを確認してください。 +ここまでの設定で、Elasticsearchにログがloadされていくはずです。 + + diff --git a/source/lambda/es_loader/aws.ini b/source/lambda/es_loader/aws.ini index 61a14141..580b3c7d 100644 --- a/source/lambda/es_loader/aws.ini +++ b/source/lambda/es_loader/aws.ini @@ -634,3 +634,30 @@ event.kind = event script_ecs = cloud.instance.id event.action event.category event.outcome source.ip source.port user.name user.id geoip = source + +[deepsecurity] +# https://cloudone.trendmicro.com/docs/workload-security/event-syslog-message-formats/ +index = log-deepsecurity +s3_key = ds_agent +format = json +scripted_fields = True + +# scriptで処理をした後にecsにmappingする機能はないため、以下に相当する処理をsf_deepsecurity.pyで実施しています。 +# +#ecs = event.action destination.ip destination.port destination.mac destination.bytes source.ip source.port source.mac source.bytes network.transport event.action server.name file.path event.count rule.category host.id event.original +#destination.ip = dst +#destination.port = dpt +#destination.mac = dmac +#destination.bytes = out +#source.ip = src +#source.port = spt +#source.mac = smac +#source.bytes = in +#network.transport = proto +#event.action = act +#server.name = fluent_hostname +#file.path = fname +#event.count = cnt +#rule.category = cs1 +#host.id = cn1 +#event.original = msg diff --git a/source/lambda/es_loader/siem/sf_deepsecurity.py b/source/lambda/es_loader/siem/sf_deepsecurity.py new file mode 100644 index 00000000..52b0e795 --- /dev/null +++ b/source/lambda/es_loader/siem/sf_deepsecurity.py @@ -0,0 +1,191 @@ +import re +import base64 +import json +import ipaddress + +def transform(logdata): + # https://cloudone.trendmicro.com/docs/workload-security/event-syslog-message-formats/ + fields = logdata['message'].split('|') + if len(fields) < 8: + print("Illegal format") + return Null + logdata.setdefault('agent', {}) + logdata['agent']['name'] = " ".join([fields[1],fields[2],fields[3]]) + logdata.setdefault('rule', {}) + logdata['rule']['name'] = " ".join([fields[4],fields[5]]) + logdata.setdefault('event', {}) + logdata['event']['severity'] = fields[6] + + # \\=を適当な文字列に置換しておく + message = re.sub('\\\\=', '____', fields[7]) + # =をdelimiterとして、順に処理していく + attributes = message.split('=') + next_ptr = attributes.pop(0) + for ptr in attributes: + values = ptr.split() + if values is None: + break + curr_ptr = next_ptr + next_ptr = values.pop() + value = ' '.join(values) + if value: + logdata[curr_ptr] = re.sub('____', '=', value) + # 末尾の処理 + logdata[curr_ptr] = re.sub('____', '=', value + next_ptr) + + if 'act' in logdata: + # IDS:Resetは、alert出力のみでpacket dropを行わない + # 誤解を招くので、置換しておく + logdata['act'] = re.sub("IDS:Reset","DetectOnly:NotReset",logdata['act']) + + # 以下はecsにmappingしていく処理 + deepsecurity_ecs_keys = { + 'destination.ip': 'dst', + 'destination.port': 'dpt', + 'destination.mac': 'dmac', + 'destination.bytes': 'out', + 'source.ip': 'src', + 'source.port': 'spt', + 'source.mac': 'smac', + 'source.bytes': 'in', + 'network.transport': 'proto', + 'event.action': 'act', + 'server.name': 'fluent_hostname', + 'file.path': 'fname', + 'event.count': 'cnt', + 'rule.category': 'cs1', + 'host.id': 'cn1', + 'event.original': 'msg', + } + + for ecs_key in deepsecurity_ecs_keys: + original_keys = deepsecurity_ecs_keys[ecs_key] + v = get_value_from_dict(logdata, original_keys) + if v: + new_ecs_dict = put_value_into_dict(ecs_key, v) + if ".ip" in ecs_key: + try: + ipaddress.ip_address(v) + except ValueError: + continue + merge(logdata, new_ecs_dict) + del logdata[original_keys] + + # source.ipが設定されていなければ、dvcで代用する + if "dvc" in logdata: + if "source" in logdata and not "ip" in logdata['source']: + logdata['source']['ip'] = logdata['dvc'] + else: + logdata['source'] = { 'ip': logdata['dvc'] } + + # packet captureをdecodeしておく + if 'TrendMicroDsPacketData' in logdata: + saved = logdata['TrendMicroDsPacketData'] + try: + logdata['TrendMicroDsPacketData'] = base64.b64decode(logdata['TrendMicroDsPacketData']).decode("utf-8", errors="backslashreplace") + except Exception as e: + print(e) + logdata['TrendMicroDsPacketData'] = saved + # filter out 'cookie' + filtered = [] + for line in logdata['TrendMicroDsPacketData'].split("\n"): + if re.findall(r'^cookie',line.lower()): + continue + filtered.append(line) + logdata['TrendMicroDsPacketData'] = "\n".join(filtered) + # X-Forwarded-Forを取り出す X-Forwarded-For: 123.123.123.234 + m = re.search(r'X-Forwarded-For: ([0-9.]+)', logdata['TrendMicroDsPacketData']) + if m: + logdata['source']['ip'] = m.group(1) + + del logdata['TrendMicroDsTenant'], logdata['TrendMicroDsTenantId'] + + return logdata + + +def put_value_into_dict(key_str, v): + """dictのkeyにドットが含まれている場合に入れ子になったdictを作成し、値としてvを入れる. + 返値はdictタイプ。vが辞書ならさらに入れ子として代入。 + TODO: 値に"が入ってると例外になる。対処方法が見つからず返値なDROPPEDにしてるので改善する。#34 + + >>> put_value_into_dict('a.b.c', 123) + {'a': {'b': {'c': '123'}}} + >>> v = {'x': 1, 'y': 2} + >>> put_value_into_dict('a.b.c', v) + {'a': {'b': {'c': {'x': 1, 'y': 2}}}} + >>> v = str({'x': "1", 'y': '2"3'}) + >>> put_value_into_dict('a.b.c', v) + {'a': {'b': {'c': 'DROPPED'}}} + """ + v = v + xkeys = key_str.split('.') + if isinstance(v, dict): + json_data = r'{{"{0}": {1} }}'.format(xkeys[-1], json.dumps(v)) + else: + json_data = r'{{"{0}": "{1}" }}'.format(xkeys[-1], v) + if len(xkeys) >= 2: + xkeys.pop() + for xkey in reversed(xkeys): + json_data = r'{{"{0}": {1} }}'.format(xkey, json_data) + try: + new_dict = json.loads(json_data, strict=False) + except json.decoder.JSONDecodeError: + new_dict = put_value_into_dict(key_str, "DROPPED") + return new_dict + +def get_value_from_dict(dct, xkeys_list): + """ 入れ子になった辞書に対して、dotを含んだkeyで値を + 抽出する。keyはリスト形式で複数含んでいたら分割する。 + 値がなければ返値なし + + >>> dct = {'a': {'b': {'c': 123}}} + >>> xkey = "a.b.c" + >>> get_value_from_dict(dct, xkey) + 123 + >>> xkey = "x.y.z" + >>> get_value_from_dict(dct, xkey) + + >>> xkeys_list = "a.b.c x.y.z" + >>> get_value_from_dict(dct, xkeys_list) + 123 + >>> dct = {'a': {'b': [{'c': 123}, {'c': 456}]}} + >>> xkeys_list = "a.b.0.c" + >>> get_value_from_dict(dct, xkeys_list) + 123 + """ + for xkeys in xkeys_list.split(): + v = dct + for k in xkeys.split('.'): + try: + k = int(k) + except ValueError: + pass + try: + v = v[k] + except (TypeError, KeyError, IndexError): + v = '' + break + if v: + return v + +def merge(a, b, path=None): + """merges b into a + """ + if path is None: + path = [] + for key in b: + if key in a: + if isinstance(a[key], dict) and isinstance(b[key], dict): + merge(a[key], b[key], path + [str(key)]) + elif a[key] == b[key]: + pass # same leaf value + elif str(a[key]) in str(b[key]): + # strで上書き。JSONだったのをstrに変換したデータ + a[key] = b[key] + else: + # conflict and override original value with new one + a[key] = b[key] + else: + a[key] = b[key] + return a + From b5d5f8b01404b8cd658670005dfe0608b4161d56 Mon Sep 17 00:00:00 2001 From: EijiSugiura Date: Wed, 16 Dec 2020 13:47:07 +0900 Subject: [PATCH 2/6] Move README for DeepSecurity --- .../contributed/deepsecurity_ja.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename source/lambda/es_loader/README_deepsecurity_ja.md => docs/contributed/deepsecurity_ja.md (100%) diff --git a/source/lambda/es_loader/README_deepsecurity_ja.md b/docs/contributed/deepsecurity_ja.md similarity index 100% rename from source/lambda/es_loader/README_deepsecurity_ja.md rename to docs/contributed/deepsecurity_ja.md From 74813a8a532a84d55648e4b2c7c8d60ed0c4e5cc Mon Sep 17 00:00:00 2001 From: EijiSugiura Date: Wed, 16 Dec 2020 14:06:53 +0900 Subject: [PATCH 3/6] Update documents around script_ecs --- docs/contributed/deepsecurity_ja.md | 60 ++++++++--------------------- 1 file changed, 17 insertions(+), 43 deletions(-) diff --git a/docs/contributed/deepsecurity_ja.md b/docs/contributed/deepsecurity_ja.md index 724d0d16..6f906750 100644 --- a/docs/contributed/deepsecurity_ja.md +++ b/docs/contributed/deepsecurity_ja.md @@ -88,12 +88,6 @@ PUT _template/log-deepsecurity "cloud.account" : { "type" : "object" }, - "cloud.region" : { - "type" : "keyword" - }, - "event.action" : { - "type" : "keyword" - }, "event.severity" : { "type" : "integer" }, @@ -103,42 +97,6 @@ PUT _template/log-deepsecurity "event.count" : { "type" : "integer" }, - "file.path" : { - "type" : "keyword" - }, - "destination.port" : { - "type" : "integer" - }, - "destination.ip" : { - "type" : "ip" - }, - "geo.country_iso_code" : { - "type" : "keyword" - }, - "host.id" : { - "type" : "keyword" - }, - "network.transport" : { - "type" : "keyword" - }, - "rule.category" : { - "type" : "keyword" - }, - "rule.name" : { - "type" : "keyword" - }, - "rule.ruleset" : { - "type" : "keyword" - }, - "service.name" : { - "type" : "keyword" - }, - "source.ip" : { - "type" : "ip" - }, - "source.port" : { - "type" : "integer" - }, "timestamp" : { "type" : "date" } @@ -156,7 +114,23 @@ aws.ini/user.iniに以下を定義します。 index = log-deepsecurity s3_key = ds_agent format = json -scripted_fields = True +script_ecs = event.action destination.ip destination.port destination.mac destination.bytes source.ip source.port source.mac source.bytes network.transport event.action server.name file.path event.count rule.category host.id event.original +event.action = act +destination.ip = dst +destination.port = dpt +destination.mac = dmac +destination.bytes = out +source.ip = src +source.port = spt +source.mac = smac +source.bytes = in +network.transport = proto +server.name = fluent_hostname +file.path = fname +event.count = cnt +rule.category = cs1 +host.id = cn1 +event.original = msg ``` lambda functionに、deepsecurityのlogを解釈する siem/sf_deepsecurity.py が存在していることを確認してください。 From 4677248508f3ec8c541a4f8d60612c87ea82f9d5 Mon Sep 17 00:00:00 2001 From: EijiSugiura Date: Wed, 16 Dec 2020 14:12:33 +0900 Subject: [PATCH 4/6] Move ini sample to use.init.sample --- source/lambda/es_loader/aws.ini | 27 ------------------------ source/lambda/es_loader/user.ini.sample | 28 ++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/source/lambda/es_loader/aws.ini b/source/lambda/es_loader/aws.ini index 580b3c7d..61a14141 100644 --- a/source/lambda/es_loader/aws.ini +++ b/source/lambda/es_loader/aws.ini @@ -634,30 +634,3 @@ event.kind = event script_ecs = cloud.instance.id event.action event.category event.outcome source.ip source.port user.name user.id geoip = source - -[deepsecurity] -# https://cloudone.trendmicro.com/docs/workload-security/event-syslog-message-formats/ -index = log-deepsecurity -s3_key = ds_agent -format = json -scripted_fields = True - -# scriptで処理をした後にecsにmappingする機能はないため、以下に相当する処理をsf_deepsecurity.pyで実施しています。 -# -#ecs = event.action destination.ip destination.port destination.mac destination.bytes source.ip source.port source.mac source.bytes network.transport event.action server.name file.path event.count rule.category host.id event.original -#destination.ip = dst -#destination.port = dpt -#destination.mac = dmac -#destination.bytes = out -#source.ip = src -#source.port = spt -#source.mac = smac -#source.bytes = in -#network.transport = proto -#event.action = act -#server.name = fluent_hostname -#file.path = fname -#event.count = cnt -#rule.category = cs1 -#host.id = cn1 -#event.original = msg diff --git a/source/lambda/es_loader/user.ini.sample b/source/lambda/es_loader/user.ini.sample index 32d70741..cf314aec 100644 --- a/source/lambda/es_loader/user.ini.sample +++ b/source/lambda/es_loader/user.ini.sample @@ -23,4 +23,30 @@ url.path = request_path static_ecs = @log_type @log_type = apache -geoip = source \ No newline at end of file +geoip = source + + +[deepsecurity] +# https://cloudone.trendmicro.com/docs/workload-security/event-syslog-message-formats/ +# See README for more details +# https://github.com/aws-samples/siem-on-amazon-elasticsearch/blob/main/docs/contributed/deepsecurity_ja.md +index = log-deepsecurity +s3_key = ds_agent +format = json +script_ecs = event.action destination.ip destination.port destination.mac destination.bytes source.ip source.port source.mac source.bytes network.transport event.action server.name file.path event.count rule.category host.id event.original +event.action = act +destination.ip = dst +destination.port = dpt +destination.mac = dmac +destination.bytes = out +source.ip = src +source.port = spt +source.mac = smac +source.bytes = in +network.transport = proto +server.name = hostname +file.path = fname +event.count = cnt +rule.category = cs1 +host.id = cn1 +event.original = msg From 62ce0e4b69c02826ea31957c8e7b748ce22e9184 Mon Sep 17 00:00:00 2001 From: EijiSugiura Date: Wed, 16 Dec 2020 17:21:39 +0900 Subject: [PATCH 5/6] fix attribute name --- docs/contributed/deepsecurity_ja.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/contributed/deepsecurity_ja.md b/docs/contributed/deepsecurity_ja.md index 6f906750..0c4ea9c5 100644 --- a/docs/contributed/deepsecurity_ja.md +++ b/docs/contributed/deepsecurity_ja.md @@ -125,7 +125,7 @@ source.port = spt source.mac = smac source.bytes = in network.transport = proto -server.name = fluent_hostname +server.name = hostname file.path = fname event.count = cnt rule.category = cs1 @@ -135,5 +135,3 @@ event.original = msg lambda functionに、deepsecurityのlogを解釈する siem/sf_deepsecurity.py が存在していることを確認してください。 ここまでの設定で、Elasticsearchにログがloadされていくはずです。 - - From 30c3faac2a4ebcafb454d1b6c0f1b9f2aa46998b Mon Sep 17 00:00:00 2001 From: EijiSugiura Date: Wed, 16 Dec 2020 18:10:43 +0900 Subject: [PATCH 6/6] import from siem --- .../lambda/es_loader/siem/sf_deepsecurity.py | 89 +------------------ 1 file changed, 1 insertion(+), 88 deletions(-) diff --git a/source/lambda/es_loader/siem/sf_deepsecurity.py b/source/lambda/es_loader/siem/sf_deepsecurity.py index 52b0e795..fe1086df 100644 --- a/source/lambda/es_loader/siem/sf_deepsecurity.py +++ b/source/lambda/es_loader/siem/sf_deepsecurity.py @@ -2,6 +2,7 @@ import base64 import json import ipaddress +from siem import merge, put_value_into_dict, get_value_from_dict def transform(logdata): # https://cloudone.trendmicro.com/docs/workload-security/event-syslog-message-formats/ @@ -101,91 +102,3 @@ def transform(logdata): del logdata['TrendMicroDsTenant'], logdata['TrendMicroDsTenantId'] return logdata - - -def put_value_into_dict(key_str, v): - """dictのkeyにドットが含まれている場合に入れ子になったdictを作成し、値としてvを入れる. - 返値はdictタイプ。vが辞書ならさらに入れ子として代入。 - TODO: 値に"が入ってると例外になる。対処方法が見つからず返値なDROPPEDにしてるので改善する。#34 - - >>> put_value_into_dict('a.b.c', 123) - {'a': {'b': {'c': '123'}}} - >>> v = {'x': 1, 'y': 2} - >>> put_value_into_dict('a.b.c', v) - {'a': {'b': {'c': {'x': 1, 'y': 2}}}} - >>> v = str({'x': "1", 'y': '2"3'}) - >>> put_value_into_dict('a.b.c', v) - {'a': {'b': {'c': 'DROPPED'}}} - """ - v = v - xkeys = key_str.split('.') - if isinstance(v, dict): - json_data = r'{{"{0}": {1} }}'.format(xkeys[-1], json.dumps(v)) - else: - json_data = r'{{"{0}": "{1}" }}'.format(xkeys[-1], v) - if len(xkeys) >= 2: - xkeys.pop() - for xkey in reversed(xkeys): - json_data = r'{{"{0}": {1} }}'.format(xkey, json_data) - try: - new_dict = json.loads(json_data, strict=False) - except json.decoder.JSONDecodeError: - new_dict = put_value_into_dict(key_str, "DROPPED") - return new_dict - -def get_value_from_dict(dct, xkeys_list): - """ 入れ子になった辞書に対して、dotを含んだkeyで値を - 抽出する。keyはリスト形式で複数含んでいたら分割する。 - 値がなければ返値なし - - >>> dct = {'a': {'b': {'c': 123}}} - >>> xkey = "a.b.c" - >>> get_value_from_dict(dct, xkey) - 123 - >>> xkey = "x.y.z" - >>> get_value_from_dict(dct, xkey) - - >>> xkeys_list = "a.b.c x.y.z" - >>> get_value_from_dict(dct, xkeys_list) - 123 - >>> dct = {'a': {'b': [{'c': 123}, {'c': 456}]}} - >>> xkeys_list = "a.b.0.c" - >>> get_value_from_dict(dct, xkeys_list) - 123 - """ - for xkeys in xkeys_list.split(): - v = dct - for k in xkeys.split('.'): - try: - k = int(k) - except ValueError: - pass - try: - v = v[k] - except (TypeError, KeyError, IndexError): - v = '' - break - if v: - return v - -def merge(a, b, path=None): - """merges b into a - """ - if path is None: - path = [] - for key in b: - if key in a: - if isinstance(a[key], dict) and isinstance(b[key], dict): - merge(a[key], b[key], path + [str(key)]) - elif a[key] == b[key]: - pass # same leaf value - elif str(a[key]) in str(b[key]): - # strで上書き。JSONだったのをstrに変換したデータ - a[key] = b[key] - else: - # conflict and override original value with new one - a[key] = b[key] - else: - a[key] = b[key] - return a -