elastic · leehinman · Nov 19, 2021 · Nov 11, 2021
diff --git a/packages/system/changelog.yml b/packages/system/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.6.3"
+  changes:
+    - description: Fix AccessList and AccessMask processing in security data_stream
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2156
 - version: "1.6.2"
   changes:
     - description: Fix missing null check in security pipeline

diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4663.json b/packages/system/data_stream/security/_dev/test/pipeline/test-4663.json
@@ -0,0 +1,74 @@
+{
+    "events": [
+        {
+            "@timestamp": "2021-11-11T04:51:32.660Z",
+            "ecs": {
+                "version": "1.11.0"
+            },
+            "host": {
+                "name": "DC01.contoso.local"
+            },
+            "agent": {
+                "version": "7.15.2",
+                "hostname": "hostname",
+                "ephemeral_id": "1e53eccd-9d5b-4001-9e6b-13b66625bb16",
+                "id": "7d1ef343-9372-428d-bd10-0a78e6894797",
+                "name": "AgentName",
+                "type": "filebeat"
+            },
+            "winlog": {
+                "event_id": "4663",
+                "opcode": "Info",
+                "time_created": "2015-09-18T22:13:54.770Z",
+                "level": "information",
+                "process": {
+                    "pid": 516,
+                    "thread": {
+                        "id": 524
+                    }
+                },
+                "keywords": [
+                    "Audit Success"
+                ],
+                "outcome": "success",
+                "event_data": {
+                    "AccessMask": "0x6",
+                    "ProcessName": "C:\\\\Windows\\\\System32\\\\notepad.exe",
+                    "SubjectDomainName": "CONTOSO",
+                    "SubjectLogonId": "0x4367b",
+                    "ObjectType": "File",
+                    "ObjectName": "C:\\\\Documents\\\\HBI Data.txt",
+                    "AccessList": "%%4417 %%4418",
+                    "ProcessId": "0x458",
+                    "ResourceAttributes": "S:AI(RA;ID;;;;WD;(\"Impact\\_MS\",TI,0x10020,3000))",
+                    "SubjectUserSid": "S-1-5-21-3457937927-2839227994-823803824-1104",
+                    "SubjectUserName": "dadmin",
+                    "ObjectServer": "Security",
+                    "HandleId": "0x1bc"
+                },
+                "computer_name": "DC01.contoso.local",
+                "provider_guid": "{54849625-5478-4994-A5BA-3E3B0328C30D}",
+                "version": 1,
+                "channel": "Security",
+                "provider_name": "Microsoft-Windows-Security-Auditing",
+                "record_id": 273866
+            },
+            "event": {
+                "code": "4663",
+                "kind": "event",
+                "provider": "Microsoft-Windows-Security-Auditing",
+                "outcome": "success"
+            },
+            "log": {
+                "file": {
+                    "path": "/file/path/4663.xml"
+                },
+                "level": "information"
+            },
+            "message": "\u003cEvent xmlns=\"http://schemas.microsoft.com/win/2004/08/events/event\"\u003e\u003cSystem\u003e \u003cProvider Name=\"Microsoft-Windows-Security-Auditing\" Guid=\"{54849625-5478-4994-A5BA-3E3B0328C30D}\" /\u003e\u003cEventID\u003e4663\u003c/EventID\u003e\u003cVersion\u003e1\u003c/Version\u003e\u003cLevel\u003e0\u003c/Level\u003e\u003cTask\u003e12800\u003c/Task\u003e\u003cOpcode\u003e0\u003c/Opcode\u003e\u003cKeywords\u003e0x8020000000000000\u003c/Keywords\u003e\u003cTimeCreated SystemTime=\"2015-09-18T22:13:54.770429700Z\" /\u003e\u003cEventRecordID\u003e273866\u003c/EventRecordID\u003e\u003cCorrelation /\u003e\u003cExecution ProcessID=\"516\" ThreadID=\"524\" /\u003e\u003cChannel\u003eSecurity\u003c/Channel\u003e\u003cComputer\u003eDC01.contoso.local\u003c/Computer\u003e\u003cSecurity /\u003e\u003c/System\u003e\u003cEventData\u003e\u003cData Name=\"SubjectUserSid\"\u003eS-1-5-21-3457937927-2839227994-823803824-1104\u003c/Data\u003e\u003cData Name=\"SubjectUserName\"\u003edadmin\u003c/Data\u003e\u003cData Name=\"SubjectDomainName\"\u003eCONTOSO\u003c/Data\u003e\u003cData Name=\"SubjectLogonId\"\u003e0x4367b\u003c/Data\u003e\u003cData Name=\"ObjectServer\"\u003eSecurity\u003c/Data\u003e\u003cData Name=\"ObjectType\"\u003eFile\u003c/Data\u003e\u003cData Name=\"ObjectName\"\u003eC:\\\\Documents\\\\HBI Data.txt\u003c/Data\u003e\u003cData Name=\"HandleId\"\u003e0x1bc\u003c/Data\u003e\u003cData Name=\"AccessList\"\u003e%%4417 %%4418\u003c/Data\u003e\u003cData Name=\"AccessMask\"\u003e0x6\u003c/Data\u003e\u003cData Name=\"ProcessId\"\u003e0x458\u003c/Data\u003e\u003cData Name=\"ProcessName\"\u003eC:\\\\Windows\\\\System32\\\\notepad.exe\u003c/Data\u003e\u003cData Name=\"ResourceAttributes\"\u003eS:AI(RA;ID;;;;WD;(\"Impact\\_MS\",TI,0x10020,3000))\u003c/Data\u003e\u003c/EventData\u003e\u003c/Event\u003e",
+            "input": {
+                "type": "log"
+            }
+        }
+    ]
+}
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4663.json-expected.json b/packages/system/data_stream/security/_dev/test/pipeline/test-4663.json-expected.json
@@ -0,0 +1,86 @@
+{
+    "expected": [
+        {
+            "input": {
+                "type": "log"
+            },
+            "agent": {
+                "name": "AgentName",
+                "hostname": "hostname",
+                "id": "7d1ef343-9372-428d-bd10-0a78e6894797",
+                "ephemeral_id": "1e53eccd-9d5b-4001-9e6b-13b66625bb16",
+                "type": "filebeat",
+                "version": "7.15.2"
+            },
+            "@timestamp": "2015-09-18T22:13:54.770Z",
+            "winlog": {
+                "computer_name": "DC01.contoso.local",
+                "process": {
+                    "pid": 516,
+                    "thread": {
+                        "id": 524
+                    }
+                },
+                "keywords": [
+                    "Audit Success"
+                ],
+                "level": "information",
+                "logon": {
+                    "id": "0x4367b"
+                },
+                "channel": "Security",
+                "event_data": {
+                    "ProcessName": "C:\\\\Windows\\\\System32\\\\notepad.exe",
+                    "SubjectLogonId": "0x4367b",
+                    "AccessMask": "0x6",
+                    "ResourceAttributes": "S:AI(RA;ID;;;;WD;(\"Impact\\_MS\",TI,0x10020,3000))",
+                    "ObjectName": "C:\\\\Documents\\\\HBI Data.txt",
+                    "ObjectType": "File",
+                    "SubjectUserName": "dadmin",
+                    "AccessListDescription": [
+                        "WriteData (or AddFile)",
+                        "AppendData (or AddSubdirectory or CreatePipeInstance)"
+                    ],
+                    "ObjectServer": "Security",
+                    "HandleId": "0x1bc",
+                    "SubjectDomainName": "CONTOSO",
+                    "ProcessId": "0x458",
+                    "AccessMaskDescription": [
+                        "Delete Child",
+                        "List Contents"
+                    ],
+                    "AccessList": "%%4417 %%4418",
+                    "SubjectUserSid": "S-1-5-21-3457937927-2839227994-823803824-1104"
+                },
+                "opcode": "Info",
+                "version": 1,
+                "record_id": "273866",
+                "event_id": "4663",
+                "provider_guid": "{54849625-5478-4994-A5BA-3E3B0328C30D}",
+                "time_created": "2015-09-18T22:13:54.770Z",
+                "provider_name": "Microsoft-Windows-Security-Auditing",
+                "outcome": "success"
+            },
+            "ecs": {
+                "version": "1.12.0"
+            },
+            "log": {
+                "level": "information",
+                "file": {
+                    "path": "/file/path/4663.xml"
+                }
+            },
+            "host": {
+                "name": "DC01.contoso.local"
+            },
+            "event": {
+                "ingested": "2021-11-11T21:31:58.908808600Z",
+                "code": "4663",
+                "provider": "Microsoft-Windows-Security-Auditing",
+                "kind": "event",
+                "outcome": "success"
+            },
+            "message": "\u003cEvent xmlns=\"http://schemas.microsoft.com/win/2004/08/events/event\"\u003e\u003cSystem\u003e \u003cProvider Name=\"Microsoft-Windows-Security-Auditing\" Guid=\"{54849625-5478-4994-A5BA-3E3B0328C30D}\" /\u003e\u003cEventID\u003e4663\u003c/EventID\u003e\u003cVersion\u003e1\u003c/Version\u003e\u003cLevel\u003e0\u003c/Level\u003e\u003cTask\u003e12800\u003c/Task\u003e\u003cOpcode\u003e0\u003c/Opcode\u003e\u003cKeywords\u003e0x8020000000000000\u003c/Keywords\u003e\u003cTimeCreated SystemTime=\"2015-09-18T22:13:54.770429700Z\" /\u003e\u003cEventRecordID\u003e273866\u003c/EventRecordID\u003e\u003cCorrelation /\u003e\u003cExecution ProcessID=\"516\" ThreadID=\"524\" /\u003e\u003cChannel\u003eSecurity\u003c/Channel\u003e\u003cComputer\u003eDC01.contoso.local\u003c/Computer\u003e\u003cSecurity /\u003e\u003c/System\u003e\u003cEventData\u003e\u003cData Name=\"SubjectUserSid\"\u003eS-1-5-21-3457937927-2839227994-823803824-1104\u003c/Data\u003e\u003cData Name=\"SubjectUserName\"\u003edadmin\u003c/Data\u003e\u003cData Name=\"SubjectDomainName\"\u003eCONTOSO\u003c/Data\u003e\u003cData Name=\"SubjectLogonId\"\u003e0x4367b\u003c/Data\u003e\u003cData Name=\"ObjectServer\"\u003eSecurity\u003c/Data\u003e\u003cData Name=\"ObjectType\"\u003eFile\u003c/Data\u003e\u003cData Name=\"ObjectName\"\u003eC:\\\\Documents\\\\HBI Data.txt\u003c/Data\u003e\u003cData Name=\"HandleId\"\u003e0x1bc\u003c/Data\u003e\u003cData Name=\"AccessList\"\u003e%%4417 %%4418\u003c/Data\u003e\u003cData Name=\"AccessMask\"\u003e0x6\u003c/Data\u003e\u003cData Name=\"ProcessId\"\u003e0x458\u003c/Data\u003e\u003cData Name=\"ProcessName\"\u003eC:\\\\Windows\\\\System32\\\\notepad.exe\u003c/Data\u003e\u003cData Name=\"ResourceAttributes\"\u003eS:AI(RA;ID;;;;WD;(\"Impact\\_MS\",TI,0x10020,3000))\u003c/Data\u003e\u003c/EventData\u003e\u003c/Event\u003e"
+        }
+    ]
+}
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4674.json b/packages/system/data_stream/security/_dev/test/pipeline/test-4674.json
@@ -0,0 +1,72 @@
+{
+    "events": [
+        {
+            "@timestamp": "2021-11-11T17:14:52.001Z",
+            "agent": {
+                "name": "AgentName",
+                "type": "filebeat",
+                "version": "7.15.2",
+                "hostname": "hostname",
+                "ephemeral_id": "8c285603-b2ba-4891-8f1a-862ca3388614",
+                "id": "7d1ef343-9372-428d-bd10-0a78e6894797"
+            },
+            "winlog": {
+                "time_created": "2015-10-09T00:22:36.237Z",
+                "event_id": "4674",
+                "provider_name": "Microsoft-Windows-Security-Auditing",
+                "keywords": [
+                    "Audit Failure"
+                ],
+                "opcode": "Info",
+                "outcome": "failure",
+                "level": "information",
+                "event_data": {
+                    "ProcessId": "0x1f0",
+                    "SubjectDomainName": "NT AUTHORITY",
+                    "SubjectLogonId": "0x3e5",
+                    "ObjectType": "-",
+                    "ObjectName": "-",
+                    "AccessMask": "16777216",
+                    "PrivilegeList": "SeSecurityPrivilege",
+                    "ProcessName": "C:\\\\Windows\\\\System32\\\\lsass.exe",
+                    "SubjectUserSid": "S-1-5-19",
+                    "SubjectUserName": "LOCAL SERVICE",
+                    "ObjectServer": "LSA",
+                    "HandleId": "0x0"
+                },
+                "process": {
+                    "pid": 496,
+                    "thread": {
+                        "id": 504
+                    }
+                },
+                "channel": "Security",
+                "record_id": 1099680,
+                "computer_name": "DC01.contoso.local",
+                "provider_guid": "{54849625-5478-4994-A5BA-3E3B0328C30D}"
+            },
+            "event": {
+                "code": "4674",
+                "kind": "event",
+                "provider": "Microsoft-Windows-Security-Auditing",
+                "outcome": "failure"
+            },
+            "log": {
+                "file": {
+                    "path": "/file/path/4674.xml"
+                },
+                "level": "information"
+            },
+            "message": "\u003cEvent xmlns=\"http://schemas.microsoft.com/win/2004/08/events/event\"\u003e\u003cSystem\u003e\u003cProvider Name=\"Microsoft-Windows-Security-Auditing\" Guid=\"{54849625-5478-4994-A5BA-3E3B0328C30D}\" /\u003e\u003cEventID\u003e4674\u003c/EventID\u003e\u003cVersion\u003e0\u003c/Version\u003e\u003cLevel\u003e0\u003c/Level\u003e\u003cTask\u003e13056\u003c/Task\u003e\u003cOpcode\u003e0\u003c/Opcode\u003e\u003cKeywords\u003e0x8010000000000000\u003c/Keywords\u003e\u003cTimeCreated SystemTime=\"2015-10-09T00:22:36.237816000Z\" /\u003e\u003cEventRecordID\u003e1099680\u003c/EventRecordID\u003e\u003cCorrelation /\u003e\u003cExecution ProcessID=\"496\" ThreadID=\"504\" /\u003e\u003cChannel\u003eSecurity\u003c/Channel\u003e\u003cComputer\u003eDC01.contoso.local\u003c/Computer\u003e\u003cSecurity /\u003e\u003c/System\u003e\u003cEventData\u003e\u003cData Name=\"SubjectUserSid\"\u003eS-1-5-19\u003c/Data\u003e\u003cData Name=\"SubjectUserName\"\u003eLOCAL SERVICE\u003c/Data\u003e\u003cData Name=\"SubjectDomainName\"\u003eNT AUTHORITY\u003c/Data\u003e\u003cData Name=\"SubjectLogonId\"\u003e0x3e5\u003c/Data\u003e\u003cData Name=\"ObjectServer\"\u003eLSA\u003c/Data\u003e\u003cData Name=\"ObjectType\"\u003e-\u003c/Data\u003e\u003cData Name=\"ObjectName\"\u003e-\u003c/Data\u003e\u003cData Name=\"HandleId\"\u003e0x0\u003c/Data\u003e\u003cData Name=\"AccessMask\"\u003e16777216\u003c/Data\u003e\u003cData Name=\"PrivilegeList\"\u003eSeSecurityPrivilege\u003c/Data\u003e\u003cData Name=\"ProcessId\"\u003e0x1f0\u003c/Data\u003e\u003cData Name=\"ProcessName\"\u003eC:\\\\Windows\\\\System32\\\\lsass.exe\u003c/Data\u003e\u003c/EventData\u003e\u003c/Event\u003e",
+            "input": {
+                "type": "log"
+            },
+            "ecs": {
+                "version": "1.11.0"
+            },
+            "host": {
+                "name": "DC01.contoso.local"
+            }
+        }
+    ]
+}
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4674.json-expected.json b/packages/system/data_stream/security/_dev/test/pipeline/test-4674.json-expected.json
@@ -0,0 +1,101 @@
+{
+    "expected": [
+        {
+            "agent": {
+                "name": "AgentName",
+                "hostname": "hostname",
+                "id": "7d1ef343-9372-428d-bd10-0a78e6894797",
+                "ephemeral_id": "8c285603-b2ba-4891-8f1a-862ca3388614",
+                "type": "filebeat",
+                "version": "7.15.2"
+            },
+            "process": {
+                "name": "lsass.exe",
+                "pid": 496,
+                "executable": "C:\\\\Windows\\\\System32\\\\lsass.exe"
+            },
+            "winlog": {
+                "computer_name": "DC01.contoso.local",
+                "process": {
+                    "pid": 496,
+                    "thread": {
+                        "id": 504
+                    }
+                },
+                "keywords": [
+                    "Audit Failure"
+                ],
+                "level": "information",
+                "logon": {
+                    "id": "0x3e5"
+                },
+                "channel": "Security",
+                "event_data": {
+                    "ObjectType": "-",
+                    "SubjectUserName": "LOCAL SERVICE",
+                    "ObjectServer": "LSA",
+                    "HandleId": "0x0",
+                    "SubjectDomainName": "NT AUTHORITY",
+                    "SubjectLogonId": "0x3e5",
+                    "AccessMaskDescription": [
+                        "ADS_RIGHT_ACCESS_SYSTEM_SECURITY"
+                    ],
+                    "AccessMask": "16777216",
+                    "PrivilegeList": [
+                        "SeSecurityPrivilege"
+                    ],
+                    "SubjectUserSid": "S-1-5-19",
+                    "ObjectName": "-"
+                },
+                "opcode": "Info",
+                "record_id": "1099680",
+                "event_id": "4674",
+                "provider_guid": "{54849625-5478-4994-A5BA-3E3B0328C30D}",
+                "time_created": "2015-10-09T00:22:36.237Z",
+                "provider_name": "Microsoft-Windows-Security-Auditing",
+                "outcome": "failure"
+            },
+            "log": {
+                "level": "information",
+                "file": {
+                    "path": "/file/path/4674.xml"
+                }
+            },
+            "message": "\u003cEvent xmlns=\"http://schemas.microsoft.com/win/2004/08/events/event\"\u003e\u003cSystem\u003e\u003cProvider Name=\"Microsoft-Windows-Security-Auditing\" Guid=\"{54849625-5478-4994-A5BA-3E3B0328C30D}\" /\u003e\u003cEventID\u003e4674\u003c/EventID\u003e\u003cVersion\u003e0\u003c/Version\u003e\u003cLevel\u003e0\u003c/Level\u003e\u003cTask\u003e13056\u003c/Task\u003e\u003cOpcode\u003e0\u003c/Opcode\u003e\u003cKeywords\u003e0x8010000000000000\u003c/Keywords\u003e\u003cTimeCreated SystemTime=\"2015-10-09T00:22:36.237816000Z\" /\u003e\u003cEventRecordID\u003e1099680\u003c/EventRecordID\u003e\u003cCorrelation /\u003e\u003cExecution ProcessID=\"496\" ThreadID=\"504\" /\u003e\u003cChannel\u003eSecurity\u003c/Channel\u003e\u003cComputer\u003eDC01.contoso.local\u003c/Computer\u003e\u003cSecurity /\u003e\u003c/System\u003e\u003cEventData\u003e\u003cData Name=\"SubjectUserSid\"\u003eS-1-5-19\u003c/Data\u003e\u003cData Name=\"SubjectUserName\"\u003eLOCAL SERVICE\u003c/Data\u003e\u003cData Name=\"SubjectDomainName\"\u003eNT AUTHORITY\u003c/Data\u003e\u003cData Name=\"SubjectLogonId\"\u003e0x3e5\u003c/Data\u003e\u003cData Name=\"ObjectServer\"\u003eLSA\u003c/Data\u003e\u003cData Name=\"ObjectType\"\u003e-\u003c/Data\u003e\u003cData Name=\"ObjectName\"\u003e-\u003c/Data\u003e\u003cData Name=\"HandleId\"\u003e0x0\u003c/Data\u003e\u003cData Name=\"AccessMask\"\u003e16777216\u003c/Data\u003e\u003cData Name=\"PrivilegeList\"\u003eSeSecurityPrivilege\u003c/Data\u003e\u003cData Name=\"ProcessId\"\u003e0x1f0\u003c/Data\u003e\u003cData Name=\"ProcessName\"\u003eC:\\\\Windows\\\\System32\\\\lsass.exe\u003c/Data\u003e\u003c/EventData\u003e\u003c/Event\u003e",
+            "input": {
+                "type": "log"
+            },
+            "@timestamp": "2015-10-09T00:22:36.237Z",
+            "ecs": {
+                "version": "1.12.0"
+            },
+            "related": {
+                "user": [
+                    "LOCAL SERVICE"
+                ]
+            },
+            "host": {
+                "name": "DC01.contoso.local"
+            },
+            "event": {
+                "ingested": "2021-11-11T21:31:59.255100300Z",
+                "code": "4674",
+                "provider": "Microsoft-Windows-Security-Auditing",
+                "kind": "event",
+                "action": "privileged-operation",
+                "category": [
+                    "iam"
+                ],
+                "type": [
+                    "admin"
+                ],
+                "outcome": "failure"
+            },
+            "user": {
+                "name": "LOCAL SERVICE",
+                "domain": "NT AUTHORITY",
+                "id": "S-1-5-19"
+            }
+        }
+    ]
+}