From 959b6bd03854acee4b85093c4a3c2746742d4dbb Mon Sep 17 00:00:00 2001 From: Derek Ditch Date: Thu, 17 Dec 2020 13:43:55 -0600 Subject: [PATCH] Adjust fields.yml to implement nested types Since #23183 was merged, `fields.yml` can now properly specify types for nested object properties --- x-pack/filebeat/module/virustotal/fields.go | 2 +- .../virustotal/livehunt/_meta/fields.yml | 484 +++++++++--------- .../livehunt/config/cleanup-empty.js | 4 +- .../virustotal/livehunt/config/livehunt.yml | 21 +- .../livehunt/config/virustotal-pe.js | 15 +- 5 files changed, 255 insertions(+), 271 deletions(-) diff --git a/x-pack/filebeat/module/virustotal/fields.go b/x-pack/filebeat/module/virustotal/fields.go index 0ccd9dd6c9e..b4c0b05f944 100644 --- a/x-pack/filebeat/module/virustotal/fields.go +++ b/x-pack/filebeat/module/virustotal/fields.go @@ -19,5 +19,5 @@ func init() { // AssetVirustotal returns asset data. // This is the base64 encoded gzipped contents of module/virustotal. func AssetVirustotal() string { - return "eJzsXd1v3DiSf89fUUgeJgN4fLcL7IsfDvDEmRtjk5kgzsztPQlsqbrFa4nUkFTbnb/+wC+1Wk1J1IeTYBE/THbdZvFXxWKxvsh+JSssijTHdI/iBjIqyabAFz/BHo83cKCiloorUrwAUFQVeAN/6t99cr/LUKaCVopydgP/9QIA4D3P6gJhywXkhGUFZTtQObbGwe2He2Bc0S1NiR4qXwBsKRaZvHlhaOifVxCG5j//CRgpsYPR/wRw+Z9fzESwFbzs4ir4Tl6fUdmSulCJwXYDW1JIbH0ssEAi8QY2qEjr9+pY4Q3sBK+r1m89g20sngfCSHGUVJ59OMiF/rl1o4CyLRelESWQDa+VYUySsirw+oLmIE9DfPXz1sdfm8eMqO5EETzqnzuiEAjLQNHSswWPRFqxfcYMNsfWMnY5PsHuhTAqkhMbUhHVXadoIlHsPtRlScQR+NYso0BZF0pa+QYGDK0WDK4YDKxam+UtoUUtQtyc6FOmcIei528iuNY/v9XlBoVm/PZPQLajDCWonCgDAR5zZH6rGKOiP6BFGJgHX7MMFaYKQ7J7NvwCKy6UBEmODVQqx8B4zClnWypKzH7SGs9r9TVFL5CkORBwUFqr8Dl+FXIiygJlaOM8GyM9azAIxeMtSUFTyutvAfAwlsYu1bL6ZiCPgPGYv4J2uynlpRpTCX98fPcfo6qsYf1UM1lXmvUva1a6ezPj7AcFDor9lZ5X/+kFH42voZSgm1phd20s4m1BlEJ2wdfQMRNx9o2w6pwygaoWrHugW8aIQO00whEVlKSqMNPS5UD0kcwyIjKQaY4lCXKd8rKsGVXHaH/meRl+4/FAiYpkRJFRz23MwTI+cHLglys7zOgYs5EMRzB9zrhBOsr1EOfwVU4Yq5PMbspGr/Se8xgsayFOvsoJ0w+4AdGH2KMVWNXKBBgzfdqUC4SUFGldEIWZDb9IUTgtqLhUdtt3QrK+fQstg8XJ5RHSAK+D5jwC8v/efryFkqg0R2miWY2s53RY7mPr/y5bZQs1MxwPkbNY93h85GLYDW1NvCK0caoWocKnsGdw2kEqzRPKEllvek/taIyPOaoc7SobyibCpEyfL5Y+cKEPoEHQG84LJGHG2iopUSW0z3WIRHzaJT9ITxTu7xYvuyO13pK34K2ilwK3KJClC1fcY5K8Fr20+lE1O7eVyFrdTThb45QzvSnOZgRFdhd2+1tL4HyipfFJC3pAyGt2zsLEfE0T8DBaVUE1jUA0LldHPniKDxioxhkju34vbEidZL0pqZQhZRrk6n3XjTRhWA+teQoSoc5jOmT32vUO+XXKa6bEMaGSJynPZqrW/cPv4CiBpqLVzE5i/lefAGDU4HQQ7zHkgUQA/IPRv2oEmiHT2mXjuFURbqmQKjGEVDgejc2xmpjYuznm/DO0QSIyfRSepf7DYAe3bEGWA33PpQKBKTJlJjvH6/xIxRdjPS1MYvRrHtrzxINsxTmQEwkbLdhGIJGw+2KCU8ZTa1xidSycqo7VWkfDBuA5OWAb7ImV+bibs4U/soKT7KzCE4X2kzF0FgfV4aTFs8GGpo82+isEQ77bqNE7U+6SZ3qTd3V7hIfzzcc3/4epAqokFluj1Zo0eNJh8AFdjkZOWCb4riYXNma5//KQ88dQfepWT0kzuP3wzyu4e/svU965/df7d2YryyvAJyVIqvXskarcyOW2gQmK866ePY/PQ1JFD1TRufvoDWeKUGY3PqmqH6QneTQzTPMR2sgOKHpOjQhYRpT/bUTp6EAtgxY5Dgyxy5mQqirCrnAkMEfIGnSTyqTMGw6wSjTbLgZAJigED2VXIqD+zylgFL4cCYYeVIKnKKWvfLcm7I8hh+PHIR707lqgBl1pC7Rm2IgbXr8k1f7lFbzM8En/Q57K4uWPixXFad0Cx89j9/qrKV0Z8KfSfkkY3aLsF/dUuD3h6wy4mtLqcAu6EUTMNlbvzHBnmIxB0PrgNHg2qJJQlnijN9PnI5Sd283VRVdSRsu6TGS2TxaZ1veWEDT1IZDZfsBax8GrSLonu5m698EOfh7BVYIfaIZips598MOtvrkE8BJ907EBPcwG9NEPXw8QlfuEskzb67mHzcmPeOR6QnkDxjDDa6lEnapa4I/Gh3pZoShfwmv9jw1f5I8GATQI+n2OvtpfKypCcaCzo4oHN3o10UolKNvJpOVgLpSvdiIESqXPbkcdtrxmq5hCRcQO1XIb0z1MHK5TSKnQhMHboLpF+pcq8afffAfjXkuTkaJB6nqaOq58ow9/fhrQzaEyT4rC5e7mns9V1SYCGSpCC3kNb3JM9/Dw8A7etD52MZrisGf8EUou0EU2VElo9uSCnaYVkenQq1CzbVkg+rBkoZ9sLL6TgUmcrGYmnuzgVh3YAhX4V00FZq2J5DX8E4/SdAKcfuvcFW38DqSo0X6e0VTParwhk+AllHl/fMuLgj/q/2fDv4lSaHSOVGRDi1B0OML3OyqV654RKJEpougBTc5Yx7HE5YJ8ZusHeTZXOAEwlFPeHBXmKHhoJw9ZgxE27lthvXMDzC7++ajwVxQccqwFlYqmYNve9B/a/pXo4N2zgE90G4j6l2co3jrCk5ponynZICWWm+K47HBoGPLkFjufaU5MLkYk4bLgFFANLQjTikTEM0wk/TzT2J/Q8Ayhh04skrIi7LggMmyBMaT6aqRxeNDUVCpO52asGziGEvRRii1NFJgMF/ynQDJ5imFyE3BtC7KTSUnkfg1Yhhr0UJuAis88R8/RBKlMQLHC3jI4Fu0ti6Te6L9eBUwvqQl41gKzDpIEnxSyFQ6LUwpwiOIEdOucYQbW0vOrjSexbXArwoJeinHoaEl2a4nL0FosL4vISK05ro0bt9Q0+csP0CGrfWF8wrRWgfrfBNyMKkoK+hmzJCOKrGHFWjTBtFcssmjUxcNruAue1jJ/oSBsV+vlnp+JbxB5WtBDKxIR7kiRpLw6CrrLl3oxhhoMUZuCSgmSYUnEXv5tFVwNvRC5ecD+vi6wELnYYgTbo1jJsFliiy1bSctVTnBNZ9kJXpI0p2wdMJbUMjxcrrRSXC5eJZtcS9Zyt1yubrnXxQXdUW29DbIVTLgnaLEtsuPVKqKqFkqoEjyrU7WGbByphVJxeNZRbQ9pcTHvHNU6LmkH3FKvVNYbeZQKy4WwhuhMRLLSKjb0Fq+jaelTpAwVhKYgGqITh6Rmz+H7nlFdwftNBRKFyfw+8lOqzFDq60ifgGZuPfYcyYIymxufBHLbM6GEmuPi8WQ8rUtkPbdTJgV7jhIEKUX7kEiEVr/l/mM/oUiPjWd0e1xDey2lZdpbZdu1TrVsu0J7iokgZ7dJn8DY8DFMaMrputJ5uuAEfSors6v3dKlQnsoK+ilF2pmqTohIc6qwrzI9ydZUNYyQi8e1OSpMuMgWr5lGpYlBH7F4TGsos4azTJc1khXceg1kkV+vcayTiNdQlubhfXxYhO9YzYkNe0iNl9ZzIvPrg/7viy6KVe/+VYJXgqIi4gjb+vPno5mZsl1n2Hhleoibi6N6CsTWFSd/XTpcPR/HUZIdTadBuYVdjVL6DqMm2r+CDZGYAWdAoOJVXRABFRG2O9sk8wUvta/7NBmmpLuSJD2Pdw030ow+D+E6d1hzgcjfqDWT2surtssANkfbdYQHFFQdr7TYEYjs4EvCT1dtagWyKqjSdEJ3f8PcBqkN3R2M41ev3SDPV5a87TRpsz31uYxUUEXTBfescrrrbvr4wSVmtO4PiseGF/xx0tiBa6nzepVOr8gASVMuMrud+u+Bje8mJQLmZ6kR/STu78y9MKgVLag66gF0x2yHlTNZx5O5OD0MSAVsKNP2Vv890V6OvIZ7BSU5wk7LwGgeKU7NTfLqYn7jiVhl1fqCOjzTq4fmlsjGNnUd/YM97opLY0rDF6Sfp/soOt167+18tkJOtSWEeUf5h5YU9bGjjbykRkfPpRgYbaVtehlTHHDTLntP249BDTS4jN37Mx9v3L3QPmFG7ByNQpGuLzC2NWy7odvWTXPkMhR93QOj7z8Vrc4De9805WXJWXE8YVwqJ3dTGpEllCUqx+SRXrypGHNX0+BrvVJjr0nzjURxMFdjqXR9ONa/dD3atmSL6kpj1WdPcdSG4Y+P76JYaVqzmxdPr/1pfO1eiXwRxUdz2pqn/WxHpmm51TS0IrQeErAu3IsuNmY6zGdCCxupoQdSS1Q5n7hS782YU295a7m0nfDwJmtRShTuuOjaqzEPx43yDupc19hKc9rcH80YP3NLEPOFYJUmVOgZwfIbsW+BaCSWiF2hZWvi4ITzWGNBzPndhNVB1VXwcvaw10WkAjtwHq6zKQO7syLp/vwCQHBfDwL1rqGl1erx1mGV7ci3fpTV2es5eLpG4jngDJsePXqJnjf3WxR3mDxC/dvpWjUfCj05bhbBa/lj3PxtH/HapDykzBDPF2bQVR+A+EuTzjgL3msTodt52iiHX90eR66Ks1zNM+HWs8xH/erVK3j77hdLsHGLpP59mDMstqvwpCfFg3G1Om9mmLtcRUF3yFKEd9q//rVmyj9P3eaVH1A8CqpMTFdf+g7dPd0nm6FdaYpd5vLRjIcvRndM6/WJHJmOKCQ1V+L9tVl3Lce/3XkN9/Z2JbqHZamCRxLKtVB9DptXGCtaYGYCypQwIIU0j5Zsyd66KyUpHolAX9WLMfNt+eRILlPgywXzqyF7dlfG6b5X1y8TrKYFCT71GREwOhYMhTHwEF83JT3vpsa8vETANL620dj1m4/HNYXNg/TedZS1KzxrguMyIRs6D9tvb9/ewaff4fYu9NJhZOPG7DqGUx7lHlleR3cW1XE73usay0M2dKC2PExjLubb1hspP9u0m7k8vCUpwuvbn+8vPJXOnae+i0o+x2Mvc/fyU/Dgh/EqcQIxrhidQtazyvrlfz797aUJGJpuQw+sc6Y0DqZJvyeV4DtBysRq08Ss8endMz3ZB0vKiarnKLtYgQ4eab35NfA8uMBgHh6ZE4FZ0vfYSmQ+3VI5PdnSehOBSoPSasfk+MA6uVhsg0XJQWx+lHVqtc70qEpUsq238rX4sXhPeR13FZ7NS5G4M71UBQ0+sBJTqG70wBMDTazftl9KvRsnXOOT+c6GVeIFr82WJmaABRqUp0v5pqoSHQn15BUHwE9PJs6PoBs25bHcXHTZDVnr6Mf4Ar7JWB7fOSPPBO5iCZwhXkeBvCkORBSu+na6GHZxmi5TpQAfX0KXnHfjGdcEphvXKj9KmpIi4dvt5UX6aQgsCeAMMir3C7AEmoOnIdEEgDLTwxVuLQi4ad7BuxzUSjOqmhQJyTJx+Q0R0yA6WuBoXTQdDHmbI36Fh7lYjB7jmTj7kE6Qdetkn9pF0EFoKEzXtOm2sTOvJjB9Wn+Mr7LXtXVrvUfjaM8pLijBq4lFqA6eEomshT2w+RYecsKYeaLFkA5j6q+Fpznt3hZcBOdNTkH+VRs3OaPS9psE2iFCuC6MPS3Xd3oszS/g9FyC/5JOT8Pmt+j0PBO4V69ewYe38fn3Cq/5AUVBjpMWd2DYlLWdvvPe5PTvIZ/L8Tx1688yRm/toBVh6DHTVekXNypq8sGu1ewf02b+3a49vL/7x1kdaTaCOd7g7879s1M7dZy8KP3ZksnezIN2W4bnnOi8dPbcor025/vx+jMI/aWyPhxtLBlu6nDWcq1va7vTM7SV4QroFlw7VmDEcNPzWRfLwNcTRKEfmyqKv/tGzVLfE9VAuwLJSzTMnx6EK8MG+Hs1crgaSWR+TWqVI1N0el7ytjVysY10zCZ9t25HO9XN8NOl3ShAvZIxr2PTlLPrgFhmt9P/SmSO9lmdcoNZhhm4dD7ouaaamQC0CBD6545u3Xd1GUyuQ3dHD8j6BAYRdY/hr6vsP49jaMfUR1vntSCPRqZ9FeDxpy2dQl6UNIad1yktbX4G/0hAuO4jaJonFboCS78+zkL0kab5h7euNNlxdsJw+l8yjPAr3cOFkW7EnGzNwAMuY32prfdaZlmzbgjRm8gcCT2+J0C/J0C/J0C/J0C/J0D/zROgFQaLvt9GVferg/teOv4ipeMKg0n4byPL/tXBfU/lPxe4iwXvfCX8gt6FClO6de/XEnb0FwAF6Ch3p9XBXtM31wV3yFAQhUAKFOcdSNNVoPc7xEdkfX92JbZ1014HIZroDJ9ksi/0iex2mDXpqta3LcxDMPJF8CNo5n7xu8Xlv7Dv/wMAAP//7arVZA==" + return "eJzsXd2T3DaOf/dfgbIfklRN5m63al/8cFUTj3OZWnvt8ji5vScVW0K3eE2RCknNTPuvv+KHPlpN6nucbK37IbG7LfAHEAQBEKBeqRIZS3NMjyhfQ0YV2TF88SMc8fQaHqislBaasBcAmmqGr+E3891n/12GKpW01FTw1/BfLwAA3ousYgh7ISEnPGOUH0Dn2HkObj7eARea7mlKzKPqBcCeIsvU6xeWhvm8gjC0+vcfgZMCexjrTwBX/fnZDgR7KYo+LiYO6vqMyp5UTCcW22vYE6aw87NEhkTha9ihJp3v9anE13CQoio739YMdrHUPBBO2ElRdfbjIBfmc+OfAsr3QhZWlEB2otKWMUWKkuH1Bc1Bnob4ivMW46/LY0Z0f6AJPJrPLdEIhGegaVGzBY9EObF9wQx2p8409jluYUchjIqkZUOiqpjuz1Q7BkelMQv8HJNPl3iBOhehhydKynzeWxpQqb5c7JKsNS0kpJaJI54ehQwDqbGmRONByNM6tG88FRD7AaWdj85N0zpsnyyNGllHkNsJEfmBckzMX9Zh/QcpsEbqiDoN2HbOPdwHlIoKvg7xb47IVwNdlZHlPwPzO6I0OELrcAeh1ICVJkEDM8lOTeLlvioK0i46b9OciQo8MLQhwOCmABMN355QVsnY7Dj6lGs8oFy5Tqpih9IwfvObnzsFOifaQoDHHHk9hdZvMT9QFgZWg694hhrTsNl/NvwSSyG1AkVODVSqxsA01lvwPZUFZj+aTVVUMVP5VUQvkaQ5EPBQOrPwZfos5EQWDFVo4TwbI5E5GITS7PWE0ZSK6s8AeBhLY5cqVf5pII+AqTH/Adrth1SXakwV/Prp3X+MqrKB9WPFVVUa1r+uWemvzUzw7zR4KO4rM675pxd8NOGM1pLuKo39uXGI94xojfyCr6FtZsLeN8Kqj/sk6kryC9/YMkYkmrgUTqihIGWJmZGuAGK2ZJ4RmYFKcyxIkOtUFEXFqe57xPHd8XkZflPjMWEFyYgmo8HhWAxnw+zkQVzO7DCjY8xOZHgC0+eMW6SjXA9xDn/IDuN0krtF2eiVWXM1Bsda2Mn8A3aYOOAGRAxxG7GVlbY5jIU+bSokQkpYWjGijTcuRQGEMa8FpTCBuVn2vWgutm6hY7AEudxCGuBV0JxPgPy/N59uoCA6zVHZ4MEgi+wO633s9UGmg5pZjofITY/NOgNvCG2cqkOo8SnsGbQrSKd5Qnmiql10156M8TFHnaObZUvZJrEoN/uLow9Cmg1oEPROCIYkzFhXJRXqhK7MJbWr5DtVE4W72/XZGUdquynvwNtELyXuUSJPV854jUmJSkZpTUdVkvSIcticR/OPM2C/o8qmvdx44AJKKjhoIZhyqdgcqfSJg1iqY8gowZlXIdhQCmwGcuimwoRgLi+jheeh5sh8G0MNk6YEphnWhdBphlzTPcXMI/5e/bAMb4Oxc9yyuad5ZiZSwY1dPRsRNDlcbP1/tmOGz9QJn9EHhLzi5yzMPFVoYmZOyzJo6SYgGperJx90BAf2uGblkUPckR9SJ1XtCqoCSeARrt73IxEbyUdoLVOQCeo8pkPOXF8fUFynouJanhKqRJKKbKFq3d1/AE8JDBWjZm4Q+6eYAGDUFPUQHzHkxE4A+Cunv1fY2h25OcI9lUonlpAOb1BTTwJtWqX2lK0LZWmDQuTGmzo7oA6DHVyyjKwH+l4oDRJT5NoOdo7XhyJarMbaTkxi9WsZ2vPcleqEypATBTsj2EYgE2HHwso2aW40LnE6Fj7tmKq1nobL4eTkAbtgW1aW4272FvHImSDZWR3CJLSfraFzOKgCQ8Dg2WFDsw5Y4+fYQ+7/qNE7U+5CZNa5mMfD+eITu/8zDhXVCtnearUhDTXpMPiALk9GTngmxaEiFzZmvf9yn4vHUBXFjRmSZnDz8e9XcPv2n9bzvfnn+3d2KasrwCctSWr07JHq3MrlpoFpHdCv4vOQVNMHqunSdfRGcE0odwuflOV3qiZ5siPM8xG6yOInxRNgWVH+txWlp2Md98VgiJvOhJQlC7vCE4F5Qs6g22w45bXhAKdEi+1iAGSCUopQgm4C1P9pcw6yLpoBSw9KKVJUqq7P6gwYT0MMpyCGeDCra4Ua9KUt0ZlhK274/iUpjy+v4GWGT+Z/5KlgL39YrShe61Y4fjX2Wn8NpSsLvi1AKwine1Rxcc+FG4lHF8A1lDaHy+hOErnYWL2zj3vD5OJ7ymsNXgyqIJQntdFb6PMRys/t5uaiKyinRVUkKjsOFOFMweoIQXPECCo7DljrafBKkh7JYaHufXQPP4/gSikeaBbOnU3BVj/elNWt1TcTG9CHxYA+1Y9vB4iqY0J5Zuz10s2m9SMehRlQvQZrmOF7pWWV6kriD9aHelmiLF7C9+Z/LnxRP1gE0CCI+xyx4+NOVITygS6OKu7905uJVmlJ+UElHQdzpXyNEyFRabN3e+qwFxXfxBRqIg+o19uY/mbicbUhpbaparODL/cvdVLvfssdjDsjTU5Yg9SXxfVc+bak9vOAbg6dFKYofe5u6f5cll0ikKEmlKlreJNjeoT7+3fwpvOzj9G0gCMXj1AIiT6yoVpBsyZXrDSjiNyEXkwvtmWB6MORhTjZqfhaA5N4WS1MPLmHO6UEDqjE3ysqMesMpK7h73hStpik/da7K8b4PRBWofs9o/ZcxXpDNsFLKK/98b1gTDyav7nwb6YU2jLtkuwoC0WHI3zXp0ASS4kKuSaaPqDNGZs4lvhcUJ3Z+k6djRVOAAzllHcnjTlKEVrJQ9ZghI27Tljv3QC7in86afwFpYAcK0mVpmnnoMuVQE0O3ptS4ye6D0T96zMUbz3hWa0ez5RsUAqLHTut2xwahmpyq53PNCc2FyOT8MnyHFANLQjTmohIZJgo+mWhsW/RiAwhQmcqkqIk/LQiMuyAsaRih57T8KA9UykFXZqxbuBYShCjNPVogmEyXDMyB5LNUwyTm4Frz8hBJQVRxy1gWWoQoTYDlVi4j56jCVKZgWKDtWVxrFpbDkm1M/96EzBRUjPwbAVmGyQJPmnkG2wWbQpwiOIMdNvsYRbW2v2riydxlZQbwoIoxWnoaEEOW4nL0lotL4fISq3Zrq0bt9Y01f0z0CNrfGF8wrTSgfO/Gbg51ZQw+gWzJCOabGHFOjTBllessmjUx8NbuAs1rXX+AiP8UJnpXp6JbxDVtCBCayIiPBCWpKI8SXrI13oxlhoMUZuDSkuSYUHkUf1lE1wNvRC5ZcD+ui2wELmphxH8iHIjw+aIrbZsBS022cENnXU7eEHSnPJtwDhS6/AItdFMCbV6llxyLdnK3fK5uvVel5D0QI31tsg2MOE1QYdtlR0vNxFVuVJCpRRZleotZONJrZSKx7ONateQVh/mnaPaxiXtgVvrlapqp05KY7ES1hCdmUg2msWG3up5tCV9mhShA6E5iIboTENS8efwfc+obuD9phKJxmR5HXmbKrOUYhXpM9AsPY89R7LimM0/nwRy2wuhhIrjpuPJRFoVyCMNTrOCPU8JgpQm+5BIpFG/9f5jnNBEj01kdH/aQnsdpXXaW2b7rXa1bL9BeYqNIBeXSbdgXPgYJjRnd91oP12xgz4VpV3VR7pWKE9FCXFKE+1MWSVEpjnVGDuZnmVrygpGyE3HtTtpTITMVs+ZQWWIQYzYdExbKLOBs06XDZIN3HoDZJVfb3Bsk4g3UNbm4ev4kIV7rJbEhhFS40frOVH59YP574s+ik17/0opSklRE3mCffXly8mOTPmh99j4yfQQNxdb9RyInRanuuM+fHo+jqMgB5rOg3IDhwqVqiuMmmj/CnZEYQaCA4FSlBUjEkoiXXW2TeZLURhf92k2TEUPBUkiV0wOF9KM3jDiK3d400BUN2XbQV3/s6sygN3JVR3hA0qqT1dG7AhE9fAl4dvPdpUGVTKqDZ1Q+3iY2yC1od7BafyauRvk+cqRd5UmXbbn3riSSqppuqLPKqeH/qKf/nCBGa3iQfHY40w8znp2oC11Wa1SexERkDQVMnPLKd4HNr6atAyYn7VG9LO8u7V9YVBpyqg+mQfogbsKK2+yTq25aK+vpRJ2lBt7a/49MV6OuoY7DQU5wcHIwGoeYW1xk7q6GN96Ik5Zjb6gCc/M7KHtEtm5oq5TfeeTb3FpTGm4Qfp5qo8mp1vv2hb69TnVjhCWbeUfO1K0t0RCKRS1OnouxcDTTtq2ljHFATftsva0e5/YQIHLWN+f/Xnn+0JjwpywcgwKTfq+wNjScOWGflk3xZHrUMSqB0avEGOdygPXb5qKohCcnVqMa+XkO6UReUJ5onNMHunFtZxTejUtvu79v7ZNWuwUygfbGkuVr8Nx/qWv0XZHtqivDFaz97CTMQy/fno3iZUXl/wwvLYeqFIZYnfFj1jOASZ/brzLM1+qsg6TG6drkIav6u6zcIlcszPX+Zlwm1GWo3716hW8ffezI9hoqTLfhzlDtt+EJzMoPljN77Uw29J6xugBeYrwzpi7Xyqu6wtnu7yKB5SPkmq7xVZ4wW1/s4jJZuiadZt7tLXgC/qQR9dcpxk4R24MvKK2Q7HuYvJV0vVtfNdw55pd0F8VSTU8kpDrS5kGYe9VKynDzO7vKeFAmLI95HtydJt3QdgjkVgnWSNF2BfcN74iksuMxHrB/GLJnpUue92v1fXr+A4pI8HL+ybs354FS2EMPExPY5PITYhTLsIgYOuQumjc/C3H48/ol0F67w/4uwm3LcEJlZAdXYbtH2/f3sLnD3BzG7q7bOI52uK0klce7a9N3UZ3VqXVe3evbzE9ZEcHUv3DNJZivum0rP/koiDby7UnKcL3Nz/dBS7NOitBj9WN1y63662L8sNE8MfpKtGCGFeMXl7xWWX98j+f/vLSBipN8UcNrLenNPeL2WxIUkpxkKRInDbNDOLba2jMYB8dKS+qyFZ2MQM9PMqFvFvgufetQcvwqJxIzJJY7/vE9Iaj0nbQd1pUqbIonXbMjjnwyV6yHXwueKPg2F67vLamvgPPQcIM1KnYLTgrnuS5bGHeP3u7/hUAt7XPoela76nVeuboYwbI0J6Ot92LNv002137g3So4eNfRYe+AuC2Sd5lAzdXotpQBvx9n6psq+gXev9fWZ28b1EzZggtd5TK/KRoSlgi9vvFnYo9RI4UCA4ZVccNsC2vy+ohM4SAcnvgHT+7jDhStRsWfrjxhKnUFWEJyTIZvp19PmxPEzzN4I3xK3zEPvTNxF3jPhP7EPoF89LkkJafbPdQW0rLtXa52e3hMISWw1B4sDVpm9oZYzk7FwP4MZaDNBGHKBeeYvTwFUhUJZ2DIPZwnxPObe+8HSKOceSSjJwu7A4Zhvcmp6B+r6wDnVHlDgYHL/aNH6i4nDCyfbCkYhBo/ZTLAZsQKxJZTToqiJ7br35bSk15m+wuPFtSr151jAavh5pSZtOETTUxMMSG9KIv9eYPr169go9vp6fgy/FE91BCe8mbdOKijqfgYzi6WDLcVeFsyFbvdbk1I3Tdyiuge/CnbjNmq49c4tAttJPQjw017R4i5yR/fFtbfNVCuwIlCrTMt/d+FEGl+HbKMXLKYYw3qXSOXNP5Bvym82Qd1/glP9+Ae2aTWHPFaEGSfbztzZgEKCoZewkiTQW/DohlcdXUL0Tl6Lqnix1mGWbg04RgxpprZgLQJoAwn1u692/1sJh8IcaBPiCPCQwmRNzDL7ZqZJv9bVE0P37ucvu35jxZkkcr09jJ0vgNRl4hL1KlQ0DHyi/PE/f1CHUvWDifLGmaJyX6xG1cHxch+kTT/ONbf+QB5ws5DCd+Yc3YxU3t/TQja3MwnhzJLg/06Y7ge99py11tzYwDwUj4/XfBlM1Y7c+H2w8zzcPy6OFNTv8aylmNWobhcGZVwPXWPfwMsMyzy+Pmn/3Tk8GMnjwHreMEJB+cyp1ZwU0QrcnOffDpOAfFr4pFkziYgFmeNbqnX6bM3cIk0UiG+avag29J329J329J3x76b0nfb0nfpfj+fZO+4RqJiSfqTXHAv/iJ+reqjPlVGd9KLgLw/jQK8syAX/QR914Xvbzs/r7ElO79xYSEn+rODglCAjkYHXL9l7YP5IAcJdEIhKE8P5yZf+4Qfb/wiNzvznqdOi2UJhAxROfnHOb3Cn4mhwNmoZfiL0Mw8pLoETRLXwrtcNVvYvr/AAAA//8g+jMq" } diff --git a/x-pack/filebeat/module/virustotal/livehunt/_meta/fields.yml b/x-pack/filebeat/module/virustotal/livehunt/_meta/fields.yml index ca7d64491ea..6beaaee7610 100644 --- a/x-pack/filebeat/module/virustotal/livehunt/_meta/fields.yml +++ b/x-pack/filebeat/module/virustotal/livehunt/_meta/fields.yml @@ -18,6 +18,33 @@ Date and time sample was analyzed by VirusTotal. type: date default_field: false + - name: results + type: nested + fields: + - name: method + description: > + Method used by VirusTotal for analysis. + type: keyword + - name: category + description: > + Category of the sample. + type: keyword + - name: result + description: > + Result of the VirusTotal analysis. + type: keyword + - name: engine_name + description: > + Name of the engine used for analysis. + type: keyword + - name: engine_version + description: > + Version of the engine used for analysis. + type: keyword + - name: engine_update + description: > + Last update of the engine used for analysis. + type: date - name: stats default_field: false description: > @@ -92,7 +119,7 @@ - name: rule description: > YARA matches for the file - type: group + type: nested fields: - name: name description: > @@ -118,6 +145,19 @@ description: > ruleset source type: keyword + - name: packers + type: nested + description: > + List of packer detection tools and their result. + fields: + - name: tool_name + description: > + Name of tool used to detect packer used. + type: keyword + - name: name + description: > + Name of identified packer(s). + type: keyword - name: notification default_field: false description: > @@ -559,54 +599,7 @@ description: > Date that VirusTotal first observed this file object on the internet, typically by URL type: keyword -- name: virustotal.analysis.results - description: > - Contains each engine's resulting data about sample - type: nested -- name: virustotal.analysis.results - type: group - fields: - - name: method - description: > - Method used by VirusTotal for analysis. - type: keyword - - name: category - description: > - Category of the sample. - type: keyword - - name: result - description: > - Result of the VirusTotal analysis. - type: keyword - - name: engine_name - description: > - Name of the engine used for analysis. - type: keyword - - name: engine_version - description: > - Version of the engine used for analysis. - type: keyword - - name: engine_update - description: > - Last update of the engine used for analysis. - type: date -- name: virustotal.packers - type: nested - description: > - List of packer detection tools and their result. -- name: virustotal.packers - type: group - description: > - List of packer detection tools and their result. - fields: - - name: tool_name - description: > - Name of tool used to detect packer used. - type: keyword - - name: name - description: > - Name of identified packer(s). - type: keyword + - name: file.hash.ssdeep default_field: false description: > @@ -690,6 +683,86 @@ description: > List of shared libraries used by this ELF object type: keyword + - name: exports + type: nested + fields: + - name: name + description: > + Name of exported symbol + type: keyword + default_field: false + - name: type + description: > + Type of exported symbol + type: keyword + default_field: false + - name: imports + default_field: false + description: > + List of imported element names and types + release: beta + type: nested + fields: + - name: name + description: > + Name of imported symbol + type: keyword + default_field: false + - name: type + description: > + Type of imported symbol + type: keyword + default_field: false + - name: sections + default_field: false + description: > + Section information of the binary executable file. + release: beta + type: nested + fields: + - name: name + description: > + Binary Section name. + type: keyword + - name: physical_offset + description: > + Binary Section offset on disk. + type: keyword + - name: physical_size + description: > + Binary Section size in bytes + type: long + format: bytes + - name: virtual_address + description: > + Binary Section virtual address. + format: string + type: long + - name: virtual_size + description: > + Binary Section virtual size in bytes. + format: bytes + type: long + - name: flags + description: > + Binary Section flags. + type: keyword + - name: type + description: > + Binary Section type. + type: keyword + - name: segment_name + description: > + Binary Section name of containing segment. + type: keyword + - name: entropy + description: > + Binary Section measurement of Shannon entropy. + type: float + - name: chi2 + description: > + Binary Section measurement of Chi squared distribution. + type: float - name: hash.telfhash description: > telfhash hash for ELF files. @@ -705,127 +778,8 @@ description: > ELF object segment list. type: flattened -- name: file.elf.exports - default_field: false - description: > - List of exported element names and types - release: beta - type: nested -- name: file.elf.exports - type: group - fields: - - name: name - description: > - Name of exported symbol - type: keyword - default_field: false - - name: type - description: > - Type of exported symbol - type: keyword - default_field: false -- name: file.elf.sections - default_field: false - description: > - Section information of the binary executable file. - release: beta - type: nested -- name: file.elf.sections - type: group - fields: - - name: name - description: > - Binary Section name. - type: keyword - - name: physical_offset - description: > - Binary Section offset on disk. - type: keyword - - name: physical_size - description: > - Binary Section size in bytes - type: long - format: bytes - - name: virtual_address - description: > - Binary Section virtual address. - format: string - type: long - - name: virtual_size - description: > - Binary Section virtual size in bytes. - format: bytes - type: long - - name: flags - description: > - Binary Section flags. - type: keyword - - name: type - description: > - Binary Section type. - type: keyword - - name: segment_name - description: > - Binary Section name of containing segment. - type: keyword - - name: entropy - description: > - Binary Section measurement of Shannon entropy. - type: float - - name: chi2 - description: > - Binary Section measurement of Chi squared distribution. - type: float -- name: file.elf.imports - default_field: false - description: > - List of imported element names and types - release: beta - type: nested -- name: file.elf.imports - type: group - fields: - - name: name - description: > - Name of imported symbol - type: keyword - default_field: false - - name: type - description: > - Type of imported symbol - type: keyword - default_field: false + ### PE file extensions ### -- name: file.pe.overlay - type: nested -- name: file.pe.overlay - type: group - fields: - - name: chi2 - description: > - Chi2 information of the PE file. - type: float - - name: entropy - description: > - Entropy information of the PE file. - type: float - - name: filetype - description: > - Filetype of the PE file. - type: keyword - - name: md5 - description: > - Overlay MD5 hash of the PE file. - type: keyword - - name: offset - description: > - Offset of the overlay information of the PE file. - type: long - - name: size - description: > - Size of the PE file. - format: bytes - type: long - name: file.pe type: group fields: @@ -890,96 +844,116 @@ description: > Machine type of the PE file. type: keyword -- name: file.pe.sections - type: nested -- name: file.pe.sections - type: group - fields: - - name: name + - name: overlay + type: nested description: > - Binary Section name. - type: keyword - - name: physical_offset - description: > - Binary Section offset on disk. - type: keyword - - name: physical_size - description: > - Binary Section size in bytes - type: long - format: bytes - - name: virtual_address - description: > - Binary Section virtual address. - format: string - type: long - - name: virtual_size - description: > - Binary Section virtual size in bytes. - format: bytes - type: long - - name: flags - description: > - Binary Section flags. - type: keyword - - name: type - description: > - Binary Section type. - type: keyword - - name: segment_name - description: > - Binary Section name of containing segment. - type: keyword - - name: entropy - description: > - Binary Section measurement of Shannon entropy. - type: float - - name: chi2 - description: > - Binary Section measurement of Chi squared distribution. - type: float -- name: file.pe.exports - description: > - List of exported element names and types - release: beta - type: nested -- name: file.pe.exports - description: > - List of exported element names and types - release: beta - type: group - fields: - - name: name - description: > - Name of exported symbol - type: keyword - default_field: false - - name: type + TODO + fields: + - name: chi2 + description: > + Chi2 information of the PE file. + type: float + - name: entropy + description: > + Entropy information of the PE file. + type: float + - name: filetype + description: > + Filetype of the PE file. + type: keyword + - name: md5 + description: > + Overlay MD5 hash of the PE file. + type: keyword + - name: offset + description: > + Offset of the overlay information of the PE file. + type: long + - name: size + description: > + Size of the PE file. + format: bytes + type: long + - name: sections + type: nested description: > - Type of exported symbol - type: keyword - default_field: false -- name: file.pe.imports - description: > - List of imported element names and types - release: beta - type: nested -- name: file.pe.imports - description: > - List of imported element names and types - release: beta - type: group - fields: - - name: name + TODO + fields: + - name: name + description: > + Binary Section name. + type: keyword + - name: physical_offset + description: > + Binary Section offset on disk. + type: keyword + - name: physical_size + description: > + Binary Section size in bytes + type: long + format: bytes + - name: virtual_address + description: > + Binary Section virtual address. + format: string + type: long + - name: virtual_size + description: > + Binary Section virtual size in bytes. + format: bytes + type: long + - name: flags + description: > + Binary Section flags. + type: keyword + - name: type + description: > + Binary Section type. + type: keyword + - name: segment_name + description: > + Binary Section name of containing segment. + type: keyword + - name: entropy + description: > + Binary Section measurement of Shannon entropy. + type: float + - name: chi2 + description: > + Binary Section measurement of Chi squared distribution. + type: float + - name: exports description: > - Name of imported symbol - type: keyword - default_field: false - - name: type + List of exported element names and types + release: beta + type: nested + fields: + - name: name + description: > + Name of exported symbol + type: keyword + default_field: false + - name: type + description: > + Type of exported symbol + type: keyword + default_field: false + - name: imports description: > - Type of imported symbol - type: keyword - default_field: false + List of imported element names and types + release: beta + type: nested + fields: + - name: name + description: > + Name of imported symbol + type: keyword + default_field: false + - name: type + description: > + Type of imported symbol + type: keyword + default_field: false - name: rule default_field: false diff --git a/x-pack/filebeat/module/virustotal/livehunt/config/cleanup-empty.js b/x-pack/filebeat/module/virustotal/livehunt/config/cleanup-empty.js index 6262bc0f300..5ab03c0c3e6 100644 --- a/x-pack/filebeat/module/virustotal/livehunt/config/cleanup-empty.js +++ b/x-pack/filebeat/module/virustotal/livehunt/config/cleanup-empty.js @@ -16,9 +16,11 @@ function isEmpty(obj) { } function process(evt) { + var console = require("console"); + console.debug("cleanup.cleanEmptyList"); - if (field == "") { + if (params.field == "") { console.debug("Empty field parameter. Skipping."); return; } diff --git a/x-pack/filebeat/module/virustotal/livehunt/config/livehunt.yml b/x-pack/filebeat/module/virustotal/livehunt/config/livehunt.yml index c506bd26f19..57083e77e91 100644 --- a/x-pack/filebeat/module/virustotal/livehunt/config/livehunt.yml +++ b/x-pack/filebeat/module/virustotal/livehunt/config/livehunt.yml @@ -1,12 +1,23 @@ #spellchecker: disable {{ if eq .input "httpjson" }} type: httpjson -url: https://www.virustotal.com/api/v3/intelligence/hunting_notification_files?limit={{ .limit }} -http_method: GET -http_headers: - x-apikey: {{ .api_key }} -json_objects_array: data +config_version: 2 interval: 1m +request: + url: https://www.virustotal.com/api/v3/intelligence/hunting_notification_files?limit={{ .limit }} + method: GET + transforms: + - set: + target: header.x-apikey + value: {{ .api_key }} +response: + split: + target: body.data + type: array + pagination: + - set: + target: url.params.cursor + value: '[[.last_response.body.meta.cursor]]' {{ end }} {{ if eq .input "file" }} diff --git a/x-pack/filebeat/module/virustotal/livehunt/config/virustotal-pe.js b/x-pack/filebeat/module/virustotal/livehunt/config/virustotal-pe.js index 9e2181ddaff..8aecaeb5804 100644 --- a/x-pack/filebeat/module/virustotal/livehunt/config/virustotal-pe.js +++ b/x-pack/filebeat/module/virustotal/livehunt/config/virustotal-pe.js @@ -59,15 +59,12 @@ var vtPE = (function () { */ var norm_exports = Array(); for (var i = 0; i < exports.length; i++) { - var libname = exports[i].library_name; - for (var j = 0; i < exports[i].imported_functions.length; j++) { - norm_exports.push( - { - "name": exports[i].imported_functions[j], - "type": "function", - } - ); - } + norm_exports.push( + { + "name": exports[i], + "type": "function", + } + ); } evt.Delete("file.pe.exports");