From f550ec5c10161fb15aedc7a1d19d9079567388ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arkadiusz=20Wr=C3=B3bel?= Date: Tue, 31 Aug 2021 17:18:01 +0200 Subject: [PATCH 01/11] Add ApiScout profile to `do_export_full` and `do_import_full` --- drakrun/drakrun/draksetup.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/drakrun/drakrun/draksetup.py b/drakrun/drakrun/draksetup.py index 12d5b8ea9..1f30880d4 100644 --- a/drakrun/drakrun/draksetup.py +++ b/drakrun/drakrun/draksetup.py @@ -1208,6 +1208,15 @@ def do_export_full(mc, bucket, name): bucket, f"{name}/profiles/{file}", os.path.join(PROFILE_DIR, file) ) + # Upload ApiScout profile + for file in os.listdir(APISCOUT_PROFILE_DIR): + logging.info("Uploading file %s", file) + mc.fput_object( + bucket, + f"{name}/apiscout_profile/{file}", + os.path.join(APISCOUT_PROFILE_DIR, file), + ) + def do_import_full(mc, name, bucket, zpool): """Perform full snapshot import, symmetric to do_export_full""" @@ -1222,14 +1231,22 @@ def do_import_full(mc, name, bucket, zpool): ["zcat", compressed_snapshot.name], stdout=snapshot, check=True ) - profile_prefix = f"{name}/profiles/" - for object in mc.list_objects(bucket, prefix=profile_prefix): - # Strip profile prefix - profile_name = object.object_name[len(profile_prefix) :] + profiles_prefix = f"{name}/profiles/" + for object in mc.list_objects(bucket, prefix=profiles_prefix): + # Strip profiles prefix + profile_name = object.object_name[len(profiles_prefix) :] mc.fget_object( bucket, object.object_name, os.path.join(PROFILE_DIR, profile_name) ) + apiscout_profile_prefix = f"{name}/apiscout_profile/" + for object in mc.list_objects(bucket, prefix=apiscout_profile_prefix): + # Strip apiscout profile prefix + filename = object.object_name[len(apiscout_profile_prefix) :] + mc.fget_object( + bucket, object.object_name, os.path.join(APISCOUT_PROFILE_DIR, filename) + ) + @click.group() def main(): From 84e8a5df05fd2aa442cd6626823c4596ee35f447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arkadiusz=20Wr=C3=B3bel?= Date: Tue, 31 Aug 2021 18:54:18 +0200 Subject: [PATCH 02/11] Make calling `build_static_apiscout_profile` in `create_missing_profiles` function --- drakrun/drakrun/draksetup.py | 12 +++++++++++- drakrun/drakrun/main.py | 16 ++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/drakrun/drakrun/draksetup.py b/drakrun/drakrun/draksetup.py index 1f30880d4..2a42f8060 100644 --- a/drakrun/drakrun/draksetup.py +++ b/drakrun/drakrun/draksetup.py @@ -51,7 +51,10 @@ delete_vm_conf, VirtualMachine, ) -from drakrun.apiscout import make_static_apiscout_profile_for_dll +from drakrun.apiscout import ( + make_static_apiscout_profile_for_dll, + build_static_apiscout_profile, +) from drakrun.util import RuntimeInfo, VmiOffsets, safe_delete from tqdm import tqdm from pathlib import Path, PureWindowsPath @@ -851,6 +854,13 @@ def create_missing_profiles(): except Exception: log.exception("Unexpected exception from create_rekall_profile!") + dll_basename_list = [dll.dest for dll in dll_file_list] + static_apiscout_profile = build_static_apiscout_profile( + APISCOUT_PROFILE_DIR, dll_basename_list + ) + with open(Path(APISCOUT_PROFILE_DIR) / "static_apiscout_profile.json", "w") as f: + json.dump(static_apiscout_profile, f) + vm.destroy() delete_vm_network( vm_id=1, net_enable=False, out_interface=out_interface, dns_server=dns_server diff --git a/drakrun/drakrun/main.py b/drakrun/drakrun/main.py index f9a6e2cd4..12bfb4b9c 100644 --- a/drakrun/drakrun/main.py +++ b/drakrun/drakrun/main.py @@ -41,7 +41,6 @@ from drakrun.vm import generate_vm_conf, VirtualMachine from drakrun.injector import Injector import drakrun.sample_startup as sample_startup -from drakrun.apiscout import build_static_apiscout_profile class LocalLogBuffer(logging.Handler): @@ -376,16 +375,6 @@ def build_profile_payload(self) -> Dict[str, LocalResource]: return Resource.from_directory(name="profiles", directory_path=tmp_dir) - def build_static_apiscout_profile_payload(self) -> Dict[str, LocalResource]: - dll_basename_list = [dll.dest for dll in dll_file_list] - static_apiscout_profile = build_static_apiscout_profile( - APISCOUT_PROFILE_DIR, dll_basename_list - ) - return LocalResource( - name="static_apiscout_profile.json", - content=json.dumps(static_apiscout_profile, indent=4, sort_keys=True), - ) - def send_raw_analysis(self, sample, outdir, metadata, dumps_metadata, quality): """ Offload drakrun-prod by sending raw analysis output to be processed by @@ -418,7 +407,10 @@ def send_raw_analysis(self, sample, outdir, metadata, dumps_metadata, quality): self.log.info("Uploading static ApiScout profile...") task.add_payload( "static_apiscout_profile.json", - self.build_static_apiscout_profile_payload(), + LocalResource( + name="static_apiscout_profile.json", + path=Path(APISCOUT_PROFILE_DIR) / "static_apiscout_profile.json", + ), ) self.log.info("Uploading artifacts...") From 85c57d3be9e195c76e154604dfa628016737c4f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arkadiusz=20Wr=C3=B3bel?= Date: Tue, 31 Aug 2021 21:38:07 +0200 Subject: [PATCH 03/11] Add checking ApiScout partial profiles existence --- drakrun/drakrun/draksetup.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drakrun/drakrun/draksetup.py b/drakrun/drakrun/draksetup.py index 2a42f8060..e1c9ddf4e 100644 --- a/drakrun/drakrun/draksetup.py +++ b/drakrun/drakrun/draksetup.py @@ -811,8 +811,10 @@ def postinstall(report, generate_usermode): logging.info(" # draksetup scale ") -def profile_exists(profile: DLL) -> bool: - return (Path(PROFILE_DIR) / f"{profile.dest}.json").is_file() +def profiles_exist(profile_name: str) -> bool: + return (Path(PROFILE_DIR) / f"{profile_name}.json").is_file() and ( + Path(APISCOUT_PROFILE_DIR) / f"{profile_name}.json" + ).is_file() def create_missing_profiles(): @@ -844,11 +846,11 @@ def create_missing_profiles(): # Ensure that all declared usermode profiles exist # This is important when upgrade defines new entries in dll_file_list and compulsory_dll_file_list for profile in compulsory_dll_file_list: - if not profile_exists(profile): + if not profiles_exist(profile.dest): create_rekall_profile(injector, profile, True) for profile in dll_file_list: - if not profile_exists(profile): + if not profiles_exist(profile.dest): try: create_rekall_profile(injector, profile) except Exception: From 954f7bb51b1b43e7d5a70e4b5654015c72b2ddff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arkadiusz=20Wr=C3=B3bel?= Date: Wed, 1 Sep 2021 17:33:00 +0200 Subject: [PATCH 04/11] fix --- drakrun/drakrun/apiscout.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drakrun/drakrun/apiscout.py b/drakrun/drakrun/apiscout.py index 671bf20ac..01c7ff5e8 100644 --- a/drakrun/drakrun/apiscout.py +++ b/drakrun/drakrun/apiscout.py @@ -54,8 +54,6 @@ def make_static_apiscout_profile_for_dll(filepath: str) -> Dict[str, Any]: pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_RESOURCE"], ] ) - if not hasattr(pe, "DIRECTORY_ENTRY_EXPORT"): - raise RuntimeError(f"DIRECTORY_ENTRY_EXPORT not found in '{filepath}'") dll_entry = {} dll_entry["base_address"] = pe.OPTIONAL_HEADER.ImageBase @@ -64,6 +62,10 @@ def make_static_apiscout_profile_for_dll(filepath: str) -> Dict[str, Any]: dll_entry["filepath"] = filepath dll_entry["aslr_offset"] = 0 dll_entry["exports"] = [] + if not hasattr(pe, "DIRECTORY_ENTRY_EXPORT"): + if pe.is_driver(): + return dll_entry + raise RuntimeError(f"DIRECTORY_ENTRY_EXPORT not found in '{filepath}'") for exp in sorted(pe.DIRECTORY_ENTRY_EXPORT.symbols, key=attrgetter("address")): export_info = {} From b03b7c31eb2503ab0d9826a654c221f38fa2ba26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arkadiusz=20Wr=C3=B3bel?= Date: Thu, 2 Sep 2021 17:08:23 +0200 Subject: [PATCH 05/11] fix --- drakrun/drakrun/apiscout.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drakrun/drakrun/apiscout.py b/drakrun/drakrun/apiscout.py index 01c7ff5e8..1409837f8 100644 --- a/drakrun/drakrun/apiscout.py +++ b/drakrun/drakrun/apiscout.py @@ -1,6 +1,7 @@ import json import logging from operator import attrgetter +import os from pathlib import Path, PureWindowsPath import pefile from typing import List, Dict, Any @@ -97,6 +98,11 @@ def build_static_apiscout_profile( for dll_basename in dll_basename_list: filepath = Path(apiscout_profile_dir) / f"{dll_basename}.json" + if not os.path.isfile(filepath): + log.warning( + f"'{filepath}' not found. Is there a problem with profiles generation?" + ) + continue with open(filepath) as f: dll_profile = json.load(f) dlls_profiles[build_apiscout_dll_key(dll_profile)] = dll_profile From 2a2b1e16e857954c3a9b76fada42c39a05c7afe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arkadiusz=20Wr=C3=B3bel?= <29861114+catsuryuu@users.noreply.github.com> Date: Mon, 13 Sep 2021 07:11:24 +0200 Subject: [PATCH 06/11] Apply suggestions from code review Co-authored-by: Hubert Jasudowicz --- drakrun/drakrun/apiscout.py | 2 +- drakrun/drakrun/draksetup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drakrun/drakrun/apiscout.py b/drakrun/drakrun/apiscout.py index 1409837f8..eda53cc43 100644 --- a/drakrun/drakrun/apiscout.py +++ b/drakrun/drakrun/apiscout.py @@ -98,7 +98,7 @@ def build_static_apiscout_profile( for dll_basename in dll_basename_list: filepath = Path(apiscout_profile_dir) / f"{dll_basename}.json" - if not os.path.isfile(filepath): + if not filepath.isfile(): log.warning( f"'{filepath}' not found. Is there a problem with profiles generation?" ) diff --git a/drakrun/drakrun/draksetup.py b/drakrun/drakrun/draksetup.py index e1c9ddf4e..985237fd2 100644 --- a/drakrun/drakrun/draksetup.py +++ b/drakrun/drakrun/draksetup.py @@ -1246,7 +1246,7 @@ def do_import_full(mc, name, bucket, zpool): profiles_prefix = f"{name}/profiles/" for object in mc.list_objects(bucket, prefix=profiles_prefix): # Strip profiles prefix - profile_name = object.object_name[len(profiles_prefix) :] + profile_name = object.object_name[len(profiles_prefix):] mc.fget_object( bucket, object.object_name, os.path.join(PROFILE_DIR, profile_name) ) @@ -1254,7 +1254,7 @@ def do_import_full(mc, name, bucket, zpool): apiscout_profile_prefix = f"{name}/apiscout_profile/" for object in mc.list_objects(bucket, prefix=apiscout_profile_prefix): # Strip apiscout profile prefix - filename = object.object_name[len(apiscout_profile_prefix) :] + filename = object.object_name[len(apiscout_profile_prefix):] mc.fget_object( bucket, object.object_name, os.path.join(APISCOUT_PROFILE_DIR, filename) ) From 956a597cec495227a3fae6163519b3ece11ded75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arkadiusz=20Wr=C3=B3bel?= Date: Mon, 13 Sep 2021 07:27:11 +0200 Subject: [PATCH 07/11] isort --- drakrun/drakrun/apiscout.py | 5 +-- drakrun/drakrun/draksetup.py | 64 ++++++++++++++++++------------------ drakrun/drakrun/main.py | 44 ++++++++++++------------- 3 files changed, 57 insertions(+), 56 deletions(-) diff --git a/drakrun/drakrun/apiscout.py b/drakrun/drakrun/apiscout.py index eda53cc43..09c246da4 100644 --- a/drakrun/drakrun/apiscout.py +++ b/drakrun/drakrun/apiscout.py @@ -1,10 +1,11 @@ import json import logging -from operator import attrgetter import os +from operator import attrgetter from pathlib import Path, PureWindowsPath +from typing import Any, Dict, List + import pefile -from typing import List, Dict, Any log = logging.getLogger(__name__) diff --git a/drakrun/drakrun/draksetup.py b/drakrun/drakrun/draksetup.py index 985237fd2..f690f597a 100644 --- a/drakrun/drakrun/draksetup.py +++ b/drakrun/drakrun/draksetup.py @@ -1,65 +1,65 @@ import configparser import hashlib -import logging import io -import sys +import json +import logging import os import re -import json -import time import secrets -import subprocess import string +import subprocess +import sys import tempfile +import time +import traceback +from pathlib import Path, PureWindowsPath from typing import Optional import click import requests -from requests import RequestException from minio import Minio from minio.error import NoSuchKey -from drakrun.drakpdb import ( - fetch_pdb, - make_pdb_profile, - dll_file_list, - compulsory_dll_file_list, - pe_codeview_data, - DLL, +from requests import RequestException +from tqdm import tqdm + +from drakrun.apiscout import ( + build_static_apiscout_profile, + make_static_apiscout_profile_for_dll, ) from drakrun.config import ( - InstallInfo, + APISCOUT_PROFILE_DIR, + ETC_DIR, LIB_DIR, - VOLUME_DIR, PROFILE_DIR, - ETC_DIR, VM_CONFIG_DIR, - APISCOUT_PROFILE_DIR, + VOLUME_DIR, + InstallInfo, +) +from drakrun.drakpdb import ( + DLL, + compulsory_dll_file_list, + dll_file_list, + fetch_pdb, + make_pdb_profile, + pe_codeview_data, ) +from drakrun.injector import Injector from drakrun.networking import ( + delete_vm_network, setup_vm_network, start_dnsmasq, - delete_vm_network, stop_dnsmasq, ) -from drakrun.storage import get_storage_backend, REGISTERED_BACKEND_NAMES -from drakrun.injector import Injector +from drakrun.storage import REGISTERED_BACKEND_NAMES, get_storage_backend +from drakrun.util import RuntimeInfo, VmiOffsets, safe_delete from drakrun.vm import ( - generate_vm_conf, FIRST_CDROM_DRIVE, SECOND_CDROM_DRIVE, - get_all_vm_conf, - delete_vm_conf, VirtualMachine, + delete_vm_conf, + generate_vm_conf, + get_all_vm_conf, ) -from drakrun.apiscout import ( - make_static_apiscout_profile_for_dll, - build_static_apiscout_profile, -) -from drakrun.util import RuntimeInfo, VmiOffsets, safe_delete -from tqdm import tqdm -from pathlib import Path, PureWindowsPath -import traceback - log = logging.getLogger(__name__) diff --git a/drakrun/drakrun/main.py b/drakrun/drakrun/main.py index 12bfb4b9c..02df10253 100644 --- a/drakrun/drakrun/main.py +++ b/drakrun/drakrun/main.py @@ -1,46 +1,46 @@ #!/usr/bin/python3 +import argparse import contextlib +import functools +import hashlib +import json import logging -import sys +import ntpath import os +import re import shutil -import argparse -import subprocess -import hashlib import socket +import subprocess +import sys +import tempfile import time import zipfile -import json -import re -import functools -import tempfile -from io import StringIO -from typing import List, Dict -from stat import S_ISREG, ST_CTIME, ST_MODE, ST_SIZE from configparser import NoOptionError +from io import StringIO from itertools import chain from pathlib import Path +from stat import S_ISREG, ST_CTIME, ST_MODE, ST_SIZE +from typing import Dict, List import magic -import ntpath -from karton.core import Karton, Config, Task, LocalResource, Resource +from karton.core import Config, Karton, LocalResource, Resource, Task -from drakrun.version import __version__ as DRAKRUN_VERSION +import drakrun.sample_startup as sample_startup +from drakrun.config import APISCOUT_PROFILE_DIR, ETC_DIR, PROFILE_DIR, InstallInfo from drakrun.drakpdb import dll_file_list -from drakrun.config import InstallInfo, ETC_DIR, PROFILE_DIR, APISCOUT_PROFILE_DIR +from drakrun.injector import Injector +from drakrun.networking import setup_vm_network, start_dnsmasq, start_tcpdump_collector from drakrun.storage import get_storage_backend -from drakrun.networking import start_tcpdump_collector, start_dnsmasq, setup_vm_network from drakrun.util import ( - patch_config, - get_xl_info, + RuntimeInfo, get_xen_commandline, + get_xl_info, graceful_exit, - RuntimeInfo, + patch_config, ) -from drakrun.vm import generate_vm_conf, VirtualMachine -from drakrun.injector import Injector -import drakrun.sample_startup as sample_startup +from drakrun.version import __version__ as DRAKRUN_VERSION +from drakrun.vm import VirtualMachine, generate_vm_conf class LocalLogBuffer(logging.Handler): From 1ac37f518ac026be74675ffc178a8e01eb87713d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arkadiusz=20Wr=C3=B3bel?= Date: Mon, 13 Sep 2021 07:46:50 +0200 Subject: [PATCH 08/11] black yuck --- drakrun/drakrun/draksetup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drakrun/drakrun/draksetup.py b/drakrun/drakrun/draksetup.py index bbbc487f7..89b5533dd 100644 --- a/drakrun/drakrun/draksetup.py +++ b/drakrun/drakrun/draksetup.py @@ -1315,7 +1315,7 @@ def do_import_full(mc, name, bucket, zpool): profiles_prefix = f"{name}/profiles/" for object in mc.list_objects(bucket, prefix=profiles_prefix): # Strip profiles prefix - profile_name = object.object_name[len(profiles_prefix):] + profile_name = object.object_name[len(profiles_prefix) :] mc.fget_object( bucket, object.object_name, os.path.join(PROFILE_DIR, profile_name) ) @@ -1323,7 +1323,7 @@ def do_import_full(mc, name, bucket, zpool): apiscout_profile_prefix = f"{name}/apiscout_profile/" for object in mc.list_objects(bucket, prefix=apiscout_profile_prefix): # Strip apiscout profile prefix - filename = object.object_name[len(apiscout_profile_prefix):] + filename = object.object_name[len(apiscout_profile_prefix) :] mc.fget_object( bucket, object.object_name, os.path.join(APISCOUT_PROFILE_DIR, filename) ) From 0ad427d50e0ee09d74b6f3e47593a443a0a00eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arkadiusz=20Wr=C3=B3bel?= Date: Mon, 13 Sep 2021 08:11:43 +0200 Subject: [PATCH 09/11] pylint --- drakrun/drakrun/apiscout.py | 1 - 1 file changed, 1 deletion(-) diff --git a/drakrun/drakrun/apiscout.py b/drakrun/drakrun/apiscout.py index 09c246da4..87b7d6b0e 100644 --- a/drakrun/drakrun/apiscout.py +++ b/drakrun/drakrun/apiscout.py @@ -1,6 +1,5 @@ import json import logging -import os from operator import attrgetter from pathlib import Path, PureWindowsPath from typing import Any, Dict, List From 0c5a3eb87f3a01a0c8c6408a90a970747971eb3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arkadiusz=20Wr=C3=B3bel?= Date: Mon, 13 Sep 2021 09:09:59 +0200 Subject: [PATCH 10/11] fix --- drakrun/drakrun/apiscout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drakrun/drakrun/apiscout.py b/drakrun/drakrun/apiscout.py index 87b7d6b0e..a76b23919 100644 --- a/drakrun/drakrun/apiscout.py +++ b/drakrun/drakrun/apiscout.py @@ -98,7 +98,7 @@ def build_static_apiscout_profile( for dll_basename in dll_basename_list: filepath = Path(apiscout_profile_dir) / f"{dll_basename}.json" - if not filepath.isfile(): + if not filepath.is_file(): log.warning( f"'{filepath}' not found. Is there a problem with profiles generation?" ) From fd4d06b5560187de705f71112a793ef904123a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arkadiusz=20Wr=C3=B3bel?= Date: Wed, 22 Sep 2021 13:26:05 +0200 Subject: [PATCH 11/11] Make calling `create_missing_profiles()` after creating `OS_INFO.json` --- drakrun/drakrun/draksetup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drakrun/drakrun/draksetup.py b/drakrun/drakrun/draksetup.py index 89b5533dd..266b12c24 100644 --- a/drakrun/drakrun/draksetup.py +++ b/drakrun/drakrun/draksetup.py @@ -782,10 +782,6 @@ def postinstall(report, generate_usermode): logging.info("Snapshotting persistent memory...") storage_backend.snapshot_vm0_volume() - if generate_usermode: - # Restore a VM and create usermode profiles - create_missing_profiles() - if report: send_usage_report( { @@ -802,6 +798,10 @@ def postinstall(report, generate_usermode): with open(os.path.join(APISCOUT_PROFILE_DIR, "OS_INFO.json"), "w") as f: f.write(json.dumps(os_info, indent=4, sort_keys=True)) + if generate_usermode: + # Restore a VM and create usermode profiles + create_missing_profiles() + logging.info("All right, drakrun setup is done.") logging.info("First instance of drakrun will be enabled automatically...") subprocess.check_output("systemctl enable drakrun@1", shell=True)