Merge pull request #32 from CERT-Polska/master

drakcore: Remove drak-archiver (CERT-Polska#491)

ci/Dockerfile-bundle

@@ -4,7 +4,7 @@ FROM debian:buster
 
 # DRAKVUF build deps
 RUN echo "deb-src http://deb.debian.org/debian buster main" >> /etc/apt/sources.list
-RUN apt-get update && apt-get install -y build-essential git wget curl cmake flex bison libjson-c-dev autoconf-archive clang python3-dev gcc-7 g++-7 lsb-release patch
+RUN apt-get update && apt-get install -y build-essential git wget curl cmake flex bison libjson-c-dev autoconf-archive clang python3-dev gcc-7 g++-7 lsb-release patch libsystemd-dev nasm bc
 
 # Install Golang
 RUN wget -q -O /usr/local/go1.15.3.linux-amd64.tar.gz https://golang.org/dl/go1.15.3.linux-amd64.tar.gz

ci/build_utils.sh

@@ -1,8 +1,12 @@
 function build_xen() {
     PREFIX=$1
     # Expects the cwd to be Xen repository
-    ./configure --prefix=$PREFIX --enable-githttp --disable-pvshim
-    make -j$(nproc) dist
+    ./configure --prefix=$PREFIX --enable-githttp --disable-pvshim --enable-systemd --enable-ovmf
+    echo CONFIG_EXPERT=y > xen/.config
+    echo CONFIG_MEM_SHARING=y >> xen/.config
+    make -C xen olddefconfig
+    make -j$(nproc) dist-xen
+    make -j$(nproc) dist-tools
     make -j$(nproc) install-xen
     make -j$(nproc) install-tools
 }

drakcore/debian/drakcore.links

@@ -1,4 +1,3 @@
-opt/venvs/drakcore/bin/drak-archiver usr/bin/drak-archiver
 opt/venvs/drakcore/bin/drak-config-setup usr/bin/drak-config-setup
 opt/venvs/drakcore/bin/drak-system usr/bin/drak-system
 opt/venvs/drakcore/bin/drak-postprocess usr/bin/drak-postprocess

drakcore/setup.py

@@ -17,8 +17,7 @@
     packages=["drakcore", "drakcore.postprocess"],
     include_package_data=True,
     install_requires=open("requirements.txt").read().splitlines(),
-    scripts=['drakcore/bin/drak-archiver',
-             'drakcore/bin/drak-system',
+    scripts=['drakcore/bin/drak-system',
              'drakcore/bin/drak-config-setup',
              'drakcore/bin/drak-postprocess',
              'drakcore/bin/drak-healthcheck',

drakrun/drakrun/config.dist.ini

@@ -60,6 +60,17 @@ syscall_filter=
 ; special version of drakvuf-bundle is required
 enable_ipt=0
 
+; (advanced) Enable testing codepaths. Test sample artifacts will not be uploaded
+; to persistent storage. Their lifetime will be bound to karton tasks produced by drakrun
+; sample_testing=0
+;
+; (advanced) override Karton test filters for this service:
+; test_filters=[ { "type": "sample-test", "platform": "win32", }, { "type": "sample-test", "platform": "win64" }]
+
+; (advanced) override Karton test headers for this service:
+; test_headers={ "type": "analysis-test", "kind": "drakrun" }
+
+
 [drakvuf_plugins]
 ; list of enabled DRAKVUF plugins that are used by default,
 ; this can be overriden for particular karton's task quality value

drakrun/drakrun/main.py

@@ -113,6 +113,22 @@ class DrakrunKarton(Karton):
         "kind": "drakrun",
     }
 
+    # Filters and headers used for testing sample analysis
+    DEFAULT_TEST_FILTERS = [
+        {
+            "type": "sample-test",
+            "platform": "win32",
+        },
+        {
+            "type": "sample-test",
+            "platform": "win64",
+        },
+    ]
+    DEFAULT_TEST_HEADERS = {
+        "type": "analysis-test",
+        "kind": "drakrun",
+    }
+
     def __init__(self, config: Config, instance_id: int):
         super().__init__(config)
 
@@ -160,6 +176,12 @@ def load_json(config, key):
         cls.identity = config.get('identity', cls.DEFAULT_IDENTITY)
         cls.filters = load_json(config, 'filters') or cls.DEFAULT_FILTERS
         cls.headers = load_json(config, 'headers') or cls.DEFAULT_HEADERS
+        cls.test_headers = load_json(config, 'test_headers') or cls.DEFAULT_TEST_HEADERS
+
+        # If testing is enabled, add additional test filters from the configuration
+        # or fall back to hardcoded
+        if config.getboolean("sample_testing", fallback=False):
+            cls.filters.extend(load_json(config, 'test_filters') or cls.DEFAULT_TEST_FILTERS)
 
     @property
     def net_enable(self) -> bool:
@@ -170,6 +192,19 @@ def enable_ipt(self) -> bool:
         # TODO: Inconsistent naming - net_enable vs enable_ipt
         return self.config.config['drakrun'].getboolean('enable_ipt', fallback=False)
 
+    @property
+    def test_run(self) -> bool:
+        # If testing is disabled, it's not a test run
+        if not self.config.config['drakrun'].getboolean('sample_testing', fallback=False):
+            return False
+
+        # Check if task matches any test filter
+        for filtr in self.test_filters:
+            if self.current_task.matches_filters(filtr):
+                return True
+
+        return False
+
     @property
     def vm_name(self) -> str:
         return f"vm-{self.instance_id}"
@@ -295,8 +330,14 @@ def upload_artifacts(self, analysis_uid, outdir, subdir=''):
             if os.path.isfile(file_path):
                 object_name = os.path.join(analysis_uid, subdir, fn)
                 res_name = os.path.join(subdir, fn)
-                resource = LocalResource(name=res_name, bucket='drakrun', path=file_path)
-                resource._uid = object_name
+                if self.test_run:
+                    # If it's a test run upload artifacts to karton-managed bucket
+                    # They'll be cleaned up by karton-system
+                    resource = LocalResource(name=res_name, path=file_path)
+                else:
+                    # If it's not a test run, put them into drakrun bucket
+                    resource = LocalResource(name=res_name, bucket='drakrun', path=file_path)
+                    resource._uid = object_name
                 yield resource
             elif os.path.isdir(file_path):
                 yield from self.upload_artifacts(analysis_uid, outdir, os.path.join(subdir, fn))
@@ -305,7 +346,11 @@ def send_analysis(self, sample, outdir, metadata, quality):
         payload = {"analysis_uid": self.analysis_uid}
         payload.update(metadata)
 
-        headers = dict(self.headers)
+        if self.test_run:
+            headers = dict(self.test_headers)
+        else:
+            headers = dict(self.headers)
+
         headers["quality"] = quality
 
         task = Task(headers, payload=payload)

drakrun/drakrun/playground.py

@@ -47,7 +47,7 @@ def cleanup(self, vm_id: int):
 
         logging.info(f"Ensuring that drakrun@{vm_id} service is stopped...")
         try:
-            subprocess.run('systemctl', 'stop', f'drakrun@{vm_id}', shell=True, stderr=subprocess.STDOUT, check=True)
+            subprocess.run(['systemctl', 'stop', f'drakrun@{vm_id}'], stderr=subprocess.STDOUT, check=True)
         except subprocess.CalledProcessError:
             raise Exception(f"drakrun@{vm_id} not stopped")
 

drakrun/drakrun/storage.py

@@ -183,7 +183,6 @@ def vm0_root_as_block(self) -> Generator[str, None, None]:
 
         yield volume_path
 
-        subprocess.run(f"umount {volume_path}", shell=True)
         subprocess.check_output(f"zfs destroy {tmp_snap}", shell=True)
 
     def export_vm0(self, file):
@@ -324,7 +323,6 @@ def vm0_root_as_block(self) -> Generator[str, None, None]:
 
         yield dev
 
-        subprocess.check_output(f"umount {dev}", shell=True)
         subprocess.check_output("qemu-nbd --disconnect /dev/nbd0", shell=True)
 
     def export_vm0(self, path: str):
@@ -499,7 +497,6 @@ def vm0_root_as_block(self) -> Generator[str, None, None]:
 
         yield volume_path
 
-        subprocess.run(f'umount {out}', shell=True)
         subprocess.run(f'losetup -d {out}', shell=True)
 
     def export_vm0(self, path: str):