Acceptance tests

.gitignore

@@ -213,4 +213,6 @@ test/run_real_world_samples.sh
 .project
 .pydevproject
 
-src/cwe_checker.plugin
+src/cwe_checker.plugin
+
+.#*

CHANGES.md

@@ -4,6 +4,7 @@
 - Refactoring: Unification of cwe_checker function interface
 - Refactoring: Created utils module for JSON functionality
 - Added check for CWE 248: Uncaught Exception
+- Added automated test suite (run with make test)
 
 0.1 (2018-10-08)
 =====

Makefile

@@ -1,10 +1,9 @@
-.phony: all
+.PHONY: all clean test uninstall
 all:
 	cd src; bapbuild -r -Is checkers,utils -pkgs yojson,unix cwe_checker.plugin; bapbundle install cwe_checker.plugin; cd ..
 
 test:
-	bapbuild -r -Is src,src/checkers,src/utils,test -pkgs yojson,unix,alcotest test/test_cwe_checker.byte
-	./test/test_cwe_checker.byte
+	pytest -v
 
 clean:
 	bapbuild -clean

README.md

@@ -41,10 +41,12 @@ The three way is to build it using the provided `Makefile`. In this case you mus
 -  Ocaml 4.05.0
 -  Opam 2.0.2
 -  BAP 1.5 (and its dependencies)
--  yojson <= 1.4.1
--  alcotest <= 0.8.3
+-  yojson >= 1.4.1
+-  alcotest >= 0.8.3
 -  Sark for IDA Pro annotations
-Just run `make all` to compile and register the plugin with BAP.
+-  pytest >= 3.5.1
+
+Just run `make all` to compile and register the plugin with BAP. You can run the test suite via `make test`.
 ## How to use cwe_checker? ##
 The usage is straight forward: adjust the `config.json` (if needed) and call BAP with *cwe_checker* as a pass.
 ``` bash

test/acceptance/cwe_checker_testlib.py

@@ -0,0 +1,14 @@
+import subprocess
+
+def build_bap_cmd(filename, target, arch):
+        cmd = 'bap test/artificial_samples/build/cwe_%s_%s.out  --pass=callsites,cwe-checker --cwe-checker-partial=CWE%s --cwe-checker-config=src/config.json' % (filename, arch, target)
+        return cmd.split()
+
+def execute_and_check_occurence(filename, target, arch, string):
+    occurence = 0
+    bap_cmd = build_bap_cmd(filename, target, arch)
+    output = subprocess.check_output(bap_cmd)
+    for l in output.splitlines():
+        if string in l:
+            occurence += 1
+    return occurence

test/acceptance/test_cwe190.py

@@ -0,0 +1,35 @@
+import unittest
+import cwe_checker_testlib
+
+class TestCwe190(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '190'
+        self.string = b'Integer Overflow or Wraparound'
+
+    def test_cwe190_01_arm(self):
+        expect_res = 3
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    def test_cwe190_01_x86(self):
+        expect_res = 3
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe190_01_x64(self):
+        expect_res = 3
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe190_01_mips(self):
+        expect_res = 3
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe190_01_ppc(self):
+        expect_res = 3
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+

test/acceptance/test_cwe215.py

@@ -0,0 +1,34 @@
+import unittest
+import cwe_checker_testlib
+
+class TestCwe215(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '215'
+        self.filename = '476'
+        self.string = b'Information Exposure Through Debug Information'
+
+    def test_cwe215_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    def test_cwe215_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe215_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    def test_cwe215_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'ppc', self.string)
+        assert res == expect_res
+
+    def test_cwe215_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'mips', self.string)
+        assert res == expect_res    

test/acceptance/test_cwe243.py

@@ -0,0 +1,61 @@
+import unittest
+import cwe_checker_testlib
+
+class TestCwe243(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '243'
+        self.string = b'The program utilizes chroot without dropping privileges and/or changing the directory'
+
+    def test_cwe243_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    def test_cwe243_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe243_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    def test_cwe243_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe243_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe243_02_arm(self):
+        expect_res = 0
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'arm', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Investigate and fix this issue")
+    def test_cwe243_02_x86(self):
+        expect_res = 0
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe243_02_x64(self):
+        expect_res = 0
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'x64', self.string)
+        assert res == expect_res
+
+    def test_cwe243_02_ppc(self):
+        expect_res = 0
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'ppc', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe476_02_mips(self):
+        expect_res = 0
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'mips', self.string)
+        assert res == expect_res

test/acceptance/test_cwe248.py

@@ -0,0 +1,36 @@
+import unittest
+import cwe_checker_testlib
+
+class TestCwe248(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '248'
+        self.string = b'Possibly Uncaught Exception'
+
+    def test_cwe248_01_arm(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Fix CPP compilation issue for x86")
+    def test_cwe248_01_x86(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe248_01_x64(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe248_01_mips(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe248_01_ppc(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+

test/acceptance/test_cwe332.py

@@ -0,0 +1,35 @@
+import unittest
+import cwe_checker_testlib
+
+class TestCwe332(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '332'
+        self.string = b'Insufficient Entropy in PRNG'
+
+    def test_cwe332_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    def test_cwe332_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe332_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe332_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe332_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+

test/acceptance/test_cwe367.py

@@ -0,0 +1,35 @@
+import unittest
+import cwe_checker_testlib
+
+class TestCwe367(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '367'
+        self.string = b'Time-of-check Time-of-use Race Condition'
+
+    def test_cwe367_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    def test_cwe367_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe367_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe367_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe367_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+

test/acceptance/test_cwe426.py

@@ -0,0 +1,36 @@
+import unittest
+import cwe_checker_testlib
+
+class TestCwe426(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '426'
+        self.string = b'Untrusted Search Path'
+
+    def test_cwe426_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    @unittest.skip("FIXME")
+    def test_cwe426_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe426_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe426_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe426_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+

test/acceptance/test_cwe457.py

@@ -0,0 +1,39 @@
+import unittest
+import cwe_checker_testlib
+
+class TestCwe457(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '457'
+        self.string = b'Use of Uninitialized Variable'
+
+    @unittest.skip("FIXME")
+    def test_cwe457_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    @unittest.skip("FIXME")
+    def test_cwe457_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    @unittest.skip("FIXME")
+    def test_cwe457_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe457_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    @unittest.skip("FIXME")
+    def test_cwe457_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+