From 0dbd2ed46bc4e86566c132e37870c8c2646be29a Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Tue, 12 May 2020 14:07:18 -0700 Subject: [PATCH 1/6] Experimental coverage changes Tells coverage.py to record coverage for individual test cases, and makes CodeCov tolerate a small drop in overall coverage percentage to account for nondeterministic gas tests. --- .coveragerc | 2 ++ .github/workflows/ci.yml | 2 +- codecov.yml | 4 ++++ setup.cfg | 2 -- 4 files changed, 7 insertions(+), 3 deletions(-) delete mode 100644 setup.cfg diff --git a/.coveragerc b/.coveragerc index 37058bd54..3a6689ecd 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,3 +2,5 @@ source = manticore omit = *__init__.py +dynamic_context = test_function +concurrency = multiprocessing diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c5b2d3c9..9a309f1dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,7 +170,7 @@ jobs: run_tests_from_dir() { DIR=$1 echo "Running only the tests from 'tests/$DIR' directory" - pytest --durations=100 --cov=manticore -n auto "tests/$DIR" + pytest --durations=100 --cov=manticore --cov-config=../../.coveragerc -n auto "tests/$DIR" coverage xml } diff --git a/codecov.yml b/codecov.yml index 9cd6e9730..bb6dccece 100644 --- a/codecov.yml +++ b/codecov.yml @@ -6,3 +6,7 @@ codecov: # If we add or remove any, we need to change this number. after_n_builds: 9 wait_for_ci: yes + status: + project: + default: + threshold: 0.2% diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b5263f3b8..000000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[nosetests] -match=(?:^|[\b_\./-])([Tt]es|[Ee]th) From 02814231d3c236898469abb252a192c8c9fb184b Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Tue, 12 May 2020 14:13:28 -0700 Subject: [PATCH 2/6] Use workspace --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a309f1dc..8e79c1e32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,7 +170,7 @@ jobs: run_tests_from_dir() { DIR=$1 echo "Running only the tests from 'tests/$DIR' directory" - pytest --durations=100 --cov=manticore --cov-config=../../.coveragerc -n auto "tests/$DIR" + pytest --durations=100 --cov=manticore --cov-config=$GITHUB_WORKSPACE/.coveragerc -n auto "tests/$DIR" coverage xml } From 560c05adaed25236dc1a7ea8baf4083757a5cbf0 Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Wed, 13 May 2020 11:22:02 -0700 Subject: [PATCH 3/6] Remove explicit concurrency type declaration Appears to break use of subprocess in integration tests --- .coveragerc | 1 - 1 file changed, 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 3a6689ecd..55ea74bd7 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,4 +3,3 @@ source = manticore omit = *__init__.py dynamic_context = test_function -concurrency = multiprocessing From c62aa847a7e1d10eecee59ff6b220a8f3b713293 Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Thu, 14 May 2020 12:57:57 -0700 Subject: [PATCH 4/6] Eliminate dynamic context --- .coveragerc | 1 - 1 file changed, 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 55ea74bd7..37058bd54 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,4 +2,3 @@ source = manticore omit = *__init__.py -dynamic_context = test_function From 06f7aa212fca1feac50ce03ece6b6bf2da1b1529 Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Fri, 15 May 2020 14:07:52 -0700 Subject: [PATCH 5/6] Make all CI respect rcfile --- .github/workflows/ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e79c1e32..ef385aa94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,13 +62,13 @@ jobs: # concolic assumes presence of ../linux/simpleassert echo "Running concolic.py..." HW=../linux/helloworld - coverage run ./concolic.py + coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run ./concolic.py if [ $? -ne 0 ]; then return 1 fi echo "Running count_instructions.py..." - coverage run ./count_instructions.py $HW |grep -q Executed + coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run ./count_instructions.py $HW |grep -q Executed if [ $? -ne 0 ]; then return 1 fi @@ -77,21 +77,21 @@ jobs: gcc -static -g src/state_explore.c -o state_explore ADDRESS=0x$(objdump -S state_explore | grep -A 1 '((value & 0xff) != 0)' | tail -n 1 | sed 's|^\s*||g' | cut -f1 -d:) - coverage run ./introduce_symbolic_bytes.py state_explore $ADDRESS + coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run ./introduce_symbolic_bytes.py state_explore $ADDRESS if [ $? -ne 0 ]; then return 1 fi echo "Running run_simple.py..." gcc -x c -static -o hello test_run_simple.c - coverage run ./run_simple.py hello + coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run ./run_simple.py hello if [ $? -ne 0 ]; then return 1 fi echo "Running run_hook.py..." MAIN_ADDR=$(nm $HW|grep 'T main' | awk '{print "0x"$1}') - coverage run ./run_hook.py $HW $MAIN_ADDR + coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run ./run_hook.py $HW $MAIN_ADDR if [ $? -ne 0 ]; then return 1 fi @@ -101,7 +101,7 @@ jobs: gcc -static -g src/state_explore.c -o state_explore SE_ADDR=0x$(objdump -S state_explore | grep -A 1 'value == 0x41' | tail -n 1 | sed 's|^\s*||g' | cut -f1 -d:) - coverage run ./state_control.py state_explore $SE_ADDR + coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run ./state_control.py state_explore $SE_ADDR if [ $? -ne 0 ]; then return 1 fi @@ -153,7 +153,7 @@ jobs: mkdir truffle_tests cd truffle_tests truffle unbox metacoin - coverage run -m manticore . --contract MetaCoin --workspace output + coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run -m manticore . --contract MetaCoin --workspace output # Truffle smoke test. We test if manticore is able to generate states # from a truffle project. if [ "$(ls output/*tx -l | wc -l)" != "34" ]; then @@ -161,7 +161,7 @@ jobs: return 1 fi echo "Truffle test succeded" - coverage xml + coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc xml cd .. cp truffle_tests/coverage.xml . return 0 @@ -171,7 +171,7 @@ jobs: DIR=$1 echo "Running only the tests from 'tests/$DIR' directory" pytest --durations=100 --cov=manticore --cov-config=$GITHUB_WORKSPACE/.coveragerc -n auto "tests/$DIR" - coverage xml + coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc xml } run_examples() { @@ -187,7 +187,7 @@ jobs: launch_examples RESULT=$? echo Ran example scripts - coverage xml + coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc xml popd cp examples/script/coverage.xml . return $RESULT From 5efa6343c2d886d73c52fac34334d63bd3538400 Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Fri, 15 May 2020 14:13:38 -0700 Subject: [PATCH 6/6] Use environment variable --- .github/workflows/ci.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef385aa94..44dd0f781 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,16 +59,17 @@ jobs: run: | # Launches all examples; this assumes PWD is examples/script launch_examples() { + COVERAGE_RCFILE=$GITHUB_WORKSPACE/.coveragerc # concolic assumes presence of ../linux/simpleassert echo "Running concolic.py..." HW=../linux/helloworld - coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run ./concolic.py + coverage run ./concolic.py if [ $? -ne 0 ]; then return 1 fi echo "Running count_instructions.py..." - coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run ./count_instructions.py $HW |grep -q Executed + coverage run ./count_instructions.py $HW |grep -q Executed if [ $? -ne 0 ]; then return 1 fi @@ -77,21 +78,21 @@ jobs: gcc -static -g src/state_explore.c -o state_explore ADDRESS=0x$(objdump -S state_explore | grep -A 1 '((value & 0xff) != 0)' | tail -n 1 | sed 's|^\s*||g' | cut -f1 -d:) - coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run ./introduce_symbolic_bytes.py state_explore $ADDRESS + coverage run ./introduce_symbolic_bytes.py state_explore $ADDRESS if [ $? -ne 0 ]; then return 1 fi echo "Running run_simple.py..." gcc -x c -static -o hello test_run_simple.c - coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run ./run_simple.py hello + coverage run ./run_simple.py hello if [ $? -ne 0 ]; then return 1 fi echo "Running run_hook.py..." MAIN_ADDR=$(nm $HW|grep 'T main' | awk '{print "0x"$1}') - coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run ./run_hook.py $HW $MAIN_ADDR + coverage run ./run_hook.py $HW $MAIN_ADDR if [ $? -ne 0 ]; then return 1 fi @@ -101,7 +102,7 @@ jobs: gcc -static -g src/state_explore.c -o state_explore SE_ADDR=0x$(objdump -S state_explore | grep -A 1 'value == 0x41' | tail -n 1 | sed 's|^\s*||g' | cut -f1 -d:) - coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run ./state_control.py state_explore $SE_ADDR + coverage run ./state_control.py state_explore $SE_ADDR if [ $? -ne 0 ]; then return 1 fi @@ -150,10 +151,11 @@ jobs: } run_truffle_tests(){ + COVERAGE_RCFILE=$GITHUB_WORKSPACE/.coveragerc mkdir truffle_tests cd truffle_tests truffle unbox metacoin - coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc run -m manticore . --contract MetaCoin --workspace output + coverage run -m manticore . --contract MetaCoin --workspace output # Truffle smoke test. We test if manticore is able to generate states # from a truffle project. if [ "$(ls output/*tx -l | wc -l)" != "34" ]; then @@ -161,7 +163,7 @@ jobs: return 1 fi echo "Truffle test succeded" - coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc xml + coverage xml cd .. cp truffle_tests/coverage.xml . return 0 @@ -169,9 +171,10 @@ jobs: run_tests_from_dir() { DIR=$1 + COVERAGE_RCFILE=$GITHUB_WORKSPACE/.coveragerc echo "Running only the tests from 'tests/$DIR' directory" pytest --durations=100 --cov=manticore --cov-config=$GITHUB_WORKSPACE/.coveragerc -n auto "tests/$DIR" - coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc xml + coverage xml } run_examples() { @@ -187,7 +190,7 @@ jobs: launch_examples RESULT=$? echo Ran example scripts - coverage --rcfile=$GITHUB_WORKSPACE/.coveragerc xml + coverage xml popd cp examples/script/coverage.xml . return $RESULT