Merge pull request #951 from ethereumjs/ci/vm-tests-on-demand

On-demand testing for VM State and Blockchain
ethereumjs · Nov 17, 2020 · c188339 · c188339
2 parents 11eb16f + c0dd7ee
commit c188339
Show file tree

Hide file tree

Showing 6 changed files with 96 additions and 24 deletions.
diff --git a/.github/workflows/vm-pr.yml b/.github/workflows/vm-pr.yml
@@ -11,7 +11,7 @@ defaults:
     working-directory: packages/vm
 
 jobs:
-  test-vm-api:
+  vm-api:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/setup-node@v1
@@ -49,21 +49,13 @@ jobs:
       - run: npm run test:API:browser
       - run: npm run lint
 
-  test-vm-state:
+  vm-state:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         fork: [
-          # 'Berlin',
           'MuirGlacier',
-          'Istanbul',
-          # 'Petersburg',
-          # 'Constantinople',
-          # 'Byzantium',
-          # 'SpuriousDragon',
-          # 'TangerineWhistle',
-          # 'Homestead',
-          # 'Chainstart'
+          'Istanbul'
         ]
       fail-fast: false
     steps:
@@ -93,7 +85,49 @@ jobs:
 
       - run: npm run test:state -- --fork=${{ matrix.fork }} --verify-test-amount-alltests
 
-  test-vm-blockchain:
+  vm-state-extended:
+    if: contains(join(github.event.pull_request.labels.*.name, ' '), 'Test all hardforks')
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        fork: [
+          'Berlin',
+          'Petersburg',
+          'Constantinople',
+          'Byzantium',
+          'SpuriousDragon',
+          'TangerineWhistle',
+          'Homestead',
+          'Chainstart'
+        ]
+      fail-fast: false
+    steps:
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+      - uses: actions/checkout@v1
+        with:
+          submodules: recursive
+      - name: Dependency cache
+        uses: actions/cache@v2
+        id: cache
+        with:
+          key: VM-${{ runner.os }}-12-${{ hashFiles('**/package-lock.json') }}
+          path: '**/node_modules'
+
+      # Installs root dependencies, ignoring Bootstrap All script.
+      # Bootstraps the current package only
+      - run: npm install --ignore-scripts && npm run bootstrap:vm
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: ${{github.workspace}}
+
+      # Builds current package and the ones it depends from.
+      - run: npm run build:vm
+        working-directory: ${{github.workspace}}
+
+      - run: npm run test:state -- --fork=${{ matrix.fork }} --verify-test-amount-alltests
+
+  vm-blockchain:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -102,15 +136,53 @@ jobs:
 
         # Tests were splitted with --dir and --excludeDir to balance execution times below the 9min mark.
         args: [
-          # '--fork=Berlin --expected-test-amount=33',
           '--fork=Istanbul --dir=GeneralStateTests/stTimeConsuming --expected-test-amount=15561',
-          '--fork=Istanbul --excludeDir=stTimeConsuming --expected-test-amount=19817',
-          # '--fork=Constantinople --dir=GeneralStateTests/stTimeConsuming --expected-test-amount=15561',
-          # '--fork=Constantinople --excludeDir=stTimeConsuming --expected-test-amount=17193',
-          # '--fork=Petersburg --dir=GeneralStateTests/stTimeConsuming --expected-test-amount=15561',
-          # '--fork=Petersburg --excludeDir=stTimeConsuming --expected-test-amount=17174',
-          # '--fork=Homestead --expected-test-amount=6997',
-          # '--fork=Chainstart --expected-test-amount=4385'
+          '--fork=Istanbul --excludeDir=stTimeConsuming --expected-test-amount=19817'
+        ]
+      fail-fast: false
+    steps:
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+      - uses: actions/checkout@v1
+        with:
+          submodules: recursive
+      - name: Dependency cache
+        uses: actions/cache@v2
+        id: cache
+        with:
+          key: VM-${{ runner.os }}-12-${{ hashFiles('**/package-lock.json') }}
+          path: '**/node_modules'
+
+      # Installs root dependencies, ignoring Bootstrap All script.
+      # Bootstraps the current package only
+      - run: npm install --ignore-scripts && npm run bootstrap:vm
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: ${{github.workspace}}
+
+      # Builds current package and the ones it depends from.
+      - run: npm run build:vm
+        working-directory: ${{github.workspace}}
+
+      - run: npm run test:blockchain -- ${{ matrix.args }}
+
+  vm-blockchain-extended:
+    if: contains(join(github.event.pull_request.labels.*.name, ' '), 'Test all hardforks')
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # Args to pass to the tester. Note that some have splitted the slow tests and only 
+        #   running those: these are only on forks where that is applicable (see PR #489 for numbers on these)
+
+        # Tests were splitted with --dir and --excludeDir to balance execution times below the 9min mark.
+        args: [
+          '--fork=Berlin --expected-test-amount=33',
+          '--fork=Constantinople --dir=GeneralStateTests/stTimeConsuming --expected-test-amount=15561',
+          '--fork=Constantinople --excludeDir=stTimeConsuming --expected-test-amount=17193',
+          '--fork=Petersburg --dir=GeneralStateTests/stTimeConsuming --expected-test-amount=15561',
+          '--fork=Petersburg --excludeDir=stTimeConsuming --expected-test-amount=17174',
+          '--fork=Homestead --expected-test-amount=6997',
+          '--fork=Chainstart --expected-test-amount=4385'
         ]
       fail-fast: false
     steps:

diff --git a/.gitmodules b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "ethereum-tests"]
-	path = ethereum-tests
+	path = packages/ethereum-tests
 	url = https://github.com/ethereum/tests.git
 	branch = develop
diff --git a/ethereum-tests b/ethereum-tests
diff --git a/packages/ethereum-tests b/packages/ethereum-tests
diff --git a/packages/tx/test/testLoader.ts b/packages/tx/test/testLoader.ts
@@ -4,7 +4,7 @@ const path = require('path')
 
 const falsePredicate = () => false
 // Load tests from git submodule
-const defaultTestsPath = path.resolve('../../ethereum-tests')
+const defaultTestsPath = path.resolve('../ethereum-tests')
 /**
  * Returns the list of test files matching the given parameters
  * @param testType the test type (path segment)

diff --git a/packages/vm/tests/testLoader.ts b/packages/vm/tests/testLoader.ts
@@ -4,7 +4,7 @@ const path = require('path')
 
 const falsePredicate = () => false
 // Load tests from git submodule
-const defaultTestsPath = path.resolve('../../ethereum-tests')
+const defaultTestsPath = path.resolve('../ethereum-tests')
 /**
  * Returns the list of test files matching the given parameters
  * @param testType the test type (path segment)