diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
index bb005afb..01bdf75c 100644
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@@ -36,7 +36,7 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
       - run: yarn install --frozen-lockfile --non-interactive
-      - run: yarn test
+      - run: ./node_modules/.bin/ember test
 
   tests_other:
     needs: lint
@@ -51,7 +51,24 @@ jobs:
       - uses: actions/checkout@v2
       - uses: volta-cli/action@v1
       - run: yarn install --frozen-lockfile --non-interactive
-      - run: yarn test
+      - run: ./node_modules/.bin/ember test
+
+  tests_compat:
+    needs: tests_linux
+    name: "Tests: Ember compatibility (ember-source@${{ matrix.ember-version }})"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ember-version:
+          ["lts-3.16", "lts-3.20", "lts-3.24", "release", "beta", "canary"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: volta-cli/action@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: yarn install --frozen-lockfile --non-interactive
+      - run: "./node_modules/.bin/ember try:one ember-${{ matrix.ember-version }}"
 
   tests_ts:
     needs: lint
diff --git a/config/ember-try.js b/config/ember-try.js
index 345d5d30..f7f02d75 100644
--- a/config/ember-try.js
+++ b/config/ember-try.js
@@ -46,6 +46,7 @@ module.exports = async function () {
       },
       {
         name: "ember-beta",
+        allowedToFail: true,
         npm: {
           devDependencies: {
             "ember-source": await getChannelURL("beta"),
@@ -54,6 +55,7 @@ module.exports = async function () {
       },
       {
         name: "ember-canary",
+        allowedToFail: true,
         npm: {
           devDependencies: {
             "ember-source": await getChannelURL("canary"),