diff --git a/.github/actions/post-build-selective/action.yml b/.github/actions/post-build-selective/action.yml
index e90413e6d47..b9d7a05554a 100644
--- a/.github/actions/post-build-selective/action.yml
+++ b/.github/actions/post-build-selective/action.yml
@@ -21,6 +21,9 @@ runs:
     - uses: actions/setup-node@v4
       with: { node-version: '22' }
 
+    - run: cat .mill-jvm-version
+      shell: ${{ inputs.shell }}
+
     - run: ./mill -i -k selective.resolve ${{ inputs.millargs }}
       shell: ${{ inputs.shell }}
 
diff --git a/.github/actions/pre-build-setup/action.yml b/.github/actions/pre-build-setup/action.yml
index 52e332bdb5d..585dc24d21b 100644
--- a/.github/actions/pre-build-setup/action.yml
+++ b/.github/actions/pre-build-setup/action.yml
@@ -62,4 +62,7 @@ runs:
         name: ${{ inputs.os }}-selective-execution-artifact
         include-hidden-files: true
 
-    - uses: actions/checkout@v4
\ No newline at end of file
+    - uses: actions/checkout@v4
+
+    - run: echo temurin:${{ inputs.java-version }} > .mill-jvm-version
+      shell: bash
\ No newline at end of file
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 569f0ef6bc5..046c4f8df2e 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -139,8 +139,11 @@ jobs:
             millargs: "contrib.__.test"
             install-android-sdk: false
 
-          - java-version: 17
-            millargs: "example.javalib.__.local.server.test"
+          # Run this one using `.native` as a smoketest. Also make sure the java-version
+          # is the same as that used in the `build-linux` job to avoid diverging code
+          # hashes (https://github.com/com-lihaoyi/mill/pull/4410)
+          - java-version: 11
+            millargs: "example.javalib.__.native.server.test"
             install-android-sdk: false
 
           - java-version: 17