diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala
index 740bca2b240..7c9cc51d298 100644
--- a/scalalib/src/mill/scalalib/JavaModule.scala
+++ b/scalalib/src/mill/scalalib/JavaModule.scala
@@ -188,7 +188,7 @@ trait JavaModule extends mill.Module with TaskModule { outer =>
       ref <- allSources()
       if exists(ref.path)
       p <- ls.rec(ref.path)
-      if p.isFile
+      if p.isFile && (p.ext == "java")
     } yield p.toNIO.toString
 
     val options = Seq("-d", javadocDir.toNIO.toString)
diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala
index c499dda5e0e..e54e2e02e6c 100644
--- a/scalalib/src/mill/scalalib/ScalaModule.scala
+++ b/scalalib/src/mill/scalalib/ScalaModule.scala
@@ -126,7 +126,7 @@ trait ScalaModule extends JavaModule { outer =>
       ref <- allSources()
       if exists(ref.path)
       p <- ls.rec(ref.path)
-      if p.isFile
+      if (p.isFile && ((p.ext == "scala") || (p.ext == "java")))
     } yield p.toNIO.toString
 
     val pluginOptions = scalacPluginClasspath().map(pluginPathRef => s"-Xplugin:${pluginPathRef.path}")