Merge pull request #80 from sourcegraph/nsc/java-stdlib-hover

src/main/kotlin/lsifjava/ExternalDocs.kt

@@ -1,83 +1,145 @@
 @file:Suppress("JAVA_MODULE_DOES_NOT_EXPORT_PACKAGE")
 package lsifjava
 
+import com.sun.tools.javac.code.*
 import com.sun.source.tree.*
 import com.sun.source.util.DocTrees
 import com.sun.source.util.JavacTask
 import com.sun.source.util.TreePathScanner
-import com.sun.tools.javac.util.Context
 import com.sun.tools.javac.api.JavacTool
 import com.sun.tools.javac.tree.JCTree.*
+import com.sun.tools.javac.util.Context
 import java.nio.charset.Charset
-import java.nio.file.Path
+import java.nio.file.*
 import javax.lang.model.element.Element
-import javax.tools.*
+import javax.tools.JavaFileObject
+import com.sun.tools.javac.file.JavacFileManager
+import javax.tools.StandardLocation
 
 data class ExternalHoverMeta(val doc: String, val tree: Tree)
 
-class ExternalDocs(private val docPaths: List<Path>) {
+private val emptyFileManager = SourceFileManager(emptySet())
 
+class ExternalDocs(private val docPaths: List<Path>, private val diagnosticListener: CountingDiagnosticListener) {
+    // we cache compilation unit trees here to reduce the number of times we have to invoke the parser
     private val fileCache = HashMap<String, Pair<JavacTask, CompilationUnitTree>?>()
 
-    private val fileManager: StandardJavaFileManager by lazy {
-        val manager = ToolProvider.getSystemJavaCompiler()
-            .getStandardFileManager(null, null, Charset.defaultCharset())
+    private val fileManager = let {
+        val manager = JavacFileManager(Context(), false, Charset.defaultCharset())
         manager.setLocation(StandardLocation.SOURCE_PATH, docPaths.map { it.toFile() })
         manager
     }
 
+    private val jdkFileManager = JDK8CompatFileManager(fileManager)
+
+    /**
+     * Returns hover metadata for the tree section that matches the given element in the class file
+     * associated with <code>containerClass</code>, which is expected to be the fully qualified name of the class file
+     */
     fun findDocForElement(containerClass: String, javac: JavacTool, element: Element): ExternalHoverMeta? {
         val context = DocumentIndexer.SimpleContext()
 
-        val (task, compUnit) = fileCache.getOrPut(containerClass) { analyzeFileFromJar(containerClass, context, javac) }
-            ?: return null
+        val (task, compUnit) = fileCache.getOrPut(containerClass) {
+            val fileObject = findFileFromJars(containerClass) ?: return@getOrPut null
+            parseFileObject(fileObject, context, javac)
+        } ?: return null
 
         return DocExtractionVisitor(task, element).scan(compUnit, null)
     }
 
-    private fun analyzeFileFromJar(containerClass: String, context: Context, javac: JavacTool): Pair<JavacTask, CompilationUnitTree>? {
-        val file = fileManager.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE)
-            ?: return null
-        val task = javac.getTask(NoopWriter, fileManager, CountingDiagnosticListener.NullWriter, listOf(), listOf(), listOf(file), context)
-        val compUnit = task.parse().iterator().next()
-        val analyzeResult = runCatching { task.analyze() }
-        analyzeResult.getOrNull() ?: run {
-            //println("${file.name} threw exception")
-            return null
-        }
+    private fun findFileFromJars(containerClass: String): JavaFileObject? =
+        fileManager.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE)
+            ?: jdkFileManager.getJavaFileForInput(containerClass)
+
+    /**
+     * Performs basic parsing of the external dependency JavaFileObject and returns a (javac task, compilation unit) pair.
+     * For reasons beyond my knowing (and many a staring at the debugger), an empty file manager
+     */
+    private fun parseFileObject(file: JavaFileObject, context: Context, javac: JavacTool): Pair<JavacTask, CompilationUnitTree>? {
+        val task = javac.getTask(NoopWriter, emptyFileManager, diagnosticListener, listOf(), listOf(), listOf(file), context)
+        val compUnit = task.parse().iterator().apply {
+            // bail out of the enclosing function
+            if(!hasNext()) return null
+        }.next()
         return Pair(task, compUnit)
     }
-
-    private class DocExtractionVisitor(task: JavacTask, private val element: Element): TreePathScanner<ExternalHoverMeta?, Unit?>() {
-        private val docs: DocTrees = DocTrees.instance(task)
-
-        override fun visitMethod(node: MethodTree?, p: Unit?): ExternalHoverMeta? {
-            (node as JCMethodDecl).sym ?: return null
 
-            if(node.sym.toString() == element.toString()) {
+    private class DocExtractionVisitor(
+        val task: JavacTask,
+        private val element: Element,
+        private val docs: DocTrees = DocTrees.instance(task)
+    ): TreePathScanner<ExternalHoverMeta?, Unit?>() {
+        // Basic flag to indicate if this DocExtractionVisitor has visited its assigned class decl yet.
+        // If set to false, we want to create a new DocExtractionVisitor for the class decl we are visiting.
+        // This way we can keep track of the owning class decl for methods/variables that are otherwise only
+        // available with fully resolved symbols, which we don't get with non-full-fat parsing+analyzing
+        private var new: Boolean = true
+        private lateinit var classDecl: ClassTree
+
+        override fun visitMethod(node: MethodTree, p: Unit?): ExternalHoverMeta? {
+            if(element !is Symbol.MethodSymbol) return null
+
+            if(element.owner.simpleName.toString() != classDecl.simpleName.toString())
+                return null
+
+            if(element.name.toString() != node.name.toString()) return null
+
+            if(element.name.toString() != "<init>" &&
+               element.returnType.toString() != node.returnType.toString()) return null
+
+            val paramsEqual = element.params.size == node.parameters.size &&
+                element.params.foldIndexed(true) { i, acc, sym ->
+                    val paramType = (node.parameters[i] as JCVariableDecl).vartype
+                    val defTypeName = when(paramType) {
+                        is JCPrimitiveTypeTree -> paramType.toString()
+                        is JCTypeApply -> paramType.clazz.toString()
+                        is JCIdent -> paramType.toString()
+                        else -> {
+                            println("param type wasn't JCPrimitiveTypeTree|JCTypeApply|JCIdent, but ${paramType::class.java}")
+                            return null
+                        }
+                    }
+                    acc && sym.type.tsym.simpleName.toString() == defTypeName
+                }
+
+            if(paramsEqual) {
                 val doc = docs.getDocComment(currentPath) ?: return null
-                return ExternalHoverMeta(doc, node)
+                return ExternalHoverMeta(doc.trim(), node)
             }
 
             return null
         }
 
-        override fun visitVariable(node: VariableTree?, p: Unit?): ExternalHoverMeta? {
-            (node as JCVariableDecl).sym ?: return null
+        override fun visitVariable(node: VariableTree, p: Unit?): ExternalHoverMeta? {
+            if(element !is Symbol.VarSymbol) return null
 
-            if(node.sym.toString() == element.toString()) {
+            if(element.owner.simpleName.toString() != classDecl.simpleName.toString())
+                return null
 
+            if(element.name.toString() == node.name.toString()) {
+                val doc = docs.getDocComment(currentPath) ?: return null
+                return ExternalHoverMeta(doc.trim(), node)
             }
+
             // filter to instance variables
             return null
         }
 
-        override fun visitClass(node: ClassTree?, p: Unit?): ExternalHoverMeta? {
-            (node as JCClassDecl).sym ?: return null
-
-            if(node.sym.toString() == element.toString()) {
+        override fun visitClass(node: ClassTree, p: Unit?): ExternalHoverMeta? {
+            // we need this logic here to stop calling scan on the same ClassTree infinitely, but we also
+            // want to start a new visitor for each nested class decl
+            if(!new) {
+                return DocExtractionVisitor(task, element, docs).scan(node, null)
+            }
+            new = false
+            classDecl = node
+
+            if(element !is Symbol.ClassSymbol) return super.visitClass(node, p)
+
+            // Assumption: no class-like name conflicts within a single class file
+            if((node as JCClassDecl).name.toString() == element.simpleName.toString()) {
                 val doc = docs.getDocComment(currentPath) ?: return super.visitClass(node, p)
-                return ExternalHoverMeta(doc, node)
+                return ExternalHoverMeta(doc.trim(), node)
             }
 
             return super.visitClass(node, p)

src/main/kotlin/lsifjava/FileCollector.kt

@@ -27,7 +27,7 @@ fun buildIndexerMap(
     val sourceVersions = buildToolInterface.javaSourceVersions
 
     // TODO(nsc) where to move this
-    val externalDocManager = ExternalDocs(buildToolInterface.sourcesList)
+    val externalDocManager = ExternalDocs(buildToolInterface.sourcesList, javacDiagListener)
 
     val fileBuildInfo = Channel<FileBuildInfo>()
 

src/main/kotlin/lsifjava/JDK8CompatFileManager.kt

@@ -0,0 +1,186 @@
+@file:Suppress("JAVA_MODULE_DOES_NOT_EXPORT_PACKAGE")
+package lsifjava
+
+import com.sun.tools.javac.util.Context
+import java.io.IOException
+import java.nio.charset.Charset
+import java.nio.file.*
+import javax.tools.*
+
+// hard-coded list for convenience. Sorry, George :)
+private val JDK_MODULES = listOf(
+    "java.activation",
+    "java.base",
+    "java.compiler",
+    "java.corba",
+    "java.datatransfer",
+    "java.desktop",
+    "java.instrument",
+    "java.jnlp",
+    "java.logging",
+    "java.management",
+    "java.management.rmi",
+    "java.naming",
+    "java.net.http",
+    "java.prefs",
+    "java.rmi",
+    "java.scripting",
+    "java.se",
+    "java.se.ee",
+    "java.security.jgss",
+    "java.security.sasl",
+    "java.smartcardio",
+    "java.sql",
+    "java.sql.rowset",
+    "java.transaction",
+    "java.transaction.xa",
+    "java.xml",
+    "java.xml.bind",
+    "java.xml.crypto",
+    "java.xml.ws",
+    "java.xml.ws.annotation",
+    "javafx.base",
+    "javafx.controls",
+    "javafx.fxml",
+    "javafx.graphics",
+    "javafx.media",
+    "javafx.swing",
+    "javafx.web",
+    "jdk.accessibility",
+    "jdk.aot",
+    "jdk.attach",
+    "jdk.charsets",
+    "jdk.compiler",
+    "jdk.crypto.cryptoki",
+    "jdk.crypto.ec",
+    "jdk.dynalink",
+    "jdk.editpad",
+    "jdk.hotspot.agent",
+    "jdk.httpserver",
+    "jdk.incubator.httpclient",
+    "jdk.internal.ed",
+    "jdk.internal.jvmstat",
+    "jdk.internal.le",
+    "jdk.internal.opt",
+    "jdk.internal.vm.ci",
+    "jdk.internal.vm.compiler",
+    "jdk.internal.vm.compiler.management",
+    "jdk.jartool",
+    "jdk.javadoc",
+    "jdk.jcmd",
+    "jdk.jconsole",
+    "jdk.jdeps",
+    "jdk.jdi",
+    "jdk.jdwp.agent",
+    "jdk.jfr",
+    "jdk.jlink",
+    "jdk.jshell",
+    "jdk.jsobject",
+    "jdk.jstatd",
+    "jdk.localedata",
+    "jdk.management",
+    "jdk.management.agent",
+    "jdk.management.cmm",
+    "jdk.management.jfr",
+    "jdk.management.resource",
+    "jdk.naming.dns",
+    "jdk.naming.rmi",
+    "jdk.net",
+    "jdk.pack",
+    "jdk.packager.services",
+    "jdk.rmic",
+    "jdk.scripting.nashorn",
+    "jdk.scripting.nashorn.shell",
+    "jdk.sctp",
+    "jdk.security.auth",
+    "jdk.security.jgss",
+    "jdk.snmp",
+    "jdk.unsupported",
+    "jdk.unsupported.desktop",
+    "jdk.xml.dom",
+    "jdk.zipfs",
+)
+
+/**
+ * FileManager that falls back to JavacPathFileManager for Java 8.
+ * Java 8 StandardJavaFileManager doesn't have the <code>setLocationFromPaths</code>
+ * method, instead it only has <code>setLocation</code> which requires an 
+ * <code>Iterable<? extends File></code>, which would cause an UnsupportedException
+ * when trying to turn the ZipFile from the in-memory FileSystem into a File.
+ * Because JavacPathFileManager doesn't exist beyond Java 8 (9?) and we build with 14+,
+ * the symbol resolver would fail for that symbol, hence we create an instance via
+ * reflection if the Java version is 8. God I hate this.
+ */
+class JDK8CompatFileManager(manager: StandardJavaFileManager): ForwardingJavaFileManager<JavaFileManager>(getFileManager(manager)) {
+    companion object {
+        /**
+         * returns a different JavaFileManager based on the current java version, with either SOURCE_PATH or MODULE_SOURCE_PATH set to "/"
+         * denoting the root of the JAR/ZIP file that this file manager is for. If JDK 8, we instantiate a <code>com.sun.tools.javac.nio.JavacPathFileManager</code>
+         * via reflection, else we use the passed file manager. See the class doc for the reason why.
+         */
+        private fun getFileManager(fileManager: StandardJavaFileManager): JavaFileManager {
+            var java8Manager: JavaFileManager? = null
+
+            if(javaVersion == 8) {
+                java8Manager = Class.forName("com.sun.tools.javac.nio.JavacPathFileManager")
+                    .constructors[0]
+                    .newInstance(Context(), false, Charset.defaultCharset())
+                    as JavaFileManager?
+            }
+
+            srcZip()?.also {
+                if(javaVersion > 8) {
+                    fileManager.setLocationFromPaths(StandardLocation.MODULE_SOURCE_PATH, setOf(it.getPath("/")))
+                } else {
+                    java8Manager!!::class.java
+                        .getMethod("setDefaultFileSystem", FileSystem::class.java)
+                        .invoke(java8Manager, it)
+                    java8Manager::class.java
+                        .getMethod("setLocation", JavaFileManager.Location::class.java, Iterable::class.java)
+                        .invoke(java8Manager, StandardLocation.SOURCE_PATH, setOf(it.getPath("/")))
+                }
+            }
+
+            return java8Manager ?: fileManager
+        }
+
+        private fun srcZip(): FileSystem? {
+            val srcZip = findSrcZip() ?: return null
+            return try {
+                FileSystems.newFileSystem(srcZip, ExternalDocs::class.java.classLoader)
+            } catch (e: IOException) {
+                throw RuntimeException(e)
+            }
+        }
+
+        private fun findSrcZip(): Path? {
+            val javaHome = JavaHomeHelper.javaHome() ?: return null
+            val locations = arrayOf("lib/src.zip", "src.zip")
+            for (rel in locations) {
+                val abs = javaHome.resolve(rel)
+                if (Files.exists(abs)) {
+                    return abs
+                }
+            }
+            return null
+        }
+    }
+
+    /**
+     * Searches for a class either by module or not depending on the current java version.
+     */
+    fun getJavaFileForInput(containerClass: String): JavaFileObject? {
+        if(javaVersion == 8) {
+            return this.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE)
+        } else {
+            for(module in JDK_MODULES) {
+                val moduleLocation = this.getLocationForModule(StandardLocation.MODULE_SOURCE_PATH, module) ?: continue
+                this.getJavaFileForInput(moduleLocation, containerClass, JavaFileObject.Kind.SOURCE)?.run {
+                    return this
+                }
+            }
+        }
+
+        return null
+    }
+}