From 3e1c1d1927d9db794b7b8de2917cb830df7c21bd Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Thu, 1 Jun 2023 09:05:56 +0200 Subject: [PATCH 1/2] Revert "Remove OldTastyInspector from scaladoc" This reverts commit 931e9f2e620be04e30254176774be70141861a27. --- .../tools/scaladoc/ScalaModuleProvider.scala | 2 +- .../tools/scaladoc/tasty/TastyParser.scala | 65 ++++----- .../tasty/inspector/DocTastyInspector.scala | 9 ++ .../tasty/inspector/OldTastyInspector.scala | 136 ++++++++++++++++++ .../tasty/comments/MemberLookupTests.scala | 15 +- 5 files changed, 186 insertions(+), 41 deletions(-) create mode 100644 scaladoc/src/scala/tasty/inspector/DocTastyInspector.scala create mode 100644 scaladoc/src/scala/tasty/inspector/OldTastyInspector.scala diff --git a/scaladoc/src/dotty/tools/scaladoc/ScalaModuleProvider.scala b/scaladoc/src/dotty/tools/scaladoc/ScalaModuleProvider.scala index edd7b41b6182..c4776f2840c2 100644 --- a/scaladoc/src/dotty/tools/scaladoc/ScalaModuleProvider.scala +++ b/scaladoc/src/dotty/tools/scaladoc/ScalaModuleProvider.scala @@ -8,7 +8,7 @@ case class Module(rootPackage: Member, members: Map[DRI, Member]) object ScalaModuleProvider: def mkModule()(using ctx: DocContext): Module = - val (result, rootDoc) = ScaladocTastyInspector.loadDocs() + val (result, rootDoc) = ScaladocTastyInspector().result() val (rootPck, rest) = result.partition(_.name == "API") val (emptyPackages, nonemptyPackages) = (rest ++ rootPck.flatMap(_.members)) .filter(p => p.members.nonEmpty || p.docs.nonEmpty).sortBy(_.name) diff --git a/scaladoc/src/dotty/tools/scaladoc/tasty/TastyParser.scala b/scaladoc/src/dotty/tools/scaladoc/tasty/TastyParser.scala index fe44e09628c1..cd1bed42f485 100644 --- a/scaladoc/src/dotty/tools/scaladoc/tasty/TastyParser.scala +++ b/scaladoc/src/dotty/tools/scaladoc/tasty/TastyParser.scala @@ -5,7 +5,7 @@ package tasty import java.util.regex.Pattern import scala.util.{Try, Success, Failure} -import scala.tasty.inspector.{TastyInspector, Inspector, Tasty} +import scala.tasty.inspector.DocTastyInspector import scala.quoted._ import dotty.tools.dotc @@ -24,12 +24,24 @@ import ScaladocSupport._ * * Delegates most of the work to [[TastyParser]] [[dotty.tools.scaladoc.tasty.TastyParser]]. */ -case class ScaladocTastyInspector()(using ctx: DocContext) extends Inspector: +case class ScaladocTastyInspector()(using ctx: DocContext) extends DocTastyInspector: private val topLevels = Seq.newBuilder[(String, Member)] private var rootDoc: Option[Comment] = None - def inspect(using Quotes)(tastys: List[scala.tasty.inspector.Tasty[quotes.type]]): Unit = + def processCompilationUnit(using Quotes)(root: reflect.Tree): Unit = () + + override def postProcess(using Quotes): Unit = + // hack into the compiler to get a list of all top-level trees + // in principle, to do this, one would collect trees in processCompilationUnit + // however, path-dependent types disallow doing so w/o using casts + inline def hackForeachTree(thunk: reflect.Tree => Unit): Unit = + given dctx: dotc.core.Contexts.Context = quotes.asInstanceOf[scala.quoted.runtime.impl.QuotesImpl].ctx + dctx.run.nn.units.foreach { compilationUnit => + // mirrors code from TastyInspector + thunk(compilationUnit.tpdTree.asInstanceOf[reflect.Tree]) + } + val symbolsToSkip: Set[reflect.Symbol] = ctx.args.identifiersToSkip.flatMap { ref => val qrSymbol = reflect.Symbol @@ -104,8 +116,7 @@ case class ScaladocTastyInspector()(using ctx: DocContext) extends Inspector: rootDoc = Some(parseCommentString(using parser.qctx, summon[DocContext])(content, topLevelPck, None)) } - for tasty <- tastys do { - val root = tasty.ast + hackForeachTree { root => if !isSkipped(root.symbol) then val treeRoot = root.asInstanceOf[parser.qctx.reflect.Tree] processRootDocIfNeeded(treeRoot) @@ -127,36 +138,15 @@ case class ScaladocTastyInspector()(using ctx: DocContext) extends Inspector: topLevels += "scala" -> Member(scalaPckg.fullName, "", scalaPckg.dri, Kind.Package) topLevels += mergeAnyRefAliasAndObject(parser) - - - def mergeAnyRefAliasAndObject(parser: TastyParser) = - import parser.qctx.reflect._ - val javaLangObjectDef = defn.ObjectClass.tree.asInstanceOf[ClassDef] - val objectMembers = parser.extractPatchedMembers(javaLangObjectDef) - val aM = parser.parseTypeDef(defn.AnyRefClass.tree.asInstanceOf[TypeDef]) - "scala" -> aM.copy( - kind = Kind.Class(Nil, Nil), - members = objectMembers - ) - -object ScaladocTastyInspector: - - def loadDocs()(using ctx: DocContext): (List[Member], Option[Comment]) = + def result(): (List[Member], Option[Comment]) = + topLevels.clear() + rootDoc = None val filePaths = ctx.args.tastyFiles.map(_.getAbsolutePath).toList val classpath = ctx.args.classpath.split(java.io.File.pathSeparator).toList - val inspector = new ScaladocTastyInspector - - val (tastyPaths, nonTastyPaths) = filePaths.partition(_.endsWith(".tasty")) - val (jarPaths, invalidPaths) = nonTastyPaths.partition(_.endsWith(".jar")) - - for invalidPath <- invalidPaths do - report.error("File extension is not `tasty` or `jar`: " + invalidPath) - - if tastyPaths.nonEmpty then - TastyInspector.inspectAllTastyFiles(tastyPaths, jarPaths, classpath)(inspector) + if filePaths.nonEmpty then inspectFilesInContext(classpath, filePaths) - val all = inspector.topLevels.result() + val all = topLevels.result() all.groupBy(_._1).map { case (pckName, members) => val (pcks, rest) = members.map(_._2).partition(_.kind == Kind.Package) val basePck = pcks.reduce( (p1, p2) => @@ -164,10 +154,17 @@ object ScaladocTastyInspector: if withNewMembers.docs.isEmpty then withNewMembers.withDocs(p2.docs) else withNewMembers ) basePck.withMembers((basePck.members ++ rest).sortBy(_.name)) - }.toList -> inspector.rootDoc - -end ScaladocTastyInspector + }.toList -> rootDoc + def mergeAnyRefAliasAndObject(parser: TastyParser) = + import parser.qctx.reflect._ + val javaLangObjectDef = defn.ObjectClass.tree.asInstanceOf[ClassDef] + val objectMembers = parser.extractPatchedMembers(javaLangObjectDef) + val aM = parser.parseTypeDef(defn.AnyRefClass.tree.asInstanceOf[TypeDef]) + "scala" -> aM.copy( + kind = Kind.Class(Nil, Nil), + members = objectMembers + ) /** Parses a single Tasty compilation unit. */ case class TastyParser( qctx: Quotes, diff --git a/scaladoc/src/scala/tasty/inspector/DocTastyInspector.scala b/scaladoc/src/scala/tasty/inspector/DocTastyInspector.scala new file mode 100644 index 000000000000..d908c2646f59 --- /dev/null +++ b/scaladoc/src/scala/tasty/inspector/DocTastyInspector.scala @@ -0,0 +1,9 @@ +package scala.tasty.inspector + +import dotty.tools.dotc.core.Contexts.Context + +abstract class DocTastyInspector extends OldTastyInspector: + def inspectFilesInDocContext( + classpath: List[String], + filePaths: List[String])( + using Context): Unit = inspectFilesInContext(classpath, filePaths) diff --git a/scaladoc/src/scala/tasty/inspector/OldTastyInspector.scala b/scaladoc/src/scala/tasty/inspector/OldTastyInspector.scala new file mode 100644 index 000000000000..16f9b0fdca1d --- /dev/null +++ b/scaladoc/src/scala/tasty/inspector/OldTastyInspector.scala @@ -0,0 +1,136 @@ +package scala.tasty.inspector + +import scala.quoted._ +import scala.quoted.runtime.impl.QuotesImpl + +import dotty.tools.dotc.Compiler +import dotty.tools.dotc.Driver +import dotty.tools.dotc.Run +import dotty.tools.dotc.core.Contexts.Context +import dotty.tools.dotc.core.Mode +import dotty.tools.dotc.core.Phases.Phase +import dotty.tools.dotc.fromtasty._ +import dotty.tools.dotc.util.ClasspathFromClassloader +import dotty.tools.dotc.CompilationUnit +import dotty.tools.unsupported +import dotty.tools.dotc.report + +import java.io.File.pathSeparator + +// COPY OF OLD IMPLEMENTATION +// TODO: update to new implementation +trait OldTastyInspector: + self => + + /** Process a TASTy file using TASTy reflect */ + protected def processCompilationUnit(using Quotes)(root: quotes.reflect.Tree): Unit + + /** Called after all compilation units are processed */ + protected def postProcess(using Quotes): Unit = () + + /** Load and process TASTy files using TASTy reflect + * + * @param tastyFiles List of paths of `.tasty` files + */ + def inspectTastyFiles(tastyFiles: List[String]): Boolean = + inspectAllTastyFiles(tastyFiles, Nil, Nil) + + /** Load and process TASTy files in a `jar` file using TASTy reflect + * + * @param jars Path of `.jar` file + */ + def inspectTastyFilesInJar(jar: String): Boolean = + inspectAllTastyFiles(Nil, List(jar), Nil) + + /** Load and process TASTy files using TASTy reflect + * + * @param tastyFiles List of paths of `.tasty` files + * @param jars List of path of `.jar` files + * @param dependenciesClasspath Classpath with extra dependencies needed to load class in the `.tasty` files + */ + def inspectAllTastyFiles(tastyFiles: List[String], jars: List[String], dependenciesClasspath: List[String]): Boolean = + def checkFile(fileName: String, ext: String): Unit = + val file = dotty.tools.io.Path(fileName) + if file.extension != ext then + throw new IllegalArgumentException(s"File extension is not `.$ext`: $file") + else if !file.exists then + throw new IllegalArgumentException(s"File not found: ${file.toAbsolute}") + tastyFiles.foreach(checkFile(_, "tasty")) + jars.foreach(checkFile(_, "jar")) + val files = tastyFiles ::: jars + files.nonEmpty && inspectFiles(dependenciesClasspath, files) + + /** Load and process TASTy files using TASTy reflect and provided context + * + * Used in doctool to reuse reporter and setup provided by sbt + * + * @param classes List of paths of `.tasty` and `.jar` files (no validation is performed) + * @param classpath Classpath with extra dependencies needed to load class in the `.tasty` files + */ + protected[inspector] def inspectFilesInContext(classpath: List[String], classes: List[String])(using Context): Unit = + if (classes.isEmpty) report.error("Parameter classes should no be empty") + inspectorDriver().process(inspectorArgs(classpath, classes), summon[Context]) + + + private def inspectorDriver() = + class InspectorDriver extends Driver: + override protected def newCompiler(implicit ctx: Context): Compiler = new TastyFromClass + + class TastyInspectorPhase extends Phase: + override def phaseName: String = "tastyInspector" + + override def run(implicit ctx: Context): Unit = + val qctx = QuotesImpl() + self.processCompilationUnit(using qctx)(ctx.compilationUnit.tpdTree.asInstanceOf[qctx.reflect.Tree]) + + class TastyInspectorFinishPhase extends Phase: + override def phaseName: String = "tastyInspectorFinish" + + override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] = + val qctx = QuotesImpl() + self.postProcess(using qctx) + units + + override def run(implicit ctx: Context): Unit = unsupported("run") + + class TastyFromClass extends TASTYCompiler: + + override protected def frontendPhases: List[List[Phase]] = + List(new ReadTasty) :: // Load classes from tasty + Nil + + override protected def picklerPhases: List[List[Phase]] = Nil + + override protected def transformPhases: List[List[Phase]] = Nil + + override protected def backendPhases: List[List[Phase]] = + List(new TastyInspectorPhase) :: // Perform a callback for each compilation unit + List(new TastyInspectorFinishPhase) :: // Perform a final callback + Nil + + override def newRun(implicit ctx: Context): Run = + reset() + val ctx2 = ctx.fresh + .addMode(Mode.ReadPositions) + .setSetting(ctx.settings.YreadComments, true) + new TASTYRun(this, ctx2) + + new InspectorDriver + + private def inspectorArgs(classpath: List[String], classes: List[String]): Array[String] = + val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader) + val fullClasspath = (classpath :+ currentClasspath).mkString(pathSeparator) + ("-from-tasty" :: "-Yretain-trees" :: "-classpath" :: fullClasspath :: classes).toArray + + + private def inspectFiles(classpath: List[String], classes: List[String]): Boolean = + if (classes.isEmpty) + throw new IllegalArgumentException("Parameter classes should no be empty") + + val reporter = inspectorDriver().process(inspectorArgs(classpath, classes)) + reporter.hasErrors + + end inspectFiles + + +end OldTastyInspector diff --git a/scaladoc/test/dotty/tools/scaladoc/tasty/comments/MemberLookupTests.scala b/scaladoc/test/dotty/tools/scaladoc/tasty/comments/MemberLookupTests.scala index 5f4b85251407..c246401c75fc 100644 --- a/scaladoc/test/dotty/tools/scaladoc/tasty/comments/MemberLookupTests.scala +++ b/scaladoc/test/dotty/tools/scaladoc/tasty/comments/MemberLookupTests.scala @@ -1,7 +1,7 @@ package dotty.tools.scaladoc package tasty.comments -import scala.quoted.* +import scala.quoted.Quotes import org.junit.{Test, Rule} import org.junit.Assert.{assertSame, assertTrue} @@ -198,11 +198,14 @@ class MemberLookupTests { @Test def test(): Unit = { - import scala.tasty.inspector.* - class MyInspector extends Inspector: + import scala.tasty.inspector.OldTastyInspector + class Inspector extends OldTastyInspector: + var alreadyRan: Boolean = false - def inspect(using Quotes)(tastys: List[Tasty[quotes.type]]): Unit = - this.test() + override def processCompilationUnit(using ctx: quoted.Quotes)(root: ctx.reflect.Tree): Unit = + if !alreadyRan then + this.test() + alreadyRan = true def test()(using q: Quotes): Unit = { import dotty.tools.scaladoc.tasty.comments.MemberLookup @@ -212,6 +215,6 @@ class MemberLookupTests { cases.testAll() } - TastyInspector.inspectTastyFiles(TestUtils.listOurClasses())(new MyInspector) + Inspector().inspectTastyFiles(TestUtils.listOurClasses()) } } From 6c7239c6b2a53b9d9f8438317b5d198746c762fc Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Thu, 1 Jun 2023 09:06:29 +0200 Subject: [PATCH 2/2] Revert "Copy TastyInspector into scaladoc" This reverts commit f36c3025c6f7241f4015717184e19dd910a755cb. --- .../src/scala/tasty/inspector/Inspector.scala | 33 ----- .../src/scala/tasty/inspector/Tasty.scala | 20 --- .../tasty/inspector/TastyInspector.scala | 126 ------------------ 3 files changed, 179 deletions(-) delete mode 100644 scaladoc/src/scala/tasty/inspector/Inspector.scala delete mode 100644 scaladoc/src/scala/tasty/inspector/Tasty.scala delete mode 100644 scaladoc/src/scala/tasty/inspector/TastyInspector.scala diff --git a/scaladoc/src/scala/tasty/inspector/Inspector.scala b/scaladoc/src/scala/tasty/inspector/Inspector.scala deleted file mode 100644 index 061c7dff0c44..000000000000 --- a/scaladoc/src/scala/tasty/inspector/Inspector.scala +++ /dev/null @@ -1,33 +0,0 @@ -// Copy of tasty-inspector/src/scala/tasty/inspector/Inspector.scala -// FIXME remove this copy of the file - -package scala.tasty.inspector - -import scala.quoted._ -import scala.quoted.runtime.impl.QuotesImpl - -import dotty.tools.dotc.Compiler -import dotty.tools.dotc.Driver -import dotty.tools.dotc.Run -import dotty.tools.dotc.core.Contexts.Context -import dotty.tools.dotc.core.Mode -import dotty.tools.dotc.core.Phases.Phase -import dotty.tools.dotc.fromtasty._ -import dotty.tools.dotc.util.ClasspathFromClassloader -import dotty.tools.dotc.CompilationUnit -import dotty.tools.unsupported -import dotty.tools.dotc.report - -import java.io.File.pathSeparator - -trait Inspector: - - /** Inspect all TASTy files using `Quotes` reflect API. - * - * Note: Within this method `quotes.reflect.SourceFile.current` will not work, hence the explicit source paths. - * - * @param tastys List of `Tasty` containing `.tasty`file path and AST - */ - def inspect(using Quotes)(tastys: List[Tasty[quotes.type]]): Unit - -end Inspector diff --git a/scaladoc/src/scala/tasty/inspector/Tasty.scala b/scaladoc/src/scala/tasty/inspector/Tasty.scala deleted file mode 100644 index b3e65bb5479e..000000000000 --- a/scaladoc/src/scala/tasty/inspector/Tasty.scala +++ /dev/null @@ -1,20 +0,0 @@ -// Copy of tasty-inspector/src/scala/tasty/inspector/Tasty.scala -// FIXME remove this copy of the file - -package scala.tasty.inspector - -import scala.quoted._ - -/** `.tasty` file representation containing file path and the AST */ -trait Tasty[Q <: Quotes & Singleton]: - - /** Instance of `Quotes` used to load the AST */ - val quotes: Q - - /** Path to the `.tasty` file */ - def path: String - - /** Abstract Syntax Tree contained in the `.tasty` file */ - def ast: quotes.reflect.Tree - -end Tasty diff --git a/scaladoc/src/scala/tasty/inspector/TastyInspector.scala b/scaladoc/src/scala/tasty/inspector/TastyInspector.scala deleted file mode 100644 index 00aa6c5e3771..000000000000 --- a/scaladoc/src/scala/tasty/inspector/TastyInspector.scala +++ /dev/null @@ -1,126 +0,0 @@ -// Copy of tasty-inspector/src/scala/tasty/inspector/TastyInspector.scala -// FIXME remove this copy of the file - -package scala.tasty.inspector - -import scala.quoted._ -import scala.quoted.runtime.impl.QuotesImpl - -import dotty.tools.dotc.Compiler -import dotty.tools.dotc.Driver -import dotty.tools.dotc.Run -import dotty.tools.dotc.core.Contexts.Context -import dotty.tools.dotc.core.Mode -import dotty.tools.dotc.core.Phases.Phase -import dotty.tools.dotc.fromtasty._ -import dotty.tools.dotc.quoted.QuotesCache -import dotty.tools.dotc.util.ClasspathFromClassloader -import dotty.tools.dotc.CompilationUnit -import dotty.tools.unsupported -import dotty.tools.dotc.report - -import java.io.File.pathSeparator - -object TastyInspector: - - /** Load and process TASTy files using TASTy reflect - * - * @param tastyFiles List of paths of `.tasty` files - * - * @return boolean value indicating whether the process succeeded - */ - def inspectTastyFiles(tastyFiles: List[String])(inspector: Inspector): Boolean = - inspectAllTastyFiles(tastyFiles, Nil, Nil)(inspector) - - /** Load and process TASTy files in a `jar` file using TASTy reflect - * - * @param jars Path of `.jar` file - * - * @return boolean value indicating whether the process succeeded - */ - def inspectTastyFilesInJar(jar: String)(inspector: Inspector): Boolean = - inspectAllTastyFiles(Nil, List(jar), Nil)(inspector) - - /** Load and process TASTy files using TASTy reflect - * - * @param tastyFiles List of paths of `.tasty` files - * @param jars List of path of `.jar` files - * @param dependenciesClasspath Classpath with extra dependencies needed to load class in the `.tasty` files - * - * @return boolean value indicating whether the process succeeded - */ - def inspectAllTastyFiles(tastyFiles: List[String], jars: List[String], dependenciesClasspath: List[String])(inspector: Inspector): Boolean = - def checkFile(fileName: String, ext: String): Unit = - val file = dotty.tools.io.Path(fileName) - if file.extension != ext then - throw new IllegalArgumentException(s"File extension is not `.$ext`: $file") - else if !file.exists then - throw new IllegalArgumentException(s"File not found: ${file.toAbsolute}") - tastyFiles.foreach(checkFile(_, "tasty")) - jars.foreach(checkFile(_, "jar")) - val files = tastyFiles ::: jars - inspectFiles(dependenciesClasspath, files)(inspector) - - private def inspectorDriver(inspector: Inspector) = - class InspectorDriver extends Driver: - override protected def newCompiler(implicit ctx: Context): Compiler = new TastyFromClass - - class TastyInspectorPhase extends Phase: - override def phaseName: String = "tastyInspector" - - override def runOn(units: List[CompilationUnit])(using ctx0: Context): List[CompilationUnit] = - val ctx = QuotesCache.init(ctx0.fresh) - runOnImpl(units)(using ctx) - - private def runOnImpl(units: List[CompilationUnit])(using Context): List[CompilationUnit] = - val quotesImpl = QuotesImpl() - class TastyImpl(val path: String, val ast: quotesImpl.reflect.Tree) extends Tasty[quotesImpl.type] { - val quotes = quotesImpl - } - val tastys = units.map(unit => new TastyImpl(unit.source.path , unit.tpdTree.asInstanceOf[quotesImpl.reflect.Tree])) - inspector.inspect(using quotesImpl)(tastys) - units - - override def run(implicit ctx: Context): Unit = unsupported("run") - end TastyInspectorPhase - - class TastyFromClass extends TASTYCompiler: - - override protected def frontendPhases: List[List[Phase]] = - List(new ReadTasty) :: // Load classes from tasty - Nil - - override protected def picklerPhases: List[List[Phase]] = Nil - - override protected def transformPhases: List[List[Phase]] = Nil - - override protected def backendPhases: List[List[Phase]] = - List(new TastyInspectorPhase) :: // Perform a callback for each compilation unit - Nil - - override def newRun(implicit ctx: Context): Run = - reset() - val ctx2 = ctx.fresh - .addMode(Mode.ReadPositions) - .setSetting(ctx.settings.YreadComments, true) - new TASTYRun(this, ctx2) - - new InspectorDriver - - private def inspectorArgs(classpath: List[String], classes: List[String]): Array[String] = - val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader) - val fullClasspath = (classpath :+ currentClasspath).mkString(pathSeparator) - ("-from-tasty" :: "-Yretain-trees" :: "-classpath" :: fullClasspath :: classes).toArray - - - private def inspectFiles(classpath: List[String], classes: List[String])(inspector: Inspector): Boolean = - classes match - case Nil => true - case _ => - val reporter = inspectorDriver(inspector).process(inspectorArgs(classpath, classes)) - !reporter.hasErrors - - end inspectFiles - - -end TastyInspector