From 607e4d59d93e4ceb0e46469923161516c4e04b60 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Thu, 25 May 2023 11:17:49 +0200 Subject: [PATCH 1/2] Loading symbols from TASTy files directly --- .../tools/backend/jvm/PostProcessor.scala | 10 ++- .../dotc/classpath/DirectoryClassPath.scala | 14 ++-- .../tools/dotc/classpath/FileUtils.scala | 36 +++++++++- .../classpath/VirtualDirectoryClassPath.scala | 11 +++- .../ZipAndJarFileLookupFactory.scala | 14 ++-- .../dotc/classpath/ZipArchiveFileLookup.scala | 9 +++ .../tools/dotc/config/JavaPlatform.scala | 3 + .../dotty/tools/dotc/config/Platform.scala | 3 + .../tools/dotc/config/ScalaSettings.scala | 1 + .../dotty/tools/dotc/core/SymbolLoaders.scala | 39 +++++++---- .../src/dotty/tools/dotc/core/Symbols.scala | 9 +-- .../dotc/core/classfile/ClassfileParser.scala | 65 +++++-------------- .../tools/dotc/fromtasty/ReadTasty.scala | 4 +- .../tools/dotc/sbt/ExtractDependencies.scala | 8 ++- .../src/dotty/tools/io/AbstractFile.scala | 6 ++ compiler/src/dotty/tools/io/ZipArchive.scala | 2 + sbt-test/tasty-compat/only-tasty/a/A.scala | 5 ++ sbt-test/tasty-compat/only-tasty/b/B.scala | 5 ++ sbt-test/tasty-compat/only-tasty/build.sbt | 16 +++++ sbt-test/tasty-compat/only-tasty/c/C.scala | 9 +++ .../project/DottyInjectedPlugin.scala | 11 ++++ .../only-tasty/project/build.properties | 1 + sbt-test/tasty-compat/only-tasty/test | 6 ++ .../tasty-inspector/tastyPaths.check | 2 +- 24 files changed, 197 insertions(+), 92 deletions(-) create mode 100644 sbt-test/tasty-compat/only-tasty/a/A.scala create mode 100644 sbt-test/tasty-compat/only-tasty/b/B.scala create mode 100644 sbt-test/tasty-compat/only-tasty/build.sbt create mode 100644 sbt-test/tasty-compat/only-tasty/c/C.scala create mode 100644 sbt-test/tasty-compat/only-tasty/project/DottyInjectedPlugin.scala create mode 100644 sbt-test/tasty-compat/only-tasty/project/build.properties create mode 100644 sbt-test/tasty-compat/only-tasty/test diff --git a/compiler/src/dotty/tools/backend/jvm/PostProcessor.scala b/compiler/src/dotty/tools/backend/jvm/PostProcessor.scala index 606b5645aa24..d1f9c6216a22 100644 --- a/compiler/src/dotty/tools/backend/jvm/PostProcessor.scala +++ b/compiler/src/dotty/tools/backend/jvm/PostProcessor.scala @@ -23,6 +23,12 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes: def postProcessAndSendToDisk(generatedDefs: GeneratedDefs): Unit = { val GeneratedDefs(classes, tasty) = generatedDefs + if !ctx.settings.YoutputOnlyTasty.value then + postProcessClassesAndSendToDisk(classes) + postProcessTastyAndSendToDisk(tasty) + } + + private def postProcessClassesAndSendToDisk(classes: List[GeneratedClass]): Unit = { for (GeneratedClass(classNode, sourceFile, isArtifact, onFileCreated) <- classes) { val bytes = try @@ -46,8 +52,10 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes: if clsFile != null then onFileCreated(clsFile) } } + } - for (GeneratedTasty(classNode, binaryGen) <- tasty){ + private def postProcessTastyAndSendToDisk(tasty: List[GeneratedTasty]): Unit = { + for (GeneratedTasty(classNode, binaryGen) <- tasty) { classfileWriter.writeTasty(classNode.name.nn, binaryGen()) } } diff --git a/compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala b/compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala index 1411493bcbfd..da1276f10dd7 100644 --- a/compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala +++ b/compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala @@ -278,15 +278,17 @@ case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[ClassFile def findClassFile(className: String): Option[AbstractFile] = { val relativePath = FileUtils.dirPath(className) - val classFile = new JFile(dir, relativePath + ".class") - if (classFile.exists) { - Some(classFile.toPath.toPlainFile) - } - else None + val tastyFile = new JFile(dir, relativePath + ".tasty") + if tastyFile.exists then Some(tastyFile.toPath.toPlainFile) + else + val classFile = new JFile(dir, relativePath + ".class") + if classFile.exists then Some(classFile.toPath.toPlainFile) + else None } protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file) - protected def isMatchingFile(f: JFile): Boolean = f.isClass + protected def isMatchingFile(f: JFile): Boolean = + f.isTasty || (f.isClass && f.classToTasty.isEmpty) private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage) } diff --git a/compiler/src/dotty/tools/dotc/classpath/FileUtils.scala b/compiler/src/dotty/tools/dotc/classpath/FileUtils.scala index d6fa6fb78d07..481419cc87d8 100644 --- a/compiler/src/dotty/tools/dotc/classpath/FileUtils.scala +++ b/compiler/src/dotty/tools/dotc/classpath/FileUtils.scala @@ -20,6 +20,10 @@ object FileUtils { def isClass: Boolean = !file.isDirectory && file.hasExtension("class") && !file.name.endsWith("$class.class") // FIXME: drop last condition when we stop being compatible with Scala 2.11 + def isTasty: Boolean = !file.isDirectory && file.hasExtension("tasty") + + def isScalaBinary: Boolean = file.isClass || file.isTasty + def isScalaOrJavaSource: Boolean = !file.isDirectory && (file.hasExtension("scala") || file.hasExtension("java")) // TODO do we need to check also other files using ZipMagicNumber like in scala.tools.nsc.io.Jar.isJarOrZip? @@ -30,17 +34,34 @@ object FileUtils { * and returning given default value in other case */ def toURLs(default: => Seq[URL] = Seq.empty): Seq[URL] = if (file.file == null) default else Seq(file.toURL) + + /** Returns the tasty file associated with this class file */ + def classToTasty: Option[AbstractFile] = + assert(file.isClass, s"non-class: $file") + val tastyName = classNameToTasty(file.name) + Option(file.resolveSibling(tastyName)) } extension (file: JFile) { def isPackage: Boolean = file.isDirectory && mayBeValidPackage(file.getName) - def isClass: Boolean = file.isFile && file.getName.endsWith(".class") && !file.getName.endsWith("$class.class") - // FIXME: drop last condition when we stop being compatible with Scala 2.11 + def isClass: Boolean = file.isFile && file.getName.endsWith(SUFFIX_CLASS) && !file.getName.endsWith("$class.class") + // FIXME: drop last condition when we stop being compatible with Scala 2.11 + + def isTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_TASTY) + + /** Returns the tasty file associated with this class file */ + def classToTasty: Option[JFile] = + assert(file.isClass, s"non-class: $file") + val tastyName = classNameToTasty(file.getName.stripSuffix(".class")) + val tastyPath = file.toPath.resolveSibling(tastyName) + if java.nio.file.Files.exists(tastyPath) then Some(tastyPath.toFile) else None + } private val SUFFIX_CLASS = ".class" private val SUFFIX_SCALA = ".scala" + private val SUFFIX_TASTY = ".tasty" private val SUFFIX_JAVA = ".java" private val SUFFIX_SIG = ".sig" @@ -81,4 +102,15 @@ object FileUtils { def mkFileFilter(f: JFile => Boolean): FileFilter = new FileFilter { def accept(pathname: JFile): Boolean = f(pathname) } + + /** Transforms a .class file name to a .tasty file name */ + private def classNameToTasty(fileName: String): String = + val classOrModuleName = fileName.stripSuffix(".class") + val className = + if classOrModuleName.endsWith("$") + && classOrModuleName != "Null$" // scala.runtime.Null$ + && classOrModuleName != "Nothing$" // scala.runtime.Nothing$ + then classOrModuleName.stripSuffix("$") + else classOrModuleName + className + SUFFIX_TASTY } diff --git a/compiler/src/dotty/tools/dotc/classpath/VirtualDirectoryClassPath.scala b/compiler/src/dotty/tools/dotc/classpath/VirtualDirectoryClassPath.scala index e750d9ccacc0..4b777444c3bf 100644 --- a/compiler/src/dotty/tools/dotc/classpath/VirtualDirectoryClassPath.scala +++ b/compiler/src/dotty/tools/dotc/classpath/VirtualDirectoryClassPath.scala @@ -41,12 +41,17 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi override def findClass(className: String): Option[ClassRepresentation] = findClassFile(className) map ClassFileEntryImpl.apply def findClassFile(className: String): Option[AbstractFile] = { - val relativePath = FileUtils.dirPath(className) + ".class" - Option(lookupPath(dir)(relativePath.split(java.io.File.separator).toIndexedSeq, directory = false)) + val pathSeq = FileUtils.dirPath(className).split(java.io.File.separator) + val parentDir = lookupPath(dir)(pathSeq.init.toSeq, directory = true) + if parentDir == null then return None + else + Option(lookupPath(parentDir)(pathSeq.last + ".tasty" :: Nil, directory = false)) + .orElse(Option(lookupPath(parentDir)(pathSeq.last + ".class" :: Nil, directory = false))) } private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage) protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file) - protected def isMatchingFile(f: AbstractFile): Boolean = f.isClass + protected def isMatchingFile(f: AbstractFile): Boolean = + f.isTasty || (f.isClass && f.classToTasty.isEmpty) } diff --git a/compiler/src/dotty/tools/dotc/classpath/ZipAndJarFileLookupFactory.scala b/compiler/src/dotty/tools/dotc/classpath/ZipAndJarFileLookupFactory.scala index 865f95551a0b..b38e1841728d 100644 --- a/compiler/src/dotty/tools/dotc/classpath/ZipAndJarFileLookupFactory.scala +++ b/compiler/src/dotty/tools/dotc/classpath/ZipAndJarFileLookupFactory.scala @@ -44,21 +44,21 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory { extends ZipArchiveFileLookup[ClassFileEntryImpl] with NoSourcePaths { - override def findClassFile(className: String): Option[AbstractFile] = { - val (pkg, simpleClassName) = PackageNameUtils.separatePkgAndClassNames(className) - file(PackageName(pkg), simpleClassName + ".class").map(_.file) - } + override def findClassFile(className: String): Option[AbstractFile] = + findClass(className).map(_.file) // This method is performance sensitive as it is used by SBT's ExtractDependencies phase. - override def findClass(className: String): Option[ClassRepresentation] = { + override def findClass(className: String): Option[ClassFileEntryImpl] = { val (pkg, simpleClassName) = PackageNameUtils.separatePkgAndClassNames(className) - file(PackageName(pkg), simpleClassName + ".class") + val binaries = files(PackageName(pkg), simpleClassName + ".tasty", simpleClassName + ".class") + binaries.find(_.file.isTasty).orElse(binaries.find(_.file.isClass)) } override private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage) override protected def createFileEntry(file: FileZipArchive#Entry): ClassFileEntryImpl = ClassFileEntryImpl(file) - override protected def isRequiredFileType(file: AbstractFile): Boolean = file.isClass + override protected def isRequiredFileType(file: AbstractFile): Boolean = + file.isTasty || (file.isClass && file.classToTasty.isEmpty) } /** diff --git a/compiler/src/dotty/tools/dotc/classpath/ZipArchiveFileLookup.scala b/compiler/src/dotty/tools/dotc/classpath/ZipArchiveFileLookup.scala index e241feee8244..8033291f5dd3 100644 --- a/compiler/src/dotty/tools/dotc/classpath/ZipArchiveFileLookup.scala +++ b/compiler/src/dotty/tools/dotc/classpath/ZipArchiveFileLookup.scala @@ -43,6 +43,15 @@ trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends Efficie } yield createFileEntry(entry) + protected def files(inPackage: PackageName, names: String*): Seq[FileEntryType] = + for { + dirEntry <- findDirEntry(inPackage).toSeq + name <- names + entry <- Option(dirEntry.lookupName(name, directory = false)) + if isRequiredFileType(entry) + } + yield createFileEntry(entry) + protected def file(inPackage: PackageName, name: String): Option[FileEntryType] = for { dirEntry <- findDirEntry(inPackage) diff --git a/compiler/src/dotty/tools/dotc/config/JavaPlatform.scala b/compiler/src/dotty/tools/dotc/config/JavaPlatform.scala index 2b2f35e49451..f611360dd4ca 100644 --- a/compiler/src/dotty/tools/dotc/config/JavaPlatform.scala +++ b/compiler/src/dotty/tools/dotc/config/JavaPlatform.scala @@ -66,4 +66,7 @@ class JavaPlatform extends Platform { def newClassLoader(bin: AbstractFile)(using Context): SymbolLoader = new ClassfileLoader(bin) + + def newTastyLoader(bin: AbstractFile)(using Context): SymbolLoader = + new TastyLoader(bin) } diff --git a/compiler/src/dotty/tools/dotc/config/Platform.scala b/compiler/src/dotty/tools/dotc/config/Platform.scala index 0faacf1bcebb..73a05fbd41c1 100644 --- a/compiler/src/dotty/tools/dotc/config/Platform.scala +++ b/compiler/src/dotty/tools/dotc/config/Platform.scala @@ -36,6 +36,9 @@ abstract class Platform { /** Create a new class loader to load class file `bin` */ def newClassLoader(bin: AbstractFile)(using Context): SymbolLoader + /** Create a new TASTy loader to load class file `bin` */ + def newTastyLoader(bin: AbstractFile)(using Context): SymbolLoader + /** The given symbol is a method with the right name and signature to be a runnable program. */ def isMainMethod(sym: Symbol)(using Context): Boolean diff --git a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala index 43df8845157c..e685d8664037 100644 --- a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala +++ b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala @@ -369,6 +369,7 @@ private sealed trait YSettings: val YnoExperimental: Setting[Boolean] = BooleanSetting("-Yno-experimental", "Disable experimental language features.") val YlegacyLazyVals: Setting[Boolean] = BooleanSetting("-Ylegacy-lazy-vals", "Use legacy (pre 3.3.0) implementation of lazy vals.") val Yscala2Stdlib: Setting[Boolean] = BooleanSetting("-Yscala2-stdlib", "Used when compiling the Scala 2 standard library.") + val YoutputOnlyTasty: Setting[Boolean] = BooleanSetting("-Youtput-only-tasty", "Used to only generate the TASTy file without the classfiles") val YprofileEnabled: Setting[Boolean] = BooleanSetting("-Yprofile-enabled", "Enable profiling.") val YprofileDestination: Setting[String] = StringSetting("-Yprofile-destination", "file", "Where to send profiling output - specify a file, default is to the console.", "") diff --git a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala index 9eb67b468cfa..8fe7b3451186 100644 --- a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala +++ b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala @@ -7,6 +7,7 @@ import java.nio.channels.ClosedByInterruptException import scala.util.control.NonFatal +import dotty.tools.dotc.classpath.FileUtils.isTasty import dotty.tools.io.{ ClassPath, ClassRepresentation, AbstractFile } import dotty.tools.backend.jvm.DottyBackendInterface.symExtensions @@ -192,10 +193,13 @@ object SymbolLoaders { if (ctx.settings.verbose.value) report.inform("[symloader] picked up newer source file for " + src.path) enterToplevelsFromSource(owner, nameOf(classRep), src) case (None, Some(src)) => - if (ctx.settings.verbose.value) report.inform("[symloader] no class, picked up source file for " + src.path) + if (ctx.settings.verbose.value) report.inform("[symloader] no class or tasty, picked up source file for " + src.path) enterToplevelsFromSource(owner, nameOf(classRep), src) case (Some(bin), _) => - enterClassAndModule(owner, nameOf(classRep), ctx.platform.newClassLoader(bin)) + val completer = + if bin.isTasty then ctx.platform.newTastyLoader(bin) + else ctx.platform.newClassLoader(bin) + enterClassAndModule(owner, nameOf(classRep), completer) } def needCompile(bin: AbstractFile, src: AbstractFile): Boolean = @@ -404,20 +408,27 @@ class ClassfileLoader(val classfile: AbstractFile) extends SymbolLoader { def description(using Context): String = "class file " + classfile.toString override def doComplete(root: SymDenotation)(using Context): Unit = - load(root) - - def load(root: SymDenotation)(using Context): Unit = { val (classRoot, moduleRoot) = rootDenots(root.asClass) val classfileParser = new ClassfileParser(classfile, classRoot, moduleRoot)(ctx) - val result = classfileParser.run() - if (mayLoadTreesFromTasty) - result match { - case Some(unpickler: tasty.DottyUnpickler) => - classRoot.classSymbol.rootTreeOrProvider = unpickler - moduleRoot.classSymbol.rootTreeOrProvider = unpickler - case _ => - } - } + classfileParser.run() +} + +class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader { + + override def sourceFileOrNull: AbstractFile | Null = tastyFile + + def description(using Context): String = "TASTy file " + tastyFile.toString + + override def doComplete(root: SymDenotation)(using Context): Unit = + val (classRoot, moduleRoot) = rootDenots(root.asClass) + val unpickler = + val tastyBytes = tastyFile.toByteArray + new tasty.DottyUnpickler(tastyBytes) + unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource)) + if mayLoadTreesFromTasty then + classRoot.classSymbol.rootTreeOrProvider = unpickler + moduleRoot.classSymbol.rootTreeOrProvider = unpickler + // TODO check TASTy UUID matches classfile private def mayLoadTreesFromTasty(using Context): Boolean = ctx.settings.YretainTrees.value || ctx.settings.fromTasty.value diff --git a/compiler/src/dotty/tools/dotc/core/Symbols.scala b/compiler/src/dotty/tools/dotc/core/Symbols.scala index c13591fb9f5a..9bfd35828500 100644 --- a/compiler/src/dotty/tools/dotc/core/Symbols.scala +++ b/compiler/src/dotty/tools/dotc/core/Symbols.scala @@ -31,6 +31,7 @@ import io.AbstractFile import util.{SourceFile, NoSource, Property, SourcePosition, SrcPos, EqHashMap} import scala.annotation.internal.sharable import config.Printers.typr +import dotty.tools.dotc.classpath.FileUtils.isScalaBinary object Symbols { @@ -151,7 +152,7 @@ object Symbols { * symbols defined by the user in a prior run of the REPL, that are still valid. */ final def isDefinedInSource(using Context): Boolean = - span.exists && isValidInCurrentRun && associatedFileMatches(_.extension != "class") + span.exists && isValidInCurrentRun && associatedFileMatches(!_.isScalaBinary) /** Is symbol valid in current run? */ final def isValidInCurrentRun(using Context): Boolean = @@ -272,7 +273,7 @@ object Symbols { /** The class file from which this class was generated, null if not applicable. */ final def binaryFile(using Context): AbstractFile | Null = { val file = associatedFile - if (file != null && file.extension == "class") file else null + if file != null && file.isScalaBinary then file else null } /** A trap to avoid calling x.symbol on something that is already a symbol. @@ -285,7 +286,7 @@ object Symbols { final def source(using Context): SourceFile = { def valid(src: SourceFile): SourceFile = - if (src.exists && src.file.extension != "class") src + if (src.exists && !src.file.isScalaBinary) src else NoSource if (!denot.exists) NoSource @@ -463,7 +464,7 @@ object Symbols { if !mySource.exists && !denot.is(Package) then // this allows sources to be added in annotations after `sourceOfClass` is first called val file = associatedFile - if file != null && file.extension != "class" then + if file != null && !file.isScalaBinary then mySource = ctx.getSource(file) else mySource = defn.patchSource(this) diff --git a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala index 71e00f985584..b3a2aaa87193 100644 --- a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala +++ b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala @@ -12,7 +12,7 @@ import SymDenotations._, unpickleScala2.Scala2Unpickler._, Constants._, Annotati import Phases._ import ast.{ tpd, untpd } import ast.tpd._, util._ -import java.io.{ ByteArrayOutputStream, IOException } +import java.io.IOException import java.lang.Integer.toHexString import java.util.UUID @@ -23,6 +23,7 @@ import scala.annotation.switch import typer.Checking.checkNonCyclic import io.{AbstractFile, ZipArchive} import scala.util.control.NonFatal +import dotty.tools.dotc.classpath.FileUtils.classToTasty object ClassfileParser { /** Marker trait for unpicklers that can be embedded in classfiles. */ @@ -918,12 +919,6 @@ class ClassfileParser( Some(unpickler) } - def unpickleTASTY(bytes: Array[Byte]): Some[Embedded] = { - val unpickler = new tasty.DottyUnpickler(bytes) - unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource)) - Some(unpickler) - } - def parseScalaSigBytes: Array[Byte] = { val tag = in.nextByte.toChar assert(tag == STRING_TAG, tag) @@ -947,49 +942,19 @@ class ClassfileParser( val attrLen = in.nextInt val bytes = in.nextBytes(attrLen) if (attrLen == 16) { // A tasty attribute with that has only a UUID (16 bytes) implies the existence of the .tasty file - val tastyBytes: Array[Byte] = classfile match { // TODO: simplify when #3552 is fixed - case classfile: io.ZipArchive#Entry => // We are in a jar - val path = classfile.parent.lookupName( - classfile.name.stripSuffix(".class") + ".tasty", directory = false - ) - if (path != null) { - val stream = path.input - try { - val tastyOutStream = new ByteArrayOutputStream() - val buffer = new Array[Byte](1024) - var read = stream.read(buffer, 0, buffer.length) - while (read != -1) { - tastyOutStream.write(buffer, 0, read) - read = stream.read(buffer, 0, buffer.length) - } - tastyOutStream.flush() - tastyOutStream.toByteArray - } finally { - stream.close() - } - } - else { - report.error(em"Could not find $path in ${classfile.underlyingSource}") - Array.empty - } - case _ => - val dir = classfile.container - val name = classfile.name.stripSuffix(".class") + ".tasty" - val tastyFileOrNull = dir.lookupName(name, false) - if (tastyFileOrNull == null) { - report.error(em"Could not find TASTY file $name under $dir") - Array.empty - } else - tastyFileOrNull.toByteArray - } - if (tastyBytes.nonEmpty) { - val reader = new TastyReader(bytes, 0, 16) - val expectedUUID = new UUID(reader.readUncompressedLong(), reader.readUncompressedLong()) - val tastyUUID = new TastyHeaderUnpickler(tastyBytes).readHeader() - if (expectedUUID != tastyUUID) - report.warning(s"$classfile is out of sync with its TASTy file. Loaded TASTy file. Try cleaning the project to fix this issue", NoSourcePosition) - return unpickleTASTY(tastyBytes) - } + classfile.classToTasty match + case None => + report.error(em"Could not find TASTY for $classfile") + case Some(tastyFile) => + val expectedUUID = + val reader = new TastyReader(bytes, 0, 16) + new UUID(reader.readUncompressedLong(), reader.readUncompressedLong()) + val tastyUUID = + val tastyBytes: Array[Byte] = tastyFile.toByteArray + new TastyHeaderUnpickler(tastyBytes).readHeader() + if (expectedUUID != tastyUUID) + report.warning(s"$classfile is out of sync with its TASTy file. Loaded TASTy file. Try cleaning the project to fix this issue", NoSourcePosition) + return None } else // Before 3.0.0 we had a mode where we could embed the TASTY bytes in the classfile. This has not been supported in any stable release. diff --git a/compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala b/compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala index 86ae99b3e0f9..98cc496681e8 100644 --- a/compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala +++ b/compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala @@ -62,8 +62,8 @@ class ReadTasty extends Phase { staticRef(className) match { case clsd: ClassDenotation => clsd.infoOrCompleter match { - case info: ClassfileLoader => - info.load(clsd) // sets cls.rootTreeOrProvider and cls.moduleClass.treeProvider as a side-effect + case info: TastyLoader => + info.doComplete(clsd) // sets cls.rootTreeOrProvider and cls.moduleClass.treeProvider as a side-effect case _ => } def moduleClass = clsd.owner.info.member(className.moduleClassName).symbol diff --git a/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala b/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala index fe5c8d061c78..5e7bedba5e4b 100644 --- a/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala +++ b/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala @@ -7,6 +7,7 @@ import java.io.File import java.util.{Arrays, EnumSet} import dotty.tools.dotc.ast.tpd +import dotty.tools.dotc.classpath.FileUtils.{isTasty, isClass} import dotty.tools.dotc.core.Contexts._ import dotty.tools.dotc.core.Decorators._ import dotty.tools.dotc.core.Flags._ @@ -141,9 +142,12 @@ class ExtractDependencies extends Phase { if (depFile != null) { // Cannot ignore inheritance relationship coming from the same source (see sbt/zinc#417) def allowLocal = dep.context == DependencyByInheritance || dep.context == LocalDependencyByInheritance - if (depFile.extension == "class") { + val depClassFile = + if depFile.isClass then depFile + else depFile.resolveSibling(dep.to.binaryClassName + ".class") + if (depClassFile != null) { // Dependency is external -- source is undefined - processExternalDependency(depFile, dep.to.binaryClassName) + processExternalDependency(depClassFile, dep.to.binaryClassName) } else if (allowLocal || depFile.file != sourceFile) { // We cannot ignore dependencies coming from the same source file because // the dependency info needs to propagate. See source-dependencies/trait-trait-211. diff --git a/compiler/src/dotty/tools/io/AbstractFile.scala b/compiler/src/dotty/tools/io/AbstractFile.scala index f34fe6f40b9c..fd9e2281181b 100644 --- a/compiler/src/dotty/tools/io/AbstractFile.scala +++ b/compiler/src/dotty/tools/io/AbstractFile.scala @@ -250,6 +250,12 @@ abstract class AbstractFile extends Iterable[AbstractFile] { file } + /** Returns the sibling abstract file in the parent of this abstract file or directory. + * If there is no such file, returns `null`. + */ + def resolveSibling(name: String): AbstractFile = + container.lookupName(name, directory = false) + private def fileOrSubdirectoryNamed(name: String, isDir: Boolean): AbstractFile = lookupName(name, isDir) match { case null => diff --git a/compiler/src/dotty/tools/io/ZipArchive.scala b/compiler/src/dotty/tools/io/ZipArchive.scala index 4383bc187979..0fb2a84b0579 100644 --- a/compiler/src/dotty/tools/io/ZipArchive.scala +++ b/compiler/src/dotty/tools/io/ZipArchive.scala @@ -72,6 +72,8 @@ abstract class ZipArchive(override val jpath: JPath, release: Option[String]) ex // have to keep this name for compat with sbt's compiler-interface def getArchive: ZipFile = null override def underlyingSource: Option[ZipArchive] = Some(self) + override def resolveSibling(name: String): AbstractFile = + parent.lookupName(name, directory = false) override def toString: String = self.path + "(" + path + ")" } diff --git a/sbt-test/tasty-compat/only-tasty/a/A.scala b/sbt-test/tasty-compat/only-tasty/a/A.scala new file mode 100644 index 000000000000..79a242ade5cd --- /dev/null +++ b/sbt-test/tasty-compat/only-tasty/a/A.scala @@ -0,0 +1,5 @@ +package a + +object A: + def f(x: Int): Int = x + 1 + inline def g(x: Int): Int = x + 1 diff --git a/sbt-test/tasty-compat/only-tasty/b/B.scala b/sbt-test/tasty-compat/only-tasty/b/B.scala new file mode 100644 index 000000000000..3241f5060d5c --- /dev/null +++ b/sbt-test/tasty-compat/only-tasty/b/B.scala @@ -0,0 +1,5 @@ +package b + +object B: + def f(x: Int): Int = x + 2 + inline def g(x: Int): Int = x + 2 diff --git a/sbt-test/tasty-compat/only-tasty/build.sbt b/sbt-test/tasty-compat/only-tasty/build.sbt new file mode 100644 index 000000000000..ae52f2b53768 --- /dev/null +++ b/sbt-test/tasty-compat/only-tasty/build.sbt @@ -0,0 +1,16 @@ +lazy val a = project.in(file("a")) + .settings( + scalacOptions += "-Youtput-only-tasty", + ) + +lazy val b = project.in(file("b")) + .settings( + scalacOptions += "-Youtput-only-tasty", + Compile / exportJars := true, + ) + +lazy val c = project.in(file("c")) + .dependsOn(a, b) + .settings( + scalacOptions += "-Ycheck:all", + ) diff --git a/sbt-test/tasty-compat/only-tasty/c/C.scala b/sbt-test/tasty-compat/only-tasty/c/C.scala new file mode 100644 index 000000000000..4d14c9134813 --- /dev/null +++ b/sbt-test/tasty-compat/only-tasty/c/C.scala @@ -0,0 +1,9 @@ +import a.A +import b.B + +object C extends App { + assert(A.f(0) == 1) + assert(A.g(0) == 1) + assert(B.f(0) == 2) + assert(B.g(0) == 2) +} diff --git a/sbt-test/tasty-compat/only-tasty/project/DottyInjectedPlugin.scala b/sbt-test/tasty-compat/only-tasty/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..fb946c4b8c61 --- /dev/null +++ b/sbt-test/tasty-compat/only-tasty/project/DottyInjectedPlugin.scala @@ -0,0 +1,11 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion") + ) +} diff --git a/sbt-test/tasty-compat/only-tasty/project/build.properties b/sbt-test/tasty-compat/only-tasty/project/build.properties new file mode 100644 index 000000000000..46e43a97ed86 --- /dev/null +++ b/sbt-test/tasty-compat/only-tasty/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.8.2 diff --git a/sbt-test/tasty-compat/only-tasty/test b/sbt-test/tasty-compat/only-tasty/test new file mode 100644 index 000000000000..31b609222ed6 --- /dev/null +++ b/sbt-test/tasty-compat/only-tasty/test @@ -0,0 +1,6 @@ +# compile library A +> a/compile +# compile library B +> b/compile +# compile library C, from source, against TASTy of A and B (where B loaded from jar) +> c/compile diff --git a/tests/run-custom-args/tasty-inspector/tastyPaths.check b/tests/run-custom-args/tasty-inspector/tastyPaths.check index 4aab4bcb2590..f9d3486040b8 100644 --- a/tests/run-custom-args/tasty-inspector/tastyPaths.check +++ b/tests/run-custom-args/tasty-inspector/tastyPaths.check @@ -1,2 +1,2 @@ -List(/tastyPaths/I8163.class) +List(/tastyPaths/I8163.tasty) `reflect.SourceFile.current` cannot be called within the TASTy ispector From f9e8b366fb8ddb516c08f9b19f4220a0deb20d47 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Tue, 6 Jun 2023 09:56:12 +0200 Subject: [PATCH 2/2] Check TASTy UUIDs in classfiles --- .../dotty/tools/dotc/core/SymbolLoaders.scala | 22 +- .../dotc/core/classfile/ClassfileParser.scala | 375 +++++++++--------- .../classfile/ClassfileTastyUUIDParser.scala | 116 ++++++ .../src/dotty/tools/io/AbstractFile.scala | 2 +- 4 files changed, 320 insertions(+), 195 deletions(-) create mode 100644 compiler/src/dotty/tools/dotc/core/classfile/ClassfileTastyUUIDParser.scala diff --git a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala index 8fe7b3451186..734688ea2ec8 100644 --- a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala +++ b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala @@ -14,7 +14,7 @@ import dotty.tools.backend.jvm.DottyBackendInterface.symExtensions import Contexts._, Symbols._, Flags._, SymDenotations._, Types._, Scopes._, Names._ import NameOps._ import StdNames._ -import classfile.ClassfileParser +import classfile.{ClassfileParser, ClassfileTastyUUIDParser} import Decorators._ import util.Stats @@ -24,6 +24,7 @@ import ast.desugar import parsing.JavaParsers.OutlineJavaParser import parsing.Parsers.OutlineParser +import dotty.tools.tasty.TastyHeaderUnpickler object SymbolLoaders { @@ -421,14 +422,25 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader { override def doComplete(root: SymDenotation)(using Context): Unit = val (classRoot, moduleRoot) = rootDenots(root.asClass) - val unpickler = - val tastyBytes = tastyFile.toByteArray - new tasty.DottyUnpickler(tastyBytes) + val tastyBytes = tastyFile.toByteArray + val unpickler = new tasty.DottyUnpickler(tastyBytes) unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource)) if mayLoadTreesFromTasty then classRoot.classSymbol.rootTreeOrProvider = unpickler moduleRoot.classSymbol.rootTreeOrProvider = unpickler - // TODO check TASTy UUID matches classfile + checkTastyUUID(tastyFile, tastyBytes) + + + private def checkTastyUUID(tastyFile: AbstractFile, tastyBytes: Array[Byte])(using Context): Unit = + var classfile = tastyFile.resolveSibling(tastyFile.name.stripSuffix(".tasty") + ".class") + if classfile == null then + classfile = tastyFile.resolveSibling(tastyFile.name.stripSuffix(".tasty") + "$.class") + if classfile != null then + val tastyUUID = new TastyHeaderUnpickler(tastyBytes).readHeader() + new ClassfileTastyUUIDParser(classfile)(ctx).checkTastyUUID(tastyUUID) + else + // This will be the case in any of our tests that compile with `-Youtput-only-tasty` + report.inform(s"No classfiles found for $tastyFile when checking TASTy UUID") private def mayLoadTreesFromTasty(using Context): Boolean = ctx.settings.YretainTrees.value || ctx.settings.fromTasty.value diff --git a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala index b3a2aaa87193..5e816502f359 100644 --- a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala +++ b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala @@ -26,6 +26,9 @@ import scala.util.control.NonFatal import dotty.tools.dotc.classpath.FileUtils.classToTasty object ClassfileParser { + + import ClassfileConstants._ + /** Marker trait for unpicklers that can be embedded in classfiles. */ trait Embedded @@ -51,6 +54,177 @@ object ClassfileParser { mapOver(tp) } } + + abstract class AbstractConstantPool(using in: DataReader) { + protected val len = in.nextChar + protected val starts = new Array[Int](len) + protected val values = new Array[AnyRef](len) + protected val internalized = new Array[NameOrString](len) + + { var i = 1 + while (i < starts.length) { + starts(i) = in.bp + i += 1 + (in.nextByte.toInt: @switch) match { + case CONSTANT_UTF8 | CONSTANT_UNICODE => + in.skip(in.nextChar) + case CONSTANT_CLASS | CONSTANT_STRING | CONSTANT_METHODTYPE => + in.skip(2) + case CONSTANT_METHODHANDLE => + in.skip(3) + case CONSTANT_FIELDREF | CONSTANT_METHODREF | CONSTANT_INTFMETHODREF + | CONSTANT_NAMEANDTYPE | CONSTANT_INTEGER | CONSTANT_FLOAT + | CONSTANT_INVOKEDYNAMIC => + in.skip(4) + case CONSTANT_LONG | CONSTANT_DOUBLE => + in.skip(8) + i += 1 + case _ => + errorBadTag(in.bp - 1) + } + } + } + + /** Return the name found at given index. */ + def getName(index: Int)(using in: DataReader): NameOrString = { + if (index <= 0 || len <= index) + errorBadIndex(index) + + values(index) match { + case name: NameOrString => name + case null => + val start = starts(index) + if (in.getByte(start).toInt != CONSTANT_UTF8) errorBadTag(start) + val len = in.getChar(start + 1).toInt + val name = new NameOrString(in.getUTF(start + 1, len + 2)) + values(index) = name + name + } + } + + /** Return the name found at given index in the constant pool, with '/' replaced by '.'. */ + def getExternalName(index: Int)(using in: DataReader): NameOrString = { + if (index <= 0 || len <= index) + errorBadIndex(index) + + if (internalized(index) == null) + internalized(index) = new NameOrString(getName(index).value.replace('/', '.')) + + internalized(index) + } + + def getClassSymbol(index: Int)(using ctx: Context, in: DataReader): Symbol + + /** Return the external name of the class info structure found at 'index'. + * Use 'getClassSymbol' if the class is sure to be a top-level class. + */ + def getClassName(index: Int)(using in: DataReader): NameOrString = { + val start = starts(index) + if (in.getByte(start).toInt != CONSTANT_CLASS) errorBadTag(start) + getExternalName(in.getChar(start + 1)) + } + + /** Return the type of a class constant entry. Since + * arrays are considered to be class types, they might + * appear as entries in 'newarray' or 'cast' opcodes. + */ + def getClassOrArrayType(index: Int)(using ctx: Context, in: DataReader): Type + + def getType(index: Int, isVarargs: Boolean = false)(using Context, DataReader): Type + + def getSuperClass(index: Int)(using Context, DataReader): Symbol = { + assert(index != 0, "attempt to parse java.lang.Object from classfile") + getClassSymbol(index) + } + + def getConstant(index: Int)(using ctx: Context, in: DataReader): Constant = { + if (index <= 0 || len <= index) errorBadIndex(index) + var value = values(index) + if (value eq null) { + val start = starts(index) + value = (in.getByte(start).toInt: @switch) match { + case CONSTANT_STRING => + Constant(getName(in.getChar(start + 1).toInt).value) + case CONSTANT_INTEGER => + Constant(in.getInt(start + 1)) + case CONSTANT_FLOAT => + Constant(in.getFloat(start + 1)) + case CONSTANT_LONG => + Constant(in.getLong(start + 1)) + case CONSTANT_DOUBLE => + Constant(in.getDouble(start + 1)) + case CONSTANT_CLASS => + getClassOrArrayType(index).typeSymbol + case _ => + errorBadTag(start) + } + values(index) = value + } + value match { + case ct: Constant => ct + case cls: Symbol => Constant(cls.typeRef) + case arr: Type => Constant(arr) + } + } + + private def getSubArray(bytes: Array[Byte]): Array[Byte] = { + val decodedLength = ByteCodecs.decode(bytes) + val arr = new Array[Byte](decodedLength) + System.arraycopy(bytes, 0, arr, 0, decodedLength) + arr + } + + def getBytes(index: Int)(using in: DataReader): Array[Byte] = { + if (index <= 0 || len <= index) errorBadIndex(index) + var value = values(index).asInstanceOf[Array[Byte]] + if (value eq null) { + val start = starts(index) + if (in.getByte(start).toInt != CONSTANT_UTF8) errorBadTag(start) + val len = in.getChar(start + 1) + val bytes = new Array[Byte](len) + in.getBytes(start + 3, bytes) + value = getSubArray(bytes) + values(index) = value + } + value + } + + def getBytes(indices: List[Int])(using in: DataReader): Array[Byte] = { + assert(!indices.isEmpty, indices) + var value = values(indices.head).asInstanceOf[Array[Byte]] + if (value eq null) { + val bytesBuffer = ArrayBuffer.empty[Byte] + for (index <- indices) { + if (index <= 0 || AbstractConstantPool.this.len <= index) errorBadIndex(index) + val start = starts(index) + if (in.getByte(start).toInt != CONSTANT_UTF8) errorBadTag(start) + val len = in.getChar(start + 1) + val buf = new Array[Byte](len) + in.getBytes(start + 3, buf) + bytesBuffer ++= buf + } + value = getSubArray(bytesBuffer.toArray) + values(indices.head) = value + } + value + } + + /** Throws an exception signaling a bad constant index. */ + protected def errorBadIndex(index: Int)(using in: DataReader) = + throw new RuntimeException("bad constant pool index: " + index + " at pos: " + in.bp) + + /** Throws an exception signaling a bad tag at given address. */ + protected def errorBadTag(start: Int)(using in: DataReader) = + throw new RuntimeException("bad constant pool tag " + in.getByte(start) + " at byte " + start) + } + + protected class NameOrString(val value: String) { + private var _name: SimpleName = null + def name: SimpleName = { + if (_name eq null) _name = termName(value) + _name + } + } } class ClassfileParser( @@ -939,26 +1113,11 @@ class ClassfileParser( } if (scan(tpnme.TASTYATTR)) { - val attrLen = in.nextInt - val bytes = in.nextBytes(attrLen) - if (attrLen == 16) { // A tasty attribute with that has only a UUID (16 bytes) implies the existence of the .tasty file - classfile.classToTasty match - case None => - report.error(em"Could not find TASTY for $classfile") - case Some(tastyFile) => - val expectedUUID = - val reader = new TastyReader(bytes, 0, 16) - new UUID(reader.readUncompressedLong(), reader.readUncompressedLong()) - val tastyUUID = - val tastyBytes: Array[Byte] = tastyFile.toByteArray - new TastyHeaderUnpickler(tastyBytes).readHeader() - if (expectedUUID != tastyUUID) - report.warning(s"$classfile is out of sync with its TASTy file. Loaded TASTy file. Try cleaning the project to fix this issue", NoSourcePosition) - return None - } - else - // Before 3.0.0 we had a mode where we could embed the TASTY bytes in the classfile. This has not been supported in any stable release. - report.error(s"Found a TASTY attribute with a length different from 16 in $classfile. This is likely a bug in the compiler. Please report.", NoSourcePosition) + val hint = + if classfile.classToTasty.isDefined then "This is likely a bug in the compiler. Please report." + else "This `.tasty` file is missing. Try cleaning the project to fix this issue." + report.error(s"Loading Scala 3 binary from $classfile. It should have been loaded from `.tasty` file. $hint", NoSourcePosition) + return None } if scan(tpnme.ScalaATTR) && !scalaUnpickleWhitelist.contains(classRoot.name) @@ -1127,78 +1286,7 @@ class ClassfileParser( private def isStatic(flags: Int) = (flags & JAVA_ACC_STATIC) != 0 private def hasAnnotation(flags: Int) = (flags & JAVA_ACC_ANNOTATION) != 0 - protected class NameOrString(val value: String) { - private var _name: SimpleName = null - def name: SimpleName = { - if (_name eq null) _name = termName(value) - _name - } - } - - def getClassSymbol(name: SimpleName)(using Context): Symbol = - if (name.endsWith("$") && (name ne nme.nothingRuntimeClass) && (name ne nme.nullRuntimeClass)) - // Null$ and Nothing$ ARE classes - requiredModule(name.dropRight(1)) - else classNameToSymbol(name) - - class ConstantPool(using in: DataReader) { - private val len = in.nextChar - private val starts = new Array[Int](len) - private val values = new Array[AnyRef](len) - private val internalized = new Array[NameOrString](len) - - { var i = 1 - while (i < starts.length) { - starts(i) = in.bp - i += 1 - (in.nextByte.toInt: @switch) match { - case CONSTANT_UTF8 | CONSTANT_UNICODE => - in.skip(in.nextChar) - case CONSTANT_CLASS | CONSTANT_STRING | CONSTANT_METHODTYPE => - in.skip(2) - case CONSTANT_METHODHANDLE => - in.skip(3) - case CONSTANT_FIELDREF | CONSTANT_METHODREF | CONSTANT_INTFMETHODREF - | CONSTANT_NAMEANDTYPE | CONSTANT_INTEGER | CONSTANT_FLOAT - | CONSTANT_INVOKEDYNAMIC => - in.skip(4) - case CONSTANT_LONG | CONSTANT_DOUBLE => - in.skip(8) - i += 1 - case _ => - errorBadTag(in.bp - 1) - } - } - } - - /** Return the name found at given index. */ - def getName(index: Int)(using in: DataReader): NameOrString = { - if (index <= 0 || len <= index) - errorBadIndex(index) - - values(index) match { - case name: NameOrString => name - case null => - val start = starts(index) - if (in.getByte(start).toInt != CONSTANT_UTF8) errorBadTag(start) - val len = in.getChar(start + 1).toInt - val name = new NameOrString(in.getUTF(start + 1, len + 2)) - values(index) = name - name - } - } - - /** Return the name found at given index in the constant pool, with '/' replaced by '.'. */ - def getExternalName(index: Int)(using in: DataReader): NameOrString = { - if (index <= 0 || len <= index) - errorBadIndex(index) - - if (internalized(index) == null) - internalized(index) = new NameOrString(getName(index).value.replace('/', '.')) - - internalized(index) - } - + class ConstantPool(using in: DataReader) extends AbstractConstantPool { def getClassSymbol(index: Int)(using ctx: Context, in: DataReader): Symbol = { if (index <= 0 || len <= index) errorBadIndex(index) var c = values(index).asInstanceOf[Symbol] @@ -1212,19 +1300,6 @@ class ClassfileParser( c } - /** Return the external name of the class info structure found at 'index'. - * Use 'getClassSymbol' if the class is sure to be a top-level class. - */ - def getClassName(index: Int)(using in: DataReader): NameOrString = { - val start = starts(index) - if (in.getByte(start).toInt != CONSTANT_CLASS) errorBadTag(start) - getExternalName(in.getChar(start + 1)) - } - - /** Return the type of a class constant entry. Since - * arrays are considered to be class types, they might - * appear as entries in 'newarray' or 'cast' opcodes. - */ def getClassOrArrayType(index: Int)(using ctx: Context, in: DataReader): Type = { if (index <= 0 || len <= index) errorBadIndex(index) val value = values(index) @@ -1252,90 +1327,12 @@ class ClassfileParser( def getType(index: Int, isVarargs: Boolean = false)(using Context, DataReader): Type = sigToType(getExternalName(index).value, isVarargs = isVarargs) + } - def getSuperClass(index: Int)(using Context, DataReader): Symbol = { - assert(index != 0, "attempt to parse java.lang.Object from classfile") - getClassSymbol(index) - } - - def getConstant(index: Int)(using ctx: Context, in: DataReader): Constant = { - if (index <= 0 || len <= index) errorBadIndex(index) - var value = values(index) - if (value eq null) { - val start = starts(index) - value = (in.getByte(start).toInt: @switch) match { - case CONSTANT_STRING => - Constant(getName(in.getChar(start + 1).toInt).value) - case CONSTANT_INTEGER => - Constant(in.getInt(start + 1)) - case CONSTANT_FLOAT => - Constant(in.getFloat(start + 1)) - case CONSTANT_LONG => - Constant(in.getLong(start + 1)) - case CONSTANT_DOUBLE => - Constant(in.getDouble(start + 1)) - case CONSTANT_CLASS => - getClassOrArrayType(index).typeSymbol - case _ => - errorBadTag(start) - } - values(index) = value - } - value match { - case ct: Constant => ct - case cls: Symbol => Constant(cls.typeRef) - case arr: Type => Constant(arr) - } - } - - private def getSubArray(bytes: Array[Byte]): Array[Byte] = { - val decodedLength = ByteCodecs.decode(bytes) - val arr = new Array[Byte](decodedLength) - System.arraycopy(bytes, 0, arr, 0, decodedLength) - arr - } - - def getBytes(index: Int)(using in: DataReader): Array[Byte] = { - if (index <= 0 || len <= index) errorBadIndex(index) - var value = values(index).asInstanceOf[Array[Byte]] - if (value eq null) { - val start = starts(index) - if (in.getByte(start).toInt != CONSTANT_UTF8) errorBadTag(start) - val len = in.getChar(start + 1) - val bytes = new Array[Byte](len) - in.getBytes(start + 3, bytes) - value = getSubArray(bytes) - values(index) = value - } - value - } - - def getBytes(indices: List[Int])(using in: DataReader): Array[Byte] = { - assert(!indices.isEmpty, indices) - var value = values(indices.head).asInstanceOf[Array[Byte]] - if (value eq null) { - val bytesBuffer = ArrayBuffer.empty[Byte] - for (index <- indices) { - if (index <= 0 || ConstantPool.this.len <= index) errorBadIndex(index) - val start = starts(index) - if (in.getByte(start).toInt != CONSTANT_UTF8) errorBadTag(start) - val len = in.getChar(start + 1) - val buf = new Array[Byte](len) - in.getBytes(start + 3, buf) - bytesBuffer ++= buf - } - value = getSubArray(bytesBuffer.toArray) - values(indices.head) = value - } - value - } - - /** Throws an exception signaling a bad constant index. */ - private def errorBadIndex(index: Int)(using in: DataReader) = - throw new RuntimeException("bad constant pool index: " + index + " at pos: " + in.bp) + def getClassSymbol(name: SimpleName)(using Context): Symbol = + if (name.endsWith("$") && (name ne nme.nothingRuntimeClass) && (name ne nme.nullRuntimeClass)) + // Null$ and Nothing$ ARE classes + requiredModule(name.dropRight(1)) + else classNameToSymbol(name) - /** Throws an exception signaling a bad tag at given address. */ - private def errorBadTag(start: Int)(using in: DataReader) = - throw new RuntimeException("bad constant pool tag " + in.getByte(start) + " at byte " + start) - } } diff --git a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileTastyUUIDParser.scala b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileTastyUUIDParser.scala new file mode 100644 index 000000000000..a9c91a68bb60 --- /dev/null +++ b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileTastyUUIDParser.scala @@ -0,0 +1,116 @@ +package dotty.tools.dotc +package core.classfile + +import scala.language.unsafeNulls + +import dotty.tools.dotc.core.Contexts._ +import dotty.tools.dotc.core.Decorators._ +import dotty.tools.dotc.core.Names._ +import dotty.tools.dotc.core.StdNames._ +import dotty.tools.dotc.core.Symbols._ +import dotty.tools.dotc.core.Types._ +import dotty.tools.dotc.util._ +import dotty.tools.io.AbstractFile +import dotty.tools.tasty.TastyReader + +import java.io.IOException +import java.lang.Integer.toHexString +import java.util.UUID + +class ClassfileTastyUUIDParser(classfile: AbstractFile)(ictx: Context) { + + import ClassfileConstants._ + + private var pool: ConstantPool = _ // the classfile's constant pool + + def checkTastyUUID(tastyUUID: UUID)(using Context): Unit = try ctx.base.reusableDataReader.withInstance { reader => + implicit val reader2 = reader.reset(classfile) + parseHeader() + this.pool = new ConstantPool + checkTastyAttr(tastyUUID) + this.pool = null + } + catch { + case e: RuntimeException => + if (ctx.debug) e.printStackTrace() + throw new IOException( + i"""class file ${classfile.canonicalPath} is broken, reading aborted with ${e.getClass} + |${Option(e.getMessage).getOrElse("")}""") + } + + private def parseHeader()(using in: DataReader): Unit = { + val magic = in.nextInt + if (magic != JAVA_MAGIC) + throw new IOException(s"class file '${classfile}' has wrong magic number 0x${toHexString(magic)}, should be 0x${toHexString(JAVA_MAGIC)}") + val minorVersion = in.nextChar.toInt + val majorVersion = in.nextChar.toInt + if ((majorVersion < JAVA_MAJOR_VERSION) || + ((majorVersion == JAVA_MAJOR_VERSION) && + (minorVersion < JAVA_MINOR_VERSION))) + throw new IOException( + s"class file '${classfile}' has unknown version $majorVersion.$minorVersion, should be at least $JAVA_MAJOR_VERSION.$JAVA_MINOR_VERSION") + } + + private def checkTastyAttr(tastyUUID: UUID)(using ctx: Context, in: DataReader): Unit = { + in.nextChar // jflags + in.nextChar // nameIdx + skipSuperclasses() + skipMembers() // fields + skipMembers() // methods + val attrs = in.nextChar + val attrbp = in.bp + + def scan(target: TypeName): Boolean = { + in.bp = attrbp + var i = 0 + while (i < attrs && pool.getName(in.nextChar).name.toTypeName != target) { + val attrLen = in.nextInt + in.skip(attrLen) + i += 1 + } + i < attrs + } + + if (scan(tpnme.TASTYATTR)) { + val attrLen = in.nextInt + val bytes = in.nextBytes(attrLen) + if (attrLen == 16) { // A tasty attribute with that has only a UUID (16 bytes) implies the existence of the .tasty file + val expectedUUID = + val reader = new TastyReader(bytes, 0, 16) + new UUID(reader.readUncompressedLong(), reader.readUncompressedLong()) + if (expectedUUID != tastyUUID) + report.warning(s"$classfile is out of sync with its TASTy file. Loaded TASTy file. Try cleaning the project to fix this issue", NoSourcePosition) + } + else + // Before 3.0.0 we had a mode where we could embed the TASTY bytes in the classfile. This has not been supported in any stable release. + report.error(s"Found a TASTY attribute with a length different from 16 in $classfile. This is likely a bug in the compiler. Please report.", NoSourcePosition) + } + + } + + private def skipAttributes()(using in: DataReader): Unit = { + val attrCount = in.nextChar + for (i <- 0 until attrCount) { + in.skip(2); in.skip(in.nextInt) + } + } + + private def skipMembers()(using in: DataReader): Unit = { + val memberCount = in.nextChar + for (i <- 0 until memberCount) { + in.skip(6); skipAttributes() + } + } + + private def skipSuperclasses()(using in: DataReader): Unit = { + in.skip(2) // superclass + val ifaces = in.nextChar + in.skip(2 * ifaces) + } + + class ConstantPool(using in: DataReader) extends ClassfileParser.AbstractConstantPool { + def getClassOrArrayType(index: Int)(using ctx: Context, in: DataReader): Type = throw new UnsupportedOperationException + def getClassSymbol(index: Int)(using ctx: Context, in: DataReader): Symbol = throw new UnsupportedOperationException + def getType(index: Int, isVarargs: Boolean)(using x$3: Context, x$4: DataReader): Type = throw new UnsupportedOperationException + } +} diff --git a/compiler/src/dotty/tools/io/AbstractFile.scala b/compiler/src/dotty/tools/io/AbstractFile.scala index fd9e2281181b..09779953fc76 100644 --- a/compiler/src/dotty/tools/io/AbstractFile.scala +++ b/compiler/src/dotty/tools/io/AbstractFile.scala @@ -253,7 +253,7 @@ abstract class AbstractFile extends Iterable[AbstractFile] { /** Returns the sibling abstract file in the parent of this abstract file or directory. * If there is no such file, returns `null`. */ - def resolveSibling(name: String): AbstractFile = + def resolveSibling(name: String): AbstractFile | Null = container.lookupName(name, directory = false) private def fileOrSubdirectoryNamed(name: String, isDir: Boolean): AbstractFile =