diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py index 9e9c5a4113c..092ca2653aa 100644 --- a/src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py +++ b/src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py @@ -279,7 +279,6 @@ def compile(self, ctx, args, classpath, upstream_analysis, self._verify_zinc_classpath(upstream_analysis.keys()) zinc_args = [] - zinc_args.extend([ '-log-level', self.get_options().level, '-analysis-cache', ctx.analysis_file, diff --git a/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala index eab5a7f63a3..0ae6c36da15 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala @@ -4,14 +4,10 @@ package org.pantsbuild.zinc.compiler -import java.io.{File, IOException} -import java.lang.{ Boolean => JBoolean } +import java.io.{File} import java.util.function.{ Function => JFunction } -import java.util.{ List => JList, Map => JMap } -import scala.collection.JavaConverters._ import scala.compat.java8.OptionConverters._ -import scala.util.matching.Regex import sbt.io.IO import sbt.util.Logger diff --git a/src/scala/org/pantsbuild/zinc/compiler/Main.scala b/src/scala/org/pantsbuild/zinc/compiler/Main.scala index 702271b03bb..4236eddd922 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Main.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Main.scala @@ -121,6 +121,14 @@ object Main { } log.info("Compile success " + Util.timing(startTime)) + + // if compile is successful, jar the contents of classesDirectory and copy to outputJar + if (settings.outputJar.isDefined) { + val outputJarPath = settings.outputJar.get.toPath + val classesDirectory = settings.classesDirectory + log.debug("Creating JAR at %s, for files at %s" format (outputJarPath, classesDirectory)) + OutputUtils.createClassesJar(classesDirectory, outputJarPath, settings.creationTime) + } } catch { case e: CompileFailed => log.error("Compile failed " + Util.timing(startTime)) diff --git a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala new file mode 100644 index 00000000000..bfeb8e50f8d --- /dev/null +++ b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala @@ -0,0 +1,126 @@ +/** + * Copyright (C) 2012 Typesafe, Inc. + */ + +package org.pantsbuild.zinc.compiler + +import java.io.File +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.{FileVisitResult, Files, Path, Paths, SimpleFileVisitor} +import java.util.jar.{JarEntry, JarInputStream, JarOutputStream} +import scala.annotation.tailrec +import scala.collection.mutable + +object OutputUtils { + + /** + * Sort the contents of the `dir` in lexicographic order. + * + * @param dir File handle containing the contents to sort + * @return sorted set of all paths within the `dir` + */ + def sort(dir:File): mutable.TreeSet[Path] = { + val sorted = new mutable.TreeSet[Path]() + + val fileSortVisitor = new SimpleFileVisitor[Path]() { + override def preVisitDirectory(path: Path, attrs: BasicFileAttributes): FileVisitResult = { + if (!path.endsWith("/")) { + sorted.add(Paths.get(path.toString, "/")) + } else { + sorted.add(path) + } + FileVisitResult.CONTINUE + } + + override def visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult = { + sorted.add(path) + FileVisitResult.CONTINUE + } + } + + Files.walkFileTree(dir.toPath, fileSortVisitor) + sorted + } + + def relativize(base: String, path: Path): String = { + new File(base.toString).toURI().relativize(new File(path.toString).toURI()).getPath() + } + + /** + * Create a JAR of of filePaths provided. + * + * @param filePaths set of all paths to be added to the JAR + * @param outputJarPath Absolute Path to the output JAR being created + * @param jarEntryTime time to be set for each JAR entry + */ + def createJar( + base: String, filePaths: mutable.TreeSet[Path], outputJarPath: Path, jarEntryTime: Long) { + + val target = new JarOutputStream(Files.newOutputStream(outputJarPath)) + + def jarEntry(name: String): JarEntry = { + val jarEntry = new JarEntry(name) + // setting jarEntry time to a fixed value for all entries within the jar so that jars are + // byte-for-byte reproducible. + jarEntry.setTime(jarEntryTime) + jarEntry + } + + def addToJar(source: Path, entryName: String): FileVisitResult = { + if (source.toFile.isDirectory) { + target.putNextEntry(jarEntry(entryName)) + } else { + target.putNextEntry(jarEntry(entryName)) + Files.copy(source, target) + } + target.closeEntry() + FileVisitResult.CONTINUE + } + + val pathToName = filePaths.zipWithIndex.map{case(k, v) => (k, relativize(base, k))}.toMap + pathToName.map(e => addToJar(e._1, e._2)) + target.close() + } + + /** + * Jar the contents of output classes (settings.classesDirectory) and copy to settings.outputJar + * + */ + def createClassesJar(classesDirectory: File, outputJarPath: Path, jarCreationTime: Long) = { + + // Sort the contents of the classesDirectory for deterministic jar creation + val sortedClasses = sort(classesDirectory) + + createJar(classesDirectory.toString, sortedClasses, outputJarPath, jarCreationTime) + } + + /** + * Determines if a file exists in a JAR provided. + * + * @param jarPath Absolute Path to the JAR being inspected + * @param fileName Name of the file, the existence of which is to be inspected + * @return + */ + def existsClass(jarPath: Path, fileName: String): Boolean = { + var jis: JarInputStream = null + var found = false + try { + jis = new JarInputStream(Files.newInputStream(jarPath)) + + @tailrec + def findClass(entry: JarEntry): Boolean = entry match { + case null => + false + case entry if entry.getName == fileName => + true + case _ => + findClass(jis.getNextJarEntry) + } + + found = findClass(jis.getNextJarEntry) + } finally { + jis.close() + } + found + } +} diff --git a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala index 4842133779b..52142bfe6de 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala @@ -5,7 +5,8 @@ package org.pantsbuild.zinc.compiler import java.io.File -import java.nio.file.Path +import sbt.internal.util.{ ConsoleLogger, ConsoleOut } +import java.nio.file.{Files, Path} import java.lang.{ Boolean => JBoolean } import java.util.function.{ Function => JFunction } import java.util.{ List => JList, logging => jlogging } @@ -38,6 +39,7 @@ case class Settings( _sources: Seq[File] = Seq.empty, classpath: Seq[File] = Seq.empty, _classesDirectory: Option[File] = None, + outputJar: Option[File] = None, scala: ScalaLocation = ScalaLocation(), scalacOptions: Seq[String] = Seq.empty, javaHome: Option[File] = None, @@ -47,7 +49,8 @@ case class Settings( compileOrder: CompileOrder = CompileOrder.Mixed, sbt: SbtJars = SbtJars(), _incOptions: IncOptions = IncOptions(), - analysis: AnalysisOptions = AnalysisOptions() + analysis: AnalysisOptions = AnalysisOptions(), + creationTime: Long = 0 ) { import Settings._ @@ -58,11 +61,7 @@ case class Settings( lazy val sources: Seq[File] = _sources map normalise lazy val classesDirectory: File = - normalise( - _classesDirectory.getOrElse { - throw new RuntimeException(s"The ${Settings.ZincCacheDirOpt} option is required.") - } - ) + normalise(_classesDirectory.getOrElse(defaultClassesDirectory())) lazy val incOptions: IncOptions = { _incOptions.copy( @@ -88,15 +87,15 @@ case class ConsoleOptions( ) { def javaLogLevel: jlogging.Level = logLevel match { case Level.Info => - jlogging.Level.INFO + jlogging.Level.INFO case Level.Warn => jlogging.Level.WARNING case Level.Error => - jlogging.Level.SEVERE + jlogging.Level.SEVERE case Level.Debug => jlogging.Level.FINE case x => - sys.error(s"Unsupported log level: $x") + sys.error(s"Unsupported log level: $x") } /** @@ -214,6 +213,7 @@ case class IncOptions( object Settings extends OptionSet[Settings] { val DestinationOpt = "-d" + val JarDestinationOpt = "-jar" val ZincCacheDirOpt = "-zinc-cache-dir" val CompilerBridgeOpt = "-compiler-bridge" val CompilerInterfaceOpt = "-compiler-interface" @@ -243,6 +243,8 @@ object Settings extends OptionSet[Settings] { header("Compile options:"), path( ("-classpath", "-cp"), "path", "Specify the classpath", (s: Settings, cp: Seq[File]) => s.copy(classpath = cp)), file( DestinationOpt, "directory", "Destination for compiled classes", (s: Settings, f: File) => s.copy(_classesDirectory = Some(f))), + file( JarDestinationOpt, "directory", "Jar destination for compiled classes", (s: Settings, f: File) => s.copy(outputJar = Some(f))), + long("-jar-creation-time", "n", "Creation timestamp for compiled jars, default is current time", (s: Settings, l: Long) => s.copy(creationTime = l)), header("Scala options:"), file( "-scala-home", "directory", "Scala home directory (for locating jars)", (s: Settings, f: File) => s.copy(scala = s.scala.copy(home = Some(f)))), @@ -311,7 +313,15 @@ object Settings extends OptionSet[Settings] { /** * By default the backup location is relative to the classes directory (for example, target/classes/../backup/classes). */ - def defaultBackupLocation(classesDir: File) = { + def defaultBackupLocation(classesDir: File): File = { classesDir.getParentFile / "backup" / classesDir.getName } + + /** + * If a settings.classesDirectory option isnt specified, create a temporary directory for output + * classes to be written to. + */ + def defaultClassesDirectory(): File = { + Files.createTempDirectory("temp-zinc-classes").toFile + } } diff --git a/src/scala/org/pantsbuild/zinc/options/OptionSet.scala b/src/scala/org/pantsbuild/zinc/options/OptionSet.scala index 6b7534dbd71..e93069dee52 100644 --- a/src/scala/org/pantsbuild/zinc/options/OptionSet.scala +++ b/src/scala/org/pantsbuild/zinc/options/OptionSet.scala @@ -58,6 +58,7 @@ trait OptionSet[T] { def boolean(opts: (String, String), desc: String, action: T => T) = new BooleanOption[T](Seq(opts._1, opts._2), desc, action) def string(opt: String, arg: String, desc: String, action: (T, String) => T) = new StringOption[T](Seq(opt), arg, desc, action) def int(opt: String, arg: String, desc: String, action: (T, Int) => T) = new IntOption[T](Seq(opt), arg, desc, action) + def long(opt: String, arg: String, desc: String, action: (T, Long) => T) = new LongOption[T](Seq(opt), arg, desc, action) def double(opt: String, arg: String, desc: String, action: (T, Double) => T) = new DoubleOption[T](Seq(opt), arg, desc, action) def fraction(opt: String, arg: String, desc: String, action: (T, Double) => T) = new FractionOption[T](Seq(opt), arg, desc, action) def file(opt: String, arg: String, desc: String, action: (T, File) => T) = new FileOption[T](Seq(opt), arg, desc, action) diff --git a/src/scala/org/pantsbuild/zinc/options/Options.scala b/src/scala/org/pantsbuild/zinc/options/Options.scala index abaeabfb051..8585cd79633 100644 --- a/src/scala/org/pantsbuild/zinc/options/Options.scala +++ b/src/scala/org/pantsbuild/zinc/options/Options.scala @@ -99,6 +99,18 @@ extends ArgumentOption[Int, Context] { } } +class LongOption[Context]( + val options: Seq[String], + val argument: String, + val description: String, + val action: (Context, Long) => Context) + extends ArgumentOption[Long, Context] { + def parse(arg: String): Option[Long] = { + try { Some(arg.toLong) } + catch { case _: NumberFormatException => None } + } +} + class DoubleOption[Context]( val options: Seq[String], val argument: String, diff --git a/tests/scala/org/pantsbuild/zinc/compiler/BUILD b/tests/scala/org/pantsbuild/zinc/compiler/BUILD new file mode 100644 index 00000000000..1be9429551f --- /dev/null +++ b/tests/scala/org/pantsbuild/zinc/compiler/BUILD @@ -0,0 +1,12 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +junit_tests( + dependencies=[ + '3rdparty/jvm/org/scala-sbt:io', + '3rdparty:scalatest', + 'src/scala/org/pantsbuild/zinc/compiler', + ], + strict_deps=True, + platform='java8', +) diff --git a/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala b/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala new file mode 100644 index 00000000000..96dfed959a0 --- /dev/null +++ b/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala @@ -0,0 +1,64 @@ +package org.pantsbuild.zinc.compiler + +import sbt.io.IO + +import java.io.File +import java.nio.file.{Files, Path, Paths} +import scala.collection.mutable +import org.junit.runner.RunWith +import org.scalatest.WordSpec +import org.scalatest.junit.JUnitRunner +import org.scalatest.MustMatchers + +@RunWith(classOf[JUnitRunner]) +class JarCreationSpec extends WordSpec with MustMatchers { + "JarCreationWithoutClasses" should { + "succeed when input classes are not provided" in { + IO.withTemporaryDirectory { tempInputDir => + val filePaths = new mutable.TreeSet[Path]() + + IO.withTemporaryDirectory { tempOutputDir => + val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-empty-output.jar") + + OutputUtils.createJar(tempInputDir.toString, filePaths, jarOutputPath, System.currentTimeMillis()) + OutputUtils.existsClass(jarOutputPath, "NonExistent.class") must be(false) + } + } + } + } + + "JarCreationWithClasses" should { + "succeed when input classes are provided" in { + IO.withTemporaryDirectory { tempInputDir => + val tempFile = File.createTempFile("Temp", ".class", tempInputDir) + val filePaths = mutable.TreeSet(tempFile.toPath) + + IO.withTemporaryDirectory { tempOutputDir => + val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-valid-output.jar") + + OutputUtils.createJar(tempInputDir.toString, filePaths, jarOutputPath, System.currentTimeMillis()) + OutputUtils.existsClass(jarOutputPath, tempFile.toString) must be(false) + OutputUtils.existsClass(jarOutputPath, OutputUtils.relativize(tempInputDir.toString, tempFile.toPath)) must be(true) + } + } + } + } + + "JarCreationWithNestedClasses" should { + "succeed when nested input directory and classes are provided" in { + IO.withTemporaryDirectory { tempInputDir => + val nestedTempDir = Files.createTempDirectory(tempInputDir.toPath, "tmp") + val nestedTempClass = File.createTempFile("NestedTemp", ".class", nestedTempDir.toFile) + val filePaths = mutable.TreeSet(nestedTempDir, nestedTempClass.toPath) + IO.withTemporaryDirectory { tempOutputDir => + val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-valid-output.jar") + + OutputUtils.createJar(tempInputDir.toString, filePaths, jarOutputPath, System.currentTimeMillis()) + OutputUtils.existsClass(jarOutputPath, OutputUtils.relativize(tempInputDir.toString, nestedTempDir)) must be(true) + OutputUtils.existsClass(jarOutputPath, OutputUtils.relativize(tempInputDir.toString, nestedTempClass.toPath)) must be(true) + } + } + } + } +} +