From 0bbfb99a9c16c4fddb57086161a0234b4648ede0 Mon Sep 17 00:00:00 2001 From: Ity Kaul Date: Tue, 10 Jul 2018 11:20:56 -0700 Subject: [PATCH 01/11] add outputJar option --- src/scala/org/pantsbuild/zinc/compiler/Main.scala | 3 +++ src/scala/org/pantsbuild/zinc/compiler/Settings.scala | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/scala/org/pantsbuild/zinc/compiler/Main.scala b/src/scala/org/pantsbuild/zinc/compiler/Main.scala index 702271b03bb..4376b03cb46 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Main.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Main.scala @@ -121,6 +121,9 @@ object Main { } log.info("Compile success " + Util.timing(startTime)) + + // if compile successful, jar the contents of classesDirectory and copy to outputJar + } catch { case e: CompileFailed => log.error("Compile failed " + Util.timing(startTime)) diff --git a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala index 4842133779b..f6c34fe3793 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala @@ -38,6 +38,7 @@ case class Settings( _sources: Seq[File] = Seq.empty, classpath: Seq[File] = Seq.empty, _classesDirectory: Option[File] = None, + outputJar: Option[File] = None, scala: ScalaLocation = ScalaLocation(), scalacOptions: Seq[String] = Seq.empty, javaHome: Option[File] = None, @@ -57,10 +58,15 @@ case class Settings( lazy val sources: Seq[File] = _sources map normalise + if _classesDirectory.isEmpty && outputJar.isEmpty { + throw new RuntimeException( + s"Either ${Settings.DestinationOpt} or ${Settings.JarDestinationOpt} option is required.") + } + lazy val classesDirectory: File = normalise( _classesDirectory.getOrElse { - throw new RuntimeException(s"The ${Settings.ZincCacheDirOpt} option is required.") + throw new RuntimeException(s"The ${Settings.DestinationOpt} option is required.") } ) @@ -214,6 +220,7 @@ case class IncOptions( object Settings extends OptionSet[Settings] { val DestinationOpt = "-d" + val JarDestinationOpt = "-jar" val ZincCacheDirOpt = "-zinc-cache-dir" val CompilerBridgeOpt = "-compiler-bridge" val CompilerInterfaceOpt = "-compiler-interface" @@ -243,6 +250,7 @@ object Settings extends OptionSet[Settings] { header("Compile options:"), path( ("-classpath", "-cp"), "path", "Specify the classpath", (s: Settings, cp: Seq[File]) => s.copy(classpath = cp)), file( DestinationOpt, "directory", "Destination for compiled classes", (s: Settings, f: File) => s.copy(_classesDirectory = Some(f))), + file( JarDestinationOpt, "directory", "Jar destination for compiled classes", (s: Settings, f: File) => s.copy(outputJar = Some(f))), header("Scala options:"), file( "-scala-home", "directory", "Scala home directory (for locating jars)", (s: Settings, f: File) => s.copy(scala = s.scala.copy(home = Some(f)))), From c44fdffcbd02cc852ecb6892943fef5368490bfc Mon Sep 17 00:00:00 2001 From: Ity Kaul Date: Wed, 11 Jul 2018 00:33:39 -0700 Subject: [PATCH 02/11] add createClassesJar functionality --- .../pantsbuild/zinc/compiler/InputUtils.scala | 37 +++++++++++++++++++ .../org/pantsbuild/zinc/compiler/Main.scala | 6 ++- .../pantsbuild/zinc/compiler/Settings.scala | 24 +++++++----- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala index eab5a7f63a3..90d9e62a008 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala @@ -6,7 +6,10 @@ package org.pantsbuild.zinc.compiler import java.io.{File, IOException} import java.lang.{ Boolean => JBoolean } +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.{FileVisitResult, Files, Path, Paths, SimpleFileVisitor} import java.util.function.{ Function => JFunction } +import java.util.jar.{Attributes, JarEntry, JarFile, JarInputStream, JarOutputStream, Manifest} import java.util.{ List => JList, Map => JMap } import scala.collection.JavaConverters._ @@ -161,6 +164,40 @@ object InputUtils { if (compiler.nonEmpty && library.nonEmpty) Some(ScalaJars(compiler(0), library(0), extra)) else None } + /** + * Jar the contents of output classes (settings.classesDirectory) and copy to settings.outputJar + */ + def createClassesJar(settings: Settings, log: Logger) = { + val classesDirectory = settings.classesDirectory + object FileVisitor extends SimpleFileVisitor[Path]() { + def done(): Path = { + target.close() + log.debug("Output jar generated at: " + jarPath) + // TODO(ity): Delete the temp classesDirectory, if one was created + jarPath + } + + val jarPath = Paths.get(classesDirectory.toString, settings.outputJar.toString) + val target = new JarOutputStream(Files.newOutputStream(jarPath)) + val entryTime = System.currentTimeMillis() + + override def visitFile(source: Path, attrs: BasicFileAttributes): FileVisitResult = { + val jarEntry = new JarEntry(source.toString) + jarEntry.setTime(entryTime) + + log.debug("Creating jar entry " + jarEntry + " for the file " + source) + + target.putNextEntry(jarEntry) + Files.copy(source, target) + target.closeEntry() + FileVisitResult.CONTINUE + } + + Files.walkFileTree(classesDirectory.toPath, FileVisitor) + FileVisitor.done() + } + } + // // Default setup // diff --git a/src/scala/org/pantsbuild/zinc/compiler/Main.scala b/src/scala/org/pantsbuild/zinc/compiler/Main.scala index 4376b03cb46..a1267986807 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Main.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Main.scala @@ -122,8 +122,10 @@ object Main { log.info("Compile success " + Util.timing(startTime)) - // if compile successful, jar the contents of classesDirectory and copy to outputJar - + // TODO(ity): if compile successful, jar the contents of classesDirectory and copy to outputJar + if (settings.outputJar.isDefined) { + InputUtils.createClassesJar(settings, log) + } } catch { case e: CompileFailed => log.error("Compile failed " + Util.timing(startTime)) diff --git a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala index f6c34fe3793..16b235c7787 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala @@ -5,7 +5,7 @@ package org.pantsbuild.zinc.compiler import java.io.File -import java.nio.file.Path +import java.nio.file.{Files, Path} import java.lang.{ Boolean => JBoolean } import java.util.function.{ Function => JFunction } import java.util.{ List => JList, logging => jlogging } @@ -58,17 +58,13 @@ case class Settings( lazy val sources: Seq[File] = _sources map normalise - if _classesDirectory.isEmpty && outputJar.isEmpty { + if (_classesDirectory.isEmpty && outputJar.isEmpty) { throw new RuntimeException( s"Either ${Settings.DestinationOpt} or ${Settings.JarDestinationOpt} option is required.") } lazy val classesDirectory: File = - normalise( - _classesDirectory.getOrElse { - throw new RuntimeException(s"The ${Settings.DestinationOpt} option is required.") - } - ) + normalise(_classesDirectory.getOrElse(defaultClassesDirectory())) lazy val incOptions: IncOptions = { _incOptions.copy( @@ -94,15 +90,15 @@ case class ConsoleOptions( ) { def javaLogLevel: jlogging.Level = logLevel match { case Level.Info => - jlogging.Level.INFO + jlogging.Level.INFO case Level.Warn => jlogging.Level.WARNING case Level.Error => - jlogging.Level.SEVERE + jlogging.Level.SEVERE case Level.Debug => jlogging.Level.FINE case x => - sys.error(s"Unsupported log level: $x") + sys.error(s"Unsupported log level: $x") } /** @@ -322,4 +318,12 @@ object Settings extends OptionSet[Settings] { def defaultBackupLocation(classesDir: File) = { classesDir.getParentFile / "backup" / classesDir.getName } + + /** + * If a settings.classesDirectory option isnt specified, create a temporary directory for output + * classes to be written to. + */ + def defaultClassesDirectory(): File = { + Files.createTempDirectory("temp-zinc-classes").toFile + } } From 7ed6e081aa5af3ee1106a8616a642c45369238d5 Mon Sep 17 00:00:00 2001 From: Ity Kaul Date: Wed, 11 Jul 2018 16:46:32 -0700 Subject: [PATCH 03/11] add a flag for jar creation time --- src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala | 5 +++-- src/scala/org/pantsbuild/zinc/compiler/Settings.scala | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala index 90d9e62a008..e991d6eb35a 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala @@ -179,11 +179,12 @@ object InputUtils { val jarPath = Paths.get(classesDirectory.toString, settings.outputJar.toString) val target = new JarOutputStream(Files.newOutputStream(jarPath)) - val entryTime = System.currentTimeMillis() override def visitFile(source: Path, attrs: BasicFileAttributes): FileVisitResult = { val jarEntry = new JarEntry(source.toString) - jarEntry.setTime(entryTime) + // setting jarEntry time to a fixed value for all entries within the jar for determinism + // and so that jarfiles are byte-for-byte reproducible. + jarEntry.setTime(settings.creationTime) log.debug("Creating jar entry " + jarEntry + " for the file " + source) diff --git a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala index 16b235c7787..4614c8eab71 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala @@ -48,7 +48,8 @@ case class Settings( compileOrder: CompileOrder = CompileOrder.Mixed, sbt: SbtJars = SbtJars(), _incOptions: IncOptions = IncOptions(), - analysis: AnalysisOptions = AnalysisOptions() + analysis: AnalysisOptions = AnalysisOptions(), + creationTime: Long = System.currentTimeMillis() ) { import Settings._ @@ -60,7 +61,7 @@ case class Settings( if (_classesDirectory.isEmpty && outputJar.isEmpty) { throw new RuntimeException( - s"Either ${Settings.DestinationOpt} or ${Settings.JarDestinationOpt} option is required.") + s"At least one of ${Settings.DestinationOpt} or ${Settings.JarDestinationOpt} option is required.") } lazy val classesDirectory: File = @@ -247,6 +248,7 @@ object Settings extends OptionSet[Settings] { path( ("-classpath", "-cp"), "path", "Specify the classpath", (s: Settings, cp: Seq[File]) => s.copy(classpath = cp)), file( DestinationOpt, "directory", "Destination for compiled classes", (s: Settings, f: File) => s.copy(_classesDirectory = Some(f))), file( JarDestinationOpt, "directory", "Jar destination for compiled classes", (s: Settings, f: File) => s.copy(outputJar = Some(f))), + long( "-jar-creation-time", "Creation time for compiled jars, default is current time", (s: Settings, l: Long) => s.copy(creationTime = l)), header("Scala options:"), file( "-scala-home", "directory", "Scala home directory (for locating jars)", (s: Settings, f: File) => s.copy(scala = s.scala.copy(home = Some(f)))), From 2cb21f36b8226058a6ba90dee6e1bd7c5cf83005 Mon Sep 17 00:00:00 2001 From: Ity Kaul Date: Thu, 12 Jul 2018 20:17:50 -0700 Subject: [PATCH 04/11] Added OutputUtils --- .../pantsbuild/zinc/compiler/InputUtils.scala | 35 ---------- .../org/pantsbuild/zinc/compiler/Main.scala | 2 +- .../zinc/compiler/OutputUtils.scala | 67 +++++++++++++++++++ .../pantsbuild/zinc/compiler/Settings.scala | 4 +- .../pantsbuild/zinc/options/OptionSet.scala | 1 + .../org/pantsbuild/zinc/options/Options.scala | 12 ++++ 6 files changed, 83 insertions(+), 38 deletions(-) create mode 100644 src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala diff --git a/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala index e991d6eb35a..c32272cb3ed 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala @@ -164,41 +164,6 @@ object InputUtils { if (compiler.nonEmpty && library.nonEmpty) Some(ScalaJars(compiler(0), library(0), extra)) else None } - /** - * Jar the contents of output classes (settings.classesDirectory) and copy to settings.outputJar - */ - def createClassesJar(settings: Settings, log: Logger) = { - val classesDirectory = settings.classesDirectory - object FileVisitor extends SimpleFileVisitor[Path]() { - def done(): Path = { - target.close() - log.debug("Output jar generated at: " + jarPath) - // TODO(ity): Delete the temp classesDirectory, if one was created - jarPath - } - - val jarPath = Paths.get(classesDirectory.toString, settings.outputJar.toString) - val target = new JarOutputStream(Files.newOutputStream(jarPath)) - - override def visitFile(source: Path, attrs: BasicFileAttributes): FileVisitResult = { - val jarEntry = new JarEntry(source.toString) - // setting jarEntry time to a fixed value for all entries within the jar for determinism - // and so that jarfiles are byte-for-byte reproducible. - jarEntry.setTime(settings.creationTime) - - log.debug("Creating jar entry " + jarEntry + " for the file " + source) - - target.putNextEntry(jarEntry) - Files.copy(source, target) - target.closeEntry() - FileVisitResult.CONTINUE - } - - Files.walkFileTree(classesDirectory.toPath, FileVisitor) - FileVisitor.done() - } - } - // // Default setup // diff --git a/src/scala/org/pantsbuild/zinc/compiler/Main.scala b/src/scala/org/pantsbuild/zinc/compiler/Main.scala index a1267986807..d54bdae34ca 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Main.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Main.scala @@ -124,7 +124,7 @@ object Main { // TODO(ity): if compile successful, jar the contents of classesDirectory and copy to outputJar if (settings.outputJar.isDefined) { - InputUtils.createClassesJar(settings, log) + OutputUtils.createClassesJar(settings, log) } } catch { case e: CompileFailed => diff --git a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala new file mode 100644 index 00000000000..6a914c73158 --- /dev/null +++ b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2012 Typesafe, Inc. + */ + +package org.pantsbuild.zinc.compiler + +import java.io.File +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.{FileVisitResult, Files, Path, Paths, SimpleFileVisitor} +import java.util.function.{Function => JFunction} +import java.util.jar.{JarEntry, JarOutputStream} +import org.pantsbuild.zinc.analysis.AnalysisMap + +import scala.collection.JavaConverters._ +import scala.compat.java8.OptionConverters._ +import scala.util.matching.Regex + +import sbt.io.IO +import sbt.util.Logger +import xsbti.{Position, Problem, Severity, ReporterConfig, ReporterUtil} +import xsbti.compile.{ + AnalysisStore, + CompileOptions, + CompileOrder, + Compilers, + Inputs, + PreviousResult, + Setup +} + +object OutputUtils { + + /** + * Jar the contents of output classes (settings.classesDirectory) and copy to settings.outputJar + */ + def createClassesJar(settings: Settings, log: Logger) = { + val classesDirectory = settings.classesDirectory + val jarCaptureVisitor = new SimpleFileVisitor[Path]() { + def done(): Path = { + target.close() + log.debug("Output jar generated at: " + jarPath) + // TODO(ity): Delete the temp classesDirectory, if one was created + jarPath + } + + val jarPath = Paths.get(classesDirectory.toString, settings.outputJar.toString) + val target = new JarOutputStream(Files.newOutputStream(jarPath)) + + override def visitFile(source: Path, attrs: BasicFileAttributes): FileVisitResult = { + val jarEntry = new JarEntry(source.toString) + // setting jarEntry time to a fixed value for all entries within the jar for determinism + // and so that jarfiles are byte-for-byte reproducible. + jarEntry.setTime(settings.creationTime) + + log.debug("Creating jar entry " + jarEntry + " for the file " + source) + + target.putNextEntry(jarEntry) + Files.copy(source, target) + target.closeEntry() + FileVisitResult.CONTINUE + } + + } + Files.walkFileTree(classesDirectory.toPath, jarCaptureVisitor) + jarCaptureVisitor.done() + } +} diff --git a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala index 4614c8eab71..9d81d2a0d29 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala @@ -61,7 +61,7 @@ case class Settings( if (_classesDirectory.isEmpty && outputJar.isEmpty) { throw new RuntimeException( - s"At least one of ${Settings.DestinationOpt} or ${Settings.JarDestinationOpt} option is required.") + s"Either ${Settings.DestinationOpt} or ${Settings.JarDestinationOpt} option is required.") } lazy val classesDirectory: File = @@ -248,7 +248,7 @@ object Settings extends OptionSet[Settings] { path( ("-classpath", "-cp"), "path", "Specify the classpath", (s: Settings, cp: Seq[File]) => s.copy(classpath = cp)), file( DestinationOpt, "directory", "Destination for compiled classes", (s: Settings, f: File) => s.copy(_classesDirectory = Some(f))), file( JarDestinationOpt, "directory", "Jar destination for compiled classes", (s: Settings, f: File) => s.copy(outputJar = Some(f))), - long( "-jar-creation-time", "Creation time for compiled jars, default is current time", (s: Settings, l: Long) => s.copy(creationTime = l)), + long("-jar-creation-time", "n", "Creation time for compiled jars, default is current time", (s: Settings, l: Long) => s.copy(creationTime = l)), header("Scala options:"), file( "-scala-home", "directory", "Scala home directory (for locating jars)", (s: Settings, f: File) => s.copy(scala = s.scala.copy(home = Some(f)))), diff --git a/src/scala/org/pantsbuild/zinc/options/OptionSet.scala b/src/scala/org/pantsbuild/zinc/options/OptionSet.scala index 6b7534dbd71..e93069dee52 100644 --- a/src/scala/org/pantsbuild/zinc/options/OptionSet.scala +++ b/src/scala/org/pantsbuild/zinc/options/OptionSet.scala @@ -58,6 +58,7 @@ trait OptionSet[T] { def boolean(opts: (String, String), desc: String, action: T => T) = new BooleanOption[T](Seq(opts._1, opts._2), desc, action) def string(opt: String, arg: String, desc: String, action: (T, String) => T) = new StringOption[T](Seq(opt), arg, desc, action) def int(opt: String, arg: String, desc: String, action: (T, Int) => T) = new IntOption[T](Seq(opt), arg, desc, action) + def long(opt: String, arg: String, desc: String, action: (T, Long) => T) = new LongOption[T](Seq(opt), arg, desc, action) def double(opt: String, arg: String, desc: String, action: (T, Double) => T) = new DoubleOption[T](Seq(opt), arg, desc, action) def fraction(opt: String, arg: String, desc: String, action: (T, Double) => T) = new FractionOption[T](Seq(opt), arg, desc, action) def file(opt: String, arg: String, desc: String, action: (T, File) => T) = new FileOption[T](Seq(opt), arg, desc, action) diff --git a/src/scala/org/pantsbuild/zinc/options/Options.scala b/src/scala/org/pantsbuild/zinc/options/Options.scala index abaeabfb051..8585cd79633 100644 --- a/src/scala/org/pantsbuild/zinc/options/Options.scala +++ b/src/scala/org/pantsbuild/zinc/options/Options.scala @@ -99,6 +99,18 @@ extends ArgumentOption[Int, Context] { } } +class LongOption[Context]( + val options: Seq[String], + val argument: String, + val description: String, + val action: (Context, Long) => Context) + extends ArgumentOption[Long, Context] { + def parse(arg: String): Option[Long] = { + try { Some(arg.toLong) } + catch { case _: NumberFormatException => None } + } +} + class DoubleOption[Context]( val options: Seq[String], val argument: String, From 06cb12d2d9ed01d4b7e7b7ab7a0de55609f7bbee Mon Sep 17 00:00:00 2001 From: Ity Kaul Date: Fri, 13 Jul 2018 02:32:57 -0700 Subject: [PATCH 05/11] Add sorting of classesDirectory for jar creation --- .../pantsbuild/zinc/compiler/InputUtils.scala | 9 +-- .../zinc/compiler/OutputUtils.scala | 68 ++++++++----------- .../pantsbuild/zinc/compiler/Settings.scala | 2 +- 3 files changed, 29 insertions(+), 50 deletions(-) diff --git a/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala index c32272cb3ed..0ae6c36da15 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/InputUtils.scala @@ -4,17 +4,10 @@ package org.pantsbuild.zinc.compiler -import java.io.{File, IOException} -import java.lang.{ Boolean => JBoolean } -import java.nio.file.attribute.BasicFileAttributes -import java.nio.file.{FileVisitResult, Files, Path, Paths, SimpleFileVisitor} +import java.io.{File} import java.util.function.{ Function => JFunction } -import java.util.jar.{Attributes, JarEntry, JarFile, JarInputStream, JarOutputStream, Manifest} -import java.util.{ List => JList, Map => JMap } -import scala.collection.JavaConverters._ import scala.compat.java8.OptionConverters._ -import scala.util.matching.Regex import sbt.io.IO import sbt.util.Logger diff --git a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala index 6a914c73158..f0df8210f21 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala @@ -4,29 +4,11 @@ package org.pantsbuild.zinc.compiler -import java.io.File import java.nio.file.attribute.BasicFileAttributes import java.nio.file.{FileVisitResult, Files, Path, Paths, SimpleFileVisitor} -import java.util.function.{Function => JFunction} import java.util.jar.{JarEntry, JarOutputStream} -import org.pantsbuild.zinc.analysis.AnalysisMap - -import scala.collection.JavaConverters._ -import scala.compat.java8.OptionConverters._ -import scala.util.matching.Regex - -import sbt.io.IO +import scala.collection.mutable import sbt.util.Logger -import xsbti.{Position, Problem, Severity, ReporterConfig, ReporterUtil} -import xsbti.compile.{ - AnalysisStore, - CompileOptions, - CompileOrder, - Compilers, - Inputs, - PreviousResult, - Setup -} object OutputUtils { @@ -35,33 +17,37 @@ object OutputUtils { */ def createClassesJar(settings: Settings, log: Logger) = { val classesDirectory = settings.classesDirectory - val jarCaptureVisitor = new SimpleFileVisitor[Path]() { - def done(): Path = { - target.close() - log.debug("Output jar generated at: " + jarPath) - // TODO(ity): Delete the temp classesDirectory, if one was created - jarPath - } + val sorted = new mutable.TreeSet[Path]() - val jarPath = Paths.get(classesDirectory.toString, settings.outputJar.toString) - val target = new JarOutputStream(Files.newOutputStream(jarPath)) + val fileSortVisitor = new SimpleFileVisitor[Path]() { + override def preVisitDirectory(path: Path, attrs: BasicFileAttributes): FileVisitResult = { + sorted.add(path) + FileVisitResult.CONTINUE + } + override def visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult = { + sorted.add(path) + FileVisitResult.CONTINUE + } + } + // Sort the contents of the classesDirectory in lexicographic order + Files.walkFileTree(classesDirectory.toPath, fileSortVisitor) - override def visitFile(source: Path, attrs: BasicFileAttributes): FileVisitResult = { - val jarEntry = new JarEntry(source.toString) - // setting jarEntry time to a fixed value for all entries within the jar for determinism - // and so that jarfiles are byte-for-byte reproducible. - jarEntry.setTime(settings.creationTime) + val jarPath = Paths.get(classesDirectory.toString, settings.outputJar.toString) + val target = new JarOutputStream(Files.newOutputStream(jarPath)) - log.debug("Creating jar entry " + jarEntry + " for the file " + source) + def createJar(source: Path): FileVisitResult = { + val jarEntry = new JarEntry(source.toString) + // setting jarEntry time to a fixed value for all entries within the jar for determinism + // and so that jar are byte-for-byte reproducible. + jarEntry.setTime(settings.creationTime) - target.putNextEntry(jarEntry) - Files.copy(source, target) - target.closeEntry() - FileVisitResult.CONTINUE - } + log.debug("Creating jar entry " + jarEntry + " for the file " + source) + target.putNextEntry(jarEntry) + Files.copy(source, target) + target.closeEntry() + FileVisitResult.CONTINUE } - Files.walkFileTree(classesDirectory.toPath, jarCaptureVisitor) - jarCaptureVisitor.done() + sorted.map(createJar(_)) } } diff --git a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala index 9d81d2a0d29..ebd96b69a94 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala @@ -248,7 +248,7 @@ object Settings extends OptionSet[Settings] { path( ("-classpath", "-cp"), "path", "Specify the classpath", (s: Settings, cp: Seq[File]) => s.copy(classpath = cp)), file( DestinationOpt, "directory", "Destination for compiled classes", (s: Settings, f: File) => s.copy(_classesDirectory = Some(f))), file( JarDestinationOpt, "directory", "Jar destination for compiled classes", (s: Settings, f: File) => s.copy(outputJar = Some(f))), - long("-jar-creation-time", "n", "Creation time for compiled jars, default is current time", (s: Settings, l: Long) => s.copy(creationTime = l)), + long("-jar-creation-time", "n", "Creation timestamp for compiled jars, default is current time", (s: Settings, l: Long) => s.copy(creationTime = l)), header("Scala options:"), file( "-scala-home", "directory", "Scala home directory (for locating jars)", (s: Settings, f: File) => s.copy(scala = s.scala.copy(home = Some(f)))), From 38a3655d6d1d253a770cf90f67a53ea8c94fcfbc Mon Sep 17 00:00:00 2001 From: Ity Kaul Date: Fri, 13 Jul 2018 09:50:29 -0700 Subject: [PATCH 06/11] fix comments --- src/scala/org/pantsbuild/zinc/compiler/Main.scala | 2 +- src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/scala/org/pantsbuild/zinc/compiler/Main.scala b/src/scala/org/pantsbuild/zinc/compiler/Main.scala index d54bdae34ca..1e8c1a900f9 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Main.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Main.scala @@ -122,7 +122,7 @@ object Main { log.info("Compile success " + Util.timing(startTime)) - // TODO(ity): if compile successful, jar the contents of classesDirectory and copy to outputJar + // if compile is successful, jar the contents of classesDirectory and copy to outputJar if (settings.outputJar.isDefined) { OutputUtils.createClassesJar(settings, log) } diff --git a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala index f0df8210f21..ac354ad72d1 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala @@ -29,7 +29,8 @@ object OutputUtils { FileVisitResult.CONTINUE } } - // Sort the contents of the classesDirectory in lexicographic order + // Sort the contents of the classesDirectory in lexicographic order for + // deterministic jar creation Files.walkFileTree(classesDirectory.toPath, fileSortVisitor) val jarPath = Paths.get(classesDirectory.toString, settings.outputJar.toString) @@ -37,8 +38,8 @@ object OutputUtils { def createJar(source: Path): FileVisitResult = { val jarEntry = new JarEntry(source.toString) - // setting jarEntry time to a fixed value for all entries within the jar for determinism - // and so that jar are byte-for-byte reproducible. + // setting jarEntry time to a fixed value for all entries within the jar so that jars are + // byte-for-byte reproducible. jarEntry.setTime(settings.creationTime) log.debug("Creating jar entry " + jarEntry + " for the file " + source) From 94ba8e91cdee042b457b88299f4d176cf5a64535 Mon Sep 17 00:00:00 2001 From: Ity Kaul Date: Tue, 17 Jul 2018 14:50:35 -0700 Subject: [PATCH 07/11] Add unit tests for jar creation --- .../tasks/jvm_compile/zinc/zinc_compile.py | 1 - .../org/pantsbuild/zinc/compiler/Main.scala | 5 +- .../zinc/compiler/OutputUtils.scala | 77 +++++++++++++++---- .../pantsbuild/zinc/compiler/Settings.scala | 2 +- .../scala/org/pantsbuild/zinc/compiler/BUILD | 12 +++ .../zinc/compiler/JarCreationSpec.scala | 41 ++++++++++ 6 files changed, 120 insertions(+), 18 deletions(-) create mode 100644 tests/scala/org/pantsbuild/zinc/compiler/BUILD create mode 100644 tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py index 9e9c5a4113c..092ca2653aa 100644 --- a/src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py +++ b/src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py @@ -279,7 +279,6 @@ def compile(self, ctx, args, classpath, upstream_analysis, self._verify_zinc_classpath(upstream_analysis.keys()) zinc_args = [] - zinc_args.extend([ '-log-level', self.get_options().level, '-analysis-cache', ctx.analysis_file, diff --git a/src/scala/org/pantsbuild/zinc/compiler/Main.scala b/src/scala/org/pantsbuild/zinc/compiler/Main.scala index 1e8c1a900f9..4236eddd922 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Main.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Main.scala @@ -124,7 +124,10 @@ object Main { // if compile is successful, jar the contents of classesDirectory and copy to outputJar if (settings.outputJar.isDefined) { - OutputUtils.createClassesJar(settings, log) + val outputJarPath = settings.outputJar.get.toPath + val classesDirectory = settings.classesDirectory + log.debug("Creating JAR at %s, for files at %s" format (outputJarPath, classesDirectory)) + OutputUtils.createClassesJar(classesDirectory, outputJarPath, settings.creationTime) } } catch { case e: CompileFailed => diff --git a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala index ac354ad72d1..9be11c67498 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala @@ -4,19 +4,23 @@ package org.pantsbuild.zinc.compiler +import java.io.File import java.nio.file.attribute.BasicFileAttributes import java.nio.file.{FileVisitResult, Files, Path, Paths, SimpleFileVisitor} -import java.util.jar.{JarEntry, JarOutputStream} +import java.util.jar.{JarEntry, JarInputStream, JarOutputStream} +import scala.annotation.tailrec import scala.collection.mutable -import sbt.util.Logger +import scala.util.Try object OutputUtils { /** - * Jar the contents of output classes (settings.classesDirectory) and copy to settings.outputJar + * Sort the contents of the `dir` in lexicographic order. + * + * @param dir File handle containing the contents to sort + * @return sorted set of all paths within the `dir` */ - def createClassesJar(settings: Settings, log: Logger) = { - val classesDirectory = settings.classesDirectory + def sort(dir:File): mutable.TreeSet[Path] = { val sorted = new mutable.TreeSet[Path]() val fileSortVisitor = new SimpleFileVisitor[Path]() { @@ -24,31 +28,74 @@ object OutputUtils { sorted.add(path) FileVisitResult.CONTINUE } + override def visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult = { sorted.add(path) FileVisitResult.CONTINUE } } - // Sort the contents of the classesDirectory in lexicographic order for - // deterministic jar creation - Files.walkFileTree(classesDirectory.toPath, fileSortVisitor) - val jarPath = Paths.get(classesDirectory.toString, settings.outputJar.toString) - val target = new JarOutputStream(Files.newOutputStream(jarPath)) + Files.walkFileTree(dir.toPath, fileSortVisitor) + sorted + } + + /** + * Create a JAR of of filePaths provided. + * + * @param filePaths set of all paths to be added to the JAR + * @param outputJarPath Path to the output JAR to be created + * @param jarEntryTime time to be set for each JAR entry + */ + def createJar(filePaths: mutable.TreeSet[Path], outputJarPath: Path, jarEntryTime: Long) { - def createJar(source: Path): FileVisitResult = { + val target = new JarOutputStream(Files.newOutputStream(outputJarPath)) + + def addToJar(source: Path): FileVisitResult = { val jarEntry = new JarEntry(source.toString) // setting jarEntry time to a fixed value for all entries within the jar so that jars are // byte-for-byte reproducible. - jarEntry.setTime(settings.creationTime) - - log.debug("Creating jar entry " + jarEntry + " for the file " + source) + jarEntry.setTime(jarEntryTime) target.putNextEntry(jarEntry) Files.copy(source, target) target.closeEntry() FileVisitResult.CONTINUE } - sorted.map(createJar(_)) + + filePaths.map(addToJar(_)) + target.close() + } + + /** + * Jar the contents of output classes (settings.classesDirectory) and copy to settings.outputJar + * + */ + def createClassesJar(classesDirectory: File, outputJarPath: Path, jarCreationTime: Long) = { + + // Sort the contents of the classesDirectory for deterministic jar creation + val sortedClasses = sort(classesDirectory) + + createJar(sortedClasses, outputJarPath, jarCreationTime) + } + + /** + * Determines if a Class exists in a JAR provided. + * + * @param jarPath Absolute Path to the JAR being inspected + * @param clazz Name of the Class, the existence of which is to be inspected + * @return + */ + def existsClass(jarPath: Path, clazz: String): Boolean = { + val jis = new JarInputStream(Files.newInputStream(jarPath)) + @tailrec + def findClass(entry: JarEntry): Boolean = entry match { + case null => + false + case entry if entry.getName == clazz => + true + case _ => + findClass(jis.getNextJarEntry) + } + findClass(jis.getNextJarEntry) } } diff --git a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala index ebd96b69a94..5df34cd7563 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala @@ -5,6 +5,7 @@ package org.pantsbuild.zinc.compiler import java.io.File +import sbt.internal.util.{ ConsoleLogger, ConsoleOut } import java.nio.file.{Files, Path} import java.lang.{ Boolean => JBoolean } import java.util.function.{ Function => JFunction } @@ -58,7 +59,6 @@ case class Settings( } lazy val sources: Seq[File] = _sources map normalise - if (_classesDirectory.isEmpty && outputJar.isEmpty) { throw new RuntimeException( s"Either ${Settings.DestinationOpt} or ${Settings.JarDestinationOpt} option is required.") diff --git a/tests/scala/org/pantsbuild/zinc/compiler/BUILD b/tests/scala/org/pantsbuild/zinc/compiler/BUILD new file mode 100644 index 00000000000..36d029f77cc --- /dev/null +++ b/tests/scala/org/pantsbuild/zinc/compiler/BUILD @@ -0,0 +1,12 @@ +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +junit_tests( + dependencies=[ + '3rdparty/jvm/org/scala-sbt:io', + '3rdparty:scalatest', + 'src/scala/org/pantsbuild/zinc/compiler', + ], + strict_deps=True, + platform='java8', +) diff --git a/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala b/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala new file mode 100644 index 00000000000..bdfc7a5dbe6 --- /dev/null +++ b/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala @@ -0,0 +1,41 @@ +package org.pantsbuild.zinc.compiler + +import java.io.File +import java.nio.file.{Path, Paths} +import scala.collection.mutable + +import sbt.io.IO + +import org.junit.runner.RunWith +import org.scalatest.WordSpec +import org.scalatest.junit.JUnitRunner +import org.scalatest.MustMatchers + +@RunWith(classOf[JUnitRunner]) +class JarCreationSpec extends WordSpec with MustMatchers { + "JarCreationWithoutClasses" should { + "succeed when input classes are provided" in { + IO.withTemporaryDirectory { tempDir => + val filePaths = new mutable.TreeSet[Path]() + val jarOutputPath = Paths.get(tempDir.toString, "spec-empty-output.jar") + OutputUtils.createJar(filePaths, jarOutputPath, System.currentTimeMillis()) + OutputUtils.existsClass(jarOutputPath, "NonExistent.class") must be(false) + } + } + } + "JarCreationWithClasses" should { + "succeed when input classes are provided" in { + IO.withTemporaryDirectory { tempDir => + + val tempDirPath = tempDir.toString + val tempFile = File.createTempFile(tempDirPath, "Clazz.class") + val filePaths = mutable.TreeSet(tempFile.toPath) + val jarOutputPath = Paths.get(tempDirPath.toString, "spec-valid-output.jar") + + OutputUtils.createJar(filePaths, jarOutputPath, System.currentTimeMillis()) + OutputUtils.existsClass(jarOutputPath, tempFile.toString) must be(true) + } + } + } +} + From 3bba4ca1f9351ccc34133d87cd32d19f8fd4212f Mon Sep 17 00:00:00 2001 From: Ity Kaul Date: Tue, 17 Jul 2018 16:43:50 -0700 Subject: [PATCH 08/11] Addressed comments --- .../zinc/compiler/OutputUtils.scala | 31 ++++++++++++------- .../zinc/compiler/JarCreationSpec.scala | 29 ++++++++++------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala index 9be11c67498..8423f62ab1a 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala @@ -43,7 +43,7 @@ object OutputUtils { * Create a JAR of of filePaths provided. * * @param filePaths set of all paths to be added to the JAR - * @param outputJarPath Path to the output JAR to be created + * @param outputJarPath Absolute Path to the output JAR being created * @param jarEntryTime time to be set for each JAR entry */ def createJar(filePaths: mutable.TreeSet[Path], outputJarPath: Path, jarEntryTime: Long) { @@ -86,16 +86,25 @@ object OutputUtils { * @return */ def existsClass(jarPath: Path, clazz: String): Boolean = { - val jis = new JarInputStream(Files.newInputStream(jarPath)) - @tailrec - def findClass(entry: JarEntry): Boolean = entry match { - case null => - false - case entry if entry.getName == clazz => - true - case _ => - findClass(jis.getNextJarEntry) + var jis: JarInputStream = null + var found = false + try { + jis = new JarInputStream(Files.newInputStream(jarPath)) + + @tailrec + def findClass(entry: JarEntry): Boolean = entry match { + case null => + false + case entry if entry.getName == clazz => + true + case _ => + findClass(jis.getNextJarEntry) + } + + found = findClass(jis.getNextJarEntry) + } finally { + jis.close() } - findClass(jis.getNextJarEntry) + found } } diff --git a/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala b/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala index bdfc7a5dbe6..b9ee52108c4 100644 --- a/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala +++ b/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala @@ -14,26 +14,31 @@ import org.scalatest.MustMatchers @RunWith(classOf[JUnitRunner]) class JarCreationSpec extends WordSpec with MustMatchers { "JarCreationWithoutClasses" should { - "succeed when input classes are provided" in { - IO.withTemporaryDirectory { tempDir => + "succeed when input classes are not provided" in { + IO.withTemporaryDirectory { tempInputDir => val filePaths = new mutable.TreeSet[Path]() - val jarOutputPath = Paths.get(tempDir.toString, "spec-empty-output.jar") - OutputUtils.createJar(filePaths, jarOutputPath, System.currentTimeMillis()) - OutputUtils.existsClass(jarOutputPath, "NonExistent.class") must be(false) + + IO.withTemporaryDirectory { tempOutputDir => + val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-empty-output.jar") + + OutputUtils.createJar(filePaths, jarOutputPath, System.currentTimeMillis()) + OutputUtils.existsClass(jarOutputPath, "NonExistent.class") must be(false) + } } } } "JarCreationWithClasses" should { "succeed when input classes are provided" in { - IO.withTemporaryDirectory { tempDir => - - val tempDirPath = tempDir.toString - val tempFile = File.createTempFile(tempDirPath, "Clazz.class") + IO.withTemporaryDirectory { tempInputDir => + val tempFile = File.createTempFile(tempInputDir.toString, "Clazz.class") val filePaths = mutable.TreeSet(tempFile.toPath) - val jarOutputPath = Paths.get(tempDirPath.toString, "spec-valid-output.jar") - OutputUtils.createJar(filePaths, jarOutputPath, System.currentTimeMillis()) - OutputUtils.existsClass(jarOutputPath, tempFile.toString) must be(true) + IO.withTemporaryDirectory { tempOutputDir => + val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-valid-output.jar") + + OutputUtils.createJar(filePaths, jarOutputPath, System.currentTimeMillis()) + OutputUtils.existsClass(jarOutputPath, tempFile.toString) must be(true) + } } } } From 1b8d8b86f7c929f84179f734ad4034d5133d49eb Mon Sep 17 00:00:00 2001 From: Ity Kaul Date: Wed, 18 Jul 2018 01:46:09 -0700 Subject: [PATCH 09/11] Relativize jar paths --- .../pantsbuild/zinc/compiler/OutputUtils.scala | 17 +++++++++++------ .../zinc/compiler/JarCreationSpec.scala | 9 +++++---- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala index 8423f62ab1a..cf0b4a27242 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala @@ -10,7 +10,6 @@ import java.nio.file.{FileVisitResult, Files, Path, Paths, SimpleFileVisitor} import java.util.jar.{JarEntry, JarInputStream, JarOutputStream} import scala.annotation.tailrec import scala.collection.mutable -import scala.util.Try object OutputUtils { @@ -39,6 +38,10 @@ object OutputUtils { sorted } + def relativize(base: String, path: Path): String = { + new File(base).toURI().relativize(new File(path.toString).toURI()).getPath() + } + /** * Create a JAR of of filePaths provided. * @@ -46,12 +49,13 @@ object OutputUtils { * @param outputJarPath Absolute Path to the output JAR being created * @param jarEntryTime time to be set for each JAR entry */ - def createJar(filePaths: mutable.TreeSet[Path], outputJarPath: Path, jarEntryTime: Long) { + def createJar( + base: String, filePaths: mutable.TreeSet[Path], outputJarPath: Path, jarEntryTime: Long) { val target = new JarOutputStream(Files.newOutputStream(outputJarPath)) - def addToJar(source: Path): FileVisitResult = { - val jarEntry = new JarEntry(source.toString) + def addToJar(source: Path, entryName: String): FileVisitResult = { + val jarEntry = new JarEntry(entryName) // setting jarEntry time to a fixed value for all entries within the jar so that jars are // byte-for-byte reproducible. jarEntry.setTime(jarEntryTime) @@ -62,7 +66,8 @@ object OutputUtils { FileVisitResult.CONTINUE } - filePaths.map(addToJar(_)) + val pathToName = filePaths.zipWithIndex.map{case(k, v) => (k, relativize(base, k))}.toMap + pathToName.map(e => addToJar(e._1, e._2)) target.close() } @@ -75,7 +80,7 @@ object OutputUtils { // Sort the contents of the classesDirectory for deterministic jar creation val sortedClasses = sort(classesDirectory) - createJar(sortedClasses, outputJarPath, jarCreationTime) + createJar(classesDirectory.toString, sortedClasses, outputJarPath, jarCreationTime) } /** diff --git a/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala b/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala index b9ee52108c4..fc0eb9d24cf 100644 --- a/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala +++ b/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala @@ -21,7 +21,7 @@ class JarCreationSpec extends WordSpec with MustMatchers { IO.withTemporaryDirectory { tempOutputDir => val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-empty-output.jar") - OutputUtils.createJar(filePaths, jarOutputPath, System.currentTimeMillis()) + OutputUtils.createJar(tempInputDir.toString, filePaths, jarOutputPath, System.currentTimeMillis()) OutputUtils.existsClass(jarOutputPath, "NonExistent.class") must be(false) } } @@ -30,14 +30,15 @@ class JarCreationSpec extends WordSpec with MustMatchers { "JarCreationWithClasses" should { "succeed when input classes are provided" in { IO.withTemporaryDirectory { tempInputDir => - val tempFile = File.createTempFile(tempInputDir.toString, "Clazz.class") + val tempFile = File.createTempFile("Temp", ".class", tempInputDir) val filePaths = mutable.TreeSet(tempFile.toPath) IO.withTemporaryDirectory { tempOutputDir => val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-valid-output.jar") - OutputUtils.createJar(filePaths, jarOutputPath, System.currentTimeMillis()) - OutputUtils.existsClass(jarOutputPath, tempFile.toString) must be(true) + OutputUtils.createJar(tempInputDir.toString, filePaths, jarOutputPath, System.currentTimeMillis()) + OutputUtils.existsClass(jarOutputPath, tempFile.toString) must be(false) + OutputUtils.existsClass(jarOutputPath, tempFile.getName) must be(true) } } } From f7a8e6f23700f722f18ae716d7fad21b873e4646 Mon Sep 17 00:00:00 2001 From: Ity Kaul Date: Wed, 18 Jul 2018 11:39:04 -0700 Subject: [PATCH 10/11] Add test for nested dir and classes --- .../zinc/compiler/OutputUtils.scala | 28 +++++++++-------- .../scala/org/pantsbuild/zinc/compiler/BUILD | 2 +- .../zinc/compiler/JarCreationSpec.scala | 30 ++++++++++++++----- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala index cf0b4a27242..d6fa0ea94fb 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala @@ -19,17 +19,17 @@ object OutputUtils { * @param dir File handle containing the contents to sort * @return sorted set of all paths within the `dir` */ - def sort(dir:File): mutable.TreeSet[Path] = { - val sorted = new mutable.TreeSet[Path]() + def sort(dir:File): mutable.TreeSet[(Path, Boolean)] = { + val sorted = new mutable.TreeSet[(Path, Boolean)]() val fileSortVisitor = new SimpleFileVisitor[Path]() { override def preVisitDirectory(path: Path, attrs: BasicFileAttributes): FileVisitResult = { - sorted.add(path) + sorted.add(path, false) FileVisitResult.CONTINUE } override def visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult = { - sorted.add(path) + sorted.add(path, true) FileVisitResult.CONTINUE } } @@ -39,7 +39,7 @@ object OutputUtils { } def relativize(base: String, path: Path): String = { - new File(base).toURI().relativize(new File(path.toString).toURI()).getPath() + new File(base.toString).toURI().relativize(new File(path.toString).toURI()).getPath() } /** @@ -50,23 +50,25 @@ object OutputUtils { * @param jarEntryTime time to be set for each JAR entry */ def createJar( - base: String, filePaths: mutable.TreeSet[Path], outputJarPath: Path, jarEntryTime: Long) { + base: String, filePaths: mutable.TreeSet[(Path, Boolean)], outputJarPath: Path, jarEntryTime: Long) { val target = new JarOutputStream(Files.newOutputStream(outputJarPath)) - def addToJar(source: Path, entryName: String): FileVisitResult = { + def addToJar(source: (Path, Boolean), entryName: String): FileVisitResult = { val jarEntry = new JarEntry(entryName) // setting jarEntry time to a fixed value for all entries within the jar so that jars are // byte-for-byte reproducible. jarEntry.setTime(jarEntryTime) target.putNextEntry(jarEntry) - Files.copy(source, target) + if (source._2) { + Files.copy(source._1, target) + } target.closeEntry() FileVisitResult.CONTINUE } - val pathToName = filePaths.zipWithIndex.map{case(k, v) => (k, relativize(base, k))}.toMap + val pathToName = filePaths.zipWithIndex.map{case(k, v) => (k, relativize(base, k._1))}.toMap pathToName.map(e => addToJar(e._1, e._2)) target.close() } @@ -84,13 +86,13 @@ object OutputUtils { } /** - * Determines if a Class exists in a JAR provided. + * Determines if a file exists in a JAR provided. * * @param jarPath Absolute Path to the JAR being inspected - * @param clazz Name of the Class, the existence of which is to be inspected + * @param fileName Name of the file, the existence of which is to be inspected * @return */ - def existsClass(jarPath: Path, clazz: String): Boolean = { + def existsClass(jarPath: Path, fileName: String): Boolean = { var jis: JarInputStream = null var found = false try { @@ -100,7 +102,7 @@ object OutputUtils { def findClass(entry: JarEntry): Boolean = entry match { case null => false - case entry if entry.getName == clazz => + case entry if entry.getName == fileName => true case _ => findClass(jis.getNextJarEntry) diff --git a/tests/scala/org/pantsbuild/zinc/compiler/BUILD b/tests/scala/org/pantsbuild/zinc/compiler/BUILD index 36d029f77cc..1be9429551f 100644 --- a/tests/scala/org/pantsbuild/zinc/compiler/BUILD +++ b/tests/scala/org/pantsbuild/zinc/compiler/BUILD @@ -1,4 +1,4 @@ -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). junit_tests( diff --git a/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala b/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala index fc0eb9d24cf..8f5f8334af9 100644 --- a/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala +++ b/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala @@ -1,11 +1,10 @@ package org.pantsbuild.zinc.compiler -import java.io.File -import java.nio.file.{Path, Paths} -import scala.collection.mutable - import sbt.io.IO +import java.io.File +import java.nio.file.{Files, Path, Paths} +import scala.collection.mutable import org.junit.runner.RunWith import org.scalatest.WordSpec import org.scalatest.junit.JUnitRunner @@ -16,7 +15,7 @@ class JarCreationSpec extends WordSpec with MustMatchers { "JarCreationWithoutClasses" should { "succeed when input classes are not provided" in { IO.withTemporaryDirectory { tempInputDir => - val filePaths = new mutable.TreeSet[Path]() + val filePaths = new mutable.TreeSet[(Path, Boolean)]() IO.withTemporaryDirectory { tempOutputDir => val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-empty-output.jar") @@ -31,14 +30,31 @@ class JarCreationSpec extends WordSpec with MustMatchers { "succeed when input classes are provided" in { IO.withTemporaryDirectory { tempInputDir => val tempFile = File.createTempFile("Temp", ".class", tempInputDir) - val filePaths = mutable.TreeSet(tempFile.toPath) + val filePaths = mutable.TreeSet((tempFile.toPath, true)) IO.withTemporaryDirectory { tempOutputDir => val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-valid-output.jar") OutputUtils.createJar(tempInputDir.toString, filePaths, jarOutputPath, System.currentTimeMillis()) OutputUtils.existsClass(jarOutputPath, tempFile.toString) must be(false) - OutputUtils.existsClass(jarOutputPath, tempFile.getName) must be(true) + OutputUtils.existsClass(jarOutputPath, OutputUtils.relativize(tempInputDir.toString, tempFile.toPath)) must be(true) + } + } + } + } + + "JarCreationWithNestedClasses" should { + "succeed when nested input directory and classes are provided" in { + IO.withTemporaryDirectory { tempInputDir => + val nestedTempDir = Files.createTempDirectory(tempInputDir.toPath, "tmp") + val nestedTempClass = File.createTempFile("NestedTemp", ".class", nestedTempDir.toFile) + val filePaths = mutable.TreeSet((nestedTempDir, false), (nestedTempClass.toPath, true)) + IO.withTemporaryDirectory { tempOutputDir => + val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-valid-output.jar") + + OutputUtils.createJar(tempInputDir.toString, filePaths, jarOutputPath, System.currentTimeMillis()) + OutputUtils.existsClass(jarOutputPath, OutputUtils.relativize(tempInputDir.toString, nestedTempDir)) must be(true) + OutputUtils.existsClass(jarOutputPath, OutputUtils.relativize(tempInputDir.toString, nestedTempClass.toPath)) must be(true) } } } From 98afdd76746c7fc68307b4862c2d0b96d8eccc7a Mon Sep 17 00:00:00 2001 From: Ity Kaul Date: Wed, 18 Jul 2018 15:48:03 -0700 Subject: [PATCH 11/11] Addressed final comments --- .../zinc/compiler/OutputUtils.scala | 31 ++++++++++++------- .../pantsbuild/zinc/compiler/Settings.scala | 8 ++--- .../zinc/compiler/JarCreationSpec.scala | 7 +++-- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala index d6fa0ea94fb..bfeb8e50f8d 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/OutputUtils.scala @@ -19,17 +19,21 @@ object OutputUtils { * @param dir File handle containing the contents to sort * @return sorted set of all paths within the `dir` */ - def sort(dir:File): mutable.TreeSet[(Path, Boolean)] = { - val sorted = new mutable.TreeSet[(Path, Boolean)]() + def sort(dir:File): mutable.TreeSet[Path] = { + val sorted = new mutable.TreeSet[Path]() val fileSortVisitor = new SimpleFileVisitor[Path]() { override def preVisitDirectory(path: Path, attrs: BasicFileAttributes): FileVisitResult = { - sorted.add(path, false) + if (!path.endsWith("/")) { + sorted.add(Paths.get(path.toString, "/")) + } else { + sorted.add(path) + } FileVisitResult.CONTINUE } override def visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult = { - sorted.add(path, true) + sorted.add(path) FileVisitResult.CONTINUE } } @@ -50,25 +54,30 @@ object OutputUtils { * @param jarEntryTime time to be set for each JAR entry */ def createJar( - base: String, filePaths: mutable.TreeSet[(Path, Boolean)], outputJarPath: Path, jarEntryTime: Long) { + base: String, filePaths: mutable.TreeSet[Path], outputJarPath: Path, jarEntryTime: Long) { val target = new JarOutputStream(Files.newOutputStream(outputJarPath)) - def addToJar(source: (Path, Boolean), entryName: String): FileVisitResult = { - val jarEntry = new JarEntry(entryName) + def jarEntry(name: String): JarEntry = { + val jarEntry = new JarEntry(name) // setting jarEntry time to a fixed value for all entries within the jar so that jars are // byte-for-byte reproducible. jarEntry.setTime(jarEntryTime) + jarEntry + } - target.putNextEntry(jarEntry) - if (source._2) { - Files.copy(source._1, target) + def addToJar(source: Path, entryName: String): FileVisitResult = { + if (source.toFile.isDirectory) { + target.putNextEntry(jarEntry(entryName)) + } else { + target.putNextEntry(jarEntry(entryName)) + Files.copy(source, target) } target.closeEntry() FileVisitResult.CONTINUE } - val pathToName = filePaths.zipWithIndex.map{case(k, v) => (k, relativize(base, k._1))}.toMap + val pathToName = filePaths.zipWithIndex.map{case(k, v) => (k, relativize(base, k))}.toMap pathToName.map(e => addToJar(e._1, e._2)) target.close() } diff --git a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala index 5df34cd7563..52142bfe6de 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala @@ -50,7 +50,7 @@ case class Settings( sbt: SbtJars = SbtJars(), _incOptions: IncOptions = IncOptions(), analysis: AnalysisOptions = AnalysisOptions(), - creationTime: Long = System.currentTimeMillis() + creationTime: Long = 0 ) { import Settings._ @@ -59,10 +59,6 @@ case class Settings( } lazy val sources: Seq[File] = _sources map normalise - if (_classesDirectory.isEmpty && outputJar.isEmpty) { - throw new RuntimeException( - s"Either ${Settings.DestinationOpt} or ${Settings.JarDestinationOpt} option is required.") - } lazy val classesDirectory: File = normalise(_classesDirectory.getOrElse(defaultClassesDirectory())) @@ -317,7 +313,7 @@ object Settings extends OptionSet[Settings] { /** * By default the backup location is relative to the classes directory (for example, target/classes/../backup/classes). */ - def defaultBackupLocation(classesDir: File) = { + def defaultBackupLocation(classesDir: File): File = { classesDir.getParentFile / "backup" / classesDir.getName } diff --git a/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala b/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala index 8f5f8334af9..96dfed959a0 100644 --- a/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala +++ b/tests/scala/org/pantsbuild/zinc/compiler/JarCreationSpec.scala @@ -15,7 +15,7 @@ class JarCreationSpec extends WordSpec with MustMatchers { "JarCreationWithoutClasses" should { "succeed when input classes are not provided" in { IO.withTemporaryDirectory { tempInputDir => - val filePaths = new mutable.TreeSet[(Path, Boolean)]() + val filePaths = new mutable.TreeSet[Path]() IO.withTemporaryDirectory { tempOutputDir => val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-empty-output.jar") @@ -26,11 +26,12 @@ class JarCreationSpec extends WordSpec with MustMatchers { } } } + "JarCreationWithClasses" should { "succeed when input classes are provided" in { IO.withTemporaryDirectory { tempInputDir => val tempFile = File.createTempFile("Temp", ".class", tempInputDir) - val filePaths = mutable.TreeSet((tempFile.toPath, true)) + val filePaths = mutable.TreeSet(tempFile.toPath) IO.withTemporaryDirectory { tempOutputDir => val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-valid-output.jar") @@ -48,7 +49,7 @@ class JarCreationSpec extends WordSpec with MustMatchers { IO.withTemporaryDirectory { tempInputDir => val nestedTempDir = Files.createTempDirectory(tempInputDir.toPath, "tmp") val nestedTempClass = File.createTempFile("NestedTemp", ".class", nestedTempDir.toFile) - val filePaths = mutable.TreeSet((nestedTempDir, false), (nestedTempClass.toPath, true)) + val filePaths = mutable.TreeSet(nestedTempDir, nestedTempClass.toPath) IO.withTemporaryDirectory { tempOutputDir => val jarOutputPath = Paths.get(tempOutputDir.toString, "spec-valid-output.jar")