From 1efefac66eb066185c2f27fc78c0ce3b7387df6e Mon Sep 17 00:00:00 2001 From: Brandon Brown Date: Fri, 19 Nov 2021 10:23:26 -0500 Subject: [PATCH] feat(scala3): WIP working on scala 3 cross compilation. #390. --- .travis.yml | 12 + .../play/twirl/api/BaseScalaTemplate.scala | 0 .../play/twirl/api/TwirlHelperImports.scala | 0 .../play/twirl/api/package.scala | 0 .../play/twirl/api/BaseScalaTemplate.scala | 43 + .../play/twirl/api/TwirlHelperImports.scala | 35 + .../scala-2.13/play/twirl/api/package.scala | 47 ++ .../play/twirl/api/BaseScalaTemplate.scala | 43 + .../play/twirl/api/TwirlHelperImports.scala | 35 + .../main/scala-3/play/twirl/api/package.scala | 47 ++ build.sbt | 35 +- .../play/twirl/compiler/TwirlCompiler.scala | 0 .../play/twirl/compiler/TwirlCompiler.scala | 784 ++++++++++++++++++ .../scala/play/twirl/parser/TwirlParser.scala | 12 +- project/Common.scala | 18 +- project/Dependencies.scala | 9 +- project/plugins.sbt | 1 + .../twirl/scalajs-compile/project/plugins.sbt | 4 +- 18 files changed, 1096 insertions(+), 29 deletions(-) rename api/shared/src/main/{scala => scala-2.12}/play/twirl/api/BaseScalaTemplate.scala (100%) rename api/shared/src/main/{scala => scala-2.12}/play/twirl/api/TwirlHelperImports.scala (100%) rename api/shared/src/main/{scala => scala-2.12}/play/twirl/api/package.scala (100%) create mode 100644 api/shared/src/main/scala-2.13/play/twirl/api/BaseScalaTemplate.scala create mode 100644 api/shared/src/main/scala-2.13/play/twirl/api/TwirlHelperImports.scala create mode 100644 api/shared/src/main/scala-2.13/play/twirl/api/package.scala create mode 100644 api/shared/src/main/scala-3/play/twirl/api/BaseScalaTemplate.scala create mode 100644 api/shared/src/main/scala-3/play/twirl/api/TwirlHelperImports.scala create mode 100644 api/shared/src/main/scala-3/play/twirl/api/package.scala rename compiler/src/main/{scala => scala-2}/play/twirl/compiler/TwirlCompiler.scala (100%) create mode 100644 compiler/src/main/scala-3/play/twirl/compiler/TwirlCompiler.scala diff --git a/.travis.yml b/.travis.yml index 9f96139e..a238d8cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,12 @@ jobs: - SCALA_VERSION=2.13.8 - ADOPTOPENJDK=11 + - name: "Run tests with Scala 3 and AdoptOpenJDK 11" + script: scripts/test-code.sh + env: + - TRAVIS_SCALA_VERSION=3.1.0 + - ADOPTOPENJDK=11 + - name: "Run tests with Scala 2.12 and AdoptOpenJDK 8" script: scripts/test-code.sh env: @@ -40,6 +46,12 @@ jobs: - SCALA_VERSION=2.13.8 - ADOPTOPENJDK=8 + - name: "Run tests with Scala 3 and AdoptOpenJDK 8" + script: scripts/test-code.sh + env: + - TRAVIS_SCALA_VERSION=3.1.0 + - ADOPTOPENJDK=8 + - stage: docs script: scripts/validate-docs.sh name: "Validate documentation" diff --git a/api/shared/src/main/scala/play/twirl/api/BaseScalaTemplate.scala b/api/shared/src/main/scala-2.12/play/twirl/api/BaseScalaTemplate.scala similarity index 100% rename from api/shared/src/main/scala/play/twirl/api/BaseScalaTemplate.scala rename to api/shared/src/main/scala-2.12/play/twirl/api/BaseScalaTemplate.scala diff --git a/api/shared/src/main/scala/play/twirl/api/TwirlHelperImports.scala b/api/shared/src/main/scala-2.12/play/twirl/api/TwirlHelperImports.scala similarity index 100% rename from api/shared/src/main/scala/play/twirl/api/TwirlHelperImports.scala rename to api/shared/src/main/scala-2.12/play/twirl/api/TwirlHelperImports.scala diff --git a/api/shared/src/main/scala/play/twirl/api/package.scala b/api/shared/src/main/scala-2.12/play/twirl/api/package.scala similarity index 100% rename from api/shared/src/main/scala/play/twirl/api/package.scala rename to api/shared/src/main/scala-2.12/play/twirl/api/package.scala diff --git a/api/shared/src/main/scala-2.13/play/twirl/api/BaseScalaTemplate.scala b/api/shared/src/main/scala-2.13/play/twirl/api/BaseScalaTemplate.scala new file mode 100644 index 00000000..d8fa9394 --- /dev/null +++ b/api/shared/src/main/scala-2.13/play/twirl/api/BaseScalaTemplate.scala @@ -0,0 +1,43 @@ +/* + * Copyright (C) Lightbend Inc. + */ +package play.twirl.api + +import java.util.Optional +import scala.collection.immutable +import scala.jdk.CollectionConverters._ +import scala.reflect.ClassTag + +case class BaseScalaTemplate[T <: Appendable[T], F <: Format[T]](format: F) { + // The overloaded methods are here for speed. The compiled templates + // can take advantage of them for a 12% performance boost + def _display_(x: AnyVal): T = format.escape(x.toString) + def _display_(x: String): T = if (x eq null) format.empty else format.escape(x) + def _display_(x: Unit): T = format.empty + def _display_(x: scala.xml.NodeSeq): T = if (x eq null) format.empty else format.raw(x.toString()) + def _display_(x: T): T = if (x eq null) format.empty else x + + def _display_(o: Any)(implicit m: ClassTag[T]): T = { + o match { + case escaped if escaped != null && escaped.getClass == m.runtimeClass => escaped.asInstanceOf[T] + case () => format.empty + case None => format.empty + case Some(v) => _display_(v) + case key: Optional[_] => + (if (key.isPresent) Some(key.get) else None) match { + case None => format.empty + case Some(v) => _display_(v) + case _ => format.empty + } + case xml: scala.xml.NodeSeq => format.raw(xml.toString()) + case escapeds: immutable.Seq[_] => format.fill(escapeds.map(_display_)) + case escapeds: TraversableOnce[_] => format.fill(escapeds.iterator.map(_display_).toList) + case escapeds: Array[_] => format.fill(escapeds.view.map(_display_).toList) + case escapeds: java.util.List[_] => + format.fill(escapeds.asScala.map(_display_).toList) + case string: String => format.escape(string) + case v if v != null => format.escape(v.toString) + case _ => format.empty + } + } +} diff --git a/api/shared/src/main/scala-2.13/play/twirl/api/TwirlHelperImports.scala b/api/shared/src/main/scala-2.13/play/twirl/api/TwirlHelperImports.scala new file mode 100644 index 00000000..d0fccdb0 --- /dev/null +++ b/api/shared/src/main/scala-2.13/play/twirl/api/TwirlHelperImports.scala @@ -0,0 +1,35 @@ +/* + * Copyright (C) Lightbend Inc. + */ +package play.twirl.api + +import scala.language.implicitConversions + +/** + * Imports for useful Twirl helpers. + */ +object TwirlHelperImports { + + /** Allows Java collections to be used as if they were Scala collections. */ + implicit def twirlJavaCollectionToScala[T](x: java.lang.Iterable[T]): Iterable[T] = { + import scala.jdk.CollectionConverters._ + x.asScala + } + + /** Allows inline formatting of java.util.Date */ + implicit class TwirlRichDate(date: java.util.Date) { + def format(pattern: String): String = { + new java.text.SimpleDateFormat(pattern).format(date) + } + } + + /** Adds a when method to Strings to control when they are rendered. */ + implicit class TwirlRichString(string: String) { + def when(predicate: => Boolean): String = { + predicate match { + case true => string + case false => "" + } + } + } +} diff --git a/api/shared/src/main/scala-2.13/play/twirl/api/package.scala b/api/shared/src/main/scala-2.13/play/twirl/api/package.scala new file mode 100644 index 00000000..c95b3074 --- /dev/null +++ b/api/shared/src/main/scala-2.13/play/twirl/api/package.scala @@ -0,0 +1,47 @@ +/* + * Copyright (C) Lightbend Inc. + */ +package play.twirl + +import scala.reflect.ClassTag + +package object api { + + /** + * Brings the template engine as a + * [[http://docs.scala-lang.org/overviews/core/string-interpolation.html string interpolator]]. + * + * Basic usage: + * + * {{{ + * import play.twirl.api.StringInterpolation + * + * val name = "Martin" + * val htmlFragment: Html = html"<div>Hello \$name</div>" + * }}} + * + * Three interpolators are available: `html`, `xml` and `js`. + */ + implicit class StringInterpolation(val sc: StringContext) extends AnyVal { + def html(args: Any*): Html = interpolate(args, HtmlFormat) + + def xml(args: Any*): Xml = interpolate(args, XmlFormat) + + def js(args: Any*): JavaScript = interpolate(args, JavaScriptFormat) + + def interpolate[A <: Appendable[A]: ClassTag](args: Seq[Any], format: Format[A]): A = { + StringContext.checkLengths(args, sc.parts) + val array = Array.ofDim[Any](args.size + sc.parts.size) + val strings = sc.parts.iterator + val expressions = args.iterator + array(0) = format.raw(strings.next()) + var i = 1 + while (strings.hasNext) { + array(i) = expressions.next() + array(i + 1) = format.raw(strings.next()) + i += 2 + } + new BaseScalaTemplate[A, Format[A]](format)._display_(array) + } + } +} diff --git a/api/shared/src/main/scala-3/play/twirl/api/BaseScalaTemplate.scala b/api/shared/src/main/scala-3/play/twirl/api/BaseScalaTemplate.scala new file mode 100644 index 00000000..e01b90ea --- /dev/null +++ b/api/shared/src/main/scala-3/play/twirl/api/BaseScalaTemplate.scala @@ -0,0 +1,43 @@ +/* + * Copyright (C) Lightbend Inc. + */ +package play.twirl.api + +import java.util.Optional +import scala.collection.immutable +import scala.jdk.CollectionConverters._ +import scala.reflect.ClassTag + +case class BaseScalaTemplate[T <: Appendable[T], F <: Format[T]](format: F) { + // The overloaded methods are here for speed. The compiled templates + // can take advantage of them for a 12% performance boost + def _display_(x: AnyVal): T = format.escape(x.toString) + def _display_(x: String): T = if (x eq null) format.empty else format.escape(x) + def _display_(x: Unit): T = format.empty + def _display_(x: scala.xml.NodeSeq): T = if (x eq null) format.empty else format.raw(x.toString()) + def _display_(x: T): T = if (x eq null) format.empty else x + + def _display_(o: Any)(implicit m: ClassTag[T]): T = { + o match { + case escaped if escaped != null && escaped.getClass == m.runtimeClass => escaped.asInstanceOf[T] + case () => format.empty + case None => format.empty + case Some(v) => _display_(v) + case key: Optional[_] => + (if (key.isPresent) Some(key.get) else None) match { + case None => format.empty + case Some(v) => _display_(v) + case null => format.empty + } + case xml: scala.xml.NodeSeq => format.raw(xml.toString()) + case escapeds: immutable.Seq[_] => format.fill(escapeds.map(_display_)) + case escapeds: TraversableOnce[_] => format.fill(escapeds.iterator.map(_display_).toList) + case escapeds: Array[_] => format.fill(escapeds.view.map(_display_).toList) + case escapeds: java.util.List[_] => + format.fill(escapeds.asScala.map(_display_).toList) + case string: String => format.escape(string) + case v if v != null => format.escape(v.toString) + case null => format.empty + } + } +} diff --git a/api/shared/src/main/scala-3/play/twirl/api/TwirlHelperImports.scala b/api/shared/src/main/scala-3/play/twirl/api/TwirlHelperImports.scala new file mode 100644 index 00000000..d0fccdb0 --- /dev/null +++ b/api/shared/src/main/scala-3/play/twirl/api/TwirlHelperImports.scala @@ -0,0 +1,35 @@ +/* + * Copyright (C) Lightbend Inc. + */ +package play.twirl.api + +import scala.language.implicitConversions + +/** + * Imports for useful Twirl helpers. + */ +object TwirlHelperImports { + + /** Allows Java collections to be used as if they were Scala collections. */ + implicit def twirlJavaCollectionToScala[T](x: java.lang.Iterable[T]): Iterable[T] = { + import scala.jdk.CollectionConverters._ + x.asScala + } + + /** Allows inline formatting of java.util.Date */ + implicit class TwirlRichDate(date: java.util.Date) { + def format(pattern: String): String = { + new java.text.SimpleDateFormat(pattern).format(date) + } + } + + /** Adds a when method to Strings to control when they are rendered. */ + implicit class TwirlRichString(string: String) { + def when(predicate: => Boolean): String = { + predicate match { + case true => string + case false => "" + } + } + } +} diff --git a/api/shared/src/main/scala-3/play/twirl/api/package.scala b/api/shared/src/main/scala-3/play/twirl/api/package.scala new file mode 100644 index 00000000..c95b3074 --- /dev/null +++ b/api/shared/src/main/scala-3/play/twirl/api/package.scala @@ -0,0 +1,47 @@ +/* + * Copyright (C) Lightbend Inc. + */ +package play.twirl + +import scala.reflect.ClassTag + +package object api { + + /** + * Brings the template engine as a + * [[http://docs.scala-lang.org/overviews/core/string-interpolation.html string interpolator]]. + * + * Basic usage: + * + * {{{ + * import play.twirl.api.StringInterpolation + * + * val name = "Martin" + * val htmlFragment: Html = html"<div>Hello \$name</div>" + * }}} + * + * Three interpolators are available: `html`, `xml` and `js`. + */ + implicit class StringInterpolation(val sc: StringContext) extends AnyVal { + def html(args: Any*): Html = interpolate(args, HtmlFormat) + + def xml(args: Any*): Xml = interpolate(args, XmlFormat) + + def js(args: Any*): JavaScript = interpolate(args, JavaScriptFormat) + + def interpolate[A <: Appendable[A]: ClassTag](args: Seq[Any], format: Format[A]): A = { + StringContext.checkLengths(args, sc.parts) + val array = Array.ofDim[Any](args.size + sc.parts.size) + val strings = sc.parts.iterator + val expressions = args.iterator + array(0) = format.raw(strings.next()) + var i = 1 + while (strings.hasNext) { + array(i) = expressions.next() + array(i + 1) = format.raw(strings.next()) + i += 2 + } + new BaseScalaTemplate[A, Format[A]](format)._display_(array) + } + } +} diff --git a/build.sbt b/build.sbt index 5346d7ec..94b492f2 100644 --- a/build.sbt +++ b/build.sbt @@ -1,8 +1,10 @@ import Dependencies._ -import sbtcrossproject.crossProject +import sbtcrossproject.CrossPlugin.autoImport.crossProject import org.scalajs.jsenv.nodejs.NodeJSEnv +Global / onChangedBuildSource := ReloadOnSourceChanges + // Binary compatibility is this version val previousVersion: Option[String] = Some("1.5.0") @@ -56,6 +58,8 @@ lazy val api = crossProject(JVMPlatform, JSPlatform) .enablePlugins(Common, Playdoc, Omnidoc) .configs(Docs) .settings( + scalaVersion := Scala212, + crossScalaVersions := ScalaVersions, mimaSettings, name := "twirl-api", jsEnv := nodeJs, @@ -83,6 +87,8 @@ lazy val parser = project .in(file("parser")) .enablePlugins(Common, Omnidoc) .settings( + scalaVersion := Scala212, + crossScalaVersions := ScalaVersions, mimaSettings, name := "twirl-parser", libraryDependencies += parserCombinators(scalaVersion.value) % Optional, @@ -93,14 +99,24 @@ lazy val parser = project lazy val compiler = project .in(file("compiler")) .enablePlugins(Common, Omnidoc) - .dependsOn(apiJvm, parser % "compile;test->test") .settings( + scalaVersion := Scala212, + crossScalaVersions := ScalaVersions, mimaSettings, - name := "twirl-compiler", - libraryDependencies += "org.scala-lang" % "scala-compiler" % scalaVersion.value, - libraryDependencies += parserCombinators(scalaVersion.value) % "optional", - run / fork := true, + name := "twirl-compiler", + libraryDependencies ++= { + CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, _)) => + // only for scala < 3 + Seq("org.scala-lang" % "scala-compiler" % scalaVersion.value) + case _ => Seq("org.scala-lang" %% "scala3-compiler" % scalaVersion.value) + } + }, + libraryDependencies += parserCombinators(scalaVersion.value) % Optional, + run / fork := true ) + .aggregate(apiJvm, parser) + .dependsOn(apiJvm, parser % "compile->compile;test->test") lazy val plugin = project .in(file("sbt-twirl")) @@ -112,7 +128,10 @@ lazy val plugin = project scalaVersion := Scala212, libraryDependencies += "org.scalatest" %%% "scalatest" % ScalaTestVersion % Test, Compile / resourceGenerators += generateVersionFile.taskValue, - scriptedLaunchOpts += version.apply { v => s"-Dproject.version=$v" }.value, + scriptedLaunchOpts := { + scriptedLaunchOpts.value ++ + Seq("-Xmx1024M", s"-Dplugin.version=${version.value}", s"-Dproject.version=${version.value}") + }, // both `locally`s are to work around sbt/sbt#6161 scriptedDependencies := { locally { val _ = scriptedDependencies.value } @@ -127,7 +146,7 @@ lazy val plugin = project } () }, - mimaFailOnNoPrevious := false, + mimaFailOnNoPrevious := false ) // Version file diff --git a/compiler/src/main/scala/play/twirl/compiler/TwirlCompiler.scala b/compiler/src/main/scala-2/play/twirl/compiler/TwirlCompiler.scala similarity index 100% rename from compiler/src/main/scala/play/twirl/compiler/TwirlCompiler.scala rename to compiler/src/main/scala-2/play/twirl/compiler/TwirlCompiler.scala diff --git a/compiler/src/main/scala-3/play/twirl/compiler/TwirlCompiler.scala b/compiler/src/main/scala-3/play/twirl/compiler/TwirlCompiler.scala new file mode 100644 index 00000000..3fb7fb75 --- /dev/null +++ b/compiler/src/main/scala-3/play/twirl/compiler/TwirlCompiler.scala @@ -0,0 +1,784 @@ +/* + * Copyright (C) Lightbend Inc. + */ +package play.twirl.compiler + +import dotty.tools.dotc.config.ScalaSettings +import dotty.tools.dotc.core.Contexts.Context +import dotty.tools.dotc.core.Contexts.ContextBase +import dotty.tools.dotc.core.Flags +import dotty.tools.dotc.interactive.InteractiveCompiler +import dotty.tools.dotc.interactive.InteractiveDriver +import dotty.tools.dotc.reporting.ConsoleReporter +import dotty.tools.dotc.semanticdb.Diagnostic.Severity +import dotty.tools.dotc.semanticdb.Tree +import java.io.File +import scala.annotation.tailrec +import scala.io.Codec +import play.twirl.parser.TwirlIO +import play.twirl.parser.TwirlParser +import scala.quoted.* +import scala.util.parsing.input.Position +import scala.util.parsing.input.OffsetPosition +import scala.util.parsing.input.NoPosition + +object Hash { + def apply(bytes: Array[Byte], imports: collection.Seq[String]): String = { + import java.security.MessageDigest + val digest = MessageDigest.getInstance("SHA-1") + digest.reset() + digest.update(bytes) + imports.foreach(i => digest.update(i.getBytes("utf-8"))) + digest.digest().map(0xff & _).map { "%02x".format(_) }.foldLeft("") { _ + _ } + } +} + +case class TemplateCompilationError(source: File, message: String, line: Int, column: Int) + extends RuntimeException(message) + +object MaybeGeneratedSource { + def unapply(source: File): Option[GeneratedSource] = apply(source, TwirlIO.defaultCodec) + + def apply(source: File, codec: Codec): Option[GeneratedSource] = { + val generated = GeneratedSource(source, codec) + if (generated.meta.isDefinedAt("SOURCE")) { + Some(generated) + } else { + None + } + } +} + +sealed trait AbstractGeneratedSource { + def content: String + + lazy val meta: Map[String, String] = { + val Meta = """([A-Z]+): (.*)""".r + val UndefinedMeta = """([A-Z]+):""".r + Map.empty[String, String] ++ { + try { + content + .split("-- GENERATED --")(1) + .trim + .split('\n') + .map { m => + m.trim match { + case Meta(key, value) => (key -> value) + case UndefinedMeta(key) => (key -> "") + case _ => ("UNDEFINED", "") + } + } + .toMap + } catch { + case _: Exception => Map.empty[String, String] + } + } + } + + lazy val matrix: Seq[(Int, Int)] = { + for { + pos <- meta("MATRIX").split('|').toIndexedSeq + c = pos.split("->") + } yield try { + Integer.parseInt(c(0)) -> Integer.parseInt(c(1)) + } catch { + case _: Exception => (0, 0) // Skip if MATRIX meta is corrupted + } + } + + lazy val lines: Seq[(Int, Int)] = { + for { + pos <- meta("LINES").split('|').toIndexedSeq + c = pos.split("->") + } yield try { + Integer.parseInt(c(0)) -> Integer.parseInt(c(1)) + } catch { + case _: Exception => (0, 0) // Skip if LINES meta is corrupted + } + } + + def mapPosition(generatedPosition: Int): Int = { + matrix.indexWhere(p => p._1 > generatedPosition) match { + case 0 => 0 + case i if i > 0 => { + val pos = matrix(i - 1) + pos._2 + (generatedPosition - pos._1) + } + case _ => { + val pos = matrix.takeRight(1)(0) + pos._2 + (generatedPosition - pos._1) + } + } + } + + def mapLine(generatedLine: Int): Int = { + lines.indexWhere(p => p._1 > generatedLine) match { + case 0 => 0 + case i if i > 0 => { + val line = lines(i - 1) + line._2 + (generatedLine - line._1) + } + case _ => { + val line = lines.takeRight(1)(0) + line._2 + (generatedLine - line._1) + } + } + } +} + +case class GeneratedSource(file: File, codec: Codec = TwirlIO.defaultCodec) extends AbstractGeneratedSource { + def content = TwirlIO.readFileAsString(file, codec) + + def needRecompilation(imports: collection.Seq[String]): Boolean = + !file.exists || + // A generated source already exist but + source.isDefined && ((source.get.lastModified > file.lastModified) || // the source has been modified + (meta("HASH") != Hash(TwirlIO.readFile(source.get), imports))) // or the hash don't match + + def toSourcePosition(marker: Int): (Int, Int) = { + try { + val targetMarker = mapPosition(marker) + val line = TwirlIO.readFileAsString(source.get, codec).substring(0, targetMarker).split('\n').size + (line, targetMarker) + } catch { + case _: Exception => (0, 0) + } + } + + def source: Option[File] = { + val s = new File(meta("SOURCE")) + if (s == null || !s.exists) { + None + } else { + Some(s) + } + } + + def sync(): Unit = { + if (file.exists && !source.isDefined) { + file.delete() + } + } +} + +case class GeneratedSourceVirtual(path: String) extends AbstractGeneratedSource { + var _content = "" + def setContent(newContent: String): Unit = { + this._content = newContent + } + def content = _content +} + +object TwirlCompiler { + val DefaultImports = Seq( + "_root_.play.twirl.api.TwirlFeatureImports._", + "_root_.play.twirl.api.TwirlHelperImports._", + "_root_.play.twirl.api.Html", + "_root_.play.twirl.api.JavaScript", + "_root_.play.twirl.api.Txt", + "_root_.play.twirl.api.Xml" + ) + + import play.twirl.parser.TreeNodes._ + + def compile( + source: File, + sourceDirectory: File, + generatedDirectory: File, + formatterType: String, + additionalImports: collection.Seq[String] = Nil, + constructorAnnotations: collection.Seq[String] = Nil, + codec: Codec = TwirlIO.defaultCodec, + inclusiveDot: Boolean = false + )(using qctx: Quotes): Option[File] = { + val resultType = formatterType + ".Appendable" + val (templateName, generatedSource) = + generatedFile(source, codec, sourceDirectory, generatedDirectory, inclusiveDot) + if (generatedSource.needRecompilation(additionalImports)) { + val generated = parseAndGenerateCode( + templateName, + TwirlIO.readFile(source), + codec, + relativePath(source), + resultType, + formatterType, + additionalImports, + constructorAnnotations, + inclusiveDot + ) + TwirlIO.writeStringToFile(generatedSource.file, generated.toString, codec) + Some(generatedSource.file) + } else { + None + } + } + + def compileVirtual( + content: String, + source: File, + sourceDirectory: File, + resultType: String, + formatterType: String, + additionalImports: collection.Seq[String] = Nil, + constructorAnnotations: collection.Seq[String] = Nil, + codec: Codec = TwirlIO.defaultCodec, + inclusiveDot: Boolean = false + )(using qctx: Quotes): GeneratedSourceVirtual = { + val (templateName, generatedSource) = generatedFileVirtual(source, sourceDirectory, inclusiveDot) + val generated = parseAndGenerateCode( + templateName, + content.getBytes(codec.charSet), + codec, + relativePath(source), + resultType, + formatterType, + additionalImports, + constructorAnnotations, + inclusiveDot + ) + generatedSource.setContent(generated) + generatedSource + } + + private def relativePath(file: File): String = + new File(".").toURI.relativize(file.toURI).getPath + + def parseAndGenerateCode( + templateName: Array[String], + content: Array[Byte], + codec: Codec, + relativePath: String, + resultType: String, + formatterType: String, + additionalImports: collection.Seq[String], + constructorAnnotations: collection.Seq[String], + inclusiveDot: Boolean + )(using qctx: Quotes): String = { + val templateParser = new TwirlParser(inclusiveDot) + templateParser.parse(new String(content, codec.charSet)) match { + case templateParser.Success(parsed: Template, rest) if rest.atEnd() => { + generateFinalTemplate( + relativePath, + content, + templateName.dropRight(1).mkString("."), + templateName.takeRight(1).mkString, + parsed, + resultType, + formatterType, + additionalImports, + constructorAnnotations + ) + } + case templateParser.Success(_, rest) => { + throw new TemplateCompilationError(new File(relativePath), "Not parsed?", rest.pos().line, rest.pos().column) + } + case templateParser.Error(_, rest, errors) => { + val firstError = errors.head + throw new TemplateCompilationError( + new File(relativePath), + firstError.str, + firstError.pos.line, + firstError.pos.column + ) + } + } + } + + def generatedFile( + template: File, + codec: Codec, + sourceDirectory: File, + generatedDirectory: File, + inclusiveDot: Boolean + ): (Array[String], GeneratedSource) = { + val templateName = { + val name = + source2TemplateName(template, sourceDirectory, template.getName.split('.').takeRight(1).head).split('.') + if (inclusiveDot) addInclusiveDotName(name) else name + } + templateName -> GeneratedSource(new File(generatedDirectory, templateName.mkString("/") + ".template.scala"), codec) + } + + def generatedFileVirtual( + template: File, + sourceDirectory: File, + inclusiveDot: Boolean + ): (Array[String], GeneratedSourceVirtual) = { + val templateName = { + val name = + source2TemplateName(template, sourceDirectory, template.getName.split('.').takeRight(1).head).split('.') + if (inclusiveDot) addInclusiveDotName(name) else name + } + templateName -> GeneratedSourceVirtual(templateName.mkString("/") + ".template.scala") + } + + def addInclusiveDotName(templateName: Array[String]): Array[String] = { + if (!templateName.isEmpty) + templateName.patch(templateName.length - 1, List(templateName.last + "$$TwirlInclusiveDot"), 1) + else + templateName + } + + @tailrec + def source2TemplateName( + f: File, + sourceDirectory: File, + ext: String, + suffix: String = "", + topDirectory: String = "views", + setExt: Boolean = true + ): String = { + val Name = """([a-zA-Z0-9_]+)[.]scala[.]([a-z]+)""".r + (f, f.getName) match { + case (f, _) if f == sourceDirectory => { + if (setExt) { + val parts = suffix.split('.') + Option(parts.dropRight(1).mkString(".")).filterNot(_.isEmpty).map(_ + ".").getOrElse("") + ext + "." + parts + .takeRight(1) + .mkString + } else suffix + } + case (f, name) if name == topDirectory => + source2TemplateName(f.getParentFile, sourceDirectory, ext, name + "." + ext + "." + suffix, topDirectory, false) + case (f, Name(name, _)) if f.isFile => + source2TemplateName(f.getParentFile, sourceDirectory, ext, name, topDirectory, setExt) + case (f, name) if !f.isFile => + source2TemplateName(f.getParentFile, sourceDirectory, ext, name + "." + suffix, topDirectory, setExt) + case (f, name) => + throw TemplateCompilationError( + f, + "Invalid template name [" + name + "], filenames must only consist of alphanumeric characters and underscores or periods.", + 0, + 0 + ) + } + } + + protected def displayVisitedChildren(children: collection.Seq[Any]): collection.Seq[Any] = { + children.size match { + case 0 => Nil + case 1 => Nil :+ "_display_(" :+ children :+ ")" + case _ => Nil :+ "_display_(Seq[Any](" :+ children :+ "))" + } + } + + private val tripleQuote = "\"" * 3 + // Scala doesn't offer a way to escape triple quoted strings inside triple quoted strings (to my knowledge), so we + // have to escape them in this rather crude way + // We need to double escape slashes, since it's a regex replacement + private val escapedTripleQuote = "\\\"" * 3 + private val doubleEscapedTripleQuote = "\\\\\"" * 3 + private val tripleQuoteReplacement = + escapedTripleQuote + " + \\\"" + doubleEscapedTripleQuote + "\\\" + " + escapedTripleQuote + private def quoteAndEscape(text: String): collection.Seq[String] = { + Seq(tripleQuote, text.replaceAll(tripleQuote, tripleQuoteReplacement), tripleQuote) + } + + def visit(elem: collection.Seq[TemplateTree], previous: collection.Seq[Any]): collection.Seq[Any] = { + elem.toList match { + case head :: tail => + visit( + tail, + head match { + case p @ Plain(text) => + // String literals may not be longer than 65536 bytes. They are encoded as UTF-8 in the classfile, each + // UTF-16 2 byte char could end up becoming up to 3 bytes, so that puts an upper limit of somewhere + // over 20000 characters. 20000 characters is a nice round number, use that. + val grouped = StringGrouper(text, 20000) + (if (previous.isEmpty) Nil else previous :+ ",") :+ + "format.raw" :+ Source("(", p.pos) :+ quoteAndEscape(grouped.head) :+ ")" :+ + grouped.tail.flatMap { t => Seq(",\nformat.raw(", quoteAndEscape(t), ")") } + case Comment(msg) => previous + case Display(exp) => + (if (previous.isEmpty) Nil else previous :+ ",") :+ displayVisitedChildren(visit(Seq(exp), Nil)) + case ScalaExp(parts) => + previous :+ parts.map { + case s @ Simple(code) => Source(code, s.pos) + case b @ Block(whitespace, args, content) if content.forall(_.isInstanceOf[ScalaExp]) => + Nil :+ Source(whitespace + "{" + args.getOrElse(""), b.pos) :+ visit(content, Nil) :+ "}" + case b @ Block(whitespace, args, content) => + Nil :+ Source(whitespace + "{" + args.getOrElse(""), b.pos) :+ displayVisitedChildren( + visit(content, Nil) + ) :+ "}" + } + } + ) + case Nil => previous + } + } + + def templateCode(template: Template, resultType: String): collection.Seq[Any] = { + val defs = (template.sub ++ template.defs).map { + case t: Template if t.name.toString == "" => templateCode(t, resultType) + case t: Template => { + Nil :+ (if (t.name.str.startsWith("implicit")) "implicit def " else "def ") :+ Source( + t.name.str, + t.name.pos + ) :+ Source( + t.params.str, + t.params.pos + ) :+ ":" :+ resultType :+ " = {_display_(" :+ templateCode(t, resultType) :+ ")};" + } + case Def(name, params, block) => { + Nil :+ (if (name.str.startsWith("implicit")) "implicit def " else "def ") :+ Source( + name.str, + name.pos + ) :+ Source( + params.str, + params.pos + ) :+ " = {" :+ block.code :+ "};" + } + } + + val imports = formatImports(template.imports) + + Nil :+ imports :+ "\n" :+ defs :+ "\n" :+ "Seq[Any](" :+ visit(template.content, Nil) :+ ")" + } + + def generateCode( + packageName: String, + name: String, + root: Template, + resultType: String, + formatterType: String, + additionalImports: collection.Seq[String], + constructorAnnotations: collection.Seq[String] + )(using qctx: Quotes): collection.Seq[Any] = { + val (renderCall, f, templateType): (String, String, String) = + TemplateAsFunctionCompiler.getFunctionMapping(root.params.str, resultType) + + // Get the imports that we need to include, filtering out empty imports + val imports: Seq[Any] = Seq(additionalImports.map(i => Seq("import ", i, "\n")), formatImports(root.topImports)) + + val classDeclaration = root.constructor.fold[Seq[Any]]( + Seq("object ", name) + ) { constructor => + Vector.empty :+ "/*" :+ constructor.comment.fold("")(_.msg) :+ """*/ +class """ :+ name :+ " " :+ constructorAnnotations :+ " " :+ Source(constructor.params.str, constructor.params.pos) + } + + val generated = { + Vector.empty :+ """ +package """ :+ packageName :+ """ + +""" :+ imports :+ """ +""" :+ classDeclaration :+ """ extends _root_.play.twirl.api.BaseScalaTemplate[""" :+ resultType :+ """,_root_.play.twirl.api.Format[""" :+ resultType :+ """]](""" :+ formatterType :+ """) with """ :+ templateType :+ """ { + + /*""" :+ root.comment.map(_.msg).getOrElse("") :+ """*/ + def apply""" :+ Source(root.params.str, root.params.pos) :+ """:""" :+ resultType :+ """ = { + _display_ { + { +""" :+ templateCode(root, resultType) :+ """ + } + } + } + + """ :+ renderCall :+ """ + + """ :+ f :+ """ + + def ref: this.type = this + +} + +""" + } + + generated + } + + def formatImports(imports: collection.Seq[Simple]): collection.Seq[Any] = { + imports.map(i => Seq(Source(i.code, i.pos), "\n")) + } + + def generateFinalTemplate( + relativePath: String, + contents: Array[Byte], + packageName: String, + name: String, + root: Template, + resultType: String, + formatterType: String, + additionalImports: collection.Seq[String], + constructorAnnotations: collection.Seq[String] + )(using qctx: Quotes): String = { + val generated = + generateCode(packageName, name, root, resultType, formatterType, additionalImports, constructorAnnotations) + + Source.finalSource(relativePath, contents, generated, Hash(contents, additionalImports)) + } + + object TemplateAsFunctionCompiler { + // Note, the presentation compiler is not thread safe, all access to it must be synchronized. If access to it + // is not synchronized, then weird things happen like FreshRunReq exceptions are thrown when multiple sub projects + // are compiled (done in parallel by default by SBT). So if adding any new methods to this object, make sure you + // make them synchronized. + + import java.io.File + import dotc.util.SourceFile + import dotc.util.Position + import dotc.util.BatchSourceFile + // For some reason they got rid of mods.isByNameParam + object ByNameParam { + def unapply(param: ValDef): Option[(String, String)] = + if (param.mods.hasFlag(Flags.BYNAMEPARAM)) { + Some((param.name.toString, param.tpt.children(1).toString)) + } else None + } + + def getFunctionMapping(signature: String, returnType: String)(using qctx: Quotes): (String, String, String) = ??? + synchronized { + import qctx.reflect.* + import quotes.reflect.report + + def filterType(t: String) = + t.replace("_root_.scala.", "Array") + .replace("", "") + + def findSignature(tree: Tree): Option[DefDef] = { + tree match { + case t: DefDef if t.name.toString == "signature" => Some(t) + case t: Tree => t.children.flatMap(findSignature).headOption + } + } + + val params: DefDef = + findSignature(PresentationCompiler.treeFrom("object FT { def signature" + signature + " }")).get + + val functionType = "(" + params + .map(group => + "(" + group + .map { + case ByNameParam(_, paramType) => " => " + paramType + case a => filterType(a.tpt.toString) + } + .mkString(",") + ")" + ) + .mkString(" => ") + " => " + returnType + ")" + + val renderCall = "def render%s: %s = apply%s".format( + "(" + params.flatten + .map { + case ByNameParam(name, paramType) => name + ":" + paramType + case a => a.name.toString + ":" + filterType(a.tpt.toString) + } + .mkString(",") + ")", + returnType, + params + .map(group => + "(" + group + .map { p => + p.name.toString + Option(p.tpt.toString) + .filter(_.startsWith("_root_.scala.")) + .map(_ => ".toIndexedSeq:_*") + .getOrElse("") + } + .mkString(",") + ")" + ) + .mkString + ) + + val templateType = "_root_.play.twirl.api.Template%s[%s%s]".format( + params.flatten.size, + params.flatten + .map { + case ByNameParam(_, paramType) => paramType + case a => filterType(a.tpt.toString) + } + .mkString(","), + (if (params.flatten.isEmpty) "" else ",") + returnType + ) + + val f = "def f:%s = %s => apply%s".format( + functionType, + params.map(group => "(" + group.map(_.name.toString).mkString(",") + ")").mkString(" => "), + params + .map(group => + "(" + group + .map { p => + p.name.toString + Option(p.tpt.toString) + .filter(_.startsWith("_root_.scala.")) + .map(_ => ".toIndexedSeq:_*") + .getOrElse("") + } + .mkString(",") + ")" + ) + .mkString + ) + + val resp = (renderCall, f, templateType) + + + val pos = Position.ofMacroExpansion + + resp match { + case None => + report.throwError(new Throwable("Timeout in getFunctionMapping"), pos) + ("", "", "") + case Some(res) => + res match { + case Right(t) => + report.throwError(new Throwable("Throwable in getFunctionMapping", t)) + ("", "", "") + case Left(res) => + res + } + } + } + + class CompilerInstance { + def additionalClassPathEntry: Option[String] = None + + private lazy val initCtx: Context = (new ContextBase).initialCtx + + lazy val compiler = { + val settings = initCtx.settings + val scalaPredefSource = Class.forName("scala.Predef").getProtectionDomain.getCodeSource + + // is null in Eclipse/OSGI but luckily we don't need it there + if (scalaPredefSource != null) { + import java.net.URL + import java.security.CodeSource + def urlToFile(url: URL): File = + try { + val file = new File(url.toURI) + if (file.exists) file else new File(url.getPath) // assume malformed URL + } catch { + case _: java.net.URISyntaxException => + // malformed URL: fallback to using the URL path directly + new File(url.getPath) + } + def toAbsolutePath(cs: CodeSource): String = urlToFile(cs.getLocation).getAbsolutePath + val compilerPath = toAbsolutePath( + Class.forName("scala.tools.nsc.Interpreter").getProtectionDomain.getCodeSource + ) + val libPath = toAbsolutePath(scalaPredefSource) + val pathList = List(compilerPath, libPath) +// val origBootclasspath = +// settings.allSettings.find(_.name == "bootclasspath").map(_.valueIn(settings.defaultState)) +// +// settings.bootclasspath.value = +// ((origBootclasspath :: pathList) ::: additionalClassPathEntry.toList).mkString(File.pathSeparator) + } + + val compiler = new InteractiveDriver(List()) + + compiler + } + } + + trait TreeCreationMethods { + val randomFileName = { + val r = new java.util.Random + () => "file" + r.nextInt + } + + def treeFrom(src: String): Tree = { + val file = new BatchSourceFile(randomFileName(), src) + treeFrom(file) + } + + def treeFrom(file: SourceFile): Tree = { + ??? + } +// val r1 = new Response[global.Tree] +// global.askParsedEntered(file, true, r1) +// r1.get match { +// case Left(result) => +// result +// case Right(error) => +// throw error +// } +// } + } + + object CompilerInstance extends CompilerInstance + + object PresentationCompiler extends TreeCreationMethods { + val global = CompilerInstance.compiler + + def shutdown(): Unit = { + global.close(???) + } + } + } +} + +/* ------- */ + +case class Source(code: String, pos: Position = NoPosition) + +object Source { + import scala.collection.mutable.ListBuffer + + def finalSource( + relativePath: String, + contents: Array[Byte], + generatedTokens: collection.Seq[Any], + hash: String + ): String = { + val scalaCode = new StringBuilder + val positions = ListBuffer.empty[(Int, Int)] + val lines = ListBuffer.empty[(Int, Int)] + serialize(generatedTokens, scalaCode, positions, lines) + scalaCode.toString + s""" + /* + -- GENERATED -- + SOURCE: ${relativePath.replace(File.separator, "/")} + HASH: $hash + MATRIX: ${positions.map(pos => s"${pos._1}->${pos._2}").mkString("|")} + LINES: ${lines.map(line => s"${line._1}->${line._2}").mkString("|")} + -- GENERATED -- + */ + """ + } + + private def serialize( + parts: collection.Seq[Any], + source: StringBuilder, + positions: ListBuffer[(Int, Int)], + lines: ListBuffer[(Int, Int)] + ): Unit = { + parts.foreach { + case s: String => source.append(s) + case Source(code, pos @ OffsetPosition(_, offset)) => { + source.append("/*" + pos + "*/") + positions += (source.length -> offset) + lines += (source.toString.split('\n').size -> pos.line) + source.append(code) + } + case Source(code, NoPosition) => source.append(code) + case s: collection.Seq[any] => serialize(s, source, positions, lines) + } + } +} + +/** + * Groups sub sections of Strings. Basically implements String.grouped, except that it guarantees that it won't break + * surrogate pairs. + */ +object StringGrouper { + + /** + * Group the given string by the given size. + * + * @param s + * The string to group. + * @param n + * The size of the groups. + * @return + * A list of strings, grouped by the specific size. + */ + def apply(s: String, n: Int): List[String] = { + if (s.length <= n + 1 /* because we'll split at n + 1 if character n - 1 is a high surrogate */ ) { + List(s) + } else { + val parts = if (s.charAt(n - 1).isHighSurrogate) { + s.splitAt(n + 1) + } else { + s.splitAt(n) + } + parts._1 :: apply(parts._2, n) + } + } +} diff --git a/parser/src/main/scala/play/twirl/parser/TwirlParser.scala b/parser/src/main/scala/play/twirl/parser/TwirlParser.scala index b885addc..73fd94b8 100644 --- a/parser/src/main/scala/play/twirl/parser/TwirlParser.scala +++ b/parser/src/main/scala/play/twirl/parser/TwirlParser.scala @@ -4,10 +4,11 @@ package play.twirl.parser import scala.annotation.tailrec -import scala.collection.mutable +import scala.collection.mutable.Buffer import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.ListBuffer import scala.util.parsing.input.OffsetPosition +import scala.reflect.ClassTag /** * TwirlParser is a recursive descent parser for a modified grammar of the Play2 template language as loosely defined @@ -328,11 +329,12 @@ class TwirlParser(val shouldParseInclusiveDot: Boolean) { } /** Match zero or more `parser` */ - def several[T, BufferType <: mutable.Buffer[T]](parser: () => T, provided: BufferType = null)(implicit - manifest: Manifest[BufferType] + def several[T, BufferType <: Buffer[T]](parser: () => T, provided: BufferType = null)(implicit + ct: ClassTag[BufferType] ): BufferType = { val ab = - if (provided != null) provided else manifest.runtimeClass.getConstructor().newInstance().asInstanceOf[BufferType] + if (provided != null) provided + else ct.runtimeClass.getDeclaredConstructor().newInstance().asInstanceOf[BufferType] var parsed = parser() while (parsed != null) { ab += parsed @@ -752,7 +754,7 @@ class TwirlParser(val shouldParseInclusiveDot: Boolean) { if (check(".")) { methodCall() match { case m: String => nextLink = m - case _ => + case null => } } diff --git a/project/Common.scala b/project/Common.scala index 14f75af0..7e537bbc 100644 --- a/project/Common.scala +++ b/project/Common.scala @@ -24,17 +24,20 @@ object Common extends AutoPlugin { "-Xlint:unchecked" ) + def scalacVersionSpecificParameters(version: String) = { + CrossVersion.partialVersion(version) match { + case Some((2, n)) if n < 12 => + Seq("-target:jvm-1.8", "-Ywarn-unused:imports", "-Xlint:nullary-unit", "-Xlint", "-Ywarn-dead-code") + case _ => Nil + } + } + val scalacParameters = Seq( "-deprecation", "-feature", "-unchecked", "-encoding", - "utf8", - "-target:jvm-1.8", - "-Ywarn-unused:imports", - "-Xlint:nullary-unit", - "-Xlint", - "-Ywarn-dead-code", + "utf8" ) override def globalSettings = @@ -46,7 +49,8 @@ object Common extends AutoPlugin { licenses := Seq("Apache-2.0" -> url("https://www.apache.org/licenses/LICENSE-2.0.html")), scalaVersion := Scala212, crossScalaVersions := ScalaVersions, - scalacOptions ++= scalacParameters, + semanticdbEnabled := scalaVersion.value != Scala212, + scalacOptions ++= scalacParameters ++ scalacVersionSpecificParameters(scalaVersion.value), javacOptions ++= javacParameters, developers += Developer( "contributors", diff --git a/project/Dependencies.scala b/project/Dependencies.scala index c527b42f..27a62e93 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,11 +1,6 @@ -import sbt.Keys.libraryDependencies - -import sbt._ - object Dependencies { - val Scala212 = "2.12.15" val Scala213 = "2.13.8" - val ScalaVersions = Seq(Scala212, Scala213) - + val Scala3 = "3.1.0" + val ScalaVersions = Seq(Scala212, Scala213, Scala3) } diff --git a/project/plugins.sbt b/project/plugins.sbt index 4ba1ff19..04670aa9 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -7,3 +7,4 @@ addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.1.0") addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "1.0.1") addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.6.0") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6") +addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.4.10") diff --git a/sbt-twirl/src/sbt-test/twirl/scalajs-compile/project/plugins.sbt b/sbt-twirl/src/sbt-test/twirl/scalajs-compile/project/plugins.sbt index 12240395..794b80c9 100644 --- a/sbt-twirl/src/sbt-test/twirl/scalajs-compile/project/plugins.sbt +++ b/sbt-twirl/src/sbt-test/twirl/scalajs-compile/project/plugins.sbt @@ -1,2 +1,2 @@ -addSbtPlugin("com.typesafe.sbt" % "sbt-twirl" % sys.props("project.version")) -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.8.0") +addSbtPlugin("com.typesafe.play" % "sbt-twirl" % sys.props("project.version")) +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.8.0")