From 08572850a6f79dbf5e79647da82ee9cda1f85d7e Mon Sep 17 00:00:00 2001 From: philwalk Date: Thu, 16 Dec 2021 06:46:20 -0700 Subject: [PATCH] Fix for 14106 cygwin failed tests (#14113) * refactor common code from scripting tests; invalid tests fail by default * dump stdout/stderr if BashScriptsTests.verifyScalaOpts fails * fix for failing cygwin tests for #14106 * normalize scala and scalac paths; set proper SHELLOPTS in cygwin bashCommandline env * improved detection of scalacPath and scalaPath; additional logging * print warnings; remove unused code * strip ansi colors from bash command line output, to fix windows tests * dist/pack before sbt test in test_windows_full and test_non_bootstrapped * squeeze redundancy from env-var-setting tests, add more log messages * further reduced redundancy; additional log messages * remove trailing java from JAVA_HOME value; shorten comand lines with relpath * Update BashScriptsTests.scala remove duplicate * Update BashScriptsTests.scala Fix not-updating property Co-authored-by: Andrzej Ratajczak <32793002+BarkingBad@users.noreply.github.com> --- .github/workflows/ci.yaml | 4 +- .../scripting/classpathReport.sc | 2 +- .../scripting/unglobClasspath.sc | 2 +- .../tools/scripting/BashScriptsTests.scala | 257 ++++++--------- .../tools/scripting/ClasspathTests.scala | 159 +++------- .../dotty/tools/scripting/ScriptTestEnv.scala | 292 ++++++++++++++++++ .../tools/scripting/ScriptingTests.scala | 78 +++-- dist/bin/scala | 10 + 8 files changed, 483 insertions(+), 321 deletions(-) create mode 100644 compiler/test/dotty/tools/scripting/ScriptTestEnv.scala diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8a2c1728496d..1e9a145127bf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -66,7 +66,7 @@ jobs: - name: Test run: | - ./project/scripts/sbt ";compile ;test" + ./project/scripts/sbt ";dist/pack; compile ;test" ./project/scripts/cmdTests test: @@ -181,7 +181,7 @@ jobs: uses: actions/checkout@v2 - name: Test - run: sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test" + run: sbt ";dist/pack ;scala3-bootstrapped/compile ;scala3-bootstrapped/test" shell: cmd - name: Scala.js Test diff --git a/compiler/test-resources/scripting/classpathReport.sc b/compiler/test-resources/scripting/classpathReport.sc index 2a240d2fc3a0..a9eacbbba1f7 100755 --- a/compiler/test-resources/scripting/classpathReport.sc +++ b/compiler/test-resources/scripting/classpathReport.sc @@ -1,4 +1,4 @@ -#!/usr/bin/env -S bin/scala -classpath 'dist/target/pack/lib/*' +#!bin/scala -classpath 'dist/target/pack/lib/*' import java.nio.file.Paths diff --git a/compiler/test-resources/scripting/unglobClasspath.sc b/compiler/test-resources/scripting/unglobClasspath.sc index d05ff78435b3..796697cdedf2 100755 --- a/compiler/test-resources/scripting/unglobClasspath.sc +++ b/compiler/test-resources/scripting/unglobClasspath.sc @@ -1,4 +1,4 @@ -#!/usr/bin/env -S bin/scala -classpath 'dist/target/pack/lib/*' +#!bin/scala -classpath 'dist/target/pack/lib/*' // won't compile unless the hashbang line sets classpath import org.jline.terminal.Terminal diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index 759430a26817..402e618733b9 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -2,33 +2,38 @@ package dotty package tools package scripting -import java.io.File -import java.nio.file.{Path, Paths, Files} -import scala.sys.process._ - -import org.junit.Test +import org.junit.{Test, AfterClass} import org.junit.Assert.assertEquals import vulpix.TestConfiguration -import dotty.tools.dotc.config.Properties._ +import ScriptTestEnv.* /** Verifies correct handling of command line arguments by `dist/bin/scala` and `dist/bin/scalac`. * +. arguments following a script path must be treated as script arguments * +. preserve script command line arguments. + * +. prevent SCALA_OPTS in build environment from infecting tests, via 'SCALA_OPTS= ' prefix + * +. test scripts must not throw execptions or exit with nonzero. */ -class BashScriptsTests: - // classpath tests managed by scripting.ClasspathTests.scala +object BashScriptsTests: + lazy val argsfile = createArgsFile() // avoid problems caused by drive letter def testFiles = scripts("/scripting") + @AfterClass def cleanup: Unit = { + val af = argsfile.toFile + if (af.exists) { + af.delete() + } + } printf("osname[%s]\n", osname) - printf("using JAVA_HOME=%s\n", javaHome) - printf("using SCALA_HOME=%s\n", scalaHome) - printf("first 5 PATH entries:\n%s\n", pathEntries.take(5).mkString("\n")) + printf("uname[%s]\n", ostypeFull) + printf("using JAVA_HOME=%s\n", envJavaHome) + printf("using SCALA_HOME=%s\n", envScalaHome) + printf("first 5 PATH entries:\n%s\n", adjustedPathEntries.take(5).mkString("\n")) printf("scala path: [%s]\n", scalaPath) printf("scalac path: [%s]\n", scalacPath) - lazy val expectedOutput = List( + val expectedOutput = List( "arg 0:[a]", "arg 1:[b]", "arg 2:[c]", @@ -37,16 +42,88 @@ class BashScriptsTests: "arg 5:[-script]", "arg 6:[-debug]", ) - lazy val testScriptArgs = Seq( + val testScriptArgs = Seq( "a", "b", "c", "-repl", "-run", "-script", "-debug" ) val showArgsScript = testFiles.find(_.getName == "showArgs.sc").get.absPath + def testFile(name: String): String = + val file = testFiles.find(_.getName == name) match { + case Some(f) => + val ff = f.absPath + printf("test file [%s] is [%s]\n", name, ff) + ff + case None => + printf("test file [%s] not found!\n", name) + name.absPath + } + file + + val Seq(envtestSc, envtestScala) = Seq("envtest.sc", "envtest.scala").map { testFile(_) } + + // create command line with given options, execute specified script, return stdout + def callScript(tag: String, script: String, keyPre: String): String = + val keyArg = s"$keyPre=$tag" + printf("pass tag [%s] via [%s] to script [%s]\n", tag, keyArg, script) + val cmd: String = Seq("SCALA_OPTS= ", scalaPath, keyArg, script).mkString(" ") + printf("cmd: [%s]\n", cmd) + val (validTest, exitCode, stdout, stderr) = bashCommand(cmd) + stderr.filter { !_.contains("Inappropriate ioctl") }.foreach { System.err.printf("stderr [%s]\n", _) } + stdout.mkString("\n") + + +class BashScriptsTests: + import BashScriptsTests.* + // classpath tests managed by scripting.ClasspathTests.scala + + ////////////////////////// begin tests ////////////////////// + + /* verify that `dist/bin/scala` correctly passes args to the jvm via -J-D for script envtest.sc */ + @Test def verifyScJProperty = + val tag = "World1" + val stdout = callScript(tag, envtestSc, s"-J-Dkey") + assertEquals( s"Hello $tag", stdout) + + /* verify that `dist/bin/scala` correctly passes args to the jvm via -J-D for script envtest.scala */ + @Test def verifyScalaJProperty = + val tag = "World2" + val stdout = callScript(tag, envtestScala, s"-J-Dkey") + assertEquals(s"Hello $tag", stdout) + + /* verify that `dist/bin/scala` can set system properties via -D for envtest.sc */ + @Test def verifyScDProperty = + val tag = "World3" + val stdout = callScript(tag, envtestSc, s"-Dkey") + assertEquals(s"Hello $tag", stdout) + + /* verify that `dist/bin/scala` can set system properties via -D for envtest.scala */ + @Test def verifyScalaDProperty = + val tag = "World4" + val stdout = callScript(tag, envtestScala, s"-Dkey") + assertEquals(s"Hello $tag", stdout) + + /* verify that `dist/bin/scala` can set system properties via -D when executing compiled script via -jar envtest.jar */ + @Test def saveAndRunWithDProperty = + val commandline = Seq("SCALA_OPTS= ", scalaPath.relpath, "-save", envtestScala.relpath).mkString(" ") + val (_, _, _, _) = bashCommand(commandline) // compile jar, discard output + val testJar = testFile("envtest.jar") // jar is created by the previous bashCommand() + if (testJar.isFile){ + printf("compiled envtest.scala to %s\n", testJar.norm) + } else { + sys.error(s"error: unable to compile envtest.scala to ${testJar.norm}") + } + + val tag = "World5" + val commandline2 = Seq("SCALA_OPTS= ", scalaPath.relpath, s"-Dkey=$tag", testJar.relpath) + printf("cmd[%s]\n", commandline2.mkString(" ")) + val (validTest, exitCode, stdout, stderr) = bashCommand(commandline2.mkString(" ")) + assertEquals(s"Hello $tag", stdout.mkString("/n")) + /* verify `dist/bin/scalac` non-interference with command line args following script name */ @Test def verifyScalacArgs = - val commandline = (Seq(scalacPath, "-script", showArgsScript) ++ testScriptArgs).mkString(" ") + val commandline = (Seq("SCALA_OPTS= ", scalacPath, "-script", showArgsScript) ++ testScriptArgs).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - if validTest then + if verifyValid(validTest) then var fail = false printf("\n") for (line, expect) <- stdout zip expectedOutput do @@ -57,46 +134,13 @@ class BashScriptsTests: if fail then assert(stdout == expectedOutput) - /* verify `dist/bin/scala` with -J setting */ - @Test def verifyScJProperty = - val commandline = Seq(scalaPath, "-J-Dkey=World", testFiles.find(_.getName == "envtest.sc").get.absPath).mkString(" ") - val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - assertEquals(stdout.mkString("/n"), "Hello World") - - /* verify `dist/bin/scala` with -J setting */ - @Test def verifyScalaJProperty = - val commandline = Seq(scalaPath, "-J-Dkey=World3", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") - val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - assertEquals(stdout.mkString("/n"), "Hello World3") - - /* verify `dist/bin/scala` with -D setting */ - @Test def verifyScDProperty = - val commandline = Seq(scalaPath, "-Dkey=World3", testFiles.find(_.getName == "envtest.sc").get.absPath).mkString(" ") - val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - assertEquals(stdout.mkString("/n"), "Hello World3") - - /* verify `dist/bin/scala` with -D setting */ - @Test def verifyScalaDProperty = - val commandline = Seq(scalaPath, "-Dkey=World4", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") - val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - assertEquals(stdout.mkString("/n"), "Hello World4") - - /* verify `dist/bin/scala` with -D setting */ - @Test def saveAndRunWithDProperty = - val commandline = Seq(scalaPath, "-save", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") - val (_, _, _, _) = bashCommand(commandline) - val commandline2 = Seq(scalaPath, "-Dkey=World5", testFiles.find(_.getName == "envtest.jar").get.absPath).mkString(" ") - val (validTest, exitCode, stdout, stderr) = bashCommand(commandline2) - assertEquals(stdout.mkString("/n"), "Hello World5") - /* verify `dist/bin/scala` non-interference with command line args following script name */ @Test def verifyScalaArgs = val commandline = (Seq("SCALA_OPTS= ", scalaPath, showArgsScript) ++ testScriptArgs).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - if validTest then + if verifyValid(validTest) then var fail = false printf("\n") - var mismatches = List.empty[(String, String)] for (line, expect) <- stdout zip expectedOutput do printf("expected: %-17s\nactual : %s\n", expect, line) if line != expect then @@ -115,7 +159,7 @@ class BashScriptsTests: printf("===> verify valid system property script.path is reported by script [%s]\n", scriptFile.getName) printf("calling scriptFile: %s\n", scriptFile) val (validTest, exitCode, stdout, stderr) = bashCommand(scriptFile.absPath) - if validTest then + if verifyValid(validTest) then stdout.foreach { printf("stdout: [%s]\n", _) } stderr.foreach { printf("stderr: [%s]\n", _) } val valid = stdout.exists { _.endsWith(expected) } @@ -131,113 +175,16 @@ class BashScriptsTests: val envPairs = List(("SCALA_OPTS", s"@$argsfile")) val (validTest, exitCode, stdout, stderr) = bashCommand(scriptFile.absPath, envPairs) printf("stdout: %s\n", stdout.mkString("\n","\n","")) - if validTest then + if verifyValid(validTest) then val expected = s"${workingDirectory.norm}" - val output = stdout.find( _.trim.startsWith("cwd") ).getOrElse("").dropWhile(_!=' ').trim - printf("output [%s]\n", output) + // stdout might be polluted with an ANSI color prefix, so be careful + val cwdline = stdout.find( _.trim.matches(".*cwd: .*") ).getOrElse("") + printf("cwdline [%s]\n", cwdline) printf("expected[%s]\n", expected) - val valid = output.startsWith(expected) + val valid = cwdline.endsWith(expected) + if (!valid) then + stdout.foreach { printf("stdout[%s]\n", _) } + stderr.foreach { printf("stderr[%s]\n", _) } if valid then printf(s"\n===> success: classpath begins with %s, as reported by [%s]\n", workingDirectory, scriptFile.getName) assert(valid, s"script ${scriptFile.absPath} did not report valid java.class.path first entry") - def existingPath: String = envOrElse("PATH", "").norm - def adjustedPath = s"$javaHome/bin$psep$scalaHome/bin$psep$existingPath" - def pathEntries = adjustedPath.split(psep).toList - - lazy val argsfile = createArgsFile() // avoid problems caused by drive letter - def createArgsFile(): String = - val utfCharset = java.nio.charset.StandardCharsets.UTF_8.name - val path = Files.createTempFile("scriptingTest", ".args") - val text = s"-classpath ${workingDirectory.absPath}" - Files.write(path, text.getBytes(utfCharset)) - path.toFile.getAbsolutePath.norm - - def fixHome(s: String): String = - s.startsWith("~") match { - case false => s - case true => s.replaceFirst("~", userHome) - } - - extension(s: String) { - def toPath: Path = Paths.get(fixHome(s)) // .toAbsolutePath - def toFile: File = s.toPath.toFile - def absPath: String = s.toFile.absPath - def norm: String = s.replace('\\', '/') // bash expects forward slash - def isFile: Boolean = s.toFile.isFile - def exists: Boolean = s.toPath.toFile.exists - def name: String = s.toFile.getName - def dropExtension: String = s.reverse.dropWhile(_ != '.').drop(1).reverse - def parent(up: Int): String = s.norm.split("/").reverse.drop(up).reverse.mkString("/") - } - - extension(p: Path) { - def listFiles: Seq[File] = p.toFile.listFiles.toList - def norm: String = p.normalize.toString.replace('\\', '/') - def name: String = p.toFile.getName - } - - extension(f: File) { - def name = f.getName - def norm: String = f.toPath.normalize.norm - def absPath: String = f.getAbsolutePath.norm - } - - lazy val psep: String = propOrElse("path.separator", "") - lazy val osname = propOrElse("os.name", "").toLowerCase - - lazy val scalacPath = s"$workingDirectory/dist/target/pack/bin/scalac".norm - lazy val scalaPath = s"$workingDirectory/dist/target/pack/bin/scala".norm - - // use optional working directory TEST_CWD, if defined - lazy val workingDirectory: String = envOrElse("TEST_CWD", userDir) - - // use optional TEST_BASH if defined, otherwise, bash must be in PATH - lazy val bashExe: String = envOrElse("TEST_BASH", whichBash) - - // test env SCALA_HOME is: - // dist/target/pack, if present - // else, SCALA_HOME if defined - // else, not defined - lazy val scalaHome = - if scalacPath.isFile then scalacPath.replaceAll("/bin/scalac", "") - else envOrElse("SCALA_HOME", "").norm - - lazy val javaHome = whichJava.parent(2) - - lazy val testEnvPairs = List( - ("JAVA_HOME", javaHome), - ("SCALA_HOME", scalaHome), - ("PATH", adjustedPath), - ).filter { case (name, valu) => valu.nonEmpty } - - lazy val whichBash: String = whichExe("bash") - lazy val whichJava: String = whichExe("java") - - def whichExe(basename: String): String = - val exeName = if (osname.toLowerCase.startsWith("windows")) s"$basename.exe" else basename - which(exeName) - - def bashCommand(cmdstr: String, additionalEnvPairs: List[(String, String)] = Nil): (Boolean, Int, Seq[String], Seq[String]) = { - var (stdout, stderr) = (List.empty[String], List.empty[String]) - if bashExe.toFile.exists then - val cmd = Seq(bashExe, "-c", cmdstr) - val envPairs = testEnvPairs ++ additionalEnvPairs - val proc = Process(cmd, None, envPairs *) - val exitVal = proc ! ProcessLogger ( - (out: String) => stdout ::= out, - (err: String) => stderr ::= err - ) - val validTest = exitVal == 0 && ! stderr.exists(_.contains("Permission denied")) - if ! validTest then - printf("\nunable to execute script, return value is %d\n", exitVal) - stderr.foreach { System.err.printf("stderr [%s]\n", _) } - (validTest, exitVal, stdout.reverse, stderr.reverse) - else - (false, -1, Nil, Nil) - } - - def execCmd(command: String, options: String *): Seq[String] = - val cmd = (command :: options.toList).toSeq - for { - line <- Process(cmd).lazyLines_! - } yield line diff --git a/compiler/test/dotty/tools/scripting/ClasspathTests.scala b/compiler/test/dotty/tools/scripting/ClasspathTests.scala index d755789543a3..767c30e60ad6 100755 --- a/compiler/test/dotty/tools/scripting/ClasspathTests.scala +++ b/compiler/test/dotty/tools/scripting/ClasspathTests.scala @@ -3,27 +3,23 @@ package tools package scripting import java.io.File -import java.nio.file.{Files, Paths, Path} - -import org.junit.Test +import java.nio.file.Path +import org.junit.{Test, Ignore, AfterClass} import vulpix.TestConfiguration - -import scala.sys.process._ -import scala.jdk.CollectionConverters._ -import dotty.tools.dotc.config.Properties._ +import ScriptTestEnv.* /** Test java command line generated by bin/scala and bin/scalac */ -class ClasspathTests: - val packBinDir = "dist/target/pack/bin" - val packLibDir = "dist/target/pack/lib" - - def exists(scriptPath: Path): Boolean = Files.exists(scriptPath) - def packBinScalaExists:Boolean = exists(Paths.get(s"$packBinDir/scala")) +class ClasspathTests: /* + * Test disabled (temporarily). * verify classpath reported by called script. + * This need to be reconceptualized. + * System property "java.class.path" does not necessarily contain the actual runtime path, + * So this test can fail even when the classpath is correct. */ + @Ignore @Test def hashbangClasspathVerifyTest = { // only interested in classpath test scripts val testScriptName = "classpathReport.sc" @@ -41,21 +37,38 @@ class ClasspathTests: cmd.foreach { printf("[%s]\n", _) } - // test script reports the classpath it sees - val scriptOutput = exec(cmd:_*) - val scriptCwd = findTaggedLine("cwd", scriptOutput) + // classpathReport.sc is expected to produce two lines: + // cwd: + // classpath: + + val scriptOutput: Seq[String] = exec(cmd:_*) + val scriptCwd: String = findTaggedLine("cwd", scriptOutput) // the value tagged "cwd: " printf("script ran in directory [%s]\n", scriptCwd) - val scriptCp = findTaggedLine("classpath", scriptOutput) + val scriptCp = findTaggedLine("classpath", scriptOutput) // the value tagged "classpath: " - val hashbangClasspathJars = scriptCp.split(psep).map { _.getName }.sorted.distinct - val packlibJars = listJars(s"$scriptCwd/$packLibDir").sorted.distinct + // convert scriptCp to a list of files + val hashbangJars: List[File] = scriptCp.split(psep).map { _.toFile }.toList + val hashbangClasspathJars = hashbangJars.map { _.name }.sorted.distinct // get jar basenames, remove duplicates + val packlibDir = s"$scriptCwd/$packLibDir" // classpathReport.sc specifies a wildcard classpath in this directory + val packlibJars: List[File] = listJars(packlibDir) // classpath entries expected to have been reported by the script printf("%d jar files in dist/target/pack/lib\n", packlibJars.size) printf("%d test script jars in classpath\n", hashbangClasspathJars.size) - // verify that the classpath set in the hashbang line is effective - if hashbangClasspathJars.size != packlibJars.size then - printf("hashbangClasspathJars: %s\n", hashbangClasspathJars.mkString("\n ", "\n ", "")) + val (diff: Set[File], msg: String) = if (packlibJars.size > hashbangClasspathJars.size) { + (packlibJars.toSet -- hashbangJars.toSet , "only in packlib classpath") + } else { + (hashbangJars.toSet -- packlibJars.toSet , "only in hashbang classpath") + } + // verify that the script hasbang classpath setting was effective at supplementing the classpath + // (a minimal subset of jars below dist/target/pack/lib are always be in the classpath) + val missingClasspathEntries = if hashbangClasspathJars.size != packlibJars.size then + printf("packlib dir [%s]\n", packlibDir) + printf("hashbangClasspathJars: %s\n", hashbangJars.map { _.relpath.norm }.mkString("\n ", "\n ", "")) + printf("# %s\n", msg) + diff.foreach { (f: File) => printf(" %s\n", f.relpath.norm) } + else + Set.empty[String] assert(hashbangClasspathJars.size == packlibJars.size) } @@ -64,7 +77,7 @@ class ClasspathTests: */ @Test def unglobClasspathVerifyTest = { val testScriptName = "unglobClasspath.sc" - val testScript = scripts("/scripting").find { _.getName.matches(testScriptName) } match + val testScript = scripts("/scripting").find { _.name.matches(testScriptName) } match case None => sys.error(s"test script not found: ${testScriptName}") case Some(file) => file @@ -73,7 +86,7 @@ class ClasspathTests: printf("bash is [%s]\n", bashExe) if packBinScalaExists then - val bashCmdline = s"SCALA_OPTS= $relpath" + val bashCmdline = s"set +x ; SCALA_OPTS= $relpath" val cmd = Array(bashExe, "-c", bashCmdline) cmd.foreach { printf("[%s]\n", _) } @@ -81,105 +94,9 @@ class ClasspathTests: // test script reports the classpath it sees val scriptOutput = exec(cmd:_*) val scriptCp = findTaggedLine("unglobbed classpath", scriptOutput) + printf("%s\n", scriptCp) val classpathJars = scriptCp.split(psep).map { _.getName }.sorted.distinct //classpathJars.foreach { printf("%s\n", _) } assert(classpathJars.size > 1) } - -//////////////// end of tests //////////////// -lazy val cwd = Paths.get(".").toAbsolutePath -lazy val wildcardEntry = "dist/target/pack/lib/*" - -def listJars(dir: String) = - val packlibDir = Paths.get(dir).toFile - if packlibDir.isDirectory then - packlibDir.listFiles.toList.map { _.getName }.filter { _.endsWith(".jar") } - else - Nil - -import scala.jdk.CollectionConverters._ -lazy val env:Map[String, String] = System.getenv.asScala.toMap - -// script output expected as ": " -def findTaggedLine(tag: String, lines: Seq[String]): String = - lines.find { _.startsWith(tag) } match - case None => - lines.foreach { System.err.printf("line[%s]\n", _) } - sys.error(s"no $tag: found in script output") - case Some(cwd) => cwd.dropWhile( _ != ' ').trim // discard tag - -def exec(cmd: String *): Seq[String] = Process(cmd).lazyLines_!.toList - -def which(str:String) = - var out = "" - path.find { entry => - val it = Paths.get(entry).toAbsolutePath - it.toFile.isDirectory && { - var testpath = s"$it/$str".norm - val test = Paths.get(testpath) - if test.toFile.exists then - out = testpath - true - else - val test = Paths.get(s"$it/$str.exe".norm) - if test.toFile.exists then - out = testpath - true - else - false - } - } - - out - -def bashExe = which("bash") -def unameExe = which("uname") -def path = envOrElse("PATH", "").split(psep).toList -def psep = sys.props("path.separator") - -def cygwin = ostype == "cygwin" -def mingw = ostype == "mingw" -def msys = ostype == "msys" -def winshell = cygwin || mingw || msys - -def ostypeFull = if unameExe.nonEmpty then exec(unameExe).mkString else "" -def ostype = ostypeFull.toLowerCase.takeWhile{ cc => cc >= 'a' && cc <='z' || cc >= 'A' && cc <= 'Z' } - -extension(p:Path) - def relpath: Path = cwd.relativize(p) - def norm: String = p.toString.replace('\\', '/') - -extension(path: String) - def getName: String = norm.replaceAll(".*/", "") - - // Normalize path separator, convert relative path to absolute - def norm: String = - path.replace('\\', '/') match { - case s if s.secondChar == ":" => s // .drop(2) // path without drive letter - case s if s.startsWith("./") => s.drop(2) - case s => s - } - - def parent: String = norm.replaceAll("/[^/]*$", "") - - // convert to absolute path relative to cwd. - def absPath: String = norm match - case str if str.isAbsolute => norm - case _ => Paths.get(userDir, norm).toString.norm - - def isDir: Boolean = Files.isDirectory(Paths.get(path)) - - def toUrl: String = Paths.get(absPath).toUri.toURL.toString - - // Treat norm paths with a leading '/' as absolute. - // Windows java.io.File#isAbsolute treats them as relative. - def isAbsolute = path.norm.startsWith("/") || (isWin && path.secondChar == ":") - def secondChar: String = path.take(2).drop(1).mkString("") - -extension (str: String) def dropExtension = - str.reverse.dropWhile(_ != '.').drop(1).reverse - -//extension(f: File) def absPath = -//f.getAbsolutePath.replace('\\', '/') - diff --git a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala new file mode 100644 index 000000000000..a37918f93ed2 --- /dev/null +++ b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala @@ -0,0 +1,292 @@ +package dotty +package tools +package scripting + +import java.io.File +import java.nio.file.{Path, Paths, Files} + +import dotty.tools.dotc.config.Properties.* + +import scala.sys.process.* +import scala.jdk.CollectionConverters.* + +/** + * Common Code for supporting scripting tests. + * To override the path to the bash executable, set TEST_BASH= + * To specify where `dist/target/pack/bin` resides, set TEST_CWD= + * Test scripts run in a bash env, so paths are converted to forward slash via .norm. + */ +object ScriptTestEnv { + def osname: String = sys.props("os.name").toLowerCase + def psep: String = sys.props("path.separator") + def userDir: String = sys.props("user.dir").norm + def testCwd = envOrElse("TEST_CWD", "").norm // optional working directory TEST_CWD + + def whichJava: String = whichExe("java") + def whichBash: String = whichExe("bash") + + lazy val workingDirectory: String = { + val dirstr = if testCwd.nonEmpty then + printf("TEST_CWD set to [%s]\n", testCwd) + testCwd + else + userDir // userDir, if TEST_CWD not set + + // issue warning if things don't look right + val test = Paths.get(s"$dirstr/dist/target/pack/bin").normalize + if !test.isDirectory then + printf("warning: not found below working directory: %s\n", test.norm) + + printf("working directory is [%s]\n", dirstr) + dirstr + } + + def envPath: String = envOrElse("PATH", "") + // remove duplicate entries in path + def supplementedPath: String = s"dist/target/pack/bin$psep$envJavaHome/bin$psep$envScalaHome/bin$psep$envPath".norm + def adjustedPathEntries: List[String] = supplementedPath.norm.split(psep).toList.distinct + def adjustedPath: String = adjustedPathEntries.mkString(psep) + def envPathEntries: List[String] = envPath.split(psep).toList.distinct + + def bashExe: String = envOrElse("TEST_BASH", whichBash) + + def unameExe = which("uname") + def ostypeFull = if unameExe.nonEmpty then exec(unameExe).mkString else "" + def ostype = ostypeFull.toLowerCase.takeWhile{ cc => cc >= 'a' && cc <='z' || cc >= 'A' && cc <= 'Z' } + + def cygwin = ostype == "cygwin" + def mingw = ostype == "mingw" + def msys = ostype == "msys" + def winshell: Boolean = cygwin || mingw || msys + + def which(str: String) = + var out = "" + // must not use adjusted path here! (causes recursive call / stack overflow) + envPathEntries.find { entry => + val it = Paths.get(entry).toAbsolutePath.normalize + it.toFile.isDirectory && { + var testpath = s"$it/$str".norm + val test = Paths.get(testpath) + if test.toFile.exists then + out = testpath + true + else + val test = Paths.get(s"$it/$str.exe".norm) + if test.toFile.exists then + out = testpath + true + else + false + } + } + out + + def whichExe(basename: String): String = + val exeName = if (osname.toLowerCase.startsWith("windows")) s"$basename.exe" else basename + which(exeName) + + /* returned values are: + * validTest: Boolean - false if permissions problems occur, true otherwise + * exitVal: Int - the conventional return error code, where zero implies "no errors". + * stdout: Seq[String] - the lines captured from STDOUT + * stderr: Seq[String] - the lines captured from STDERR + */ + def bashCommand(cmdstr: String, additionalEnvPairs: List[(String, String)] = Nil): (Boolean, Int, Seq[String], Seq[String]) = { + var (stdout, stderr) = (List.empty[String], List.empty[String]) + if bashExe.toFile.exists then + def q = "\"" + printf("bashCmd: %s -c %s\n", bashExe, s"$q$cmdstr$q") + val cmd = Seq(bashExe, "-c", cmdstr) + val envPairs = testEnvPairs ++ additionalEnvPairs + val proc = Process(cmd, None, envPairs *) + val exitVal = proc ! ProcessLogger ( + (out: String) => stdout ::= out, + (err: String) => stderr ::= err + ) + // a misconfigured environment (e.g., script is not executable) can prevent script execution + val validTest = !stderr.exists(_.contains("Permission denied")) + if ! validTest then + printf("\nunable to execute script, return value is %d\n", exitVal) + stderr.foreach { System.err.printf("stderr [%s]\n", _) } + + (validTest, exitVal, stdout.reverse, stderr.reverse) + else + (false, -1, Nil, Nil) + } + + def execCmd(command: String, options: String *): Seq[String] = + val cmd = (command :: options.toList).toSeq + for { + line <- Process(cmd).lazyLines_! + } yield line + + + def packBinDir = "dist/target/pack/bin" + def packLibDir = "dist/target/pack/lib" + def packBinScalaExists: Boolean = Files.exists(Paths.get(s"$packBinDir/scala")) + + def listJars(dir: String): List[File] = + val packlibDir = Paths.get(dir).toFile + if packlibDir.isDirectory then + packlibDir.listFiles.toList.filter { _.getName.endsWith(".jar") } + else + Nil + + // script output expected as ": " + def findTaggedLine(tag: String, lines: Seq[String]): String = + lines.map { stripColors(_) }.find { _.startsWith(tag) } match + case None => + lines.foreach { System.err.printf("line[%s]\n", _) } + sys.error(s"no $tag: found in script output") + case Some(cwd) => cwd.dropWhile( _ != ' ').trim // discard tag + + def stripColors(line:String): String = + // ESC has be seen in the wild replaced by "\u2190" + // Also, BOM marker appears as  + lazy val colorsRegex = "(\u001b|\u2190)\\[[0-9;]*m|".r + colorsRegex.replaceAllIn(line,"") + + def exec(cmd: String *): Seq[String] = Process(cmd).lazyLines_!.toList + + def script2jar(scriptFile: File) = + val jarName = s"${scriptFile.getName.dropExtension}.jar" + File(scriptFile.getParent, jarName) + + def showScriptUnderTest(scriptFile: File): Unit = + printf("===> test script name [%s]\n", scriptFile.getName) + + def callExecutableJar(script: File, jar: File, scriptArgs: Array[String] = Array.empty[String]) = { + import scala.sys.process._ + val cmd = Array("java", s"-Dscript.path=${script.getName}", "-jar", jar.absPath) + ++ scriptArgs + Process(cmd).lazyLines_!.foreach { println } + } + + //////////////////////////////////////////////////////////////////////////////// + + def createArgsFile(): String = + val utfCharset = java.nio.charset.StandardCharsets.UTF_8.name + val path = Files.createTempFile("scriptingTest", ".args") + val text = s"-classpath ${workingDirectory.absPath}" + Files.write(path, text.getBytes(utfCharset)) + path.toFile.getAbsolutePath.norm + + def fixHome(s: String): String = + s.startsWith("~") match { + case false => s + case true => s.replaceFirst("~", userHome) + } + + extension(s: String) { + def norm: String = s.replace('\\', '/') // bash expects forward slash + def noDrive = if s.secondChar == ":" then s.drop(2).norm else s.norm + def toPath: Path = Paths.get(fixHome(s)) // .toAbsolutePath + def toFile: File = File(s) + def absPath: String = s.toFile.absPath + def relpath: String = s.norm.replaceFirst(s"${cwd.norm}/","") + def isFile: Boolean = s.toFile.isFile + def isDirectory: Boolean = s.toFile.isDirectory + def exists: Boolean = s.toFile.exists + def name: String = s.toFile.getName + def getName: String = s.toFile.getName + def dropExtension: String = s.reverse.dropWhile(_ != '.').drop(1).reverse + def parent(up: Int): String = s.norm.split("/").reverse.drop(up).reverse.mkString("/") + def secondChar: String = s.take(2).drop(1).mkString("") + } + + extension(p: Path) { + def norm: String = p.normalize.toString.replace('\\', '/') + def noDrive = p.norm match { + case str if str.drop(1).take(1) == ":" => str.drop(2) + case str => str + } + def name: String = p.toFile.getName + def relpath: Path = cwd.relativize(p).normalize + def files: Seq[File] = p.toFile.files + def parent: String = norm.replaceAll("/[^/]*$", "") + + // convert to absolute path relative to cwd. + def absPath: String = if (p.isAbsolute) p.norm else Paths.get(userDir, p.norm).norm + + def isDir: Boolean = Files.isDirectory(p) + def isDirectory: Boolean = p.toFile.isDirectory + def isFile: Boolean = p.toFile.isFile + + def toUrl: String = Paths.get(absPath).toUri.toURL.toString + + // Treat norm paths with a leading '/' as absolute (Windows java.io.File#isAbsolute treats them as relative) + def isAbsolute = p.norm.startsWith("/") || (isWin && p.norm.secondChar == ":") + } + + extension(f: File) { + def name = f.getName + def norm: String = f.toPath.normalize.norm + def absPath: String = f.getAbsolutePath.norm + def relpath: Path = f.toPath.relpath + def files: Seq[File] = f.listFiles.toList + def parentDir: Path = f.toPath.getParent + } + + lazy val cwd: Path = Paths.get(".").toAbsolutePath.normalize + + lazy val (scalacPath: String, scalaPath: String) = { + val scalac = s"$workingDirectory/dist/target/pack/bin/scalac".toPath.normalize + val scala = s"$workingDirectory/dist/target/pack/bin/scala".toPath.normalize + (scalac.norm, scala.norm) + } + + + // use optional TEST_BASH if defined, otherwise, bash must be in PATH + + // envScalaHome is: + // dist/target/pack, if present + // else, SCALA_HOME if defined + // else, not defined + lazy val envScalaHome = + printf("scalacPath: %s\n", scalacPath.norm) + if scalacPath.isFile then scalacPath.replaceAll("/bin/scalac", "") + else envOrElse("SCALA_HOME", "not-found").norm + + lazy val javaParent: String = whichJava.parent(2) + lazy val envJavaHome: String = envOrElse("JAVA_HOME", javaParent) + lazy val cyghome = envOrElse("CYGWIN", "") + lazy val msyshome = envOrElse("MSYS", "") + + // remove xtrace, if present, add :igncr: if not present + lazy val shellopts: String = { + val value: String = envOrElse("SHELLOPTS", "braceexpand:hashall:igncr:ignoreeof:monitor:vi") + val list: List[String] = value.split(":").toList + val minlist = list.filter { + case "igncr" | "xtrace" => false + case _ => true + } + if isWin then + "igncr" :: minlist + else + minlist + }.mkString(":") + + lazy val testEnvPairs = { + val pairs = List( + ("JAVA_HOME", envJavaHome), + ("SCALA_HOME", envScalaHome), + ("PATH", adjustedPath), + ("CYGWIN", cyghome), + ("MSYS", msyshome), + ("SHELLOPTS", shellopts), + ).filter { case (name, valu) => valu.nonEmpty } + for (k, v) <- pairs do + printf("%s : %s\n", k ,v) + pairs + } + + // if unable to execute bash commands, this prevents invalid tests from failing + lazy val passInvalidTests = envOrElse("PASS_INVALID_TESTS", "").nonEmpty + + def verifyValid(validTest: Boolean): Boolean = + // !validTest implies unable to execute scripts via bash (e.g., permissions, or bash not found, etc.) + if !validTest && !passInvalidTests then + assert(validTest == true, s"unable to call script via bash -c") + + validTest +} diff --git a/compiler/test/dotty/tools/scripting/ScriptingTests.scala b/compiler/test/dotty/tools/scripting/ScriptingTests.scala index 922564d4ff58..7ddab2b55424 100644 --- a/compiler/test/dotty/tools/scripting/ScriptingTests.scala +++ b/compiler/test/dotty/tools/scripting/ScriptingTests.scala @@ -8,49 +8,13 @@ import java.nio.file.Path import org.junit.Test import vulpix.TestConfiguration - +import ScriptTestEnv.* /** Runs all tests contained in `compiler/test-resources/scripting/` */ class ScriptingTests: - extension (str: String) def dropExtension = - str.reverse.dropWhile(_ != '.').drop(1).reverse - - extension(f: File) def absPath = - f.getAbsolutePath.replace('\\', '/') - // classpath tests managed by scripting.ClasspathTests.scala def testFiles = scripts("/scripting").filter { ! _.getName.toLowerCase.contains("classpath") } - def script2jar(scriptFile: File) = - val jarName = s"${scriptFile.getName.dropExtension}.jar" - File(scriptFile.getParent, jarName) - - def showScriptUnderTest(scriptFile: File): Unit = - printf("===> test script name [%s]\n", scriptFile.getName) - - val argss: Map[String, Array[String]] = ( - for - argFile <- testFiles - if argFile.getName.endsWith(".args") - name = argFile.getName.dropExtension - scriptArgs = readLines(argFile).toArray - yield name -> scriptArgs).toMap - - def scalaFilesWithArgs(extension: String) = ( - for - scriptFile <- testFiles - if scriptFile.getName.endsWith(extension) - name = scriptFile.getName.dropExtension - scriptArgs = argss.getOrElse(name, Array.empty[String]) - yield scriptFile -> scriptArgs).toList.sortBy { (file, args) => file.getName } - - def callExecutableJar(script: File, jar: File, scriptArgs: Array[String] = Array.empty[String]) = { - import scala.sys.process._ - val cmd = Array("java", s"-Dscript.path=${script.getName}", "-jar", jar.absPath) - ++ scriptArgs - Process(cmd).lazyLines_!.foreach { println } - } - /* * Call .scala scripts without -save option, verify no jar created */ @@ -71,7 +35,7 @@ class ScriptingTests: printf("mainClass from ScriptingDriver: %s\n", mainClass) true // call compiled script main method } - assert(! unexpectedJar.exists, s"not expecting jar file: ${unexpectedJar.absPath}") + assert( !unexpectedJar.exists, s"not expecting jar file: ${unexpectedJar.absPath}" ) /* * Call .sc scripts without -save option, verify no jar created @@ -89,7 +53,7 @@ class ScriptingTests: ) ++ scriptArgs Main.main(mainArgs) - assert(! unexpectedJar.exists, s"not expecting jar file: ${unexpectedJar.absPath}") + assert( !unexpectedJar.exists, s"not expecting jar file: ${unexpectedJar.absPath}") /* * Call .sc scripts with -save option, verify jar is created. @@ -125,7 +89,7 @@ class ScriptingTests: // verify main method not called when false is returned printf("testing script compile, with no call to script main method.\n") touchedFile.delete - assert(!touchedFile.exists, s"unable to delete ${touchedFile}") + assert( !touchedFile.exists, s"unable to delete ${touchedFile}" ) ScriptingDriver( compilerArgs = Array("-classpath", TestConfiguration.basicClasspath), scriptFile = scriptFile, @@ -174,12 +138,44 @@ class ScriptingTests: assert(expectedJar.exists, s"unable to create executable jar [$expectedJar]") touchedFile.delete - assert(!touchedFile.exists, s"unable to delete ${touchedFile}") + assert( !touchedFile.exists, s"unable to delete ${touchedFile}" ) printf("calling executable jar %s\n", expectedJar) callExecutableJar(scriptFile, expectedJar) if touchedFile.exists then printf("success: executable jar created file %s\n", touchedFile) assert( touchedFile.exists, s"expected to find file ${touchedFile}" ) +/////////////////////////////////// def touchFileScript = testFiles.find(_.getName == "touchFile.sc").get + def touchedFile = File("touchedFile.out") + + def script2jar(scriptFile: File) = + val jarName = s"${scriptFile.getName.dropExtension}.jar" + File(scriptFile.getParent, jarName) + + def showScriptUnderTest(scriptFile: File): Unit = + printf("===> test script name [%s]\n", scriptFile.getName) + + def argss: Map[String, Array[String]] = ( + for + argFile <- testFiles + if argFile.getName.endsWith(".args") + name = argFile.getName.dropExtension + scriptArgs = readLines(argFile).toArray + yield name -> scriptArgs).toMap + + def scalaFilesWithArgs(extension: String) = ( + for + scriptFile <- testFiles + if scriptFile.getName.endsWith(extension) + name = scriptFile.getName.dropExtension + scriptArgs = argss.getOrElse(name, Array.empty[String]) + yield scriptFile -> scriptArgs).toList.sortBy { (file, args) => file.getName } + + def callExecutableJar(script: File, jar: File, scriptArgs: Array[String] = Array.empty[String]) = { + import scala.sys.process._ + val cmd = Array("java", s"-Dscript.path=${script.getName}", "-jar", jar.absPath) + ++ scriptArgs + Process(cmd).lazyLines_!.foreach { println } + } diff --git a/dist/bin/scala b/dist/bin/scala index 8a87d6d0d8a0..b3116b2706b3 100755 --- a/dist/bin/scala +++ b/dist/bin/scala @@ -45,6 +45,16 @@ while [[ $# -gt 0 ]]; do addScala "$1" shift ;; + -classpath*) + if [ "$1" != "${1##* }" ]; then + # hashbang-combined args "-classpath 'lib/*'" + A=$1 ; shift # consume $1 before adding its substrings back + set -- $A "$@" # split $1 on whitespace and put it back + else + addScala "$1" + shift + fi + ;; *) addScala "$1" shift