diff --git a/.github/workflows/publish-artifacts.yml b/.github/workflows/publish-artifacts.yml index 7f1db0be4a7..01b1d9a8f39 100644 --- a/.github/workflows/publish-artifacts.yml +++ b/.github/workflows/publish-artifacts.yml @@ -50,7 +50,7 @@ jobs: java-version: '11' distribution: temurin - - run: ci/release-maven.sh + - run: ./mill -i mill.scalalib.PublishModule/ release-github: # when in master repo, publish all tags and manual runs on main @@ -72,4 +72,4 @@ jobs: java-version: '11' distribution: temurin - - run: ./mill -i uploadToGithub --authKey $REPO_ACCESS_TOKEN + - run: ./mill -i dist.uploadToGithub --authKey $REPO_ACCESS_TOKEN diff --git a/build.mill b/build.mill index 95698d2e95a..5f0c3a8457b 100644 --- a/build.mill +++ b/build.mill @@ -5,19 +5,15 @@ import coursier.maven.MavenRepository import de.tobiasroeser.mill.vcs.version.VcsVersion import com.goyeau.mill.scalafix.ScalafixModule import mill._ -import mill.api.JarManifest import mill.define.NamedTask import mill.main.Tasks import mill.scalalib._ import mill.scalalib.api.ZincWorkerUtil import mill.scalalib.publish._ -import mill.util.Jvm import mill.resolve.SelectMode import mill.T import mill.define.Cross -import scala.util.matching.Regex - // plugins and dependencies import $meta._ import $file.ci.shared @@ -622,337 +618,13 @@ def formatDep(dep: Dep) = { s"${d.module.organization.value}:${d.module.name.value}:${d.version}" } -val DefaultLocalMillReleasePath = - s"target/mill-release${if (scala.util.Properties.isWin) ".bat" else ""}" def listIn(path: os.Path) = interp.watchValue(os.list(path).map(_.last)) -def launcherScript( - shellJvmArgs: Seq[String], - cmdJvmArgs: Seq[String], - shellClassPath: Agg[String], - cmdClassPath: Agg[String] -) = { - - val millMainClass = "mill.runner.client.MillClientMain" - - Jvm.universalScript( - shellCommands = { - val jvmArgsStr = shellJvmArgs.mkString(" ") - val classpathStr = shellClassPath.mkString(":") - - s"""if [ -z "$$JAVA_HOME" ] ; then - | JAVACMD="java" - |else - | JAVACMD="$$JAVA_HOME/bin/java" - |fi - | - |# Client-server mode doesn't seem to work on WSL, just disable it for now - |# https://stackoverflow.com/a/43618657/871202 - |if grep -qEi "(Microsoft|WSL)" /proc/version > /dev/null 2> /dev/null ; then - | if [ -z $$COURSIER_CACHE ] ; then - | COURSIER_CACHE=.coursier - | fi - |fi - |exec "$$JAVACMD" $jvmArgsStr $$JAVA_OPTS -cp "$classpathStr" $millMainClass "$$@" - |""".stripMargin - }, - cmdCommands = { - val jvmArgsStr = cmdJvmArgs.mkString(" ") - val classpathStr = cmdClassPath.mkString(";") - s"""setlocal EnableDelayedExpansion - |set "JAVACMD=java.exe" - |if not "%JAVA_HOME%"=="" set "JAVACMD=%JAVA_HOME%\\bin\\java.exe" - |if "%1" == "-i" set _I_=true - |if "%1" == "--interactive" set _I_=true - |if "%1" == "--repl" set _I_=true - |if "%1" == "--no-server" set _I_=true - |if "%1" == "--bsp" set _I_=true - | - |"%JAVACMD%" $jvmArgsStr %JAVA_OPTS% -cp "$classpathStr" $millMainClass %* - | - |endlocal - |""".stripMargin - } - ) -} object idea extends MillPublishScalaModule { def moduleDeps = Seq(build.scalalib, build.runner) } -/** - * Version of [[dist]] meant for local integration testing within the Mill - * repo. Looks mostly the same as [[dist]], except it does not have a reference - * to itself in its [[testTransitiveDeps]], to avoid a circular dependency. - */ -object dist0 extends MillPublishJavaModule { - // disable scalafix here because it crashes when a module has no sources - def fix(args: String*): Command[Unit] = Task.Command {} - def moduleDeps = Seq(build.runner, idea) - - def testTransitiveDeps = build.runner.testTransitiveDeps() ++ Seq( - build.main.graphviz.testDep(), - build.runner.linenumbers.testDep(), - build.scalalib.backgroundwrapper.testDep(), - build.contrib.bloop.testDep(), - build.contrib.buildinfo.testDep(), - build.contrib.scoverage.testDep(), - build.contrib.scoverage.worker2.testDep(), - build.contrib.jmh.testDep(), - build.contrib.playlib.testDep(), - build.contrib.playlib.worker("2.8").testDep(), - build.contrib.testng.testDep(), - build.bsp.worker.testDep(), - build.testkit.testDep() - ) -} - -object dist extends MillPublishJavaModule { - def jar = rawAssembly() - def moduleDeps = Seq(build.runner, idea, build.main.init) - - def testTransitiveDeps = dist0.testTransitiveDeps() ++ Seq( - (s"com.lihaoyi-${dist.artifactId()}", dist0.runClasspath().map(_.path).mkString("\n")) - ) - - def genTask(m: ScalaModule) = Task.Anon { Seq(m.jar(), m.sourceJar()) ++ m.runClasspath() } - - def forkArgs: T[Seq[String]] = Task { - val genIdeaArgs = - genTask(build.main.define)() ++ - genTask(build.main.eval)() ++ - genTask(build.main)() ++ - genTask(build.scalalib)() ++ - genTask(build.kotlinlib)() ++ - genTask(build.scalajslib)() ++ - genTask(build.scalanativelib)() - - testArgs() ++ - Seq( - "-DMILL_CLASSPATH=" + runClasspath().map(_.path.toString).mkString(","), - "-DMILL_BUILD_LIBRARIES=" + genIdeaArgs.map(_.path).mkString(","), - s"-DBSP4J_VERSION=${Deps.bsp4j.dep.version}" - ) - } - - def launcher = Task { - val isWin = scala.util.Properties.isWin - val outputPath = Task.dest / (if (isWin) "run.bat" else "run") - - os.write(outputPath, prependShellScript()) - if (!isWin) os.perms.set(outputPath, "rwxrwxrwx") - - PathRef(outputPath) - } - - def extraPublish: T[Seq[PublishInfo]] = Task { - Seq(PublishInfo(file = assembly(), classifier = Some("assembly"), ivyConfig = "compile")) - } - - def assemblyRules = super.assemblyRules ++ Seq( - mill.scalalib.Assembly.Rule.ExcludePattern("mill/local-test-overrides/.*") - ) - - // All modules that we want to aggregate as part of this `dev` assembly. - // Excluding itself, and the `dist` module that uses it - lazy val allPublishModules = build.millInternal.modules.collect { - case m: PublishModule if (m ne this) && (m ne dist) => m - } - - def rawAssembly = Task { - val version = millVersion() - val devRunClasspath = runClasspath().map(_.path) - val filename = if (scala.util.Properties.isWin) "mill.bat" else "mill" - val commonArgs = Seq( - // Workaround for Zinc/JNA bug - // https://github.com/sbt/sbt/blame/6718803ee6023ab041b045a6988fafcfae9d15b5/main/src/main/scala/sbt/Main.scala#L130 - "-Djna.nosys=true" - ) - val shellArgs = Seq("-DMILL_CLASSPATH=$0") ++ commonArgs - val cmdArgs = Seq(""""-DMILL_CLASSPATH=%~dpnx0"""") ++ commonArgs - os.move( - mill.scalalib.Assembly.createAssembly( - devRunClasspath, - prependShellScript = launcherScript(shellArgs, cmdArgs, Agg("$0"), Agg("%~dpnx0")), - assemblyRules = assemblyRules - ).path, - Task.dest / filename - ) - PathRef(Task.dest / filename) - } - def assembly = Task { - Task.traverse(allPublishModules)(m => m.publishLocalCached)() - val raw = rawAssembly().path - os.copy(raw, Task.dest / raw.last) - PathRef(Task.dest / raw.last) - } - - def prependShellScript = Task { - val (millArgs, otherArgs) = - forkArgs().partition(arg => arg.startsWith("-DMILL") && !arg.startsWith("-DMILL_VERSION")) - // Pass Mill options via file, due to small max args limit in Windows - val vmOptionsFile = Task.dest / "mill.properties" - val millOptionsContent = - millArgs.map(_.drop(2).replace("\\", "/")).mkString( - "\r\n" - ) // drop -D prefix, replace \ with / - os.write(vmOptionsFile, millOptionsContent) - val jvmArgs = otherArgs ++ List(s"-DMILL_OPTIONS_PATH=$vmOptionsFile") - val classpath = runClasspath().map(_.path.toString) - launcherScript( - jvmArgs, - jvmArgs, - classpath, - Agg(pathingJar().path.toString) // TODO not working yet on Windows! see #791 - ) - } - - def pathingJar = Task { - // see http://todayguesswhat.blogspot.com/2011/03/jar-manifestmf-class-path-referencing.html - // for more detailed explanation - val isWin = scala.util.Properties.isWin - val classpath = runClasspath().map { pathRef => - val path = - if (isWin) "/" + pathRef.path.toString.replace("\\", "/") - else pathRef.path.toString - if (path.endsWith(".jar")) path - else path + "/" - }.mkString(" ") - val manifestEntries = Map[String, String]( - java.util.jar.Attributes.Name.MANIFEST_VERSION.toString -> "1.0", - "Created-By" -> "Scala mill", - "Class-Path" -> classpath - ) - Jvm.createJar(Agg(), JarManifest(manifestEntries)) - } - - def run(args: Task[Args] = Task.Anon(Args())) = Task.Command(exclusive = true) { - args().value match { - case Nil => mill.api.Result.Failure("Need to pass in cwd as first argument to dev.run") - case wd0 +: rest => - val wd = os.Path(wd0, Task.workspace) - os.makeDir.all(wd) - try { - Jvm.runSubprocess( - Seq(launcher().path.toString) ++ rest, - forkEnv(), - workingDir = wd - ) - mill.api.Result.Success(()) - } catch { - case e: Throwable => - mill.api.Result.Failure(s"dev.run failed with an exception. ${e.getMessage()}") - } - } - } -} - -/** - * Build and install Mill locally. - * - * @param binFile The location where the Mill binary should be installed - * @param ivyRepo The local Ivy repository where Mill modules should be published to - */ -def installLocal(binFile: String = DefaultLocalMillReleasePath, ivyRepo: String = null) = - Task.Command { - PathRef(installLocalTask(Task.Anon(binFile), ivyRepo)()) - } - -def installLocalCache() = Task.Command { - val path = installLocalTask( - Task.Anon((os.home / ".cache" / "mill" / "download" / millVersion()).toString()) - )() - Task.log.outputStream.println(path.toString()) - PathRef(path) -} - -def installLocalTask(binFile: Task[String], ivyRepo: String = null): Task[os.Path] = Task.Anon { - val millBin = dist.assembly() - val targetFile = os.Path(binFile(), Task.workspace) - if (os.exists(targetFile)) - Task.log.info(s"Overwriting existing local Mill binary at ${targetFile}") - os.copy.over(millBin.path, targetFile, createFolders = true) - Task.log.info(s"Published ${dist.allPublishModules.size} modules and installed ${targetFile}") - targetFile -} - -def millBootstrap = Task.Sources(Task.workspace / "mill") - -def bootstrapLauncher = Task { - val outputPath = Task.dest / "mill" - val millBootstrapGrepPrefix = "(\n *DEFAULT_MILL_VERSION=)" - val millDownloadUrlPrefix = "(\n *MILL_DOWNLOAD_URL=)" - - os.write( - outputPath, - os.read(millBootstrap().head.path) - .replaceAll( - millBootstrapGrepPrefix + "[^\\n]+", - "$1" + millVersion() - ) - ) - os.perms.set(outputPath, "rwxrwxrwx") - PathRef(outputPath) -} - -def examplePathsWithArtifactName:Task[Seq[(os.Path,String)]] = Task.Anon{ - for { - exampleMod <- build.example.exampleModules - path = exampleMod.millSourcePath - } yield { - val example = path.subRelativeTo(Task.workspace) - val artifactName = millVersion() + "-" + example.segments.mkString("-") - (path, artifactName) - } -} - - -def exampleZips: T[Seq[PathRef]] = Task { - examplePathsWithArtifactName().map{ case (examplePath, exampleStr) => - os.copy(examplePath, Task.dest / exampleStr, createFolders = true) - os.write(Task.dest / exampleStr / ".mill-version", millLastTag()) - os.copy(bootstrapLauncher().path, Task.dest / exampleStr / "mill") - val zip = Task.dest / s"$exampleStr.zip" - os.proc("zip", "-r", zip, exampleStr).call(cwd = Task.dest) - PathRef(zip) - } -} - -def uploadToGithub(authKey: String) = Task.Command { - val vcsState = VcsVersion.vcsState() - val label = vcsState.format() - if (label != millVersion()) sys.error("Modified mill version detected, aborting upload") - val releaseTag = vcsState.lastTag.getOrElse(sys.error( - "Incomplete git history. No tag found.\nIf on CI, make sure your git checkout job includes enough history." - )) - - if (releaseTag == label) { - // TODO: check if the tag already exists (e.g. because we created it manually) and do not fail - requests.post( - s"https://api.github.com/repos/${Settings.githubOrg}/${Settings.githubRepo}/releases", - data = ujson.Obj("tag_name" -> releaseTag, "name" -> releaseTag), - headers = Seq("Authorization" -> ("token " + authKey)) - ) - } - - val examples = exampleZips().map(z => (z.path, z.path.last)) - - val zips = examples ++ Seq( - (dist.assembly().path, label + "-assembly"), - (bootstrapLauncher().path, label) - ) - - for ((zip, name) <- zips) { - upload.apply( - zip, - releaseTag, - name, - authKey, - Settings.githubOrg, - Settings.githubRepo - ) - } -} private def resolveTasks[T](taskNames: String*): Seq[NamedTask[T]] = { mill.resolve.Resolve.Tasks.resolve( diff --git a/ci/release-maven.sh b/ci/release-maven.sh deleted file mode 100755 index 9ee1e16a2f8..00000000000 --- a/ci/release-maven.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -eu - - -./mill -i mill.scalalib.PublishModule/ \ No newline at end of file diff --git a/ci/test-mill-bootstrap.sh b/ci/test-mill-bootstrap.sh index 50047066a95..924c92632c0 100755 --- a/ci/test-mill-bootstrap.sh +++ b/ci/test-mill-bootstrap.sh @@ -7,7 +7,7 @@ git stash -u git stash -a # Build Mill -./mill -i installLocal +./mill -i dist.installLocal # Clean up git stash -a -m "preserve mill-release" -- target/mill-release diff --git a/dist/package.mill b/dist/package.mill new file mode 100644 index 00000000000..c74944f2594 --- /dev/null +++ b/dist/package.mill @@ -0,0 +1,333 @@ +package build.dist +import mill._, scalalib._, publish._ +import mill.util.Jvm +import mill.api.JarManifest +import de.tobiasroeser.mill.vcs.version.VcsVersion +import $file.ci.upload + +object `package` extends RootModule with build.MillPublishJavaModule { + + /** + * Version of [[dist]] meant for local integration testing within the Mill + * repo. Looks mostly the same as [[dist]], except it does not have a reference + * to itself in its [[testTransitiveDeps]], to avoid a circular dependency. + */ + object dist0 extends build.MillPublishJavaModule { + // disable scalafix here because it crashes when a module has no sources + def fix(args: String*): Command[Unit] = Task.Command {} + def moduleDeps = Seq(build.runner, build.idea) + + def testTransitiveDeps = build.runner.testTransitiveDeps() ++ Seq( + build.main.graphviz.testDep(), + build.runner.linenumbers.testDep(), + build.scalalib.backgroundwrapper.testDep(), + build.contrib.bloop.testDep(), + build.contrib.buildinfo.testDep(), + build.contrib.scoverage.testDep(), + build.contrib.scoverage.worker2.testDep(), + build.contrib.jmh.testDep(), + build.contrib.playlib.testDep(), + build.contrib.playlib.worker("2.8").testDep(), + build.contrib.testng.testDep(), + build.bsp.worker.testDep(), + build.testkit.testDep() + ) + } + + def jar = rawAssembly() + def moduleDeps = Seq(build.runner, build.idea, build.main.init) + + def testTransitiveDeps = dist0.testTransitiveDeps() ++ Seq( + (s"com.lihaoyi-${build.dist.artifactId()}", dist0.runClasspath().map(_.path).mkString("\n")) + ) + + def genTask(m: ScalaModule) = Task.Anon { Seq(m.jar(), m.sourceJar()) ++ m.runClasspath() } + + def forkArgs: T[Seq[String]] = Task { + val genIdeaArgs = + genTask(build.main.define)() ++ + genTask(build.main.eval)() ++ + genTask(build.main)() ++ + genTask(build.scalalib)() ++ + genTask(build.kotlinlib)() ++ + genTask(build.scalajslib)() ++ + genTask(build.scalanativelib)() + + testArgs() ++ + Seq( + "-DMILL_CLASSPATH=" + runClasspath().map(_.path.toString).mkString(","), + "-DMILL_BUILD_LIBRARIES=" + genIdeaArgs.map(_.path).mkString(","), + s"-DBSP4J_VERSION=${build.Deps.bsp4j.dep.version}" + ) + } + + def launcher = Task { + val isWin = scala.util.Properties.isWin + val outputPath = Task.dest / (if (isWin) "run.bat" else "run") + + os.write(outputPath, prependShellScript()) + if (!isWin) os.perms.set(outputPath, "rwxrwxrwx") + + PathRef(outputPath) + } + + def extraPublish: T[Seq[PublishInfo]] = Task { + Seq(PublishInfo(file = assembly(), classifier = Some("assembly"), ivyConfig = "compile")) + } + + def assemblyRules = super.assemblyRules ++ Seq( + mill.scalalib.Assembly.Rule.ExcludePattern("mill/local-test-overrides/.*") + ) + + // All modules that we want to aggregate as part of this `dev` assembly. + // Excluding itself, and the `dist` module that uses it + lazy val allPublishModules = build.millInternal.modules.collect { + case m: PublishModule if (m ne this) && (m ne build.dist) => m + } + + def rawAssembly = Task { + val version = build.millVersion() + val devRunClasspath = runClasspath().map(_.path) + val filename = if (scala.util.Properties.isWin) "mill.bat" else "mill" + val commonArgs = Seq( + // Workaround for Zinc/JNA bug + // https://github.com/sbt/sbt/blame/6718803ee6023ab041b045a6988fafcfae9d15b5/main/src/main/scala/sbt/Main.scala#L130 + "-Djna.nosys=true" + ) + val shellArgs = Seq("-DMILL_CLASSPATH=$0") ++ commonArgs + val cmdArgs = Seq(""""-DMILL_CLASSPATH=%~dpnx0"""") ++ commonArgs + os.move( + mill.scalalib.Assembly.createAssembly( + devRunClasspath, + prependShellScript = launcherScript(shellArgs, cmdArgs, Agg("$0"), Agg("%~dpnx0")), + assemblyRules = assemblyRules + ).path, + Task.dest / filename + ) + PathRef(Task.dest / filename) + } + def assembly = Task { + Task.traverse(allPublishModules)(m => m.publishLocalCached)() + val raw = rawAssembly().path + os.copy(raw, Task.dest / raw.last) + PathRef(Task.dest / raw.last) + } + + def prependShellScript = Task { + val (millArgs, otherArgs) = + forkArgs().partition(arg => arg.startsWith("-DMILL") && !arg.startsWith("-DMILL_VERSION")) + // Pass Mill options via file, due to small max args limit in Windows + val vmOptionsFile = Task.dest / "mill.properties" + val millOptionsContent = + millArgs.map(_.drop(2).replace("\\", "/")).mkString( + "\r\n" + ) // drop -D prefix, replace \ with / + os.write(vmOptionsFile, millOptionsContent) + val jvmArgs = otherArgs ++ List(s"-DMILL_OPTIONS_PATH=$vmOptionsFile") + val classpath = runClasspath().map(_.path.toString) + launcherScript( + jvmArgs, + jvmArgs, + classpath, + Agg(pathingJar().path.toString) // TODO not working yet on Windows! see #791 + ) + } + + def pathingJar = Task { + // see http://todayguesswhat.blogspot.com/2011/03/jar-manifestmf-class-path-referencing.html + // for more detailed explanation + val isWin = scala.util.Properties.isWin + val classpath = runClasspath().map { pathRef => + val path = + if (isWin) "/" + pathRef.path.toString.replace("\\", "/") + else pathRef.path.toString + if (path.endsWith(".jar")) path + else path + "/" + }.mkString(" ") + val manifestEntries = Map[String, String]( + java.util.jar.Attributes.Name.MANIFEST_VERSION.toString -> "1.0", + "Created-By" -> "Scala mill", + "Class-Path" -> classpath + ) + Jvm.createJar(Agg(), JarManifest(manifestEntries)) + } + + def run(args: Task[Args] = Task.Anon(Args())) = Task.Command(exclusive = true) { + args().value match { + case Nil => mill.api.Result.Failure("Need to pass in cwd as first argument to dev.run") + case wd0 +: rest => + val wd = os.Path(wd0, Task.workspace) + os.makeDir.all(wd) + try { + Jvm.runSubprocess( + Seq(launcher().path.toString) ++ rest, + forkEnv(), + workingDir = wd + ) + mill.api.Result.Success(()) + } catch { + case e: Throwable => + mill.api.Result.Failure(s"dev.run failed with an exception. ${e.getMessage()}") + } + } + } + def launcherScript( + shellJvmArgs: Seq[String], + cmdJvmArgs: Seq[String], + shellClassPath: Agg[String], + cmdClassPath: Agg[String] + ) = { + + val millMainClass = "mill.runner.client.MillClientMain" + + Jvm.universalScript( + shellCommands = { + val jvmArgsStr = shellJvmArgs.mkString(" ") + val classpathStr = shellClassPath.mkString(":") + + s"""if [ -z "$$JAVA_HOME" ] ; then + | JAVACMD="java" + |else + | JAVACMD="$$JAVA_HOME/bin/java" + |fi + | + |# Client-server mode doesn't seem to work on WSL, just disable it for now + |# https://stackoverflow.com/a/43618657/871202 + |if grep -qEi "(Microsoft|WSL)" /proc/version > /dev/null 2> /dev/null ; then + | if [ -z $$COURSIER_CACHE ] ; then + | COURSIER_CACHE=.coursier + | fi + |fi + |exec "$$JAVACMD" $jvmArgsStr $$JAVA_OPTS -cp "$classpathStr" $millMainClass "$$@" + |""".stripMargin + }, + cmdCommands = { + val jvmArgsStr = cmdJvmArgs.mkString(" ") + val classpathStr = cmdClassPath.mkString(";") + s"""setlocal EnableDelayedExpansion + |set "JAVACMD=java.exe" + |if not "%JAVA_HOME%"=="" set "JAVACMD=%JAVA_HOME%\\bin\\java.exe" + |if "%1" == "-i" set _I_=true + |if "%1" == "--interactive" set _I_=true + |if "%1" == "--repl" set _I_=true + |if "%1" == "--no-server" set _I_=true + |if "%1" == "--bsp" set _I_=true + | + |"%JAVACMD%" $jvmArgsStr %JAVA_OPTS% -cp "$classpathStr" $millMainClass %* + | + |endlocal + |""".stripMargin + } + ) + } + + val DefaultLocalMillReleasePath = + s"target/mill-release${if (scala.util.Properties.isWin) ".bat" else ""}" + + /** + * Build and install Mill locally. + * + * @param binFile The location where the Mill binary should be installed + * @param ivyRepo The local Ivy repository where Mill modules should be published to + */ + def installLocal(binFile: String = DefaultLocalMillReleasePath, ivyRepo: String = null) = + Task.Command { + PathRef(installLocalTask(Task.Anon(binFile), ivyRepo)()) + } + + def installLocalCache() = Task.Command { + val path = installLocalTask( + Task.Anon((os.home / ".cache" / "mill" / "download" / build.millVersion()).toString()) + )() + Task.log.outputStream.println(path.toString()) + PathRef(path) + } + + def installLocalTask(binFile: Task[String], ivyRepo: String = null): Task[os.Path] = Task.Anon { + val millBin = build.dist.assembly() + val targetFile = os.Path(binFile(), Task.workspace) + if (os.exists(targetFile)) + Task.log.info(s"Overwriting existing local Mill binary at ${targetFile}") + os.copy.over(millBin.path, targetFile, createFolders = true) + Task.log.info(s"Published ${build.dist.allPublishModules.size} modules and installed ${targetFile}") + targetFile + } + + def millBootstrap = Task.Sources(Task.workspace / "mill") + + def bootstrapLauncher = Task { + val outputPath = Task.dest / "mill" + val millBootstrapGrepPrefix = "(\n *DEFAULT_MILL_VERSION=)" + val millDownloadUrlPrefix = "(\n *MILL_DOWNLOAD_URL=)" + + os.write( + outputPath, + os.read(millBootstrap().head.path) + .replaceAll( + millBootstrapGrepPrefix + "[^\\n]+", + "$1" + build.millVersion() + ) + ) + os.perms.set(outputPath, "rwxrwxrwx") + PathRef(outputPath) + } + + def examplePathsWithArtifactName:Task[Seq[(os.Path,String)]] = Task.Anon{ + for { + exampleMod <- build.example.exampleModules + path = exampleMod.millSourcePath + } yield { + val example = path.subRelativeTo(Task.workspace) + val artifactName = build.millVersion() + "-" + example.segments.mkString("-") + (path, artifactName) + } + } + + + def exampleZips: T[Seq[PathRef]] = Task { + examplePathsWithArtifactName().map{ case (examplePath, exampleStr) => + os.copy(examplePath, Task.dest / exampleStr, createFolders = true) + os.write(Task.dest / exampleStr / ".mill-version", build.millLastTag()) + os.copy(bootstrapLauncher().path, Task.dest / exampleStr / "mill") + val zip = Task.dest / s"$exampleStr.zip" + os.proc("zip", "-r", zip, exampleStr).call(cwd = Task.dest) + PathRef(zip) + } + } + + def uploadToGithub(authKey: String) = Task.Command { + val vcsState = VcsVersion.vcsState() + val label = vcsState.format() + if (label != build.millVersion()) sys.error("Modified mill version detected, aborting upload") + val releaseTag = vcsState.lastTag.getOrElse(sys.error( + "Incomplete git history. No tag found.\nIf on CI, make sure your git checkout job includes enough history." + )) + + if (releaseTag == label) { + // TODO: check if the tag already exists (e.g. because we created it manually) and do not fail + requests.post( + s"https://api.github.com/repos/${build.Settings.githubOrg}/${build.Settings.githubRepo}/releases", + data = ujson.Obj("tag_name" -> releaseTag, "name" -> releaseTag), + headers = Seq("Authorization" -> ("token " + authKey)) + ) + } + + val examples = exampleZips().map(z => (z.path, z.path.last)) + + val zips = examples ++ Seq( + (build.dist.assembly().path, label + "-assembly"), + (bootstrapLauncher().path, label) + ) + + for ((zip, name) <- zips) { + upload.apply( + zip, + releaseTag, + name, + authKey, + build.Settings.githubOrg, + build.Settings.githubRepo + ) + } + } +} diff --git a/integration/package.mill b/integration/package.mill index d4c4ade0529..8d82542a9f7 100644 --- a/integration/package.mill +++ b/integration/package.mill @@ -89,6 +89,6 @@ object `package` extends RootModule { /** Deploy freshly build mill for use in tests */ def testMill: T[PathRef] = { val name = if (scala.util.Properties.isWin) "mill.bat" else "mill" - T { PathRef(build.installLocalTask(binFile = T.task((T.dest / name).toString()))()) } + T { PathRef(build.dist.installLocalTask(binFile = T.task((T.dest / name).toString()))()) } } } diff --git a/main/init/package.mill b/main/init/package.mill index 8053cb809ae..0970b00fdd6 100644 --- a/main/init/package.mill +++ b/main/init/package.mill @@ -25,7 +25,7 @@ object `package` extends RootModule with build.MillPublishScalaModule { case None => "0.0.0" } - val data: Seq[(os.SubPath, String)] = build.examplePathsWithArtifactName().map { case (path, str) => + val data: Seq[(os.SubPath, String)] = build.dist.examplePathsWithArtifactName().map { case (path, str) => val downloadUrl = build.Settings.projectUrl + "/releases/download/" + lastTag + "/" + str + ".zip" val subPath = path.subRelativeTo(T.workspace / "example") (subPath, downloadUrl) diff --git a/readme.adoc b/readme.adoc index e8975b4fd46..b9a8978340b 100644 --- a/readme.adoc +++ b/readme.adoc @@ -51,9 +51,10 @@ If you use Mill and like it, you will probably enjoy the following book by the A * https://www.handsonscala.com/[_Hands-on Scala Programming_] -The remainder of this readme is developer-documentation targeted at people who wish to work on Mill's own codebase. -The developer docs assume you have read through the user-facing documentation linked above. -It's also worth spending a few minutes reading the following blog posts to get a sense of Mill's design & motivation: +The remainder of this readme is developer-documentation targeted at people who wish to work +on Mill's own codebase. The developer docs assume you have read through the user-facing +documentation linked above. It's also worth spending a few minutes reading the following +blog posts to get a sense of Mill's design & motivation: * http://www.lihaoyi.com/post/SowhatswrongwithSBT.html["So, what's wrong with SBT?"] * http://www.lihaoyi.com/post/BuildToolsasPureFunctionalPrograms.html[Build Tools as Pure Functional Programs] @@ -84,7 +85,7 @@ The following table contains the main ways you can test the code in | In-Process Tests | `main.__.test`, `scalalib.test`, `contrib.buildinfo.test`, etc. | | | Sub-Process w/o packaging/publishing| `example.\\__.local`, `integration.__.local` | `dist.run` | `test-mill-dev.sh` | Sub-Process w/ packaging/publishing | `example.\\__.server`, `integration.__.server` | `dist.assembly` | `test-mill-release.sh` -| Bootstrapping: Building Mill with your current checkout of Mill | | `installLocal` | `test-mill-bootstrap.sh` +| Bootstrapping: Building Mill with your current checkout of Mill | | `dist.installLocal` | `test-mill-bootstrap.sh` |=== In general, `println` or `pprint.log` should work in most places and be sufficient for @@ -98,34 +99,45 @@ debug statements). === In-Process Tests In-process tests live in the `.test` sub-modules of the various Mill modules. -These range from tiny unit tests, to larger integration tests that instantiate a `mill.testkit.BaseModule` in-process and a `UnitTester` to evaluate tasks on it. +These range from tiny unit tests, to larger integration tests that instantiate a +`mill.testkit.BaseModule` in-process and a `UnitTester` to evaluate tasks on it. -Most "core" tests live in `main.__test`; these should run almost instantly, and cover most of Mill's functionality that is not specific to Scala/Scala.js/etc.. +Most "core" tests live in `main.__test`; these should run almost instantly, and cover +most of Mill's functionality that is not specific to Scala/Scala.js/etc.. Tests specific to Scala/Scala.js/Scala-Native live in -`scalalib.test`/`scalajslib.test`/`scalanativelib.test` respectively, and take a lot longer to run because running them involves actually compiling Scala code. +`scalalib.test`/`scalajslib.test`/`scalanativelib.test` respectively, and take a lot longer +to run because running them involves actually compiling Scala code. The various `contrib` modules also have tests in this style, e.g. `contrib.buildinfo.test` -Note that the in-memory tests compile the `BaseModule` together with the test suite, and do not exercise the Mill script-file bootstrapping, transformation, and compilation process. +Note that the in-memory tests compile the `BaseModule` together with the test suite, +and do not exercise the Mill script-file bootstrapping, transformation, and compilation process. === Sub-Process Tests *without* Packaging/Publishing -`example.\\__.local` and `integration.__.local` tests run Mill end-to-end in a subprocess, but *without* the expensive/slow steps of packaging the core packages into an assembly jar and publishing the remaining packages to +`example.\\__.local` and `integration.__.local` tests run Mill end-to-end in a subprocess, +but *without* the expensive/slow steps of packaging the core packages into an assembly jar +and publishing the remaining packages to `~/.ivy2/local`. You can reproduce these tests manually using `./mill dist.run `. -`example` tests are written in a single `build.mill` file, with the test commands written in a comment with a bash-like syntax together with the build code and comments that explain the example. +`example` tests are written in a single `build.mill` file, with the test commands written +in a comment with a bash-like syntax together with the build code and comments that explain +the example. These serve three purposes: -1. Basic smoke-tests to make sure the functionality works at all, without covering every edge case +1. Basic smoke-tests to make sure the functionality works at all, without covering every + edge case -2. User-facing documentation, with the test cases, test commands, and explanatory comments included in the Mill documentation site +2. User-facing documentation, with the test cases, test commands, and explanatory comments + included in the Mill documentation site 3. Example repositories, that Mill users can download to bootstrap their own projects -The `integration` tests are similar to `example` tests and share most of their test infrastructure, but with differences: +The `integration` tests are similar to `example` tests and share most of their test +infrastructure, but with differences: 1. `integration` tests are meant to test features more thoroughly then `example` tests, covering more and deeper edge cases even at the expense of readability @@ -159,8 +171,11 @@ You can reproduce any of the tests manually using `dist.run`, e.g. === Sub-Process Tests *with* Packaging/Publishing `example.\\__.server`, `integration.__.server`, `example.\\__.fork` and -`integration.__.fork` cover the same test cases as the `.local` tests described above, but they perform packaging of the Mill core modules into an assembly jar, and publish the remaining modules to `~/.ivy2/local`. -This results in a more realistic test environment, but at the cost of taking tens-of-seconds more to run a test after making a code change. +`integration.__.fork` cover the same test cases as the `.local` tests described above, but +they perform packaging of the Mill core modules into an assembly jar, and publish the +remaining modules to `~/.ivy2/local`. +This results in a more realistic test environment, but at the cost of taking tens-of-seconds +more to run a test after making a code change. You can reproduce these tests manually using `dist.assembly`: @@ -171,11 +186,14 @@ You can reproduce these tests manually using `dist.assembly`: There are two flavors of these tests: -1. `.server` test run the test cases with the default configuration, so consecutive commands run in the same long-lived background server process +1. `.server` test run the test cases with the default configuration, so consecutive commands + run in the same long-lived background server process -2. `.fork` test run the test cases with `--no-server`, meaning each command runs in a newly spawned Mill process +2. `.fork` test run the test cases with `--no-server`, meaning each command runs in a newly + spawned Mill process -In general you should spend most of your time working with the `.local` version of the `example` and `integration` tests to save time, and only run `.fork` +In general you should spend most of your time working with the `.local` version of the +`example` and `integration` tests to save time, and only run `.fork` or `.server` once `.local` is passing. === Bootstrapping: Building Mill with your current checkout of Mill @@ -184,22 +202,30 @@ To test bootstrapping of Mill's own Mill build using a version of Mill built fro [source,bash] ---- -./mill installLocal +./mill dist.installLocal ci/patch-mill-bootstrap.sh ---- -This creates a standalone assembly at `target/mill-release` you can use, which references jars published locally in your `~/.ivy2/local` cache, and applies any necessary patches to `build.mill` to deal with changes in Mill between the version specified in `.config/mill-version` that is normally used to build Mill and the `HEAD` version your assembly was created from. -You can then use this standalone assembly to build & re-build your current Mill checkout without worrying about stomping over compiled code that the assembly is using. +This creates a standalone assembly at `target/mill-release` you can use, which references jars +published locally in your `~/.ivy2/local` cache, and applies any necessary patches to +`build.mill` to deal with changes in Mill between the version specified in `.config/mill-version` +that is normally used to build Mill and the `HEAD` version your assembly was created from. +You can then use this standalone assembly to build & re-build your current Mill checkout without +worrying about stomping over compiled code that the assembly is using. + +You can also use `./mill dist.installLocalCache` to provide a "stable" version of Mill that +can be used locally in bootstrap scripts. This assembly is design to work on bash, bash-like shells and Windows Cmd. -If you have another default shell like zsh or fish, you probably need to invoke it with `sh ~/mill-release` or prepend the file with a proper shebang. +If you have another default shell like zsh or fish, you probably need to invoke it with +`sh ~/mill-release` or prepend the file with a proper shebang. If you want to install into a different location or a different Ivy repository, you can set its optional parameters. .Install into `/tmp` [source,bash] ---- -$ ./mill installLocal --binFile /tmp/mill --ivyRepo /tmp/millRepo +$ ./mill dist.installLocal --binFile /tmp/mill --ivyRepo /tmp/millRepo ... Published 44 modules and installed /tmp/mill ---- @@ -237,30 +263,42 @@ The Mill project is organized roughly as follows: * `runner`, `main.*`, `scalalib`, `scalajslib`, `scalanativelib`. -These are general lightweight and dependency-free: mostly configuration & wiring of a Mill build and without the heavy lifting. +These are general lightweight and dependency-free: mostly configuration & wiring of a Mill +build and without the heavy lifting. -Heavy lifting is delegated to the worker modules (described below), which the core modules resolve from Maven Central (or from the local filesystem in dev) and load into isolated classloaders. +Heavy lifting is delegated to the worker modules (described below), which the core modules +resolve from Maven Central (or from the local filesystem in dev) and load into isolated +classloaders. === Worker modules that are resolved from Maven Central * `scalalib.worker`, `scalajslib.worker[0.6]`, `scalajslib.worker[1.0]` -These modules are where the heavy-lifting happens, and include heavy dependencies like the Scala compiler, Scala.js optimizer, etc.. Rather than being bundled in the main assembly & classpath, these are resolved separately from Maven Central (or from the local filesystem in dev) and kept in isolated classloaders. +These modules are where the heavy-lifting happens, and include heavy dependencies like the +Scala compiler, Scala.js optimizer, etc.. Rather than being bundled in the main assembly & +classpath, these are resolved separately from Maven Central (or from the local filesystem +in dev) and kept in isolated classloaders. -This allows a single Mill build to use multiple versions of e.g. the Scala.js optimizer without classpath conflicts. +This allows a single Mill build to use multiple versions of e.g. the Scala.js optimizer +without classpath conflicts. === Contrib modules * `contrib/bloop/`, `contrib/flyway/`, `contrib/scoverage/`, etc. -These are modules that help integrate Mill with the wide variety of different tools and utilities available in the JVM ecosystem. +These are modules that help integrate Mill with the wide variety of different tools and +utilities available in the JVM ecosystem. -These modules are not as stringently reviewed as the main Mill core/worker codebase, and are primarily maintained by their individual contributors. -These are maintained as part of the primary Mill Github repo for easy testing/updating as the core Mill APIs evolve, ensuring that they are always tested and passing against the corresponding version of Mill. +These modules are not as stringently reviewed as the main Mill core/worker codebase, and +are primarily maintained by their individual contributors. +These are maintained as part of the primary Mill Github repo for easy testing/updating as +the core Mill APIs evolve, ensuring that they are always tested and passing against the +corresponding version of Mill. == Compatibility & Stability -Mill maintains backward binary compatibility for each major version (`major.minor.point`), enforced with Mima, for the following packages: +Mill maintains backward binary compatibility for each major version (`major.minor.point`), +enforced with Mima, for the following packages: - `mill.api` - `mill.util` @@ -271,7 +309,8 @@ Mill maintains backward binary compatibility for each major version (`major.mino - `mill.scalajslib` - `mill.scalanativelib` -Other packages like `mill.runner`, `mill.bsp`, etc. are on the classpath but offer no compatibility guarantees. +Other packages like `mill.runner`, `mill.bsp`, etc. are on the classpath but offer no +compatibility guarantees. Currently, Mill does not offer compatibility guarantees for `mill.contrib` packages, although they tend to evolve slowly.