Skip to content

Commit

Permalink
Move object dist and related build logic into dist/package.mill f…
Browse files Browse the repository at this point in the history 
…ile (#3892)

Since #3818 created a `dist/` sub-folder, we may as well use it. This
removes 300+ lines from the root `build.mill` and should speed up
incremental compiles of the build code
  • Loading branch information
@lihaoyi
lihaoyi authored Nov 2, 2024
1 parent 2330f34 commit 94d28d2
Show file tree
Hide file tree
Showing 8 changed files with 409 additions and 371 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/publish-artifacts.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
java-version: '11'
distribution: temurin

- run: ci/release-maven.sh
- run: ./mill -i mill.scalalib.PublishModule/

release-github:
# when in master repo, publish all tags and manual runs on main
Expand All @@ -72,4 +72,4 @@ jobs:
java-version: '11'
distribution: temurin

- run: ./mill -i uploadToGithub --authKey $REPO_ACCESS_TOKEN
- run: ./mill -i dist.uploadToGithub --authKey $REPO_ACCESS_TOKEN
328 changes: 0 additions & 328 deletions build.mill
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,15 @@ import coursier.maven.MavenRepository
import de.tobiasroeser.mill.vcs.version.VcsVersion
import com.goyeau.mill.scalafix.ScalafixModule
import mill._
import mill.api.JarManifest
import mill.define.NamedTask
import mill.main.Tasks
import mill.scalalib._
import mill.scalalib.api.ZincWorkerUtil
import mill.scalalib.publish._
import mill.util.Jvm
import mill.resolve.SelectMode
import mill.T
import mill.define.Cross

import scala.util.matching.Regex

// plugins and dependencies
import $meta._
import $file.ci.shared
Expand Down Expand Up @@ -622,337 +618,13 @@ def formatDep(dep: Dep) = {
s"${d.module.organization.value}:${d.module.name.value}:${d.version}"
}

val DefaultLocalMillReleasePath =
s"target/mill-release${if (scala.util.Properties.isWin) ".bat" else ""}"

def listIn(path: os.Path) = interp.watchValue(os.list(path).map(_.last))
def launcherScript(
shellJvmArgs: Seq[String],
cmdJvmArgs: Seq[String],
shellClassPath: Agg[String],
cmdClassPath: Agg[String]
) = {

val millMainClass = "mill.runner.client.MillClientMain"

Jvm.universalScript(
shellCommands = {
val jvmArgsStr = shellJvmArgs.mkString(" ")
val classpathStr = shellClassPath.mkString(":")

s"""if [ -z "$$JAVA_HOME" ] ; then
| JAVACMD="java"
|else
| JAVACMD="$$JAVA_HOME/bin/java"
|fi
|
|# Client-server mode doesn't seem to work on WSL, just disable it for now
|# https://stackoverflow.com/a/43618657/871202
|if grep -qEi "(Microsoft|WSL)" /proc/version > /dev/null 2> /dev/null ; then
| if [ -z $$COURSIER_CACHE ] ; then
| COURSIER_CACHE=.coursier
| fi
|fi
|exec "$$JAVACMD" $jvmArgsStr $$JAVA_OPTS -cp "$classpathStr" $millMainClass "$$@"
|""".stripMargin
},
cmdCommands = {
val jvmArgsStr = cmdJvmArgs.mkString(" ")
val classpathStr = cmdClassPath.mkString(";")
s"""setlocal EnableDelayedExpansion
|set "JAVACMD=java.exe"
|if not "%JAVA_HOME%"=="" set "JAVACMD=%JAVA_HOME%\\bin\\java.exe"
|if "%1" == "-i" set _I_=true
|if "%1" == "--interactive" set _I_=true
|if "%1" == "--repl" set _I_=true
|if "%1" == "--no-server" set _I_=true
|if "%1" == "--bsp" set _I_=true
|
|"%JAVACMD%" $jvmArgsStr %JAVA_OPTS% -cp "$classpathStr" $millMainClass %*
|
|endlocal
|""".stripMargin
}
)
}

object idea extends MillPublishScalaModule {
def moduleDeps = Seq(build.scalalib, build.runner)
}

/**
* Version of [[dist]] meant for local integration testing within the Mill
* repo. Looks mostly the same as [[dist]], except it does not have a reference
* to itself in its [[testTransitiveDeps]], to avoid a circular dependency.
*/
object dist0 extends MillPublishJavaModule {
// disable scalafix here because it crashes when a module has no sources
def fix(args: String*): Command[Unit] = Task.Command {}
def moduleDeps = Seq(build.runner, idea)

def testTransitiveDeps = build.runner.testTransitiveDeps() ++ Seq(
build.main.graphviz.testDep(),
build.runner.linenumbers.testDep(),
build.scalalib.backgroundwrapper.testDep(),
build.contrib.bloop.testDep(),
build.contrib.buildinfo.testDep(),
build.contrib.scoverage.testDep(),
build.contrib.scoverage.worker2.testDep(),
build.contrib.jmh.testDep(),
build.contrib.playlib.testDep(),
build.contrib.playlib.worker("2.8").testDep(),
build.contrib.testng.testDep(),
build.bsp.worker.testDep(),
build.testkit.testDep()
)
}

object dist extends MillPublishJavaModule {
def jar = rawAssembly()
def moduleDeps = Seq(build.runner, idea, build.main.init)

def testTransitiveDeps = dist0.testTransitiveDeps() ++ Seq(
(s"com.lihaoyi-${dist.artifactId()}", dist0.runClasspath().map(_.path).mkString("\n"))
)

def genTask(m: ScalaModule) = Task.Anon { Seq(m.jar(), m.sourceJar()) ++ m.runClasspath() }

def forkArgs: T[Seq[String]] = Task {
val genIdeaArgs =
genTask(build.main.define)() ++
genTask(build.main.eval)() ++
genTask(build.main)() ++
genTask(build.scalalib)() ++
genTask(build.kotlinlib)() ++
genTask(build.scalajslib)() ++
genTask(build.scalanativelib)()

testArgs() ++
Seq(
"-DMILL_CLASSPATH=" + runClasspath().map(_.path.toString).mkString(","),
"-DMILL_BUILD_LIBRARIES=" + genIdeaArgs.map(_.path).mkString(","),
s"-DBSP4J_VERSION=${Deps.bsp4j.dep.version}"
)
}

def launcher = Task {
val isWin = scala.util.Properties.isWin
val outputPath = Task.dest / (if (isWin) "run.bat" else "run")

os.write(outputPath, prependShellScript())
if (!isWin) os.perms.set(outputPath, "rwxrwxrwx")

PathRef(outputPath)
}

def extraPublish: T[Seq[PublishInfo]] = Task {
Seq(PublishInfo(file = assembly(), classifier = Some("assembly"), ivyConfig = "compile"))
}

def assemblyRules = super.assemblyRules ++ Seq(
mill.scalalib.Assembly.Rule.ExcludePattern("mill/local-test-overrides/.*")
)

// All modules that we want to aggregate as part of this `dev` assembly.
// Excluding itself, and the `dist` module that uses it
lazy val allPublishModules = build.millInternal.modules.collect {
case m: PublishModule if (m ne this) && (m ne dist) => m
}

def rawAssembly = Task {
val version = millVersion()
val devRunClasspath = runClasspath().map(_.path)
val filename = if (scala.util.Properties.isWin) "mill.bat" else "mill"
val commonArgs = Seq(
// Workaround for Zinc/JNA bug
// https://github.com/sbt/sbt/blame/6718803ee6023ab041b045a6988fafcfae9d15b5/main/src/main/scala/sbt/Main.scala#L130
"-Djna.nosys=true"
)
val shellArgs = Seq("-DMILL_CLASSPATH=$0") ++ commonArgs
val cmdArgs = Seq(""""-DMILL_CLASSPATH=%~dpnx0"""") ++ commonArgs
os.move(
mill.scalalib.Assembly.createAssembly(
devRunClasspath,
prependShellScript = launcherScript(shellArgs, cmdArgs, Agg("$0"), Agg("%~dpnx0")),
assemblyRules = assemblyRules
).path,
Task.dest / filename
)
PathRef(Task.dest / filename)
}
def assembly = Task {
Task.traverse(allPublishModules)(m => m.publishLocalCached)()
val raw = rawAssembly().path
os.copy(raw, Task.dest / raw.last)
PathRef(Task.dest / raw.last)
}

def prependShellScript = Task {
val (millArgs, otherArgs) =
forkArgs().partition(arg => arg.startsWith("-DMILL") && !arg.startsWith("-DMILL_VERSION"))
// Pass Mill options via file, due to small max args limit in Windows
val vmOptionsFile = Task.dest / "mill.properties"
val millOptionsContent =
millArgs.map(_.drop(2).replace("\\", "/")).mkString(
"\r\n"
) // drop -D prefix, replace \ with /
os.write(vmOptionsFile, millOptionsContent)
val jvmArgs = otherArgs ++ List(s"-DMILL_OPTIONS_PATH=$vmOptionsFile")
val classpath = runClasspath().map(_.path.toString)
launcherScript(
jvmArgs,
jvmArgs,
classpath,
Agg(pathingJar().path.toString) // TODO not working yet on Windows! see #791
)
}

def pathingJar = Task {
// see http://todayguesswhat.blogspot.com/2011/03/jar-manifestmf-class-path-referencing.html
// for more detailed explanation
val isWin = scala.util.Properties.isWin
val classpath = runClasspath().map { pathRef =>
val path =
if (isWin) "/" + pathRef.path.toString.replace("\\", "/")
else pathRef.path.toString
if (path.endsWith(".jar")) path
else path + "/"
}.mkString(" ")
val manifestEntries = Map[String, String](
java.util.jar.Attributes.Name.MANIFEST_VERSION.toString -> "1.0",
"Created-By" -> "Scala mill",
"Class-Path" -> classpath
)
Jvm.createJar(Agg(), JarManifest(manifestEntries))
}

def run(args: Task[Args] = Task.Anon(Args())) = Task.Command(exclusive = true) {
args().value match {
case Nil => mill.api.Result.Failure("Need to pass in cwd as first argument to dev.run")
case wd0 +: rest =>
val wd = os.Path(wd0, Task.workspace)
os.makeDir.all(wd)
try {
Jvm.runSubprocess(
Seq(launcher().path.toString) ++ rest,
forkEnv(),
workingDir = wd
)
mill.api.Result.Success(())
} catch {
case e: Throwable =>
mill.api.Result.Failure(s"dev.run failed with an exception. ${e.getMessage()}")
}
}
}
}

/**
* Build and install Mill locally.
*
* @param binFile The location where the Mill binary should be installed
* @param ivyRepo The local Ivy repository where Mill modules should be published to
*/
def installLocal(binFile: String = DefaultLocalMillReleasePath, ivyRepo: String = null) =
Task.Command {
PathRef(installLocalTask(Task.Anon(binFile), ivyRepo)())
}

def installLocalCache() = Task.Command {
val path = installLocalTask(
Task.Anon((os.home / ".cache" / "mill" / "download" / millVersion()).toString())
)()
Task.log.outputStream.println(path.toString())
PathRef(path)
}

def installLocalTask(binFile: Task[String], ivyRepo: String = null): Task[os.Path] = Task.Anon {
val millBin = dist.assembly()
val targetFile = os.Path(binFile(), Task.workspace)
if (os.exists(targetFile))
Task.log.info(s"Overwriting existing local Mill binary at ${targetFile}")
os.copy.over(millBin.path, targetFile, createFolders = true)
Task.log.info(s"Published ${dist.allPublishModules.size} modules and installed ${targetFile}")
targetFile
}

def millBootstrap = Task.Sources(Task.workspace / "mill")

def bootstrapLauncher = Task {
val outputPath = Task.dest / "mill"
val millBootstrapGrepPrefix = "(\n *DEFAULT_MILL_VERSION=)"
val millDownloadUrlPrefix = "(\n *MILL_DOWNLOAD_URL=)"

os.write(
outputPath,
os.read(millBootstrap().head.path)
.replaceAll(
millBootstrapGrepPrefix + "[^\\n]+",
"$1" + millVersion()
)
)
os.perms.set(outputPath, "rwxrwxrwx")
PathRef(outputPath)
}

def examplePathsWithArtifactName:Task[Seq[(os.Path,String)]] = Task.Anon{
for {
exampleMod <- build.example.exampleModules
path = exampleMod.millSourcePath
} yield {
val example = path.subRelativeTo(Task.workspace)
val artifactName = millVersion() + "-" + example.segments.mkString("-")
(path, artifactName)
}
}


def exampleZips: T[Seq[PathRef]] = Task {
examplePathsWithArtifactName().map{ case (examplePath, exampleStr) =>
os.copy(examplePath, Task.dest / exampleStr, createFolders = true)
os.write(Task.dest / exampleStr / ".mill-version", millLastTag())
os.copy(bootstrapLauncher().path, Task.dest / exampleStr / "mill")
val zip = Task.dest / s"$exampleStr.zip"
os.proc("zip", "-r", zip, exampleStr).call(cwd = Task.dest)
PathRef(zip)
}
}

def uploadToGithub(authKey: String) = Task.Command {
val vcsState = VcsVersion.vcsState()
val label = vcsState.format()
if (label != millVersion()) sys.error("Modified mill version detected, aborting upload")
val releaseTag = vcsState.lastTag.getOrElse(sys.error(
"Incomplete git history. No tag found.\nIf on CI, make sure your git checkout job includes enough history."
))

if (releaseTag == label) {
// TODO: check if the tag already exists (e.g. because we created it manually) and do not fail
requests.post(
s"https://api.github.com/repos/${Settings.githubOrg}/${Settings.githubRepo}/releases",
data = ujson.Obj("tag_name" -> releaseTag, "name" -> releaseTag),
headers = Seq("Authorization" -> ("token " + authKey))
)
}

val examples = exampleZips().map(z => (z.path, z.path.last))

val zips = examples ++ Seq(
(dist.assembly().path, label + "-assembly"),
(bootstrapLauncher().path, label)
)

for ((zip, name) <- zips) {
upload.apply(
zip,
releaseTag,
name,
authKey,
Settings.githubOrg,
Settings.githubRepo
)
}
}

private def resolveTasks[T](taskNames: String*): Seq[NamedTask[T]] = {
mill.resolve.Resolve.Tasks.resolve(
Expand Down
6 changes: 0 additions & 6 deletions ci/release-maven.sh
View file

This file was deleted.

2 changes: 1 addition & 1 deletion ci/test-mill-bootstrap.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ git stash -u
git stash -a

# Build Mill
./mill -i installLocal
./mill -i dist.installLocal

# Clean up
git stash -a -m "preserve mill-release" -- target/mill-release
Expand Down
Loading

0 comments on commit 94d28d2

Please sign in to comment.