From 93761e431ded967121d304eab8eb98dba2623be1 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 10 Aug 2024 18:56:35 -0700 Subject: [PATCH 1/5] . --- .../test/src/MultiLevelBuildTests.scala | 3 +- .../src/mill/main/client/MillClientMain.java | 13 ++-- .../client/src/mill/main/client/OutFiles.java | 50 +++++++++++++++ .../src/mill/main/client/ServerFiles.java | 63 +++++++++++++++++++ .../src/mill/main/client/lock/Locks.java | 8 ++- main/eval/src/mill/eval/EvaluatorCore.scala | 5 +- .../src/mill/runner/MillBuildBootstrap.scala | 3 +- runner/src/mill/runner/MillServerMain.scala | 36 ++++++++--- 8 files changed, 161 insertions(+), 20 deletions(-) create mode 100644 main/client/src/mill/main/client/OutFiles.java create mode 100644 main/client/src/mill/main/client/ServerFiles.java diff --git a/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala b/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala index 6a0de6a8754..6efbabbaf3a 100644 --- a/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala +++ b/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala @@ -1,5 +1,6 @@ package mill.integration +import mill.main.client.OutFiles import mill.runner.RunnerState import utest._ @@ -47,7 +48,7 @@ object MultiLevelBuildTests extends IntegrationTestSuite { def loadFrames(n: Int) = { for (depth <- Range(0, n)) yield { - val path = wsRoot / "out" / Seq.fill(depth)("mill-build") / "mill-runner-state.json" + val path = wsRoot / "out" / Seq.fill(depth)(OutFiles.millBuild()) / OutFiles.millRunnerState() if (os.exists(path)) upickle.default.read[RunnerState.Frame.Logged](os.read(path)) -> path else RunnerState.Frame.Logged(Map(), Seq(), Seq(), Map(), None, Seq(), 0) -> path } diff --git a/main/client/src/mill/main/client/MillClientMain.java b/main/client/src/mill/main/client/MillClientMain.java index 5687c8b251c..b751a528fc3 100644 --- a/main/client/src/mill/main/client/MillClientMain.java +++ b/main/client/src/mill/main/client/MillClientMain.java @@ -1,5 +1,6 @@ package mill.main.client; +import com.sun.security.ntlm.Server; import mill.main.client.lock.Locked; import mill.main.client.lock.Locks; import org.newsclub.net.unix.AFUNIXSocket; @@ -112,7 +113,7 @@ public static int main0(String[] args) throws Exception { int index = 0; while (index < serverProcessesLimit) { index++; - final String lockBase = "out/mill-worker-" + versionAndJvmHomeEncoding + "-" + index; + final String lockBase = "out/" + OutFiles.millWorker() + versionAndJvmHomeEncoding + "-" + index; java.io.File lockBaseFile = new java.io.File(lockBase); final File stdout = new java.io.File(lockBaseFile, "stdout"); final File stderr = new java.io.File(lockBaseFile, "stderr"); @@ -185,7 +186,7 @@ public static int run( String[] args, Map env) throws Exception { - try (FileOutputStream f = new FileOutputStream(lockBase + "/run")) { + try (FileOutputStream f = new FileOutputStream(ServerFiles.runArgs(lockBase))) { f.write(System.console() != null ? 1 : 0); Util.writeString(f, BuildInfo.millVersion); Util.writeArgs(args, f); @@ -199,7 +200,7 @@ public static int run( } while (locks.processLock.probe()) Thread.sleep(3); - String socketName = lockBase + "/mill-" + Util.md5hex(new File(lockBase).getCanonicalPath()) + "-io"; + String socketName = ServerFiles.pipe(lockBase); AFUNIXSocketAddress addr = AFUNIXSocketAddress.of(new File(socketName)); long retryStart = System.currentTimeMillis(); @@ -241,7 +242,7 @@ public static int run( outPump.getLastData().waitForSilence(50); try { - return Integer.parseInt(Files.readAllLines(Paths.get(lockBase + "/exitCode")).get(0)); + return Integer.parseInt(Files.readAllLines(Paths.get(ServerFiles.exitCode(lockBase))).get(0)); } catch (Throwable e) { return ExitClientCodeCannotReadFromExitCodeFile(); } finally { @@ -252,8 +253,8 @@ public static int run( // 5 processes max private static int getServerProcessesLimit(String jvmHomeEncoding) { File outFolder = new File("out"); - String[] totalProcesses = outFolder.list((dir, name) -> name.startsWith("mill-worker-")); - String[] thisJdkProcesses = outFolder.list((dir, name) -> name.startsWith("mill-worker-" + jvmHomeEncoding)); + String[] totalProcesses = outFolder.list((dir, name) -> name.startsWith(OutFiles.millWorker())); + String[] thisJdkProcesses = outFolder.list((dir, name) -> name.startsWith(OutFiles.millWorker() + jvmHomeEncoding)); int processLimit = 5; if (totalProcesses != null) { diff --git a/main/client/src/mill/main/client/OutFiles.java b/main/client/src/mill/main/client/OutFiles.java new file mode 100644 index 00000000000..1233aba84dc --- /dev/null +++ b/main/client/src/mill/main/client/OutFiles.java @@ -0,0 +1,50 @@ +package mill.main.client; + +/** + * Central place containing all the files that live inside the `out/` folder + * and documentation about what they do + */ +public class OutFiles { + /** + * Path of the Mill "meta-build", used to compile the `build.sc` file so we can + * run the primary Mill build. Can be nested for multiple stages of bootstrapping + */ + public static String millBuild(){ + return "mill-build"; + } + + /** + * A parallel performance and timing profile generated for every Mill execution. + * Can be loaded into the Chrome browser chrome://tracing page to visualize where + * time in a build is being spent + */ + public static String millChromeProfile(){ + return "mill-chrome-profile.json"; + } + + /** + * A sequential profile containing rich information about the tasks that were run + * as part of a build: name, duration, cached, dependencies, etc.. Useful to help + * understand what tasks are taking time in a build run and why those tasks are + * being executed + */ + public static String millProfile(){ + return "mill-profile.json"; + } + + /** + * Long lived metadata about the Mill bootstrap process that persists between runs: + * workers, watched files, classpaths, etc. + */ + public static String millRunnerState(){ + return "mill-runner-state.json"; + } + + /** + * Subfolder of `out/` that contains the machinery necessary for a single Mill background + * server: metadata files, pipes, logs, etc. + */ + public static String millWorker(){ + return "mill-worker-"; + } +} diff --git a/main/client/src/mill/main/client/ServerFiles.java b/main/client/src/mill/main/client/ServerFiles.java new file mode 100644 index 00000000000..45d8b3c369c --- /dev/null +++ b/main/client/src/mill/main/client/ServerFiles.java @@ -0,0 +1,63 @@ +package mill.main.client; + +/** + * Central place containing all the files that live inside the `out/mill-worker-*` folder + * and documentation about what they do + */ +public class ServerFiles { + + /** + * Lock file used to ensure a single server is running in a particular + * mill-worker folder. + */ + public static String processLock(String base){ + return base + "/processLock"; + } + + public static String clientLock(String base){ + return base + "/clientLock"; + } + + public static String serverLock(String base){ + return base + "/serverLock"; + } + + + /** + * The pipe by which the client snd server exchange IO + * + * Use uniquely-named pipes based on the fully qualified path of the project folder + * because on Windows the un-qualified name of the pipe must be globally unique + * across the whole filesystem + */ + public static String pipe(String base) { + try { + return base + "/mill-" + Util.md5hex(new java.io.File(base).getCanonicalPath()) + "-io"; + }catch (Exception e){ + throw new RuntimeException(e); + } + } + + /** + * Log file containing server housekeeping information + */ + public static String serverLog(String base){ + return base + "/server.log"; + } + + /** + * File that the client writes to pass the arguments, environment variables, + * and other necessary metadata to the Mill server to kick off a run + */ + public static String runArgs(String base){ + return base + "/runArgs"; + } + + /** + * File the server writes to pass the exit code of a completed run back to the + * client + */ + public static String exitCode(String base){ + return base + "/exitCode"; + } +} diff --git a/main/client/src/mill/main/client/lock/Locks.java b/main/client/src/mill/main/client/lock/Locks.java index 6d74812351f..a6483d719bc 100644 --- a/main/client/src/mill/main/client/lock/Locks.java +++ b/main/client/src/mill/main/client/lock/Locks.java @@ -1,5 +1,7 @@ package mill.main.client.lock; +import mill.main.client.ServerFiles; + public class Locks implements AutoCloseable { public Lock processLock; @@ -8,9 +10,9 @@ public class Locks implements AutoCloseable { public static Locks files(String lockBase) throws Exception { return new Locks(){{ - processLock = new FileLock(lockBase + "/pid"); - serverLock = new FileLock(lockBase + "/serverLock"); - clientLock = new FileLock(lockBase + "/clientLock"); + processLock = new FileLock(ServerFiles.processLock(lockBase)); + serverLock = new FileLock(ServerFiles.serverLock(lockBase)); + clientLock = new FileLock(ServerFiles.clientLock(lockBase)); }}; } diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index e33143749ff..26c5ace2d94 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -5,6 +5,7 @@ import mill.api.Strict.Agg import mill.api._ import mill.define._ import mill.eval.Evaluator.TaskResult +import mill.main.client.OutFiles import mill.util._ import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} @@ -69,8 +70,8 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { contextLoggerMsg0: Int => String ): Evaluator.Results = { os.makeDir.all(outPath) - val chromeProfileLogger = new ChromeProfileLogger(outPath / "mill-chrome-profile.json") - val profileLogger = new ProfileLogger(outPath / "mill-profile.json") + val chromeProfileLogger = new ChromeProfileLogger(outPath / OutFiles.millChromeProfile()) + val profileLogger = new ProfileLogger(outPath / OutFiles.millProfile()) val threadNumberer = new ThreadNumberer() val (sortedGroups, transitive) = Plan.plan(goals) val interGroupDeps = findInterGroupDeps(sortedGroups) diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index bbf636daf30..c9d2a993aaf 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -7,6 +7,7 @@ import mill.eval.Evaluator import mill.main.RunScript import mill.resolve.SelectMode import mill.define.{BaseModule, Discover, Segments} +import mill.main.client.OutFiles import java.net.URLClassLoader @@ -50,7 +51,7 @@ class MillBuildBootstrap( for ((frame, depth) <- runnerState.frames.zipWithIndex) { os.write.over( - recOut(projectRoot, depth) / "mill-runner-state.json", + recOut(projectRoot, depth) / OutFiles.millRunnerState(), upickle.default.write(frame.loggedData, indent = 4), createFolders = true ) diff --git a/runner/src/mill/runner/MillServerMain.scala b/runner/src/mill/runner/MillServerMain.scala index 0a8fd606293..8fcc8f02c2b 100644 --- a/runner/src/mill/runner/MillServerMain.scala +++ b/runner/src/mill/runner/MillServerMain.scala @@ -13,6 +13,7 @@ import mill.api.internal import mill.main.client.lock.{Lock, Locks} import mill.api.SystemStreams +import java.nio.file.StandardOpenOption import scala.util.Try @internal @@ -94,17 +95,32 @@ class Server[T]( locks: Locks ) { - val originalStdout = System.out + val serverLog0 = new OutputStreamWriter( + os.write.outputStream( + os.Path(ServerFiles.serverLog(lockBase)), + openOptions = Seq(StandardOpenOption.APPEND) + ) + ) + + def serverLog(s: String) = { + serverLog0.write(s + "\n") + serverLog0.flush() + } + def run(): Unit = { val initialSystemProperties = sys.props.toMap Server.tryLockBlock(locks.processLock) { + serverLog("taken process lock") var running = true while (running) { Server.lockBlock(locks.serverLock) { - val socketName = - lockBase + "/mill-" + Util.md5hex(new File(lockBase).getCanonicalPath()) + "-io" + serverLog("taken server lock") + + val socketName = ServerFiles.pipe(lockBase) + new File(socketName).delete() val addr = AFUNIXSocketAddress.of(new File(socketName)) + serverLog("bound socket") val serverSocket = AFUNIXServerSocket.bindOn(addr) val socketClose = () => serverSocket.close() @@ -116,12 +132,17 @@ class Server[T]( ) sockOpt match { - case None => running = false + case None => + serverLog("socket none") + running = false case Some(sock) => try { + serverLog("handling run") handleRun(sock, initialSystemProperties) serverSocket.close() - } catch { case e: Throwable => e.printStackTrace(originalStdout) } + } catch { case e: Throwable => + serverLog(e.toString + "\n" + e.getStackTrace.mkString("\n")) + } } } // Make sure you give an opportunity for the client to probe the lock @@ -152,7 +173,7 @@ class Server[T]( // that relies on that method val proxiedSocketInput = proxyInputStreamThroughPumper(clientSocket.getInputStream) - val argStream = new FileInputStream(lockBase + "/run") + val argStream = new FileInputStream(ServerFiles.runArgs(lockBase)) val interactive = argStream.read() != 0 val clientMillVersion = Util.readString(argStream) val serverMillVersion = BuildInfo.millVersion @@ -161,7 +182,7 @@ class Server[T]( s"Mill version changed ($serverMillVersion -> $clientMillVersion), re-starting server" ) java.nio.file.Files.write( - java.nio.file.Paths.get(lockBase + "/exitCode"), + java.nio.file.Paths.get(ServerFiles.exitCode(lockBase)), s"${MillClientMain.ExitServerCodeWhenVersionMismatch()}".getBytes() ) System.exit(MillClientMain.ExitServerCodeWhenVersionMismatch()) @@ -226,6 +247,7 @@ class Server[T]( } object Server { + val uniqueServerId = scala.util.Random.nextLong() def lockBlock[T](lock: Lock)(t: => T): T = { val l = lock.lock() From 0499ad047c90622104462349fc66d974449cb57a Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 10 Aug 2024 18:59:57 -0700 Subject: [PATCH 2/5] . --- main/client/src/mill/main/client/MillClientMain.java | 1 - 1 file changed, 1 deletion(-) diff --git a/main/client/src/mill/main/client/MillClientMain.java b/main/client/src/mill/main/client/MillClientMain.java index b751a528fc3..90035e8fff6 100644 --- a/main/client/src/mill/main/client/MillClientMain.java +++ b/main/client/src/mill/main/client/MillClientMain.java @@ -1,6 +1,5 @@ package mill.main.client; -import com.sun.security.ntlm.Server; import mill.main.client.lock.Locked; import mill.main.client.lock.Locks; import org.newsclub.net.unix.AFUNIXSocket; From a3750b9940b5519f8b5f1eb8db747554d3d47950 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 10 Aug 2024 19:53:16 -0700 Subject: [PATCH 3/5] . --- .../test/src/MultiLevelBuildTests.scala | 3 ++- runner/src/mill/runner/MillServerMain.scala | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala b/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala index 6efbabbaf3a..a5dd3a1f40f 100644 --- a/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala +++ b/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala @@ -48,7 +48,8 @@ object MultiLevelBuildTests extends IntegrationTestSuite { def loadFrames(n: Int) = { for (depth <- Range(0, n)) yield { - val path = wsRoot / "out" / Seq.fill(depth)(OutFiles.millBuild()) / OutFiles.millRunnerState() + val path = + wsRoot / "out" / Seq.fill(depth)(OutFiles.millBuild()) / OutFiles.millRunnerState() if (os.exists(path)) upickle.default.read[RunnerState.Frame.Logged](os.read(path)) -> path else RunnerState.Frame.Logged(Map(), Seq(), Seq(), Map(), None, Seq(), 0) -> path } diff --git a/runner/src/mill/runner/MillServerMain.scala b/runner/src/mill/runner/MillServerMain.scala index 8fcc8f02c2b..6d71ad0a4d8 100644 --- a/runner/src/mill/runner/MillServerMain.scala +++ b/runner/src/mill/runner/MillServerMain.scala @@ -102,7 +102,7 @@ class Server[T]( ) ) - def serverLog(s: String) = { + def serverLog(s: String): Unit = { serverLog0.write(s + "\n") serverLog0.flush() } @@ -140,8 +140,9 @@ class Server[T]( serverLog("handling run") handleRun(sock, initialSystemProperties) serverSocket.close() - } catch { case e: Throwable => - serverLog(e.toString + "\n" + e.getStackTrace.mkString("\n")) + } catch { + case e: Throwable => + serverLog(e.toString + "\n" + e.getStackTrace.mkString("\n")) } } } @@ -247,7 +248,7 @@ class Server[T]( } object Server { - val uniqueServerId = scala.util.Random.nextLong() + val uniqueServerId: Long = scala.util.Random.nextLong() def lockBlock[T](lock: Lock)(t: => T): T = { val l = lock.lock() From 69e6924221dd8f06748de971c73ccf648774310f Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 10 Aug 2024 20:14:17 -0700 Subject: [PATCH 4/5] . --- runner/src/mill/runner/MillServerMain.scala | 27 +++------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/runner/src/mill/runner/MillServerMain.scala b/runner/src/mill/runner/MillServerMain.scala index 6d71ad0a4d8..dd6fa1acc30 100644 --- a/runner/src/mill/runner/MillServerMain.scala +++ b/runner/src/mill/runner/MillServerMain.scala @@ -13,7 +13,6 @@ import mill.api.internal import mill.main.client.lock.{Lock, Locks} import mill.api.SystemStreams -import java.nio.file.StandardOpenOption import scala.util.Try @internal @@ -95,32 +94,18 @@ class Server[T]( locks: Locks ) { - val serverLog0 = new OutputStreamWriter( - os.write.outputStream( - os.Path(ServerFiles.serverLog(lockBase)), - openOptions = Seq(StandardOpenOption.APPEND) - ) - ) - - def serverLog(s: String): Unit = { - serverLog0.write(s + "\n") - serverLog0.flush() - } - + val originalStdout = System.out def run(): Unit = { val initialSystemProperties = sys.props.toMap Server.tryLockBlock(locks.processLock) { - serverLog("taken process lock") var running = true while (running) { Server.lockBlock(locks.serverLock) { - serverLog("taken server lock") val socketName = ServerFiles.pipe(lockBase) new File(socketName).delete() val addr = AFUNIXSocketAddress.of(new File(socketName)) - serverLog("bound socket") val serverSocket = AFUNIXServerSocket.bindOn(addr) val socketClose = () => serverSocket.close() @@ -132,18 +117,12 @@ class Server[T]( ) sockOpt match { - case None => - serverLog("socket none") - running = false + case None => running = false case Some(sock) => try { - serverLog("handling run") handleRun(sock, initialSystemProperties) serverSocket.close() - } catch { - case e: Throwable => - serverLog(e.toString + "\n" + e.getStackTrace.mkString("\n")) - } + } catch { case e: Throwable => e.printStackTrace(originalStdout) } } } // Make sure you give an opportunity for the client to probe the lock From 470fbcd88f65e57df0b050a069cce5a1b34fd763 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 10 Aug 2024 20:35:04 -0700 Subject: [PATCH 5/5] . --- runner/src/mill/runner/MillServerMain.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/runner/src/mill/runner/MillServerMain.scala b/runner/src/mill/runner/MillServerMain.scala index dd6fa1acc30..bdc9d818102 100644 --- a/runner/src/mill/runner/MillServerMain.scala +++ b/runner/src/mill/runner/MillServerMain.scala @@ -227,7 +227,6 @@ class Server[T]( } object Server { - val uniqueServerId: Long = scala.util.Random.nextLong() def lockBlock[T](lock: Lock)(t: => T): T = { val l = lock.lock()