Skip to content

Commit

Permalink
Parallelize windows CI (#3467)
Browse files Browse the repository at this point in the history
* Remove `-j1` parallelism limit on windows, enable parallelism for
windows tests by changing `.test` to `.testCached`

* Clean up coursier retry logic and add more cases

* Two of our example tests were conflicting on port `8088`, moved one of
them to `8089`
lihaoyi authored Sep 7, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 0637b86 commit 72267c2
Showing 6 changed files with 41 additions and 41 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/actions.yml
Original file line number Diff line number Diff line change
@@ -133,15 +133,15 @@ jobs:
# just run a subset of examples/ on Windows, because for some reason running
# the whole suite can take hours on windows v.s. half an hour on linux
- java-version: 11
millargs: '"{main,scalalib,bsp}.__.test"'
millargs: '"{main,scalalib,bsp}.__.testCached"'
- java-version: 11
millargs: '"example.scalalib.{basic,web}.__.fork.test"'
millargs: '"example.scalalib.{basic,web}.__.fork.testCached"'
- java-version: 17
millargs: "'integration.{feature,failure}[_].fork.test'"
millargs: "'integration.{feature,failure}[_].fork.testCached'"
- java-version: 11
millargs: "'integration.invalidation[_].server.test'"
millargs: "'integration.invalidation[_].server.testCached'"
- java-version: 11
millargs: "contrib.__.test"
millargs: "contrib.__.testCached"

uses: ./.github/workflows/run-mill-action.yml
with:
2 changes: 1 addition & 1 deletion .github/workflows/run-mill-action.yml
Original file line number Diff line number Diff line change
@@ -75,7 +75,7 @@ jobs:
if: inputs.millargs != '' && !startsWith(inputs.os, 'windows')

- name: Run Mill (on Windows) '${{ inputs.millargs }}'
run: cmd /C %GITHUB_WORKSPACE%\ci\mill.bat -ij1 __.resolvedIvyDeps; cmd /C %GITHUB_WORKSPACE%\ci\mill.bat -i -j1 -k ${{ inputs.millargs }}
run: cmd /C %GITHUB_WORKSPACE%\ci\mill.bat -i -k ${{ inputs.millargs }}
if: inputs.millargs != '' && startsWith(inputs.os, 'windows')

- name: Run Mill (on Windows) Worker Cleanup
2 changes: 1 addition & 1 deletion example/javalib/web/5-todo-micronaut/build.mill
Original file line number Diff line number Diff line change
@@ -92,7 +92,7 @@ trait MicronautModule extends MavenModule{

> mill runBackground

> curl http://localhost:8088
> curl http://localhost:8089
...<h1>todos</h1>...

> mill clean runBackground
Original file line number Diff line number Diff line change
@@ -14,4 +14,4 @@ micronaut.router.static-resources.learnjson.paths=classpath\:public
micronaut.router.static-resources.learnjson.mapping=/*.json
# Views
micronaut.views.dir=templates
micronaut.server.port=8088
micronaut.server.port=8089
3 changes: 3 additions & 0 deletions main/client/src/mill/main/client/ProxyStream.java
Original file line number Diff line number Diff line change
@@ -141,6 +141,9 @@ public void run() {
flush();
}
}
} catch (org.newsclub.net.unix.ConnectionResetSocketException e) {
// This happens when you run mill shutdown and the server exits gracefully
break;
} catch (IOException e) {
e.printStackTrace();
System.exit(1);
63 changes: 30 additions & 33 deletions main/util/src/mill/util/CoursierSupport.scala
Original file line number Diff line number Diff line change
@@ -7,7 +7,6 @@ import coursier.{Dependency, Repository, Resolution}
import mill.api.{Ctx, PathRef, Result}
import mill.api.Loose.Agg
import java.io.File
import java.nio.file.NoSuchFileException
import scala.annotation.tailrec
import scala.collection.mutable
import scala.util.{Failure, Success, Try}
@@ -18,6 +17,16 @@ trait CoursierSupport {
private val CoursierRetryCount = 5
private val CoursierRetryWait = 100

private def retryableCoursierError(s: String) = s match {
case s"${_}concurrent download${_}" => true
case s"${_}checksum not found${_}" => true
case s"${_}download error${_}" => true
case s"${_}The process cannot access the file because it is being used by another process${_}" =>
true
case s"${_}->${_}__sha1.computed" => true
case _ => false
}

/**
* Somewhat generic way to retry some action and a Workaround for https://github.com/com-lihaoyi/mill/issues/1028
*
@@ -33,45 +42,30 @@ trait CoursierSupport {
@tailrec
private def retry[T](
retryCount: Int = CoursierRetryCount,
ctx: Option[Ctx.Log],
debug: String => Unit,
errorMsgExtractor: T => Seq[String]
)(f: () => T): T = {
val tried = Try(f())
tried match {
case Failure(e)
if retryCount > 0
&& e.getMessage.contains("__sha1.computed")
&& (e.isInstanceOf[NoSuchFileException] ||
e.isInstanceOf[java.nio.file.AccessDeniedException] ||
e.isInstanceOf[java.io.FileNotFoundException]) =>
case Failure(e) if retryCount > 0 && retryableCoursierError(e.getMessage) =>
// this one is not detected by coursier itself, so we try-catch handle it
// I assume, this happens when another coursier thread already moved or rename dthe temporary file
ctx.foreach(_.log.debug(
s"Detected a concurrent download issue in coursier. Attempting a retry (${retryCount} left)"
))
debug(s"Attempting to retry coursier failure (${retryCount} left): ${e.getMessage}")
Thread.sleep(CoursierRetryWait)
retry(retryCount - 1, ctx, errorMsgExtractor)(f)
retry(retryCount - 1, debug, errorMsgExtractor)(f)

case Success(res) if retryCount > 0 =>
val errors = errorMsgExtractor(res)
if (errors.exists(e => e.contains("concurrent download"))) {
ctx.foreach(_.log.debug(
s"Detected a concurrent download issue in coursier. Attempting a retry (${retryCount} left)"
))
Thread.sleep(CoursierRetryWait)
retry(retryCount - 1, ctx, errorMsgExtractor)(f)
} else if (errors.exists(e => e.contains("checksum not found"))) {
ctx.foreach(_.log.debug(
s"Detected a checksum download issue in coursier. Attempting a retry (${retryCount} left)"
))
Thread.sleep(CoursierRetryWait)
retry(retryCount - 1, ctx, errorMsgExtractor)(f)
} else if (errors.exists(e => e.contains("download error"))) {
ctx.foreach(_.log.debug(
s"Detected a download error by coursier. Attempting a retry (${retryCount} left)"
))
Thread.sleep(CoursierRetryWait)
retry(retryCount - 1, ctx, errorMsgExtractor)(f)
} else res
errors.filter(retryableCoursierError) match {
case Nil => res
case retryable =>
for (r <- retryable) {
debug(s"Attempting to retry coursier failure (${retryCount} left): $r")
}
Thread.sleep(CoursierRetryWait)
retry(retryCount - 1, debug, errorMsgExtractor)(f)
}

case r => r.get
}
}
@@ -188,7 +182,7 @@ trait CoursierSupport {
)

val (errors, successes) = retry(
ctx = ctx,
debug = ctx.map(c => c.log.debug(_)).getOrElse(_ => ()),
errorMsgExtractor = (res: (Seq[ArtifactError], Seq[File])) => res._1.map(_.describe)
) {
() => load(sourceOrJar)
@@ -259,7 +253,10 @@ trait CoursierSupport {
import scala.concurrent.ExecutionContext.Implicits.global

val resolution =
retry(ctx = ctx, errorMsgExtractor = (r: Resolution) => r.errors.flatMap(_._2)) {
retry(
debug = ctx.map(c => c.log.debug(_)).getOrElse(_ => ()),
errorMsgExtractor = (r: Resolution) => r.errors.flatMap(_._2)
) {
() => start.process.run(fetch).unsafeRun()
}

0 comments on commit 72267c2

Please sign in to comment.