From 25cdd4420bf8eaa06d07e84ff7f870d52a311f00 Mon Sep 17 00:00:00 2001 From: Matthew de Detrich Date: Thu, 28 Sep 2023 14:49:44 +0400 Subject: [PATCH] Make clean respect githubWorkflowOSes --- .github/workflows/clean.yml | 3 +- README.md | 2 +- .../scala/sbtghactions/GenerativePlugin.scala | 78 +++++++++-- .../check-and-regenerate/expected-clean.yml | 3 +- .../.github/workflows/ci.yml | 128 ++++++++++++++++++ .../.github/workflows}/clean.yml | 5 +- .../githubworkflowoses-clean/build.sbt | 6 + .../project/plugins.sbt | 5 + .../githubworkflowoses-clean/test | 1 + .../.github/workflows/clean.yml | 3 +- .../.github/workflows/clean.yml | 3 +- .../expected-clean.yml | 3 +- 12 files changed, 223 insertions(+), 17 deletions(-) create mode 100644 src/sbt-test/sbtghactions/githubworkflowoses-clean/.github/workflows/ci.yml rename src/{main/resources => sbt-test/sbtghactions/githubworkflowoses-clean/.github/workflows}/clean.yml (96%) create mode 100644 src/sbt-test/sbtghactions/githubworkflowoses-clean/build.sbt create mode 100644 src/sbt-test/sbtghactions/githubworkflowoses-clean/project/plugins.sbt create mode 100644 src/sbt-test/sbtghactions/githubworkflowoses-clean/test diff --git a/.github/workflows/clean.yml b/.github/workflows/clean.yml index 547aaa4..bfc865d 100644 --- a/.github/workflows/clean.yml +++ b/.github/workflows/clean.yml @@ -17,6 +17,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Delete artifacts + shell: bash {0} run: | # Customize those three lines with your repository and credentials: REPO=${GITHUB_API_URL}/repos/${{ github.repository }} @@ -25,7 +26,7 @@ jobs: ghapi() { curl --silent --location --user _:$GITHUB_TOKEN "$@"; } # A temporary file which receives HTTP response headers. - TMPFILE=/tmp/tmp.$$ + TMPFILE=$(mktemp) # An associative array, key: artifact name, value: number of artifacts of that name. declare -A ARTCOUNT diff --git a/README.md b/README.md index 10ad0f8..3042dca 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ Any and all settings which affect the behavior of the generative plugin should b - `githubWorkflowBuild` : `Seq[WorkflowStep]` – The steps which invoke sbt (or whatever else you want) to build and test your project. This defaults to just `[sbt test]`, but can be overridden to anything. For example, sbt plugin projects probably want to redefine this to be `Seq(WorkflowStep.Sbt(List("test", "scripted")))`, which would run the `test` and `scripted` sbt tasks, in order. Note that all uses of `WorkflowStep.Sbt` are compiled using the configured `githubWorkflowSbtCommand` invocation, and prepended with `githubWorkflowBuildSbtStepPreamble` (default: `[++{matrix.scala}]`). - `githubWorkflowJavaVersions` : `Seq[JavaSpec]` – A list of Java versions to be used for the build job. The publish job will use the *first* of these versions. Defaults to `JavaSpec.temurin("8")`). - `githubWorkflowScalaVersions` : `Seq[String]` – A list of Scala versions which will be used to `build` your project. Defaults to `crossScalaVersions` in `build`, and simply `scalaVersion` in `publish`. -- `githubWorkflowOSes` : `Seq[String]` – A list of operating systems, which will be ultimately passed to [the `runs-on:` directive](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idruns-on), on which to `build` your project. Defaults to `ubuntu-latest`. Note that, regardless of the value of this setting, only `ubuntu-latest` will be used for the `publish` job. This setting only affects `build`. +- `githubWorkflowOSes` : `Seq[String]` – A list of operating systems, which will be ultimately passed to [the `runs-on:` directive](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idruns-on), on which to `build` your project. Defaults to `ubuntu-latest`. Note that, regardless of the value of this setting, only `ubuntu-latest` will be used for the `publish` job. The first value of this setting will be used for the clean job. - `githubWorkflowBuildRunsOnExtraLabels` : `Seq[String]` - A list of additional runs-on labels, which will be combined with the matrix.os from `githubWorkflowOSes` above allowing for singling out more specific runners. - `githubWorkflowBuildTimeout` : `Option[FiniteDuration]` - [The maximum duration](https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes) to let the build job run before GitHub automatically cancels it. Defaults to `None`. diff --git a/src/main/scala/sbtghactions/GenerativePlugin.scala b/src/main/scala/sbtghactions/GenerativePlugin.scala index dcc5fb1..b305c74 100644 --- a/src/main/scala/sbtghactions/GenerativePlugin.scala +++ b/src/main/scala/sbtghactions/GenerativePlugin.scala @@ -783,13 +783,73 @@ ${indent(jobs.map(compileJob(_, sbt)).mkString("\n\n"), 1)} sbt) } - private val readCleanContents = Def task { - val src = Source.fromURL(getClass.getResource("/clean.yml")) - try { - src.mkString - } finally { - src.close() - } + private def generateCleanContents(runsOnOs: String) = { + val first = + """|# This file was automatically generated by sbt-github-actions using the + |# githubWorkflowGenerate task. You should add and commit this file to + |# your git repository. It goes without saying that you shouldn't edit + |# this file by hand! Instead, if you wish to make changes, you should + |# change your sbt build configuration to revise the workflow description + |# to meet your needs, then regenerate this file. + | + |name: Clean + | + |on: push + | + |jobs: + | delete-artifacts: + | name: Delete Artifacts + |""".stripMargin + + val last = + """ + | env: + | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + | steps: + | - name: Delete artifacts + | shell: bash {0} + | run: | + | # Customize those three lines with your repository and credentials: + | REPO=${GITHUB_API_URL}/repos/${{ github.repository }} + | + | # A shortcut to call GitHub API. + | ghapi() { curl --silent --location --user _:$GITHUB_TOKEN "$@"; } + | + | # A temporary file which receives HTTP response headers. + | TMPFILE=$(mktemp) + | + | # An associative array, key: artifact name, value: number of artifacts of that name. + | declare -A ARTCOUNT + | + | # Process all artifacts on this repository, loop on returned "pages". + | URL=$REPO/actions/artifacts + | while [[ -n "$URL" ]]; do + | + | # Get current page, get response headers in a temporary file. + | JSON=$(ghapi --dump-header $TMPFILE "$URL") + | + | # Get URL of next page. Will be empty if we are at the last page. + | URL=$(grep '^Link:' "$TMPFILE" | tr ',' '\n' | grep 'rel="next"' | head -1 | sed -e 's/.*.*//') + | rm -f $TMPFILE + | + | # Number of artifacts on this page: + | COUNT=$(( $(jq <<<$JSON -r '.artifacts | length') )) + | + | # Loop on all artifacts on this page. + | for ((i=0; $i < $COUNT; i++)); do + | + | # Get name of artifact and count instances of this name. + | name=$(jq <<<$JSON -r ".artifacts[$i].name?") + | ARTCOUNT[$name]=$(( $(( ${ARTCOUNT[$name]} )) + 1)) + | + | id=$(jq <<<$JSON -r ".artifacts[$i].id?") + | size=$(( $(jq <<<$JSON -r ".artifacts[$i].size_in_bytes?") )) + | printf "Deleting '%s' #%d, %'d bytes\n" $name ${ARTCOUNT[$name]} $size + | ghapi -X DELETE $REPO/actions/artifacts/$id + | done + | done + |""".stripMargin + first + s" runs-on: $runsOnOs" ++ last } private val workflowsDirTask = Def task { @@ -829,7 +889,7 @@ ${indent(jobs.map(compileJob(_, sbt)).mkString("\n\n"), 1)} githubWorkflowGenerate := { val ciContents = generateCiContents.value val includeClean = githubWorkflowIncludeClean.value - val cleanContents = readCleanContents.value + val cleanContents = generateCleanContents(githubWorkflowOSes.value.head) val ciYml = ciYmlFile.value val cleanYml = cleanYmlFile.value @@ -843,7 +903,7 @@ ${indent(jobs.map(compileJob(_, sbt)).mkString("\n\n"), 1)} githubWorkflowCheck := { val expectedCiContents = generateCiContents.value val includeClean = githubWorkflowIncludeClean.value - val expectedCleanContents = readCleanContents.value + val expectedCleanContents = generateCleanContents(githubWorkflowOSes.value.head) val ciYml = ciYmlFile.value val cleanYml = cleanYmlFile.value diff --git a/src/sbt-test/sbtghactions/check-and-regenerate/expected-clean.yml b/src/sbt-test/sbtghactions/check-and-regenerate/expected-clean.yml index 547aaa4..bfc865d 100644 --- a/src/sbt-test/sbtghactions/check-and-regenerate/expected-clean.yml +++ b/src/sbt-test/sbtghactions/check-and-regenerate/expected-clean.yml @@ -17,6 +17,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Delete artifacts + shell: bash {0} run: | # Customize those three lines with your repository and credentials: REPO=${GITHUB_API_URL}/repos/${{ github.repository }} @@ -25,7 +26,7 @@ jobs: ghapi() { curl --silent --location --user _:$GITHUB_TOKEN "$@"; } # A temporary file which receives HTTP response headers. - TMPFILE=/tmp/tmp.$$ + TMPFILE=$(mktemp) # An associative array, key: artifact name, value: number of artifacts of that name. declare -A ARTCOUNT diff --git a/src/sbt-test/sbtghactions/githubworkflowoses-clean/.github/workflows/ci.yml b/src/sbt-test/sbtghactions/githubworkflowoses-clean/.github/workflows/ci.yml new file mode 100644 index 0000000..5cd33ca --- /dev/null +++ b/src/sbt-test/sbtghactions/githubworkflowoses-clean/.github/workflows/ci.yml @@ -0,0 +1,128 @@ +# This file was automatically generated by sbt-github-actions using the +# githubWorkflowGenerate task. You should add and commit this file to +# your git repository. It goes without saying that you shouldn't edit +# this file by hand! Instead, if you wish to make changes, you should +# change your sbt build configuration to revise the workflow description +# to meet your needs, then regenerate this file. + +name: Continuous Integration + +on: + pull_request: + branches: ['**'] + push: + branches: ['**'] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + build: + name: Build and Test + strategy: + matrix: + os: [windows-latest] + scala: [2.13.10, 2.12.17] + java: [temurin@8] + runs-on: ${{ matrix.os }} + steps: + - name: Ignore line ending differences in git + if: contains(runner.os, 'windows') + shell: bash + run: git config --global core.autocrlf false + + - name: Configure pagefile for Windows + if: contains(runner.os, 'windows') + uses: al-cheb/configure-pagefile-action@v1.3 + with: + minimum-size: 2GB + maximum-size: 8GB + + - name: Checkout current branch (full) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Java (temurin@8) + if: matrix.java == 'temurin@8' + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 8 + cache: sbt + + - name: Check that workflows are up to date + shell: bash + run: sbt '++ ${{ matrix.scala }}' githubWorkflowCheck + + - name: Build project + shell: bash + run: sbt '++ ${{ matrix.scala }}' test + + - name: Compress target directories + shell: bash + run: tar cf targets.tar target project/target + + - name: Upload target directories + uses: actions/upload-artifact@v3 + with: + name: target-${{ matrix.os }}-${{ matrix.scala }}-${{ matrix.java }} + path: targets.tar + + publish: + name: Publish Artifacts + needs: [build] + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main') + strategy: + matrix: + os: [ubuntu-latest] + scala: [2.13.10] + java: [temurin@8] + runs-on: ${{ matrix.os }} + steps: + - name: Ignore line ending differences in git + if: contains(runner.os, 'windows') + run: git config --global core.autocrlf false + + - name: Configure pagefile for Windows + if: contains(runner.os, 'windows') + uses: al-cheb/configure-pagefile-action@v1.3 + with: + minimum-size: 2GB + maximum-size: 8GB + + - name: Checkout current branch (full) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Java (temurin@8) + if: matrix.java == 'temurin@8' + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 8 + cache: sbt + + - name: Download target directories (2.13.10) + uses: actions/download-artifact@v3 + with: + name: target-${{ matrix.os }}-2.13.10-${{ matrix.java }} + + - name: Inflate target directories (2.13.10) + run: | + tar xf targets.tar + rm targets.tar + + - name: Download target directories (2.12.17) + uses: actions/download-artifact@v3 + with: + name: target-${{ matrix.os }}-2.12.17-${{ matrix.java }} + + - name: Inflate target directories (2.12.17) + run: | + tar xf targets.tar + rm targets.tar + + - name: Publish project + run: sbt +publish diff --git a/src/main/resources/clean.yml b/src/sbt-test/sbtghactions/githubworkflowoses-clean/.github/workflows/clean.yml similarity index 96% rename from src/main/resources/clean.yml rename to src/sbt-test/sbtghactions/githubworkflowoses-clean/.github/workflows/clean.yml index 547aaa4..c310b19 100644 --- a/src/main/resources/clean.yml +++ b/src/sbt-test/sbtghactions/githubworkflowoses-clean/.github/workflows/clean.yml @@ -12,11 +12,12 @@ on: push jobs: delete-artifacts: name: Delete Artifacts - runs-on: ubuntu-latest + runs-on: windows-latest env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Delete artifacts + shell: bash {0} run: | # Customize those three lines with your repository and credentials: REPO=${GITHUB_API_URL}/repos/${{ github.repository }} @@ -25,7 +26,7 @@ jobs: ghapi() { curl --silent --location --user _:$GITHUB_TOKEN "$@"; } # A temporary file which receives HTTP response headers. - TMPFILE=/tmp/tmp.$$ + TMPFILE=$(mktemp) # An associative array, key: artifact name, value: number of artifacts of that name. declare -A ARTCOUNT diff --git a/src/sbt-test/sbtghactions/githubworkflowoses-clean/build.sbt b/src/sbt-test/sbtghactions/githubworkflowoses-clean/build.sbt new file mode 100644 index 0000000..3dbf66a --- /dev/null +++ b/src/sbt-test/sbtghactions/githubworkflowoses-clean/build.sbt @@ -0,0 +1,6 @@ +organization := "com.github.sbt" +version := "0.0.1" + +ThisBuild / crossScalaVersions := Seq("2.13.10", "2.12.17") +ThisBuild / scalaVersion := crossScalaVersions.value.head +ThisBuild / githubWorkflowOSes := Seq("windows-latest") diff --git a/src/sbt-test/sbtghactions/githubworkflowoses-clean/project/plugins.sbt b/src/sbt-test/sbtghactions/githubworkflowoses-clean/project/plugins.sbt new file mode 100644 index 0000000..e1c93ca --- /dev/null +++ b/src/sbt-test/sbtghactions/githubworkflowoses-clean/project/plugins.sbt @@ -0,0 +1,5 @@ +sys.props.get("plugin.version") match { + case Some(x) => addSbtPlugin("com.github.sbt" % "sbt-github-actions" % x) + case _ => sys.error("""|The system property 'plugin.version' is not defined. + |Specify this property using the scriptedLaunchOpts -D.""".stripMargin) +} diff --git a/src/sbt-test/sbtghactions/githubworkflowoses-clean/test b/src/sbt-test/sbtghactions/githubworkflowoses-clean/test new file mode 100644 index 0000000..d3f3f6b --- /dev/null +++ b/src/sbt-test/sbtghactions/githubworkflowoses-clean/test @@ -0,0 +1 @@ +> githubWorkflowCheck diff --git a/src/sbt-test/sbtghactions/non-existent-target/.github/workflows/clean.yml b/src/sbt-test/sbtghactions/non-existent-target/.github/workflows/clean.yml index 547aaa4..bfc865d 100644 --- a/src/sbt-test/sbtghactions/non-existent-target/.github/workflows/clean.yml +++ b/src/sbt-test/sbtghactions/non-existent-target/.github/workflows/clean.yml @@ -17,6 +17,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Delete artifacts + shell: bash {0} run: | # Customize those three lines with your repository and credentials: REPO=${GITHUB_API_URL}/repos/${{ github.repository }} @@ -25,7 +26,7 @@ jobs: ghapi() { curl --silent --location --user _:$GITHUB_TOKEN "$@"; } # A temporary file which receives HTTP response headers. - TMPFILE=/tmp/tmp.$$ + TMPFILE=$(mktemp) # An associative array, key: artifact name, value: number of artifacts of that name. declare -A ARTCOUNT diff --git a/src/sbt-test/sbtghactions/sbt-native-thin-client/.github/workflows/clean.yml b/src/sbt-test/sbtghactions/sbt-native-thin-client/.github/workflows/clean.yml index 547aaa4..bfc865d 100644 --- a/src/sbt-test/sbtghactions/sbt-native-thin-client/.github/workflows/clean.yml +++ b/src/sbt-test/sbtghactions/sbt-native-thin-client/.github/workflows/clean.yml @@ -17,6 +17,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Delete artifacts + shell: bash {0} run: | # Customize those three lines with your repository and credentials: REPO=${GITHUB_API_URL}/repos/${{ github.repository }} @@ -25,7 +26,7 @@ jobs: ghapi() { curl --silent --location --user _:$GITHUB_TOKEN "$@"; } # A temporary file which receives HTTP response headers. - TMPFILE=/tmp/tmp.$$ + TMPFILE=$(mktemp) # An associative array, key: artifact name, value: number of artifacts of that name. declare -A ARTCOUNT diff --git a/src/sbt-test/sbtghactions/suppressed-scala-version/expected-clean.yml b/src/sbt-test/sbtghactions/suppressed-scala-version/expected-clean.yml index 547aaa4..bfc865d 100644 --- a/src/sbt-test/sbtghactions/suppressed-scala-version/expected-clean.yml +++ b/src/sbt-test/sbtghactions/suppressed-scala-version/expected-clean.yml @@ -17,6 +17,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Delete artifacts + shell: bash {0} run: | # Customize those three lines with your repository and credentials: REPO=${GITHUB_API_URL}/repos/${{ github.repository }} @@ -25,7 +26,7 @@ jobs: ghapi() { curl --silent --location --user _:$GITHUB_TOKEN "$@"; } # A temporary file which receives HTTP response headers. - TMPFILE=/tmp/tmp.$$ + TMPFILE=$(mktemp) # An associative array, key: artifact name, value: number of artifacts of that name. declare -A ARTCOUNT