From d064019b42ea044ecd2efdaaddbb85fdb4da0654 Mon Sep 17 00:00:00 2001 From: Georges Lebreton Date: Mon, 16 Oct 2023 19:19:11 +0100 Subject: [PATCH 01/13] Initial setup --- .github/workflows/ci.yml | 69 ++++++++++++++++++++++++++++++++ app/views/healthcheck.scala.html | 1 - app/views/layout.scala.html | 1 - build.sbt | 44 ++++++++++---------- project/plugins.sbt | 4 +- 5 files changed, 92 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..91f501e44 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,69 @@ +name: build + +on: + push: + branches: ["main"] + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + permissions: # required by aws-actions/configure-aws-credentials + id-token: write + contents: read + + steps: + # Seed the build number with last number from TeamCity. + # This env var is used by the JS, and SBT builds, and guardian/actions-riff-raff. + # Set the value early, rather than `buildNumberOffset` in guardian/actions-riff-raff, to ensure each usage has the same number. + # For some reason, it's not possible to mutate GITHUB_RUN_NUMBER, so set BUILD_NUMBER instead. + - name: Set BUILD_NUMBER environment variable + run: | + LAST_TEAMCITY_BUILD=1603 + echo "BUILD_NUMBER=$(( $GITHUB_RUN_NUMBER + $LAST_TEAMCITY_BUILD ))" >> $GITHUB_ENV + - uses: actions/checkout@v4 + - uses: actions/setup-java@v3 + with: + distribution: "corretto" + java-version: "8" + cache: "sbt" + + - name: AWS Auth + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.GU_RIFF_RAFF_ROLE_ARN }} + aws-region: eu-west-1 + + + + + - name: Install Node + uses: actions/setup-node@v3 + with: + node-version-file: '.nvmrc' + - name: Install dependencies, build & run tests + run: | + set -e + yarn install --frozen-lockfile + yarn build + + - name: Build and Test project + run: | + sbt clean compile test Debian/packageBin + + + - uses: guardian/actions-riff-raff@v2 + with: + projectName: Editorial Tools::Workflow::Workflow Frontend + buildNumber: ${{ env.BUILD_NUMBER }} + configPath: riff-raff.yaml + contentDirectories: | + workflow-frontend: + - workflow-frontend/target/workflow-frontend_0.1_all.deb + workflow-frontend-fluentbit: + - fluentbit/parsers.conf + - fluentbit/td-agent-bit.conf + workflow-notification: + - notification/target/workflow-notification.zip \ No newline at end of file diff --git a/app/views/healthcheck.scala.html b/app/views/healthcheck.scala.html index f8b439420..1c03e9302 100644 --- a/app/views/healthcheck.scala.html +++ b/app/views/healthcheck.scala.html @@ -1,3 +1,2 @@

Healthcheck OK

build-commit-id: @build.BuildInfo.gitCommitId

-

build-number: @build.BuildInfo.buildNumber

diff --git a/app/views/layout.scala.html b/app/views/layout.scala.html index a4571e3d5..a3015ef6c 100644 --- a/app/views/layout.scala.html +++ b/app/views/layout.scala.html @@ -41,7 +41,6 @@

@content - diff --git a/build.sbt b/build.sbt index b6187efc6..59f81648d 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,8 @@ import play.sbt.PlayImport.PlayKeys._ import Dependencies._ -import com.gu.riffraff.artifact.BuildInfo +import sbtbuildinfo.BuildInfo +import scala.sys.env +// import com.gu.riffraff.artifact.BuildInfo Test / parallelExecution := false @@ -8,16 +10,12 @@ val scalaVersionNumber = "2.12.15" val buildInfo = Seq( buildInfoPackage := "build", - buildInfoKeys ++= { - val buildInfo = BuildInfo(baseDirectory.value) - - Seq[BuildInfoKey]( - "gitCommitId" -> buildInfo.revision, - "buildNumber" -> buildInfo.buildIdentifier - ) - } + buildInfoKeys += + "gitCommitId" -> env.getOrElse("GITHUB_SHA", "Unknown"), ) + + val commonSettings = Seq( scalaVersion := scalaVersionNumber, ThisBuild / scalaVersion := scalaVersionNumber, @@ -63,7 +61,7 @@ lazy val commonLib = project("common-lib") val application = "workflow-frontend" lazy val root = playProject(application) - .enablePlugins(RiffRaffArtifact, JDebPackaging) + .enablePlugins(JDebPackaging, BuildInfoPlugin) .settings( libraryDependencies ++= awsDependencies @@ -84,19 +82,19 @@ lazy val root = playProject(application) s"-Dpidfile.path=/var/run/${packageName.value}/play.pid" ), debianPackageDependencies := Seq("openjdk-8-jre-headless"), - riffRaffPackageType := (Debian / packageBin).value, - riffRaffUploadArtifactBucket := Option("riffraff-artifact"), - riffRaffUploadManifestBucket := Option("riffraff-builds"), - riffRaffPackageName := s"editorial-tools:workflow:$application", - riffRaffManifestProjectName := riffRaffPackageName.value, - fileDescriptorLimit := Option("32768"), - riffRaffArtifactResources := Seq( - riffRaffPackageType.value -> s"$application/${riffRaffPackageType.value.getName}", - (notificationLambda / Universal / packageBin).value -> s"${(notificationLambda / name).value}/${(notificationLambda / Universal / packageBin).value.getName}", - baseDirectory.value / "conf" / "riff-raff.yaml" -> "riff-raff.yaml", - baseDirectory.value / "fluentbit/td-agent-bit.conf" -> "workflow-frontend-fluentbit/td-agent-bit.conf", - baseDirectory.value / "fluentbit/parsers.conf" -> "workflow-frontend-fluentbit/parsers.conf" - ), + // riffRaffPackageType := (Debian / packageBin).value, + // riffRaffUploadArtifactBucket := Option("riffraff-artifact"), + // riffRaffUploadManifestBucket := Option("riffraff-builds"), + // riffRaffPackageName := s"editorial-tools:workflow:$application", + // riffRaffManifestProjectName := riffRaffPackageName.value, + // fileDescriptorLimit := Option("32768"), + // riffRaffArtifactResources := Seq( + // riffRaffPackageType.value -> s"$application/${riffRaffPackageType.value.getName}", + // (notificationLambda / Universal / packageBin).value -> s"${(notificationLambda / name).value}/${(notificationLambda / Universal / packageBin).value.getName}", + // baseDirectory.value / "conf" / "riff-raff.yaml" -> "riff-raff.yaml", + // baseDirectory.value / "fluentbit/td-agent-bit.conf" -> "workflow-frontend-fluentbit/td-agent-bit.conf", + // baseDirectory.value / "fluentbit/parsers.conf" -> "workflow-frontend-fluentbit/parsers.conf" + // ), Universal / javaOptions ++= Seq( "-Dpidfile.path=/dev/null" ), diff --git a/project/plugins.sbt b/project/plugins.sbt index 6307189ce..2b0ca80e4 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -6,12 +6,12 @@ resolvers += "Typesafe repository" at "https://repo.typesafe.com/typesafe/releas libraryDependencies += "org.vafer" % "jdeb" % "1.8" artifacts (Artifact("jdeb", "jar", "jar")) -addSbtPlugin("com.gu" % "sbt-riffraff-artifact" % "1.1.9") +// addSbtPlugin("com.gu" % "sbt-riffraff-artifact" % "1.1.9") // Use the Play sbt plugin for Play projects addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.11") -addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0") +addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0") addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.10.0-RC1") From 67245dc98c7f280e798c06f82842442ae5779f7b Mon Sep 17 00:00:00 2001 From: Georges Lebreton Date: Mon, 16 Oct 2023 19:34:21 +0100 Subject: [PATCH 02/13] fixed riffraff yaml path --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91f501e44..8932c32ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: with: projectName: Editorial Tools::Workflow::Workflow Frontend buildNumber: ${{ env.BUILD_NUMBER }} - configPath: riff-raff.yaml + configPath: conf/riff-raff.yaml contentDirectories: | workflow-frontend: - workflow-frontend/target/workflow-frontend_0.1_all.deb From 1a356f2897ea6202fb3326cc0fc9c4903d3bf87d Mon Sep 17 00:00:00 2001 From: Georges Lebreton Date: Mon, 16 Oct 2023 20:01:35 +0100 Subject: [PATCH 03/13] fixed .deb path --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8932c32ad..f95c4cf8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,9 +61,9 @@ jobs: configPath: conf/riff-raff.yaml contentDirectories: | workflow-frontend: - - workflow-frontend/target/workflow-frontend_0.1_all.deb + - target/workflow-frontend_0.1_all.deb workflow-frontend-fluentbit: - fluentbit/parsers.conf - fluentbit/td-agent-bit.conf - workflow-notification: - - notification/target/workflow-notification.zip \ No newline at end of file + # workflow-notification: + # - notification/target/workflow-notification.zip \ No newline at end of file From 74f3169091e2deab8f1c4e71ab2d1f4a325b8e90 Mon Sep 17 00:00:00 2001 From: Georges Lebreton Date: Mon, 16 Oct 2023 20:31:37 +0100 Subject: [PATCH 04/13] added notification lambda logic --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f95c4cf8d..dda235ff3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,10 @@ jobs: - name: Build and Test project run: | sbt clean compile test Debian/packageBin + - name: Notification Lambda + working-directory: ./notification + run: | + sbt clean compile test Universal/packageBin - uses: guardian/actions-riff-raff@v2 @@ -65,5 +69,5 @@ jobs: workflow-frontend-fluentbit: - fluentbit/parsers.conf - fluentbit/td-agent-bit.conf - # workflow-notification: - # - notification/target/workflow-notification.zip \ No newline at end of file + workflow-notification: + - notification/target/workflow-notification.zip \ No newline at end of file From ca10e5148705a944e924a0730ee428fd07402423 Mon Sep 17 00:00:00 2001 From: Georges Lebreton Date: Mon, 16 Oct 2023 22:05:49 +0100 Subject: [PATCH 05/13] trying to specify notificaiton path --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dda235ff3..7c88d47fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,10 +52,12 @@ jobs: - name: Build and Test project run: | sbt clean compile test Debian/packageBin - - name: Notification Lambda - working-directory: ./notification - run: | - sbt clean compile test Universal/packageBin + cd notification && sbt clean compile test Universal/packageBin + + # - name: Notification Lambda + # working-directory: ./notification + # run: | + # sbt clean compile test Universal/packageBin - uses: guardian/actions-riff-raff@v2 From 7fb9220b2078b9eccb3ac0bb25e62b6dd4369537 Mon Sep 17 00:00:00 2001 From: Georges Lebreton Date: Tue, 17 Oct 2023 01:21:16 +0100 Subject: [PATCH 06/13] build steps for lambda --- .github/workflows/ci.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c88d47fb..30a312511 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,12 +52,9 @@ jobs: - name: Build and Test project run: | sbt clean compile test Debian/packageBin - cd notification && sbt clean compile test Universal/packageBin - - # - name: Notification Lambda - # working-directory: ./notification - # run: | - # sbt clean compile test Universal/packageBin + - name: Notification Lambda + run: | + sbt project notification clean compile Universal/packageBin - uses: guardian/actions-riff-raff@v2 @@ -72,4 +69,4 @@ jobs: - fluentbit/parsers.conf - fluentbit/td-agent-bit.conf workflow-notification: - - notification/target/workflow-notification.zip \ No newline at end of file + - notification/target/universal/workflow-notification.zip \ No newline at end of file From b37d8ed0f43249dd711bfea70ee00adf236ed8ea Mon Sep 17 00:00:00 2001 From: Georges Lebreton Date: Tue, 17 Oct 2023 01:26:40 +0100 Subject: [PATCH 07/13] tweaked path --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30a312511..13efb98ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: sbt clean compile test Debian/packageBin - name: Notification Lambda run: | - sbt project notification clean compile Universal/packageBin + sbt "project/notification" clean compile Universal/packageBin - uses: guardian/actions-riff-raff@v2 From 4b9df8d05afbcc2006f5ca2f29e9735f23508094 Mon Sep 17 00:00:00 2001 From: Georges Lebreton Date: Tue, 17 Oct 2023 01:31:46 +0100 Subject: [PATCH 08/13] tweaked path again --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13efb98ce..2ab1f2070 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: sbt clean compile test Debian/packageBin - name: Notification Lambda run: | - sbt "project/notification" clean compile Universal/packageBin + sbt "project notification" clean compile Universal/packageBin - uses: guardian/actions-riff-raff@v2 From b38337ef1e267883df62c46c6e67f1ea98d11e99 Mon Sep 17 00:00:00 2001 From: Georges Lebreton Date: Tue, 17 Oct 2023 10:12:41 +0100 Subject: [PATCH 09/13] removed sbt riffraff artifact --- .github/workflows/ci.yml | 6 ++---- build.sbt | 14 -------------- project/plugins.sbt | 2 -- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ab1f2070..46f2d65ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,13 +36,11 @@ jobs: role-to-assume: ${{ secrets.GU_RIFF_RAFF_ROLE_ARN }} aws-region: eu-west-1 - - - - name: Install Node uses: actions/setup-node@v3 with: node-version-file: '.nvmrc' + - name: Install dependencies, build & run tests run: | set -e @@ -52,11 +50,11 @@ jobs: - name: Build and Test project run: | sbt clean compile test Debian/packageBin + - name: Notification Lambda run: | sbt "project notification" clean compile Universal/packageBin - - uses: guardian/actions-riff-raff@v2 with: projectName: Editorial Tools::Workflow::Workflow Frontend diff --git a/build.sbt b/build.sbt index 59f81648d..3c3930cc9 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,6 @@ import play.sbt.PlayImport.PlayKeys._ import Dependencies._ import sbtbuildinfo.BuildInfo import scala.sys.env -// import com.gu.riffraff.artifact.BuildInfo Test / parallelExecution := false @@ -82,19 +81,6 @@ lazy val root = playProject(application) s"-Dpidfile.path=/var/run/${packageName.value}/play.pid" ), debianPackageDependencies := Seq("openjdk-8-jre-headless"), - // riffRaffPackageType := (Debian / packageBin).value, - // riffRaffUploadArtifactBucket := Option("riffraff-artifact"), - // riffRaffUploadManifestBucket := Option("riffraff-builds"), - // riffRaffPackageName := s"editorial-tools:workflow:$application", - // riffRaffManifestProjectName := riffRaffPackageName.value, - // fileDescriptorLimit := Option("32768"), - // riffRaffArtifactResources := Seq( - // riffRaffPackageType.value -> s"$application/${riffRaffPackageType.value.getName}", - // (notificationLambda / Universal / packageBin).value -> s"${(notificationLambda / name).value}/${(notificationLambda / Universal / packageBin).value.getName}", - // baseDirectory.value / "conf" / "riff-raff.yaml" -> "riff-raff.yaml", - // baseDirectory.value / "fluentbit/td-agent-bit.conf" -> "workflow-frontend-fluentbit/td-agent-bit.conf", - // baseDirectory.value / "fluentbit/parsers.conf" -> "workflow-frontend-fluentbit/parsers.conf" - // ), Universal / javaOptions ++= Seq( "-Dpidfile.path=/dev/null" ), diff --git a/project/plugins.sbt b/project/plugins.sbt index 2b0ca80e4..6c16285d5 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -6,8 +6,6 @@ resolvers += "Typesafe repository" at "https://repo.typesafe.com/typesafe/releas libraryDependencies += "org.vafer" % "jdeb" % "1.8" artifacts (Artifact("jdeb", "jar", "jar")) -// addSbtPlugin("com.gu" % "sbt-riffraff-artifact" % "1.1.9") - // Use the Play sbt plugin for Play projects addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.11") From 4612d9cf5877519678d19d766a48755ec13c1b51 Mon Sep 17 00:00:00 2001 From: Georges Lebreton Date: Tue, 17 Oct 2023 16:21:05 +0100 Subject: [PATCH 10/13] clarified and split up build steps --- .github/workflows/ci.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46f2d65ad..2fcd3ff28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,17 +41,20 @@ jobs: with: node-version-file: '.nvmrc' - - name: Install dependencies, build & run tests - run: | - set -e - yarn install --frozen-lockfile - yarn build + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Yarn build + run: yarn build + + - name: Yarn test + run: yarn test - - name: Build and Test project + - name: Build, test and package the scala app run: | sbt clean compile test Debian/packageBin - - name: Notification Lambda + - name: Build and package the notification Lambda run: | sbt "project notification" clean compile Universal/packageBin From b4aaca58cae4275f687bc25cfd7a5b529310f083 Mon Sep 17 00:00:00 2001 From: Georges Lebreton Date: Tue, 17 Oct 2023 17:33:03 +0100 Subject: [PATCH 11/13] tweaked steps --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fcd3ff28..0d60ef433 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,9 +47,6 @@ jobs: - name: Yarn build run: yarn build - - name: Yarn test - run: yarn test - - name: Build, test and package the scala app run: | sbt clean compile test Debian/packageBin From a1e952fd99c35b0ebbcb50a7a4d6e5e0f88f5a6c Mon Sep 17 00:00:00 2001 From: Georges Lebreton <102960844+Georges-GNM@users.noreply.github.com> Date: Wed, 18 Oct 2023 11:31:33 +0100 Subject: [PATCH 12/13] Apply naming suggestions from code review Co-authored-by: Tom Richards --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d60ef433..7b76976e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,10 +41,10 @@ jobs: with: node-version-file: '.nvmrc' - - name: Install dependencies + - name: Install JS dependencies run: yarn install --frozen-lockfile - - name: Yarn build + - name: JS build run: yarn build - name: Build, test and package the scala app From 20ca4a494175679d0c0dea170d2d6ad10980e114 Mon Sep 17 00:00:00 2001 From: Tom Richards Date: Wed, 18 Oct 2023 11:35:31 +0100 Subject: [PATCH 13/13] run JS tests in CI --- .github/workflows/ci.yml | 3 +++ karma.conf.js | 2 +- package.json | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b76976e1..eedb74e0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,9 @@ jobs: - name: Install JS dependencies run: yarn install --frozen-lockfile + - name: test JS + run: yarn test + - name: JS build run: yarn build diff --git a/karma.conf.js b/karma.conf.js index a603bf78b..e1d0bd6f1 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -67,6 +67,6 @@ module.exports = function(config) { // Continuous Integration mode // if true, Karma captures browsers, runs the tests and exits - singleRun: false + singleRun: true }); }; diff --git a/package.json b/package.json index a295cab5a..be7fe9afd 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "build-dev": "webpack --config ./conf/webpack.conf.js --watch", "build": "webpack --config ./conf/webpack.prod.conf.js", "test": "karma start", + "test-watch": "yarn test --no-single-run", "lint": "eslint ./public/**/**/*.js" }, "devDependencies": {