parallel_iocs

Add building archiver inside docker container Building epics in a docker file Add more tests to the parallel epics integration tests category Add SampleBufferOverFlowTest to ParallelEpicsIntegrationTests Use a static variable to track status of default TomcatSetup Running some integration tests in parallel Add a new test group ParallelEpicsIntegrationTests which is for tests with multiple iocs running at the same time and/or a single tomcat instance to test against.
archiver-appliance · Aug 25, 2023 · 5f526eb · 5f526eb
1 parent 9aeb9b5
commit 5f526eb
Show file tree

Hide file tree

Showing 125 changed files with 23,034 additions and 23,073 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,39 @@
+RELEASE_NOTES
+bin/
+stage/
+.DS_Store
+*.swp
+/mts
+
+.gradle
+**/build/
+!src/**/build/
+
+# Ignore Gradle GUI config
+gradle-app.setting
+
+# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored)
+!gradle-wrapper.jar
+
+# Avoid ignore Gradle wrappper properties
+!gradle-wrapper.properties
+
+# Cache of project
+.gradletasknamecache
+
+# Eclipse Gradle plugin generated files
+# Eclipse Core
+.project
+# JDT-specific (Eclipse Java Development Tools)
+.classpath
+# Eclipse settings
+.settings
+
+# vscode settings
+.vscode
+
+# Integration tests folders
+tomcat_*
+
+# War files
+wars
diff --git a/.github/workflows/ant.yml b/.github/workflows/ant.yml
@@ -1,7 +1,7 @@
 # This workflow will build a Java project with Ant
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-ant
 
-name: Java CI
+name: Ant build
 
 on:
   push:

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -0,0 +1,82 @@
+name: Docker Run Integration Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+permissions:
+  contents: read
+  checks: write
+
+jobs:
+  epicstests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+        fetch-depth: 0
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+    - name: Create Build Reports Folder
+      run: mkdir -p build/reports/tests
+    - name: Build and push
+      uses: docker/build-push-action@v4
+      with:
+        file: docker/Dockerfile
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+    - name: Run tests with docker compose
+      run: docker compose -f docker/docker-compose.epicsTests.yml run epicsarchiver-test
+      env:
+        DOCKER_DEFAULT_PLATFORM: linux/amd64
+        COMPOSE_DOCKER_CLI_BUILD: 1
+        DOCKER_BUILDKIT: 1
+    - name: Publish Test Report
+      uses: mikepenz/action-junit-report@v3
+      if: success() || failure() # always run even if the previous step fails
+      with:
+        report_paths: '**/build/test-results/test/TEST-*.xml'
+    - name: Upload build reports
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: build-reports
+        path: build/reports/
+
+  epicstests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+        fetch-depth: 0
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+    - name: Create Build Reports Folder
+      run: mkdir -p build/reports/tests
+    - name: Build and push
+      uses: docker/build-push-action@v4
+      with:
+        file: docker/Dockerfile
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+    - name: Run tests with docker compose
+      run: docker compose -f docker/docker-compose.integrationTests.yml run epicsarchiver-test
+      env:
+        DOCKER_DEFAULT_PLATFORM: linux/amd64
+        COMPOSE_DOCKER_CLI_BUILD: 1
+        DOCKER_BUILDKIT: 1
+    - name: Publish Test Report
+      uses: mikepenz/action-junit-report@v3
+      if: success() || failure() # always run even if the previous step fails
+      with:
+        report_paths: '**/build/test-results/test/TEST-*.xml'
+    - name: Upload build reports
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: build-reports
+        path: build/reports/
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -5,7 +5,7 @@
 # This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-gradle
 
-name: Java CI with Gradle
+name: Gradle build and test
 
 on:
   push:
@@ -15,6 +15,7 @@ on:
 
 permissions:
   contents: read
+  checks: write
 
 jobs:
   build:
@@ -24,6 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
+          submodules: true
           fetch-depth: 0
       - name: Set up JDK 17
         uses: actions/setup-java@v3

diff --git a/.gitignore b/.gitignore
@@ -36,4 +36,7 @@ gradle-app.setting
 .vscode
 
 # Integration tests folders
-tomcat_*
+tomcat_*
+
+# War files
+wars
diff --git a/build.gradle b/build.gradle
@@ -41,10 +41,10 @@ java {
 }
 
 version gitVersion()
-def stageDir = layout.projectDir.file("stage")
-def warDir = layout.projectDir.file("build")
-def srcDir = layout.projectDir.file("src/main")
-def libDir = layout.projectDir.file("lib")
+def stageDir = layout.projectDirectory.file("build/stage")
+def srcDir = layout.projectDirectory.file("src/main")
+def libDir = layout.projectDirectory.file("lib")
+def apiDocsDir = layout.projectDirectory.file("docs/api")
 def archapplsite = System.getenv("ARCHAPPL_SITEID") ?: "tests"
 
 
@@ -107,36 +107,21 @@ dependencies {
     implementation("org.phoebus:core-pva:4.7.2")
 }
 
-tasks.register("cleanWars", Delete) {
+tasks.register("cleanApiDocs", Delete) {
     group = "Clean"
-    description = "Remove any war files created, and the staging directory"
-    delete file(stageDir)
-    delete file("${warDir}/retrieval.war")
-    delete file("${warDir}/engine.war")
-    delete file("${warDir}/etl.war")
-    delete file("${warDir}/mgmt.war")
-    delete file("${warDir}/archappl_v${version}.tar.gz")
-    delete file("RELEASE_NOTES")
+    description = "Remove the generated java api docs."
+    delete file(apiDocsDir)
 }
 
-Collection getListOfFolders(String dir, String pattern) {
-    file(dir).listFiles({ file -> !file.isFile() && file.name ==~ pattern } as FileFilter)
-}
-
-tasks.register("cleanTestData", Delete) {
-    group = "Clean"
-    description = "Remove any integration test folders created, and the staging directory"
-    delete getListOfFolders("./", /tomcat_*/)
-}
-clean.dependsOn("cleanTestData")
-clean.dependsOn "cleanWars"
+clean.dependsOn("cleanApiDocs")
 
 tasks.register("generateBPLActionsMappings", JavaExec) {
     group = "Documentation"
     description = "Generates the rest api documentation."
+    it.outputs.file("${apiDocsDir}/mgmtpathmappings.txt")
     doFirst {
-        mkdir "docs/api"
-        standardOutput new FileOutputStream("${projectDir}/docs/api/mgmtpathmappings.txt")
+        mkdir apiDocsDir
+        standardOutput = new FileOutputStream("${apiDocsDir}/mgmtpathmappings.txt")
     }
     mainClass = "org.epics.archiverappliance.mgmt.BPLServlet"
 
@@ -145,8 +130,10 @@ tasks.register("generateBPLActionsMappings", JavaExec) {
 
 tasks.register("generateJavaDocTagletScriptables", JavaExec) {
     group = "Documentation"
+    dependsOn "generateBPLActionsMappings"
     mainClass = "org.epics.archiverappliance.common.taglets.ProcessMgmtScriptables"
-
+    it.inputs.file("${apiDocsDir}/mgmt_scriptables.txt")
+    it.outputs.file("${apiDocsDir}/mgmt_scriptables.html")
     classpath = sourceSets.main.runtimeClasspath
     workingDir = project.projectDir
 }
@@ -158,10 +145,11 @@ javadoc {
     doFirst {
         copy {
             from layout.projectDirectory.file("LICENSE"),
-                    file("${srcDir}/main/edu/stanford/slac/archiverappliance/PB/EPICSEvent.proto")
+                    file("${srcDir}/edu/stanford/slac/archiverappliance/PB/EPICSEvent.proto")
             into layout.projectDirectory.dir("docs")
         }
     }
+    it.outputs.file("${apiDocsDir}/mgmt_scriptables.txt")
     source = sourceSets.main.allJava
     failOnError = false
     options {
@@ -181,8 +169,8 @@ tasks.register("syncStaticContentHeaderFooters", JavaExec) {
     group = "Staging"
     dependsOn "compileJava"
     mainClass = "org.epics.archiverappliance.mgmt.bpl.SyncStaticContentHeadersFooters"
-    args "${srcDir}/main/org/epics/archiverappliance/mgmt/staticcontent/index.html",
-            "${srcDir}/main/org/epics/archiverappliance/mgmt/staticcontent/"
+    args "${srcDir}/org/epics/archiverappliance/mgmt/staticcontent/index.html",
+            "${srcDir}/org/epics/archiverappliance/mgmt/staticcontent/"
 
     classpath = sourceSets.main.runtimeClasspath
 }
@@ -205,9 +193,14 @@ tasks.register("zipSvgViewer", Zip) {
 tasks.register("stage") {
     group = "Staging"
     description = "Copy static content from each of the projects into the staging directory."
+    dependsOn javadoc
     finalizedBy "zipSvgViewer"
     mkdir stageDir
 
+    copy {
+        from "${docsDir}/*.*"
+        into apiDocsDir
+    }
     copy {
         from file("${srcDir}/org/epics/archiverappliance/staticcontent")
         into file("${stageDir}/org/epics/archiverappliance/staticcontent")
@@ -226,8 +219,9 @@ tasks.register("stage") {
 tasks.register("generateReleaseNotes", Exec) {
     group = "Staging"
     description = "Generate the Release Notes."
+    it.outputs.file("${stageDir}/RELEASE_NOTES")
     doFirst {
-        standardOutput new FileOutputStream("${projectDir}/RELEASE_NOTES")
+        standardOutput new FileOutputStream("${stageDir}/RELEASE_NOTES")
     }
     commandLine "git", "log", "--oneline", "remotes/origin/master"
 }
@@ -240,6 +234,11 @@ tasks.withType(War).configureEach {
     from("${stageDir}/org/epics/archiverappliance/staticcontent") {
         into "ui/comm"
     }
+    if (archapplsite == "tests") {
+        from("src/resources/test/log4j2.xml") {
+            into "WEB-INF/classes"
+        }
+    }
     from("src/sitespecific/${archapplsite}/classpathfiles") {
         into "WEB-INF/classes"
     }
@@ -248,7 +247,7 @@ tasks.withType(War).configureEach {
 
 tasks.register("mgmtWar", War) {
     group = "Wars"
-    dependsOn javadoc
+    dependsOn "generateJavaDocTagletScriptables"
     from("${stageDir}/org/epics/archiverappliance/mgmt/staticcontent") {
         into "ui"
     }
@@ -417,46 +416,51 @@ tasks.register("shutdownAllTomcats", Exec) {
     group = "test"
     description "task to shut down all tomcats after running integration tests," +
             " if they didn't shut down correctly."
+    ignoreExitValue true
     commandLine "pkill", "-9", "-f", "tomcat"
 }
 
-tasks.register("integrationSetup") {
-    group = "Test"
-    description = "Setup for Integration Tests."
-    dependsOn "buildRelease"
-    doFirst {
+tasks.register("integrationTestSetup", Copy) {
+    dependsOn("buildRelease")
+    it.inputs.dir("$System.env.TOMCAT_HOME/conf")
+    it.outputs.dir("$System.env.TOMCAT_HOME/conf_original")
+    // backup the original tomcat conf folder
+    // if not already done. See org.epics.archiverappliance.TomcatSetup
+    def catalinaConf = file("$System.env.TOMCAT_HOME/conf_original")
+    if (!catalinaConf.exists()) {
         copy {
-            from layout.buildDirectory.file("libs/mgmt.war")
-            from layout.buildDirectory.file("libs/engine.war")
-            from layout.buildDirectory.file("libs/etl.war")
-            from layout.buildDirectory.file("libs/retrieval.war")
-            into file("${warDir}")
-        }
-        // backup the original tomcat conf folder
-        // if not already done. See org.epics.archiverappliance.TomcatSetup
-        def catalinaConf = file("$System.env.TOMCAT_HOME/conf_original")
-        if (!catalinaConf.exists()) {
-            copy {
-                from file("$System.env.TOMCAT_HOME/conf")
-                into file("$System.env.TOMCAT_HOME/conf_original")
-            }
+            from "$System.env.TOMCAT_HOME/conf"
+            into "$System.env.TOMCAT_HOME/conf_original"
         }
     }
-
 }
 
 tasks.register("integrationTests", Test) {
     group = "Test"
     description = "Run the integration tests, ones that require a tomcat installation."
-    dependsOn "integrationSetup"
     forkEvery 1
     maxParallelForks = 1
+    dependsOn "integrationTestSetup"
     useJUnit {
         includeCategories "org.epics.archiverappliance.IntegrationTests",
                 "org.epics.archiverappliance.LocalEpicsTests"
         excludeCategories "org.epics.archiverappliance.SlowTests",
                 "org.epics.archiverappliance.FlakyTests"
     }
+    finalizedBy "shutdownAllTomcats"
+}
+
+tasks.register("epicsTests", Test) {
+    group = "Test"
+    description = "Run the epics integration tests with parallel iocs."
+    dependsOn "integrationTestSetup"
+    useJUnit {
+        includeCategories "org.epics.archiverappliance.LocalEpicsTests"
+        excludeCategories "org.epics.archiverappliance.SlowTests",
+                "org.epics.archiverappliance.FlakyTests",
+                "org.epics.archiverappliance.IntegrationTests"
+    }
+    finalizedBy "shutdownAllTomcats"
 }
 
 tasks.register("singleForkTests", Test) {
@@ -496,7 +500,7 @@ tasks.register("allTests", Test) {
 tasks.register("testRun", JavaExec) {
     group = "test"
     description = "Runs the application same as for integration tests."
-    dependsOn "integrationSetup"
+    dependsOn "integrationTestSetup"
 
     mainClass = "org.epics.archiverappliance.TestRun"