Merge branch 'master' into #848-auto-update-dependencies

Flank · Sep 21, 2020 · 1b72732 · 1b72732
2 parents e45959e + 457032c
commit 1b72732
Show file tree

Hide file tree

Showing 21 changed files with 241 additions and 143 deletions.
diff --git a/.github/workflows/ubuntu-workflow.yml b/.github/workflows/ubuntu-workflow.yml
@@ -46,7 +46,6 @@ jobs:
         uses: eskatos/gradle-command-action@v1
         with:
           arguments: "--info :integration_tests:test --tests IntegrationTests.shouldMatchAndroidSuccessExitCodeAndPattern -Dflank-path=../test_runner/build/libs/flank.jar -Dyml-path=./src/test/resources/flank_android.yml"
-
       - name: Gradle Integration Tests iOS
         uses: eskatos/gradle-command-action@v1
         env:

diff --git a/.github/workflows/windows_workflow.yml b/.github/workflows/windows_workflow.yml
@@ -0,0 +1,36 @@
+name: windows-workflow
+
+on:
+  push:
+    branches: 
+      - master
+  pull_request:
+    branches:
+      - '*'
+    paths:
+      - '**'
+      - '!flank-scripts/**'
+
+jobs:
+  build:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Display the java version
+        run: java -version
+        shell: bash
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+
+      - name: Gradle clean build
+        uses: eskatos/gradle-command-action@v1
+        with:
+          arguments: "clean build"
diff --git a/test_runner/.gitignore b/test_runner/.gitignore
@@ -8,3 +8,4 @@ src/test/kotlin/ftl/fixtures/error_result/*.xml
 src/test/kotlin/ftl/fixtures/success_result/*.xml
 *.json
 *-describe.adoc
+should_exists.txt
diff --git a/test_runner/build.gradle.kts b/test_runner/build.gradle.kts
@@ -1,12 +1,12 @@
+import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 import com.jfrog.bintray.gradle.BintrayExtension
 import groovy.util.Node
 import groovy.util.NodeList
 import org.gradle.api.tasks.testing.logging.TestExceptionFormat
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
-import java.util.Date
-import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 import java.io.ByteArrayOutputStream
-import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask
+import java.util.*
 
 plugins {
     application
@@ -148,6 +148,12 @@ detekt {
     }
 }
 
+subprojects {
+    tasks.withType<Test> {
+        maxParallelForks = Runtime.getRuntime().availableProcessors() / 2
+    }
+}
+
 // Kotlin dsl
 tasks.withType<io.gitlab.arturbosch.detekt.Detekt> {
     // Target version of the generated JVM bytecode. It is used for type resolution.

diff --git a/test_runner/gradle.properties b/test_runner/gradle.properties
@@ -9,7 +9,8 @@
 
 # Specifies the JVM arguments used for the daemon process.
 # The setting is particularly useful for tweaking memory settings.
-org.gradle.jvmargs=-Xmx2560m
+org.gradle.jvmargs=-Xmx2560m -Dfile.encoding=UTF-8
 kotlin.code.style=official
 org.gradle.parallel=true
-
+org.gradle.daemon=true
+org.gradle.caching=true
diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt
@@ -10,12 +10,14 @@ import com.google.cloud.storage.BucketInfo
 import com.google.cloud.storage.Storage
 import com.google.cloud.storage.StorageClass
 import com.google.cloud.storage.StorageOptions
+import ftl.args.ArgsHelper.convertToWindowsPath
 import ftl.args.IArgs.Companion.AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE
 import ftl.args.yml.YamlObjectMapper
 import ftl.config.FtlConstants
 import ftl.config.FtlConstants.GCS_PREFIX
 import ftl.config.FtlConstants.JSON_FACTORY
 import ftl.config.FtlConstants.defaultCredentialPath
+import ftl.config.FtlConstants.isWindows
 import ftl.config.FtlConstants.useMock
 import ftl.gc.GcStorage
 import ftl.gc.GcToolResults
@@ -31,6 +33,7 @@ import ftl.util.assertNotEmpty
 import java.io.File
 import java.net.URI
 import java.nio.file.Files
+import java.nio.file.InvalidPathException
 import java.nio.file.Path
 import java.nio.file.Paths
 import java.util.regex.Pattern
@@ -46,14 +49,16 @@ object ArgsHelper {
             throw FlankGeneralError("'$file' $name doesn't exist")
     }
 
+    private fun String.convertToWindowsPath() = this.replace("/", "\\").replaceFirst("~", System.getProperty("user.home"))
+
     private fun String.exist() =
-        if (startsWith(GCS_PREFIX)) GcStorage.exist(this) else File(this).exists()
+            if (startsWith(GCS_PREFIX)) GcStorage.exist(this) else File(this).exists()
 
     fun assertCommonProps(args: IArgs) {
         assertNotEmpty(
-            args.project, "The project is not set. Define GOOGLE_CLOUD_PROJECT, set project in flank.yml\n" +
-                    "or save service account credential to ${defaultCredentialPath}\n" +
-                    " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id"
+                args.project, "The project is not set. Define GOOGLE_CLOUD_PROJECT, set project in flank.yml\n" +
+                "or save service account credential to ${defaultCredentialPath}\n" +
+                " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id"
         )
 
         if (args.maxTestShards !in AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE && args.maxTestShards != -1)
@@ -72,11 +77,18 @@ object ArgsHelper {
         }
     }
 
+    private fun String.inferCorrectPath() = if (isWindows) this.trim().convertToWindowsPath()
+    else this.trim().replaceFirst(Regex("^~"), System.getProperty("user.home"))
+
     fun evaluateFilePath(filePath: String): String {
-        var file = filePath.trim().replaceFirst(Regex("^~"), System.getProperty("user.home"))
+        var file = filePath.inferCorrectPath()
         file = evaluateEnvVars(file)
         // avoid File(..).canonicalPath since that will resolve symlinks
-        file = Paths.get(file).toAbsolutePath().normalize().toString()
+        try {
+            file = Paths.get(file).toAbsolutePath().normalize().toString()
+        } catch (e: InvalidPathException) {
+            throw FlankGeneralError("Invalid path exception: $e")
+        }
 
         // Avoid walking the folder's parent dir if we know it exists already.
         if (File(file).exists()) return file
@@ -103,12 +115,7 @@ object ArgsHelper {
         GcStorage.storage.get(bucket, path) ?: throw FlankGeneralError("The file at '$uri' does not exist")
     }
 
-    fun validateTestMethods(
-        testTargets: List<String>,
-        validTestMethods: Collection<String>,
-        from: String,
-        skipValidation: Boolean = useMock
-    ) {
+    fun validateTestMethods(testTargets: List<String>, validTestMethods: Collection<String>, from: String, skipValidation: Boolean = useMock) {
         val missingMethods = testTargets - validTestMethods
 
         if (!skipValidation && missingMethods.isNotEmpty()) throw FlankConfigurationError("$from is missing methods: $missingMethods.\nValid methods:\n$validTestMethods")
@@ -125,17 +132,17 @@ object ArgsHelper {
     // Due to permission issues, the user may not be able to list or create buckets.
     fun createGcsBucket(projectId: String, bucket: String): String {
         if (bucket.isEmpty()) return GcToolResults.getDefaultBucket(projectId)
-            ?: throw FlankGeneralError("Failed to make bucket for $projectId")
+                ?: throw FlankGeneralError("Failed to make bucket for $projectId")
         if (useMock) return bucket
 
         // test lab supports using a special free storage bucket
         // because we don't have access to the root account, it won't show up in the storage list.
         if (bucket.startsWith("test-lab-")) return bucket
 
         val storage = StorageOptions.newBuilder()
-            .setCredentials(FtlConstants.credential)
-            .setProjectId(projectId)
-            .build().service
+                .setCredentials(FtlConstants.credential)
+                .setProjectId(projectId)
+                .build().service
         val bucketLabel = mapOf("flank" to "")
         val storageLocation = "us-central1"
 
@@ -152,11 +159,11 @@ object ArgsHelper {
 
         try {
             storage.create(
-                BucketInfo.newBuilder(bucket)
-                    .setStorageClass(StorageClass.REGIONAL)
-                    .setLocation(storageLocation)
-                    .setLabels(bucketLabel)
-                    .build()
+                    BucketInfo.newBuilder(bucket)
+                            .setStorageClass(StorageClass.REGIONAL)
+                            .setLocation(storageLocation)
+                            .setLabels(bucketLabel)
+                            .build()
             )
         } catch (e: Exception) {
             println("Warning: Failed to make bucket for $projectId\nCause: ${e.message}")
@@ -170,9 +177,9 @@ object ArgsHelper {
             if (!defaultCredentialPath.toFile().exists()) return null
 
             return JsonObjectParser(JSON_FACTORY).parseAndClose(
-                Files.newInputStream(defaultCredentialPath),
-                Charsets.UTF_8,
-                GenericJson::class.java
+                    Files.newInputStream(defaultCredentialPath),
+                    Charsets.UTF_8,
+                    GenericJson::class.java
             )["project_id"] as String
         } catch (e: Exception) {
             println("Parsing $defaultCredentialPath failed:")
@@ -226,9 +233,9 @@ object ArgsHelper {
         val shards = if (args.disableSharding) {
             listOf(Chunk(testsToExecute.map {
                 TestMethod(
-                    name = it.testName,
-                    isParameterized = it.isParameterizedClass,
-                    time = 0.0
+                        name = it.testName,
+                        isParameterized = it.isParameterizedClass,
+                        time = 0.0
                 )
             }))
         } else {
@@ -238,8 +245,8 @@ object ArgsHelper {
         }
 
         return CalculateShardsResult(
-            testMethodsAlwaysRun(shards, args),
-            ignoredTestCases = ignoredTests.map { it.testName })
+                testMethodsAlwaysRun(shards, args),
+                ignoredTestCases = ignoredTests.map { it.testName })
     }
 
     private fun testMethodsAlwaysRun(shards: List<Chunk>, args: IArgs): List<Chunk> {
@@ -251,9 +258,9 @@ object ArgsHelper {
 }
 
 fun String.normalizeFilePath(): String =
-    if (startsWith(GCS_PREFIX)) this
-    else try {
-        ArgsHelper.evaluateFilePath(this)
-    } catch (e: Throwable) {
-        this
-    }
+        if (startsWith(GCS_PREFIX)) this
+        else try {
+            ArgsHelper.evaluateFilePath(this)
+        } catch (e: Throwable) {
+            this
+        }
diff --git a/test_runner/src/main/kotlin/ftl/doctor/Doctor.kt b/test_runner/src/main/kotlin/ftl/doctor/Doctor.kt
@@ -26,7 +26,7 @@ fun validateYaml(args: IArgs.ICompanion, data: Path) =
 @VisibleForTesting
 internal fun validateYaml(args: IArgs.ICompanion, data: Reader) =
     runCatching { ArgsHelper.yamlMapper.readTree(data) }
-        .onFailure { return it.message ?: "Unknown error when parsing tree" }
+        .onFailure { return it.message?.replace(System.lineSeparator(), "\n") ?: "Unknown error when parsing tree" }
         .getOrNull()
         ?.run { validateYamlKeys(args).plus(validateDevices()) }
         .orEmpty()

diff --git a/test_runner/src/main/kotlin/ftl/reports/util/MatrixPath.kt b/test_runner/src/main/kotlin/ftl/reports/util/MatrixPath.kt
@@ -5,13 +5,10 @@ import java.nio.file.Paths
 
 fun File.getMatrixPath(objectName: String) = getShardName(objectName)?.asObjectPath()
 
-private fun File.getShardName(
-    objectName: String
-) = shardNameRegex(objectName)
+private fun File.getShardName(objectName: String) = shardNameRegex(objectName)
     .find(toString())
     ?.value
-    ?.removePrefix("/")
-    ?.removeSuffix("/")
+    ?.removePrefix("/")?.removeSuffix("/")
 
 private fun shardNameRegex(objectName: String) = "/($objectName)/(shard_|matrix_)\\d+(-rerun_\\d+)?/".toRegex()
 

diff --git a/test_runner/src/test/kotlin/ftl/args/ArgsHelperFilePathTest.kt b/test_runner/src/test/kotlin/ftl/args/ArgsHelperFilePathTest.kt
@@ -68,6 +68,8 @@ class ArgsHelperFilePathTest {
 
     @Test
     fun evaluateRelativeFilePath() {
+        assumeFalse(isWindows)
+
         val expected = makeTmpFile("/tmp/app-debug.apk")
         val testApkPath = "~/../../../../../../../../../tmp/app-debug.apk"
         val actual = ArgsHelper.evaluateFilePath(testApkPath)

diff --git a/test_runner/src/test/kotlin/ftl/args/yml/ErrorParserTest.kt b/test_runner/src/test/kotlin/ftl/args/yml/ErrorParserTest.kt
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
 import com.fasterxml.jackson.dataformat.yaml.snakeyaml.error.MarkedYAMLException
 import ftl.args.AndroidArgs
 import ftl.args.yml.errors.ConfigurationErrorMessageBuilder
+import ftl.doctor.assertEqualsIgnoreNewlineStyle
 import ftl.test.util.TestHelper
 import ftl.test.util.TestHelper.getThrowable
 import ftl.run.exception.FlankConfigurationError
@@ -63,7 +64,7 @@ At line: 23, column: 3
               "version" : "test"
             }
         """.trimIndent()
-        Assert.assertEquals(exceptedMessage, actualMessage)
+        assertEqualsIgnoreNewlineStyle(exceptedMessage, actualMessage)
     }
 
     @Test
@@ -80,7 +81,7 @@ Error node: {
   }
 }
         """.trimIndent()
-        Assert.assertEquals(exceptedMessage, actualMessage)
+        assertEqualsIgnoreNewlineStyle(exceptedMessage, actualMessage)
     }
 
     @Test
@@ -108,7 +109,7 @@ Error node:
         val actualMessage = buildErrorMessage(exception)
 
         // then
-        Assert.assertEquals(expectedMessage, actualMessage)
+        assertEqualsIgnoreNewlineStyle(expectedMessage, actualMessage)
     }
 
     @Test(expected = FlankConfigurationError::class)

diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/CancelCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/CancelCommandTest.kt
@@ -43,6 +43,7 @@ class CancelCommandTest {
 
     @Test
     fun cancelCommandRuns() {
+        CancelCommand().run()
         val runCmd = AndroidRunCommand()
         runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-android-flank.yml"
         runCmd.run()

diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/RefreshCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/RefreshCommandTest.kt
@@ -20,7 +20,7 @@ class RefreshCommandTest {
 
     /** Create one result dir with matrix_ids.json for refresh command tests */
     private fun setupResultsDir() {
-        val parent = "results/2018-09-07_01:21:14.201000_SUfE"
+        val parent = "results/2018-09-07_01-21-14.201000_SUfE"
         val matrixIds = Paths.get(parent, "matrix_ids.json")
         val yamlCfg = Paths.get(parent, "flank.yml")
         matrixIds.parent.toFile().mkdirs()

diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/IPBlocksCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/IPBlocksCommandTest.kt
@@ -1,6 +1,6 @@
 package ftl.cli.firebase.test
 
-import org.junit.Assert.assertEquals
+import ftl.doctor.assertEqualsIgnoreNewlineStyle
 import org.junit.Rule
 import org.junit.Test
 import org.junit.contrib.java.lang.system.SystemOutRule
@@ -23,6 +23,6 @@ Commands:
 
         val actual = out.log.trim()
 
-        assertEquals(expected, actual)
+        assertEqualsIgnoreNewlineStyle(expected, actual)
     }
 }