Allow blob, tilde and env vars in app paths

release_notes.md

@@ -1,5 +1,6 @@
-## v4.1
-- ?
+## v4.1 (unreleased)
+- `app`, `test`, and `xctestrun-file` now support `~`, environment variables, and globs (`*`, `**`) when resolving paths. #386
+- Update `flank android run` to support `--app`, `--test`, `--test-targets`, `--use-orchestrator` and `--no-use-orchestrator`.
 
 ## v4.0.0
 

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -10,6 +10,7 @@ import ftl.android.UnsupportedVersionId
 import ftl.args.ArgsHelper.assertFileExists
 import ftl.args.ArgsHelper.assertGcsFileExists
 import ftl.args.ArgsHelper.calculateShards
+import ftl.args.ArgsHelper.evaluateFilePath
 import ftl.args.ArgsHelper.getGcsBucket
 import ftl.args.ArgsHelper.mergeYmlMaps
 import ftl.args.ArgsHelper.yamlMapper
@@ -47,8 +48,8 @@ class AndroidArgs(
     override val resultsHistoryName = gcloud.resultsHistoryName
 
     private val androidGcloud = androidGcloudYml.gcloud
-    val appApk = cli.app ?: androidGcloud.app
-    val testApk = cli.test ?: androidGcloud.test
+    var appApk = cli.app ?: androidGcloud.app
+    var testApk = cli.test ?: androidGcloud.test
     val autoGoogleLogin = androidGcloud.autoGoogleLogin
 
     // We use not() on noUseOrchestrator because if the flag is on, useOrchestrator needs to be false
@@ -89,12 +90,14 @@ class AndroidArgs(
         if (appApk.startsWith(FtlConstants.GCS_PREFIX)) {
             assertGcsFileExists(appApk)
         } else {
+            appApk = evaluateFilePath(appApk)
             assertFileExists(appApk, "appApk")
         }
 
         if (testApk.startsWith(FtlConstants.GCS_PREFIX)) {
             assertGcsFileExists(testApk)
         } else {
+            testApk = evaluateFilePath(testApk)
             assertFileExists(testApk, "testApk")
         }
 

test_runner/src/main/kotlin/ftl/args/ArgsFileVisitor.kt

@@ -0,0 +1,51 @@
+package ftl.args
+
+import ftl.util.Utils.fatalError
+import java.io.IOException
+import java.nio.file.FileSystems
+import java.nio.file.FileVisitOption
+import java.nio.file.FileVisitResult
+import java.nio.file.Files
+import java.nio.file.LinkOption
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.SimpleFileVisitor
+import java.nio.file.attribute.BasicFileAttributes
+import java.util.EnumSet
+
+class ArgsFileVisitor(glob: String) : SimpleFileVisitor<Path>() {
+    private val pathMatcher = FileSystems.getDefault().getPathMatcher(glob)
+    private val result: MutableList<Path> = mutableListOf()
+
+    @Throws(IOException::class)
+    override fun visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult {
+        if (pathMatcher.matches(path)) {
+            result.add(path)
+        }
+        return FileVisitResult.CONTINUE
+    }
+
+    override fun visitFileFailed(file: Path?, exc: IOException?): FileVisitResult {
+        fatalError("Failed to visit $file $exc")
+        return FileVisitResult.CONTINUE
+    }
+
+    companion object {
+        private const val RECURSE = "/**"
+        private const val SINGLE_GLOB = "/*"
+    }
+
+    @Throws(java.nio.file.NoSuchFileException::class)
+    fun walk(searchPath: Path): List<Path> {
+        val searchString = searchPath.toString()
+        // /Users/tmp/code/flank/test_app/** => /Users/tmp/code/flank/test_app/
+        // /Users/tmp/code/*                 => /Users/tmp/code/
+        val beforeGlob = Paths.get(searchString.substringBefore(SINGLE_GLOB))
+        // must not follow links when resolving paths or /tmp turns into /private/tmp
+        val realPath = beforeGlob.toRealPath(LinkOption.NOFOLLOW_LINKS)
+        val searchDepth = if (searchString.contains(RECURSE)) Integer.MAX_VALUE else 1
+
+        Files.walkFileTree(realPath, EnumSet.of(FileVisitOption.FOLLOW_LINKS), searchDepth, this)
+        return this.result
+    }
+}

test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt

@@ -23,6 +23,9 @@ import java.io.File
 import java.math.RoundingMode
 import java.net.URI
 import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.util.regex.Pattern
 
 object ArgsHelper {
 
@@ -45,6 +48,25 @@ object ArgsHelper {
         }
     }
 
+    fun evaluateFilePath(filePath: String): String {
+        var file = filePath.trim().replaceFirst(Regex("^~"), System.getProperty("user.home"))
+        file = evaluateEnvVars(file)
+        // avoid File(..).canonicalPath since that will resolve symlinks
+        file = Paths.get(file).toAbsolutePath().normalize().toString()
+
+        // Avoid walking the folder's parent dir if we know it exists already.
+        if (File(file).exists()) return file
+
+        val filePaths = walkFileTree(file)
+        if (filePaths.size > 1) {
+            Utils.fatalError("'$file' ($filePath) matches multiple files: $filePaths")
+        } else if (filePaths.isEmpty()) {
+            Utils.fatalError("'$file' not found ($filePath)")
+        }
+
+        return filePaths.first().toAbsolutePath().normalize().toString()
+    }
+
     fun assertGcsFileExists(uri: String) {
         if (!uri.startsWith(GCS_PREFIX)) {
             throw IllegalArgumentException("must start with $GCS_PREFIX uri: $uri")
@@ -135,7 +157,8 @@ object ArgsHelper {
             return JsonObjectParser(JSON_FACTORY).parseAndClose(
                 Files.newInputStream(defaultCredentialPath),
                 Charsets.UTF_8,
-                GenericJson::class.java)["project_id"] as String
+                GenericJson::class.java
+            )["project_id"] as String
         } catch (e: Exception) {
             println("Parsing $defaultCredentialPath failed:")
             println(e.printStackTrace())
@@ -150,4 +173,27 @@ object ArgsHelper {
         // Allow users control over projectId by checking using Google's logic first before falling back to JSON.
         return ServiceOptions.getDefaultProjectId() ?: serviceAccountProjectId()
     }
+
+    // https://stackoverflow.com/a/2821201/2450315
+    private val envRegex = Pattern.compile("\\$([a-zA-Z_]+[a-zA-Z0-9_]*)")
+    private fun evaluateEnvVars(text: String): String {
+        val buffer = StringBuffer()
+        val matcher = envRegex.matcher(text)
+        while (matcher.find()) {
+            val envName = matcher.group(1)
+            val envValue: String? = System.getenv(envName)
+            if (envValue == null) {
+                println("WARNING: $envName not found")
+            }
+            matcher.appendReplacement(buffer, envValue ?: "")
+        }
+        matcher.appendTail(buffer)
+        return buffer.toString()
+    }
+
+    private fun walkFileTree(filePath: String): List<Path> {
+        val searchDir = Paths.get(filePath).parent
+
+        return ArgsFileVisitor("glob:$filePath").walk(searchDir)
+    }
 }

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -2,6 +2,7 @@ package ftl.args
 
 import ftl.args.ArgsHelper.assertFileExists
 import ftl.args.ArgsHelper.assertGcsFileExists
+import ftl.args.ArgsHelper.evaluateFilePath
 import ftl.args.ArgsHelper.mergeYmlMaps
 import ftl.args.ArgsHelper.validateTestMethods
 import ftl.args.ArgsHelper.yamlMapper
@@ -36,8 +37,8 @@ class IosArgs(
     override val resultsHistoryName = gcloud.resultsHistoryName
 
     private val iosGcloud = iosGcloudYml.gcloud
-    val xctestrunZip = iosGcloud.test
-    val xctestrunFile = iosGcloud.xctestrunFile
+    var xctestrunZip = iosGcloud.test
+    var xctestrunFile = iosGcloud.xctestrunFile
     val xcodeVersion = iosGcloud.xcodeVersion
     val devices = iosGcloud.device
 
@@ -70,8 +71,10 @@ class IosArgs(
         if (xctestrunZip.startsWith(FtlConstants.GCS_PREFIX)) {
             assertGcsFileExists(xctestrunZip)
         } else {
+            xctestrunZip = evaluateFilePath(xctestrunZip)
             assertFileExists(xctestrunZip, "xctestrunZip")
         }
+        xctestrunFile = evaluateFilePath(xctestrunFile)
         assertFileExists(xctestrunFile, "xctestrunFile")
 
         devices.forEach { device -> assertDeviceSupported(device) }

test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt

@@ -17,6 +17,7 @@ import org.junit.contrib.java.lang.system.SystemErrRule
 import org.junit.contrib.java.lang.system.SystemOutRule
 import org.junit.rules.ExpectedException
 import org.junit.runner.RunWith
+import ftl.test.util.TestHelper.absolutePath
 
 @RunWith(FlankTestRunner::class)
 class AndroidArgsFileTest {
@@ -42,6 +43,8 @@ class AndroidArgsFileTest {
     private val testName = "class com.example.app.ExampleUiTest#testPasses"
     private val directoryToPull = "/sdcard/screenshots"
 
+    private val appApkAbsolutePath = appApkLocal.absolutePath()
+    private val testApkAbsolutePath = testApkLocal.absolutePath()
     // NOTE: Change working dir to '%MODULE_WORKING_DIR%' in IntelliJ to match gradle for this test to pass.
     @Test
     fun localConfigLoadsSuccessfully() {
@@ -58,10 +61,10 @@ class AndroidArgsFileTest {
     private fun checkConfig(args: AndroidArgs, local: Boolean) {
 
         with(args) {
-            if (local) assert(getString(testApk), testApkLocal)
+            if (local) assert(getString(testApk), getString(testApkAbsolutePath))
             else assert(testApk, testApkGcs)
 
-            if (local) assert(getString(appApk), appApkLocal)
+            if (local) assert(getString(appApk), getString(appApkAbsolutePath))
             else assert(appApk, appApkGcs)
 
             assert(autoGoogleLogin, true)

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -4,6 +4,7 @@ import com.google.common.truth.Truth.assertThat
 import ftl.cli.firebase.test.android.AndroidRunCommand
 import ftl.config.Device
 import ftl.test.util.FlankTestRunner
+import ftl.test.util.TestHelper.absolutePath
 import ftl.test.util.TestHelper.assert
 import org.junit.Rule
 import org.junit.Test
@@ -17,6 +18,8 @@ class AndroidArgsTest {
     private val appApk = "../test_app/apks/app-debug.apk"
     private val testApk = "../test_app/apks/app-debug-androidTest.apk"
     private val testErrorApk = "../test_app/apks/error-androidTest.apk"
+    private val appApkAbsolutePath = appApk.absolutePath()
+    private val testApkAbsolutePath = testApk.absolutePath()
 
     private val androidNonDefault = """
         gcloud:
@@ -114,8 +117,6 @@ class AndroidArgsTest {
     @Test
     fun androidArgs() {
         val androidArgs = AndroidArgs.load(androidNonDefault)
-        val expectedAppApk = appApk
-        val expectedTestApk = testApk
 
         with(androidArgs) {
             // GcloudYml
@@ -127,8 +128,8 @@ class AndroidArgsTest {
             assert(resultsHistoryName ?: "", "android-history")
 
             // AndroidGcloudYml
-            assert(appApk, expectedAppApk)
-            assert(testApk, expectedTestApk)
+            assert(appApk, appApkAbsolutePath)
+            assert(testApk, testApkAbsolutePath)
             assert(autoGoogleLogin, false)
             assert(useOrchestrator, false)
             assert(environmentVariables, linkedMapOf("clearPackageData" to "true", "randomEnvVar" to "false"))
@@ -174,8 +175,8 @@ AndroidArgs
       project: projectFoo
       results-history-name: android-history
       # Android gcloud
-      app: ../test_app/apks/app-debug.apk
-      test: ../test_app/apks/app-debug-androidTest.apk
+      app: $appApkAbsolutePath
+      test: $testApkAbsolutePath
       auto-google-login: false
       use-orchestrator: false
       environment-variables:
@@ -227,8 +228,8 @@ AndroidArgs
             assert(projectId, "mockProjectId")
 
             // AndroidGcloudYml
-            assert(appApk, appApk)
-            assert(testApk, testApk)
+            assert(appApk, appApkAbsolutePath)
+            assert(testApk, testApkAbsolutePath)
             assert(autoGoogleLogin, true)
             assert(useOrchestrator, true)
             assert(environmentVariables, emptyMap<String, String>())
@@ -276,8 +277,8 @@ AndroidArgs
       """
         )
 
-        assertThat(androidArgs.appApk).isEqualTo(appApk)
-        assertThat(androidArgs.testApk).isEqualTo(testApk)
+        assertThat(androidArgs.appApk).isEqualTo(appApkAbsolutePath)
+        assertThat(androidArgs.testApk).isEqualTo(testApkAbsolutePath)
     }
 
     @Test
@@ -294,7 +295,7 @@ AndroidArgs
             cli
         )
 
-        assertThat(androidArgs.appApk).isEqualTo(testApk)
+        assertThat(androidArgs.appApk).isEqualTo(testApkAbsolutePath)
         assertThat(androidArgs.cli).isEqualTo(cli)
     }
 
@@ -312,7 +313,7 @@ AndroidArgs
             cli
         )
 
-        assertThat(androidArgs.testApk).isEqualTo(appApk)
+        assertThat(androidArgs.testApk).isEqualTo(appApkAbsolutePath)
         assertThat(androidArgs.cli).isEqualTo(cli)
     }