From 0ff99a3e977899a51c836113362356046fb10de7 Mon Sep 17 00:00:00 2001 From: bootstraponline Date: Sun, 11 Nov 2018 13:01:48 -0500 Subject: [PATCH] Path evaluation fixes Add ArgsFileVisitor Fix paths --- .../src/main/kotlin/ftl/args/AndroidArgs.kt | 4 +- .../main/kotlin/ftl/args/ArgsFileVisitor.kt | 52 ++++++++++++ .../src/main/kotlin/ftl/args/ArgsHelper.kt | 79 ++++++------------- .../src/main/kotlin/ftl/args/IosArgs.kt | 4 +- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 6 +- .../test/kotlin/ftl/args/ArgsHelperTest.kt | 78 +++++++++++++----- .../src/test/kotlin/ftl/args/IosArgsTest.kt | 6 +- .../test/kotlin/ftl/test/util/TestHelper.kt | 4 +- 8 files changed, 149 insertions(+), 84 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/args/ArgsFileVisitor.kt diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 504b5b0eb2..5e41ff4fef 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -86,14 +86,14 @@ class AndroidArgs( if (appApk.startsWith(FtlConstants.GCS_PREFIX)) { assertGcsFileExists(appApk) } else { - appApk = evaluateFilePath(appApk, "appApk") + appApk = evaluateFilePath(appApk) assertFileExists(appApk, "appApk") } if (testApk.startsWith(FtlConstants.GCS_PREFIX)) { assertGcsFileExists(testApk) } else { - testApk = evaluateFilePath(testApk, "testApk") + testApk = evaluateFilePath(testApk) assertFileExists(testApk, "testApk") } diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsFileVisitor.kt b/test_runner/src/main/kotlin/ftl/args/ArgsFileVisitor.kt new file mode 100644 index 0000000000..938625d75d --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/ArgsFileVisitor.kt @@ -0,0 +1,52 @@ +package ftl.args + +import ftl.util.Utils.fatalError +import java.io.IOException +import java.nio.file.FileSystems +import java.nio.file.FileVisitOption +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.LinkOption +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes +import java.util.EnumSet + +class ArgsFileVisitor(glob: String) : SimpleFileVisitor() { + private val pathMatcher = FileSystems.getDefault().getPathMatcher(glob) + val result: MutableList = mutableListOf() + + @Throws(IOException::class) + override fun visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult { + if (pathMatcher.matches(path)) { + result.add(path) + } + return FileVisitResult.CONTINUE + } + + override fun visitFileFailed(file: Path?, exc: IOException?): FileVisitResult { + fatalError("Failed to visit $file $exc") + return FileVisitResult.CONTINUE + } + + private val RECURSE = "/**" + + @Throws(java.nio.file.NoSuchFileException::class) + fun walk(searchPath: Path): List { + val searchString = searchPath.toString() + // /Users/tmp/code/flank/test_app/** => /Users/tmp/code/flank/test_app/ + val beforeGlob = Paths.get(searchString.substringBefore(RECURSE)) + // must not follow links when resolving paths or /tmp turns into /private/tmp + val realPath = beforeGlob.toRealPath(LinkOption.NOFOLLOW_LINKS) + + val searchDepth = if (searchString.contains(RECURSE)) { + Integer.MAX_VALUE + } else { + 1 + } + + Files.walkFileTree(realPath, EnumSet.of(FileVisitOption.FOLLOW_LINKS), searchDepth, this) + return this.result + } +} diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt index 612067ba23..6a2aca303d 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt @@ -20,17 +20,11 @@ import ftl.config.FtlConstants.defaultCredentialPath import ftl.gc.GcStorage import ftl.util.Utils import java.io.File -import java.io.IOException import java.math.RoundingMode import java.net.URI -import java.nio.file.FileVisitResult import java.nio.file.Files -import java.nio.file.FileSystems -import java.nio.file.SimpleFileVisitor -import java.nio.file.attribute.BasicFileAttributes import java.nio.file.Path import java.nio.file.Paths -import java.util.HashSet import java.util.regex.Pattern object ArgsHelper { @@ -54,15 +48,20 @@ object ArgsHelper { } } - fun evaluateFilePath(fileRegEx: String, name: String): String { - var file = fileRegEx.trim().replaceFirst("~", System.getProperty("user.home")) + fun evaluateFilePath(filePath: String): String { + var file = filePath.trim().replaceFirst("~", System.getProperty("user.home")) file = substituteEnvVars(file) - val searchDirectoryPath = getSearchDirectoryPath(file) - val filePaths = getAbsoluteFilePaths(searchDirectoryPath, file) + // avoid File(..).canonicalPath since that will resolve symlinks + file = Paths.get(file).toAbsolutePath().normalize().toString() + + // Avoid walking the folder's parent dir if we know it exists already. + if (File(file).exists()) return file + + val filePaths = walkFileTree(file) if (filePaths.size > 1) { - Utils.fatalError("$name multiple files found with expression: `$fileRegEx`: $filePaths") + Utils.fatalError("'$file' ($filePath) matches multiple files: $filePaths") } else if (filePaths.isEmpty()) { - Utils.fatalError("'$fileRegEx' $name doesn't exist") + Utils.fatalError("'$file' not found ($filePath)") } return filePaths[0].toAbsolutePath().toString() } @@ -157,7 +156,8 @@ object ArgsHelper { return JsonObjectParser(JSON_FACTORY).parseAndClose( Files.newInputStream(defaultCredentialPath), Charsets.UTF_8, - GenericJson::class.java)["project_id"] as String + GenericJson::class.java + )["project_id"] as String } catch (e: Exception) { println("Parsing $defaultCredentialPath failed:") println(e.printStackTrace()) @@ -173,52 +173,23 @@ object ArgsHelper { return ServiceOptions.getDefaultProjectId() ?: serviceAccountProjectId() } - private fun getSearchDirectoryPath(path: String): String { - var searchDirectoryPath = String() - val pattern = "([^*]*/)" - val matcher = Pattern.compile(pattern).matcher(path) - if (matcher.find()) { - searchDirectoryPath = matcher.group(1) - } - return searchDirectoryPath - } - + // https://stackoverflow.com/a/2821201/2450315 + private val envRegex = Pattern.compile("\\$([a-zA-Z_]+[a-zA-Z0-9_]*)") private fun substituteEnvVars(text: String): String { - val sb = StringBuffer() - // https://stackoverflow.com/a/2821201/2450315 - val pattern = "\\$([a-zA-Z_]{1,}[a-zA-Z0-9_]{0,})" - val matcher = Pattern.compile(pattern).matcher(text) + val buffer = StringBuffer() + val matcher = envRegex.matcher(text) while (matcher.find()) { - val varname = matcher.group(1) - val envValue: String = System.getenv(varname) ?: "" - matcher.appendReplacement(sb, envValue) + val envName = matcher.group(1) + val envValue = System.getenv(envName) ?: "" + matcher.appendReplacement(buffer, envValue) } - matcher.appendTail(sb) - return sb.toString() + matcher.appendTail(buffer) + return buffer.toString() } - private fun getAbsoluteFilePaths(searchDir: String, globPath: String): List { - val maxDepth = if (globPath.contains("/*")) Integer.MAX_VALUE else 1 - val glob = "glob:$globPath" - val paths = ArrayList() - val pathMatcher = FileSystems.getDefault().getPathMatcher(glob) + private fun walkFileTree(filePath: String): List { + val searchDir = Paths.get(filePath).parent - Files.walkFileTree(Paths.get(searchDir), HashSet(), maxDepth, object : SimpleFileVisitor() { - - @Throws(IOException::class) - override fun visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult { - if (pathMatcher.matches(path)) { - paths.add(path) - } - return FileVisitResult.CONTINUE - } - - @Throws(IOException::class) - override fun visitFileFailed(file: Path, exc: IOException?): FileVisitResult { - Utils.fatalError("'$file' doesn't exist") - return FileVisitResult.CONTINUE - } - }) - return paths + return ArgsFileVisitor("glob:$filePath").walk(searchDir) } } diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index b8458b08e5..f7b881f58f 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -71,10 +71,10 @@ class IosArgs( if (xctestrunZip.startsWith(FtlConstants.GCS_PREFIX)) { assertGcsFileExists(xctestrunZip) } else { - xctestrunZip = evaluateFilePath(xctestrunZip, "xctestrunZip") + xctestrunZip = evaluateFilePath(xctestrunZip) assertFileExists(xctestrunZip, "xctestrunZip") } - xctestrunFile = evaluateFilePath(xctestrunFile, "xctestrunFile") + xctestrunFile = evaluateFilePath(xctestrunFile) assertFileExists(xctestrunFile, "xctestrunFile") devices.forEach { device -> assertDeviceSupported(device) } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 6142513437..8bc6d99fb3 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -3,12 +3,12 @@ package ftl.args import com.google.common.truth.Truth.assertThat import ftl.config.Device import ftl.test.util.FlankTestRunner +import ftl.test.util.TestHelper.absolutePath import ftl.test.util.TestHelper.assert import org.junit.Rule import org.junit.Test import org.junit.rules.ExpectedException import org.junit.runner.RunWith -import java.io.File @RunWith(FlankTestRunner::class) class AndroidArgsTest { @@ -16,8 +16,8 @@ class AndroidArgsTest { private val appApk = "../test_app/apks/app-debug.apk" private val testApk = "../test_app/apks/app-debug-androidTest.apk" private val testErrorApk = "../test_app/apks/error-androidTest.apk" - private val appApkAbsolutePath = File(appApk).absolutePath - private val testApkAbsolutePath = File(testApk).absolutePath + private val appApkAbsolutePath = appApk.absolutePath() + private val testApkAbsolutePath = testApk.absolutePath() private val androidNonDefault = """ gcloud: diff --git a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt index 3e98363f4d..efbf003470 100644 --- a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt @@ -12,12 +12,18 @@ import ftl.args.yml.IosGcloudYml import ftl.test.util.FlankTestRunner import org.junit.Rule import org.junit.Test +import org.junit.contrib.java.lang.system.EnvironmentVariables import org.junit.contrib.java.lang.system.SystemErrRule import org.junit.rules.ExpectedException import org.junit.runner.RunWith import java.io.File -import org.junit.contrib.java.lang.system.EnvironmentVariables -import java.lang.RuntimeException +import java.io.IOException +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes @RunWith(FlankTestRunner::class) class ArgsHelperTest { @@ -170,40 +176,74 @@ class ArgsHelperTest { @Test fun evaluateBlobInFilePath() { - val testApkRelativePath = "../test_app/apks/app-debug-androidTest.apk" val testApkBlobPath = "../test_app/**/app-debug-*.apk" + val actual = ArgsHelper.evaluateFilePath(testApkBlobPath) + + val testApkRelativePath = "../test_app/apks/app-debug-androidTest.apk" + val expected = testApkRelativePath.absolutePath() + + assertThat(actual).isEqualTo(expected) + } + + private fun makeTmpFile(filePath: String): String { + val file = File(filePath) + file.parentFile.mkdirs() + + file.apply { + createNewFile() + deleteOnExit() + } - assertThat(File(testApkRelativePath).absolutePath) - .isEqualTo(ArgsHelper.evaluateFilePath(testApkBlobPath, "test")) + return file.absolutePath + } + + private fun String.absolutePath(): String { + return Paths.get(this).toAbsolutePath().normalize().toString() + } + + @Test + fun tmpTest() { + Files.walkFileTree(Paths.get("/tmp"), object : SimpleFileVisitor() { + @Throws(IOException::class) + override fun visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult { + // hits '/tmp' once and doesn't iterate through the files + return FileVisitResult.CONTINUE + } + + override fun visitFileFailed(file: Path?, exc: IOException?): FileVisitResult { + return FileVisitResult.CONTINUE + } + + override fun preVisitDirectory(dir: Path?, attrs: BasicFileAttributes?): FileVisitResult { + return FileVisitResult.CONTINUE + } + }) } @Test fun evaluateTildeInFilePath() { - val testApkPath = "~/flank_test_app/random.xctestrun" - val file = File(testApkPath.replaceFirst("~", System.getProperty("user.home"))) - file.getParentFile().mkdirs() - file.createNewFile() + val expected = makeTmpFile("/tmp/random.xctestrun") - assertThat(file.absolutePath) - .isEqualTo(ArgsHelper.evaluateFilePath(testApkPath, "xctestrun-file")) + val inputPath = "~/../../tmp/random.xctestrun" + val actual = ArgsHelper.evaluateFilePath(inputPath) + + assertThat(actual).isEqualTo(expected) } @Test fun evaluateEnvVarInFilePath() { environmentVariables.set("TEST_APK_DIR", "test_app/apks") val testApkPath = "../\$TEST_APK_DIR/app-debug-androidTest.apk" - val expectedPath = File("../test_app/apks/app-debug-androidTest.apk").absolutePath - val file = File(testApkPath.replaceFirst("~", System.getProperty("user.home"))) - file.getParentFile().mkdirs() - file.createNewFile() + val actual = ArgsHelper.evaluateFilePath(testApkPath) + + val expected = "../test_app/apks/app-debug-androidTest.apk".absolutePath() - assertThat(expectedPath) - .isEqualTo(ArgsHelper.evaluateFilePath(testApkPath, "test")) + assertThat(actual).isEqualTo(expected) } - @Test(expected = RuntimeException::class) + @Test(expected = java.nio.file.NoSuchFileException::class) fun evaluateInvalidFilePath() { val testApkPath = "~/flank_test_app/invalid_path/app-debug-*.xctestrun" - ArgsHelper.evaluateFilePath(testApkPath, "test") + ArgsHelper.evaluateFilePath(testApkPath) } } diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 0e1c716640..49fdf2b93a 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -8,13 +8,13 @@ import ftl.args.yml.IosGcloudYml import ftl.args.yml.IosGcloudYmlParams import ftl.config.Device import ftl.test.util.FlankTestRunner +import ftl.test.util.TestHelper.absolutePath import ftl.test.util.TestHelper.assert import org.junit.Rule import org.junit.Test import org.junit.contrib.java.lang.system.SystemErrRule import org.junit.rules.ExpectedException import org.junit.runner.RunWith -import java.io.File @RunWith(FlankTestRunner::class) class IosArgsTest { @@ -22,8 +22,8 @@ class IosArgsTest { private val testPath = "./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExample.zip" private val xctestrunFile = "./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun" - val xctestrunFileAbsolutePath = File(xctestrunFile).absolutePath - val testAbsolutePath = File(testPath).absolutePath + private val xctestrunFileAbsolutePath = xctestrunFile.absolutePath() + private val testAbsolutePath = testPath.absolutePath() private val iosNonDefault = """ gcloud: results-bucket: mockBucket diff --git a/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt b/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt index 56007f92bc..dbf979a720 100644 --- a/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt +++ b/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt @@ -10,8 +10,10 @@ object TestHelper { Truth.assertThat(actual).isEqualTo(expected) fun getPath(path: String): Path = - Paths.get(path).normalize().toAbsolutePath() + Paths.get(path).toAbsolutePath().normalize() fun getString(path: String): String = getPath(path).toString() + + fun String.absolutePath(): String = Paths.get(this).toAbsolutePath().normalize().toString() }