Update catalog to use projectId

test_runner/src/main/kotlin/ftl/android/AndroidCatalog.kt

@@ -1,6 +1,7 @@
 package ftl.android
 
 import com.google.api.services.testing.model.AndroidDevice
+import com.google.api.services.testing.model.AndroidDeviceCatalog
 import ftl.gc.GcTesting
 import ftl.http.executeWithRetry
 
@@ -9,16 +10,43 @@ import ftl.http.executeWithRetry
  * to validate device configuration
  */
 object AndroidCatalog {
-    private val androidDeviceCatalog by lazy {
-        GcTesting.get.testEnvironmentCatalog().get("android").executeWithRetry().androidDeviceCatalog
+    private val catalogMap: MutableMap<String, AndroidDeviceCatalog> = mutableMapOf()
+    private val modelMap: MutableMap<String, List<String>> = mutableMapOf()
+    private val versionMap: MutableMap<String, List<String>> = mutableMapOf()
+
+    private fun androidDeviceCatalog(projectId: String): AndroidDeviceCatalog {
+        val cached = catalogMap[projectId]
+        if (cached != null) return cached
+
+        val newCatalog = GcTesting.get.testEnvironmentCatalog()
+            .get("android")
+            .setProjectId(projectId)
+            .executeWithRetry().androidDeviceCatalog
+        catalogMap[projectId] = newCatalog
+        return newCatalog
     }
 
-    val androidModelIds by lazy { androidDeviceCatalog.models.map { it.id } }
-    val androidVersionIds by lazy { androidDeviceCatalog.versions.map { it.id } }
+    fun androidModelIds(projectId: String): List<String> {
+        val cached = modelMap[projectId]
+        if (cached != null) return cached
+
+        val newModels = androidDeviceCatalog(projectId).models.map { it.id }
+        modelMap[projectId] = newModels
+        return newModels
+    }
+
+    fun androidVersionIds(projectId: String): List<String> {
+        val cached = versionMap[projectId]
+        if (cached != null) return cached
+
+        val newVersions = androidDeviceCatalog(projectId).versions.map { it.id }
+        versionMap[projectId] = newVersions
+        return newVersions
+    }
 
-    fun supportedDeviceConfig(modelId: String, versionId: String): DeviceConfigCheck {
-        val foundModel = androidDeviceCatalog.models.find { it.id == modelId } ?: return UnsupportedModelId
-        if (!androidVersionIds.contains(versionId)) return UnsupportedVersionId
+    fun supportedDeviceConfig(modelId: String, versionId: String, projectId: String): DeviceConfigCheck {
+        val foundModel = androidDeviceCatalog(projectId).models.find { it.id == modelId } ?: return UnsupportedModelId
+        if (!androidVersionIds(projectId).contains(versionId)) return UnsupportedVersionId
 
         val supportedVersionIds = foundModel.supportedVersionIds
         supportedVersionIds?.let {
@@ -28,9 +56,9 @@ object AndroidCatalog {
         return SupportedDeviceConfig
     }
 
-    fun isVirtualDevice(device: AndroidDevice?): Boolean {
+    fun isVirtualDevice(device: AndroidDevice?, projectId: String): Boolean {
         val modelId = device?.androidModelId ?: return false
-        val form = androidDeviceCatalog.models.find { it.id == modelId }?.form ?: "PHYSICAL"
+        val form = androidDeviceCatalog(projectId).models.find { it.id == modelId }?.form ?: "PHYSICAL"
         return form == "VIRTUAL"
     }
 }

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -101,11 +101,11 @@ class AndroidArgs(
     }
 
     private fun assertDeviceSupported(device: Device) {
-        when (val deviceConfigTest = AndroidCatalog.supportedDeviceConfig(device.model, device.version)) {
+        when (val deviceConfigTest = AndroidCatalog.supportedDeviceConfig(device.model, device.version, this.project)) {
             SupportedDeviceConfig -> {
             }
-            UnsupportedModelId -> throw RuntimeException("Unsupported model id, '${device.model}'\nSupported model ids: ${AndroidCatalog.androidModelIds}")
-            UnsupportedVersionId -> throw RuntimeException("Unsupported version id, '${device.version}'\nSupported Version ids: ${AndroidCatalog.androidVersionIds}")
+            UnsupportedModelId -> throw RuntimeException("Unsupported model id, '${device.model}'\nSupported model ids: ${AndroidCatalog.androidModelIds(this.project)}")
+            UnsupportedVersionId -> throw RuntimeException("Unsupported version id, '${device.version}'\nSupported Version ids: ${AndroidCatalog.androidVersionIds(this.project)}")
             is IncompatibleModelVersion -> throw RuntimeException("Incompatible model, '${device.model}', and version, '${device.version}'\nSupported version ids for '${device.model}': $deviceConfigTest")
         }
     }

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -20,7 +20,6 @@ import ftl.config.Device
 import ftl.config.FtlConstants
 import ftl.ios.IosCatalog
 import ftl.ios.Xctestrun
-import ftl.util.Utils
 import ftl.util.Utils.fatalError
 import java.nio.file.Files
 import java.nio.file.Path
@@ -95,14 +94,14 @@ class IosArgs(
 
     private fun assertXcodeSupported(xcodeVersion: String?) {
         if (xcodeVersion == null) return
-        if (!IosCatalog.supportedXcode(xcodeVersion)) {
-            Utils.fatalError(("Xcode $xcodeVersion is not a supported Xcode version"))
+        if (!IosCatalog.supportedXcode(xcodeVersion, this.project)) {
+            fatalError(("Xcode $xcodeVersion is not a supported Xcode version"))
         }
     }
 
     private fun assertDeviceSupported(device: Device) {
-        if (!IosCatalog.supportedDevice(device.model, device.version)) {
-            Utils.fatalError("iOS ${device.version} on ${device.model} is not a supported device")
+        if (!IosCatalog.supportedDevice(device.model, device.version, this.project)) {
+            fatalError("iOS ${device.version} on ${device.model} is not a supported device")
         }
     }
 

test_runner/src/main/kotlin/ftl/ios/IosCatalog.kt

@@ -1,5 +1,6 @@
 package ftl.ios
 
+import com.google.api.services.testing.model.IosDeviceCatalog
 import ftl.gc.GcTesting
 import ftl.http.executeWithRetry
 
@@ -9,17 +10,34 @@ import ftl.http.executeWithRetry
  * note:  500 Internal Server Error is returned on invalid model id/version
  **/
 object IosCatalog {
-    private val xcodeVersions by lazy {
-        iosDeviceCatalog.xcodeVersions.map { it.version }
+    private val catalogMap: MutableMap<String, IosDeviceCatalog> = mutableMapOf()
+    private val xcodeMap: MutableMap<String, List<String>> = mutableMapOf()
+
+    fun xcodeVersions(projectId: String): List<String> {
+        val cached = xcodeMap[projectId]
+        if (cached != null) return cached
+
+        val newVersions = iosDeviceCatalog(projectId).xcodeVersions.map { it.version }
+        xcodeMap[projectId] = newVersions
+        return newVersions
     }
 
-    private val iosDeviceCatalog by lazy {
+    // Device catalogMap is different depending on the project id
+    fun iosDeviceCatalog(projectId: String): IosDeviceCatalog {
+        val cached = catalogMap[projectId]
+        if (cached != null) return cached
+
         try {
-            GcTesting.get.testEnvironmentCatalog().get("ios").executeWithRetry().iosDeviceCatalog
+            val newCatalog = GcTesting.get.testEnvironmentCatalog()
+                .get("ios")
+                .setProjectId(projectId)
+                .executeWithRetry().iosDeviceCatalog
+            catalogMap[projectId] = newCatalog
+            return newCatalog
         } catch (e: java.lang.Exception) {
             throw java.lang.RuntimeException(
                 """
-Unable to access the test environment catalog. Firebase Test Lab for iOS is currently in beta.
+Unable to access the test environment catalogMap. Firebase Test Lab for iOS is currently in beta.
 Request access for your project via the following form:
   https://goo.gl/forms/wAxbiNEP2pxeIRG82
 
@@ -32,12 +50,12 @@ If you are still having issues, please email [email protected] for sup
         }
     }
 
-    fun supportedXcode(version: String): Boolean {
-        return xcodeVersions.contains(version)
+    fun supportedXcode(version: String, projectId: String): Boolean {
+        return xcodeVersions(projectId).contains(version)
     }
 
-    fun supportedDevice(modelId: String, versionId: String): Boolean {
-        val model = iosDeviceCatalog.models.find { it.id == modelId }
+    fun supportedDevice(modelId: String, versionId: String, projectId: String): Boolean {
+        val model = iosDeviceCatalog(projectId).models.find { it.id == modelId }
         return model?.supportedVersionIds?.contains(versionId) ?: false
     }
 }

test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt

@@ -84,7 +84,7 @@ class SavedMatrix(matrix: TestMatrix) {
 
             val billableMinutes = Billing.billableMinutes(testTimeSeconds)
 
-            if (AndroidCatalog.isVirtualDevice(it.environment?.androidDevice)) {
+            if (AndroidCatalog.isVirtualDevice(it.environment?.androidDevice, matrix.projectId ?: "")) {
                 billableVirtualMinutes += billableMinutes
             } else {
                 billablePhysicalMinutes += billableMinutes

test_runner/src/test/kotlin/Tmp.kt

@@ -1,4 +1,3 @@
-import com.google.api.services.testing.model.TestMatrix
 import com.google.api.services.testing.model.ToolResultsStep
 import com.google.api.services.toolresults.model.Step
 import com.google.api.services.toolresults.model.TestSuiteOverview
@@ -15,6 +14,7 @@ import java.nio.file.Files
 import java.nio.file.Paths
 import kotlin.system.exitProcess
 
+@Suppress("UnusedPrivateMember")
 object Tmp {
 
     fun Step.testSuiteName(): String {
@@ -93,7 +93,6 @@ object Tmp {
         val overview = result.testExecutionStep.testSuiteOverviews.first()
         println("overview name: ${overview.name}") // null on android
 
-
         val testCases = mutableListOf<JUnitTestCase>()
         tests.testCases.forEach { testCase ->
             // null on android, EarlGreyExampleSwiftTests on iOS

test_runner/src/test/kotlin/ftl/android/AndroidCatalogTest.kt

@@ -10,19 +10,21 @@ import org.mockito.Mockito
 @RunWith(FlankTestRunner::class)
 class AndroidCatalogTest {
 
+    private val projectId = ""
+
     @Test
     fun androidModelIds() {
-        assertThat(AndroidCatalog.androidModelIds).isNotEmpty()
+        assertThat(AndroidCatalog.androidModelIds(projectId)).isNotEmpty()
     }
 
     @Test
     fun supportedDeviceConfig() {
-        assertThat(AndroidCatalog.supportedDeviceConfig("ios", "23")).isEqualTo(UnsupportedModelId)
-        assertThat(AndroidCatalog.supportedDeviceConfig("NexusLowRes", "twenty-three")).isEqualTo(UnsupportedVersionId)
-        assertThat(AndroidCatalog.supportedDeviceConfig("NexusLowRes", "21")).isEqualTo(IncompatibleModelVersion)
-        assertThat(AndroidCatalog.supportedDeviceConfig("NexusLowRes", "23")).isEqualTo(SupportedDeviceConfig)
-        assertThat(AndroidCatalog.supportedDeviceConfig("brokenModel", "23")).isEqualTo(UnsupportedModelId)
-        assertThat(AndroidCatalog.supportedDeviceConfig("does not exist", "23")).isEqualTo(UnsupportedModelId)
+        assertThat(AndroidCatalog.supportedDeviceConfig("ios", "23", projectId)).isEqualTo(UnsupportedModelId)
+        assertThat(AndroidCatalog.supportedDeviceConfig("NexusLowRes", "twenty-three", projectId)).isEqualTo(UnsupportedVersionId)
+        assertThat(AndroidCatalog.supportedDeviceConfig("NexusLowRes", "21", projectId)).isEqualTo(IncompatibleModelVersion)
+        assertThat(AndroidCatalog.supportedDeviceConfig("NexusLowRes", "23", projectId)).isEqualTo(SupportedDeviceConfig)
+        assertThat(AndroidCatalog.supportedDeviceConfig("brokenModel", "23", projectId)).isEqualTo(UnsupportedModelId)
+        assertThat(AndroidCatalog.supportedDeviceConfig("does not exist", "23", projectId)).isEqualTo(UnsupportedModelId)
     }
 
     @Test
@@ -32,29 +34,29 @@ class AndroidCatalogTest {
         val shamu = AndroidDevice()
         shamu.androidModelId = "shamu"
 
-        assertThat(AndroidCatalog.isVirtualDevice(nexus)).isEqualTo(true)
-        assertThat(AndroidCatalog.isVirtualDevice(shamu)).isEqualTo(false)
-        assertThat(AndroidCatalog.isVirtualDevice(null)).isEqualTo(false)
+        assertThat(AndroidCatalog.isVirtualDevice(nexus, projectId)).isEqualTo(true)
+        assertThat(AndroidCatalog.isVirtualDevice(shamu, projectId)).isEqualTo(false)
+        assertThat(AndroidCatalog.isVirtualDevice(null, projectId)).isEqualTo(false)
     }
 
     @Test
     fun isVirtualDeviceNullModel() {
         val mockDevice = Mockito.mock(AndroidDevice::class.java)
         Mockito.`when`(mockDevice.androidModelId).thenReturn(null)
-        assertThat(AndroidCatalog.isVirtualDevice(mockDevice)).isEqualTo(false)
+        assertThat(AndroidCatalog.isVirtualDevice(mockDevice, projectId)).isEqualTo(false)
     }
 
     @Test
     fun isVirtualDeviceUnknownModel() {
         val mockDevice = Mockito.mock(AndroidDevice::class.java)
         Mockito.`when`(mockDevice.androidModelId).thenReturn("zz")
-        assertThat(AndroidCatalog.isVirtualDevice(mockDevice)).isEqualTo(false)
+        assertThat(AndroidCatalog.isVirtualDevice(mockDevice, projectId)).isEqualTo(false)
     }
 
     @Test
     fun isVirtualDeviceBrokenModel() {
         val brokenModel = AndroidDevice()
         brokenModel.androidModelId = "brokenModel"
-        assertThat(AndroidCatalog.isVirtualDevice(brokenModel)).isEqualTo(false)
+        assertThat(AndroidCatalog.isVirtualDevice(brokenModel, projectId)).isEqualTo(false)
     }
 }

test_runner/src/test/kotlin/ftl/ios/IosCatalogTest.kt

@@ -8,16 +8,18 @@ import org.junit.runner.RunWith
 @RunWith(FlankTestRunner::class)
 class IosCatalogTest {
 
+    private val projectId = ""
+
     @Test
     fun supportedDevice() {
-        assertThat(IosCatalog.supportedDevice("bogus", "11.2")).isFalse()
-        assertThat(IosCatalog.supportedDevice("iphone8", "bogus")).isFalse()
-        assertThat(IosCatalog.supportedDevice("iphone8", "11.2")).isTrue()
+        assertThat(IosCatalog.supportedDevice("bogus", "11.2", projectId)).isFalse()
+        assertThat(IosCatalog.supportedDevice("iphone8", "bogus", projectId)).isFalse()
+        assertThat(IosCatalog.supportedDevice("iphone8", "11.2", projectId)).isTrue()
     }
 
     @Test
     fun supportedXcode() {
-        assertThat(IosCatalog.supportedXcode("9.2")).isTrue()
-        assertThat(IosCatalog.supportedXcode("0.1")).isFalse()
+        assertThat(IosCatalog.supportedXcode("9.2", projectId)).isTrue()
+        assertThat(IosCatalog.supportedXcode("0.1", projectId)).isFalse()
     }
 }