Try to fix e2e flakiness (#59)

* Try to fix e2e flakiness * refactor * fml * . * format * fml * . * remove dumb aware * .. * . * . * fml * . * . * .
metalbear-co · Jul 21, 2023 · 5ba0bee · 5ba0bee
1 parent f6000d7
commit 5ba0bee
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 47 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -53,4 +53,22 @@ Now open the plugin's code in IntelliJ IDEA. Create a new Gradle run configurati
 Running this configuration in debug will open a new IDE window.
 You can set breakpoints in the plugin's code in the first window, and use the plugin in the second window to reach the breakpoints.
 
-You can control which IDE is opened with a `PLATFORMTYPE` environment variable. For example, set `PLATFORMTYPE=IU` for IntelliJ IDEA Ultimate.
+You can control which IDE is opened with a `PLATFORMTYPE` environment variable. For example, set `PLATFORMTYPE=IU` for IntelliJ IDEA Ultimate.
+
+
+## Adding E2E tests
+
+We use the [intellij-ui-test-robot](https://github.com/JetBrains/intellij-ui-test-robot) to automate UI tests for the extension.
+
+- If you have made a change to the UI, to add new tests, run `./gradlew runIdeForUITests` and open a test project,
+go to [localhost:8082](http://localhost:8082) and choose the elements you want to click on by their Xpath.
+
+- To make sure the test doesn't flake in the CI, check if all the elements related to UI component have loaded by using
+ a `waitFor` fixture or if a change depends on files to be indexed, use the `dumbAware` fixture.
+
+- As a rule of thumb try to encapsulate functionality in fixtures in the `utils` folder.
+
+- To run the tests locally from scratch run, `./gradlew test` which download the latest stable pycharm IDE. 
+
+
+
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -27,15 +27,14 @@ version = properties("pluginVersion")
 repositories {
     mavenCentral()
     maven {
-        url = uri("https://packages.jetbrains.team/maven/p/ij/intellij-dependencies")
+        url = uri("https://packages.jetbrains.team/maven/p/iuia/qa-automation-maven")
     }
-
     maven {
-        url = uri("https://packages.jetbrains.team/maven/p/iuia/qa-automation-maven")
+        url = uri("https://packages.jetbrains.team/maven/p/ij/intellij-dependencies")
     }
 }
 
-val remoteRobotVersion = "0.11.19.416"
+val remoteRobotVersion = "0.11.19"
 
 dependencies {
     implementation(project(":mirrord-products-idea"))
@@ -184,8 +183,7 @@ tasks {
         // binaries to copy from $projectDir/bin to $pluginDir/bin with same path.
         // we have custom delve until delve 20 is widely used
         val binaries = listOf("macos/arm64/dlv", "macos/x86-64/dlv")
-        binaries.forEach {
-                binary ->
+        binaries.forEach { binary ->
             from(file(project.projectDir.resolve("bin").resolve(binary))) {
                 // into treats last part as directory, so need to drop it.
                 into(Paths.get(pluginName.get(), "bin", binary).parent.toString())

diff --git a/changelog.d/+e2e-fix.internal.md b/changelog.d/+e2e-fix.internal.md
@@ -0,0 +1 @@
+Fix e2e flakiness
diff --git a/src/test/kotlin/com/metalbear/mirrord/MirrordPluginTest.kt b/src/test/kotlin/com/metalbear/mirrord/MirrordPluginTest.kt
@@ -35,6 +35,7 @@ internal class MirrordPluginTest {
     init {
         StepsLogger.init()
     }
+
     companion object {
         private var ideaProcess: Process? = null
         private var tmpDir: Path = Files.createTempDirectory("launcher")
@@ -92,16 +93,22 @@ internal class MirrordPluginTest {
         }
         idea {
             step("Create config file") {
+                waitFor(ofSeconds(60)) {
+                    mirrordDropdownButton.isShowing
+                    // issue here is that elements move when git is visible
+                    git.isShowing
+                }
+                // as per the extension this doesn't need to be in the dumbAware block
+                // however, there can be a loading page which can only be ignored by the
+                // dumbAware block
                 dumbAware {
-                    waitFor(ofSeconds(30)) {
-                        mirrordDropdownButton.isShowing
-                    }
+                    mirrordDropdownButton.click()
                 }
-                mirrordDropdownButton.click()
 
                 waitFor(ofSeconds(30)) {
                     mirrordDropdownMenu.isShowing
                 }
+
                 mirrordDropdownMenu.findText("Settings").click()
 
                 editorTabs {
@@ -112,11 +119,16 @@ internal class MirrordPluginTest {
             }
 
             step("Open `app.py`") {
-                openFileByName("app.py")
-
-                editorTabs {
+                with(projectViewTree) {
                     waitFor(ofSeconds(30)) {
-                        isFileOpened("app.py")
+                        hasText("app.py")
+                        hasText(".mirrord")
+                    }
+                    findText("app.py").doubleClick()
+                    editorTabs {
+                        waitFor {
+                            isFileOpened("app.py")
+                        }
                     }
                 }
 
@@ -156,8 +168,8 @@ internal class MirrordPluginTest {
                     enableMirrord.isShowing
                     startDebugging.isShowing
                 }
+                enableMirrord.click()
                 dumbAware {
-                    enableMirrord.click()
                     startDebugging.click()
                 }
                 step("Select pod to mirror traffic from") {

diff --git a/src/test/kotlin/com/metalbear/mirrord/utils/IdeaFrame.kt b/src/test/kotlin/com/metalbear/mirrord/utils/IdeaFrame.kt
@@ -5,7 +5,6 @@ import com.intellij.remoterobot.data.RemoteComponent
 import com.intellij.remoterobot.fixtures.*
 import com.intellij.remoterobot.search.locators.byXpath
 import com.intellij.remoterobot.stepsProcessing.step
-import com.intellij.remoterobot.utils.keyboard
 import com.intellij.remoterobot.utils.waitFor
 import java.time.Duration
 
@@ -29,6 +28,15 @@ class IdeaFrame(remoteRobot: RemoteRobot, remoteComponent: RemoteComponent) :
             Duration.ofSeconds(30)
         )
 
+    val git
+        get() = find<ContainerFixture>(byXpath("//div[@visible_text='Git:' and @class='MyLabel']"), Duration.ofSeconds(30))
+
+    val projectViewTree
+        get() = find<ContainerFixture>(
+            byXpath("ProjectViewTree", "//div[@class='ProjectViewTree']"),
+            Duration.ofSeconds(60)
+        )
+
     val mirrordDropdownMenu
         get() = find<ContainerFixture>(
             byXpath("//div[@class='MyList' and (@visible_text='Disabled || Select Active Config || Configuration || Settings' or @visible_text='Disabled || Settings || Configuration')]"),
@@ -57,27 +65,6 @@ class IdeaFrame(remoteRobot: RemoteRobot, remoteComponent: RemoteComponent) :
     val xDebuggerFramesList
         get() = find<ContainerFixture>(byXpath("//div[@class='XDebuggerFramesList']"))
 
-    fun openFileByName(name: String) {
-        find<ContainerFixture>(byXpath("//div[@class='ActionMenu' and @text='Navigate']")).click()
-        waitFor {
-            findAll<ContainerFixture>(byXpath("//div[@class='ActionMenuItem' and @text='Search Everywhere' and @defaulticon='find.svg']"))
-                .isNotEmpty()
-        }
-        findAll<ContainerFixture>(byXpath("//div[@class='ActionMenuItem' and @text='Search Everywhere' and @defaulticon='find.svg']"))
-            .first()
-            .click()
-        find<ContainerFixture>(byXpath("//div[@class='SearchField' and @visible_text='Type / to see commands']")).click()
-        keyboard {
-            enterText(name)
-        }
-        var listElems = emptyList<ContainerFixture>()
-        waitFor(Duration.ofSeconds(30)) {
-            listElems = findAll<ContainerFixture>(byXpath("//div[@class='JBList']")).filter { it.hasText(name) }
-            listElems.isNotEmpty()
-        }
-        listElems.first().findText(name).click()
-    }
-
     // dumb and smart mode refer to the state of the IDE when it is indexing and not indexing respectively
     @JvmOverloads
     fun dumbAware(timeout: Duration = Duration.ofMinutes(5), function: () -> Unit) {
@@ -158,12 +145,3 @@ class StatusBar(remoteRobot: RemoteRobot, remoteComponent: RemoteComponent) :
         ).isEmpty()
     }
 }
-
-fun RemoteRobot.leftStripe(function: LeftStripe.() -> Unit) {
-    find<LeftStripe>(timeout = Duration.ofSeconds(60)).apply(function)
-}
-
-// reprsents the slim bar on the left showing bookmarks, project, etc.
-@DefaultXpath("Stripe type", "//div[@class='Stripe'][.//div[contains(@text.key, 'project.scheme')]]")
-class LeftStripe(remoteRobot: RemoteRobot, remoteComponent: RemoteComponent) :
-    CommonContainerFixture(remoteRobot, remoteComponent)