test: change some skipped tests to use retries

.circleci/config.yml

@@ -27,8 +27,7 @@ mainBuildFilters: &mainBuildFilters
     branches:
       only:
         - develop
-        - fix-ci-deps
-        - issue-23843_electron_21_upgrade
+        - retry-flake
 
 # usually we don't build Mac app - it takes a long time
 # but sometimes we want to really confirm we are doing the right thing
@@ -37,8 +36,7 @@ macWorkflowFilters: &darwin-workflow-filters
   when:
     or:
     - equal: [ develop, << pipeline.git.branch >> ]
-    - equal: [ 'correct-dashboard-results', << pipeline.git.branch >> ]
-    - equal: [ 'issue-23843_electron_21_upgrade', << pipeline.git.branch >> ]
+    - equal: [ 'retry-flake', << pipeline.git.branch >> ]
     - matches:
           pattern: "-release$"
           value: << pipeline.git.branch >>
@@ -47,7 +45,7 @@ linuxArm64WorkflowFilters: &linux-arm64-workflow-filters
   when:
     or:
     - equal: [ develop, << pipeline.git.branch >> ]
-    - equal: [ 'issue-23843_electron_21_upgrade', << pipeline.git.branch >> ]
+    - equal: [ 'retry-flake', << pipeline.git.branch >> ]
     - matches:
           pattern: "-release$"
           value: << pipeline.git.branch >>
@@ -65,8 +63,7 @@ windowsWorkflowFilters: &windows-workflow-filters
   when:
     or:
     - equal: [ develop, << pipeline.git.branch >> ]
-    - equal: [ linux-arm64, << pipeline.git.branch >> ]
-    - equal: [ 'issue-23843_electron_21_upgrade', << pipeline.git.branch >> ]
+    - equal: [ retry-flake, << pipeline.git.branch >> ]
     - matches:
           pattern: "-release$"
           value: << pipeline.git.branch >>
@@ -131,7 +128,7 @@ commands:
       - run:
           name: Check current branch to persist artifacts
           command: |
-            if [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "issue-23843_electron_21_upgrade" ]]; then
+            if [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "retry-flake" ]]; then
               echo "Not uploading artifacts or posting install comment for this branch."
               circleci-agent step halt
             fi

npm/webpack-dev-server/cypress/e2e/angular.cy.ts

@@ -77,7 +77,7 @@ for (const project of WEBPACK_REACT) {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23455
-    it.skip('should detect new spec', () => {
+    it('should detect new spec', { retries: 15 }, () => {
       cy.visitApp()
 
       cy.withCtx(async (ctx) => {

npm/webpack-dev-server/cypress/e2e/next.cy.ts

@@ -76,7 +76,7 @@ for (const project of WEBPACK_REACT) {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23417
-    it.skip('should detect new spec', () => {
+    it('should detect new spec', { retries: 15 }, () => {
       cy.visitApp()
 
       cy.withCtx(async (ctx) => {
@@ -94,7 +94,7 @@ for (const project of WEBPACK_REACT) {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23417
-    it.skip('should allow import of global styles in support file', () => {
+    it('should allow import of global styles in support file', { retries: 15 }, () => {
       cy.visitApp()
       cy.contains('styles.cy.js').click()
       cy.waitForSpecToFinish({ passCount: 1 })

npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts

@@ -71,7 +71,7 @@ for (const project of PROJECTS) {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23455
-    it.skip('should detect new spec', () => {
+    it('should detect new spec', { retries: 15 }, () => {
       cy.visitApp()
 
       cy.withCtx(async (ctx) => {

packages/app/cypress/e2e/cypress-in-cypress-component.cy.ts

@@ -79,7 +79,7 @@ describe('Cypress In Cypress CT', { viewportWidth: 1500, defaultCommandTimeout:
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23159
-    it.skip('redirects to the specs list with error if a spec is not found', () => {
+    it('redirects to the specs list with error if a spec is not found', { retries: 15 }, () => {
       cy.visitApp()
       const { title, intro, explainer } = defaultMessages.specPage.noSpecError
       const badFilePath = 'src/DoesNotExist.spec.js'

packages/app/cypress/e2e/cypress-in-cypress-run-mode.cy.ts

@@ -34,7 +34,7 @@ describe('Cypress In Cypress - run mode', { viewportWidth: 1200 }, () => {
   })
 
   // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23306
-  it.skip('component testing run mode spec runner header is correct', () => {
+  it('component testing run mode spec runner header is correct', { retries: 15 }, () => {
     cy.scaffoldProject('cypress-in-cypress')
     cy.findBrowsers()
     cy.openProject('cypress-in-cypress')

packages/app/cypress/e2e/cypress-in-cypress.cy.ts

@@ -93,7 +93,7 @@ describe('Cypress in Cypress', { viewportWidth: 1500, defaultCommandTimeout: 100
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23307
-    it.skip(`scales the AUT correctly in ${testingType}`, () => {
+    it(`scales the AUT correctly in ${testingType}`, { retries: 15 }, () => {
       const assertNoScaleShown = () => {
         // check that no message about scale % is shown,
         // meaning the AUT is at 100% scale

packages/app/cypress/e2e/specs_list_e2e.cy.ts

@@ -246,7 +246,7 @@ describe('App: Spec List (E2E)', () => {
       })
 
       // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23305
-      it.skip('saves the filter when navigating to a spec and back', function () {
+      it('saves the filter when navigating to a spec and back', { retries: 15 }, function () {
         const targetSpecFile = 'accounts_list.spec.js'
 
         clearSearchAndType(targetSpecFile)

packages/app/cypress/e2e/specs_list_latest_runs.cy.ts

@@ -675,7 +675,7 @@ describe('App/Cloud Integration - Latest runs and Average duration', { viewportW
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23419
-    it.skip('shows offline alert then hides it after coming online', () => {
+    it('shows offline alert then hides it after coming online', { retries: 15 }, () => {
       cy.findByTestId('offline-alert')
       .should('contain.text', defaultMessages.specPage.offlineWarning.title)
       .and('contain.text', defaultMessages.specPage.offlineWarning.explainer)

packages/app/cypress/e2e/specs_list_no_git_repo.cy.ts

@@ -7,7 +7,7 @@ describe('Spec List - Last updated with no git info', () => {
   })
 
   // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23474
-  it.skip('shows no icon and file system timestamp for files', () => {
+  it('shows no icon and file system timestamp for files', { retries: 15 }, () => {
     cy.get('[data-cy-row="blank-contents.spec.js"] [data-cy="git-info-row"] svg')
     .should('not.exist')
 

packages/data-context/test/unit/sources/GitDataSource.spec.ts

@@ -43,8 +43,10 @@ describe('GitDataSource', () => {
     sinon.restore()
   })
 
-  // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317
-  it.skip(`gets correct status for files on ${os.platform()}`, async () => {
+  it(`gets correct status for files on ${os.platform()}`, async function () {
+    // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317
+    this.retries(15)
+
     const onBranchChange = sinon.stub()
     const onGitInfoChange = sinon.stub()
     const onError = sinon.stub()

packages/driver/cypress/e2e/commands/actions/type_special_chars.cy.js

@@ -85,7 +85,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
 
   context('{{}', () => {
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-    it.skip('sets which and keyCode to 219', (done) => {
+    it('sets which and keyCode to 219', { retries: 15 }, (done) => {
       cy.$$(':text:first').on('keydown', (e) => {
         expect(e.which).to.eq(219)
         expect(e.keyCode).to.eq(219)
@@ -238,7 +238,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-    it.skip('sets which and keyCode to 8 and does not fire keypress events', (done) => {
+    it('sets which and keyCode to 8 and does not fire keypress events', { retries: 15 }, (done) => {
       cy.$$(':text:first').on('keypress', () => {
         done('should not have received keypress')
       })
@@ -255,7 +255,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-    it.skip('does not fire textInput event', (done) => {
+    it('does not fire textInput event', { retries: 15 }, (done) => {
       cy.$$(':text:first').on('textInput', (e) => {
         done(new Error('textInput should not have fired'))
       })
@@ -266,7 +266,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-    it.skip('can prevent default backspace movement', (done) => {
+    it('can prevent default backspace movement', { retries: 15 }, (done) => {
       cy.$$(':text:first').on('keydown', (e) => {
         if (e.keyCode === 8) {
           e.preventDefault()
@@ -581,7 +581,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-    it.skip('can prevent default del movement', (done) => {
+    it('can prevent default del movement', { retries: 15 }, (done) => {
       cy.$$(':text:first').on('keydown', (e) => {
         if (e.keyCode === 46) {
           e.preventDefault()
@@ -1240,7 +1240,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
 
     // https://github.com/cypress-io/cypress/issues/3405
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-    it.skip('does fire input event when text inserted', (done) => {
+    it('does fire input event when text inserted', { retries: 15 }, (done) => {
       cy.$$('[contenteditable]:first').on('input', (e) => {
         done()
       })
@@ -1281,7 +1281,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
 
     context('1 input, no \'submit\' elements', () => {
       // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-      it.skip('triggers form submit', function (done) {
+      it('triggers form submit', { retries: 15 }, function (done) {
         this.foo = {}
 
         this.$forms.find('#single-input').submit((e) => {
@@ -1470,7 +1470,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => {
 
     context('2 inputs, 1 \'submit\' element input[type=submit]', () => {
       // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160
-      it.skip('triggers form submit', function (done) {
+      it('triggers form submit', { retries: 15 }, function (done) {
         this.$forms.find('#multiple-inputs-and-input-submit').submit((e) => {
           e.preventDefault()
 

packages/driver/cypress/e2e/commands/cookies.cy.js

@@ -436,7 +436,7 @@ describe('src/cy/commands/cookies', () => {
   })
 
   // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444
-  context.skip('#setCookie', () => {
+  context('#setCookie', { retries: 15 }, () => {
     beforeEach(() => {
       cy.stub(Cypress.utils, 'addTwentyYears').returns(12345)
     })
@@ -510,8 +510,9 @@ describe('src/cy/commands/cookies', () => {
     })
 
     describe('timeout', () => {
-      // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444
-      it.skip('sets timeout to Cypress.config(responseTimeout)', {
+      it('sets timeout to Cypress.config(responseTimeout)', {
+        // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444
+        retries: 15,
         responseTimeout: 2500,
       }, () => {
         Cypress.automation.resolves(null)
@@ -524,7 +525,7 @@ describe('src/cy/commands/cookies', () => {
       })
 
       // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444
-      it.skip('can override timeout', () => {
+      it('can override timeout', { retries: 15 }, () => {
         Cypress.automation.resolves(null)
 
         const timeout = cy.spy(Promise.prototype, 'timeout')
@@ -535,7 +536,7 @@ describe('src/cy/commands/cookies', () => {
       })
 
       // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444
-      it.skip('clears the current timeout and restores after success', () => {
+      it('clears the current timeout and restores after success', { retries: 15 }, () => {
         Cypress.automation.resolves(null)
 
         cy.timeout(100)
@@ -568,7 +569,7 @@ describe('src/cy/commands/cookies', () => {
       })
 
       // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444
-      it.skip('logs once on error', function (done) {
+      it('logs once on error', { retries: 15 }, function (done) {
         const error = new Error('some err message')
 
         error.name = 'foo'
@@ -589,7 +590,7 @@ describe('src/cy/commands/cookies', () => {
       })
 
       // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444
-      it.skip('throws after timing out', function (done) {
+      it('throws after timing out', { retries: 15 }, function (done) {
         Cypress.automation.resolves(Promise.delay(1000))
 
         cy.on('fail', (err) => {

packages/driver/cypress/e2e/commands/navigation.cy.js

@@ -272,7 +272,7 @@ describe('src/cy/commands/navigation', () => {
   })
 
   // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23308
-  context.skip('#go', () => {
+  context('#go', { retries: 15 }, () => {
     // TODO: fix this
     it('sets timeout to Cypress.config(pageLoadTimeout)', {
       pageLoadTimeout: 4567,
@@ -846,7 +846,7 @@ describe('src/cy/commands/navigation', () => {
 
     // https://github.com/cypress-io/cypress/issues/1311
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23201
-    it.skip('window immediately resolves and doesn\'t reload when visiting the same URL with hashes', () => {
+    it('window immediately resolves and doesn\'t reload when visiting the same URL with hashes', { retries: 15 }, () => {
       const onLoad = cy.stub()
 
       cy