feat: Improve UX when Test Replay upload fails due to slow upload (#3…

…0235) * new error message for stream stall upload failures * new unknown error msg, extract printProtocolUploadError for testing, consolidatd args for stream stall msg * rename env more appropriately, add sampling interval param to putProtocolArtifact * default to 5000, override with app capture protocol value, override that with env var * update snapshots * fix math in upload stall error msg * changelog * update gql schema w/ new error msg, fix protocol tscheck * update test to use fn for app capture supplied interval * increase default stream stall interval to 10 seconds * typos * snapshots * rename env var * do not use user-supplied interval if it parses to NaN * use the more standard makeErr for unknown/uncategorized upload error in visual snapshots * rearrange upload stall error message * fix typo * changelog --------- Co-authored-by: Jennifer Shehane <[email protected]>
cypress-io · Sep 25, 2024 · 1d84532 · 1d84532 · cypress-bot · Sep 25, 2024
1 parent fbe51fc
commit 1d84532
Show file tree

Hide file tree

Showing 20 changed files with 455 additions and 66 deletions.
diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md
@@ -1,8 +1,12 @@
 <!-- See the ../guides/writing-the-cypress-changelog.md for details on writing the changelog. -->
-## 13.14.3
+## 13.15.0
 
 _Released 9/24/2024 (PENDING)_
 
+**Features:**
+
+- Cypress now displays more actionable errors when a Test Replay upload takes too long, and more verbose messages when uncategorized errors occur during the upload process. Addressed in [#30235](https://github.com/cypress-io/cypress/pull/30235).
+
 **Bugfixes:**
 
 - Fixed an issue where Firefox was incorrectly mutating the state of click events on checkboxes after Firefox version `129` and up. Addressed in [#30245](https://github.com/cypress-io/cypress/pull/30245).

diff --git a/...ges/errors/__snapshot-html__/CLOUD_PROTOCOL_UPLOAD_AGGREGATE_ERROR - withSystemError.html b/...ges/errors/__snapshot-html__/CLOUD_PROTOCOL_UPLOAD_AGGREGATE_ERROR - withSystemError.html
diff --git a/...CLOUD_PROTOCOL_UPLOAD_NEWORK_FAILURE.html → ...LOUD_PROTOCOL_UPLOAD_NETWORK_FAILURE.html b/...CLOUD_PROTOCOL_UPLOAD_NEWORK_FAILURE.html → ...LOUD_PROTOCOL_UPLOAD_NETWORK_FAILURE.html
diff --git a/packages/errors/__snapshot-html__/CLOUD_PROTOCOL_UPLOAD_STREAM_STALL_FAILURE.html b/packages/errors/__snapshot-html__/CLOUD_PROTOCOL_UPLOAD_STREAM_STALL_FAILURE.html
diff --git a/packages/errors/__snapshot-html__/CLOUD_PROTOCOL_UPLOAD_UNKNOWN_ERROR.html b/packages/errors/__snapshot-html__/CLOUD_PROTOCOL_UPLOAD_UNKNOWN_ERROR.html
diff --git a/packages/errors/src/errors.ts b/packages/errors/src/errors.ts
@@ -574,6 +574,17 @@ export const AllCypressErrors = {
         This error will not affect or change the exit code.
     `
   },
+  CLOUD_PROTOCOL_UPLOAD_UNKNOWN_ERROR: (error: Error) => {
+    return errTemplate`\
+        Warning: We encountered an error while uploading the Test Replay recording of this spec.
+
+        These results will not display Test Replay recordings.
+
+        This error will not affect or change the exit code.
+
+        ${fmt.highlightSecondary(error)}
+    `
+  },
   CLOUD_PROTOCOL_UPLOAD_HTTP_FAILURE: (error: Error & { url: string, status: number, statusText: string, responseBody: string }) => {
     return errTemplate`\
         Warning: We encountered an HTTP error while uploading the Test Replay recording for this spec.
@@ -586,7 +597,7 @@ export const AllCypressErrors = {
         
         ${fmt.highlightTertiary(error.responseBody)}`
   },
-  CLOUD_PROTOCOL_UPLOAD_NEWORK_FAILURE: (error: Error & { url: string }) => {
+  CLOUD_PROTOCOL_UPLOAD_NETWORK_FAILURE: (error: Error & { url: string }) => {
     return errTemplate`\
         Warning: We encountered a network error while uploading the Test Replay recording for this spec.
         
@@ -598,14 +609,29 @@ export const AllCypressErrors = {
 
         ${fmt.highlightSecondary(error)}`
   },
+  CLOUD_PROTOCOL_UPLOAD_STREAM_STALL_FAILURE: (error: Error & { chunkSizeKB: number, maxActivityDwellTime: number }) => {
+    const kbpsThreshold = (error.chunkSizeKB * 8) / (error.maxActivityDwellTime / 1000)
+
+    return errTemplate`\
+        Warning: We encountered slow network conditions while uploading the Test Replay recording for this spec.
+
+        The upload transfer rate fell below ${fmt.highlightSecondary(`${kbpsThreshold}kbps`)} over a sampling period of ${fmt.highlightSecondary(`${error.maxActivityDwellTime}ms`)}.
+
+        To prevent long CI execution durations, this Test Replay recording will not be uploaded.
+
+        The results for this spec will not display Test Replay recordings.
+
+        If this error occurs often, the sampling period may be configured by setting the ${fmt.highlightSecondary('CYPRESS_TEST_REPLAY_UPLOAD_SAMPLING_INTERVAL')} environment variable to a higher value than ${fmt.stringify(error.maxActivityDwellTime)}.
+    `
+  },
   CLOUD_PROTOCOL_UPLOAD_AGGREGATE_ERROR: (error: {
     errors: (Error & { kind?: 'SystemError', url: string } | Error & { kind: 'HttpError', url: string, status?: string, statusText?: string, responseBody?: string })[]
   }) => {
     if (error.errors.length === 1) {
       const firstError = error.errors[0]
 
       if (firstError?.kind === 'SystemError') {
-        return AllCypressErrors.CLOUD_PROTOCOL_UPLOAD_NEWORK_FAILURE(firstError as Error & { url: string })
+        return AllCypressErrors.CLOUD_PROTOCOL_UPLOAD_NETWORK_FAILURE(firstError as Error & { url: string })
       }
 
       return AllCypressErrors.CLOUD_PROTOCOL_UPLOAD_HTTP_FAILURE(error.errors[0] as Error & { url: string, status: number, statusText: string, responseBody: string})

diff --git a/packages/errors/test/unit/visualSnapshotErrors_spec.ts b/packages/errors/test/unit/visualSnapshotErrors_spec.ts
@@ -685,7 +685,7 @@ describe('visual error templates', () => {
         default: [err],
       }
     },
-    CLOUD_PROTOCOL_UPLOAD_NEWORK_FAILURE: () => {
+    CLOUD_PROTOCOL_UPLOAD_NETWORK_FAILURE: () => {
       // @ts-expect-error
       const err: Error & { url: string } = makeErr()
 
@@ -695,6 +695,17 @@ describe('visual error templates', () => {
         default: [err],
       }
     },
+    CLOUD_PROTOCOL_UPLOAD_STREAM_STALL_FAILURE: () => {
+      // @ts-expect-error
+      const err: Error & { chunkSizeKB: number, maxActivityDwellTime: number } = new Error('stream stall')
+
+      err.chunkSizeKB = 64
+      err.maxActivityDwellTime = 5000
+
+      return {
+        default: [err],
+      }
+    },
     CLOUD_PROTOCOL_UPLOAD_AGGREGATE_ERROR: () => {
       // @ts-expect-error
       const aggregateError: Error & { errors: any[] } = makeErr()
@@ -719,6 +730,13 @@ describe('visual error templates', () => {
         withSystemError: [aggregateErrorWithSystemError],
       }
     },
+    CLOUD_PROTOCOL_UPLOAD_UNKNOWN_ERROR: () => {
+      const error = makeErr()
+
+      return {
+        default: [error],
+      }
+    },
     CLOUD_RECORD_KEY_NOT_VALID: () => {
       return {
         default: ['record-key-123', 'project-id-123'],

diff --git a/packages/graphql/schemas/schema.graphql b/packages/graphql/schemas/schema.graphql
@@ -1153,7 +1153,9 @@ enum ErrorTypeEnum {
   CLOUD_PROTOCOL_INITIALIZATION_FAILURE
   CLOUD_PROTOCOL_UPLOAD_AGGREGATE_ERROR
   CLOUD_PROTOCOL_UPLOAD_HTTP_FAILURE
-  CLOUD_PROTOCOL_UPLOAD_NEWORK_FAILURE
+  CLOUD_PROTOCOL_UPLOAD_NETWORK_FAILURE
+  CLOUD_PROTOCOL_UPLOAD_STREAM_STALL_FAILURE
+  CLOUD_PROTOCOL_UPLOAD_UNKNOWN_ERROR
   CLOUD_RECORD_KEY_NOT_VALID
   CLOUD_RUN_GROUP_NAME_NOT_UNIQUE
   CLOUD_STALE_RUN

diff --git a/packages/server/lib/cloud/api/put_protocol_artifact.ts b/packages/server/lib/cloud/api/put_protocol_artifact.ts
@@ -7,23 +7,18 @@ import { putFetch, ParseKinds } from '../network/put_fetch'
 import { isRetryableError } from '../network/is_retryable_error'
 const debug = Debug('cypress:server:cloud:api:protocol-artifact')
 
-// the upload will get canceled if the stream pipeline
-// stalls (does not push data to the `fetch` sink) for more
-// than 5 seconds
-const MAX_ACTIVITY_DWELL_TIME = 5000
-
 export const _delay = linearDelay(500)
 
 export const putProtocolArtifact = asyncRetry(
-  async (artifactPath: string, maxFileSize: number, destinationUrl: string) => {
+  async (artifactPath: string, maxFileSize: number, destinationUrl: string, uploadMonitorSamplingRate: number) => {
     debug(`Atttempting to upload Test Replay archive from ${artifactPath} to ${destinationUrl})`)
     const { size } = await fsAsync.stat(artifactPath)
 
     if (size > maxFileSize) {
       throw new Error(`Spec recording too large: artifact is ${size} bytes, limit is ${maxFileSize} bytes`)
     }
 
-    const activityMonitor = new StreamActivityMonitor(MAX_ACTIVITY_DWELL_TIME)
+    const activityMonitor = new StreamActivityMonitor(uploadMonitorSamplingRate)
     const fileStream = fs.createReadStream(artifactPath)
     const controller = activityMonitor.getController()
 

diff --git a/packages/server/lib/cloud/artifacts/print_protocol_upload_error.ts b/packages/server/lib/cloud/artifacts/print_protocol_upload_error.ts
@@ -0,0 +1,24 @@
+import { HttpError } from '../network/http_error'
+import { SystemError } from '../network/system_error'
+import { StreamStalledError } from '../upload/stream_stalled_error'
+import Debug from 'debug'
+import * as errors from '../../errors'
+
+const debug = Debug('cypress:server:cloud:artifacts')
+
+export const printProtocolUploadError = (error: Error) => {
+  debug('protocol error: %O', error)
+  // eslint-disable-next-line no-console
+  console.log('')
+  if ((error as AggregateError).errors) {
+    errors.warning('CLOUD_PROTOCOL_UPLOAD_AGGREGATE_ERROR', error as AggregateError)
+  } else if (HttpError.isHttpError(error)) {
+    errors.warning('CLOUD_PROTOCOL_UPLOAD_HTTP_FAILURE', error)
+  } else if (SystemError.isSystemError(error)) {
+    errors.warning('CLOUD_PROTOCOL_UPLOAD_NETWORK_FAILURE', error)
+  } else if (StreamStalledError.isStreamStalledError(error)) {
+    errors.warning('CLOUD_PROTOCOL_UPLOAD_STREAM_STALL_FAILURE', error)
+  } else {
+    errors.warning('CLOUD_PROTOCOL_UPLOAD_UNKNOWN_ERROR', error)
+  }
+}
diff --git a/packages/server/lib/cloud/artifacts/upload_artifacts.ts b/packages/server/lib/cloud/artifacts/upload_artifacts.ts
@@ -12,7 +12,7 @@ import { createScreenshotArtifactBatch } from './screenshot_artifact'
 import { createVideoArtifact } from './video_artifact'
 import { createProtocolArtifact, composeProtocolErrorReportFromOptions } from './protocol_artifact'
 import { HttpError } from '../network/http_error'
-import { SystemError } from '../network/system_error'
+import { printProtocolUploadError } from './print_protocol_upload_error'
 
 const debug = Debug('cypress:server:cloud:artifacts')
 
@@ -230,20 +230,7 @@ export const uploadArtifacts = async (options: UploadArtifactOptions) => {
     if (postUploadProtocolFatalError && postUploadProtocolFatalError.captureMethod === 'uploadCaptureArtifact') {
       const error = postUploadProtocolFatalError.error
 
-      debug('protocol error: %O', error)
-      if ((error as AggregateError).errors) {
-        // eslint-disable-next-line no-console
-        console.log('')
-        errors.warning('CLOUD_PROTOCOL_UPLOAD_AGGREGATE_ERROR', postUploadProtocolFatalError.error as AggregateError)
-      } else if (HttpError.isHttpError(error)) {
-        // eslint-disable-next-line no-console
-        console.log('')
-        errors.warning('CLOUD_PROTOCOL_UPLOAD_HTTP_FAILURE', error)
-      } else if (SystemError.isSystemError(error)) {
-        // eslint-disable-next-line no-console
-        console.log('')
-        errors.warning('CLOUD_PROTOCOL_UPLOAD_NEWORK_FAILURE', error)
-      }
+      printProtocolUploadError(error)
     }
 
     // there is no upload results entry for protocol if we did not attempt to upload protocol due to a fatal error

diff --git a/packages/server/lib/cloud/protocol.ts b/packages/server/lib/cloud/protocol.ts
@@ -26,6 +26,8 @@ const DELETE_DB = !process.env.CYPRESS_LOCAL_PROTOCOL_PATH
 
 export const DB_SIZE_LIMIT = 5000000000
 
+export const DEFAULT_STREAM_SAMPLING_INTERVAL = 10000
+
 const dbSizeLimit = () => {
   return env.get('CYPRESS_INTERNAL_SYSTEM_TESTS') === '1' ?
     200 : DB_SIZE_LIMIT
@@ -320,7 +322,12 @@ export class ProtocolManager implements ProtocolManagerShape {
     debug(`uploading %s to %s with a file size of %s`, filePath, uploadUrl, fileSize)
 
     try {
-      await putProtocolArtifact(filePath, dbSizeLimit(), uploadUrl)
+      const environmentSuppliedInterval = parseInt(process.env.CYPRESS_TEST_REPLAY_UPLOAD_SAMPLING_INTERVAL || '', 10)
+      const samplingInterval = !Number.isNaN(environmentSuppliedInterval) ?
+        environmentSuppliedInterval :
+        this._protocol.uploadStallSamplingInterval ? this._protocol.uploadStallSamplingInterval() : DEFAULT_STREAM_SAMPLING_INTERVAL
+
+      await putProtocolArtifact(filePath, dbSizeLimit(), uploadUrl, samplingInterval)
 
       return {
         fileSize,