✅ [RUM-253] add E2E tests related to compression (#2416)

* ✅♻️ [RUM-253] move segment file infos into a property

* ✅ [RUM-253] read encoded requests and expose the 'encoding' on IntakeRequest objects

* ✅ [RUM-253] add E2E tests related to compression

test/e2e/lib/framework/intakeRegistry.ts

@@ -4,27 +4,31 @@ import type { TelemetryEvent, TelemetryErrorEvent, TelemetryConfigurationEvent }
 import type { BrowserSegment } from '@datadog/browser-rum/src/types'
 import type { BrowserSegmentMetadataAndSegmentSizes } from '@datadog/browser-rum/src/domain/segmentCollection'
 
+type BaseIntakeRequest = {
+  isBridge: boolean
+  encoding: string | null
+}
+
 export type LogsIntakeRequest = {
   intakeType: 'logs'
-  isBridge: boolean
   events: LogsEvent[]
-}
+} & BaseIntakeRequest
 
 export type RumIntakeRequest = {
   intakeType: 'rum'
-  isBridge: boolean
   events: Array<RumEvent | TelemetryEvent>
-}
+} & BaseIntakeRequest
 
 export type ReplayIntakeRequest = {
   intakeType: 'replay'
-  isBridge: false
   segment: BrowserSegment
   metadata: BrowserSegmentMetadataAndSegmentSizes
-  filename: string
-  encoding: string
-  mimetype: string
-}
+  segmentFile: {
+    filename: string
+    encoding: string
+    mimetype: string
+  }
+} & BaseIntakeRequest
 
 export type IntakeRequest = LogsIntakeRequest | RumIntakeRequest | ReplayIntakeRequest
 

test/e2e/lib/framework/serverApps/intake.ts

@@ -1,16 +1,23 @@
-import { createInflate } from 'zlib'
+import { createInflate, inflateSync } from 'zlib'
 import https from 'https'
 import connectBusboy from 'connect-busboy'
 import express from 'express'
 
 import cors from 'cors'
 import type { BrowserSegmentMetadataAndSegmentSizes } from '@datadog/browser-rum/src/domain/segmentCollection'
 import type { BrowserSegment } from '@datadog/browser-rum/src/types'
-import type { IntakeRegistry, IntakeRequest, ReplayIntakeRequest } from '../intakeRegistry'
+import type {
+  IntakeRegistry,
+  IntakeRequest,
+  LogsIntakeRequest,
+  ReplayIntakeRequest,
+  RumIntakeRequest,
+} from '../intakeRegistry'
 
 interface IntakeRequestInfos {
   isBridge: boolean
   intakeType: IntakeRequest['intakeType']
+  encoding: string | null
 }
 
 export function createIntakeServerApp(intakeRegistry: IntakeRegistry) {
@@ -42,45 +49,60 @@ function computeIntakeRequestInfos(req: express.Request): IntakeRequestInfos {
   if (!ddforward) {
     throw new Error('ddforward is missing')
   }
+  const { pathname, searchParams } = new URL(ddforward, 'https://example.org')
+
+  const encoding = req.headers['content-encoding'] || searchParams.get('dd-evp-encoding')
 
   if (req.query.bridge === 'true') {
     const eventType = req.query.event_type
     return {
       isBridge: true,
+      encoding,
       intakeType: eventType === 'log' ? 'logs' : 'rum',
     }
   }
 
   let intakeType: IntakeRequest['intakeType']
-  // ddforward = /api/v2/rum?key=value
-  const endpoint = ddforward.split(/[/?]/)[3]
+  // pathname = /api/v2/rum
+  const endpoint = pathname.split(/[/?]/)[3]
   if (endpoint === 'logs' || endpoint === 'rum' || endpoint === 'replay') {
     intakeType = endpoint
   } else {
     throw new Error("Can't find intake type")
   }
   return {
     isBridge: false,
+    encoding,
     intakeType,
   }
 }
 
-async function readIntakeRequest(req: express.Request, infos: IntakeRequestInfos): Promise<IntakeRequest> {
-  if (infos.intakeType === 'replay') {
-    return readReplayIntakeRequest(req)
-  }
+function readIntakeRequest(req: express.Request, infos: IntakeRequestInfos): Promise<IntakeRequest> {
+  return infos.intakeType === 'replay'
+    ? readReplayIntakeRequest(req, infos as IntakeRequestInfos & { intakeType: 'replay' })
+    : readRumOrLogsIntakeRequest(req, infos as IntakeRequestInfos & { intakeType: 'rum' | 'logs' })
+}
+
+async function readRumOrLogsIntakeRequest(
+  req: express.Request,
+  infos: IntakeRequestInfos & { intakeType: 'rum' | 'logs' }
+): Promise<RumIntakeRequest | LogsIntakeRequest> {
+  const rawBody = await readStream(req)
+  const encodedBody = infos.encoding === 'deflate' ? inflateSync(rawBody) : rawBody
 
   return {
-    intakeType: infos.intakeType,
-    isBridge: infos.isBridge,
-    events: (await readStream(req))
+    ...infos,
+    events: encodedBody
       .toString('utf-8')
       .split('\n')
       .map((line): any => JSON.parse(line)),
   }
 }
 
-function readReplayIntakeRequest(req: express.Request): Promise<ReplayIntakeRequest> {
+function readReplayIntakeRequest(
+  req: express.Request,
+  infos: IntakeRequestInfos & { intakeType: 'replay' }
+): Promise<ReplayIntakeRequest> {
   return new Promise((resolve, reject) => {
     let segmentPromise: Promise<{
       encoding: string
@@ -108,11 +130,11 @@ function readReplayIntakeRequest(req: express.Request): Promise<ReplayIntakeRequ
 
     req.busboy.on('finish', () => {
       Promise.all([segmentPromise, metadataPromise])
-        .then(([segmentEntry, metadata]) => ({
-          intakeType: 'replay' as const,
-          isBridge: false as const,
+        .then(([{ segment, ...segmentFile }, metadata]) => ({
+          ...infos,
+          segmentFile,
           metadata,
-          ...segmentEntry,
+          segment,
         }))
         .then(resolve, reject)
     })

test/e2e/scenario/recorder/recorder.scenario.ts

@@ -34,7 +34,11 @@ describe('recorder', () => {
       await flushEvents()
 
       expect(intakeRegistry.replaySegments.length).toBe(1)
-      const { segment, metadata, encoding, filename, mimetype } = intakeRegistry.replayRequests[0]
+      const {
+        segment,
+        metadata,
+        segmentFile: { encoding, filename, mimetype },
+      } = intakeRegistry.replayRequests[0]
       expect(metadata).toEqual({
         application: { id: jasmine.stringMatching(UUID_RE) },
         creation_reason: 'init',

test/e2e/scenario/transport.scenario.ts

@@ -0,0 +1,55 @@
+import { ExperimentalFeature } from '@datadog/browser-core'
+import { createTest, flushEvents } from '../lib/framework'
+import { getBrowserName, getPlatformName, withBrowserLogs } from '../lib/helpers/browser'
+
+describe('transport', () => {
+  describe('data compression', () => {
+    createTest('send RUM data compressed')
+      .withRum({
+        enableExperimentalFeatures: [ExperimentalFeature.COMPRESS_BATCH],
+      })
+      .run(async ({ intakeRegistry }) => {
+        await flushEvents()
+
+        expect(intakeRegistry.rumRequests.length).toBe(2)
+
+        const plainRequest = intakeRegistry.rumRequests.find((request) => request.encoding === null)
+        const deflateRequest = intakeRegistry.rumRequests.find((request) => request.encoding === 'deflate')
+
+        // The last view update should be sent without compression
+        expect(plainRequest!.events).toEqual([
+          jasmine.objectContaining({
+            type: 'view',
+          }),
+        ])
+
+        // Other data should be sent encoded
+        expect(deflateRequest!.events.length).toBeGreaterThan(0)
+      })
+
+    // Ignore this test on Safari desktop and Firefox because the Worker actually works even with
+    // CSP restriction.
+    // TODO: Remove this condition when upgrading to Safari 15 and Firefox 99
+    if (!((getBrowserName() === 'safari' && getPlatformName() === 'macos') || getBrowserName() === 'firefox')) {
+      createTest("displays a message if the worker can't be started")
+        .withRum({
+          enableExperimentalFeatures: [ExperimentalFeature.COMPRESS_BATCH],
+        })
+        .withBasePath('/no-blob-worker-csp')
+        .run(async ({ intakeRegistry }) => {
+          await flushEvents()
+
+          // Some non-deflate request can still be sent because on some browsers the Worker fails
+          // asynchronously
+          expect(intakeRegistry.rumRequests.filter((request) => request.encoding === 'deflate').length).toBe(0)
+
+          await withBrowserLogs((logs) => {
+            const failedToStartLog = logs.find((log) => log.message.includes('Datadog RUM failed to start'))
+            const cspDocLog = logs.find((log) => log.message.includes('Please make sure CSP'))
+            expect(failedToStartLog).withContext("'Failed to start' log").toBeTruthy()
+            expect(cspDocLog).withContext("'CSP doc' log").toBeTruthy()
+          })
+        })
+    }
+  })
+})