✅ [RUMF-804] add support for session replay in E2E tests

and add a recorder test

package.json

@@ -27,6 +27,7 @@
     "rum-events-format:sync": "scripts/cli update_submodule && scripts/cli build_json2type && node scripts/generate-schema-types.js"
   },
   "devDependencies": {
+    "@types/connect-busboy": "0.0.2",
     "@types/cors": "2.8.7",
     "@types/express": "4.17.8",
     "@types/jasmine": "3.5.10",
@@ -41,6 +42,7 @@
     "ajv": "6.12.6",
     "browserstack-local": "1.4.5",
     "codecov": "3.7.1",
+    "connect-busboy": "0.0.2",
     "cors": "2.8.5",
     "emoji-name-map": "1.2.8",
     "express": "4.17.1",

packages/core/src/domain/configuration.spec.ts

@@ -30,6 +30,7 @@ describe('configuration', () => {
       expect(configuration.rumEndpoint).toEqual('<<< E2E RUM ENDPOINT >>>')
       expect(configuration.logsEndpoint).toEqual('<<< E2E LOGS ENDPOINT >>>')
       expect(configuration.internalMonitoringEndpoint).toEqual('<<< E2E INTERNAL MONITORING ENDPOINT >>>')
+      expect(configuration.sessionReplayEndpoint).toEqual('<<< E2E SESSION REPLAY ENDPOINT >>>')
     })
   })
 

packages/core/src/domain/configuration.ts

@@ -190,6 +190,7 @@ export function buildConfiguration(userConfiguration: UserConfiguration, buildEn
     configuration.internalMonitoringEndpoint = '<<< E2E INTERNAL MONITORING ENDPOINT >>>'
     configuration.logsEndpoint = '<<< E2E LOGS ENDPOINT >>>'
     configuration.rumEndpoint = '<<< E2E RUM ENDPOINT >>>'
+    configuration.sessionReplayEndpoint = '<<< E2E SESSION REPLAY ENDPOINT >>>'
   }
 
   if (transportConfiguration.buildMode === BuildMode.STAGING) {

packages/rum-recorder/src/index.ts

@@ -1,2 +1,9 @@
 export * from '@datadog/browser-rum'
 export { datadogRum } from './boot/recorder.entry'
+export {
+  Segment as internal_Segment,
+  CreationReason as internal_CreationReason,
+  RecordType as internal_RecordType,
+  IncrementalSource as internal_IncrementalSource,
+  MouseMoveRecord as internal_MouseMoveRecord,
+} from './types'

test/app/yarn.lock

@@ -2,30 +2,37 @@
 # yarn lockfile v1
 
 
-"@datadog/browser-core@1.26.2", "@datadog/browser-core@file:../../packages/core":
-  version "1.26.2"
+"@datadog/browser-core@2.1.1", "@datadog/browser-core@file:../../packages/core":
+  version "2.1.1"
   dependencies:
     tslib "^1.10.0"
 
 "@datadog/browser-logs@file:../../packages/logs":
-  version "1.26.2"
+  version "2.1.1"
   dependencies:
-    "@datadog/browser-core" "1.26.2"
+    "@datadog/browser-core" "2.1.1"
     tslib "^1.10.0"
 
 "@datadog/browser-rum-recorder@file:../../packages/rum-recorder":
-  version "1.26.2"
+  version "2.1.1"
   dependencies:
-    "@datadog/browser-core" "1.26.2"
-    "@datadog/browser-rum" "1.26.2"
+    "@datadog/browser-core" "2.1.1"
+    "@datadog/browser-rum" "2.1.1"
+    "@types/css-font-loading-module" "0.0.4"
+    rrweb-snapshot "1.0.1"
     tslib "^1.10.0"
 
-"@datadog/browser-rum@1.26.2", "@datadog/browser-rum@file:../../packages/rum":
-  version "1.26.2"
+"@datadog/browser-rum@2.1.1", "@datadog/browser-rum@file:../../packages/rum":
+  version "2.1.1"
   dependencies:
-    "@datadog/browser-core" "1.26.2"
+    "@datadog/browser-core" "2.1.1"
     tslib "^1.10.0"
 
+"@types/[email protected]":
+  version "0.0.4"
+  resolved "https://registry.yarnpkg.com/@types/css-font-loading-module/-/css-font-loading-module-0.0.4.tgz#94a835e27d1af444c65cba88523533c174463d64"
+  integrity sha512-ENdXf7MW4m9HeDojB2Ukbi7lYMIuQNBHVf98dbzaiG4EEJREBd6oleVAjrLRCrp7dm6CK1mmdmU9tcgF61acbw==
+
 "@webassemblyjs/[email protected]":
   version "1.8.5"
   resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.8.5.tgz#51b1c5fe6576a34953bf4b253df9f0d490d9e359"
@@ -1845,6 +1852,11 @@ ripemd160@^2.0.0, ripemd160@^2.0.1:
     hash-base "^3.0.0"
     inherits "^2.0.1"
 
+[email protected]:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/rrweb-snapshot/-/rrweb-snapshot-1.0.1.tgz#f3800f4deb434da02b6086533c5c38e23e7e796b"
+  integrity sha512-HKbfbTTDSTy4T0h/w9SubI1E5kL/qLiK08pJZRLA0MmGtqJ5nCTMtpEZXgCKMUncH3lb1rbiu+sDIoxKD+Anng==
+
 run-queue@^1.0.0, run-queue@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/run-queue/-/run-queue-1.0.3.tgz#e848396f057d223f24386924618e25694161ec47"

test/e2e/lib/framework/createTest.ts

@@ -34,6 +34,7 @@ type TestRunner = (testContext: TestContext) => Promise<void>
 
 class TestBuilder {
   private rumOptions: RumSetupOptions | undefined = undefined
+  private rumRecorderOptions: RumSetupOptions | undefined = undefined
   private logsOptions: LogsSetupOptions | undefined = undefined
   private head: string = ''
   private body: string = ''
@@ -46,6 +47,11 @@ class TestBuilder {
     return this
   }
 
+  withRumRecorder(rumRecorderOptions?: RumSetupOptions) {
+    this.rumRecorderOptions = { ...DEFAULT_RUM_OPTIONS, ...rumRecorderOptions }
+    return this
+  }
+
   withLogs(logsOptions?: LogsSetupOptions) {
     this.logsOptions = { ...DEFAULT_LOGS_OPTIONS, ...logsOptions }
     return this
@@ -77,6 +83,7 @@ class TestBuilder {
       head: this.head,
       logs: this.logsOptions,
       rum: this.rumOptions,
+      rumRecorder: this.rumRecorderOptions,
     }
 
     if (setups.length > 1) {
@@ -131,6 +138,7 @@ function createTestContext(servers: Servers): TestContext {
       internalMonitoring: `${servers.intake.url}/v1/input/internalMonitoring`,
       logs: `${servers.intake.url}/v1/input/logs`,
       rum: `${servers.intake.url}/v1/input/rum`,
+      sessionReplay: `${servers.intake.url}/v1/input/sessionReplay`,
     },
     events: new EventRegistry(),
   }

test/e2e/lib/framework/eventsRegistry.ts

@@ -5,14 +5,16 @@ import {
   isRumResourceEvent,
   isRumUserActionEvent,
   isRumViewEvent,
+  SerssionReplayCall,
   ServerInternalMonitoringMessage,
 } from '../types/serverEvents'
 
-type IntakeType = 'logs' | 'rum' | 'internalMonitoring'
+type IntakeType = 'logs' | 'rum' | 'internalMonitoring' | 'sessionReplay'
 
 export class EventRegistry {
   readonly rum: RumEvent[] = []
   readonly logs: LogsEvent[] = []
+  readonly sessionReplay: SerssionReplayCall[] = []
   readonly internalMonitoring: ServerInternalMonitoringMessage[] = []
 
   push(type: IntakeType, event: any) {
@@ -21,7 +23,7 @@ export class EventRegistry {
   }
 
   get count() {
-    return this.logs.length + this.rum.length + this.internalMonitoring.length
+    return this.logs.length + this.rum.length + this.internalMonitoring.length + this.sessionReplay.length
   }
 
   get rumActions() {

test/e2e/lib/framework/pageSetups.ts

@@ -15,6 +15,7 @@ export interface LogsSetupOptions {
 
 export interface SetupOptions {
   rum?: RumSetupOptions
+  rumRecorder?: RumSetupOptions
   logs?: LogsSetupOptions
   head?: string
   body?: string
@@ -56,12 +57,13 @@ n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
     `
   }
 
-  if (options.rum) {
+  const rumOptions = options.rumRecorder || options.rum
+  if (rumOptions) {
     body += html`
       <script type="text/javascript">
-        ${formatSnippet('./datadog-rum.js', 'DD_RUM')}
+        ${formatSnippet(options.rumRecorder ? './datadog-rum-recorder.js' : './datadog-rum.js', 'DD_RUM')}
         DD_RUM.onReady(function () {
-          DD_RUM.init(${formatRumOptions(options.rum)})
+          DD_RUM.init(${formatRumOptions(rumOptions)})
         })
       </script>
     `
@@ -84,11 +86,16 @@ export function bundleSetup(options: SetupOptions) {
       </script>
     `
   }
-  if (options.rum) {
+
+  const rumOptions = options.rumRecorder || options.rum
+  if (rumOptions) {
     header += html`
-      <script type="text/javascript" src="./datadog-rum.js"></script>
+      <script
+        type="text/javascript"
+        src="${options.rumRecorder ? './datadog-rum-recorder.js' : './datadog-rum.js'}"
+      ></script>
       <script type="text/javascript">
-        DD_RUM.init(${formatRumOptions(options.rum)})
+        DD_RUM.init(${formatRumOptions(rumOptions)})
       </script>
     `
   }
@@ -109,10 +116,12 @@ export function npmSetup(options: SetupOptions) {
       </script>
     `
   }
-  if (options.rum) {
+
+  const rumOptions = options.rumRecorder || options.rum
+  if (rumOptions) {
     header += html`
       <script type="text/javascript">
-        window.RUM_CONFIG = ${formatRumOptions(options.rum)}
+        window.RUM_CONFIG = ${formatRumOptions(rumOptions)}
       </script>
     `
   }
@@ -147,6 +156,7 @@ export function html(parts: ReadonlyArray<string>, ...vars: string[]) {
 function formatLogsOptions(options: LogsSetupOptions) {
   return JSON.stringify(options)
 }
+
 function formatRumOptions(options: RumSetupOptions) {
   return JSON.stringify(options).replace('"LOCATION_ORIGIN"', 'location.origin')
 }

test/e2e/lib/framework/sdkBuilds.ts

@@ -7,18 +7,24 @@ const readFile = promisify(fs.readFile)
 const ROOT = path.join(__dirname, '../../../..')
 const RUM_BUNDLE = path.join(ROOT, 'packages/rum/bundle/datadog-rum.js')
 const LOGS_BUNDLE = path.join(ROOT, 'packages/logs/bundle/datadog-logs.js')
+const RUM_RECORDER_BUNDLE = path.join(ROOT, 'packages/rum-recorder/bundle/datadog-rum-recorder.js')
 const NPM_BUNDLE = path.join(ROOT, 'test/app/dist/app.js')
 
 export interface Endpoints {
   rum: string
   logs: string
   internalMonitoring: string
+  sessionReplay: string
 }
 
 export async function buildRum(endpoints: Endpoints) {
   return replaceEndpoints(await readFile(RUM_BUNDLE), endpoints)
 }
 
+export async function buildRumRecorder(endpoints: Endpoints) {
+  return replaceEndpoints(await readFile(RUM_RECORDER_BUNDLE), endpoints)
+}
+
 export async function buildLogs(endpoints: Endpoints) {
   return replaceEndpoints(await readFile(LOGS_BUNDLE), endpoints)
 }
@@ -32,6 +38,7 @@ function replaceEndpoints(content: Buffer, endpoints: Endpoints) {
     '<<< E2E INTERNAL MONITORING ENDPOINT >>>': endpoints.internalMonitoring,
     '<<< E2E LOGS ENDPOINT >>>': endpoints.logs,
     '<<< E2E RUM ENDPOINT >>>': endpoints.rum,
+    '<<< E2E SESSION REPLAY ENDPOINT >>>': endpoints.sessionReplay,
   })
 }
 

test/e2e/lib/framework/serverApps/intake.ts

@@ -1,18 +1,76 @@
+import connectBusboy from 'connect-busboy'
 import express from 'express'
+import { createInflate } from 'zlib'
+
+import { SegmentFile, SerssionReplayCall } from '../../types/serverEvents'
 import { EventRegistry } from '../eventsRegistry'
 
 export function createIntakeServerApp(events: EventRegistry) {
   const app = express()
 
   app.use(express.text())
+  app.use(connectBusboy({ immediate: true }))
 
-  app.post('/v1/input/:endpoint', (req, res) => {
+  app.post('/v1/input/:endpoint', async (req, res) => {
     const endpoint = req.params.endpoint
     if (endpoint === 'rum' || endpoint === 'logs' || endpoint === 'internalMonitoring') {
       ;(req.body as string).split('\n').map((rawEvent) => events.push(endpoint, JSON.parse(rawEvent) as any))
     }
+
+    if (endpoint === 'sessionReplay' && req.busboy) {
+      events.push('sessionReplay', await readSessionReplay(req))
+    }
+
     res.end()
   })
 
   return app
 }
+
+async function readSessionReplay(req: express.Request): Promise<SerssionReplayCall> {
+  return new Promise((resolve, reject) => {
+    const meta: {
+      [field: string]: string
+    } = {}
+    let segmentPromise: Promise<SegmentFile>
+
+    req.busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
+      if (fieldname === 'segment') {
+        segmentPromise = readStream(file.pipe(createInflate())).then((data) => ({
+          encoding,
+          filename,
+          mimetype,
+          data: JSON.parse(data.toString()),
+        }))
+      }
+    })
+
+    req.busboy.on('field', (key: string, value: string) => {
+      meta[key] = value
+    })
+
+    req.busboy.on('finish', async () => {
+      try {
+        const segment = await segmentPromise
+        resolve({ meta, segment })
+      } catch (e) {
+        reject(e)
+      }
+    })
+  })
+}
+
+async function readStream(stream: NodeJS.ReadableStream): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    const buffers: Buffer[] = []
+    stream.on('data', (data: Buffer) => {
+      buffers.push(data)
+    })
+    stream.on('error', (error) => {
+      reject(error)
+    })
+    stream.on('end', () => {
+      resolve(Buffer.concat(buffers))
+    })
+  })
+}

test/e2e/lib/framework/serverApps/mock.ts

@@ -1,7 +1,7 @@
 import cors from 'cors'
 import express from 'express'
 import * as url from 'url'
-import { buildLogs, buildNpm, buildRum, Endpoints } from '../sdkBuilds'
+import { buildLogs, buildNpm, buildRum, buildRumRecorder, Endpoints } from '../sdkBuilds'
 
 export function createMockServerApp(endpoints: Endpoints, setup: string) {
   const app = express()
@@ -59,6 +59,10 @@ export function createMockServerApp(endpoints: Endpoints, setup: string) {
     res.header('content-type', 'application/javascript').send(await buildRum(endpoints))
   })
 
+  app.get('/datadog-rum-recorder.js', async (req, res) => {
+    res.header('content-type', 'application/javascript').send(await buildRumRecorder(endpoints))
+  })
+
   app.get('/app.js', async (req, res) => {
     res.header('content-type', 'application/javascript').send(await buildNpm(endpoints))
   })

test/e2e/lib/types/serverEvents.ts

@@ -1,4 +1,5 @@
 import { RumActionEvent, RumErrorEvent, RumEvent, RumResourceEvent, RumViewEvent } from '@datadog/browser-rum'
+import { internal_Segment as Segment } from '@datadog/browser-rum-recorder'
 
 export interface ServerInternalMonitoringMessage {
   message: string
@@ -23,3 +24,15 @@ export function isRumViewEvent(event: RumEvent): event is RumViewEvent {
 export function isRumErrorEvent(event: RumEvent): event is RumErrorEvent {
   return event.type === 'error'
 }
+
+export interface SegmentFile {
+  filename: string
+  encoding: string
+  mimetype: string
+  data: Segment
+}
+
+export interface SerssionReplayCall {
+  segment: SegmentFile
+  meta: { [key: string]: string }
+}