ci(workflow): enable test trace upload (vercel#51107)

### What?

WEB-1150. 

This PR is an attempt to upload next.js's test results into datadog test trace to track its status. 

Originally it tried to use automatic trace injection (dd-trace/ci/init). However, due to some of custom environments / setup we use it was not compatible out of the box. Instead, this PR injects a new test reportert to generate junit report, then upload it at once at the end of the testing pipeline.

The reporter is configured to run when necessary variables set, local run should not be affected. 

One thing to note is this report will not count retry results, as it'll create duplicated test entry with multiple results since we runts jest per individual test. This'll allow to detect flaky test easier, but also it means we'll get bit of skewed test results compare to the real world as first failure will be accounted as test fail immediately.

.github/workflows/build_and_test.yml

@@ -23,6 +23,7 @@ env:
   # canary next-swc binaries in the monorepo
   NEXT_SKIP_NATIVE_POSTINSTALL: 1
   DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
+  DATADOG_TRACE_NEXTJS_TEST: 'true'
   DD_ENV: 'ci'
   TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
   NEXT_TEST_JOB: 1
@@ -181,6 +182,27 @@ jobs:
       afterBuild: pnpm playwright install && BROWSER_NAME=firefox node run-tests.js test/integration/production/test/index.test.js && BROWSER_NAME=safari NEXT_TEST_MODE=start node run-tests.js -c 1 test/integration/production/test/index.test.js test/e2e/basepath.test.ts && BROWSER_NAME=safari DEVICE_NAME='iPhone XR' node run-tests.js -c 1 test/production/prerender-prefetch/index.test.ts
     secrets: inherit
 
+  report-test-results:
+    needs: ['test-dev', 'test-prod', 'test-integration', 'test-turbopack-dev']
+    runs-on: [self-hosted, linux, x64]
+    name: upload test trace to datadog
+    # We'll not block the CI if the upload fails
+    continue-on-error: true
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: Test trace reports
+          path: test
+      - name: Upload traces
+        run: |
+          ls -al ./test
+          npm install -g [email protected] @datadog/[email protected]
+          jrm ./nextjs-test-result-junit.xml "test/test-junit-report/**/*.xml"
+          jrm ./turbopack-test-result-junit.xml "test/turbopack-test-junit-report/**/*.xml"
+          # Put a separate tag for the tests with turbopack to distinguish between same test names
+          DD_ENV=ci datadog-ci junit upload --tags test.type:nextjs --service nextjs ./nextjs-test-result-junit.xml
+          DD_ENV=ci datadog-ci junit upload --tags test.type:turbopack --service nextjs ./turbopack-test-result-junit.xml
+
   tests-pass:
     needs:
       [

.github/workflows/build_reusable.yml

@@ -53,6 +53,7 @@ env:
   # canary next-swc binaries in the monorepo
   NEXT_SKIP_NATIVE_POSTINSTALL: 1
   DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
+  DATADOG_TRACE_NEXTJS_TEST: 'true'
   DD_ENV: 'ci'
   TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
   NEXT_TEST_JOB: 1
@@ -140,3 +141,13 @@ jobs:
         with:
           name: turbo run summary
           path: .turbo/runs
+
+      - name: Upload test reports artifact
+        uses: actions/upload-artifact@v3
+        if: ${{ inputs.afterBuild }}
+        with:
+          name: Test trace reports
+          path: |
+            test/test-junit-report
+            test/turbopack-test-junit-report
+          if-no-files-found: ignore

jest.config.js

@@ -25,5 +25,33 @@ const customJestConfig = {
   },
 }
 
+// Check if the environment variable is set to enable test trace,
+// Insert a reporter to generate a junit report to upload.
+// This won't count for the retry to avoid duplicated test being reported twice
+// - which means our test trace will report test results for the flaky test as failed without retry.
+const shouldEnableTestTrace =
+  process.env.DATADOG_API_KEY &&
+  process.env.DATADOG_TRACE_NEXTJS_TEST &&
+  !process.env.IS_RETRY
+
+if (shouldEnableTestTrace) {
+  if (!customJestConfig.reporters) {
+    customJestConfig.reporters = ['default']
+  }
+
+  const outputDirectory = process.env.__INTERNAL_NEXT_DEV_TEST_TURBO_DEV
+    ? '<rootDir>/turbopack-test-junit-report'
+    : '<rootDir>/test-junit-report'
+  customJestConfig.reporters.push([
+    'jest-junit',
+    {
+      outputDirectory,
+      reportTestSuiteErrors: 'true',
+      uniqueOutputName: 'true',
+      outputName: 'nextjs-test-junit',
+    },
+  ])
+}
+
 // createJestConfig is exported in this way to ensure that next/jest can load the Next.js config which is async
 module.exports = createJestConfig(customJestConfig)

package.json

@@ -152,6 +152,7 @@
     "isomorphic-unfetch": "3.0.0",
     "jest": "27.0.6",
     "jest-extended": "1.2.1",
+    "jest-junit": "16.0.0",
     "json5": "2.2.3",
     "ky": "0.19.1",
     "ky-universal": "0.6.0",

run-tests.js

@@ -322,33 +322,6 @@ async function main() {
 
       const shouldRecordTestWithReplay = process.env.RECORD_REPLAY && isRetry
 
-      let shouldEnableTestTrace = false
-      let traceInitPath = null
-
-      try {
-        traceInitPath = require.resolve('dd-trace/ci/init')
-        // Enable trace only if tracer (dd-trace) can be resolved
-        shouldEnableTestTrace =
-          process.env.DATADOG_API_KEY && process.env.DATADOG_TRACE_NEXTJS_TEST
-      } catch (e) {
-        shouldEnableTestTrace = false
-      }
-
-      const traceEnv =
-        shouldEnableTestTrace && traceInitPath
-          ? {
-              DD_API_KEY: process.env.DATADOG_API_KEY,
-              DD_CIVISIBILITY_AGENTLESS_ENABLED: 1,
-              DD_ENV: 'ci',
-              DD_SERVICE: 'nextjs',
-              NODE_OPTIONS: `-r ${traceInitPath}`,
-            }
-          : {}
-
-      if (shouldEnableTestTrace) {
-        console.log(`Running test with Datadog tracing enabled`)
-      }
-
       const child = spawn(
         jestPath,
         [
@@ -368,7 +341,7 @@ async function main() {
           stdio: ['ignore', 'pipe', 'pipe'],
           env: {
             ...process.env,
-            ...traceEnv,
+            IS_RETRY: isRetry ? 'true' : undefined,
             RECORD_REPLAY: shouldRecordTestWithReplay,
             // run tests in headless mode by default
             HEADLESS: 'true',

test/.gitignore

@@ -2,3 +2,5 @@
 .vscode
 
 e2e/**/tsconfig.json
+test-junit-report/
+turbopack-test-junit-report/