Merge pull request #819 from gemini-testing/HERMIONE-1276.fix_x_req_id

feat: generate "X-Request-ID" header for each browser request

README.md

@@ -1089,7 +1089,7 @@ module.exports = (hermione) => {
 Allows to use a custom `http`/`https`/`http2` [agent](https://www.npmjs.com/package/got#agent) to make requests. Default value is `null` (it means that will be used default http-agent from got).
 
 ####  headers
-Allows to set custom [headers](https://github.com/sindresorhus/got/blob/main/documentation/2-options.md#headers) to pass into every webdriver request. These headers aren't passed into browser request. Read more about this option in [wdio](https://github.com/sindresorhus/got/blob/main/documentation/2-options.md#headers). Default value is `{'X-Request-ID': '<some-uniq-request-id>'}`. One `X-Request-ID` is generated for the entire test run. It can be useful if you manage the cloud with browsers yourself and collect logs with requests.
+Allows to set custom [headers](https://github.com/sindresorhus/got/blob/main/documentation/2-options.md#headers) to pass into every webdriver request. These headers aren't passed into browser request. Read more about this option in [wdio](https://github.com/sindresorhus/got/blob/main/documentation/2-options.md#headers). Default value is `null`.
 
 ####  transformRequest
 Allows to intercept [HTTP request options](https://github.com/sindresorhus/got#options) before a WebDriver request is made. Default value is `null`. If function is passed then it takes `RequestOptions` as the first argument and should return modified `RequestOptions`. Example:
@@ -1098,6 +1098,20 @@ Allows to intercept [HTTP request options](https://github.com/sindresorhus/got#o
 (RequestOptions) => RequestOptions
 ```
 
+In runtime a unique `X-Request-ID` header is generated for each browser request which consists of `${FIRST_X_REQ_ID}__${LAST_X_REQ_ID}`, where:
+- `FIRST_X_REQ_ID` - unique uuid for each test (different for each retry), allows to find all requests related to a single test run;
+- `LAST_X_REQ_ID` - unique uuid for each browser request, allows to find one unique request in one test (together with `FIRST_X_REQ_ID`).
+
+Header `X-Request-ID` can be useful if you manage the cloud with browsers yourself and collect logs with requests. Real-world example: `2f31ffb7-369d-41f4-bbb8-77744615d2eb__e8d011d8-bb76-42b9-b80e-02f03b8d6fe1`.
+
+To override generated `X-Request-ID` to your own value you need specify it in `transformRequest` handler. Example:
+
+```javascript
+transformRequest: (req) => {
+    req.handler["X-Request-ID"] = "my_x_req_id";
+}
+```
+
 ####  transformResponse
 Allows to intercept [HTTP response object](https://github.com/sindresorhus/got#response) after a WebDriver response has arrived. Default value is `null`. If function is passed then it takes `Response` (original response object) as the first and `RequestOptions` as the second argument. Should return modified `Response`. Example:
 

package.json

@@ -73,7 +73,6 @@
     "uglifyify": "^3.0.4",
     "urijs": "^1.19.11",
     "url-join": "^4.0.1",
-    "uuid": "^9.0.1",
     "webdriverio": "8.13.4",
     "worker-farm": "^1.7.0",
     "yallist": "^3.1.1"

src/browser-pool/basic-pool.js

@@ -23,8 +23,7 @@ module.exports = class BasicPool extends Pool {
     }
 
     async getBrowser(id, opts = {}) {
-        const { version } = opts;
-        const browser = Browser.create(this._config, id, version);
+        const browser = Browser.create(this._config, { ...opts, id });
 
         try {
             await browser.init();

src/browser/browser.js

@@ -1,7 +1,9 @@
 "use strict";
 
+const crypto = require("crypto");
 const _ = require("lodash");
 const { SAVE_HISTORY_MODE } = require("../constants/config");
+const { X_REQUEST_ID_DELIMITER } = require("../constants/browser");
 const history = require("./history");
 const addRunStepCommand = require("./commands/runStep");
 
@@ -19,13 +21,14 @@ const CUSTOM_SESSION_OPTS = [
 ];
 
 module.exports = class Browser {
-    static create(config, id, version) {
-        return new this(config, id, version);
+    static create(config, opts) {
+        return new this(config, opts);
     }
 
-    constructor(config, id, version) {
-        this.id = id;
-        this.version = version;
+    constructor(config, opts) {
+        this.id = opts.id;
+        this.version = opts.version;
+        this.testXReqId = opts.testXReqId;
 
         this._config = config.forBrowser(this.id);
         this._debug = config.system.debug;
@@ -82,7 +85,21 @@ module.exports = class Browser {
 
     _getSessionOptsFromConfig(optNames = CUSTOM_SESSION_OPTS) {
         return optNames.reduce((options, optName) => {
-            if (!_.isNull(this._config[optName])) {
+            if (optName === "transformRequest") {
+                options[optName] = req => {
+                    if (!_.isNull(this._config[optName])) {
+                        req = this._config[optName](req);
+                    }
+
+                    if (!req.headers["X-Request-ID"]) {
+                        req.headers["X-Request-ID"] = `${
+                            this.testXReqId
+                        }${X_REQUEST_ID_DELIMITER}${crypto.randomUUID()}`;
+                    }
+
+                    return req;
+                };
+            } else if (!_.isNull(this._config[optName])) {
                 options[optName] = this._config[optName];
             }
 

src/browser/existing-browser.js

@@ -19,14 +19,14 @@ const { isSupportIsolation } = require("../utils/browser");
 const OPTIONAL_SESSION_OPTS = ["transformRequest", "transformResponse"];
 
 module.exports = class ExistingBrowser extends Browser {
-    static create(config, id, version, emitter) {
-        return new this(config, id, version, emitter);
+    static create(config, opts) {
+        return new this(config, opts);
     }
 
-    constructor(config, id, version, emitter) {
-        super(config, id, version);
+    constructor(config, opts) {
+        super(config, opts);
 
-        this._emitter = emitter;
+        this._emitter = opts.emitter;
         this._camera = Camera.create(this._config.screenshotMode, () => this._takeScreenshot());
 
         this._meta = this._initMeta();
@@ -169,7 +169,7 @@ module.exports = class ExistingBrowser extends Browser {
         return {
             pid: process.pid,
             browserVersion: this.version,
-            "X-Request-ID": this._config.headers["X-Request-ID"],
+            testXReqId: this.testXReqId,
             ...this._config.meta,
         };
     }

src/browser/new-browser.js

@@ -31,8 +31,8 @@ const headlessBrowserOptions = {
 };
 
 module.exports = class NewBrowser extends Browser {
-    constructor(config, id, version) {
-        super(config, id, version);
+    constructor(config, opts) {
+        super(config, opts);
 
         signalHandler.on("exit", () => this.quit());
     }

src/config/browser-options.js

@@ -2,7 +2,6 @@
 
 const _ = require("lodash");
 const option = require("gemini-configparser").option;
-const uuid = require("uuid");
 const defaults = require("./defaults");
 const optionsBuilder = require("./options-builder");
 const utils = require("./utils");
@@ -306,13 +305,7 @@ function buildBrowserOptions(defaultFactory, extra) {
         outputDir: options.optionalString("outputDir"),
 
         agent: options.optionalObject("agent"),
-        headers: option({
-            parseEnv: JSON.parse,
-            parseCli: JSON.parse,
-            defaultValue: defaultFactory("headers"),
-            validate: value => utils.assertObject(value, "headers"),
-            map: value => (value["X-Request-ID"] ? value : { "X-Request-ID": uuid.v4(), ...value }),
-        }),
+        headers: options.optionalObject("headers"),
         transformRequest: options.optionalFunction("transformRequest"),
         transformResponse: options.optionalFunction("transformResponse"),
         strictSSL: options.optionalBoolean("strictSSL"),

src/config/defaults.js

@@ -83,7 +83,7 @@ module.exports = {
     fileExtensions: [".js", ".mjs", ".ts", ".mts"],
     outputDir: null,
     agent: null,
-    headers: {},
+    headers: null,
     transformRequest: null,
     transformResponse: null,
     strictSSL: null,

src/config/utils.js

@@ -20,12 +20,6 @@ exports.assertNonNegativeNumber = (value, name) => {
     }
 };
 
-exports.assertObject = (value, name) => {
-    if (!_.isPlainObject(value)) {
-        throw new Error(`"${name}" must be an object`);
-    }
-};
-
 exports.assertOptionalObject = (value, name) => {
     if (!_.isNull(value) && !_.isPlainObject(value)) {
         throw new Error(`"${name}" must be an object`);

src/constants/browser.ts

@@ -1 +1,2 @@
 export const MIN_CHROME_VERSION_SUPPORT_ISOLATION = 93;
+export const X_REQUEST_ID_DELIMITER = "__";

src/runner/browser-agent.js

@@ -1,11 +1,11 @@
 "use strict";
 
 module.exports = class BrowserAgent {
-    static create(id, version, pool) {
-        return new this(id, version, pool);
+    static create(opts = {}) {
+        return new this(opts);
     }
 
-    constructor(id, version, pool) {
+    constructor({ id, version, pool }) {
         this.browserId = id;
 
         this._version = version;

src/runner/browser-runner.ts

@@ -60,7 +60,11 @@ export class BrowserRunner extends Runner {
     }
 
     private async _runTest(test: Test): Promise<void> {
-        const browserAgent = BrowserAgent.create(this._browserId, test.browserVersion, this.browserPool);
+        const browserAgent = BrowserAgent.create({
+            id: this._browserId,
+            version: test.browserVersion,
+            pool: this.browserPool,
+        });
         const runner = TestRunner.create(test, this.config, browserAgent);
 
         runner.on(MasterEvents.TEST_BEGIN, (test: Test) => this.suiteMonitor.testBegin(test));

src/runner/test-runner/high-priority-browser-agent.js

@@ -9,8 +9,8 @@ module.exports = class HighPriorityBrowserAgent {
         this._browserAgent = browserAgent;
     }
 
-    getBrowser() {
-        return this._browserAgent.getBrowser({ highPriority: true });
+    getBrowser(opts = {}) {
+        return this._browserAgent.getBrowser({ ...opts, highPriority: true });
     }
 
     freeBrowser(...args) {

src/runner/test-runner/regular-test-runner.js

@@ -1,5 +1,6 @@
 "use strict";
 
+const crypto = require("crypto");
 const _ = require("lodash");
 const { Runner } = require("../runner");
 const logger = require("../../utils/logger");
@@ -64,6 +65,7 @@ module.exports = class RegularTestRunner extends Runner {
             sessionCaps: this._browser.capabilities,
             sessionOpts: this._browser.publicAPI.options,
             file: this._test.file,
+            testXReqId: this._browser.testXReqId,
         });
     }
 
@@ -80,7 +82,7 @@ module.exports = class RegularTestRunner extends Runner {
 
     async _getBrowser() {
         try {
-            this._browser = await this._browserAgent.getBrowser();
+            this._browser = await this._browserAgent.getBrowser({ testXReqId: crypto.randomUUID() });
             this._test.sessionId = this._browser.sessionId;
 
             return this._browser;

src/worker/hermione.ts

@@ -11,6 +11,7 @@ export interface WorkerRunTestOpts {
     sessionId: string;
     sessionCaps: WdioBrowser["capabilities"];
     sessionOpts: WdioBrowser["options"];
+    testXReqId: string;
 }
 
 export interface AssertViewResultsSuccess {