Add performance measurement

This commit adds a script for performance measurement using the Chrome DevTools tracing.
Currently the startup time of the browser-app is being measured.
The script uses Puppeteer so other processes can be measured in the future as well.
For more information see the README in the scripts/performance directory.

Contributed on behalf of STMicroelectronics

Signed-off-by: Simon Graband <[email protected]>

.npmrc

@@ -0,0 +1 @@
+scripts-prepend-node-path=true

CHANGELOG.md

@@ -6,6 +6,7 @@
 
  - [workspace] added support for multiple selections in 'Add folder to workspace' dialog. [#9684](https://github.com/eclipse-theia/theia/pull/9684)
  - [debug] added support for managing debug sessions for extensions from debug panel (previously only possible using `Hosted Plugin` commands) [#8706](https://github.com/eclipse-theia/theia/pull/8706)
+ - [scripts] added startup performance measurement script [#9777](https://github.com/eclipse-theia/theia/pull/9777) - Contributed on behalf of STMicroelectronics
 
 <a name="breaking_changes_1.16.0">[Breaking Changes:](#breaking_changes_1.16.0)</a>
 

package.json

@@ -76,7 +76,8 @@
     "start:electron": "yarn rebuild:electron && yarn --cwd examples/electron start",
     "debug:browser": "yarn rebuild:browser && yarn --cwd examples/browser start:debug",
     "debug:electron": "yarn rebuild:electron && yarn --cwd examples/electron start:debug",
-    "download:plugins": "theia download:plugins"
+    "download:plugins": "theia download:plugins",
+    "performance:startup": "concurrently --success first -k -r \"cd scripts/performance && node measure-performance.js --name Startup --folder startup --runs 2\" \"yarn start:browser\""
   },
   "workspaces": [
     "dev-packages/*",

scripts/performance/.gitignore

@@ -0,0 +1,2 @@
+profiles
+workspace

scripts/performance/README.md

@@ -0,0 +1,30 @@
+# Performance measurements
+
+This directory contains a script that measures the performance of Theia.
+Currently the support is limited to measuring the `browser-app`'s startup time using the `Largest contentful paint (LCP)` value.
+
+## Running the script
+
+### Quick Start
+
+Execute `yarn run performance:startup` in the root directory to startup the backend and execute the script.
+
+### Prerequisites
+
+To run the script the Theia backend needs to be started.
+This can either be done with the `Launch Browser Backend` launch config or by running `yarn start` in the `examples/browser-app` directory.
+
+### Executing the script
+
+The script can be exectued using `node measure-performance.js` in this directory.
+
+The script accepts the following optional parameters:
+
+-   `--name`: Specify a name for the current measurement (default: `Measurement`)
+-   `--url`: Point Theia to a url for example for specifying a specifc workspace (default: `http://localhost:3000/#/<pathToMeasurementScript>/workspace`)
+-   `--folder`: Folder name for the generated tracing files in the `profiles` folder (default: `profile`)
+-   `--runs`: Number of runs for the measurement (default: `10`)
+
+Additionally, the `--headful` flag can be set to run the script headful.
+
+_**Note**: When multiple runs are specified the script will calculate the mean and the standard deviation of all values._

scripts/performance/measure-performance.js

@@ -0,0 +1,183 @@
+/********************************************************************************
+ * Copyright (C) 2021 STMicroelectronics and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+// @ts-check
+const puppeteer = require('puppeteer');
+const fs = require('fs');
+const resolve = require('path').resolve;
+const path = resolve('./workspace');
+const profilesPath = './profiles/';
+
+const lcp = 'Largest Contentful Paint (LCP)';
+const performanceTag = braceText('Performance');
+
+let name = 'Measurement';
+let url = 'http://localhost:3000/#' + path;
+let folder = 'profile';
+let headless = true;
+let runs = 10;
+
+(async () => {
+    // Wait until the browser page is available
+    const browser = await puppeteer.launch({ headless: true });
+    const page = await browser.newPage();
+    await retryConnection(() => page.goto('http://localhost:3000'), 1000);
+    browser.close();
+
+    let defaultUrl = true;
+    const args = process.argv.slice(2);
+    for (let index = 0; index < args.length; index++) {
+        if (args[index].startsWith('--')) {
+            let next = args[index + 1];
+            if (!next.startsWith('--')) {
+                if (args[index] === '--name') {
+                    name = args[index + 1];
+                    index++;
+                }
+                if (args[index] === '--url') {
+                    url = args[index + 1];
+                    defaultUrl = false;
+                    index++;
+                }
+                if (args[index] === '--folder') {
+                    folder = args[index + 1];
+                    index++;
+                }
+                if (args[index] === '--headful') {
+                    headless = false;
+                    index++;
+                }
+                if (args[index] === '--runs') {
+                    runs = parseInt(args[index + 1]);
+                    index++;
+                }
+            }
+        }
+    }
+    if (defaultUrl) getOrCreateFolder(path);
+    getOrCreateFolder(profilesPath);
+    const folderPath = profilesPath + folder;
+    getOrCreateFolder(folderPath);
+    await measurePerformance(name, url, folderPath, headless, runs);
+})();
+
+async function measurePerformance(name, url, folder, headless, runs) {
+    const multipleRuns = runs > 1 ? true : false;
+    let runNr = 1;
+    let durations = [];
+    while (runs > 0) {
+        const browser = await puppeteer.launch({ headless: headless });
+        const page = await browser.newPage();
+
+        const file = folder + '/' + runNr + '.json';
+
+        await page.tracing.start({ path: file, screenshots: true });
+
+        await page.goto(url);
+        // This selector is for the problems indicator in the status bar, as this is one of the last elements to be rendered.
+        await page.waitForSelector('.fa-exclamation-triangle', { visible: true });
+
+        await page.tracing.stop();
+
+        await browser.close();
+
+        const time = await analyzeStartup(file)
+        durations.push(time);
+        logDuration(name, runNr, lcp, time, multipleRuns);
+
+        runs--;
+        runNr++;
+    }
+
+    if (multipleRuns) {
+        const mean = calculateMean(durations);
+        logDuration(name, 'MEAN', lcp, mean);
+        logDuration(name, 'STDEV', lcp, calculateStandardDeviation(mean, durations));
+    }
+}
+
+async function analyzeStartup(profilePath) {
+    let startEvent;
+    const tracing = JSON.parse(fs.readFileSync('./' + profilePath, 'utf8'));
+    const lcpEvents = tracing.traceEvents.filter(x => {
+        if (isStart(x)) {
+            startEvent = x;
+            return false;
+        }
+        return isLCP(x);
+    });
+
+    if (startEvent !== null) {
+        return duration(lcpEvents[lcpEvents.length - 1], startEvent);
+    }
+    throw new Error('Could not analyze startup');
+}
+
+function isLCP(x) {
+    return x.name === 'largestContentfulPaint::Candidate';
+}
+
+function isStart(x) {
+    return x.name === 'TracingStartedInBrowser';
+}
+
+function duration(event, startEvent) {
+    return parseFloat(((event.ts - startEvent.ts) / 1000000).toFixed(3));
+}
+
+function logDuration(name, run, metric, duration, multipleRuns = true) {
+    let runText = '';
+    if (multipleRuns) {
+        runText = braceText(run);
+    }
+    console.log(performanceTag + braceText(name) + runText + ' ' + metric + ': ' + duration + ' seconds');
+}
+
+function calculateMean(array) {
+    let sum = 0;
+    array.forEach(x => {
+        sum += x;
+    });
+    return (sum / array.length).toFixed(3);
+};
+
+function calculateStandardDeviation(mean, array) {
+    let count = 0;
+    array.forEach(time => {
+        count += Math.pow((time - mean), 2)
+    });
+    const variance = count / array.length;
+    return Math.sqrt(variance).toFixed(3);
+}
+
+function braceText(text) {
+    return '[' + text + ']';
+}
+
+function getOrCreateFolder(path) {
+    if (!fs.existsSync(path)) {
+        fs.mkdirSync(path);
+    }
+}
+
+const retryConnection = (fn, ms) => new Promise(url => {
+    fn()
+        .then(url)
+        .catch(() => {
+            setTimeout(() => {
+                retryConnection(fn, ms).then(url);
+            }, ms);
+        })
+});