From 05ff7e8b88cc647f7e4f84a52a5522ef685bdcb3 Mon Sep 17 00:00:00 2001 From: David Thompson Date: Tue, 13 Jul 2021 11:06:14 -0400 Subject: [PATCH] Handle OOM in Java language server Add parameters to the java server to make lemminx crash when it runs out of memory. Detect when the java server shuts down due to running out of memory, display a message to the user that this happened, and don't attempt to restart the server. Closes #527 Signed-off-by: David Thompson --- docs/Troubleshooting.md | 19 +++++++ package-lock.json | 46 ++++++++++----- package.json | 3 +- src/client/clientErrorHandler.ts | 72 +++++++++++++++++++++++- src/client/xmlClient.ts | 12 ++-- src/extension.ts | 7 ++- src/server/binary/binaryServerStarter.ts | 4 +- src/server/java/javaServerStarter.ts | 31 +++++++++- src/server/java/jvmArguments.ts | 16 ++++++ src/utils/contextUtils.ts | 30 ++++++++++ 10 files changed, 212 insertions(+), 28 deletions(-) create mode 100644 src/server/java/jvmArguments.ts create mode 100644 src/utils/contextUtils.ts diff --git a/docs/Troubleshooting.md b/docs/Troubleshooting.md index 7dc942aa..7d7a6ee3 100644 --- a/docs/Troubleshooting.md +++ b/docs/Troubleshooting.md @@ -21,3 +21,22 @@ You can kill the process by: * on Windows OS: run `taskkill /F /PID ...` all instances * on other OS: run `kill -9 ...` all instances + +### The Language Server Crashes Due to an Out Of Memory Error + +If you are working with large XML files or referencing large schema files, +this may lead to the language server running out of memory. +The Java language server is more likely to run out memory than the binary language server. +Switching to the binary language server +or increasing the memory available to the Java language server could resolve this issue. + +If you get an Out of Memory Error, but aren't working with large XML files, +then there may be a memory leak in the language server. +Please [file a issue](https://github.com/redhat-developer/vscode-xml/issues) with a description of what you were doing if this is the case. + +#### How to increase the amount of memory available to the Java Language Server + +1. Go to settings +2. Navigate to the setting `xml.server.vmargs` +3. Add `-Xmx512m` to the setting string. This allows the the language server to use at most 512 megabytes of memory. +4. If the problem persists, you can increase the `512m` to `1G` or higher diff --git a/package-lock.json b/package-lock.json index 76169914..a9c5a248 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "vscode-xml", - "version": "0.17.0", + "version": "0.17.1", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -127,20 +127,36 @@ "dev": true }, "@types/fs-extra": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.1.tgz", - "integrity": "sha512-TcUlBem321DFQzBNuz8p0CLLKp0VvF/XH9E4KHNmgwyp4E3AfgI5cjiIVZWlbfThBop2qxFIh4+LeY6hVWWZ2w==", + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.2.tgz", + "integrity": "sha512-SvSrYXfWSc7R4eqnOzbQF4TZmfpNSM9FrSWLU3EUnWBuyZqNBOrv1B1JA3byUDPUl9z4Ab3jeZG2eDdySlgNMg==", "dev": true, "requires": { "@types/node": "*" } }, + "@types/glob": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.4.tgz", + "integrity": "sha512-w+LsMxKyYQm347Otw+IfBXOv9UWVjpHpCDdbBMt8Kz/xbvCYNjP+0qPh91Km3iKfSRLBB0P7fAMf0KHrPu+MyA==", + "dev": true, + "requires": { + "@types/minimatch": "*", + "@types/node": "*" + } + }, "@types/json-schema": { "version": "7.0.7", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.7.tgz", "integrity": "sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==", "dev": true }, + "@types/minimatch": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz", + "integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==", + "dev": true + }, "@types/node": { "version": "10.17.54", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.54.tgz", @@ -155,7 +171,7 @@ }, "@types/yauzl": { "version": "2.9.1", - "resolved": "https://repository.engineering.redhat.com/nexus/repository/registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz", "integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==", "dev": true, "requires": { @@ -387,7 +403,7 @@ }, "ajv-keywords": { "version": "3.5.2", - "resolved": "https://repository.engineering.redhat.com/nexus/repository/registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", "dev": true }, @@ -853,7 +869,7 @@ }, "buffer-crc32": { "version": "0.2.13", - "resolved": "https://repository.engineering.redhat.com/nexus/repository/registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" }, "buffer-equal": { @@ -1704,13 +1720,13 @@ }, "fast-deep-equal": { "version": "3.1.3", - "resolved": "https://repository.engineering.redhat.com/nexus/repository/registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true }, "fast-json-stable-stringify": { "version": "2.1.0", - "resolved": "https://repository.engineering.redhat.com/nexus/repository/registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "dev": true }, @@ -1728,7 +1744,7 @@ }, "fd-slicer": { "version": "1.1.0", - "resolved": "https://repository.engineering.redhat.com/nexus/repository/registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", "requires": { "pend": "~1.2.0" @@ -2959,7 +2975,7 @@ }, "mkdirp": { "version": "0.5.5", - "resolved": "https://repository.engineering.redhat.com/nexus/repository/registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { @@ -3026,7 +3042,7 @@ }, "neo-async": { "version": "2.6.2", - "resolved": "https://repository.engineering.redhat.com/nexus/repository/registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "dev": true }, @@ -3361,7 +3377,7 @@ }, "pend": { "version": "1.2.0", - "resolved": "https://repository.engineering.redhat.com/nexus/repository/registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" }, "picomatch": { @@ -3940,7 +3956,7 @@ }, "source-map-support": { "version": "0.5.19", - "resolved": "https://repository.engineering.redhat.com/nexus/repository/registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", "dev": true, "requires": { @@ -4964,7 +4980,7 @@ }, "yauzl": { "version": "2.10.0", - "resolved": "https://repository.engineering.redhat.com/nexus/repository/registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", "requires": { "buffer-crc32": "~0.2.3", diff --git a/package.json b/package.json index 9b4cbb55..c6b0c871 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,8 @@ "Snippets" ], "devDependencies": { - "@types/fs-extra": "^8.0.0", + "@types/fs-extra": "^8.1.2", + "@types/glob": "^7.1.4", "@types/node": "^10.14.16", "@types/vscode": "^1.37.0", "@types/yauzl": "^2.9.1", diff --git a/src/client/clientErrorHandler.ts b/src/client/clientErrorHandler.ts index 344bc2ad..696c74b5 100644 --- a/src/client/clientErrorHandler.ts +++ b/src/client/clientErrorHandler.ts @@ -1,9 +1,15 @@ -import { window } from "vscode"; +import * as fs from "fs-extra"; +import { commands, ExtensionContext, window, workspace } from "vscode"; import { CloseAction, ErrorAction, ErrorHandler, Message } from "vscode-languageclient"; +import { ClientCommandConstants } from "../commands/commandConstants"; +import { HEAP_DUMP_LOCATION } from "../server/java/jvmArguments"; +import { getGlobalStoragePath } from "../utils/contextUtils"; +import glob = require("glob"); /** * An error handler that restarts the language server, - * unless it has been restarted 5 times in the last 3 minutes + * unless it has been restarted 5 times in the last 3 minutes, + * or if it crashed due to an Out Of Memory Error * * Adapted from [vscode-java](https://github.com/redhat-developer/vscode-java) */ @@ -11,10 +17,14 @@ export class ClientErrorHandler implements ErrorHandler { private restarts: number[]; private name: string; + private context: ExtensionContext; + private heapDumpFolder: string; - constructor(name: string) { + constructor(name: string, context: ExtensionContext) { this.name = name; this.restarts = []; + this.context = context; + this.heapDumpFolder = getHeapDumpFolderFromSettings() || getGlobalStoragePath(context); } error(_error: Error, _message: Message, _count: number): ErrorAction { @@ -23,6 +33,17 @@ export class ClientErrorHandler implements ErrorHandler { closed(): CloseAction { this.restarts.push(Date.now()); + const heapProfileGlob = new glob.GlobSync(`${this.heapDumpFolder}/java_*.hprof`); + if (heapProfileGlob.found.length) { + // Only clean heap dumps that are generated in the default location. + // The default location is the extension global storage + // This means that if users change the folder where the heap dumps are placed, + // then they will be able to read the heap dumps, + // since they aren't immediately deleted. + cleanUpHeapDumps(this.context); + showOOMMessage(); + return CloseAction.DoNotRestart; + } if (this.restarts.length < 5) { return CloseAction.Restart; } else { @@ -37,3 +58,48 @@ export class ClientErrorHandler implements ErrorHandler { } } + +/** + * Deletes all the heap dumps generated by Out Of Memory errors + * + * @returns when the heap dumps have been deleted + */ +export async function cleanUpHeapDumps(context: ExtensionContext): Promise { + const heapProfileGlob = new glob.GlobSync(`${getGlobalStoragePath(context)}/java_*.hprof`); + for (let heapProfile of heapProfileGlob.found) { + await fs.remove(heapProfile); + } +} + +/** + * Shows a message about the server crashing due to an out of memory issue + */ +async function showOOMMessage(): Promise { + const DOCS = 'More info...'; + const result = await window.showErrorMessage('The XML Language Server crashed due to an Out Of Memory Error, and will not be restarted. ', // + DOCS); + if (result === DOCS) { + await commands.executeCommand(ClientCommandConstants.OPEN_DOCS, + { + page: 'Troubleshooting', + section: 'the-language-server-crashes-due-to-an-out-of-memory-error' + } + ); + } +} + +const HEAP_DUMP_FOLDER_EXTRACTOR = new RegExp(`${HEAP_DUMP_LOCATION}(?:'([^']+)'|"([^"]+)"|([^\\s]+))`); + +/** + * Returns the heap dump folder defined in the user's preferences, or undefined if the user does not set the heap dump folder + * + * @returns the heap dump folder defined in the user's preferences, or undefined if the user does not set the heap dump folder + */ +function getHeapDumpFolderFromSettings(): string { + const jvmArgs: string = workspace.getConfiguration('xml.server').get('vmargs'); + const results = HEAP_DUMP_FOLDER_EXTRACTOR.exec(jvmArgs); + if (!results || !results[0]) { + return undefined; + } + return results[1] || results[2] || results[3]; +} \ No newline at end of file diff --git a/src/client/xmlClient.ts b/src/client/xmlClient.ts index e4779dd7..454fd1db 100644 --- a/src/client/xmlClient.ts +++ b/src/client/xmlClient.ts @@ -3,7 +3,7 @@ import { commands, ExtensionContext, extensions, Position, TextDocument, TextEdi import { Command, ConfigurationParams, ConfigurationRequest, DidChangeConfigurationNotification, ExecuteCommandParams, LanguageClientOptions, MessageType, NotificationType, RequestType, RevealOutputChannelOn, TextDocumentPositionParams } from "vscode-languageclient"; import { Executable, LanguageClient } from 'vscode-languageclient/node'; import { XMLFileAssociation } from '../api/xmlExtensionApi'; -import { ClientCommandConstants, ServerCommandConstants } from '../commands/commandConstants'; +import { ServerCommandConstants } from '../commands/commandConstants'; import { registerClientServerCommands } from '../commands/registerCommands'; import { onExtensionChange } from '../plugin'; import { RequirementsData } from "../server/requirements"; @@ -37,7 +37,7 @@ let languageClient: LanguageClient; export async function startLanguageClient(context: ExtensionContext, executable: Executable, logfile: string, externalXmlSettings: ExternalXmlSettings, requirementsData: RequirementsData): Promise { - const languageClientOptions: LanguageClientOptions = getLanguageClientOptions(logfile, externalXmlSettings, requirementsData); + const languageClientOptions: LanguageClientOptions = getLanguageClientOptions(logfile, externalXmlSettings, requirementsData, context); languageClient = new LanguageClient('xml', 'XML Support', executable, languageClientOptions); languageClient.onTelemetry(async (e: TelemetryEvent) => { @@ -106,7 +106,11 @@ export async function startLanguageClient(context: ExtensionContext, executable: return languageClient; } -function getLanguageClientOptions(logfile: string, externalXmlSettings: ExternalXmlSettings, requirementsData: RequirementsData): LanguageClientOptions { +function getLanguageClientOptions( + logfile: string, + externalXmlSettings: ExternalXmlSettings, + requirementsData: RequirementsData, + context: ExtensionContext): LanguageClientOptions { return { // Register the server for xml and xsl documentSelector: [ @@ -133,7 +137,7 @@ function getLanguageClientOptions(logfile: string, externalXmlSettings: External shouldLanguageServerExitOnShutdown: true } }, - errorHandler: new ClientErrorHandler('XML'), + errorHandler: new ClientErrorHandler('XML', context), synchronize: { //preferences starting with these will trigger didChangeConfiguration configurationSection: ['xml', '[xml]', 'files.trimFinalNewlines', 'files.trimTrailingWhitespace', 'files.insertFinalNewline'] diff --git a/src/extension.ts b/src/extension.ts index 5975b135..7068473f 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -10,21 +10,24 @@ * Microsoft Corporation - Auto Closing Tags */ +import * as fs from 'fs-extra'; import * as os from 'os'; import * as path from 'path'; import { ExtensionContext, extensions, languages } from "vscode"; import { Executable, LanguageClient } from 'vscode-languageclient/node'; import { XMLExtensionApi } from './api/xmlExtensionApi'; import { getXmlExtensionApiImplementation } from './api/xmlExtensionApiImplementation'; +import { cleanUpHeapDumps } from './client/clientErrorHandler'; import { getIndentationRules } from './client/indentation'; import { startLanguageClient } from './client/xmlClient'; +import { registerClientOnlyCommands } from './commands/registerCommands'; import { collectXmlJavaExtensions } from './plugin'; import * as requirements from './server/requirements'; import { prepareExecutable } from './server/serverStarter'; import { ExternalXmlSettings } from "./settings/externalXmlSettings"; import { getXMLConfiguration } from './settings/settings'; import { Telemetry } from './telemetry'; -import { registerClientOnlyCommands } from './commands/registerCommands'; +import { getGlobalStoragePath } from './utils/contextUtils'; let languageClient: LanguageClient; @@ -52,6 +55,8 @@ export async function activate(context: ExtensionContext): Promise { await commands.executeCommand(ClientCommandConstants.OPEN_DOCS, { page: "Proxy" }); -} \ No newline at end of file +} diff --git a/src/server/java/javaServerStarter.ts b/src/server/java/javaServerStarter.ts index e12c5141..1611e2d0 100644 --- a/src/server/java/javaServerStarter.ts +++ b/src/server/java/javaServerStarter.ts @@ -1,16 +1,28 @@ import * as os from 'os'; import * as path from 'path'; -import { ExtensionContext, workspace } from 'vscode'; +import { ExtensionContext, window, workspace } from 'vscode'; import { Executable } from 'vscode-languageclient/node'; import { getProxySettings, getProxySettingsAsJVMArgs, jvmArgsContainsProxySettings, ProxySettings } from '../../settings/proxySettings'; import { getJavaagentFlag, getKey, getXMLConfiguration, IS_WORKSPACE_VMARGS_XML_ALLOWED, xmlServerVmargs } from '../../settings/settings'; +import { getGlobalStoragePath } from '../../utils/contextUtils'; import { RequirementsData } from '../requirements'; -const glob = require('glob'); +import { HEAP_DUMP_LOCATION } from './jvmArguments'; +import glob = require('glob'); declare var v8debug; const DEBUG = (typeof v8debug === 'object') || startedInDebugMode(); +/** + * Argument that tells the program to crash when an OutOfMemoryError is raised + */ +const CRASH_ON_OOM = '-XX:+ExitOnOutOfMemoryError'; + +/** + * Argument that tells the program to generate a heap dump file when an OutOfMemoryError is raised + */ +const HEAP_DUMP = '-XX:+HeapDumpOnOutOfMemoryError'; + export async function prepareJavaExecutable( context: ExtensionContext, requirements: RequirementsData, @@ -63,6 +75,21 @@ function prepareParams(requirements: RequirementsData, xmlJavaExtensions: string params.push(watchParentProcess + 'false'); } } + if (vmargs.indexOf(CRASH_ON_OOM) < 0) { + params.push(CRASH_ON_OOM); + } + if (vmargs.indexOf(HEAP_DUMP) < 0) { + params.push(HEAP_DUMP); + } + if (vmargs.indexOf(HEAP_DUMP_LOCATION) < 0) { + params.push(`${HEAP_DUMP_LOCATION}${getGlobalStoragePath(context)}`); + } else { + window.showWarningMessage('Heap dump location has been modified. ' + + 'vscode-xml won\'t delete the heap dumps. ' + + 'vscode-xml\'s Out Of Memory detection won\'t work properly, ' + + 'unless you manually delete the heap dumps after each Out Of Memory crash.'); + } + // "OpenJDK 64-Bit Server VM warning: Options -Xverify:none and -noverify // were deprecated in JDK 13 and will likely be removed in a future release." // so only add -noverify for older versions diff --git a/src/server/java/jvmArguments.ts b/src/server/java/jvmArguments.ts new file mode 100644 index 00000000..adc0b289 --- /dev/null +++ b/src/server/java/jvmArguments.ts @@ -0,0 +1,16 @@ +/** + * Copyright (c) 2021 Red Hat, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * Contributors: + * Red Hat Inc. - initial API and implementation + * Microsoft Corporation - Auto Closing Tags + */ + +/** + * Argument that tells the program where to generate the heap dump that is created when an OutOfMemoryError is raised and `HEAP_DUMP` has been passed + */ +export const HEAP_DUMP_LOCATION = '-XX:HeapDumpPath='; diff --git a/src/utils/contextUtils.ts b/src/utils/contextUtils.ts new file mode 100644 index 00000000..889c98f4 --- /dev/null +++ b/src/utils/contextUtils.ts @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2021 Red Hat, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * Contributors: + * Red Hat Inc. - initial API and implementation + * Microsoft Corporation - Auto Closing Tags + */ + +import { platform } from "os"; +import { ExtensionContext } from "vscode"; + +/** + * Returns the path to the extension storage given the extension context + * + * @param context the extension context + * @returns the path to the extension storage + */ +export function getGlobalStoragePath(context: ExtensionContext): string { + let storagePath = context.globalStorageUri.path; + if (platform() !== "win32") { + return storagePath; + } + // fix the path from vscode api to be a proper windows path + storagePath = storagePath.substring(1); + return storagePath.replace(/\//g, '\\'); +} \ No newline at end of file