Skip to content

Commit

Permalink
Convert die with dignity test to new rest test infra (#97734)
Browse files Browse the repository at this point in the history 
This commit rewrites the DieWithDignity test to use the new test infra.
A side effect of this change is that it no longer relies on jps, which
appears to have issues on Windows.

closes #77282
  • Loading branch information
@rjernst
rjernst authored Jul 17, 2023
1 parent c60edaf commit 1d995eb
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 99 deletions.
17 changes: 1 addition & 16 deletions test/external-modules/die-with-dignity/build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import org.elasticsearch.gradle.internal.info.BuildParams
import org.elasticsearch.gradle.util.GradleUtils

apply plugin: 'elasticsearch.legacy-java-rest-test'
apply plugin: 'elasticsearch.internal-java-rest-test'
apply plugin: 'elasticsearch.internal-es-plugin'

esplugin {
Expand All @@ -12,21 +12,6 @@ esplugin {
// let the javaRestTest see the classpath of main
GradleUtils.extendSourceSet(project, "main", "javaRestTest", tasks.named("javaRestTest"))

tasks.named("javaRestTest").configure {
it.onlyIf("snapshot build") { BuildParams.isSnapshotBuild() }
systemProperty 'tests.security.manager', 'false'
systemProperty 'tests.system_call_filter', 'false'
nonInputProperties.systemProperty 'log', testClusters.named("javaRestTest").map(c -> c.singleNode().getServerLog())
systemProperty 'runtime.java.home', BuildParams.runtimeJavaHome
}

testClusters.matching { it.name == "javaRestTest" }.configureEach {
systemProperty "die.with.dignity.test", "true"
// disable exit on out of memory error to let DieWithDignityIT verify that OOM handling without that works (including OOMs that are not caused by
// memory like native threads. We leave it to the JVM to test that exit on OOM works via the flag.
jvmArgs '-XX:-ExitOnOutOfMemoryError'
}

tasks.named("test").configure {
enabled = false
}
Expand Down
144 changes: 61 additions & 83 deletions ...dignity/src/javaRestTest/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,14 @@

package org.elasticsearch.qa.die_with_dignity;

import org.apache.lucene.util.Constants;
import org.elasticsearch.client.Request;
import org.elasticsearch.core.PathUtils;
import org.elasticsearch.test.cluster.ElasticsearchCluster;
import org.elasticsearch.test.cluster.LogType;
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.hamcrest.Matcher;
import org.junit.ClassRule;

import java.io.BufferedReader;
import java.io.IOException;
Expand All @@ -20,110 +24,84 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.not;

public class DieWithDignityIT extends ESRestTestCase {

@ClassRule
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
.distribution(DistributionType.INTEG_TEST)
.module("test-die-with-dignity")
.setting("xpack.security.enabled", "false")
.environment("CLI_JAVA_OPTS", "-Ddie.with.dignity.test=true")
.jvmArg("-XX:-ExitOnOutOfMemoryError")
.build();

@Override
protected String getTestRestCluster() {
return cluster.getHttpAddresses();
}

public void testDieWithDignity() throws Exception {
assumeFalse("Mute on Windows, see https://github.com/elastic/elasticsearch/issues/77282", Constants.WINDOWS);
// there should be an Elasticsearch process running with the die.with.dignity.test system property
{
final Map<String, String> esCommandLines = getElasticsearchCommandLines();
final boolean found = esCommandLines.values().stream().anyMatch(line -> line.contains("-Ddie.with.dignity.test=true"));
assertTrue(esCommandLines.toString(), found);
}
final long pid = cluster.getPid(0);
assertJvmArgs(pid, containsString("-Ddie.with.dignity.test=true"));

expectThrows(IOException.class, () -> client().performRequest(new Request("GET", "/_die_with_dignity")));

// the Elasticsearch process should die and disappear from the output of jps
assertBusy(() -> {
final Map<String, String> esCommandLines = getElasticsearchCommandLines();
final boolean notFound = esCommandLines.values().stream().noneMatch(line -> line.contains("-Ddie.with.dignity.test=true"));
assertTrue(esCommandLines.toString(), notFound);
});
// the Elasticsearch process should die
assertBusy(() -> assertJvmArgs(pid, not(containsString("-Ddie.with.dignity.test=true"))));

// parse the logs and ensure that Elasticsearch died with the expected cause
final List<String> lines = Files.readAllLines(PathUtils.get(System.getProperty("log")));
final Iterator<String> it = lines.iterator();

boolean fatalError = false;
boolean fatalErrorInThreadExiting = false;
for (String line : readLines(cluster.getNodeLog(0, LogType.SERVER_JSON))) {
if (containsAll(line, ".*ERROR.*", ".*ExceptionsHelper.*", ".*fatal error.*")) {
fatalError = true;
} else if (containsAll(
line,
".*ERROR.*",
".*ElasticsearchUncaughtExceptionHandler.*",
".*fatal error in thread \\[Thread-\\d+\\], exiting.*",
".*java.lang.OutOfMemoryError: Requested array size exceeds VM limit.*"
)) {
fatalErrorInThreadExiting = true;
}
}
assertTrue(fatalError);
assertTrue(fatalErrorInThreadExiting);
}

private void assertJvmArgs(long pid, Matcher<String> matcher) throws IOException {
final String jcmdPath = PathUtils.get(System.getProperty("tests.runtime.java"), "bin/jcmd").toString();
final Process jcmdProcess = new ProcessBuilder().command(jcmdPath, Long.toString(pid), "VM.command_line")
.redirectErrorStream(true)
.start();
List<String> outputLines = readLines(jcmdProcess.getInputStream());

String jvmArgs = null;
try {
while (it.hasNext() && (fatalError == false || fatalErrorInThreadExiting == false)) {
final String line = it.next();
if (containsAll(line, ".*ERROR.*", ".*ExceptionsHelper.*", ".*javaRestTest-0.*", ".*fatal error.*")) {
fatalError = true;
} else if (containsAll(
line,
".*ERROR.*",
".*ElasticsearchUncaughtExceptionHandler.*",
".*javaRestTest-0.*",
".*fatal error in thread \\[Thread-\\d+\\], exiting.*",
".*java.lang.OutOfMemoryError: Requested array size exceeds VM limit.*"
)) {
fatalErrorInThreadExiting = true;
for (String line : outputLines) {
if (line.startsWith("jvm_args")) {
jvmArgs = line;
break;
}
}

assertTrue(fatalError);
assertTrue(fatalErrorInThreadExiting);
assertThat(jvmArgs, matcher);

} catch (AssertionError ae) {
Path path = PathUtils.get(System.getProperty("log"));
debugLogs(path);
logger.error("Failed matcher for jvm pid " + pid);
logger.error("jcmd output: " + String.join("\n", outputLines));
throw ae;
}
}

private Map<String, String> getElasticsearchCommandLines() throws IOException {
/*
* jps will truncate the command line to 1024 characters; so we collect the pids and then run jcmd <pid> VM.command_line to get the
* full command line.
*/
final String jpsPath = PathUtils.get(System.getProperty("runtime.java.home"), "bin/jps").toString();
final Process process = new ProcessBuilder().command(jpsPath, "-q").start();

final List<String> pids = new ArrayList<>();
try (
InputStream is = process.getInputStream();
BufferedReader in = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))
) {
String line;
while ((line = in.readLine()) != null) {
pids.add(line);
}
}

final String jcmdPath = PathUtils.get(System.getProperty("runtime.java.home"), "bin/jcmd").toString();
final Map<String, String> esCommandLines = new HashMap<>();
for (final String pid : pids) {
final Process jcmdProcess = new ProcessBuilder().command(jcmdPath, pid, "VM.command_line").start();
try (
InputStream is = jcmdProcess.getInputStream();
BufferedReader in = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))
) {
boolean isElasticsearch = false;
String jvmArgs = null;
String line;
while ((line = in.readLine()) != null) {
if (line.equals("java_command: org.elasticsearch.server/org.elasticsearch.bootstrap.Elasticsearch")) {
isElasticsearch = true;
}
if (line.startsWith("jvm_args")) {
jvmArgs = line;
}
}
if (isElasticsearch) {
assertNotNull(pid, jvmArgs);
esCommandLines.put(pid, jvmArgs);
}
}
private List<String> readLines(InputStream is) throws IOException {
try (BufferedReader in = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
return in.lines().toList();
}
return esCommandLines;
}

private boolean containsAll(String line, String... subStrings) {
Expand Down

0 comments on commit 1d995eb

Please sign in to comment.