Skip to content

Commit

Permalink
Adjust initial and max memory usage of builder.
Browse files Browse the repository at this point in the history
  • Loading branch information
@fniephaus
fniephaus committed Apr 17, 2023
1 parent e03ab4a commit 44ab73c
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 56 deletions.
72 changes: 72 additions & 0 deletions substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/*
* Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package com.oracle.svm.driver;

import java.lang.management.ManagementFactory;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.oracle.svm.core.OS;

class MemoryUtil {
private static final double MAX_RAM_MIN_PERCENTAGE = 50.0;
private static final double MAX_RAM_MAX_PERCENTAGE = 80.0;
private static final double MAX_RAM_HEADROOM_PERCENTAGE = 10.0;

private static final Pattern MEMINFO_AVAILABLE_REGEX = Pattern.compile("^MemAvailable:\\s+(\\d+) kB");

public static double determineMaxRAMPercentage() {
double availableRAMPercentage = Math.floor(getAvailableRAMPercentage() - MAX_RAM_HEADROOM_PERCENTAGE);
return Math.max(MAX_RAM_MIN_PERCENTAGE, Math.min(availableRAMPercentage, MAX_RAM_MAX_PERCENTAGE));
}

private static double getAvailableRAMPercentage() {
var osBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
double usableMemory = osBean.getFreeMemorySize();
// TODO: drop debug code
System.out.println("Free memory percentage: " + (usableMemory / osBean.getTotalMemorySize()));
if (OS.LINUX.isCurrent()) {
// Increase value if more memory is available on Linux.
usableMemory = Math.max(usableMemory, getLinuxAvailableMemory());
// TODO: drop debug code
System.out.println("Available memory percentage: " + (usableMemory / osBean.getTotalMemorySize()));
}
return (usableMemory / osBean.getTotalMemorySize()) * 100;
}

private static long getLinuxAvailableMemory() {
try {
String memAvailableLine = Files.readAllLines(Paths.get("/proc/meminfo")).stream().filter(l -> l.startsWith("MemAvailable")).findFirst().orElse("");
Matcher m = MEMINFO_AVAILABLE_REGEX.matcher(memAvailableLine);
if (m.matches()) {
return Long.parseLong(m.group(1)) * 1024 /* kB to B */;
}
} catch (Exception e) {
}
return -1;
}
}
63 changes: 7 additions & 56 deletions substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
Expand Down Expand Up @@ -58,7 +57,6 @@
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.jar.Attributes;
import java.util.jar.JarFile;
import java.util.jar.Manifest;
Expand Down Expand Up @@ -808,11 +806,12 @@ static void ensureDirectoryExists(Path dir) {

private void prepareImageBuildArgs() {
addImageBuilderJavaArgs("-Xss10m");
addImageBuilderJavaArgs(oXms + getXmsValue());
String xmxVal = getXmxValue(1);
if (!"0".equals(xmxVal)) {
addImageBuilderJavaArgs(oXmx + xmxVal);
}

/* Builder needs at least 2GB of memory. */
addImageBuilderJavaArgs("-Xms2g");

/* Adjust max memory usage depending on availability. */
addImageBuilderJavaArgs("-XX:MaxRAMPercentage=" + MemoryUtil.determineMaxRAMPercentage());

/* Let builder exit on first OutOfMemoryError. */
addImageBuilderJavaArgs("-XX:+ExitOnOutOfMemoryError");
Expand Down Expand Up @@ -863,27 +862,6 @@ protected static boolean replaceArg(Collection<String> args, String argPrefix, S
return elementsRemoved;
}

private static <T> T consolidateArgs(Collection<String> args, String argPrefix,
Function<String, T> fromSuffix, Function<T, String> toSuffix,
Supplier<T> init, BiFunction<T, T, T> combiner) {
T consolidatedValue = null;
boolean needsConsolidate = false;
for (String arg : args) {
if (arg.startsWith(argPrefix)) {
if (consolidatedValue == null) {
consolidatedValue = init.get();
} else {
needsConsolidate = true;
}
consolidatedValue = combiner.apply(consolidatedValue, fromSuffix.apply(arg.substring(argPrefix.length())));
}
}
if (consolidatedValue != null && needsConsolidate) {
replaceArg(args, argPrefix, toSuffix.apply(consolidatedValue));
}
return consolidatedValue;
}

private static LinkedHashSet<String> collectListArgs(Collection<String> args, String argPrefix, String delimiter) {
LinkedHashSet<String> allEntries = new LinkedHashSet<>();
for (String arg : args) {
Expand Down Expand Up @@ -1052,14 +1030,7 @@ private int completeImageBuild() {
}
}
}
/* Perform JavaArgs consolidation - take the maximum of -Xmx, minimum of -Xms */
Long xmxValue = consolidateArgs(imageBuilderJavaArgs, oXmx, SubstrateOptionsParser::parseLong, String::valueOf, () -> 0L, Math::max);
Long xmsValue = consolidateArgs(imageBuilderJavaArgs, oXms, SubstrateOptionsParser::parseLong, String::valueOf, () -> SubstrateOptionsParser.parseLong(getXmsValue()), Math::max);
if (xmxValue != null) {
if (Long.compareUnsigned(xmsValue, xmxValue) > 0) {
replaceArg(imageBuilderJavaArgs, oXms, Long.toUnsignedString(xmxValue));
}
}

addImageBuilderJavaArgs(customJavaArgs.toArray(new String[0]));

/* Perform option consolidation of imageBuilderArgs */
Expand Down Expand Up @@ -2088,26 +2059,6 @@ List<String> getNativeImageArgs() {
return Stream.concat(getDefaultNativeImageArgs().stream(), config.getBuildArgs().stream()).toList();
}

protected String getXmsValue() {
return "1g";
}

@SuppressWarnings("deprecation") // getTotalPhysicalMemorySize is deprecated after JDK 11
private static long getPhysicalMemorySize() {
OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean();
long totalPhysicalMemorySize = ((com.sun.management.OperatingSystemMXBean) osMXBean).getTotalPhysicalMemorySize();
return totalPhysicalMemorySize;
}

protected String getXmxValue(int maxInstances) {
Long memMax = Long.divideUnsigned(Long.divideUnsigned(getPhysicalMemorySize(), 10) * 8, maxInstances);
String maxXmx = "14g";
if (Long.compareUnsigned(memMax, SubstrateOptionsParser.parseLong(maxXmx)) >= 0) {
return maxXmx;
}
return Long.toUnsignedString(memMax);
}

private static boolean isDumbTerm() {
String term = System.getenv().getOrDefault("TERM", "");
return term.isEmpty() || term.equals("dumb") || term.equals("unknown");
Expand Down

0 comments on commit 44ab73c

Please sign in to comment.