Merge pull request #21613 from cescoffier/upx-support

Add support for UPX compression

core/deployment/src/main/java/io/quarkus/deployment/pkg/NativeConfig.java

@@ -3,6 +3,7 @@
 import java.io.File;
 import java.util.List;
 import java.util.Optional;
+import java.util.OptionalInt;
 
 import io.quarkus.runtime.annotations.ConfigGroup;
 import io.quarkus.runtime.annotations.ConfigItem;
@@ -382,6 +383,34 @@ public static class Debug {
     @ConfigItem
     public boolean enableDashboardDump;
 
+    /**
+     * Configure native executable compression using UPX.
+     */
+    @ConfigItem
+    public Compression compression;
+
+    @ConfigGroup
+    public static class Compression {
+        /**
+         * The compression level in [1, 10].
+         * 10 means <em>best</em>
+         *
+         * Higher compression level requires more time to compress the executable.
+         */
+        @ConfigItem
+        public OptionalInt level;
+
+        /**
+         * Allows passing extra arguments to the UPX command line (like --brute).
+         * The arguments are comma-separated.
+         *
+         * The exhaustive list of parameters can be found in
+         * <a href="https://github.com/upx/upx/blob/devel/doc/upx.pod">https://github.com/upx/upx/blob/devel/doc/upx.pod</a>.
+         */
+        @ConfigItem
+        public Optional<List<String>> additionalArgs;
+    }
+
     /**
      * Supported Container runtimes
      */

core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildContainerRunner.java

@@ -131,7 +131,7 @@ protected String[] buildCommand(String dockerCmd, List<String> containerRuntimeA
      *         executable exists in the environment or if the docker executable is an alias to podman
      * @throws IllegalStateException if no container runtime was found to build the image
      */
-    private static NativeConfig.ContainerRuntime detectContainerRuntime() {
+    public static NativeConfig.ContainerRuntime detectContainerRuntime() {
         // Docker version 19.03.14, build 5eb3275d40
         String dockerVersionOutput = getVersionOutputFor(NativeConfig.ContainerRuntime.DOCKER);
         boolean dockerAvailable = dockerVersionOutput.contains("Docker version");

core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/UpxCompressionBuildStep.java

@@ -0,0 +1,195 @@
+package io.quarkus.deployment.pkg.steps;
+
+import static io.quarkus.deployment.pkg.steps.LinuxIDUtil.getLinuxID;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.SystemUtils;
+import org.jboss.logging.Logger;
+
+import io.quarkus.deployment.annotations.BuildProducer;
+import io.quarkus.deployment.annotations.BuildStep;
+import io.quarkus.deployment.pkg.NativeConfig;
+import io.quarkus.deployment.pkg.builditem.ArtifactResultBuildItem;
+import io.quarkus.deployment.pkg.builditem.NativeImageBuildItem;
+import io.quarkus.deployment.util.FileUtil;
+import io.quarkus.deployment.util.ProcessUtil;
+
+public class UpxCompressionBuildStep {
+
+    private static final Logger log = Logger.getLogger(UpxCompressionBuildStep.class);
+
+    /**
+     * The name of the environment variable containing the system path.
+     */
+    private static final String PATH = "PATH";
+
+    @BuildStep(onlyIf = NativeBuild.class)
+    public void compress(NativeConfig nativeConfig, NativeImageBuildItem image,
+            BuildProducer<ArtifactResultBuildItem> artifactResultProducer) {
+        if (nativeConfig.compression.level.isEmpty()) {
+            log.debug("UPX compression disabled");
+            return;
+        }
+
+        Optional<File> upxPathFromSystem = getUpxFromSystem();
+        boolean inContainerUpx = upxPathFromSystem.isEmpty() || nativeConfig.isContainerBuild();
+        if (inContainerUpx) {
+            log.infof("Running UPX from a container using the builder image: " + nativeConfig.builderImage);
+            if (!runUpxInContainer(image, nativeConfig)) {
+                throw new IllegalStateException("Unable to compress the native executable");
+            }
+        } else {
+            log.debug("Running UPX from system path");
+            if (!runUpxFromHost(upxPathFromSystem.get(), image.getPath().toFile(), nativeConfig)) {
+                throw new IllegalStateException("Unable to compress the native executable");
+            }
+        }
+        log.infof("Native executable compressed: %s", image.getPath().toFile().getAbsolutePath());
+    }
+
+    private boolean runUpxFromHost(File upx, File executable, NativeConfig nativeConfig) {
+        String level = getCompressionLevel(nativeConfig.compression.level.getAsInt());
+        List<String> extraArgs = nativeConfig.compression.additionalArgs.orElse(Collections.emptyList());
+        List<String> args = Stream.concat(
+                Stream.concat(Stream.of(upx.getAbsolutePath(), level), extraArgs.stream()),
+                Stream.of(executable.getAbsolutePath()))
+                .collect(Collectors.toList());
+        log.infof("Executing %s", String.join(" ", args));
+        final ProcessBuilder processBuilder = new ProcessBuilder(args)
+                .directory(executable.getAbsoluteFile().getParentFile())
+                .redirectOutput(ProcessBuilder.Redirect.PIPE)
+                .redirectError(ProcessBuilder.Redirect.PIPE);
+        Process process = null;
+        try {
+            process = processBuilder.start();
+            ProcessUtil.streamOutputToSysOut(process);
+            final int exitCode = process.waitFor();
+            if (exitCode != 0) {
+                log.errorf("Command: " + String.join(" ", args) + " failed with exit code " + exitCode);
+                return false;
+            }
+            return true;
+        } catch (Exception e) {
+            log.errorf("Command: " + String.join(" ", args) + " failed", e);
+            return false;
+        } finally {
+            if (process != null) {
+                process.destroy();
+            }
+        }
+
+    }
+
+    private boolean runUpxInContainer(NativeImageBuildItem nativeImage, NativeConfig nativeConfig) {
+        String level = getCompressionLevel(nativeConfig.compression.level.getAsInt());
+        List<String> extraArgs = nativeConfig.compression.additionalArgs.orElse(Collections.emptyList());
+
+        List<String> commandLine = new ArrayList<>();
+        NativeConfig.ContainerRuntime containerRuntime = nativeConfig.containerRuntime
+                .orElseGet(NativeImageBuildContainerRunner::detectContainerRuntime);
+        commandLine.add(containerRuntime.getExecutableName());
+
+        commandLine.add("run");
+        commandLine.add("--env");
+        commandLine.add("LANG=C");
+        commandLine.add("--rm");
+        commandLine.add("--entrypoint=upx");
+
+        String containerName = "upx-" + RandomStringUtils.random(5, true, false);
+        commandLine.add("--name");
+        commandLine.add(containerName);
+
+        String volumeOutputPath = nativeImage.getPath().toFile().getParentFile().getAbsolutePath();
+        if (SystemUtils.IS_OS_WINDOWS) {
+            volumeOutputPath = FileUtil.translateToVolumePath(volumeOutputPath);
+        } else if (SystemUtils.IS_OS_LINUX) {
+            String uid = getLinuxID("-ur");
+            String gid = getLinuxID("-gr");
+            if (uid != null && gid != null && !uid.isEmpty() && !gid.isEmpty()) {
+                Collections.addAll(commandLine, "--user", uid + ":" + gid);
+                if (containerRuntime == NativeConfig.ContainerRuntime.PODMAN) {
+                    // Needed to avoid AccessDeniedExceptions
+                    commandLine.add("--userns=keep-id");
+                }
+            }
+        }
+
+        Collections.addAll(commandLine, "-v",
+                volumeOutputPath + ":" + NativeImageBuildStep.CONTAINER_BUILD_VOLUME_PATH + ":z");
+
+        commandLine.add(nativeConfig.builderImage);
+        commandLine.add(level);
+        commandLine.addAll(extraArgs);
+
+        commandLine.add(nativeImage.getPath().toFile().getName());
+
+        log.infof("Compress native executable using: %s", String.join(" ", commandLine));
+        final ProcessBuilder processBuilder = new ProcessBuilder(commandLine)
+                .redirectOutput(ProcessBuilder.Redirect.PIPE)
+                .redirectError(ProcessBuilder.Redirect.PIPE);
+        Process process = null;
+        try {
+            process = processBuilder.start();
+            ProcessUtil.streamOutputToSysOut(process);
+            final int exitCode = process.waitFor();
+            if (exitCode != 0) {
+                if (exitCode == 127) {
+                    log.errorf("Command: %s failed because the builder image does not provide the `upx` executable",
+                            String.join(" ", commandLine));
+                } else {
+                    log.errorf("Command: %s failed with exit code %d", String.join(" ", commandLine), exitCode);
+                }
+                return false;
+            }
+            return true;
+        } catch (Exception e) {
+            log.errorf("Command: " + String.join(" ", commandLine) + " failed", e);
+            return false;
+        } finally {
+            if (process != null) {
+                process.destroy();
+            }
+        }
+
+    }
+
+    private String getCompressionLevel(int level) {
+        if (level == 10) {
+            return "--best";
+        }
+        if (level > 0 && level < 10) {
+            return "-" + level;
+        }
+        throw new IllegalArgumentException("Invalid compression level, " + level + " is not in [1, 10]");
+    }
+
+    private Optional<File> getUpxFromSystem() {
+        String exec = getUpxExecutableName();
+        String systemPath = System.getenv(PATH);
+        if (systemPath != null) {
+            String[] pathDirs = systemPath.split(File.pathSeparator);
+            for (String pathDir : pathDirs) {
+                File dir = new File(pathDir);
+                if (dir.isDirectory()) {
+                    File file = new File(dir, exec);
+                    if (file.exists()) {
+                        return Optional.of(file);
+                    }
+                }
+            }
+        }
+        return Optional.empty();
+    }
+
+    private static String getUpxExecutableName() {
+        return SystemUtils.IS_OS_WINDOWS ? "upx.exe" : "upx";
+    }
+}

docs/src/main/asciidoc/building-native-image.adoc

@@ -602,6 +602,8 @@ NOTE: To use Mandrel instead of GraalVM CE, update the `FROM` clause to: `FROM q
 
 === Using a distroless base image
 
+IMPORTANT: Distroless image support is experimental.
+
 If you are looking for small container images, the https://github.com/GoogleContainerTools/distroless[distroless] approach reduces the size of the base layer.
 The idea behind _distroless_ is the usage of a single and minimal base image containing all the requirements, and sometimes even the application itself.
 
@@ -625,6 +627,11 @@ Quarkus provides the `quay.io/quarkus/quarkus-distroless-image:1.0` image.
 It contains the required packages to run a native executable and is only **9Mb**.
 Just add your application on top of this image, and you will get a tiny container image.
 
+=== Native executable compression
+
+Quarkus can compress the produced native executable using UPX.
+More details on xref:./upx.adoc[UPX Compression documentation].
+
 === Separating Java and native image compilation
 
 In certain circumstances, you may want to build the native image in a separate step.

docs/src/main/asciidoc/upx.adoc

@@ -0,0 +1,67 @@
+////
+This guide is maintained in the main Quarkus repository
+and pull requests should be submitted there:
+https://github.com/quarkusio/quarkus/tree/main/docs/src/main/asciidoc
+////
+
+= Compressing native executables using UPX
+
+include::./attributes.adoc[]
+
+https://upx.github.io/[Ultimate Packer for eXecutables (UPX)] is a compression tool reducing the size of executables.
+Quarkus can compress the produced native executable to reduce its size.
+Such compression is interesting when:
+
+* building CLI tools, and you want to reduce the disk footprint,
+* building small container images.
+
+Note that UPX compression:
+
+1. increases your build time, mainly if you use high-compression levels
+2. increases the startup RSS usage of the application
+
+== System vs. Container
+
+The UPX compression requires:
+
+* the `upx` command to be available in the system `PATH`;
+* or to have built the native executable using an in-container build.
+
+If you have the `upx` command available on your path, Quarkus uses it.
+
+Otherwise, if you built the native image using an in-container build (using `quarkus.native.container-build=true`) and if the builder image provides the `upx` command, Quarkus compresses the executable from inside the container.
+
+If you are not in one of these cases, the compression fails.
+
+== Configuring the UPX compression
+
+Then, in your application configuration, enable the compression by configuring the _compression level_ you want:
+
+[source, properties]
+----
+quarkus.native.compression.level=5
+----
+
+If the compression level is not set, the compression is disabled.
+The compression will happen once the native executable is built and will replace the executable.
+
+== Compression level
+
+The compression level goes from 1 to 10:
+
+* `1`: faster compression
+* `9`: better compression
+* `10`: best compression (can be slow for big files)
+
+== Extra parameters
+
+You can pass extra parameter to upx, such as `--brute` or `--ultra-brute` using the `quarkus.native.compression.additional-args` parameter.
+The value is a comma-separated list of arguments:
+
+[source, properties]
+----
+quarkus.native.compression.level=3
+quarkus.native.compression.additional-args=--ultra-brute,-v
+----
+
+The exhaustive list of parameters can be found in https://github.com/upx/upx/blob/devel/doc/upx.pod[the UPX documentation].