Skip to content

Commit

Permalink
Merge pull request #41201 from Karm/issue-41020
Browse files Browse the repository at this point in the history 
Adds -H:+GenerateBuildArtifactsFile, copies .so from remote container
  • Loading branch information
@zakkak
zakkak authored Jul 10, 2024
2 parents 389f694 + 3150791 commit 57e1bd6
Show file tree
Hide file tree
Showing 12 changed files with 422 additions and 22 deletions.
6 changes: 3 additions & 3 deletions .github/native-tests.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,9 +145,9 @@
"os-name": "ubuntu-latest"
},
{
"category": "AWT, ImageIO and Java2D",
"timeout": 30,
"test-modules": "awt, no-awt",
"category": "AWT, ImageIO and Java2D, Packaging .so files",
"timeout": 40,
"test-modules": "awt, no-awt, awt-packaging",
"os-name": "ubuntu-latest"
}
]
Expand Down
71 changes: 56 additions & 15 deletions .../src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildRemoteContainerRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,19 @@
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import org.jboss.logging.Logger;

import io.quarkus.builder.JsonReader;
import io.quarkus.builder.json.JsonArray;
import io.quarkus.builder.json.JsonObject;
import io.quarkus.builder.json.JsonString;
import io.quarkus.builder.json.JsonValue;
import io.quarkus.deployment.pkg.NativeConfig;

public class NativeImageBuildRemoteContainerRunner extends NativeImageBuildContainerRunner {
Expand All @@ -34,59 +40,94 @@ protected void preBuild(Path outputDir, List<String> buildArgs) throws Interrupt
final List<String> containerRuntimeArgs = Arrays.asList("-v",
CONTAINER_BUILD_VOLUME_NAME + ":" + NativeImageBuildStep.CONTAINER_BUILD_VOLUME_PATH);
final String[] createTempContainerCommand = buildCommand("create", containerRuntimeArgs, Collections.emptyList());
containerId = runCommandAndReadOutput(createTempContainerCommand, "Failed to create temp container.");
try {
containerId = runCommandAndReadOutput(createTempContainerCommand).get(0);
} catch (RuntimeException | InterruptedException | IOException e) {
throw new RuntimeException("Failed to create temp container.", e);
}
// docker cp <files> <containerID>:/project
String[] copyCommand = new String[] { containerRuntime.getExecutableName(), "cp", outputDir.toAbsolutePath() + "/.",
final String[] copyCommand = new String[] {
containerRuntime.getExecutableName(), "cp", outputDir.toAbsolutePath() + "/.",
containerId + ":" + NativeImageBuildStep.CONTAINER_BUILD_VOLUME_PATH };
runCommand(copyCommand, "Failed to copy source-jar and libs from host to builder container", null);
runCommand(copyCommand, "Failed to copy source-jar and libs from host to builder container");
super.preBuild(outputDir, buildArgs);
}

private String runCommandAndReadOutput(String[] command, String errorMsg) throws IOException, InterruptedException {
private List<String> runCommandAndReadOutput(String[] command) throws IOException, InterruptedException {
log.info(String.join(" ", command).replace("$", "\\$"));
Process process = new ProcessBuilder(command).start();
final Process process = new ProcessBuilder(command).start();
if (process.waitFor() != 0) {
throw new RuntimeException(errorMsg);
throw new RuntimeException("Command failed: " + String.join(" ", command));
}
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
return reader.readLine();
return reader.lines().toList();
}
}

@Override
protected void postBuild(Path outputDir, String nativeImageName, String resultingExecutableName) {
copyFromContainerVolume(outputDir, resultingExecutableName,
"Failed to copy native image from container volume back to the host.");

// Note that podman cp does not support globbing i.e. cp /project/*.so will not work.
// Why only .so? How about .dynlib and .lib? Regardless of the host platform,
// the builder container is always Linux. So, we only need to copy .so files.
//
// We could either start the container again, exec `find' or `ls' to list the .so files,
// stop the container again and use that list. We could also use the build-artifacts.json
// to get the list of artifacts straight away which is what ended up doing here:
copyFromContainerVolume(outputDir, "build-artifacts.json", null);
try {
final Path buildArtifactsFile = outputDir.resolve("build-artifacts.json");
if (Files.exists(buildArtifactsFile)) {
// The file is small enough to afford this read
final String buildArtifactsJson = Files.readString(buildArtifactsFile);
final JsonObject jsonRead = JsonReader.of(buildArtifactsJson).read();
final JsonValue jdkLibraries = jsonRead.get("jdk_libraries");
// The jdk_libraries field is optional, there might not be any.
if (jdkLibraries instanceof JsonArray) {
for (JsonValue lib : ((JsonArray) jdkLibraries).value()) {
copyFromContainerVolume(outputDir, ((JsonString) lib).value(),
"Failed to copy " + lib + " from container volume back to the host.");
}
}
}
} catch (IOException e) {
log.errorf(e, "Failed to list .so files in the build-artifacts.json. Skipping the step.");
}

if (nativeConfig.debug().enabled()) {
copyFromContainerVolume(outputDir, "sources", "Failed to copy sources from container volume back to the host.");
String symbols = String.format("%s.debug", nativeImageName);
copyFromContainerVolume(outputDir, symbols, "Failed to copy debug symbols from container volume back to the host.");
copyFromContainerVolume(outputDir, "sources",
"Failed to copy sources from container volume back to the host.");
final String symbols = String.format("%s.debug", nativeImageName);
copyFromContainerVolume(outputDir, symbols,
"Failed to copy debug symbols from container volume back to the host.");
}
// docker container rm <containerID>
final String[] rmTempContainerCommand = new String[] { containerRuntime.getExecutableName(), "container", "rm",
containerId };
runCommand(rmTempContainerCommand, "Failed to remove container: " + containerId, null);
runCommand(rmTempContainerCommand, "Failed to remove container: " + containerId);
// docker volume rm <volumeID>
rmVolume("Failed to remove volume: " + CONTAINER_BUILD_VOLUME_NAME);
}

private void rmVolume(String errorMsg) {
final String[] rmVolumeCommand = new String[] { containerRuntime.getExecutableName(), "volume", "rm",
CONTAINER_BUILD_VOLUME_NAME };
runCommand(rmVolumeCommand, errorMsg, null);
runCommand(rmVolumeCommand, errorMsg);
}

private void copyFromContainerVolume(Path outputDir, String path, String errorMsg) {
// docker cp <containerID>:/project/<path> <dest>
String[] copyCommand = new String[] { containerRuntime.getExecutableName(), "cp",
final String[] copyCommand = new String[] { containerRuntime.getExecutableName(), "cp",
containerId + ":" + NativeImageBuildStep.CONTAINER_BUILD_VOLUME_PATH + "/" + path,
outputDir.toAbsolutePath().toString() };
runCommand(copyCommand, errorMsg, null);
runCommand(copyCommand, errorMsg);
}

@Override
protected List<String> getContainerRuntimeBuildArgs(Path outputDir) {
List<String> containerRuntimeArgs = super.getContainerRuntimeBuildArgs(outputDir);
final List<String> containerRuntimeArgs = super.getContainerRuntimeBuildArgs(outputDir);
Collections.addAll(containerRuntimeArgs, "-v",
CONTAINER_BUILD_VOLUME_NAME + ":" + NativeImageBuildStep.CONTAINER_BUILD_VOLUME_PATH);
return containerRuntimeArgs;
Expand Down
25 changes: 21 additions & 4 deletions core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;

import org.apache.commons.lang3.SystemUtils;
import org.jboss.logging.Logger;
Expand Down Expand Up @@ -132,22 +133,28 @@ static void runCommand(String[] command, String errorMsg, File workingDirectory)
log.info(String.join(" ", command).replace("$", "\\$"));
Process process = null;
try {
final ProcessBuilder processBuilder = new ProcessBuilder(command);
final ProcessBuilder processBuilder = new ProcessBuilder(command)
.redirectErrorStream(true);
if (workingDirectory != null) {
processBuilder.directory(workingDirectory);
}
process = processBuilder.start();
final int exitCode = process.waitFor();
if (exitCode != 0) {
final String out;
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
out = reader.lines().collect(Collectors.joining("\n"));
}
if (errorMsg != null) {
log.error(errorMsg);
log.error(errorMsg + " Output: " + out);
} else {
log.debugf("Command: " + String.join(" ", command) + " failed with exit code " + exitCode);
log.debugf(
"Command: " + String.join(" ", command) + " failed with exit code " + exitCode + " Output: " + out);
}
}
} catch (IOException | InterruptedException e) {
if (errorMsg != null) {
log.error(errorMsg);
log.errorf(e, errorMsg);
} else {
log.debugf(e, "Command: " + String.join(" ", command) + " failed.");
}
Expand All @@ -158,6 +165,16 @@ static void runCommand(String[] command, String errorMsg, File workingDirectory)
}
}

/**
* Run {@code command} and log error if {@code errorMsg} is not null.
*
* @param command
* @param errorMsg
*/
static void runCommand(String[] command, String errorMsg) {
runCommand(command, errorMsg, null);
}

static class Result {
private final int exitCode;

Expand Down
5 changes: 5 additions & 0 deletions core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -815,6 +815,11 @@ public NativeImageInvokerInfo build() {
"-H:BuildOutputJSONFile=" + nativeImageName + "-build-output-stats.json");
}

// only available in GraalVM 23.0+, we want a file with the list of built artifacts
if (graalVMVersion.compareTo(GraalVM.Version.VERSION_23_0_0) >= 0) {
addExperimentalVMOption(nativeImageArgs, "-H:+GenerateBuildArtifactsFile");
}

// only available in GraalVM 23.1.0+
if (graalVMVersion.compareTo(GraalVM.Version.VERSION_23_1_0) >= 0) {
if (graalVMVersion.compareTo(GraalVM.Version.VERSION_24_0_0) < 0) {
Expand Down
125 changes: 125 additions & 0 deletions integration-tests/awt-packaging/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>quarkus-integration-tests-parent</artifactId>
<groupId>io.quarkus</groupId>
<version>999-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>quarkus-integration-test-awt-packaging</artifactId>
<name>Quarkus - Integration Tests - AWT Packaging</name>

<dependencies>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-resteasy</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-resteasy-jaxb</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-awt</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-amazon-lambda</artifactId>
</dependency>

<!-- test dependencies -->
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-junit5</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-junit5-mockito</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.rest-assured</groupId>
<artifactId>rest-assured</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-test-amazon-lambda</artifactId>
<scope>test</scope>
</dependency>

<!-- Minimal test dependencies to *-deployment artifacts for consistent build order -->
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-resteasy-deployment</artifactId>
<version>${project.version}</version>
<type>pom</type>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-resteasy-jaxb-deployment</artifactId>
<version>${project.version}</version>
<type>pom</type>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-awt-deployment</artifactId>
<version>${project.version}</version>
<type>pom</type>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-amazon-lambda-deployment</artifactId>
<version>${project.version}</version>
<type>pom</type>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>build</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
36 changes: 36 additions & 0 deletions integration-tests/awt-packaging/src/main/java/io/quarkus/it/jaxb/Book.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package io.quarkus.it.jaxb;

import java.awt.Image;

import jakarta.xml.bind.annotation.XmlRootElement;

@XmlRootElement
public class Book {

private String title;
private Image cover;

public Book() {
}

public Book(String title, Image cover) {
this.title = title;
this.cover = cover;
}

public String getTitle() {
return title;
}

public void setTitle(String title) {
this.title = title;
}

public Image getCover() {
return cover;
}

public void setCover(Image cover) {
this.cover = cover;
}
}
Loading

0 comments on commit 57e1bd6

Please sign in to comment.