diff --git a/.github/native-tests.json b/.github/native-tests.json index 95d36db4af702..5cb3b5ada7a27 100644 --- a/.github/native-tests.json +++ b/.github/native-tests.json @@ -145,9 +145,9 @@ "os-name": "ubuntu-latest" }, { - "category": "AWT, ImageIO and Java2D", - "timeout": 30, - "test-modules": "awt, no-awt", + "category": "AWT, ImageIO and Java2D, Packaging .so files", + "timeout": 40, + "test-modules": "awt, no-awt, awt-packaging", "os-name": "ubuntu-latest" } ] diff --git a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildRemoteContainerRunner.java b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildRemoteContainerRunner.java index cd433fb365c68..94d402bb2d211 100644 --- a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildRemoteContainerRunner.java +++ b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildRemoteContainerRunner.java @@ -3,6 +3,7 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; @@ -10,6 +11,11 @@ import org.jboss.logging.Logger; +import io.quarkus.builder.JsonReader; +import io.quarkus.builder.json.JsonArray; +import io.quarkus.builder.json.JsonObject; +import io.quarkus.builder.json.JsonString; +import io.quarkus.builder.json.JsonValue; import io.quarkus.deployment.pkg.NativeConfig; public class NativeImageBuildRemoteContainerRunner extends NativeImageBuildContainerRunner { @@ -34,22 +40,27 @@ protected void preBuild(Path outputDir, List buildArgs) throws Interrupt final List containerRuntimeArgs = Arrays.asList("-v", CONTAINER_BUILD_VOLUME_NAME + ":" + NativeImageBuildStep.CONTAINER_BUILD_VOLUME_PATH); final String[] createTempContainerCommand = buildCommand("create", containerRuntimeArgs, Collections.emptyList()); - containerId = runCommandAndReadOutput(createTempContainerCommand, "Failed to create temp container."); + try { + containerId = runCommandAndReadOutput(createTempContainerCommand).get(0); + } catch (RuntimeException | InterruptedException | IOException e) { + throw new RuntimeException("Failed to create temp container.", e); + } // docker cp :/project - String[] copyCommand = new String[] { containerRuntime.getExecutableName(), "cp", outputDir.toAbsolutePath() + "/.", + final String[] copyCommand = new String[] { + containerRuntime.getExecutableName(), "cp", outputDir.toAbsolutePath() + "/.", containerId + ":" + NativeImageBuildStep.CONTAINER_BUILD_VOLUME_PATH }; - runCommand(copyCommand, "Failed to copy source-jar and libs from host to builder container", null); + runCommand(copyCommand, "Failed to copy source-jar and libs from host to builder container"); super.preBuild(outputDir, buildArgs); } - private String runCommandAndReadOutput(String[] command, String errorMsg) throws IOException, InterruptedException { + private List runCommandAndReadOutput(String[] command) throws IOException, InterruptedException { log.info(String.join(" ", command).replace("$", "\\$")); - Process process = new ProcessBuilder(command).start(); + final Process process = new ProcessBuilder(command).start(); if (process.waitFor() != 0) { - throw new RuntimeException(errorMsg); + throw new RuntimeException("Command failed: " + String.join(" ", command)); } try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { - return reader.readLine(); + return reader.lines().toList(); } } @@ -57,15 +68,45 @@ private String runCommandAndReadOutput(String[] command, String errorMsg) throws protected void postBuild(Path outputDir, String nativeImageName, String resultingExecutableName) { copyFromContainerVolume(outputDir, resultingExecutableName, "Failed to copy native image from container volume back to the host."); + + // Note that podman cp does not support globbing i.e. cp /project/*.so will not work. + // Why only .so? How about .dynlib and .lib? Regardless of the host platform, + // the builder container is always Linux. So, we only need to copy .so files. + // + // We could either start the container again, exec `find' or `ls' to list the .so files, + // stop the container again and use that list. We could also use the build-artifacts.json + // to get the list of artifacts straight away which is what ended up doing here: + copyFromContainerVolume(outputDir, "build-artifacts.json", null); + try { + final Path buildArtifactsFile = outputDir.resolve("build-artifacts.json"); + if (Files.exists(buildArtifactsFile)) { + // The file is small enough to afford this read + final String buildArtifactsJson = Files.readString(buildArtifactsFile); + final JsonObject jsonRead = JsonReader.of(buildArtifactsJson).read(); + final JsonValue jdkLibraries = jsonRead.get("jdk_libraries"); + // The jdk_libraries field is optional, there might not be any. + if (jdkLibraries instanceof JsonArray) { + for (JsonValue lib : ((JsonArray) jdkLibraries).value()) { + copyFromContainerVolume(outputDir, ((JsonString) lib).value(), + "Failed to copy " + lib + " from container volume back to the host."); + } + } + } + } catch (IOException e) { + log.errorf(e, "Failed to list .so files in the build-artifacts.json. Skipping the step."); + } + if (nativeConfig.debug().enabled()) { - copyFromContainerVolume(outputDir, "sources", "Failed to copy sources from container volume back to the host."); - String symbols = String.format("%s.debug", nativeImageName); - copyFromContainerVolume(outputDir, symbols, "Failed to copy debug symbols from container volume back to the host."); + copyFromContainerVolume(outputDir, "sources", + "Failed to copy sources from container volume back to the host."); + final String symbols = String.format("%s.debug", nativeImageName); + copyFromContainerVolume(outputDir, symbols, + "Failed to copy debug symbols from container volume back to the host."); } // docker container rm final String[] rmTempContainerCommand = new String[] { containerRuntime.getExecutableName(), "container", "rm", containerId }; - runCommand(rmTempContainerCommand, "Failed to remove container: " + containerId, null); + runCommand(rmTempContainerCommand, "Failed to remove container: " + containerId); // docker volume rm rmVolume("Failed to remove volume: " + CONTAINER_BUILD_VOLUME_NAME); } @@ -73,20 +114,20 @@ protected void postBuild(Path outputDir, String nativeImageName, String resultin private void rmVolume(String errorMsg) { final String[] rmVolumeCommand = new String[] { containerRuntime.getExecutableName(), "volume", "rm", CONTAINER_BUILD_VOLUME_NAME }; - runCommand(rmVolumeCommand, errorMsg, null); + runCommand(rmVolumeCommand, errorMsg); } private void copyFromContainerVolume(Path outputDir, String path, String errorMsg) { // docker cp :/project/ - String[] copyCommand = new String[] { containerRuntime.getExecutableName(), "cp", + final String[] copyCommand = new String[] { containerRuntime.getExecutableName(), "cp", containerId + ":" + NativeImageBuildStep.CONTAINER_BUILD_VOLUME_PATH + "/" + path, outputDir.toAbsolutePath().toString() }; - runCommand(copyCommand, errorMsg, null); + runCommand(copyCommand, errorMsg); } @Override protected List getContainerRuntimeBuildArgs(Path outputDir) { - List containerRuntimeArgs = super.getContainerRuntimeBuildArgs(outputDir); + final List containerRuntimeArgs = super.getContainerRuntimeBuildArgs(outputDir); Collections.addAll(containerRuntimeArgs, "-v", CONTAINER_BUILD_VOLUME_NAME + ":" + NativeImageBuildStep.CONTAINER_BUILD_VOLUME_PATH); return containerRuntimeArgs; diff --git a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildRunner.java b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildRunner.java index 899cf9c280a01..0b999594e66d3 100644 --- a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildRunner.java +++ b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildRunner.java @@ -11,6 +11,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.stream.Collectors; import org.apache.commons.lang3.SystemUtils; import org.jboss.logging.Logger; @@ -132,22 +133,28 @@ static void runCommand(String[] command, String errorMsg, File workingDirectory) log.info(String.join(" ", command).replace("$", "\\$")); Process process = null; try { - final ProcessBuilder processBuilder = new ProcessBuilder(command); + final ProcessBuilder processBuilder = new ProcessBuilder(command) + .redirectErrorStream(true); if (workingDirectory != null) { processBuilder.directory(workingDirectory); } process = processBuilder.start(); final int exitCode = process.waitFor(); if (exitCode != 0) { + final String out; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + out = reader.lines().collect(Collectors.joining("\n")); + } if (errorMsg != null) { - log.error(errorMsg); + log.error(errorMsg + " Output: " + out); } else { - log.debugf("Command: " + String.join(" ", command) + " failed with exit code " + exitCode); + log.debugf( + "Command: " + String.join(" ", command) + " failed with exit code " + exitCode + " Output: " + out); } } } catch (IOException | InterruptedException e) { if (errorMsg != null) { - log.error(errorMsg); + log.errorf(e, errorMsg); } else { log.debugf(e, "Command: " + String.join(" ", command) + " failed."); } @@ -158,6 +165,16 @@ static void runCommand(String[] command, String errorMsg, File workingDirectory) } } + /** + * Run {@code command} and log error if {@code errorMsg} is not null. + * + * @param command + * @param errorMsg + */ + static void runCommand(String[] command, String errorMsg) { + runCommand(command, errorMsg, null); + } + static class Result { private final int exitCode; diff --git a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildStep.java b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildStep.java index 1c1d8f54b73e1..59cf46bf025ab 100644 --- a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildStep.java +++ b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/NativeImageBuildStep.java @@ -815,6 +815,11 @@ public NativeImageInvokerInfo build() { "-H:BuildOutputJSONFile=" + nativeImageName + "-build-output-stats.json"); } + // only available in GraalVM 23.0+, we want a file with the list of built artifacts + if (graalVMVersion.compareTo(GraalVM.Version.VERSION_23_0_0) >= 0) { + addExperimentalVMOption(nativeImageArgs, "-H:+GenerateBuildArtifactsFile"); + } + // only available in GraalVM 23.1.0+ if (graalVMVersion.compareTo(GraalVM.Version.VERSION_23_1_0) >= 0) { if (graalVMVersion.compareTo(GraalVM.Version.VERSION_24_0_0) < 0) { diff --git a/integration-tests/awt-packaging/pom.xml b/integration-tests/awt-packaging/pom.xml new file mode 100644 index 0000000000000..6fed85e314d3f --- /dev/null +++ b/integration-tests/awt-packaging/pom.xml @@ -0,0 +1,125 @@ + + + + quarkus-integration-tests-parent + io.quarkus + 999-SNAPSHOT + + 4.0.0 + + quarkus-integration-test-awt-packaging + Quarkus - Integration Tests - AWT Packaging + + + + io.quarkus + quarkus-resteasy + + + io.quarkus + quarkus-resteasy-jaxb + + + io.quarkus + quarkus-awt + + + io.quarkus + quarkus-amazon-lambda + + + + + io.quarkus + quarkus-junit5 + test + + + io.quarkus + quarkus-junit5-mockito + test + + + io.rest-assured + rest-assured + test + + + io.quarkus + quarkus-test-amazon-lambda + test + + + + + io.quarkus + quarkus-resteasy-deployment + ${project.version} + pom + test + + + * + * + + + + + io.quarkus + quarkus-resteasy-jaxb-deployment + ${project.version} + pom + test + + + * + * + + + + + io.quarkus + quarkus-awt-deployment + ${project.version} + pom + test + + + * + * + + + + + io.quarkus + quarkus-amazon-lambda-deployment + ${project.version} + pom + test + + + * + * + + + + + + + + + io.quarkus + quarkus-maven-plugin + + + + build + + + + + + + diff --git a/integration-tests/awt-packaging/src/main/java/io/quarkus/it/jaxb/Book.java b/integration-tests/awt-packaging/src/main/java/io/quarkus/it/jaxb/Book.java new file mode 100644 index 0000000000000..efa5b83d3aeaa --- /dev/null +++ b/integration-tests/awt-packaging/src/main/java/io/quarkus/it/jaxb/Book.java @@ -0,0 +1,36 @@ +package io.quarkus.it.jaxb; + +import java.awt.Image; + +import jakarta.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +public class Book { + + private String title; + private Image cover; + + public Book() { + } + + public Book(String title, Image cover) { + this.title = title; + this.cover = cover; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public Image getCover() { + return cover; + } + + public void setCover(Image cover) { + this.cover = cover; + } +} diff --git a/integration-tests/awt-packaging/src/main/java/io/quarkus/it/jaxb/Lambda.java b/integration-tests/awt-packaging/src/main/java/io/quarkus/it/jaxb/Lambda.java new file mode 100644 index 0000000000000..f887628550afd --- /dev/null +++ b/integration-tests/awt-packaging/src/main/java/io/quarkus/it/jaxb/Lambda.java @@ -0,0 +1,31 @@ +package io.quarkus.it.jaxb; + +import java.io.StringReader; + +import jakarta.inject.Named; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; + +import org.jboss.logging.Logger; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; + +@Named("test") +public class Lambda implements RequestHandler { + + private static final Logger LOGGER = Logger.getLogger(Lambda.class); + + @Override + public String handleRequest(String input, Context context) { + try { + final JAXBContext jaxbContext = JAXBContext.newInstance(Book.class); + final Book book = (Book) jaxbContext.createUnmarshaller().unmarshal(new StringReader(input)); + return String.valueOf(book.getCover().getHeight(null)); + } catch (JAXBException e) { + context.getLogger().log("Error: " + e.getMessage()); + LOGGER.error(e); + } + return null; + } +} diff --git a/integration-tests/awt-packaging/src/main/java/io/quarkus/it/jaxb/Resource.java b/integration-tests/awt-packaging/src/main/java/io/quarkus/it/jaxb/Resource.java new file mode 100644 index 0000000000000..dce419c63f1b7 --- /dev/null +++ b/integration-tests/awt-packaging/src/main/java/io/quarkus/it/jaxb/Resource.java @@ -0,0 +1,33 @@ +package io.quarkus.it.jaxb; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +import org.jboss.logging.Logger; + +@Path("/jaxb") +@ApplicationScoped +public class Resource { + + private static final Logger LOGGER = Logger.getLogger(Resource.class); + + @Path("/book") + @POST + @Consumes(MediaType.APPLICATION_XML) + @Produces(MediaType.TEXT_PLAIN) + public Response postBook(Book book) { + LOGGER.info("Received book: " + book); + try { + return Response.accepted().entity(book.getCover().getHeight(null)).build(); + } catch (Exception e) { + LOGGER.error(e); + return Response.serverError().entity(e.getMessage()).build(); + } + } + +} diff --git a/integration-tests/awt-packaging/src/main/resources/application.properties b/integration-tests/awt-packaging/src/main/resources/application.properties new file mode 100644 index 0000000000000..f78355769f04a --- /dev/null +++ b/integration-tests/awt-packaging/src/main/resources/application.properties @@ -0,0 +1,2 @@ +quarkus.lambda.handler=test +quarkus.native.remote-container-build=true diff --git a/integration-tests/awt-packaging/src/test/java/io/quarkus/it/jaxb/AwtJaxbTest.java b/integration-tests/awt-packaging/src/test/java/io/quarkus/it/jaxb/AwtJaxbTest.java new file mode 100644 index 0000000000000..8010793a0b4a8 --- /dev/null +++ b/integration-tests/awt-packaging/src/test/java/io/quarkus/it/jaxb/AwtJaxbTest.java @@ -0,0 +1,50 @@ +package io.quarkus.it.jaxb; + +import static io.restassured.RestAssured.given; +import static jakarta.ws.rs.core.MediaType.APPLICATION_XML; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.http.HttpStatus; +import org.junit.jupiter.api.Test; + +import io.quarkus.amazon.lambda.test.LambdaClient; +import io.quarkus.test.junit.QuarkusTest; + +@QuarkusTest +public class AwtJaxbTest { + + public static final String BOOK_WITH_IMAGE = "" + + "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAIElEQVR4XmNgGCngPxSgi6MAZAU4FeOUQAdEKwQBdKsBOgof4SXid6kAAAAASUVORK5CYII=" + + + "Foundation" + + ""; + + /** + * Smoke tests that we have .so files + * copied over from the remote build container. + */ + @Test + public void book() { + given() + .when() + .header("Content-Type", APPLICATION_XML) + .body(BOOK_WITH_IMAGE) + .when() + .post("/jaxb/book") + .then() + .statusCode(HttpStatus.SC_ACCEPTED) + // The height in pixels of the book's cover image. + .body(is("10")); + } + + /** + * Smoke tests that our Lambda function makes at + * least some sense, but it doesn't talk to real AWS API. + */ + @Test + public void testLambdaStream() { + assertEquals("10", LambdaClient.invoke(String.class, BOOK_WITH_IMAGE)); + } + +} diff --git a/integration-tests/awt-packaging/src/test/java/io/quarkus/it/jaxb/AwtJaxbTestIT.java b/integration-tests/awt-packaging/src/test/java/io/quarkus/it/jaxb/AwtJaxbTestIT.java new file mode 100644 index 0000000000000..f8d98358f481a --- /dev/null +++ b/integration-tests/awt-packaging/src/test/java/io/quarkus/it/jaxb/AwtJaxbTestIT.java @@ -0,0 +1,59 @@ +package io.quarkus.it.jaxb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.junit.jupiter.api.Test; + +import io.quarkus.test.junit.QuarkusIntegrationTest; + +@QuarkusIntegrationTest +public class AwtJaxbTestIT extends AwtJaxbTest { + + /** + * Test is native image only, we need all artifacts to be packaged + * already, e.g. function.zip + *
+ * Tests that the same set of .so files that was copied over + * from the remote build container is also packaged into the + * zip file that will be deployed to AWS Lambda. + * + * @throws java.io.IOException + */ + @Test + public void testPackaging() throws IOException { + final Path targetPath = Paths.get(".", "target").toAbsolutePath(); + final Set localLibs = new HashSet<>(); + try (DirectoryStream stream = Files.newDirectoryStream(targetPath, "*.so")) { + for (Path entry : stream) { + localLibs.add(entry.getFileName().toString()); + } + } + + final Path zipPath = targetPath.resolve("function.zip").toAbsolutePath(); + assertTrue(Files.exists(zipPath), "Expected " + zipPath + " to exist"); + final Set awsLambdaLibs = new HashSet<>(); + try (ZipInputStream zipInputStream = new ZipInputStream(Files.newInputStream(zipPath))) { + ZipEntry entry; + while ((entry = zipInputStream.getNextEntry()) != null) { + if (entry.getName().endsWith(".so")) { + awsLambdaLibs.add(entry.getName()); + } + zipInputStream.closeEntry(); + } + } + assertEquals(localLibs, awsLambdaLibs, + "The sets of .so libs produced by the build and the set in .zip file MUST be the same. It was: " + + localLibs + " vs. " + awsLambdaLibs); + } +} diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 5c1a676bc3b1d..487eb8dc9de25 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -164,6 +164,7 @@ avro-reload awt + awt-packaging no-awt bouncycastle bouncycastle-fips