Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable caching for quarkus:build and declare dependencies as inputs #37622

Merged
merged 10 commits into from
Feb 16, 2024
Merged

Enable caching for quarkus:build and declare dependencies as inputs #37622

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
b56cbba
Dump all Quarkus app dependencies along with their checksums as part …
aloubyansky Dec 8, 2023
ae18537
Execute quarkus:track-config-changes for all the IT modules to dump q…
aloubyansky Dec 8, 2023
f6faadb
Only compute checksum if snapshot dependency
gsmet Feb 1, 2024
fe16620
Enable caching of quarkus:build outcome for the main integration test
aloubyansky Aug 25, 2023
8f7dec8
Fix name of config property for consistency
gsmet Feb 1, 2024
057ef33
Enable config tracking for all ITs
gsmet Feb 1, 2024
970c62f
Avoid NPE in CodeGenerator.dumpCurrentConfigValues
gsmet Feb 1, 2024
c76e348
Configure additional input for cached plugins
gsmet Feb 1, 2024
471c4fc
Revert "Enable config tracking for all ITs"
gsmet Feb 15, 2024
a171754
Some refinements to make sure the execution only runs once
gsmet Feb 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/ci-actions-incremental.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1072,6 +1072,11 @@ jobs:
build-scan-capture-strategy: ON_DEMAND
job-name: "Native Tests - ${{matrix.category}}"
wrapper-init: true
- name: Cache Quarkus metadata
uses: actions/cache@v3
with:
path: '**/.quarkus/quarkus-prod-config-dump'
key: ${{ runner.os }}-quarkus-metadata
- name: Build
env:
TEST_MODULES: ${{matrix.test-modules}}
Expand Down
5 changes: 5 additions & 0 deletions .mvn/extensions.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,9 @@
<artifactId>common-custom-user-data-maven-extension</artifactId>
<version>1.12.5</version>
</extension>
<extension>
<groupId>com.gradle</groupId>
<artifactId>quarkus-build-caching-extension</artifactId>
<version>0.10</version>
</extension>
</extensions>
138 changes: 109 additions & 29 deletions devtools/maven/src/main/java/io/quarkus/maven/TrackConfigChangesMojo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
package io.quarkus.maven;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Method;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.zip.Adler32;
import java.util.zip.Checksum;

import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
Expand All @@ -18,6 +25,7 @@
import io.quarkus.bootstrap.app.CuratedApplication;
import io.quarkus.bootstrap.classloading.QuarkusClassLoader;
import io.quarkus.bootstrap.model.ApplicationModel;
import io.quarkus.maven.dependency.DependencyFlags;
import io.quarkus.runtime.LaunchMode;

/**
Expand Down Expand Up @@ -58,6 +66,18 @@ public class TrackConfigChangesMojo extends QuarkusBootstrapMojo {
@Parameter(defaultValue = "false", property = "quarkus.track-config-changes.dump-current-when-recorded-unavailable")
boolean dumpCurrentWhenRecordedUnavailable;

/**
* Whether to dump Quarkus application dependencies along with their checksums
*/
@Parameter(defaultValue = "true", property = "quarkus.track-config-changes.dump-dependencies")
boolean dumpDependencies;

/**
* Dependency dump file
*/
@Parameter(property = "quarkus.track-config-changes.dependencies-file")
File dependenciesFile;

@Override
protected boolean beforeExecute() throws MojoExecutionException, MojoFailureException {
if (skip) {
Expand All @@ -82,16 +102,6 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException {
getLog().debug("Bootstrapping Quarkus application in mode " + launchMode);
}

Path targetFile;
if (outputFile == null) {
targetFile = outputDirectory.toPath()
.resolve("quarkus-" + launchMode.getDefaultProfile() + "-config-check");
} else if (outputFile.isAbsolute()) {
targetFile = outputFile.toPath();
} else {
targetFile = outputDirectory.toPath().resolve(outputFile.toPath());
}

Path compareFile;
if (this.recordedBuildConfigFile == null) {
compareFile = recordedBuildConfigDirectory.toPath()
Expand All @@ -102,34 +112,64 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException {
compareFile = recordedBuildConfigDirectory.toPath().resolve(this.recordedBuildConfigFile.toPath());
}

final Properties compareProps = new Properties();
if (Files.exists(compareFile)) {
try (BufferedReader reader = Files.newBufferedReader(compareFile)) {
compareProps.load(reader);
} catch (IOException e) {
throw new RuntimeException("Failed to read " + compareFile, e);
}
} else if (!dumpCurrentWhenRecordedUnavailable) {
getLog().info(compareFile + " not found");
final boolean prevConfigExists = Files.exists(compareFile);
if (!prevConfigExists && !dumpCurrentWhenRecordedUnavailable && !dumpDependencies) {
getLog().info("Config dump from the previous build does not exist at " + compareFile);
return;
}

CuratedApplication curatedApplication = null;
QuarkusClassLoader deploymentClassLoader = null;
final ClassLoader originalCl = Thread.currentThread().getContextClassLoader();
Properties actualProps;
final boolean clearPackageTypeSystemProperty = setPackageTypeSystemPropertyIfNativeProfileEnabled();
try {
curatedApplication = bootstrapApplication(launchMode);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW, I benchmarked things a bit and resolving the dependencies is quite slow. Even for a relatively simple IT as the Hibernate Validator one, it takes ~ 0.8 s.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is, this is something i've been looking into for the last few weeks. But there is not much we can do especially for relatively simple projects, since most of the time is spent in the Maven artifact resolver itself.

However, the app model resolution will typically be necessary either way. It's about at which point it's performed. Here, we are moving this step to an earlier phase and it's cached for later phases if they get executed.

deploymentClassLoader = curatedApplication.createDeploymentClassLoader();
Thread.currentThread().setContextClassLoader(deploymentClassLoader);

final Class<?> codeGenerator = deploymentClassLoader.loadClass("io.quarkus.deployment.CodeGenerator");
final Method dumpConfig = codeGenerator.getMethod("dumpCurrentConfigValues", ApplicationModel.class, String.class,
Properties.class, QuarkusClassLoader.class, Properties.class, Path.class);
dumpConfig.invoke(null, curatedApplication.getApplicationModel(),
launchMode.name(), getBuildSystemProperties(true),
deploymentClassLoader, compareProps, targetFile);
if (prevConfigExists || dumpCurrentWhenRecordedUnavailable) {
final Path targetFile = getOutputFile(outputFile, launchMode.getDefaultProfile(), "-config-check");
Properties compareProps = new Properties();
if (prevConfigExists) {
try (BufferedReader reader = Files.newBufferedReader(compareFile)) {
compareProps.load(reader);
} catch (IOException e) {
throw new RuntimeException("Failed to read " + compareFile, e);
}
}

deploymentClassLoader = curatedApplication.createDeploymentClassLoader();
Thread.currentThread().setContextClassLoader(deploymentClassLoader);

final Class<?> codeGenerator = deploymentClassLoader.loadClass("io.quarkus.deployment.CodeGenerator");
final Method dumpConfig = codeGenerator.getMethod("dumpCurrentConfigValues", ApplicationModel.class,
String.class,
Properties.class, QuarkusClassLoader.class, Properties.class, Path.class);
dumpConfig.invoke(null, curatedApplication.getApplicationModel(),
launchMode.name(), getBuildSystemProperties(true),
deploymentClassLoader, compareProps, targetFile);
}

if (dumpDependencies) {
final List<String> deps = new ArrayList<>();
for (var d : curatedApplication.getApplicationModel().getDependencies(DependencyFlags.DEPLOYMENT_CP)) {
gsmet marked this conversation as resolved.
Show resolved Hide resolved
StringBuilder entry = new StringBuilder(d.toGACTVString());
if (d.isSnapshot()) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I decided to avoid computing the checksum if the dependency is not a snapshot one. They should be immutable in this case.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Final versions could still be built locally. For example before releasing but not necessarily then.

var adler32 = new Adler32();
updateChecksum(adler32, d.getResolvedPaths());
entry.append(" ").append(adler32.getValue());
}

deps.add(entry.toString());
}
Collections.sort(deps);
final Path targetFile = getOutputFile(dependenciesFile, launchMode.getDefaultProfile(),
"-dependency-checksums.txt");
Files.createDirectories(targetFile.getParent());
try (BufferedWriter writer = Files.newBufferedWriter(targetFile)) {
for (var s : deps) {
writer.write(s);
writer.newLine();
}
}
}
} catch (Exception any) {
throw new MojoExecutionException("Failed to bootstrap Quarkus application", any);
} finally {
Expand All @@ -142,4 +182,44 @@ protected void doExecute() throws MojoExecutionException, MojoFailureException {
}
}
}

private Path getOutputFile(File outputFile, String profile, String fileNameSuffix) {
if (outputFile == null) {
return outputDirectory.toPath().resolve("quarkus-" + profile + fileNameSuffix);
}
if (outputFile.isAbsolute()) {
return outputFile.toPath();
}
return outputDirectory.toPath().resolve(outputFile.toPath());
}

private static void updateChecksum(Checksum checksum, Iterable<Path> pc) throws IOException {
for (var path : sort(pc)) {
if (Files.isDirectory(path)) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
updateChecksum(checksum, stream);
}
} else {
checksum.update(Files.readAllBytes(path));
}
}
}

private static Iterable<Path> sort(Iterable<Path> original) {
var i = original.iterator();
if (!i.hasNext()) {
return List.of();
}
var o = i.next();
if (!i.hasNext()) {
return List.of(o);
}
final List<Path> sorted = new ArrayList<>();
sorted.add(o);
while (i.hasNext()) {
sorted.add(i.next());
}
Collections.sort(sorted);
return sorted;
}
}
5 changes: 5 additions & 0 deletions docs/src/main/asciidoc/config-reference.adoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,11 @@
The `track-config-changes` goal looks for `${project.basedir}/.quarkus/quarkus-prod-config-dump` (file name and directory are configurable) and, if the file already exists, checks whether the values stored in the config dump have changed.
It will log the changed options and save the current values of each of the options present in `${project.basedir}/.quarkus/quarkus-prod-config-dump` in `${project.basedir}/target/quarkus-prod-config.check` (the target file name and location can be configured). If the build time configuration has not changed since the last build both `${project.basedir}/.quarkus/quarkus-prod-config-dump` and `${project.basedir}/.quarkus/quarkus-prod-config-dump` will be identical.

==== Dump Quarkus application dependencies

In addition to dumping configuration values, `track-config-changes` goal also dumps all the Quarkus application dependencies, including Quarkus build time dependencies, along with their checksums (Adler32). This file could be used to check whether Quarkus build classpath has changed since the previous run.
By default, the dependency checksums will be stored under `target/quarkus-prod-dependency-checksums.txt` file. A different location could be configured using plugin parameters.

Check warning on line 744 in docs/src/main/asciidoc/config-reference.adoc

View workflow job for this annotation

GitHub Actions / Linting with Vale 

[vale] reported by reviewdog 🐶
[Quarkus.TermsSuggestions] Depending on the context, consider using 'by using' or 'that uses' rather than 'using'.

Raw Output:
{"message": "[Quarkus.TermsSuggestions] Depending on the context, consider using 'by using' or 'that uses' rather than 'using'.", "location": {"path": "docs/src/main/asciidoc/config-reference.adoc", "range": {"start": {"line": 744, "column": 152}}}, "severity": "INFO"}

==== Dump current build configuration when the recorded configuration isn't found

By default, `track-config-changes` looks for the configuration recorded during previous build and does nothing if it's not found. Enabling `dumpCurrentWhenRecordedUnavailable` parameter will make it dump the current build configuration
Expand Down
4 changes: 4 additions & 0 deletions ...rojects/bootstrap/app-model/src/main/java/io/quarkus/maven/dependency/ArtifactCoords.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ default boolean isJar() {
return TYPE_JAR.equals(getType());
}

default boolean isSnapshot() {
return getVersion() != null && getVersion().endsWith("-SNAPSHOT");
}

default String toGACTVString() {
return getGroupId() + ":" + getArtifactId() + ":" + getClassifier() + ":" + getType() + ":" + getVersion();
}
Expand Down
57 changes: 57 additions & 0 deletions independent-projects/parent/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,63 @@
</runtimeClassPath>
</normalization>
<plugins>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<inputs>
<fileSets>
<fileSet>
<name>dependency-checksums</name>
<paths>
<path>${project.build.directory}</path>
</paths>
<includes>
<include>quarkus-*-dependency-checksums.txt</include>
</includes>
<normalization>RELATIVE_PATH</normalization>
</fileSet>
</fileSets>
</inputs>
</plugin>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<inputs>
<fileSets>
<fileSet>
<name>dependency-checksums</name>
<paths>
<path>${project.build.directory}</path>
</paths>
<includes>
<include>quarkus-*-dependency-checksums.txt</include>
</includes>
<normalization>RELATIVE_PATH</normalization>
</fileSet>
</fileSets>
</inputs>
</plugin>
<plugin>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-maven-plugin</artifactId>
<executions>
<execution>
<id>default</id>
<inputs>
<fileSets>
<fileSet>
<name>dependency-checksums</name>
<paths>
<path>${project.build.directory}</path>
</paths>
<includes>
<include>quarkus-*-dependency-checksums.txt</include>
</includes>
<normalization>RELATIVE_PATH</normalization>
</fileSet>
</fileSets>
</inputs>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-maven-plugin</artifactId>
Expand Down
13 changes: 13 additions & 0 deletions integration-tests/main/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@

<!-- do not update this dependency, it is only used for testing -->
<webjar.jquery-ui.version>1.13.0</webjar.jquery-ui.version>

<quarkus.config-tracking.enabled>true</quarkus.config-tracking.enabled>
<quarkus.native.container-build>true</quarkus.native.container-build>
zakkak marked this conversation as resolved.
Show resolved Hide resolved
</properties>

<dependencies>
Expand Down Expand Up @@ -507,6 +510,16 @@
<groupId>io.quarkus</groupId>
<artifactId>quarkus-maven-plugin</artifactId>
<executions>
<execution>
<id>track-config-changes</id>
<phase>process-resources</phase>
<goals>
<goal>track-config-changes</goal>
</goals>
<configuration>
<dumpCurrentWhenRecordedUnavailable>true</dumpCurrentWhenRecordedUnavailable>
</configuration>
</execution>
<execution>
<goals>
<goal>build</goal>
Expand Down
13 changes: 12 additions & 1 deletion integration-tests/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,17 @@
<groupId>io.quarkus</groupId>
<artifactId>quarkus-maven-plugin</artifactId>
<version>${project.version}</version>
<executions>
<execution>
<!-- for now this execution will only dump Quarkus application
dependency checksums that could indicate build time classpath changes -->
<id>track-config-changes</id>
<phase>process-resources</phase>
<goals>
<goal>track-config-changes</goal>
</goals>
</execution>
</executions>
<configuration>
<noDeps>true</noDeps>
<skip>${quarkus.build.skip}</skip>
Expand Down Expand Up @@ -304,7 +315,7 @@
<module>amazon-lambda-rest-resteasy-reactive</module>
<module>amazon-lambda-rest-reactive-routes</module>
<module>amazon-lambda-rest-funqy</module>
<module>amazon-lambda-rest-servlet</module>
<module>amazon-lambda-rest-servlet</module>
<module>amazon-lambda-http-resteasy</module>
<module>amazon-lambda-http-resteasy-reactive</module>
<module>container-image</module>
Expand Down
Loading