Make QuarkusBuild not pollute Gradle's build cache

Currently the `QuarkusBuild` task implementation adds even large build artifacts and unmodified dependency jars to Gradle's build cache. This pollutes the Gradle build cache quite a lot and therefore lets archived build caches become unnecessary huge.

This change updates the build logic to fix this behavior by not adding dependencies and large build artifacts, like uber-jar and native binary, to the Gradle build cache.

The `QuarkusBuild` task has been split into three tasks:
1. a new `QuarkusBuildDependencies` task that only collects the contents for `build/quarkus-app/lib`
2. a new `QuarkusBuildApp` task that to collect everything else from a Quarkus build (everything except the `build/quarkus-app/lib`)
3. the `QuarkusBuild` task now combines the outputs of the above two tasks

`QuarkusBuildDependencies` (named 'quarkusDependenciesBuild`) is not cacheable, because it only collects dependencies, which come either from a repository (and are already available locally elsewhere) or are built by other Gradle tasks. This task is only executed if the configured Quarkus package type requires the "quarkus-app" directory (`fast-jar` + `native`). It's "build working directory" is `build/quarkus-build/dep`.

`QuarkusBuildApp` (named `quarkusAppBuild`) collects the contents of the "quarkus-app" directory _excluding_ the `lib/` directory are cacheable, which is the default for CI environments. Non-CI environments still cache all outputs, even uber-jars and native binaries to retain the existing behavior for developers and keep build turn-around times low. CI environments can opt-in to add even huge artifacts to Gradle's build cache by explicitly setting the `cacheUberAndNativeRunners` property in the Quarkus extension to `true`. It's "build working directory" is `build/quarkus-build/app`.

Since `QuarkusBuild` only combines the outputs of the above two tasks, the same "CI vs local" caching behavior as for the `QuarkusBuildApp` task applies. To make "up to date" checks (kind of) more reliable, all outputs are removed first. This means, that for example an existing uber-jar in `build/` will disappear, when the build's package type is "fast-jar". This behavior can be disabled by setting the `cleanupBuildOutput` property on the Quarkus extension to `false`.

Both `QuarkusBuildDependencies` and `QuarkusBuildApp` can trigger an actual Quarkus application build. That Quarkus app build will only be triggered when needed and only once per Gradle build.

The task names `quarkusDependenciesBuild` and `quarkusAppBuild` are intentionally "that way around". Letting the names of these tasks begin with `quarkusBuild...` could confuse users, who use abbreviated task names on the command line (for example `./gradlew qB` is automagically expanded to `./gradlew quarkusBuild`, which would become ambiguous with `quarkusBuildDependencies` and `quarkusBuildApp`).

Unless the `cacheLargeArtifacts` property on the `quarkus` extension is set to `true`, the output of/for the package type `fast-jar` minus the dependency jars is cached by Gradle's build cache (similar for `legacy-jar`).

Basically everything is cacheable to allow fast(er) local development turn-around cycles.

To be able to investigate which configuration settings were effectively used, there's another new task called `quarkusShowEffectiveConfig` that shows all the `quarkus.*` properties and some more information, including the loaded `application.(properties|yaml|yml)`. This task is intended to debug build issues. The task can optionally save the effecitve config properties as a properties file in the `build/` directory, if run with the command line option `./gradlew quarkusShowEffectiveConfig --save-config-properties`.

Relates to: #30852

core/deployment/src/main/java/io/quarkus/deployment/pkg/PackageConfig.java

@@ -78,6 +78,15 @@ private BuiltInType(final String name) {
         public String toString() {
             return name;
         }
+
+        public static BuiltInType fromString(String name) {
+            for (PackageConfig.BuiltInType type : values()) {
+                if (type.toString().equals(name)) {
+                    return type;
+                }
+            }
+            throw new IllegalArgumentException("Unknown Quarkus package type '" + name + "'");
+        }
     }
 
     /**

core/deployment/src/test/java/io/quarkus/deployment/pkg/BuiltInTypeTest.java

@@ -0,0 +1,23 @@
+package io.quarkus.deployment.pkg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+public class BuiltInTypeTest {
+
+    @ParameterizedTest
+    @EnumSource(PackageConfig.BuiltInType.class)
+    void packageTypeConversion(PackageConfig.BuiltInType packageType) {
+        assertThat(PackageConfig.BuiltInType.fromString(packageType.toString())).isSameAs(packageType);
+    }
+
+    @Test
+    void invalidPackageType() {
+        assertThatIllegalArgumentException().isThrownBy(() -> PackageConfig.BuiltInType.fromString("not-a-package-type"))
+                .withMessage("Unknown Quarkus package type 'not-a-package-type'");
+    }
+}

core/runtime/src/main/java/io/quarkus/runtime/configuration/ConfigInstantiator.java

@@ -54,6 +54,10 @@ public static <T> T handleObject(Supplier<T> supplier) {
 
     public static void handleObject(Object o) {
         final SmallRyeConfig config = (SmallRyeConfig) ConfigProvider.getConfig();
+        handleObject(o, config);
+    }
+
+    public static void handleObject(Object o, SmallRyeConfig config) {
         final String clsNameSuffix = getClassNameSuffix(o);
         if (clsNameSuffix == null) {
             // unsupported object type
@@ -190,7 +194,11 @@ private static Converter<?> getConverterFor(Type type, SmallRyeConfig config) {
         } else if (rawType == Optional.class) {
             return Converters.newOptionalConverter(getConverterFor(typeOfParameter(type, 0), config));
         } else if (rawType == List.class) {
-            return Converters.newCollectionConverter(getConverterFor(typeOfParameter(type, 0), config), ArrayList::new);
+            return Converters.newCollectionConverter(getConverterFor(typeOfParameter(type, 0), config),
+                    ConfigUtils.listFactory());
+        } else if (rawType == Set.class) {
+            return Converters.newCollectionConverter(getConverterFor(typeOfParameter(type, 0), config),
+                    ConfigUtils.setFactory());
         } else {
             return config.requireConverter(rawTypeOf(type));
         }

devtools/gradle/build.gradle

@@ -48,6 +48,7 @@ subprojects {
 
     javadoc {
         options.addStringOption('encoding', 'UTF-8')
+        options.addStringOption("Xdoclint:-reference", "-quiet")
     }
 }
 

devtools/gradle/gradle-application-plugin/build.gradle

@@ -7,6 +7,7 @@ dependencies {
     implementation "io.quarkus:quarkus-devtools-common:${version}"
     implementation "io.quarkus:quarkus-core-deployment:${version}"
     implementation "io.quarkus:quarkus-bootstrap-gradle-resolver:${version}"
+    implementation "io.smallrye.config:smallrye-config-source-yaml:${smallrye_config_version}"
 
     implementation project(":gradle-model")
 

devtools/gradle/gradle-application-plugin/pom.xml

@@ -54,6 +54,10 @@
             <artifactId>quarkus-devmode-test-utils</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>io.smallrye.config</groupId>
+            <artifactId>smallrye-config-source-yaml</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.jetbrains.kotlin</groupId>
             <artifactId>kotlin-gradle-plugin</artifactId>

devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/QuarkusPlugin.java

@@ -36,7 +36,8 @@
 import io.quarkus.gradle.tasks.ImagePush;
 import io.quarkus.gradle.tasks.QuarkusAddExtension;
 import io.quarkus.gradle.tasks.QuarkusBuild;
-import io.quarkus.gradle.tasks.QuarkusBuildConfiguration;
+import io.quarkus.gradle.tasks.QuarkusBuildCacheableAppParts;
+import io.quarkus.gradle.tasks.QuarkusBuildDependencies;
 import io.quarkus.gradle.tasks.QuarkusDev;
 import io.quarkus.gradle.tasks.QuarkusGenerateCode;
 import io.quarkus.gradle.tasks.QuarkusGoOffline;
@@ -46,6 +47,7 @@
 import io.quarkus.gradle.tasks.QuarkusListPlatforms;
 import io.quarkus.gradle.tasks.QuarkusRemoteDev;
 import io.quarkus.gradle.tasks.QuarkusRemoveExtension;
+import io.quarkus.gradle.tasks.QuarkusShowEffectiveConfig;
 import io.quarkus.gradle.tasks.QuarkusTest;
 import io.quarkus.gradle.tasks.QuarkusTestConfig;
 import io.quarkus.gradle.tasks.QuarkusUpdate;
@@ -55,7 +57,10 @@
 public class QuarkusPlugin implements Plugin<Project> {
 
     public static final String ID = "io.quarkus";
-    public static final String QUARKUS_PACKAGE_TYPE = "quarkus.package.type";
+    public static final String DEFAULT_OUTPUT_DIRECTORY = "quarkus-app";
+    public static final String CLASS_LOADING_REMOVED_ARTIFACTS = "quarkus.class-loading.removed-artifacts";
+    public static final String CLASS_LOADING_PARENT_FIRST_ARTIFACTS = "quarkus.class-loading.parent-first-artifacts";
+    public static final String QUARKUS_ARTIFACT_PROPERTIES = "quarkus-artifact.properties";
 
     public static final String EXTENSION_NAME = "quarkus";
     public static final String LIST_EXTENSIONS_TASK_NAME = "listExtensions";
@@ -66,6 +71,9 @@ public class QuarkusPlugin implements Plugin<Project> {
     public static final String QUARKUS_GENERATE_CODE_TASK_NAME = "quarkusGenerateCode";
     public static final String QUARKUS_GENERATE_CODE_DEV_TASK_NAME = "quarkusGenerateCodeDev";
     public static final String QUARKUS_GENERATE_CODE_TESTS_TASK_NAME = "quarkusGenerateCodeTests";
+    public static final String QUARKUS_BUILD_DEP_TASK_NAME = "quarkusDependenciesBuild";
+    public static final String QUARKUS_BUILD_APP_TASK_NAME = "quarkusAppBuild";
+    public static final String QUARKUS_SHOW_EFFECTIVE_CONFIG_TASK_NAME = "quarkusShowEffectiveConfig";
     public static final String QUARKUS_BUILD_TASK_NAME = "quarkusBuild";
     public static final String QUARKUS_DEV_TASK_NAME = "quarkusDev";
     public static final String QUARKUS_REMOTE_DEV_TASK_NAME = "quarkusRemoteDev";
@@ -96,6 +104,7 @@ public class QuarkusPlugin implements Plugin<Project> {
 
     private final ToolingModelBuilderRegistry registry;
 
+    @SuppressWarnings("CdiInjectionPointsInspection")
     @Inject
     public QuarkusPlugin(ToolingModelBuilderRegistry registry) {
         this.registry = registry;
@@ -148,22 +157,38 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) {
         TaskProvider<QuarkusGenerateCode> quarkusGenerateCodeTests = tasks.register(QUARKUS_GENERATE_CODE_TESTS_TASK_NAME,
                 QuarkusGenerateCode.class, task -> task.setTest(true));
 
-        QuarkusBuildConfiguration buildConfig = new QuarkusBuildConfiguration(project);
+        TaskProvider<QuarkusBuildDependencies> quarkusBuildDep = tasks.register(QUARKUS_BUILD_DEP_TASK_NAME,
+                QuarkusBuildDependencies.class,
+                task -> task.getOutputs().doNotCacheIf("Dependencies are never cached", t -> true));
+
+        tasks.register(QUARKUS_SHOW_EFFECTIVE_CONFIG_TASK_NAME,
+                QuarkusShowEffectiveConfig.class, task -> {
+                    task.setDescription("Show effective Quarkus build configuration.");
+                });
+
+        TaskProvider<QuarkusBuildCacheableAppParts> quarkusBuildApp = tasks.register(QUARKUS_BUILD_APP_TASK_NAME,
+                QuarkusBuildCacheableAppParts.class, task -> {
+                    task.dependsOn(quarkusGenerateCode);
+                    task.getOutputs().doNotCacheIf(
+                            "Not adding uber-jars, native binaries and mutable-jar package type to Gradle " +
+                                    "build cache by default. To allow caching of uber-jars, native binaries and mutable-jar " +
+                                    "package type, set 'cacheUberAndNativeRunners' in the 'quarkus' Gradle extension to 'true'.",
+                            t -> !task.isCachedByDefault() && !quarkusExt.getCacheLargeArtifacts().get());
+                });
 
         TaskProvider<QuarkusBuild> quarkusBuild = tasks.register(QUARKUS_BUILD_TASK_NAME, QuarkusBuild.class, build -> {
-            build.dependsOn(quarkusGenerateCode);
-            build.getForcedProperties().set(buildConfig.getForcedProperties());
+            build.dependsOn(quarkusBuildDep, quarkusBuildApp);
+            build.getOutputs().doNotCacheIf(
+                    "Only collects and combines the outputs of " + QUARKUS_BUILD_APP_TASK_NAME + " and "
+                            + QUARKUS_BUILD_DEP_TASK_NAME,
+                    t -> !quarkusExt.getCacheLargeArtifacts().get());
         });
 
-        TaskProvider<ImageBuild> imageBuild = tasks.register(IMAGE_BUILD_TASK_NAME, ImageBuild.class, buildConfig);
-        imageBuild.configure(task -> {
-            task.finalizedBy(quarkusBuild);
-        });
+        tasks.register(IMAGE_BUILD_TASK_NAME, ImageBuild.class,
+                task -> task.finalizedBy(quarkusBuildApp, quarkusBuildDep, quarkusBuild));
 
-        TaskProvider<ImagePush> imagePush = tasks.register(IMAGE_PUSH_TASK_NAME, ImagePush.class, buildConfig);
-        imagePush.configure(task -> {
-            task.finalizedBy(quarkusBuild);
-        });
+        tasks.register(IMAGE_PUSH_TASK_NAME, ImagePush.class,
+                task -> task.finalizedBy(quarkusBuildApp, quarkusBuildDep, quarkusBuild));
 
         TaskProvider<QuarkusDev> quarkusDev = tasks.register(QUARKUS_DEV_TASK_NAME, QuarkusDev.class, devRuntimeDependencies,
                 quarkusExt);
@@ -180,7 +205,6 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) {
 
         });
 
-        configureBuildNativeTask(project);
         project.getPlugins().withType(
                 BasePlugin.class,
                 basePlugin -> tasks.named(BasePlugin.ASSEMBLE_TASK_NAME, task -> task.dependsOn(quarkusBuild)));
@@ -242,7 +266,7 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) {
                                 quarkusGenerateCode,
                                 quarkusGenerateCodeTests);
                     });
-                    quarkusBuild.configure(
+                    quarkusBuildApp.configure(
                             task -> task.dependsOn(classesTask, resourcesTask, tasks.named(JavaPlugin.JAR_TASK_NAME)));
 
                     SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
@@ -310,7 +334,7 @@ public void execute(Task task) {
                         t.useJUnitPlatform();
                     });
                     // quarkusBuild is expected to run after the project has passed the tests
-                    quarkusBuild.configure(task -> task.shouldRunAfter(tasks.withType(Test.class)));
+                    quarkusBuildApp.configure(task -> task.shouldRunAfter(tasks.withType(Test.class)));
 
                     SourceSet generatedSourceSet = sourceSets.create(QuarkusGenerateCode.QUARKUS_GENERATED_SOURCES);
                     SourceSet generatedTestSourceSet = sourceSets.create(QuarkusGenerateCode.QUARKUS_TEST_GENERATED_SOURCES);
@@ -378,16 +402,6 @@ private void verifyGradleVersion() {
         }
     }
 
-    private void configureBuildNativeTask(Project project) {
-        project.getGradle().getTaskGraph().whenReady(taskGraph -> {
-            if (taskGraph.hasTask(project.getPath() + BUILD_NATIVE_TASK_NAME)
-                    || taskGraph.hasTask(project.getPath() + TEST_NATIVE_TASK_NAME)) {
-                project.getExtensions().getExtraProperties()
-                        .set(QUARKUS_PACKAGE_TYPE, "native");
-            }
-        });
-    }
-
     private void afterEvaluate(Project project) {
 
         visitProjectDependencies(project, project, new HashSet<>());