ci: install and test only necessary modules (#10803)

* ci: install and test only necessary modules * ci: debugging mvn install * ci: remove quotes from INTEGRATION_TEST_ARGS * ci: remove debugging lines
googleapis · May 7, 2024 · 2145857 · 2145857
1 parent 6b1e9fa
commit 2145857
Show file tree

Hide file tree

Showing 5 changed files with 92 additions and 64 deletions.
diff --git a/.cloud/setup.sh b/.cloud/setup.sh
@@ -29,5 +29,5 @@ popd >/dev/null
 # Ensure all SNAPSHOTs are available in the local mvn repository
 pushd "$scriptDir/.." >/dev/null
 source ./.kokoro/common.sh
-install_modules
+install_modules "$1"
 popd >/dev/null
diff --git a/.kokoro/build.sh b/.kokoro/build.sh
@@ -38,16 +38,17 @@ RETURN_CODE=0
 case ${JOB_TYPE} in
   test)
     retry_with_backoff 3 10 \
-      mvn -B -ntp \
-      -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
-      -Dclirr.skip=true \
-      -Denforcer.skip=true \
-      -Dcheckstyle.skip=true \
-      -Dflatten.skip=true \
-      -Danimal.sniffer.skip=true \
-      -Dmaven.wagon.http.retryHandler.count=5 \
-      -T 1C \
-      test
+      mvn test \
+        -B -ntp \
+        -Dorg.slf4j.simpleLogger.showDateTime=true \
+        -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+        -Dclirr.skip=true \
+        -Denforcer.skip=true \
+        -Dcheckstyle.skip=true \
+        -Dflatten.skip=true \
+        -Danimal.sniffer.skip=true \
+        -Dmaven.wagon.http.retryHandler.count=5 \
+        -T 1C
     RETURN_CODE=$?
     echo "Finished running unit tests"
     ;;
@@ -59,7 +60,7 @@ case ${JOB_TYPE} in
         echo "${modified_module_list[*]}"
       )
       setup_cloud "$module_list"
-      install_modules
+      install_modules "$module_list"
       run_integration_tests "$module_list"
     else
       echo "No Integration Tests to run"
@@ -69,31 +70,20 @@ case ${JOB_TYPE} in
     generate_graalvm_presubmit_modules_list
     printf "Running GraalVM presubmit checks for:\n%s\n" "${module_list}"
     setup_cloud "$module_list"
-    install_modules
+    install_modules "$module_list"
     run_graalvm_tests "$module_list"
     ;;
   graalvm)
     generate_graalvm_modules_list
     if [ ! -z "${module_list}" ]; then
       printf "Running GraalVM checks for:\n%s\n" "${module_list}"
       setup_cloud "$module_list"
-      install_modules
+      install_modules "$module_list"
       run_graalvm_tests "$module_list"
     else
       echo "Not running GraalVM checks -- No changes in relevant modules"
     fi
     ;;
-  graalvm17)
-    generate_graalvm_modules_list
-    if [ ! -z "${module_list}" ]; then
-      printf "Running GraalVM 17 checks for:\n%s\n" "${module_list}"
-      setup_cloud "$module_list"
-      install_modules
-      run_graalvm_tests "$module_list"
-    else
-      echo "Not running GraalVM 17 checks -- No changes in relevant modules"
-    fi
-    ;;
   *) ;;
 
 esac

diff --git a/.kokoro/common.sh b/.kokoro/common.sh
@@ -13,9 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO: remove java-core once we figure out how setup_cloud understands Maven's
-# "--also-make-dependents" option. https://github.com/googleapis/google-cloud-java/issues/9088
-excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent' 'java-core')
+excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent')
 
 function retry_with_backoff {
   attempts_left=$1
@@ -128,46 +126,48 @@ function generate_modified_modules_list() {
 function run_integration_tests() {
   printf "Running Integration Tests for:\n%s\n" "$1"
   # --also-make-dependents to run other modules that use the affected module
-  mvn -B ${INTEGRATION_TEST_ARGS} \
-    -pl "$1" \
-    --also-make-dependents \
-    -ntp \
-    -Penable-integration-tests \
+  mvn verify -Penable-integration-tests --projects "$1" \
+    ${INTEGRATION_TEST_ARGS} \
+    -B -ntp -fae \
     -DtrimStackTrace=false \
     -Dclirr.skip=true \
     -Denforcer.skip=true \
-    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+    -Dorg.slf4j.simpleLogger.showDateTime=true \
+    -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
     -Dcheckstyle.skip=true \
     -Dflatten.skip=true \
     -Danimal.sniffer.skip=true \
     -Djacoco.skip=true \
     -DskipUnitTests=true \
     -Dmaven.wagon.http.retryHandler.count=5 \
-    -fae \
-    -T 1C \
-    verify
+    -T 1C
 
   RETURN_CODE=$?
   printf "Finished Integration Tests for:\n%s\n" "$1"
 }
 
 function run_graalvm_tests() {
-  printf "Running GraalVM ITs on:\n%s\n" "$1"
+  IFS=, read -ra individual_modules <<< "$1"
+  projects_to_install=()
+  for module in "${individual_modules[@]}"; do
+    p=$(echo "$module" | cut -d '-' -f2-) # Convert from 'java-x-y-z' to 'x-y-z'
+    projects_to_install+=("java-$p/google-cloud-$p"); # Create reference to GAPIC project
+  done
+  IFS=, gapic_projects="${projects_to_install[*]}" # Convert to comma-separated string
+
+  printf "Running GraalVM ITs on:\n%s\n" "$gapic_projects"
 
-  mvn -B ${INTEGRATION_TEST_ARGS} \
-    -pl "$1" \
-    --also-make-dependents \
-    -ntp \
+  mvn test -Pnative --projects "$gapic_projects" \
+    ${INTEGRATION_TEST_ARGS} \
+    -B -ntp -fae \
     -DtrimStackTrace=false \
     -Dclirr.skip=true \
     -Denforcer.skip=true \
-    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+    -Dorg.slf4j.simpleLogger.showDateTime=true \
+    -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
     -Dcheckstyle.skip=true \
     -Dflatten.skip=true \
-    -Danimal.sniffer.skip=true \
-    -Pnative \
-    -fae \
-    test
+    -Danimal.sniffer.skip=true
 
   RETURN_CODE=$?
   printf "Finished Unit and Integration Tests for GraalVM:\n%s\n" "$1"
@@ -176,7 +176,7 @@ function run_graalvm_tests() {
 function generate_graalvm_presubmit_modules_list() {
   modules_assigned_list=()
   generate_modified_modules_list
-  if [[ ${#modified_module_list[@]} -gt 0 && ${#modified_module_list[@]} -lt 10 ]]; then
+  if [[ ${#modified_module_list[@]} -gt 0 && ${#modified_module_list[@]} -lt 5 ]]; then
     # If only a few modules have been modified, focus presubmit testing only on them.
     module_list=$(
       IFS=,
@@ -229,18 +229,56 @@ function generate_graalvm_modules_list() {
 }
 
 function install_modules() {
-  retry_with_backoff 3 10 \
-    mvn -B \
-    -ntp \
-    -DtrimStackTrace=false \
-    -Dclirr.skip=true \
-    -Denforcer.skip=true \
-    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
-    -Dcheckstyle.skip=true \
-    -Dflatten.skip=true \
-    -Danimal.sniffer.skip=true \
-    -DskipTests=true \
-    -Djacoco.skip=true \
-    -T 1C \
-    install
+  if [ -z "$1" ]; then
+    mvn install \
+      -B -ntp \
+      -DtrimStackTrace=false \
+      -Dclirr.skip=true \
+      -Denforcer.skip=true \
+      -Dorg.slf4j.simpleLogger.showDateTime=true \
+      -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+      -Dcheckstyle.skip=true \
+      -Dflatten.skip=true \
+      -Danimal.sniffer.skip=true \
+      -DskipTests=true \
+      -Djacoco.skip=true \
+      -T 1C
+  else
+    IFS=, read -ra individual_modules <<< "$1"
+    projects_to_install=()
+    for module in "${individual_modules[@]}"; do
+      p=$(echo "$module" | cut -d '-' -f2-) # Convert from 'java-x-y-z' to 'x-y-z'
+      projects_to_install+=("java-$p/google-cloud-$p"); # Create reference to GAPIC project
+    done
+    IFS=, gapic_projects="${projects_to_install[*]}" # Convert to comma-separated string
+
+    # When working with a maven multi-module project containing other multi-module projects,
+    # to build a module with its dependencies and without building its dependents:
+    # Perform the install command on a grandchild module with the --also-make flag.
+    #
+    # Examples:
+    #
+    #   mvn install --projects java-asset --also-make
+    #      ! Does not work. Maven reactor will not build java-asset's child modules, such as the
+    #        gapic, proto, and grpc modules.
+    #
+    #   mvn install --projects java-kms --also-make-dependents
+    #      ! Does not work. Maven reactor will include java-kmsinventory in its build.
+    #
+    #   mvn install --projects java-kms/google-cloud-kms --also-make
+    #      Correctly builds dependencies without building dependents.
+    mvn install --projects "$gapic_projects" --also-make \
+      -B -ntp \
+      -DtrimStackTrace=false \
+      -Dclirr.skip=true \
+      -Denforcer.skip=true \
+      -Dorg.slf4j.simpleLogger.showDateTime=true \
+      -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+      -Dcheckstyle.skip=true \
+      -Dflatten.skip=true \
+      -Danimal.sniffer.skip=true \
+      -DskipTests=true \
+      -Djacoco.skip=true \
+      -T 1C
+  fi
 }
diff --git a/.kokoro/nightly/graalvm-native-17.cfg b/.kokoro/nightly/graalvm-native-17.cfg
@@ -8,7 +8,7 @@ env_vars: {
 
 env_vars: {
   key: "JOB_TYPE"
-  value: "graalvm17"
+  value: "graalvm"
 }
 
 # TODO: remove this after we've migrated all tests and scripts

diff --git a/.kokoro/nightly/graalvm-sub-jobs/native-17/common.cfg b/.kokoro/nightly/graalvm-sub-jobs/native-17/common.cfg
@@ -29,7 +29,7 @@ env_vars: {
 
 env_vars: {
   key: "JOB_TYPE"
-  value: "graalvm17"
+  value: "graalvm"
 }
 
 # TODO: remove this after we've migrated all tests and scripts