Revert "ci: install and test only necessary modules (#10803)" (#10810)

This reverts commit 2145857.
googleapis · May 8, 2024 · 55367d6 · 55367d6
1 parent 475c9ba
commit 55367d6
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 92 deletions.
diff --git a/.cloud/setup.sh b/.cloud/setup.sh
@@ -29,5 +29,5 @@ popd >/dev/null
 # Ensure all SNAPSHOTs are available in the local mvn repository
 pushd "$scriptDir/.." >/dev/null
 source ./.kokoro/common.sh
-install_modules "$1"
+install_modules
 popd >/dev/null
diff --git a/.kokoro/build.sh b/.kokoro/build.sh
@@ -38,17 +38,16 @@ RETURN_CODE=0
 case ${JOB_TYPE} in
   test)
     retry_with_backoff 3 10 \
-      mvn test \
-        -B -ntp \
-        -Dorg.slf4j.simpleLogger.showDateTime=true \
-        -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
-        -Dclirr.skip=true \
-        -Denforcer.skip=true \
-        -Dcheckstyle.skip=true \
-        -Dflatten.skip=true \
-        -Danimal.sniffer.skip=true \
-        -Dmaven.wagon.http.retryHandler.count=5 \
-        -T 1C
+      mvn -B -ntp \
+      -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+      -Dclirr.skip=true \
+      -Denforcer.skip=true \
+      -Dcheckstyle.skip=true \
+      -Dflatten.skip=true \
+      -Danimal.sniffer.skip=true \
+      -Dmaven.wagon.http.retryHandler.count=5 \
+      -T 1C \
+      test
     RETURN_CODE=$?
     echo "Finished running unit tests"
     ;;
@@ -60,7 +59,7 @@ case ${JOB_TYPE} in
         echo "${modified_module_list[*]}"
       )
       setup_cloud "$module_list"
-      install_modules "$module_list"
+      install_modules
       run_integration_tests "$module_list"
     else
       echo "No Integration Tests to run"
@@ -70,20 +69,31 @@ case ${JOB_TYPE} in
     generate_graalvm_presubmit_modules_list
     printf "Running GraalVM presubmit checks for:\n%s\n" "${module_list}"
     setup_cloud "$module_list"
-    install_modules "$module_list"
+    install_modules
     run_graalvm_tests "$module_list"
     ;;
   graalvm)
     generate_graalvm_modules_list
     if [ ! -z "${module_list}" ]; then
       printf "Running GraalVM checks for:\n%s\n" "${module_list}"
       setup_cloud "$module_list"
-      install_modules "$module_list"
+      install_modules
       run_graalvm_tests "$module_list"
     else
       echo "Not running GraalVM checks -- No changes in relevant modules"
     fi
     ;;
+  graalvm17)
+    generate_graalvm_modules_list
+    if [ ! -z "${module_list}" ]; then
+      printf "Running GraalVM 17 checks for:\n%s\n" "${module_list}"
+      setup_cloud "$module_list"
+      install_modules
+      run_graalvm_tests "$module_list"
+    else
+      echo "Not running GraalVM 17 checks -- No changes in relevant modules"
+    fi
+    ;;
   *) ;;
 
 esac

diff --git a/.kokoro/common.sh b/.kokoro/common.sh
@@ -13,7 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent')
+# TODO: remove java-core once we figure out how setup_cloud understands Maven's
+# "--also-make-dependents" option. https://github.com/googleapis/google-cloud-java/issues/9088
+excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent' 'java-core')
 
 function retry_with_backoff {
   attempts_left=$1
@@ -126,48 +128,46 @@ function generate_modified_modules_list() {
 function run_integration_tests() {
   printf "Running Integration Tests for:\n%s\n" "$1"
   # --also-make-dependents to run other modules that use the affected module
-  mvn verify -Penable-integration-tests --projects "$1" \
-    ${INTEGRATION_TEST_ARGS} \
-    -B -ntp -fae \
+  mvn -B ${INTEGRATION_TEST_ARGS} \
+    -pl "$1" \
+    --also-make-dependents \
+    -ntp \
+    -Penable-integration-tests \
     -DtrimStackTrace=false \
     -Dclirr.skip=true \
     -Denforcer.skip=true \
-    -Dorg.slf4j.simpleLogger.showDateTime=true \
-    -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
     -Dcheckstyle.skip=true \
     -Dflatten.skip=true \
     -Danimal.sniffer.skip=true \
     -Djacoco.skip=true \
     -DskipUnitTests=true \
     -Dmaven.wagon.http.retryHandler.count=5 \
-    -T 1C
+    -fae \
+    -T 1C \
+    verify
 
   RETURN_CODE=$?
   printf "Finished Integration Tests for:\n%s\n" "$1"
 }
 
 function run_graalvm_tests() {
-  IFS=, read -ra individual_modules <<< "$1"
-  projects_to_install=()
-  for module in "${individual_modules[@]}"; do
-    p=$(echo "$module" | cut -d '-' -f2-) # Convert from 'java-x-y-z' to 'x-y-z'
-    projects_to_install+=("java-$p/google-cloud-$p"); # Create reference to GAPIC project
-  done
-  IFS=, gapic_projects="${projects_to_install[*]}" # Convert to comma-separated string
-
-  printf "Running GraalVM ITs on:\n%s\n" "$gapic_projects"
+  printf "Running GraalVM ITs on:\n%s\n" "$1"
 
-  mvn test -Pnative --projects "$gapic_projects" \
-    ${INTEGRATION_TEST_ARGS} \
-    -B -ntp -fae \
+  mvn -B ${INTEGRATION_TEST_ARGS} \
+    -pl "$1" \
+    --also-make-dependents \
+    -ntp \
     -DtrimStackTrace=false \
     -Dclirr.skip=true \
     -Denforcer.skip=true \
-    -Dorg.slf4j.simpleLogger.showDateTime=true \
-    -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
     -Dcheckstyle.skip=true \
     -Dflatten.skip=true \
-    -Danimal.sniffer.skip=true
+    -Danimal.sniffer.skip=true \
+    -Pnative \
+    -fae \
+    test
 
   RETURN_CODE=$?
   printf "Finished Unit and Integration Tests for GraalVM:\n%s\n" "$1"
@@ -176,7 +176,7 @@ function run_graalvm_tests() {
 function generate_graalvm_presubmit_modules_list() {
   modules_assigned_list=()
   generate_modified_modules_list
-  if [[ ${#modified_module_list[@]} -gt 0 && ${#modified_module_list[@]} -lt 5 ]]; then
+  if [[ ${#modified_module_list[@]} -gt 0 && ${#modified_module_list[@]} -lt 10 ]]; then
     # If only a few modules have been modified, focus presubmit testing only on them.
     module_list=$(
       IFS=,
@@ -229,56 +229,18 @@ function generate_graalvm_modules_list() {
 }
 
 function install_modules() {
-  if [ -z "$1" ]; then
-    mvn install \
-      -B -ntp \
-      -DtrimStackTrace=false \
-      -Dclirr.skip=true \
-      -Denforcer.skip=true \
-      -Dorg.slf4j.simpleLogger.showDateTime=true \
-      -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
-      -Dcheckstyle.skip=true \
-      -Dflatten.skip=true \
-      -Danimal.sniffer.skip=true \
-      -DskipTests=true \
-      -Djacoco.skip=true \
-      -T 1C
-  else
-    IFS=, read -ra individual_modules <<< "$1"
-    projects_to_install=()
-    for module in "${individual_modules[@]}"; do
-      p=$(echo "$module" | cut -d '-' -f2-) # Convert from 'java-x-y-z' to 'x-y-z'
-      projects_to_install+=("java-$p/google-cloud-$p"); # Create reference to GAPIC project
-    done
-    IFS=, gapic_projects="${projects_to_install[*]}" # Convert to comma-separated string
-
-    # When working with a maven multi-module project containing other multi-module projects,
-    # to build a module with its dependencies and without building its dependents:
-    # Perform the install command on a grandchild module with the --also-make flag.
-    #
-    # Examples:
-    #
-    #   mvn install --projects java-asset --also-make
-    #      ! Does not work. Maven reactor will not build java-asset's child modules, such as the
-    #        gapic, proto, and grpc modules.
-    #
-    #   mvn install --projects java-kms --also-make-dependents
-    #      ! Does not work. Maven reactor will include java-kmsinventory in its build.
-    #
-    #   mvn install --projects java-kms/google-cloud-kms --also-make
-    #      Correctly builds dependencies without building dependents.
-    mvn install --projects "$gapic_projects" --also-make \
-      -B -ntp \
-      -DtrimStackTrace=false \
-      -Dclirr.skip=true \
-      -Denforcer.skip=true \
-      -Dorg.slf4j.simpleLogger.showDateTime=true \
-      -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
-      -Dcheckstyle.skip=true \
-      -Dflatten.skip=true \
-      -Danimal.sniffer.skip=true \
-      -DskipTests=true \
-      -Djacoco.skip=true \
-      -T 1C
-  fi
+  retry_with_backoff 3 10 \
+    mvn -B \
+    -ntp \
+    -DtrimStackTrace=false \
+    -Dclirr.skip=true \
+    -Denforcer.skip=true \
+    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+    -Dcheckstyle.skip=true \
+    -Dflatten.skip=true \
+    -Danimal.sniffer.skip=true \
+    -DskipTests=true \
+    -Djacoco.skip=true \
+    -T 1C \
+    install
 }
diff --git a/.kokoro/nightly/graalvm-native-17.cfg b/.kokoro/nightly/graalvm-native-17.cfg
@@ -8,7 +8,7 @@ env_vars: {
 
 env_vars: {
   key: "JOB_TYPE"
-  value: "graalvm"
+  value: "graalvm17"
 }
 
 # TODO: remove this after we've migrated all tests and scripts

diff --git a/.kokoro/nightly/graalvm-sub-jobs/native-17/common.cfg b/.kokoro/nightly/graalvm-sub-jobs/native-17/common.cfg
@@ -29,7 +29,7 @@ env_vars: {
 
 env_vars: {
   key: "JOB_TYPE"
-  value: "graalvm"
+  value: "graalvm17"
 }
 
 # TODO: remove this after we've migrated all tests and scripts