chore: recursive comparison in hermetic build IT (#2542)

Fixes #2540 

Sample error message when running
`library_generation/test/integration_tests.py`
```
Generation finished successfully. Will now compare differences between generated and existing libraries
******************************
Checking for differences in 'java-apigee-connect'.
  The expected library is in /usr/local/google/home/diegomarquezp/Desktop/sdk-platform-java/library_generation/test/resources/integration/golden/java-apigee-connect.
  The actual library is in /usr/local/google/home/diegomarquezp/Desktop/sdk-platform-java/library_generation/output/google-cloud-java/java-apigee-connect. 
  Some files (found in both folders) are differing:
   -  README.md
   -  proto-google-cloud-apigee-connect-v1/src/main/java/com/google/cloud/apigeeconnect/v1/ConnectionProto.java
```

library_generation/test/integration_tests.py

@@ -76,7 +76,7 @@ def test_get_commit_message_success(self):
                 cmp(
                     f"{config_dir}/{repo}/pr-description-golden.txt",
                     f"{description_file}",
-                )
+                ), "The generated PR description does not match the expected golden file"
             )
             os.remove(f"{description_file}")
 
@@ -107,16 +107,17 @@ def test_generate_repo(self):
                 generation_config_yaml=config_file, repository_path=repo_dest
             )
             # compare result
+            print(
+                "Generation finished successfully. Will now compare differences between generated and existing libraries"
+            )
             for library_name in library_names:
                 actual_library = (
                     f"{repo_dest}/{library_name}" if config.is_monorepo else repo_dest
                 )
-                print(
-                    f"Generation finished. Will now compare "
-                    f"the expected library in {golden_dir}/{library_name}, "
-                    f"with the actual library in {actual_library}. "
-                    f"Compare generation result: "
-                )
+                print("*" * 50)
+                print(f"Checking for differences in '{library_name}'.")
+                print(f"  The expected library is in {golden_dir}/{library_name}.")
+                print(f"  The actual library is in {actual_library}. ")
                 target_repo_dest = (
                     f"{repo_dest}/{library_name}" if config.is_monorepo else repo_dest
                 )
@@ -125,20 +126,40 @@ def test_generate_repo(self):
                     target_repo_dest,
                     ignore=[".repo-metadata.json"],
                 )
+                diff_files = []
+                golden_only = []
+                generated_only = []
                 # compare source code
-                self.assertEqual([], compare_result.left_only)
-                self.assertEqual([], compare_result.right_only)
-                self.assertEqual([], compare_result.diff_files)
-                print("Source code comparison succeed.")
+                self.__recursive_diff_files(
+                    compare_result, diff_files, golden_only, generated_only
+                )
+
+                # print all found differences for inspection
+                print_file = lambda f: print(f"   -  {f}")
+                if len(diff_files) > 0:
+                    print("  Some files (found in both folders) are differing:")
+                    [print_file(f) for f in diff_files]
+                if len(golden_only) > 0:
+                    print("  There were files found only in the golden dir:")
+                    [print_file(f) for f in golden_only]
+                if len(generated_only) > 0:
+                    print("  Some files were found to have differences:")
+                    [print_file(f) for f in generated_only]
+
+                self.assertTrue(len(golden_only) == 0)
+                self.assertTrue(len(generated_only) == 0)
+                self.assertTrue(len(diff_files) == 0)
+
+                print("  No differences found in {library_name}")
                 # compare .repo-metadata.json
                 self.assertTrue(
                     self.__compare_json_files(
                         f"{golden_dir}/{library_name}/.repo-metadata.json",
                         f"{target_repo_dest}/.repo-metadata.json",
                     ),
-                    msg=f"The generated {library_name}/.repo-metadata.json is different from golden.",
+                    msg=f"  The generated {library_name}/.repo-metadata.json is different from golden.",
                 )
-                print(".repo-metadata.json comparison succeed.")
+                print("  .repo-metadata.json comparison succeed.")
 
                 if not config.is_monorepo:
                     continue
@@ -151,15 +172,15 @@ def test_generate_repo(self):
                         False,
                     )
                 )
-                print("gapic-libraries-bom/pom.xml comparison succeed.")
+                print("  gapic-libraries-bom/pom.xml comparison succeed.")
                 self.assertFalse(
                     compare_xml(
                         f"{golden_dir}/pom.xml",
                         f"{repo_dest}/pom.xml",
                         False,
                     )
                 )
-                print("pom.xml comparison succeed.")
+                print("  pom.xml comparison succeed.")
 
     @classmethod
     def __pull_repo_to(cls, default_dest: Path, repo: str, committish: str) -> str:
@@ -203,7 +224,7 @@ def __get_config_files(cls, path: str) -> List[tuple[str, str]]:
             if sub_dir.is_file():
                 continue
             repo = sub_dir.name
-            if repo == "golden" or repo == "java-bigtable":
+            if repo in ["golden", "java-bigtable"]:
                 continue
             config = f"{sub_dir}/{config_name}"
             config_files.append((repo, config))
@@ -223,3 +244,24 @@ def __load_json_to_sorted_list(cls, path: str) -> List[tuple]:
         res = [(key, value) for key, value in data.items()]
 
         return sorted(res, key=lambda x: x[0])
+
+    @classmethod
+    def __recursive_diff_files(
+        self,
+        dcmp: dircmp,
+        diff_files: List[str],
+        left_only: List[str],
+        right_only: List[str],
+        dirname: str = "",
+    ):
+        """
+        recursively compares two subdirectories. The found differences are passed to three expected list references
+        """
+        append_dirname = lambda d: dirname + d
+        diff_files.extend(map(append_dirname, dcmp.diff_files))
+        left_only.extend(map(append_dirname, dcmp.left_only))
+        right_only.extend(map(append_dirname, dcmp.right_only))
+        for sub_dirname, sub_dcmp in dcmp.subdirs.items():
+            self.__recursive_diff_files(
+                sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/"
+            )

library_generation/test/resources/integration/google-cloud-java/generation_config.yaml

@@ -1,6 +1,6 @@
-gapic_generator_version: 2.34.0
+gapic_generator_version: 2.37.0
 protobuf_version: 25.2
-googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
+googleapis_commitish: 9868a57470a969ffa1d21194a5c05d7a6e4e98cc
 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
 synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e
 template_excludes:
@@ -48,17 +48,3 @@ libraries:
     - proto_path: google/cloud/alloydb/connectors/v1
     - proto_path: google/cloud/alloydb/connectors/v1alpha
     - proto_path: google/cloud/alloydb/connectors/v1beta
-
-  - api_shortname: documentai
-    name_pretty: Document AI
-    product_documentation: https://cloud.google.com/compute/docs/documentai/
-    api_description: allows developers to unlock insights from your documents with machine
-        learning.
-    library_name: document-ai
-    release_level: stable
-    issue_tracker: https://issuetracker.google.com/savedsearches/559755
-    GAPICs:
-    - proto_path: google/cloud/documentai/v1
-    - proto_path: google/cloud/documentai/v1beta1
-    - proto_path: google/cloud/documentai/v1beta2
-    - proto_path: google/cloud/documentai/v1beta3

library_generation/test/resources/integration/google-cloud-java/pr-description-golden.txt

@@ -1,19 +1,24 @@
-This pull request is generated with proto changes between googleapis commit a17d4caf184b050d50cacf2b0d579ce72c31ce74 (exclusive) and 1a45bf7393b52407188c82e63101db7dc9c72026 (inclusive).
+This pull request is generated with proto changes between googleapis commit a17d4caf184b050d50cacf2b0d579ce72c31ce74 (exclusive) and 9868a57470a969ffa1d21194a5c05d7a6e4e98cc (inclusive).
 Qualified commits are:
-[googleapis/googleapis@7a9a855](https://github.com/googleapis/googleapis/commit/7a9a855287b5042410c93e5a510f40efd4ce6cb1)
-[googleapis/googleapis@c7fd8bd](https://github.com/googleapis/googleapis/commit/c7fd8bd652ac690ca84f485014f70b52eef7cb9e)
+[googleapis/googleapis@aa16fda](https://github.com/googleapis/googleapis/commit/aa16fdad909bc33e2d4ff04cfde56a46d0e52b13)
+[googleapis/googleapis@0733fdb](https://github.com/googleapis/googleapis/commit/0733fdb5f745192f9f3c95f8d08039286567cbcc)
 BEGIN_NESTED_COMMIT
-feat: [document-ai] expose model_type in v1 processor, so that user can see the model_type after get or list processor version
+feat: [alloydb] support for obtaining the public IP address of an Instance
+feat: [alloydb] support for getting PSC DNS name from the GetConnectionInfo API
+feat: [alloydb] add PSC cluster and instance configuration settings to enable/disable PSC and obtain the PSC endpoint name
+feat: [alloydb] add new API to list the databases in a project and location
+docs: [alloydb] clarified read pool config is for read pool type instances
 
-PiperOrigin-RevId: 603727585
+PiperOrigin-RevId: 610475013
 
 END_NESTED_COMMIT
 BEGIN_NESTED_COMMIT
-feat: [document-ai] add model_type in v1beta3 processor proto
+feat: [alloydb] support for obtaining the public IP address of an Instance
+feat: [alloydb] support for getting PSC DNS name from the GetConnectionInfo API
 
-PiperOrigin-RevId: 603726122
+PiperOrigin-RevId: 610415824
 
 END_NESTED_COMMIT
 BEGIN_NESTED_COMMIT
-feat: Regenerate with the Java code generator (gapic-generator-java) v2.34.0
+feat: Regenerate with the Java code generator (gapic-generator-java) v2.37.0
 END_NESTED_COMMIT

library_generation/test/resources/integration/java-bigtable/generation_config.yaml

@@ -1,6 +1,6 @@
-gapic_generator_version: 2.35.0
+gapic_generator_version: 2.37.0
 protobuf_version: 25.2
-googleapis_commitish: fc3043ebe12fb6bc1729c175e1526c859ce751d8
+googleapis_commitish: 9868a57470a969ffa1d21194a5c05d7a6e4e98cc
 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
 synthtool_commitish: a6fb7d5f072b75698af1cbf06c5b001565753cfb
 template_excludes: