From 89381c78c7d4f4dd72d9497684b480a3c570a244 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Fri, 8 Mar 2024 11:37:45 -0500 Subject: [PATCH] chore: recursive comparison in hermetic build IT (#2542) Fixes https://github.com/googleapis/sdk-platform-java/issues/2540 Sample error message when running `library_generation/test/integration_tests.py` ``` Generation finished successfully. Will now compare differences between generated and existing libraries ****************************** Checking for differences in 'java-apigee-connect'. The expected library is in /usr/local/google/home/diegomarquezp/Desktop/sdk-platform-java/library_generation/test/resources/integration/golden/java-apigee-connect. The actual library is in /usr/local/google/home/diegomarquezp/Desktop/sdk-platform-java/library_generation/output/google-cloud-java/java-apigee-connect. Some files (found in both folders) are differing: - README.md - proto-google-cloud-apigee-connect-v1/src/main/java/com/google/cloud/apigeeconnect/v1/ConnectionProto.java ``` --- library_generation/test/integration_tests.py | 74 +++++++++++++++---- .../google-cloud-java/generation_config.yaml | 18 +---- .../pr-description-golden.txt | 21 ++++-- .../java-bigtable/generation_config.yaml | 4 +- 4 files changed, 75 insertions(+), 42 deletions(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 5c0a3b4523..b10aeeb301 100755 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -76,7 +76,7 @@ def test_get_commit_message_success(self): cmp( f"{config_dir}/{repo}/pr-description-golden.txt", f"{description_file}", - ) + ), "The generated PR description does not match the expected golden file" ) os.remove(f"{description_file}") @@ -107,16 +107,17 @@ def test_generate_repo(self): generation_config_yaml=config_file, repository_path=repo_dest ) # compare result + print( + "Generation finished successfully. Will now compare differences between generated and existing libraries" + ) for library_name in library_names: actual_library = ( f"{repo_dest}/{library_name}" if config.is_monorepo else repo_dest ) - print( - f"Generation finished. Will now compare " - f"the expected library in {golden_dir}/{library_name}, " - f"with the actual library in {actual_library}. " - f"Compare generation result: " - ) + print("*" * 50) + print(f"Checking for differences in '{library_name}'.") + print(f" The expected library is in {golden_dir}/{library_name}.") + print(f" The actual library is in {actual_library}. ") target_repo_dest = ( f"{repo_dest}/{library_name}" if config.is_monorepo else repo_dest ) @@ -125,20 +126,40 @@ def test_generate_repo(self): target_repo_dest, ignore=[".repo-metadata.json"], ) + diff_files = [] + golden_only = [] + generated_only = [] # compare source code - self.assertEqual([], compare_result.left_only) - self.assertEqual([], compare_result.right_only) - self.assertEqual([], compare_result.diff_files) - print("Source code comparison succeed.") + self.__recursive_diff_files( + compare_result, diff_files, golden_only, generated_only + ) + + # print all found differences for inspection + print_file = lambda f: print(f" - {f}") + if len(diff_files) > 0: + print(" Some files (found in both folders) are differing:") + [print_file(f) for f in diff_files] + if len(golden_only) > 0: + print(" There were files found only in the golden dir:") + [print_file(f) for f in golden_only] + if len(generated_only) > 0: + print(" Some files were found to have differences:") + [print_file(f) for f in generated_only] + + self.assertTrue(len(golden_only) == 0) + self.assertTrue(len(generated_only) == 0) + self.assertTrue(len(diff_files) == 0) + + print(" No differences found in {library_name}") # compare .repo-metadata.json self.assertTrue( self.__compare_json_files( f"{golden_dir}/{library_name}/.repo-metadata.json", f"{target_repo_dest}/.repo-metadata.json", ), - msg=f"The generated {library_name}/.repo-metadata.json is different from golden.", + msg=f" The generated {library_name}/.repo-metadata.json is different from golden.", ) - print(".repo-metadata.json comparison succeed.") + print(" .repo-metadata.json comparison succeed.") if not config.is_monorepo: continue @@ -151,7 +172,7 @@ def test_generate_repo(self): False, ) ) - print("gapic-libraries-bom/pom.xml comparison succeed.") + print(" gapic-libraries-bom/pom.xml comparison succeed.") self.assertFalse( compare_xml( f"{golden_dir}/pom.xml", @@ -159,7 +180,7 @@ def test_generate_repo(self): False, ) ) - print("pom.xml comparison succeed.") + print(" pom.xml comparison succeed.") @classmethod def __pull_repo_to(cls, default_dest: Path, repo: str, committish: str) -> str: @@ -203,7 +224,7 @@ def __get_config_files(cls, path: str) -> List[tuple[str, str]]: if sub_dir.is_file(): continue repo = sub_dir.name - if repo == "golden" or repo == "java-bigtable": + if repo in ["golden", "java-bigtable"]: continue config = f"{sub_dir}/{config_name}" config_files.append((repo, config)) @@ -223,3 +244,24 @@ def __load_json_to_sorted_list(cls, path: str) -> List[tuple]: res = [(key, value) for key, value in data.items()] return sorted(res, key=lambda x: x[0]) + + @classmethod + def __recursive_diff_files( + self, + dcmp: dircmp, + diff_files: List[str], + left_only: List[str], + right_only: List[str], + dirname: str = "", + ): + """ + recursively compares two subdirectories. The found differences are passed to three expected list references + """ + append_dirname = lambda d: dirname + d + diff_files.extend(map(append_dirname, dcmp.diff_files)) + left_only.extend(map(append_dirname, dcmp.left_only)) + right_only.extend(map(append_dirname, dcmp.right_only)) + for sub_dirname, sub_dcmp in dcmp.subdirs.items(): + self.__recursive_diff_files( + sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/" + ) diff --git a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml index eb2dcf7258..2adc11323e 100644 --- a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml @@ -1,6 +1,6 @@ -gapic_generator_version: 2.34.0 +gapic_generator_version: 2.37.0 protobuf_version: 25.2 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +googleapis_commitish: 9868a57470a969ffa1d21194a5c05d7a6e4e98cc owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e template_excludes: @@ -48,17 +48,3 @@ libraries: - proto_path: google/cloud/alloydb/connectors/v1 - proto_path: google/cloud/alloydb/connectors/v1alpha - proto_path: google/cloud/alloydb/connectors/v1beta - - - api_shortname: documentai - name_pretty: Document AI - product_documentation: https://cloud.google.com/compute/docs/documentai/ - api_description: allows developers to unlock insights from your documents with machine - learning. - library_name: document-ai - release_level: stable - issue_tracker: https://issuetracker.google.com/savedsearches/559755 - GAPICs: - - proto_path: google/cloud/documentai/v1 - - proto_path: google/cloud/documentai/v1beta1 - - proto_path: google/cloud/documentai/v1beta2 - - proto_path: google/cloud/documentai/v1beta3 diff --git a/library_generation/test/resources/integration/google-cloud-java/pr-description-golden.txt b/library_generation/test/resources/integration/google-cloud-java/pr-description-golden.txt index c095ab12a1..47be5aa6f1 100644 --- a/library_generation/test/resources/integration/google-cloud-java/pr-description-golden.txt +++ b/library_generation/test/resources/integration/google-cloud-java/pr-description-golden.txt @@ -1,19 +1,24 @@ -This pull request is generated with proto changes between googleapis commit a17d4caf184b050d50cacf2b0d579ce72c31ce74 (exclusive) and 1a45bf7393b52407188c82e63101db7dc9c72026 (inclusive). +This pull request is generated with proto changes between googleapis commit a17d4caf184b050d50cacf2b0d579ce72c31ce74 (exclusive) and 9868a57470a969ffa1d21194a5c05d7a6e4e98cc (inclusive). Qualified commits are: -[googleapis/googleapis@7a9a855](https://github.com/googleapis/googleapis/commit/7a9a855287b5042410c93e5a510f40efd4ce6cb1) -[googleapis/googleapis@c7fd8bd](https://github.com/googleapis/googleapis/commit/c7fd8bd652ac690ca84f485014f70b52eef7cb9e) +[googleapis/googleapis@aa16fda](https://github.com/googleapis/googleapis/commit/aa16fdad909bc33e2d4ff04cfde56a46d0e52b13) +[googleapis/googleapis@0733fdb](https://github.com/googleapis/googleapis/commit/0733fdb5f745192f9f3c95f8d08039286567cbcc) BEGIN_NESTED_COMMIT -feat: [document-ai] expose model_type in v1 processor, so that user can see the model_type after get or list processor version +feat: [alloydb] support for obtaining the public IP address of an Instance +feat: [alloydb] support for getting PSC DNS name from the GetConnectionInfo API +feat: [alloydb] add PSC cluster and instance configuration settings to enable/disable PSC and obtain the PSC endpoint name +feat: [alloydb] add new API to list the databases in a project and location +docs: [alloydb] clarified read pool config is for read pool type instances -PiperOrigin-RevId: 603727585 +PiperOrigin-RevId: 610475013 END_NESTED_COMMIT BEGIN_NESTED_COMMIT -feat: [document-ai] add model_type in v1beta3 processor proto +feat: [alloydb] support for obtaining the public IP address of an Instance +feat: [alloydb] support for getting PSC DNS name from the GetConnectionInfo API -PiperOrigin-RevId: 603726122 +PiperOrigin-RevId: 610415824 END_NESTED_COMMIT BEGIN_NESTED_COMMIT -feat: Regenerate with the Java code generator (gapic-generator-java) v2.34.0 +feat: Regenerate with the Java code generator (gapic-generator-java) v2.37.0 END_NESTED_COMMIT \ No newline at end of file diff --git a/library_generation/test/resources/integration/java-bigtable/generation_config.yaml b/library_generation/test/resources/integration/java-bigtable/generation_config.yaml index 55092330d6..48afd9eef7 100644 --- a/library_generation/test/resources/integration/java-bigtable/generation_config.yaml +++ b/library_generation/test/resources/integration/java-bigtable/generation_config.yaml @@ -1,6 +1,6 @@ -gapic_generator_version: 2.35.0 +gapic_generator_version: 2.37.0 protobuf_version: 25.2 -googleapis_commitish: fc3043ebe12fb6bc1729c175e1526c859ce751d8 +googleapis_commitish: 9868a57470a969ffa1d21194a5c05d7a6e4e98cc owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 synthtool_commitish: a6fb7d5f072b75698af1cbf06c5b001565753cfb template_excludes: