chore: re-enable HW processing for hermetic build scripts (#2505)

re-enables the changes from
#2342 as part of
[milestone
3.2](https://docs.google.com/document/d/1v-sJBmdNEVBRryL8n90XK8OIiqVphaJemXVhANw14Jk/edit?pli=1&resourcekey=0-QGUb2do-JBlDWKvptWBp_g&tab=t.0#bookmark=id.kl5fzqdav6u7)
of Hermetic Build

### Changes:
 - add support for HW libs in `integration_tests.py`
 - adapt configuration of docker volumes
 - adapt usage of `fix-poms.py` to use monorepo and split_repo modes
- fix temp_destination_path in `generate_composed_library.py` in order
to avoid collisions
- For example `google/bigtable/v2` and `google/bigtable/admin/v2` would
both be sent to a temporal `java-bigtable-v2` folder, spoiling the files
being transferred via `copy-code`.
- The fix is to have a `proto_path`-based folder name to achieve
uniqueness

### After this PR:
- [ ] Reconfigure the nightly CI in google-cloud-java to use the updated
volume mapping

.gitignore

@@ -25,6 +25,7 @@ target/
 **/output/
 **/googleapis
 library_generation/test/**/golden*/
-library_generation/test/resources/test_repo_level_postprocess/
+library_generation/test/resources/test_monorepo_postprocessing/
 **/*egg-info/
 **/build/
+**/dist/

library_generation/generate_composed_library.py

@@ -62,7 +62,6 @@ def generate_composed_library(
         config=config, library=library, output_folder=output_folder
     )
 
-    is_monorepo = util.check_monorepo(config=config)
     base_arguments = __construct_tooling_arg(config=config)
     owlbot_cli_source_folder = util.sh_util("mktemp -d")
     os.makedirs(f"{library_path}", exist_ok=True)
@@ -80,10 +79,7 @@ def generate_composed_library(
             transport=gapic_inputs.transport,
             library_path=library_path,
         )
-        service_version = gapic.proto_path.split("/")[-1]
-        temp_destination_path = (
-            f"java-{util.get_library_name(library)}-{service_version}"
-        )
+        temp_destination_path = f"java-{gapic.proto_path.replace('/','-')}"
         effective_arguments = __construct_effective_arg(
             base_arguments=base_arguments,
             gapic=gapic,
@@ -115,7 +111,7 @@ def generate_composed_library(
             owlbot_cli_source_folder,
             config.owlbot_cli_image,
             config.synthtool_commitish,
-            str(is_monorepo).lower(),
+            str(config.is_monorepo).lower(),
             config.path_to_yaml,
         ],
         "Library postprocessing",

library_generation/generate_repo.py

@@ -15,6 +15,7 @@
 
 import click
 import library_generation.utilities as util
+import os
 from library_generation.generate_composed_library import generate_composed_library
 from library_generation.model.generation_config import from_yaml
 
@@ -78,6 +79,11 @@ def generate_from_yaml(
     Parses a config yaml and generates libraries via
     generate_composed_library.py
     """
+    # convert paths to absolute paths so they can be correctly referenced in
+    # downstream scripts
+    generation_config_yaml = os.path.abspath(generation_config_yaml)
+    repository_path = os.path.abspath(repository_path)
+
     config = from_yaml(generation_config_yaml)
     target_libraries = config.libraries
     if target_library_api_shortname is not None:
@@ -102,7 +108,11 @@ def generate_from_yaml(
             versions_file=repo_config.versions_file,
         )
 
-    util.repo_level_post_process(
+    # we skip monorepo_postprocessing if not in a monorepo
+    if not config.is_monorepo:
+        return
+
+    util.monorepo_postprocessing(
         repository_path=repository_path, versions_file=repo_config.versions_file
     )
 

library_generation/model/generation_config.py

@@ -46,6 +46,8 @@ def __init__(
         self.libraries = libraries
         self.grpc_version = grpc_version
         self.protobuf_version = protobuf_version
+        # monorepos have more than one library defined in the config yaml
+        self.is_monorepo = len(libraries) > 1
 
 
 def from_yaml(path_to_yaml: str) -> GenerationConfig:
@@ -92,6 +94,9 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig:
             rpc_documentation=__optional(library, "rpc_documentation", None),
             cloud_api=__optional(library, "cloud_api", True),
             requires_billing=__optional(library, "requires_billing", True),
+            extra_versioned_modules=__optional(
+                library, "extra_versioned_modules", None
+            ),
         )
         parsed_libraries.append(new_library)
 

library_generation/model/library_config.py

@@ -46,6 +46,7 @@ def __init__(
         rpc_documentation: Optional[str] = None,
         cloud_api: Optional[bool] = True,
         requires_billing: Optional[bool] = True,
+        extra_versioned_modules: Optional[str] = None,
     ):
         self.api_shortname = api_shortname
         self.api_description = api_description
@@ -69,3 +70,4 @@ def __init__(
         self.rpc_documentation = rpc_documentation
         self.cloud_api = cloud_api
         self.requires_billing = requires_billing
+        self.extra_versioned_modules = extra_versioned_modules

library_generation/owlbot/bin/entrypoint.sh

@@ -28,6 +28,7 @@ scripts_root=$1
 versions_file=$2
 configuration_yaml=$3
 
+
 # This script can be used to process HW libraries and monorepo
 # (google-cloud-java) libraries, which require a slightly different treatment
 # monorepo folders have an .OwlBot.yaml file in the module folder (e.g.
@@ -65,7 +66,7 @@ echo "...done"
 
 # write or restore pom.xml files
 echo "Generating missing pom.xml..."
-python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true"
+python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "${monorepo}"
 echo "...done"
 
 # write or restore clirr-ignored-differences.xml
@@ -78,16 +79,7 @@ echo "Fixing missing license headers..."
 python3 "${scripts_root}/owlbot/src/fix-license-headers.py"
 echo "...done"
 
-# TODO: re-enable this once we resolve thrashing
-# restore license headers years
-# echo "Restoring copyright years..."
-# /owlbot/bin/restore_license_headers.sh
-# echo "...done"
-
 # ensure formatting on all .java files in the repository
 echo "Reformatting source..."
 mvn fmt:format -V --batch-mode --no-transfer-progress
 echo "...done"
-
-
-

library_generation/owlbot/src/fix-poms.py

@@ -559,7 +559,5 @@ def main(versions_file, monorepo):
 
 if __name__ == "__main__":
     versions_file = sys.argv[1]
-    monorepo = sys.argv[2]
-    if monorepo == "true":
-        monorepo = True
+    monorepo = True if sys.argv[2].lower() == "true" else False
     main(versions_file, monorepo)

library_generation/postprocess_library.sh

@@ -68,8 +68,8 @@ else
 fi
 
 # Default values for running copy-code directly from host
-repo_binding="${postprocessing_target}"
-repo_workspace="/repo"
+repo_bindings="-v ${postprocessing_target}:/workspace"
+repo_workspace="/workspace"
 preprocessed_libraries_binding="${owlbot_cli_source_folder}"
 
 # When running docker inside docker, we run into the issue of volume bindings
@@ -86,16 +86,20 @@ preprocessed_libraries_binding="${owlbot_cli_source_folder}"
 
 if [[ -n "${RUNNING_IN_DOCKER}" ]]; then
   set -u # temporarily fail on unset variables
-  repo_binding="${REPO_BINDING_VOLUME}"
+  repo_bindings="${REPO_BINDING_VOLUMES}"
   set +u
+  library_name=$(echo "${postprocessing_target}" | rev | cut -d'/' -f1 | rev)
+  repo_workspace="/workspace/"
   if [[ "${is_monorepo}" == "true" ]]; then
-    repo_workspace="/repo/$(echo "${postprocessing_target}" | rev | cut -d'/' -f1 | rev)"
+    monorepo_name=$(echo "${postprocessing_target}" | rev | cut -d'/' -f2 | rev)
+    repo_workspace+="${monorepo_name}/"
   fi
+  repo_workspace+="${library_name}"
 fi
 
 docker run --rm \
   --user "$(id -u)":"$(id -g)" \
-  -v "${repo_binding}:/repo" \
+  ${repo_bindings} \
   -v "/tmp:/tmp" \
   -w "${repo_workspace}" \
   --env HOME=/tmp \
@@ -115,6 +119,7 @@ fi
 git config --global --add safe.directory /tmp/synthtool/synthtool
 pushd "synthtool"
 
+git fetch --all
 git reset --hard "${synthtool_commitish}"
 
 python3 -m pip install -e .

library_generation/test/container_integration_tests.sh

@@ -8,32 +8,42 @@ if [[ -z "${TEST_IMAGE_ID}" ]]; then
   exit 1
 fi
 
-if [[ ! -d google-cloud-java ]]; then
-  git clone https://github.com/googleapis/google-cloud-java
-fi
-pushd google-cloud-java
-git reset --hard main
-popd
+repo_volumes=""
+for repo in google-cloud-java java-bigtable; do
+  if [[ ! -d "${repo}" ]]; then
+    git clone "https://github.com/googleapis/${repo}"
+  fi
+  pushd "${repo}"
+  git reset --hard main
+  popd
 
-# We use a volume to hold the google-cloud-java repository used in the
-# integration tests. This is because the test container creates a child
-# container using the host machine's docker socket, meaning that we can only
-# reference volumes created from within the host machine (i.e. the machine
-# running this script)
-#
-# To summarize, we create a special volume that can be referenced both in the
-# main container and in any child containers created by this one.
-if [[ $(docker volume inspect repo) != '[]' ]]; then
-  docker volume rm repo
-fi
-docker volume create --name "repo" --opt "type=none" --opt "device=$(pwd)/google-cloud-java" --opt "o=bind"
+  # We use a volume to hold the repositories used in the
+  # integration tests. This is because the test container creates a child
+  # container using the host machine's docker socket, meaning that we can only
+  # reference volumes created from within the host machine (i.e. the machine
+  # running this script)
+  #
+  # To summarize, we create a special volume that can be referenced both in the
+  # main container and in any child containers created by this one.
+  volume_name="repo-${repo}"
+  if [[ $(docker volume inspect "${volume_name}") != '[]' ]]; then
+    docker volume rm "${volume_name}"
+  fi
+  docker volume create \
+    --name "${volume_name}" \
+    --opt "type=none" \
+    --opt "device=$(pwd)/${repo}" \
+    --opt "o=bind"
+
+  repo_volumes="${repo_volumes} -v ${volume_name}:/workspace/${repo}"
+done
 
 docker run --rm \
-  -v repo:/workspace \
+  ${repo_volumes} \
   -v /tmp:/tmp \
   -v /var/run/docker.sock:/var/run/docker.sock \
   -e "RUNNING_IN_DOCKER=true" \
-  -e "REPO_BINDING_VOLUME=repo" \
+  -e "REPO_BINDING_VOLUMES=${repo_volumes}" \
   -w "/src" \
   "${TEST_IMAGE_ID}" \
   python -m unittest /src/test/integration_tests.py