From 8cd7ff739cefe6fcc366168817051640a5552e88 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Tue, 3 Sep 2024 19:13:40 -0400 Subject: [PATCH] feat: bake gapic-generator-java into the hermetic build docker image (#3067) Makes the gapic-generator-java jar a prepared binary in the Docker image whose location is now assumed by the scripts. Developers need now to prepare this well-know location (specified in `library_generation/DEVELOPMENT.md`. --------- Co-authored-by: Blake Li Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> --- .../library_generation.Dockerfile | 25 ++++- .github/workflows/ci.yaml | 11 +++ .gitignore | 2 + .../downstream-compatibility-spring.sh | 12 ++- library_generation/DEVELOPMENT.md | 70 +++++++------- .../gapic-generator-java-wrapper | 5 +- .../generate_composed_library.py | 1 - library_generation/generate_library.sh | 17 +--- .../test/generate_library_unit_tests.py | 96 +++++++++++++++++++ .../test/generate_library_unit_tests.sh | 80 +++------------- library_generation/test/integration_tests.py | 55 ++++++++++- .../integration/test_generator_coordinates | 1 + library_generation/test/test_utilities.sh | 2 +- library_generation/utils/utilities.py | 43 +++++++-- library_generation/utils/utilities.sh | 89 +++++++---------- showcase/scripts/generate_showcase.sh | 26 +++-- 16 files changed, 336 insertions(+), 199 deletions(-) create mode 100644 library_generation/test/generate_library_unit_tests.py create mode 100644 library_generation/test/resources/integration/test_generator_coordinates diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index ebc400348c2..ef788284a97 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -12,11 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +# install gapic-generator-java in a separate layer so we don't overload the image +# with the transferred source code and jars +FROM gcr.io/cloud-devrel-public-resources/java21 AS ggj-build + +WORKDIR /sdk-platform-java +COPY . . +# {x-version-update-start:gapic-generator-java:current} +ENV DOCKER_GAPIC_GENERATOR_VERSION="2.44.1-SNAPSHOT" +# {x-version-update-end:gapic-generator-java:current} + +RUN mvn install -DskipTests -Dclirr.skip -Dcheckstyle.skip +RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \ + "./gapic-generator-java.jar" + # build from the root of this repo: FROM gcr.io/cloud-devrel-public-resources/python SHELL [ "/bin/bash", "-c" ] + ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550 ARG PROTOC_VERSION=25.4 ARG GRPC_VERSION=1.66.0 @@ -47,7 +62,15 @@ RUN source /src/utils/utilities.sh \ ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe" ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}" -# use python 3.11 (the base image has several python versions; here we define the default one) + +# Here we transfer gapic-generator-java from the previous stage. +# Note that the destination is a well-known location that will be assumed at runtime +# We hard-code the location string to avoid making it configurable (via ARG) as +# well as to avoid it making it overridable at runtime (via ENV). +COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar" +RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar" + +# use python 3.11 (the base image has several python versions; here we define the default one) RUN rm $(which python3) RUN ln -s $(which python3.11) /usr/local/bin/python RUN ln -s $(which python3.11) /usr/local/bin/python3 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6e3a6b85d51..5dcedf37df7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -232,6 +232,17 @@ jobs: - name: Showcase golden tests working-directory: showcase run: | + # The golden test directly calls + # library_generation/generate_library.sh, which expects the jar to be + # located in its well-known location. More info in + # library_generation/DEVELOPMENT.md + # Here we prepare the jar in such location + generator_version=$(grep "gapic-generator-java:" "../versions.txt" \ + | cut -d: -f3) # the 3rd field is the snapshot version + mkdir -p "${HOME}/.library_generation" + cp \ + "${HOME}/.m2/repository/com/google/api/gapic-generator-java/${generator_version}/gapic-generator-java-${generator_version}.jar" \ + "${HOME}/.library_generation/gapic-generator-java.jar" mvn test \ -P enable-golden-tests \ --batch-mode \ diff --git a/.gitignore b/.gitignore index 98c7b77b4a8..6768e8cd69f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ target/ **/*egg-info/ **/build/ **/dist/ +library_generation/**/*.jar + diff --git a/.kokoro/presubmit/downstream-compatibility-spring.sh b/.kokoro/presubmit/downstream-compatibility-spring.sh index 2c0ae2b62ca..edc99d6cb75 100755 --- a/.kokoro/presubmit/downstream-compatibility-spring.sh +++ b/.kokoro/presubmit/downstream-compatibility-spring.sh @@ -34,8 +34,8 @@ git clone "https://github.com/GoogleCloudPlatform/spring-cloud-gcp.git" --depth= update_all_poms_dependency "spring-cloud-gcp" "gapic-generator-java-bom" "${GAPIC_GENERATOR_VERSION}" # Install spring-cloud-gcp modules -pushd spring-cloud-gcp/spring-cloud-generator -../mvnw \ +pushd spring-cloud-gcp +./mvnw \ -U \ --batch-mode \ --no-transfer-progress \ @@ -47,10 +47,16 @@ pushd spring-cloud-gcp/spring-cloud-generator # Generate showcase autoconfig +pushd spring-cloud-generator +# The script is not executable for non-owners. Here we manually chmod it. +# TODO(diegomarquezp): remove this line after +# https://github.com/GoogleCloudPlatform/spring-cloud-gcp/pull/3183 is merged and released. +chmod 755 ./scripts/generate-showcase.sh ./scripts/generate-showcase.sh pushd showcase/showcase-spring-starter mvn verify popd # showcase/showcase-spring-starter -popd # spring-cloud-gcp/spring-cloud-generator +popd # spring-cloud-generator +popd # spring-cloud-gcp popd # gapic-generator-java/target diff --git a/library_generation/DEVELOPMENT.md b/library_generation/DEVELOPMENT.md index 39992824fca..858b467797a 100644 --- a/library_generation/DEVELOPMENT.md +++ b/library_generation/DEVELOPMENT.md @@ -4,8 +4,7 @@ # Linting -When contributing, ensure your changes to python code have a valid -format. +When contributing, ensure your changes to python code have a valid format. ``` python -m pip install black @@ -30,9 +29,9 @@ python -m unittest test/integration_tests.py # Running the unit tests The unit tests of the hermetic build scripts are contained in several scripts, -corresponding to a specific component. Every unit test script ends with -`unit_tests.py`. To avoid them specifying them -individually, we can use the following command: +corresponding to a specific component. +Every unit test script ends with `unit_tests.py`. +To avoid them specifying them individually, we can use the following command: ```bash python -m unittest discover -s test/ -p "*unit_tests.py" @@ -45,35 +44,41 @@ python -m unittest discover -s test/ -p "*unit_tests.py" # Running the scripts in your local environment Although the scripts are designed to be run in a Docker container, you can also -run them directly. This section explains how to run the entrypoint script +run them directly. +This section explains how to run the entrypoint script (`library_generation/cli/entry_point.py`). -## Installing prerequisites +## Assumptions made by the scripts +### The Hermetic Build's well-known folder +Located in `${HOME}/.library_generation`, this folder is assumed by the scripts +to contain the generator JAR. +Please note that this is a recent feature and only this jar is expected to be +there. +Developers must make sure this folder is properly configured before running the +scripts locally. +Note that this relies on the `HOME` en var which is always defined as per +[POSIX env var definition](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html). -In order to run the generation scripts directly, there are a few tools we -need to install beforehand. +#### Put the gapic-generator-java jar in its well-known location -### Install synthtool +Run `cd sdk-platform-java && mvn install -DskipTests -Dclirr.skip +-Dcheckstyle.skip`. +This will generate a jar located in +`~/.m2/repository/com/google/api/gapic-generator-java/{version}/gapic-generator-java-{version}.jar` -It requires python 3.x to be installed. -You will need to specify a committish of the synthtool repo in order to have -your generation results matching exactly what the docker image would produce. -You can achieve this by inspecting `SYNTHTOOL_COMMITISH` in -`.cloudbuild/library_generation/library_generation.Dockerfile`. +Then `mv` the jar into the well-known location of the jar. +The generation scripts will assume the jar is there. -```bash -# obtained from .cloudbuild/library_generation/library_generation.Dockerfile -export SYNTHTOOL_COMMITTISH=6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 +```shell +mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar" ``` -```bash -git clone https://github.com/googleapis/synthtool -cd synthtool -git checkout "${SYNTHTOOL_COMMITTISH}" -python -m pip install --require-hashes -r requirements.txt -python -m pip install --no-deps -e . -python -m synthtool --version -``` + + +## Installing prerequisites + +In order to run the generation scripts directly, there are a few tools we +need to install beforehand. ### Install the owl-bot CLI @@ -93,6 +98,7 @@ owl-bot copy-code --version The key step is `npm link`, which will make the command available in you current shell session. + ## Running the script The entrypoint script (`library_generation/cli/entry_point.py`) allows you to update the target repository with the latest changes starting from the @@ -132,9 +138,9 @@ This will create an `image-id` file at the root of the repo with the hash ID of the image. ## Run the docker image -The docker image will perform changes on its internal `/workspace` folder, to which you -need to map a folder on your host machine (i.e. map your downloaded repo to this -folder). +The docker image will perform changes on its internal `/workspace` folder, +to which you need to map a folder on your host machine (i.e. map your downloaded +repo to this folder). To run the docker container on the google-cloud-java repo, you must run: ```bash @@ -151,9 +157,9 @@ docker run -u "$(id -u)":"$(id -g)" -v/path/to/google-cloud-java:/workspace $(c ## Debug the created containers If you are working on changing the way the containers are created, you may want -to inspect the containers to check the setup. It would be convenient in such -case to have a text editor/viewer available. You can achieve this by modifying -the Dockerfile as follows: +to inspect the containers to check the setup. +It would be convenient in such case to have a text editor/viewer available. +You can achieve this by modifying the Dockerfile as follows: ```docker # install OS tools diff --git a/library_generation/gapic-generator-java-wrapper b/library_generation/gapic-generator-java-wrapper index 3e26a675c0d..3f1e7bd2e7b 100755 --- a/library_generation/gapic-generator-java-wrapper +++ b/library_generation/gapic-generator-java-wrapper @@ -1,6 +1,7 @@ #!/usr/bin/env bash - set -e +wrapper_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")") +source "${wrapper_dir}/utils/utilities.sh" # Wrap gapic-generator-java.jar because protoc requires the plugin to be executable. -exec java -classpath "gapic-generator-java-${gapic_generator_version}.jar" com.google.api.generator.Main +exec java -classpath "$(get_gapic_generator_location)" com.google.api.generator.Main diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index 5b503450a94..2fd014f791c 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -131,7 +131,6 @@ def __construct_tooling_arg(config: GenerationConfig) -> List[str]: :return: arguments containing tooling versions """ arguments = [] - arguments += util.create_argument("gapic_generator_version", config) arguments += util.create_argument("grpc_version", config) arguments += util.create_argument("protoc_version", config) diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index 1b113ba11b2..c28b7f72a9d 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -14,12 +14,6 @@ case $key in destination_path="$2" shift ;; - --gapic_generator_version) - gapic_generator_version="$2" - # export this variable so that it can be used in gapic-generator-java-wrapper.sh - export gapic_generator_version - shift - ;; --protoc_version) protoc_version="$2" shift @@ -77,17 +71,12 @@ script_dir=$(dirname "$(readlink -f "$0")") source "${script_dir}"/utils/utilities.sh output_folder="$(get_output_folder)" -if [ -z "${gapic_generator_version}" ]; then - echo 'missing required argument --gapic_generator_version' - exit 1 -fi - if [ -z "${protoc_version}" ]; then - protoc_version=$(get_protoc_version "${gapic_generator_version}") + protoc_version=$(get_protoc_version) fi if [ -z "${grpc_version}" ]; then - grpc_version=$(get_grpc_version "${gapic_generator_version}") + grpc_version=$(get_grpc_version) fi if [ -z "${proto_only}" ]; then @@ -185,7 +174,7 @@ esac # download gapic-generator-java, protobuf and grpc plugin. # the download_tools function will create the environment variables "protoc_path" # and "grpc_path", to be used in the protoc calls below. -download_tools "${gapic_generator_version}" "${protoc_version}" "${grpc_version}" "${os_architecture}" +download_tools "${protoc_version}" "${grpc_version}" "${os_architecture}" ##################### Section 1 ##################### # generate grpc-*/ ##################################################### diff --git a/library_generation/test/generate_library_unit_tests.py b/library_generation/test/generate_library_unit_tests.py new file mode 100644 index 00000000000..86a6a5d637a --- /dev/null +++ b/library_generation/test/generate_library_unit_tests.py @@ -0,0 +1,96 @@ +import subprocess +import unittest +import os +from library_generation.utils.utilities import ( + run_process_and_print_output as bash_call, + run_process_and_get_output_string as get_bash_call_output, +) + +script_dir = os.path.dirname(os.path.realpath(__file__)) + + +class GenerateLibraryUnitTests(unittest.TestCase): + """ + Confirms the correct behavior of `library_generation/utils/utilities.sh`. + + Note that there is an already existing, shell-based, test suite for + generate_library.sh, but these tests will soon be transferred to this one as + an effort to unify the implementation of the Hermetic Build scripts as + python-only. New tests for `utilities.sh` should be added in this file. + """ + + TEST_ARCHITECTURE = "linux-x86_64" + + def setUp(self): + # we create a simulated home folder that has a fake generator jar + # in its well-known location + self.simulated_home = get_bash_call_output("mktemp -d") + bash_call(f"mkdir {self.simulated_home}/.library_generation") + bash_call( + f"touch {self.simulated_home}/.library_generation/gapic-generator-java.jar" + ) + + # We create a per-test directory where all output files will be created into. + # Each folder will be deleted after its corresponding test finishes. + test_dir = get_bash_call_output("mktemp -d") + self.output_folder = self._run_command_and_get_sdout( + "get_output_folder", + cwd=test_dir, + ) + bash_call(f"mkdir {self.output_folder}") + + def tearDown(self): + bash_call(f"rm -rdf {self.simulated_home}") + + def _run_command(self, command, **kwargs): + env = os.environ.copy() + env["HOME"] = self.simulated_home + if "cwd" not in kwargs: + kwargs["cwd"] = self.output_folder + return bash_call( + [ + "bash", + "-exc", + f"source {script_dir}/../utils/utilities.sh " + f"&& {command}", + ], + exit_on_fail=False, + env=env, + **kwargs, + ) + + def _run_command_and_get_sdout(self, command, **kwargs): + return self._run_command( + command, stderr=subprocess.PIPE, **kwargs + ).stdout.decode()[:-1] + + def test_get_grpc_version_with_no_env_var_fails(self): + # the absence of DOCKER_GRPC_VERSION will make this function to fail + result = self._run_command("get_grpc_version") + self.assertEquals(1, result.returncode) + self.assertRegex(result.stdout.decode(), "DOCKER_GRPC_VERSION is not set") + + def test_get_protoc_version_with_no_env_var_fails(self): + # the absence of DOCKER_PROTOC_VERSION will make this function to fail + result = self._run_command("get_protoc_version") + self.assertEquals(1, result.returncode) + self.assertRegex(result.stdout.decode(), "DOCKER_PROTOC_VERSION is not set") + + def test_download_tools_without_baked_generator_fails(self): + # This test has the same structure as + # download_tools_succeed_with_baked_protoc, but meant for + # gapic-generator-java. + + test_protoc_version = "1.64.0" + test_grpc_version = "1.64.0" + jar_location = ( + f"{self.simulated_home}/.library_generation/gapic-generator-java.jar" + ) + # we expect the function to fail because the generator jar is not found in + # its well-known location. To achieve this, we temporarily remove the fake + # generator jar + bash_call(f"rm {jar_location}") + result = self._run_command( + f"download_tools {test_protoc_version} {test_grpc_version} {self.TEST_ARCHITECTURE}" + ) + self.assertEquals(1, result.returncode) + self.assertRegex(result.stdout.decode(), "Please configure your environment") diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index 0261a041c38..2e4912536e7 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -5,7 +5,14 @@ set -xeo pipefail # Unit tests against ../utilities.sh script_dir=$(dirname "$(readlink -f "$0")") source "${script_dir}"/test_utilities.sh -source "${script_dir}"/../utils/utilities.sh + +# we simulate a properly prepared environment (i.e. generator jar in its +# well-known location). Tests confirming the opposite case will make sure this +# environment is restored +readonly SIMULATED_HOME=$(mktemp -d) +mkdir "${SIMULATED_HOME}/.library_generation" +touch "${SIMULATED_HOME}/.library_generation/gapic-generator-java.jar" +HOME="${SIMULATED_HOME}" source "${script_dir}"/../utils/utilities.sh # Unit tests extract_folder_name_test() { @@ -15,25 +22,12 @@ extract_folder_name_test() { assertEquals "google-cloud-aiplatform-v1-java" "${folder_name}" } -get_grpc_version_succeed_with_valid_generator_version_test() { - local actual_version - actual_version=$(get_grpc_version "2.24.0") - rm "gapic-generator-java-pom-parent-2.24.0.pom" - assertEquals "1.56.1" "${actual_version}" -} - -get_grpc_version_failed_with_invalid_generator_version_test() { - local res=0 - $(get_grpc_version "1.99.0") || res=$? - assertEquals 1 $((res)) -} - get_grpc_version_succeed_docker_env_var_test() { local version_with_docker local version_without_docker export DOCKER_GRPC_VERSION="9.9.9" # get_grpc_version should prioritize DOCKER_GRPC_VERSION - version_with_docker=$(get_grpc_version "2.24.0") + version_with_docker=$(get_grpc_version) assertEquals "${DOCKER_GRPC_VERSION}" "${version_with_docker}" unset DOCKER_GRPC_VERSION } @@ -43,24 +37,11 @@ get_protoc_version_succeed_docker_env_var_test() { local version_without_docker export DOCKER_PROTOC_VERSION="9.9.9" # get_protoc_version should prioritize DOCKER_PROTOC_VERSION - version_with_docker=$(get_protoc_version "2.24.0") + version_with_docker=$(get_protoc_version) assertEquals "${DOCKER_PROTOC_VERSION}" "${version_with_docker}" unset DOCKER_PROTOC_VERSION } -get_protoc_version_succeed_with_valid_generator_version_test() { - local actual_version - actual_version=$(get_protoc_version "2.24.0") - assertEquals "23.2" "${actual_version}" - rm "gapic-generator-java-pom-parent-2.24.0.pom" -} - -get_protoc_version_failed_with_invalid_generator_version_test() { - local res=0 - $(get_protoc_version "1.99.0") || res=$? - assertEquals 1 $((res)) -} - get_gapic_opts_with_rest_test() { local proto_path="${script_dir}/resources/gapic_options" local transport="grpc" @@ -108,28 +89,6 @@ remove_grpc_version_test() { rm "${destination_path}/QueryServiceGrpc.java" } -download_generator_success_with_valid_version_test() { - local version="2.24.0" - local artifact="gapic-generator-java-${version}.jar" - download_generator_artifact "${version}" "${artifact}" - assertFileOrDirectoryExists "${artifact}" - rm "${artifact}" -} - -download_generator_failed_with_invalid_version_test() { - # The download function will exit the shell - # if download failed. Test the exit code instead of - # downloaded file (there will be no downloaded file). - # Use $() to execute the function in subshell so that - # the other tests can continue executing in the current - # shell. - local res=0 - local version="1.99.0" - local artifact="gapic-generator-java-${version}.jar" - $(download_generator_artifact "${version}" "${artifact}") || res=$? - assertEquals 1 $((res)) -} - download_protoc_succeed_with_valid_version_linux_test() { download_protoc "23.2" "linux-x86_64" assertFileOrDirectoryExists "protoc-23.2" @@ -162,8 +121,6 @@ download_tools_succeed_with_baked_protoc() { # the version passed to `download_protoc`, then we will not download protoc # but simply have the variable `protoc_path` pointing to DOCKER_PROTOC_LOCATION # (which we manually created in this test) - local test_dir=$(mktemp -d) - pushd "${test_dir}" export DOCKER_PROTOC_LOCATION=$(mktemp -d) export DOCKER_PROTOC_VERSION="99.99" export output_folder=$(get_output_folder) @@ -171,14 +128,13 @@ download_tools_succeed_with_baked_protoc() { local protoc_bin_folder="${DOCKER_PROTOC_LOCATION}/protoc-99.99/bin" mkdir -p "${protoc_bin_folder}" - local test_ggj_version="2.40.0" local test_grpc_version="1.64.0" # we expect download_tools to decide to use DOCKER_PROTOC_LOCATION because # the protoc version we want to download is the same as DOCKER_PROTOC_VERSION. # Note that `protoc_bin_folder` is just the expected formatted value that # download_tools will format using DOCKER_PROTOC_VERSION (via # download_protoc). - download_tools "${test_ggj_version}" "99.99" "${test_grpc_version}" "linux-x86_64" + download_tools "99.99" "${test_grpc_version}" "linux-x86_64" assertEquals "${protoc_bin_folder}" "${protoc_path}" rm -rdf "${output_folder}" @@ -191,18 +147,15 @@ download_tools_succeed_with_baked_protoc() { download_tools_succeed_with_baked_grpc() { # This test has the same structure as # download_tools_succeed_with_baked_protoc, but meant for the grpc plugin. - local test_dir=$(mktemp -d) - pushd "${test_dir}" export DOCKER_GRPC_LOCATION=$(mktemp -d) export DOCKER_GRPC_VERSION="99.99" export output_folder=$(get_output_folder) mkdir "${output_folder}" - local test_ggj_version="2.40.0" local test_protoc_version="1.64.0" # we expect download_tools to decide to use DOCKER_GRPC_LOCATION because # the protoc version we want to download is the same as DOCKER_GRPC_VERSION - download_tools "${test_ggj_version}" "${test_protoc_version}" "99.99" "linux-x86_64" + download_tools "${test_protoc_version}" "99.99" "linux-x86_64" assertEquals "${DOCKER_GRPC_LOCATION}" "${grpc_path}" rm -rdf "${output_folder}" @@ -243,7 +196,6 @@ generate_library_failed_with_invalid_generator_version() { bash "${script_dir}"/../generate_library.sh \ -p google/cloud/alloydb/v1 \ -d ../"${destination}" \ - --gapic_generator_version 1.99.0 \ --protoc_version 23.2 \ --grpc_version 1.55.1 \ --transport grpc+rest \ @@ -260,7 +212,6 @@ generate_library_failed_with_invalid_protoc_version() { bash "${script_dir}"/../generate_library.sh \ -p google/cloud/alloydb/v1 \ -d ../"${destination}" \ - --gapic_generator_version 2.24.0 \ --protoc_version 22.99 \ --grpc_version 1.55.1 \ --transport grpc+rest \ @@ -277,7 +228,6 @@ generate_library_failed_with_invalid_grpc_version() { bash "${script_dir}"/../generate_library.sh \ -p google/cloud/alloydb/v1 \ -d ../output/"${destination}" \ - --gapic_generator_version 2.24.0 \ --grpc_version 0.99.0 \ --transport grpc+rest \ --rest_numeric_enums true || res=$? @@ -332,18 +282,12 @@ get_proto_path_from_preprocessed_sources_multiple_proto_dirs_fails() { # One line per test. test_list=( extract_folder_name_test - get_grpc_version_succeed_with_valid_generator_version_test - get_grpc_version_failed_with_invalid_generator_version_test get_grpc_version_succeed_docker_env_var_test get_protoc_version_succeed_docker_env_var_test - get_protoc_version_succeed_with_valid_generator_version_test - get_protoc_version_failed_with_invalid_generator_version_test get_gapic_opts_with_rest_test get_gapic_opts_without_rest_test get_gapic_opts_with_non_default_test remove_grpc_version_test - download_generator_success_with_valid_version_test - download_generator_failed_with_invalid_version_test download_protoc_succeed_with_valid_version_linux_test download_protoc_succeed_with_valid_version_macos_test download_protoc_failed_with_invalid_version_linux_test diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 5e5d219c61a..ebb8c66afa0 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -29,7 +29,9 @@ from library_generation.utils.utilities import sh_util as shell_call script_dir = os.path.dirname(os.path.realpath(__file__)) -golden_dir = os.path.join(script_dir, "resources", "integration", "golden") +config_dir = os.path.join(script_dir, "resources", "integration") +golden_dir = os.path.join(config_dir, "golden") +generator_jar_coordinates_file = os.path.join(config_dir, "test_generator_coordinates") repo_root_dir = os.path.join(script_dir, "..", "..") build_file = os.path.join( repo_root_dir, ".cloudbuild", "library_generation", "library_generation.Dockerfile" @@ -42,15 +44,25 @@ "google-cloud-java": "chore/test-hermetic-build", "java-bigtable": "chore/test-hermetic-build", } -config_dir = f"{script_dir}/resources/integration" baseline_config_name = "baseline_generation_config.yaml" current_config_name = "current_generation_config.yaml" +# This variable is used to override the jar created by building the image +# with our own downloaded jar in order to lock the integration test to use +# a constant version specified in +# library_generation/test/resources/integration/test_generator_coordinates +# This allows us to decouple the generation workflow testing with what the +# generator jar will actually generate. +# See library_generation/DEVELOPMENT.md ("The Hermetic Build's +# well-known folder). +WELL_KNOWN_GENERATOR_JAR_FILENAME = "gapic-generator-java.jar" + class IntegrationTest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - IntegrationTest.__build_image(docker_file=build_file, cwd=repo_root_dir) + cls.__download_generator_jar(coordinates_file=generator_jar_coordinates_file) + cls.__build_image(docker_file=build_file, cwd=repo_root_dir) @classmethod def setUp(cls) -> None: @@ -186,6 +198,41 @@ def __build_image(cls, docker_file: str, cwd: str): cwd=cwd, ) + @classmethod + def __download_generator_jar(cls, coordinates_file: str) -> None: + """ + Downloads the jar at the version specified in the + coordinates file + :param coordinates_file: path to the file containing the coordinates + """ + with open(coordinates_file, "r") as coordinates_file_handle: + # make this var available in the function scope + # nonlocal coordinates + coordinates = coordinates_file_handle.read() + # download the jar + subprocess.check_call( + [ + "mvn", + "dependency:copy", + f"-Dartifact={coordinates}", + f"-DoutputDirectory={config_dir}", + ] + ) + + # compute the filename of the downloaded jar + split_coordinates = coordinates.split(":") + artifact_id = split_coordinates[1] + version = split_coordinates[2] + jar_filename = f"{artifact_id}-{version}.jar" + + # rename the jar to its well-known filename defined at the top of this + # script file + source_jar_path = os.path.join(config_dir, jar_filename) + destination_jar_path = os.path.join( + config_dir, WELL_KNOWN_GENERATOR_JAR_FILENAME + ) + shutil.move(source_jar_path, destination_jar_path) + @classmethod def __remove_generated_files(cls): shutil.rmtree(f"{output_dir}", ignore_errors=True) @@ -247,6 +294,8 @@ def __run_entry_point_in_docker_container( f"{repo_location}:/workspace/repo", "-v", f"{config_location}:/workspace/config", + "-v", + f"{config_dir}/{WELL_KNOWN_GENERATOR_JAR_FILENAME}:/home/.library_generation/{WELL_KNOWN_GENERATOR_JAR_FILENAME}", "-w", "/workspace/repo", image_tag, diff --git a/library_generation/test/resources/integration/test_generator_coordinates b/library_generation/test/resources/integration/test_generator_coordinates new file mode 100644 index 00000000000..00af5f647f5 --- /dev/null +++ b/library_generation/test/resources/integration/test_generator_coordinates @@ -0,0 +1 @@ +com.google.api:gapic-generator-java:2.38.1 \ No newline at end of file diff --git a/library_generation/test/test_utilities.sh b/library_generation/test/test_utilities.sh index 007dc8e6d93..bbcdee1ebc1 100755 --- a/library_generation/test/test_utilities.sh +++ b/library_generation/test/test_utilities.sh @@ -27,7 +27,7 @@ __test_failed() { } -############# Functions used in test execution ############# +############# Functions used in test execution. They can only be called once per test ############# assertEquals() { local expected=$1 diff --git a/library_generation/utils/utilities.py b/library_generation/utils/utilities.py index 59f238aaea9..1aa60d21c02 100755 --- a/library_generation/utils/utilities.py +++ b/library_generation/utils/utilities.py @@ -17,6 +17,8 @@ import os import shutil from pathlib import Path +from typing import Any + from library_generation.model.generation_config import GenerationConfig from library_generation.model.library_config import LibraryConfig from typing import List @@ -39,20 +41,43 @@ def create_argument(arg_key: str, arg_container: object) -> List[str]: return [] -def run_process_and_print_output(arguments: List[str], job_name: str = "Job"): +def run_process_and_print_output( + arguments: List[str] | str, job_name: str = "Job", exit_on_fail=True, **kwargs +) -> Any: """ Runs a process with the given "arguments" list and prints its output. If the process fails, then the whole program exits + :param arguments: list of strings or string containing the arguments + :param job_name: optional job name to be printed + :param exit_on_fail: True if the main program should exit when the + subprocess exits with non-zero. Used for testing. + :param kwargs: passed to the underlying subprocess.run() call """ - # check_output() raises an exception if it exited with a nonzero code - try: - output = subprocess.check_output(arguments, stderr=subprocess.STDOUT) - print(output.decode(), end="", flush=True) - print(f"{job_name} finished successfully") - except subprocess.CalledProcessError as ex: - print(ex.output.decode(), end="", flush=True) - print(f"{job_name} failed") + # split "arguments" if passed as a single string + if isinstance(arguments, str): + arguments = arguments.split(" ") + if "stderr" not in kwargs: + kwargs["stderr"] = subprocess.STDOUT + proc_info = subprocess.run(arguments, stdout=subprocess.PIPE, **kwargs) + print(proc_info.stdout.decode(), end="", flush=True) + print( + f"{job_name} {'finished successfully' if proc_info.returncode == 0 else 'failed'}" + ) + if exit_on_fail and proc_info.returncode != 0: sys.exit(1) + return proc_info + + +def run_process_and_get_output_string( + arguments: List[str] | str, job_name: str = "Job", exit_on_fail=True, **kwargs +) -> Any: + """ + Wrapper of run_process_and_print_output() that returns the merged + stdout and stderr in a single string without its trailing newline char. + """ + return run_process_and_print_output( + arguments, job_name, exit_on_fail, stderr=subprocess.PIPE, **kwargs + ).stdout.decode()[:-1] def sh_util(statement: str, **kwargs) -> str: diff --git a/library_generation/utils/utilities.sh b/library_generation/utils/utilities.sh index 72fefec76a6..ae285b1b04b 100755 --- a/library_generation/utils/utilities.sh +++ b/library_generation/utils/utilities.sh @@ -2,6 +2,8 @@ set -eo pipefail utilities_script_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")") +# The $HOME variable is always set in the OS env as per POSIX specification. +GAPIC_GENERATOR_LOCATION="${HOME}/.library_generation/gapic-generator-java.jar" # Utility functions used in `generate_library.sh` and showcase generation. extract_folder_name() { @@ -97,49 +99,34 @@ remove_grpc_version() { sed -i.bak 's/value = \"by gRPC proto compiler.*/value = \"by gRPC proto compiler\",/g' {} \; -exec rm {}.bak \; } -download_gapic_generator_pom_parent() { - local gapic_generator_version=$1 - download_generator_artifact "${gapic_generator_version}" "gapic-generator-java-pom-parent-${gapic_generator_version}.pom" "gapic-generator-java-pom-parent" -} - # This function returns the version of the grpc plugin to generate the libraries. If -# DOCKER_GRPC_VERSION is set, this will be the version. Otherwise, it will be -# computed from the gapic-generator-pom-parent artifact at the specified -# gapic_generator_version. +# DOCKER_GRPC_VERSION is set, this will be the version. Otherwise, the script +# will exit since this is a necessary env var get_grpc_version() { - local gapic_generator_version=$1 local grpc_version if [[ -n "${DOCKER_GRPC_VERSION}" ]]; then >&2 echo "Using grpc version baked into the container: ${DOCKER_GRPC_VERSION}" echo "${DOCKER_GRPC_VERSION}" return + else + >&2 echo "Cannot infer grpc version because DOCKER_GRPC_VERSION is not set" + exit 1 fi - pushd "${output_folder}" > /dev/null - # get grpc version from gapic-generator-java-pom-parent/pom.xml - download_gapic_generator_pom_parent "${gapic_generator_version}" - grpc_version=$(grep grpc.version "gapic-generator-java-pom-parent-${gapic_generator_version}.pom" | sed 's/\(.*\)<\/grpc\.version>/\1/' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - popd > /dev/null - echo "${grpc_version}" } # This function returns the version of protoc to generate the libraries. If -# DOCKER_PROTOC_VERSION is set, this will be the version. Otherwise, it will be -# computed from the gapic-generator-pom-parent artifact at the specified -# gapic_generator_version. +# DOCKER_PROTOC_VERSION is set, this will be the version. Otherwise, the script +# will exit since this is a necessary env var get_protoc_version() { - local gapic_generator_version=$1 local protoc_version if [[ -n "${DOCKER_PROTOC_VERSION}" ]]; then >&2 echo "Using protoc version baked into the container: ${DOCKER_PROTOC_VERSION}" echo "${DOCKER_PROTOC_VERSION}" return + else + >&2 echo "Cannot infer protoc version because DOCKER_PROTOC_VERSION is not set" + exit 1 fi - pushd "${output_folder}" > /dev/null - # get protobuf version from gapic-generator-java-pom-parent/pom.xml - download_gapic_generator_pom_parent "${gapic_generator_version}" - protoc_version=$(grep protobuf.version "gapic-generator-java-pom-parent-${gapic_generator_version}.pom" | sed 's/\(.*\)<\/protobuf\.version>/\1/' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | cut -d "." -f2-) - popd > /dev/null - echo "${protoc_version}" } # Given the versions of the gapic generator, protoc and the protoc-grpc plugin, @@ -149,16 +136,16 @@ get_protoc_version() { # DOCKER_GRPC_VERSION respectively, this function will instead set "protoc_path" # and "grpc_path" to DOCKER_PROTOC_PATH and DOCKER_GRPC_PATH respectively (no # download), since the docker image will have downloaded these tools beforehand. +# # For the case of gapic-generator-java, no env var will be exported for the -# upstream flow, but instead it will be assigned a default filename that will be -# referenced by the file `library_generation/gapic-generator-java-wrapper`. +# upstream flow. Instead, the jar must be located in the well-known location +# (${HOME}/.library_generation/gapic-generator-java.jar). More information in +# `library_generation/DEVELOPMENT.md` download_tools() { - local gapic_generator_version=$1 - local protoc_version=$2 - local grpc_version=$3 - local os_architecture=$4 + local protoc_version=$1 + local grpc_version=$2 + local os_architecture=$3 pushd "${output_folder}" - download_generator_artifact "${gapic_generator_version}" "gapic-generator-java-${gapic_generator_version}.jar" # the variable protoc_path is used in generate_library.sh. It is explicitly # exported to make clear that it is used outside this utilities file. @@ -179,30 +166,19 @@ download_tools() { export grpc_path=$(download_grpc_plugin "${grpc_version}" "${os_architecture}") fi - popd -} - -download_generator_artifact() { - local gapic_generator_version=$1 - local artifact=$2 - local project=${3:-"gapic-generator-java"} - if [ ! -f "${artifact}" ]; then - # first, try to fetch the generator locally - local local_fetch_successful - local_fetch_successful=$(copy_from "$HOME/.m2/repository/com/google/api/${project}/${gapic_generator_version}/${artifact}" \ - "${artifact}") - if [[ "${local_fetch_successful}" == "false" ]];then - # download gapic-generator-java artifact from Google maven central mirror if not - # found locally - >&2 echo "${artifact} not found locally. Attempting a download from Maven Central" - download_from \ - "https://maven-central.storage-download.googleapis.com/maven2/com/google/api/${project}/${gapic_generator_version}/${artifact}" \ - "${artifact}" - >&2 echo "${artifact} found and downloaded from Maven Central" - else - >&2 echo "${artifact} found copied from local repository (~/.m2)" - fi + # Here we check whether the jar is stored in the expected location. + # The docker image will prepare the jar in this location. Developers must + # prepare their environment by creating + # $HOME/.library_generation/gapic_generator_java.jar + # This check is meant to ensure integrity of the downstream workflow. (i.e. + # ensure the generator wrapper succeeds) + if [[ ! -f "${GAPIC_GENERATOR_LOCATION}" ]]; then + >&2 echo "File ${GAPIC_GENERATOR_LOCATION} not found in the " + >&2 echo "filesystem. Please configure your environment and store the " + >&2 echo "generator jar in this location" + exit 1 fi + popd } download_protoc() { @@ -401,3 +377,6 @@ download_googleapis_files_and_folders() { cp -r grafeas "${output_folder}" } +get_gapic_generator_location() { + echo "${GAPIC_GENERATOR_LOCATION}" +} diff --git a/showcase/scripts/generate_showcase.sh b/showcase/scripts/generate_showcase.sh index d524aaa77f7..f5558e7b041 100755 --- a/showcase/scripts/generate_showcase.sh +++ b/showcase/scripts/generate_showcase.sh @@ -14,12 +14,11 @@ readonly perform_cleanup=$1 cd "${SCRIPT_DIR}" mkdir -p "${SCRIPT_DIR}/output" -# takes a versions.txt file and returns its version -get_version_from_versions_txt() { - versions=$1 - key=$2 - version=$(grep "$key:" "${versions}" | cut -d: -f3) # 3rd field is snapshot - echo "${version}" +get_version_from_pom() { + target_pom="$1" + key="$2" + # prints the result to stdout + grep -e "<${key}>" "${target_pom}" | cut -d'>' -f2 | cut -d'<' -f1 } # clone gapic-showcase @@ -30,8 +29,10 @@ if [ ! -d schema ]; then # looks at sdk-platform-java/showcase/gapic-showcase/pom.xml to extract the # version of gapic-showcase # see https://github.com/googleapis/gapic-showcase/releases - showcase_version=$(grep -e '' "${SCRIPT_DIR}/../gapic-showcase/pom.xml" | cut -d'>' -f2 | cut -d'<' -f1) - sparse_clone https://github.com/googleapis/gapic-showcase.git "schema/google/showcase/v1beta1" "v${showcase_version}" + showcase_version=$(get_version_from_pom \ + "${SCRIPT_DIR}/../gapic-showcase/pom.xml" "gapic-showcase.version" + ) + sparse_clone https://github.com/googleapis/gapic-showcase.git "schema/google/showcase/v1beta1" "v${showcase_version}" cd gapic-showcase mv schema ../output cd .. @@ -46,8 +47,12 @@ if [ ! -d google ];then rm -rdf googleapis fi -ggj_version=$(get_version_from_versions_txt ../../versions.txt "gapic-generator-java") gapic_additional_protos="google/iam/v1/iam_policy.proto google/cloud/location/locations.proto" + +path_to_generator_parent_pom="${SCRIPT_DIR}/../../gapic-generator-java-pom-parent/pom.xml" +protoc_version=$(get_version_from_pom "${path_to_generator_parent_pom}" "protobuf.version" \ + | cut -d. -f2-) +grpc_version=$(get_version_from_pom "${path_to_generator_parent_pom}" "grpc.version") rest_numeric_enums="false" transport="grpc+rest" gapic_yaml="" @@ -58,9 +63,10 @@ rm -rdf output/showcase-output mkdir output/showcase-output set +e bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \ + --protoc_version "${protoc_version}" \ + --grpc_version "${grpc_version}" \ --proto_path "schema/google/showcase/v1beta1" \ --destination_path "showcase-output" \ - --gapic_generator_version "${ggj_version}" \ --gapic_additional_protos "${gapic_additional_protos}" \ --rest_numeric_enums "${rest_numeric_enums}" \ --gapic_yaml "${gapic_yaml}" \