chore: bake synthtool and owl-bot into library_generation docker image

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -15,26 +15,68 @@
 # build from the root of this repo:
 FROM gcr.io/cloud-devrel-public-resources/python
 
-# install tools
+ARG SYNTHTOOL_COMMITTISH=63cc541da2c45fcfca2136c43e638da1fbae174d
+ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
+ENV HOME=/home
+
+# install OS tools
 RUN apt-get update && apt-get install -y \
-	unzip openjdk-17-jdk rsync maven jq \
+	unzip openjdk-17-jdk rsync maven jq less vim \
 	&& apt-get clean
 
-COPY library_generation /src
-
+# use python 3.11 (the base image has several python versions; here we define the default one)
 RUN rm $(which python3)
 RUN ln -s $(which python3.11) /usr/local/bin/python
 RUN ln -s $(which python3.11) /usr/local/bin/python3
 RUN python -m pip install --upgrade pip
-RUN cd /src && python -m pip install -r requirements.txt
-RUN cd /src && python -m pip install .
 
-# set dummy git credentials for empty commit used in postprocessing
-RUN git config --global user.email "[email protected]"
-RUN git config --global user.name "Cloud Java Bot"
+# copy source code
+COPY library_generation /src
 
-WORKDIR /workspace
-RUN chmod 750 /workspace
-RUN chmod 750 /src/generate_repo.py
+# install scripts as a python package
+WORKDIR /src
+RUN python -m pip install -r requirements.txt
+RUN python -m pip install .
+
+# install synthtool
+WORKDIR /tools
+RUN git clone https://github.com/googleapis/synthtool
+WORKDIR /tools/synthtool
+RUN git checkout "${SYNTHTOOL_COMMITTISH}"
+RUN python3 -m pip install --no-deps -e .
+RUN python3 -m pip install -r requirements.in
 
-CMD [ "/src/generate_repo.py" ]
+# Install nvm with node and npm
+ENV NODE_VERSION 20.12.0
+WORKDIR /home
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
+RUN chmod o+rx /home/.nvm
+ENV NODE_PATH=/home/.nvm/versions/node/v${NODE_VERSION}/bin
+ENV PATH=${PATH}:${NODE_PATH}
+RUN node --version
+RUN npm --version
+
+# install the owl-bot CLI
+WORKDIR /tools
+RUN git clone https://github.com/googleapis/repo-automation-bots
+WORKDIR /tools/repo-automation-bots/packages/owl-bot
+RUN git checkout "${OWLBOT_CLI_COMMITTISH}"
+RUN npm i && npm run compile && npm link
+RUN owl-bot copy-code --version
+RUN chmod -R o+rx ${NODE_PATH}
+RUN ln -sf ${NODE_PATH}/* /usr/local/bin
+
+# allow users to access the script folders
+RUN chmod -R o+rx /src
+
+# set dummy git credentials for the empty commit used in postprocessing
+# we use system so all users using the container will use this configuration
+RUN git config --system user.email "[email protected]"
+RUN git config --system user.name "Cloud Java Bot"
+
+# allow read-write for /home and execution for binaries in /home/.nvm
+RUN chmod -R a+rw /home
+RUN chmod -R a+rx /home/.nvm
+
+WORKDIR /workspace
+ENTRYPOINT [ "python", "/src/cli/entry_point.py", "generate" ]

library_generation/DEVELOPMENT.md

@@ -0,0 +1,101 @@
+# Linting
+
+When contributing, ensure your changes to python code have a valid
+format.
+
+```
+python -m pip install black
+black library_generation
+```
+
+# Running the integration tests
+
+The integration tests build the docker image declared in
+`.cloudbuild/library_generation/library_generation.Dockerfile`, pull GAPIC
+repositories, generate the libraries and compares the results with the source
+code declared in a "golden branch" of the repo.
+
+It requires docker and python 3.x to be installed.
+
+```
+cd library_generation
+python -m pip install .
+python -m pip install -r requirements.txt
+python -m unittest test/integration_tests.py
+```
+
+# Running the scripts in your local environment
+
+Although the scripts are designed to be run in a Docker container, you can also
+run them directly. This section explains how to run the entrypoint script
+(`library_generation/cli/entry_point.py`).
+
+## Installing prerequisites
+
+In order to run the generation scripts directly, there are a few tools we
+need to install beforehand.
+
+### Install synthtool
+
+It requires python 3.x to be installed.
+You will need to specify a committish of the synthtool repo in order to have
+your generation results matching exactly what the docker image would produce.
+You can achieve this by inspecting `SYNTHTOOL_COMMITISH` in
+`.cloudbuild/library_generation/library_generation.Dockerfile`.
+
+```bash
+# obtained from .cloudbuild/library_generation/library_generation.Dockerfile
+export SYNTHTOOL_COMMITTISH=6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
+```
+
+```bash
+git clone https://github.com/googleapis/synthtool
+cd synthtool
+git checkout "${SYNTHTOOL_COMMITTISH}"
+python -m pip install --require-hashes -r requirements.txt
+python -m pip install --no-deps -e .
+python -m synthtool --version
+```
+
+### Install the owl-bot CLI
+
+Requires node.js to be installed.
+Check this [installation guide](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script)
+for NVM, Node.js's version manager.
+
+After you install it, you can install the owl-bot CLI with the following
+commands:
+```bash
+git clone https://github.com/googleapis/repo-automation-bots
+cd repo-automation-bots/packages/owl-bot
+npm i && npm run compile && npm link
+owl-bot copy-code --version
+```
+
+The key step is `npm link`, which will make the command available in you current
+shell session.
+
+## Running the script
+The entrypoint script (`library_generation/cli/entry_point.py`) allows you to
+update the target repository with the latest changes starting from the
+googleapis committish declared in `generation_config.yaml`.
+
+### Download the repo
+For example, google-cloud-java
+```
+git clone https://github.com/googleapis/google-cloud-java
+export path_to_repo="$(pwd)/google-cloud-java"
+```
+
+### Install the scripts
+```
+cd library_generation
+python -m pip install .
+```
+
+### Run the script
+```
+cd library_generation
+python cli/entry_point.py --repository-path "${path_to_repo}"
+```
+

library_generation/README.md

@@ -98,8 +98,6 @@ They are shared by library level parameters.
 | grpc_version            |    No    | inferred from the generator if not specified |
 | googleapis-commitish    |   Yes    |                                              |
 | libraries_bom_version   |   Yes    |                                              |
-| owlbot-cli-image        |   Yes    |                                              |
-| synthtool-commitish     |   Yes    |                                              |
 | template_excludes       |   Yes    |                                              |
 
 ### Library level parameters
@@ -149,8 +147,6 @@ gapic_generator_version: 2.34.0
 protobuf_version: 25.2
 googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
 libraries_bom_version: 26.37.0
-owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
-synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
 destination_path: google-cloud-java
 template_excludes:
   - ".github/*"

library_generation/generate_composed_library.py

@@ -112,8 +112,6 @@ def generate_composed_library(
             "",
             versions_file,
             owlbot_cli_source_folder,
-            config.owlbot_cli_image,
-            config.synthtool_commitish,
             str(config.is_monorepo()).lower(),
             config_path,
         ],

library_generation/generate_repo.py

@@ -12,12 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
 import click as click
 import library_generation.utils.utilities as util
 from library_generation.generate_composed_library import generate_composed_library
 from library_generation.model.generation_config import GenerationConfig, from_yaml
 from library_generation.model.library_config import LibraryConfig
 from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing
+from pathlib import Path
 
 
 @click.group(invoke_without_command=False)
@@ -30,11 +32,12 @@ def main(ctx):
 @main.command()
 @click.option(
     "--generation-config-yaml",
-    required=True,
+    required=False,
     type=str,
     help="""
     Path to generation_config.yaml that contains the metadata about
-    library generation
+    library generation. If not specified, it will be assumed to be found
+    at the root of --repository-path
     """,
 )
 @click.option(
@@ -72,6 +75,10 @@ def generate(
     target_library_names: str,
     repository_path: str,
 ):
+    if generation_config_yaml is None:
+        generation_config_yaml = str(
+            Path(os.path.join(repository_path, "generation_config.yaml")).resolve()
+        )
     config = from_yaml(generation_config_yaml)
     generate_from_yaml(
         config_path=generation_config_yaml,

library_generation/model/generation_config.py

@@ -30,8 +30,6 @@ def __init__(
         gapic_generator_version: str,
         googleapis_commitish: str,
         libraries_bom_version: str,
-        owlbot_cli_image: str,
-        synthtool_commitish: str,
         template_excludes: List[str],
         libraries: List[LibraryConfig],
         grpc_version: Optional[str] = None,
@@ -40,8 +38,6 @@ def __init__(
         self.gapic_generator_version = gapic_generator_version
         self.googleapis_commitish = googleapis_commitish
         self.libraris_bom_version = libraries_bom_version
-        self.owlbot_cli_image = owlbot_cli_image
-        self.synthtool_commitish = synthtool_commitish
         self.template_excludes = template_excludes
         self.libraries = libraries
         self.grpc_version = grpc_version
@@ -119,8 +115,6 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig:
         protobuf_version=__optional(config, "protobuf_version", None),
         googleapis_commitish=__required(config, "googleapis_commitish"),
         libraries_bom_version=__required(config, "libraries_bom_version"),
-        owlbot_cli_image=__required(config, "owlbot_cli_image"),
-        synthtool_commitish=__required(config, "synthtool_commitish"),
         template_excludes=__required(config, "template_excludes"),
         libraries=parsed_libraries,
     )

library_generation/owlbot/bin/entrypoint.sh

@@ -70,7 +70,12 @@ echo "Fixing missing license headers..."
 python3 "${scripts_root}/owlbot/src/fix-license-headers.py"
 echo "...done"
 
-# ensure formatting on all .java files in the repository
+# Ensure formatting on all .java files in the repository.
+# Here we manually set the user.home system variable. Unfortunately, Maven
+# user.home inference involves the /etc/passwd file (confirmed empirically),
+# instead of the presumable $HOME env var, which may not work properly
+# when `docker run`ning with the -u flag because we may incur in users
+# not registered in the container's /etc/passwd file
 echo "Reformatting source..."
-mvn fmt:format -V --batch-mode --no-transfer-progress
+mvn fmt:format -Duser.home="${HOME}" -V --batch-mode --no-transfer-progress
 echo "...done"