feat: enable generation with postprocessing of multiple service versions

.github/workflows/verify_library_generation.yaml

@@ -25,7 +25,7 @@ jobs:
         cache: maven
     - uses: actions/setup-python@v4
       with:
-        python-version: '3.11'
+        python-version: 3.11
     - name: install pyenv
       shell: bash
       run: |
@@ -36,10 +36,23 @@ jobs:
         export PATH="$PYENV_ROOT/bin:$PATH"
         echo "PYENV_ROOT=${PYENV_ROOT}" >> $GITHUB_ENV
         echo "PATH=${PATH}" >> $GITHUB_ENV
-        # init pyenv
-        eval "$(pyenv init --path)"
-        eval "$(pyenv init -)"
+
         set +ex
+    - name: install python dependencies
+      shell: bash
+      run: |
+        set -ex
+        pushd library_generation
+        pip install -r requirements.in
+        popd
+
+    - name: install utils (macos)
+      if: matrix.os == 'macos-12'
+      shell: bash
+      run: |
+        brew update --preinstall
+        # we need the `realpath` command to be available
+        brew install coreutils
     - name: install docker (ubuntu)
       if: matrix.os == 'ubuntu-22.04'
       shell: bash
@@ -69,10 +82,30 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v3
-    - name: Run unit tests
+    - name: install utils (macos)
+      if: matrix.os == 'macos-12'
+      shell: bash
+      run: |
+        brew update --preinstall
+        brew install coreutils
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.11
+    - name: install python dependencies
+      shell: bash
+      run: |
+        set -ex
+        pushd library_generation
+        pip install -r requirements.in
+        popd
+    - name: Run shell unit tests
       run: |
         set -x
         library_generation/test/generate_library_unit_tests.sh
+    - name: Run python unit tests
+      run: |
+        set -x
+        python -m unittest library_generation/test/unit_tests.py
   lint:
     runs-on: ubuntu-22.04
     steps:

library_generation/.gitignore

@@ -0,0 +1 @@
+__pycache__/

library_generation/generate_composed_library.py

@@ -0,0 +1,144 @@
+"""
+This script allows generation of libraries that are composed of more than one
+service version. It is achieved by calling `generate_library.sh` without
+postprocessing for all service versions and then calling
+postprocess_library.sh at the end, once all libraries are ready.
+
+Prerequisites
+- Needs a folder named `output` in current working directory. This folder
+is automatically detected by `generate_library.sh` and this script ensures it
+contains the necessary folders and files, specifically:
+  - A "google" folder found in the googleapis/googleapis repository
+  - A "grafeas" folder found in the googleapis/googleapis repository
+Note: googleapis repo is found in https://github.com/googleapis/googleapis.
+"""
+
+import click
+import utilities as util
+import os
+import sys
+import subprocess
+import json
+from model.GenerationConfig import GenerationConfig
+from model.Library import Library
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+MONOREPO_NAME = 'google-cloud-java'
+
+"""
+Main function in charge of generating libraries composed of more than one
+service or service version.
+Arguments
+ - config: a GenerationConfig object representing a parsed configuration
+ yaml
+ - library: a Library object contained inside config, passed here for
+   convenience and to prevent all libraries to be processed
+ - enable_postprocessing: true if postprocessing should be done on the generated
+   libraries
+ - repository_path: path to the repository where the generated files will be
+   sent. If not specified, it will default to the one defined in the configuration yaml
+   and will be downloaded. The versions file will be inferred from this folder
+"""
+def generate_composed_library(
+    config: GenerationConfig,
+    library: Library,
+    repository_path: str,
+    enable_postprocessing: bool = True,
+) -> None:
+  output_folder = util.sh_util('get_output_folder')
+
+  print(f'output_folder: {output_folder}')
+  print('library: ', library)
+  os.makedirs(output_folder, exist_ok=True)
+
+  googleapis_commitish = config.googleapis_commitish
+  if library.googleapis_commitish is not None:
+    googleapis_commitish = library.googleapis_commitish
+    print('using library-specific googleapis commitish: ' + googleapis_commitish)
+  else:
+    print('using common googleapis_commitish')
+
+  print('removing old googleapis folders and files')
+  util.delete_if_exists(f'{output_folder}/google')
+  util.delete_if_exists(f'{output_folder}/grafeas')
+
+  print('downloading googleapis')
+  util.sh_util(f'download_googleapis_files_and_folders "{output_folder}" "{googleapis_commitish}"')
+
+  is_monorepo = config.destination_path == MONOREPO_NAME
+
+  base_arguments = []
+  base_arguments += util.create_argument('gapic_generator_version', config)
+  base_arguments += util.create_argument('grpc_version', config)
+  base_arguments += util.create_argument('protobuf_version', config)
+
+  destination_path = f'java-{library.api_shortname}'
+  if is_monorepo:
+    destination_path = config.destination_path + '/' + destination_path
+
+  versions_file = ''
+  if is_monorepo:
+    print('this is a monorepo library')
+    library_folder = destination_path.split('/')[-1]
+    if repository_path is None:
+      repository_path = f'{output_folder}/{MONOREPO_NAME}'
+      clone_out = util.sh_util(f'sparse_clone "https://github.com/googleapis/{MONOREPO_NAME}.git" "{library_folder} google-cloud-pom-parent google-cloud-jar-parent versions.txt .github"', cwd=output_folder)
+      print(clone_out)
+    versions_file = f'{repository_path}/versions.txt'
+  else:
+    print('this is a HW library')
+    if repository_path is None:
+      repository_path = f'{output_folder}/{destination_path}'
+      util.delete_if_exists(f'{output_folder}/{destination_path}')
+      clone_out = util.sh_util(f'git clone "https://github.com/googleapis/{destination_path}.git"', cwd=output_folder)
+      print(clone_out)
+    versions_file = f'{repository_path}/versions.txt'
+
+  owlbot_cli_source_folder = util.sh_util('mktemp -d')
+  for gapic in library.GAPICs:
+
+    effective_arguments = list(base_arguments)
+    effective_arguments += util.create_argument('proto_path', gapic)
+
+    build_file = f'{gapic.proto_path}/BUILD.bazel'
+    print(f'build_file: {build_file}')
+    proto_only = util.sh_util(f'get_proto_only_from_BUILD {build_file}')
+    gapic_additional_protos=util.sh_util(f'get_gapic_additional_protos_from_BUILD {build_file}')
+    transport = util.sh_util(f'get_transport_from_BUILD {build_file}')
+    rest_numeric_enums = util.sh_util(f'get_rest_numeric_enums_from_BUILD {build_file}')
+    gapic_yaml = util.sh_util(f'get_gapic_yaml_from_BUILD {build_file}')
+    service_config = util.sh_util(f'get_service_config_from_BUILD {build_file}')
+    service_yaml = util.sh_util(f'get_service_yaml_from_BUILD {build_file}')
+    include_samples = util.sh_util(f'get_include_samples_from_BUILD {build_file}')
+    effective_arguments += [
+        '--proto_only', proto_only,
+        '--gapic_additional_protos', gapic_additional_protos,
+        '--transport', transport,
+        '--rest_numeric_enums', rest_numeric_enums,
+        '--gapic_yaml', gapic_yaml,
+        '--service_config', service_config,
+        '--service_yaml', service_yaml,
+        '--include_samples', include_samples,
+    ]
+    service_version = gapic.proto_path.split('/')[-1]
+    temp_destination_path = f'java-{library.api_shortname}-{service_version}'
+    effective_arguments += [ '--destination_path', temp_destination_path ]
+    print('arguments: ')
+    print(effective_arguments)
+    print(f'Generating library from {gapic.proto_path} to {destination_path}...')
+    util.run_process_and_print_output(['bash', '-x', f'{script_dir}/generate_library.sh',
+      *effective_arguments], 'Library postprocessing')
+
+
+    if enable_postprocessing:
+      util.sh_util(f'build_owlbot_cli_source_folder "{output_folder}/{destination_path}"'
+                   + f' "{owlbot_cli_source_folder}" "{output_folder}/{temp_destination_path}"'
+                   + f' "{gapic.proto_path}"',
+                   cwd=output_folder)
+
+  if enable_postprocessing:
+    # call postprocess library
+    util.run_process_and_print_output([f'{script_dir}/postprocess_library.sh',
+              f'{output_folder}/{destination_path}', '', versions_file, owlbot_cli_source_folder,
+        config.owlbot_cli_image, config.synthtool_commitish], 'Library postprocessing')
+

library_generation/generate_library.sh

@@ -60,18 +60,10 @@ case $key in
     include_samples="$2"
     shift
     ;;
-  --enable_postprocessing)
-    enable_postprocessing="$2"
-    shift
-    ;;
   --os_architecture)
     os_architecture="$2"
     shift
     ;;
-  --versions_file)
-    versions_file="$2"
-    shift
-    ;;
   *)
     echo "Invalid option: [$1]"
     exit 1
@@ -85,6 +77,11 @@ script_dir=$(dirname "$(readlink -f "$0")")
 source "${script_dir}"/utilities.sh
 output_folder="$(get_output_folder)"
 
+if [ -z "${gapic_generator_version}" ]; then
+  echo 'missing required argument --gapic_generator_version'
+  exit 1
+fi
+
 if [ -z "${protobuf_version}" ]; then
   protobuf_version=$(get_protobuf_version "${gapic_generator_version}")
 fi
@@ -125,10 +122,6 @@ if [ -z "${include_samples}" ]; then
   include_samples="true"
 fi
 
-if [ -z "$enable_postprocessing" ]; then
-  enable_postprocessing="true"
-fi
-
 if [ -z "${os_architecture}" ]; then
   os_architecture=$(detect_os_architecture)
 fi
@@ -298,34 +291,7 @@ popd # output_folder
 pushd "${temp_destination_path}"
 rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar
 popd # destination path
-##################### Section 5 #####################
-# post-processing
-#####################################################
-if [ "${enable_postprocessing}" != "true" ];
-then
-  echo "post processing is disabled"
-  cp -r ${temp_destination_path}/* "${output_folder}/${destination_path}"
-  rm -rdf "${temp_destination_path}"
-  exit 0
-fi
-if [ -z "${versions_file}" ];then
-  echo "no versions.txt argument provided. Please provide one in order to enable post-processing"
-  exit 1
-fi
-workspace="${output_folder}/workspace"
-if [ -d "${workspace}" ]; then
-  rm -rdf "${workspace}"
-fi
-
-mkdir -p "${workspace}"
-
-# if destination_path is not empty, it will be used as a starting workspace for
-# postprocessing
-if [[ $(find "${output_folder}/${destination_path}" -mindepth 1 -maxdepth 1 -type d,f | wc -l) -gt 0 ]];then
-  workspace="${output_folder}/${destination_path}"
-fi
-
-bash -x "${script_dir}/postprocess_library.sh" "${workspace}" \
-  "${temp_destination_path}" \
-  "${versions_file}"
 
+cp -r ${temp_destination_path}/* "${output_folder}/${destination_path}"
+rm -rdf "${temp_destination_path}"
+exit 0

library_generation/main.py

@@ -0,0 +1,79 @@
+"""
+Parses a config yaml and generates libraries via generate_composed_library.py
+"""
+
+import click
+from generate_composed_library import generate_composed_library
+from typing import Dict
+from model.GenerationConfig import GenerationConfig
+from model.Library import Library
+from model.GAPIC import GAPIC
+from collections.abc import Sequence
+from absl import app
+
+@click.group(invoke_without_command=False)
+@click.pass_context
+@click.version_option(message="%(version)s")
+def main(ctx):
+    pass
+
+@main.command()
+@click.option(
+    "--generation-config-yaml",
+    required=True,
+    type=str,
+    help="""
+    Path to generation_config.yaml that contains the metadata about library generation
+    """
+)
+@click.option(
+    "--enable-postprocessing",
+    required=False,
+    default=True,
+    type=bool,
+    help="""
+    Path to repository where generated files will be merged into, via owlbot copy-code.
+    Specifying this option enables postprocessing
+    """
+)
+@click.option(
+    "--target-library-api-shortname",
+    required=False,
+    type=str,
+    help="""
+    If specified, only the `library` with api_shortname = target-library-api-shortname will
+    be generated. If not specified, all libraries in the configuration yaml will be generated
+    """
+)
+@click.option(
+    "--repository-path",
+    required=False,
+    type=str,
+    help="""
+    If specified, the generated files will be sent to this location. If not specified, the
+    repository will be pulled into output_folder and move the generated files there
+    """
+)
+def generate_from_yaml(
+    generation_config_yaml: str,
+    enable_postprocessing: bool,
+    target_library_api_shortname: str,
+    repository_path: str
+) -> None:
+  config = GenerationConfig.from_yaml(generation_config_yaml)
+  target_libraries = config.libraries
+  if target_library_api_shortname is not None:
+    target_libraries = [library for library in config.libraries
+                              if library.api_shortname == target_library_api_shortname]
+  for library in target_libraries:
+    print(f'generating library {library.api_shortname}')
+    generate_composed_library(
+        config, library, repository_path, enable_postprocessing
+    )
+
+
+
+
+
+if __name__ == "__main__":
+  main()

library_generation/model/GAPIC.py

@@ -0,0 +1,9 @@
+"""
+Class that represents a GAPIC entry, inside a `Library` in a generation_config.yaml
+"""
+class GAPIC:
+  def __init__(
+      self,
+      proto_path: str,
+  ):
+    self.proto_path = proto_path