From 349f035709e9d6649eb58513665c3f869b5c6a54 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 26 Apr 2024 20:44:30 -0400 Subject: [PATCH 1/6] deps: update library generation image tag --- .../generate-from-configuration.yaml | 2 +- .github/workflows/generated_files_sync.yaml | 2 +- .../hermetic_library_generation.yaml | 2 +- generation_config.yaml | 32 +++++++++---------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/generate-from-configuration.yaml b/.github/workflows/generate-from-configuration.yaml index 42c0ef125d9e..1ebef96f6982 100644 --- a/.github/workflows/generate-from-configuration.yaml +++ b/.github/workflows/generate-from-configuration.yaml @@ -13,7 +13,7 @@ jobs: base_branch: main # the branch with which the pull request is associated. branch_name: generate-libraries-main - library_generation_image_tag: 2.39.0 + library_generation_image_tag: 2.40.0 repo_volumes: "-v repo-google-cloud-java:/workspace/google-cloud-java" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/generated_files_sync.yaml b/.github/workflows/generated_files_sync.yaml index f3a48566d93f..f5f3fc69f19c 100644 --- a/.github/workflows/generated_files_sync.yaml +++ b/.github/workflows/generated_files_sync.yaml @@ -20,7 +20,7 @@ on: pull_request: name: generation diff env: - library_generation_image_tag: 2.39.0 + library_generation_image_tag: 2.40.0 repo_volumes: "-v repo-google-cloud-java:/workspace/google-cloud-java" jobs: root-pom: diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index 3376b16a01e8..852a7d2defe1 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -24,7 +24,7 @@ jobs: library_generation: runs-on: ubuntu-latest env: - library_generation_image_tag: 2.39.0 + library_generation_image_tag: 2.40.0 steps: - uses: actions/checkout@v4 with: diff --git a/generation_config.yaml b/generation_config.yaml index 3e7304774156..842a1da55eb3 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,5 +1,5 @@ gapic_generator_version: 2.39.0 -protobuf_version: '25.3' +protoc_version: '25.3' googleapis_commitish: 0179dcc4d2b4d7376e7f0292b15c72aab3100dd6 libraries_bom_version: 26.37.0 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 @@ -40,21 +40,21 @@ libraries: - proto_path: google/identity/accesscontextmanager/v1 - proto_path: google/identity/accesscontextmanager/type -#- api_shortname: admanager -# name_pretty: Google Ad Manager API -# product_documentation: https://developers.google.com/ad-manager/api/beta -# api_description: The Ad Manager API enables an app to integrate with Google Ad Manager. -# You can read Ad Manager data and run reports using the API. -# client_documentation: https://cloud.google.com/java/docs/reference/ad-manager/latest/overview -# release_level: preview -# distribution_name: com.google.api-ads:ad-manager -# api_id: admanager.googleapis.com -# library_type: GAPIC_AUTO -# group_id: com.google.api-ads -# cloud_api: false -# GAPICs: -# - proto_path: google/ads/admanager/v1 -# requires_billing: true +- api_shortname: admanager + name_pretty: Google Ad Manager API + product_documentation: https://developers.google.com/ad-manager/api/beta + api_description: The Ad Manager API enables an app to integrate with Google Ad Manager. + You can read Ad Manager data and run reports using the API. + client_documentation: https://cloud.google.com/java/docs/reference/ad-manager/latest/overview + release_level: preview + distribution_name: com.google.api-ads:ad-manager + api_id: admanager.googleapis.com + library_type: GAPIC_AUTO + group_id: com.google.api-ads + cloud_api: false + GAPICs: + - proto_path: google/ads/admanager/v1 + requires_billing: true - api_shortname: advisorynotifications name_pretty: Advisory Notifications API From 048cd7c0d927eed9e1d498a1db23312feb4e46b9 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 30 Apr 2024 17:22:14 -0400 Subject: [PATCH 2/6] remove obsolate workflow --- .../generate-from-configuration.yaml | 113 ------------------ 1 file changed, 113 deletions(-) delete mode 100644 .github/workflows/generate-from-configuration.yaml diff --git a/.github/workflows/generate-from-configuration.yaml b/.github/workflows/generate-from-configuration.yaml deleted file mode 100644 index 1ebef96f6982..000000000000 --- a/.github/workflows/generate-from-configuration.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: Generate GAPIC libraries from configuration -on: - schedule: - - cron: '0 2 * * *' # nightly at 2 am UTC - workflow_dispatch: - - -jobs: - generate-from-configuration: - runs-on: ubuntu-22.04 - env: - # the branch into which pull request is created. - base_branch: main - # the branch with which the pull request is associated. - branch_name: generate-libraries-main - library_generation_image_tag: 2.40.0 - repo_volumes: "-v repo-google-cloud-java:/workspace/google-cloud-java" - steps: - - uses: actions/checkout@v4 - - name: get baseline commit - id: get-baseline - shell: bash - run: | - echo "baseline_commit=$(grep googleapis_commitish generation_config.yaml | cut -d " " -f2 | xargs)" >> "$GITHUB_ENV" - - name: setup branch for pull request - id: setup-branch - shell: bash - run: | - [ -z "`git config user.email`" ] && git config --global user.email "cloud-java-bot@google.com" - [ -z "`git config user.name`" ] && git config --global user.name "cloud-java-bot" - # try to find a open pull request associated with the branch - pr_num=$(gh pr list -s open -H "${branch_name}" -q . --json number | jq ".[] | .number") - # create a branch if there's no open pull request associated with the - # branch; otherwise checkout the pull request. - if [ -z "${pr_num}" ]; then - git checkout -b "${branch_name}" - else - gh pr checkout "${pr_num}" - fi - echo "pr_num=${pr_num}" >> "$GITHUB_ENV" - env: - GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} - - name: setup docker environment - shell: bash - run: | - # we create a volume pointing to `pwd` (google-cloud-java) that will - # be referenced by the container and its children - if [[ $(docker volume inspect repo-google-cloud-java) != '[]' ]]; then - docker volume rm repo-google-cloud-java - fi - docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind" - - name: update googleapis commit to latest - id: update-commit - shell: bash - run: | - mkdir tmp-googleapis - # use partial clone because only commit history is needed. - git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis - pushd tmp-googleapis - git pull - latest_commit=$(git rev-parse HEAD) - popd - rm -rf tmp-googleapis - sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" generation_config.yaml - - name: generate from configuration - shell: bash - run: | - docker run --rm \ - ${repo_volumes} \ - -v /tmp:/tmp \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e "RUNNING_IN_DOCKER=true" \ - -e "REPO_BINDING_VOLUMES=${repo_volumes}" \ - gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \ - python /src/generate_repo.py generate --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml --repository-path=/workspace/google-cloud-java - - name: generate pull request description - id: generate-description - shell: bash - run: | - # the pr description (body) will be available in pr-description.txt after - # running this command. - set -x - docker run --rm \ - ${repo_volumes} \ - -v /tmp:/tmp \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e "RUNNING_IN_DOCKER=true" \ - -e "REPO_BINDING_VOLUMES=${repo_volumes}" \ - gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \ - python /src/generate_pr_description.py generate --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml --baseline-commit="${baseline_commit}" - env: - baseline_commit: ${{ env.baseline_commit }} - - name: create or update the pull request - shell: bash - run: | - title="chore: generate libraries at $(date)" - git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt generation_config.yaml - # use --allow-empty because (rarely) there's no change. - git commit --allow-empty -m "${title}" - if [ -z "${pr_num}" ]; then - git remote add monorepo https://cloud-java-bot:${GH_TOKEN}@github.com/${{ github.repository }}.git - git fetch -q --unshallow monorepo - git push -f monorepo "${branch_name}" - set -x - gh pr create --base "${base_branch}" --title "${title}" --head "${branch_name}" --body "$(cat pr_description.txt)" - else - git push - gh pr edit "${pr_num}" --title "${title}" - gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" - fi - env: - GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} - pr_num: ${{ env.pr_num }} From 69b8a7a097592271e4a8e350f2d0d5e4d4dae5aa Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 3 May 2024 08:33:51 -0400 Subject: [PATCH 3/6] change docker run command --- generation/hermetic_library_generation.sh | 24 ++++------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh index b37b898c7364..b087a1af78eb 100755 --- a/generation/hermetic_library_generation.sh +++ b/generation/hermetic_library_generation.sh @@ -70,9 +70,7 @@ if [ -z "${generation_config}" ]; then echo "Use default generation config: ${generation_config}" fi -volume_name="repo" -workspace_name="/workspace/repo" -repo_volumes="${volume_name}:${workspace_name}" +workspace_name="/workspace" baseline_generation_config="baseline_generation_config.yaml" message="chore: generate libraries at $(date)" @@ -89,28 +87,14 @@ fi git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" config_diff=$(diff "${generation_config}" "${baseline_generation_config}") -# bind docker volume to include the repository in docker running environment. -if [[ $(docker volume inspect ${volume_name}) != '[]' ]]; then - docker volume rm ${volume_name} -fi -docker volume create \ - --name ${volume_name} \ - --opt "type=none" \ - --opt "device=$(pwd)" \ - --opt "o=bind" # run hermetic code generation docker image. docker run \ --rm \ - -v "${repo_volumes}" \ - -v /tmp:/tmp \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e "RUNNING_IN_DOCKER=true" \ - -e "REPO_BINDING_VOLUMES=-v ${repo_volumes}" \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):${workspace_name}" \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ - python /src/cli/entry_point.py generate \ --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ - --current-generation-config-path="${workspace_name}/${generation_config}" \ - --repository-path="${workspace_name}" + --current-generation-config-path="${workspace_name}/${generation_config}" # commit the change to the pull request. git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt changed_files=$(git diff --cached --name-only) From f5c3bbcfce0e63c18ebd8c213fd73931d3f9dea9 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 3 May 2024 08:41:55 -0400 Subject: [PATCH 4/6] change docker run command --- .github/workflows/generated_files_sync.yaml | 50 +++++++-------------- 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/.github/workflows/generated_files_sync.yaml b/.github/workflows/generated_files_sync.yaml index f5f3fc69f19c..3f11b68d9be2 100644 --- a/.github/workflows/generated_files_sync.yaml +++ b/.github/workflows/generated_files_sync.yaml @@ -21,33 +21,24 @@ on: name: generation diff env: library_generation_image_tag: 2.40.0 - repo_volumes: "-v repo-google-cloud-java:/workspace/google-cloud-java" jobs: root-pom: # root pom.xml does not have diff from generated one runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: setup docker environment - shell: bash - run: | - # we create a volume pointing to `pwd` (google-cloud-java) that will - # be referenced by the container and its children - if [[ $(docker volume inspect repo-google-cloud-java) != '[]' ]]; then - docker volume rm repo-google-cloud-java - fi - docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind" - name: Generate root pom.xml file shell: bash run: | - docker run --rm \ - ${repo_volumes} \ - -v /tmp:/tmp \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e "RUNNING_IN_DOCKER=true" \ - -e "REPO_BINDING_VOLUMES=${repo_volumes}" \ + docker run \ + --rm \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):/workspace" \ + --entrypoint python \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \ - python /src/cli/generate_monorepo_root_pom.py generate --repository-path=/workspace/google-cloud-java + /src/cli/generate_monorepo_root_pom.py \ + generate \ + --repository-path=/workspace - name: Fail if there's any difference run: git --no-pager diff --exit-code @@ -56,26 +47,19 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: setup docker environment - shell: bash - run: | - # we create a volume pointing to `pwd` (google-cloud-java) that will - # be referenced by the container and its children - if [[ $(docker volume inspect repo-google-cloud-java) != '[]' ]]; then - docker volume rm repo-google-cloud-java - fi - docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind" - name: Generate gapic-libraries-bom/pom.xml shell: bash run: | - docker run --rm \ - ${repo_volumes} \ - -v /tmp:/tmp \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e "RUNNING_IN_DOCKER=true" \ - -e "REPO_BINDING_VOLUMES=${repo_volumes}" \ + docker run \ + --rm \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):/workspace" \ + --entrypoint python \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \ - python /src/cli/generate_monorepo_gapic_bom.py generate --repository-path=/workspace/google-cloud-java --versions-file=/workspace/google-cloud-java/versions.txt + /src/cli/generate_monorepo_gapic_bom.py \ + generate \ + --repository-path=/workspace \ + --versions-file=/workspace/versions.txt - name: Fail if there's any difference run: git --no-pager diff --exit-code From 1c14fd07717eaca886c508f2f37d46c4185e18a2 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 3 May 2024 08:43:09 -0400 Subject: [PATCH 5/6] restore plugin version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aba7f9fe2127..3a811bd99a07 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ org.apache.maven.plugins maven-deploy-plugin - 3.1.1 + 3.1.2 true From 89ade69f03aa068b5c423f249465a734c0ea1053 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 3 May 2024 09:06:12 -0400 Subject: [PATCH 6/6] change readme --- .../new_client_hermetic_build/README.md | 44 +++++-------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/generation/new_client_hermetic_build/README.md b/generation/new_client_hermetic_build/README.md index 7137a7326f09..6725c82b6efe 100644 --- a/generation/new_client_hermetic_build/README.md +++ b/generation/new_client_hermetic_build/README.md @@ -1,9 +1,9 @@ # New client generation (GitHub Action) This new generation workflow enables generation of new libraries by - 1. appending a new library to our [generation_config.yaml](https://github.com/googleapis/google-cloud-java/blob/c7429c0eec419c01d4e2fe14d063b9335efb810b/generation_config.yaml). - 2. running the hermetic build scripts docker image and - generate the newly added library. - 3. create a PR with the changes. + 1. Appending a new library to our [generation_config.yaml](/generation_config.yaml). + 2. Creating a PR with the changes. + +The new client will be generated by [Hermetic library generation workflow](/.github/workflows/hermetic_library_generation.yaml). ## Components @@ -14,18 +14,15 @@ A new entry will be added to `libraries` with the necessary generation configuration. ### `.github/workflows/generate_new_client_hermetic_build.yaml` -This workflow orchestrates the `add-new-client-config.py` script and also uses our docker -image -([gcr.io/cloud-devrel-public-resources/java-library-generation](https://pantheon.corp.google.com/gcr/images/cloud-devrel-public-resources/global/java-library-generation)) -to generate a new library. -It also creates the pull request. +This workflow orchestrates the `add-new-client-config.py` script and creates +a pull request. ## Execute the GitHub Action In order to run the -[GitHub Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client_hermetic_build.yaml) -, you need to specify a few parameters. +[GitHub Action](/.github/workflows/generate_new_client_hermetic_build.yaml), +you need to specify a few parameters. These parameters will be available in the Cloud Drop link (a YAML file) included in the Buganizer request. The example in this README uses AlloyDB's [Cloud Drop](https://github.com/googleapis/googleapis/blob/master/google/cloud/alloydb/v1/alloydb_v1.yaml) @@ -156,26 +153,5 @@ python3.9 generation/new_client_hermetic_build/add-new-client-config.py generate After you run the script, you will see that the `generation_config.yaml` file was modified (or the script exited because the library already existed). -The last step you need is to `cd` into the root of `google-cloud-java` and run -``` -docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind" -repo_volumes="-v repo-google-cloud-java:/workspace/google-cloud-java" -docker run --rm \ - ${repo_volumes} \ - -v /tmp:/tmp \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e "RUNNING_IN_DOCKER=true" \ - -e "REPO_BINDING_VOLUMES=${repo_volumes}" \ - gcr.io/cloud-devrel-public-resources/java-library-generation:latest \ - python /src/generate_repo.py generate \ - --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml \ - --repository-path=/workspace/google-cloud-java \ - --target-library-names= - -``` - -This docker container will run the generation scripts and generate the library -in your repo. -You can create a PR explaining what commands you used (the docker command is -not as informative as the `add-new-client-config.py` call, so make sure at least -the add-new-client-config.py arguments were listed). +Please create a PR explaining what commands you used and make sure the +add-new-client-config.py arguments were listed).