Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add libraries_bom_version to generation configuration #2639

Merged
merged 16 commits into from
Apr 16, 2024
Merged
10 changes: 8 additions & 2 deletions library_generation/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ They are shared by library level parameters.
| protobuf_version | No | inferred from the generator if not specified |
| grpc_version | No | inferred from the generator if not specified |
| googleapis-commitish | Yes | |
| libraries_bom_version | Yes | |
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you please add it to our spec doc as well?

| owlbot-cli-image | Yes | |
| synthtool-commitish | Yes | |
| template_excludes | Yes | |
Expand Down Expand Up @@ -147,6 +148,7 @@ The GAPIC level parameters define how to generate a GAPIC library.
gapic_generator_version: 2.34.0
protobuf_version: 25.2
googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
libraries_bom_version: 26.37.0
owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
destination_path: google-cloud-java
Expand Down Expand Up @@ -192,11 +194,15 @@ libraries:

# Local Environment Setup before running `entry_point.py`

1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment. The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case).
2. Assuming the virtual environment is installed under `sdk-platform-java`. Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation`
1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment.
The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case).
2. Assuming the virtual environment is installed under `sdk-platform-java`.
Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation`

```bash
python -m pip install -r library_generation/requirements.txt
```

3. Run the following command to install `library_generation` as a module, which allows the `library_generation` module to be imported from anywhere
```bash
python -m pip install library_generation/
Expand Down
3 changes: 3 additions & 0 deletions library_generation/model/generation_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ def __init__(
self,
gapic_generator_version: str,
googleapis_commitish: str,
libraries_bom_version: str,
owlbot_cli_image: str,
synthtool_commitish: str,
template_excludes: List[str],
Expand All @@ -38,6 +39,7 @@ def __init__(
):
self.gapic_generator_version = gapic_generator_version
self.googleapis_commitish = googleapis_commitish
self.libraris_bom_version = libraries_bom_version
self.owlbot_cli_image = owlbot_cli_image
self.synthtool_commitish = synthtool_commitish
self.template_excludes = template_excludes
Expand Down Expand Up @@ -116,6 +118,7 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig:
grpc_version=__optional(config, "grpc_version", None),
protobuf_version=__optional(config, "protobuf_version", None),
googleapis_commitish=__required(config, "googleapis_commitish"),
libraries_bom_version=__required(config, "libraries_bom_version"),
owlbot_cli_image=__required(config, "owlbot_cli_image"),
synthtool_commitish=__required(config, "synthtool_commitish"),
template_excludes=__required(config, "template_excludes"),
Expand Down
1 change: 1 addition & 0 deletions library_generation/owlbot/src/apply_repo_templates.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ def apply_repo_templates(configuration_yaml_path: str, monorepo: bool) -> None:
excludes=config.template_excludes,
template_path=Path(repo_templates_path),
monorepo=monorepo,
libraries_bom_version=config.libraris_bom_version,
)


Expand Down
2 changes: 1 addition & 1 deletion library_generation/owlbot/templates/java_library/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file:
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>libraries-bom</artifactId>
<version>{{ metadata['latest_bom_version'] }}</version>
<version>{{ metadata['libraries_bom_version'] }}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
Expand Down
1 change: 1 addition & 0 deletions library_generation/test/generate_repo_unit_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ def __get_an_empty_generation_config() -> GenerationConfig:
return GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
synthtool_commitish="",
owlbot_cli_image="",
template_excludes=[],
Expand Down
1 change: 1 addition & 0 deletions library_generation/test/model/config_change_unit_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ def __get_a_gen_config(
return GenerationConfig(
gapic_generator_version="",
googleapis_commitish=googleapis_commitish,
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[],
Expand Down
60 changes: 59 additions & 1 deletion library_generation/test/model/generation_config_unit_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import os
import unittest
from pathlib import Path

from library_generation.model.generation_config import from_yaml, GenerationConfig
from library_generation.model.library_config import LibraryConfig

Expand All @@ -39,6 +38,63 @@


class GenerationConfigTest(unittest.TestCase):
def test_from_yaml_succeeds(self):
config = from_yaml(f"{test_config_dir}/generation_config.yaml")
self.assertEqual("2.34.0", config.gapic_generator_version)
self.assertEqual(25.2, config.protobuf_version)
self.assertEqual(
"1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish
)
self.assertEqual("26.37.0", config.libraris_bom_version)
self.assertEqual(
"sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409",
config.owlbot_cli_image,
)
self.assertEqual(
"6612ab8f3afcd5e292aecd647f0fa68812c9f5b5", config.synthtool_commitish
)
self.assertEqual(
[
".github/*",
".kokoro/*",
"samples/*",
"CODE_OF_CONDUCT.md",
"CONTRIBUTING.md",
"LICENSE",
"SECURITY.md",
"java.header",
"license-checks.xml",
"renovate.json",
".gitignore",
],
config.template_excludes,
)
library = config.libraries[0]
self.assertEqual("cloudasset", library.api_shortname)
self.assertEqual("Cloud Asset Inventory", library.name_pretty)
self.assertEqual(
"https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
library.product_documentation,
)
self.assertEqual(
"provides inventory services based on a time series database.",
library.api_description,
)
self.assertEqual("asset", library.library_name)
self.assertEqual("@googleapis/analytics-dpe", library.codeowner_team)
self.assertEqual(
"proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1",
library.excluded_poms,
)
self.assertEqual("google-iam-policy", library.excluded_dependencies)
gapics = library.gapic_configs
self.assertEqual(5, len(gapics))
self.assertEqual("google/cloud/asset/v1", gapics[0].proto_path)
self.assertEqual("google/cloud/asset/v1p1beta1", gapics[1].proto_path)
self.assertEqual("google/cloud/asset/v1p2beta1", gapics[2].proto_path)
self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path)
self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path)

def test_get_proto_path_to_library_name_success(self):
paths = from_yaml(
f"{test_config_dir}/generation_config.yaml"
Expand All @@ -58,6 +114,7 @@ def test_is_monorepo_with_one_library_returns_false(self):
config = GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[],
Expand All @@ -69,6 +126,7 @@ def test_is_monorepo_with_two_libraries_returns_true(self):
config = GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[],
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
gapic_generator_version: 2.37.0
gapic_generator_version: 2.38.1
protobuf_version: 25.2
googleapis_commitish: a17d4caf184b050d50cacf2b0d579ce72c31ce74
libraries_bom_version: 26.37.0
owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e
synthtool_commitish: 63cc541da2c45fcfca2136c43e638da1fbae174d
template_excludes:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just a side note that the old commitish 5e1fb2 belongs to a branch (not up to date with main) that had the java templates removed. Using a committish that contains java templates should not affect the generation because we have this logic in postprocessing:

export SYNTHTOOL_TEMPLATES=$(mktemp -d)
# defaults to run owlbot.py
python3 owlbot.py
export SYNTHTOOL_TEMPLATES=""

- ".github/*"
- ".kokoro/*"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
gapic_generator_version: 2.38.1
protobuf_version: 25.2
googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc
libraries_bom_version: 26.37.0
owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
synthtool_commitish: 63cc541da2c45fcfca2136c43e638da1fbae174d
template_excludes:
- ".github/*"
- ".kokoro/*"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
gapic_generator_version: 2.34.0
googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
libraries:
- api_shortname: apigeeconnect
name_pretty: Apigee Connect
api_description: "allows the Apigee hybrid management"
product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/"
GAPICs:
- proto_path: google/cloud/apigeeconnect/v1
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
gapic_generator_version: 2.34.0
googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
libraries_bom_version: 26.37.0
libraries:
- api_shortname: apigeeconnect
name_pretty: Apigee Connect
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
gapic_generator_version: 2.34.0
googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
libraries_bom_version: 26.37.0
owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
libraries:
- api_shortname: apigeeconnect
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
gapic_generator_version: 2.34.0
googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
libraries_bom_version: 26.37.0
owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
libraries:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
gapic_generator_version: 2.34.0
protobuf_version: 25.2
googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
libraries_bom_version: 26.37.0
owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
template_excludes:
Expand Down
61 changes: 5 additions & 56 deletions library_generation/test/utilities_unit_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ def test_eprint_valid_input_succeeds(self):
"googleapis_commitish",
f"{test_config_dir}/config_without_googleapis.yaml",
),
(
"libraries_bom_version",
f"{test_config_dir}/config_without_libraries_bom_version.yaml",
),
("owlbot_cli_image", f"{test_config_dir}/config_without_owlbot.yaml"),
("synthtool_commitish", f"{test_config_dir}/config_without_synthtool.yaml"),
(
Expand All @@ -159,62 +163,6 @@ def test_from_yaml_without_key_fails(self, error_message_contains, path_to_yaml)
path_to_yaml,
)

def test_from_yaml_succeeds(self):
config = from_yaml(f"{test_config_dir}/generation_config.yaml")
self.assertEqual("2.34.0", config.gapic_generator_version)
self.assertEqual(25.2, config.protobuf_version)
self.assertEqual(
"1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish
)
self.assertEqual(
"sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409",
config.owlbot_cli_image,
)
self.assertEqual(
"6612ab8f3afcd5e292aecd647f0fa68812c9f5b5", config.synthtool_commitish
)
self.assertEqual(
[
".github/*",
".kokoro/*",
"samples/*",
"CODE_OF_CONDUCT.md",
"CONTRIBUTING.md",
"LICENSE",
"SECURITY.md",
"java.header",
"license-checks.xml",
"renovate.json",
".gitignore",
],
config.template_excludes,
)
library = config.libraries[0]
self.assertEqual("cloudasset", library.api_shortname)
self.assertEqual("Cloud Asset Inventory", library.name_pretty)
self.assertEqual(
"https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
library.product_documentation,
)
self.assertEqual(
"provides inventory services based on a time series database.",
library.api_description,
)
self.assertEqual("asset", library.library_name)
self.assertEqual("@googleapis/analytics-dpe", library.codeowner_team)
self.assertEqual(
"proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1",
library.excluded_poms,
)
self.assertEqual("google-iam-policy", library.excluded_dependencies)
gapics = library.gapic_configs
self.assertEqual(5, len(gapics))
self.assertEqual("google/cloud/asset/v1", gapics[0].proto_path)
self.assertEqual("google/cloud/asset/v1p1beta1", gapics[1].proto_path)
self.assertEqual("google/cloud/asset/v1p2beta1", gapics[2].proto_path)
self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path)
self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path)

@parameterized.expand(
[
("BUILD_no_additional_protos.bazel", " "),
Expand Down Expand Up @@ -481,6 +429,7 @@ def __get_a_gen_config(
return GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ def setUp(self) -> None:
self.baseline_config = GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[],
Expand All @@ -49,6 +50,7 @@ def setUp(self) -> None:
self.current_config = GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[],
Expand Down Expand Up @@ -91,6 +93,20 @@ def test_compare_config_generator_update(self):
self.assertEqual("gapic_generator_version", config_change.changed_param)
self.assertEqual("1.2.4", config_change.current_value)

def test_compare_config_libraries_bom_update(self):
self.baseline_config.libraris_bom_version = "26.36.0"
self.current_config.libraris_bom_version = "26.37.0"
result = compare_config(
baseline_config=self.baseline_config,
current_config=self.current_config,
)
self.assertTrue(
len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1
)
config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0]
self.assertEqual("libraris_bom_version", config_change.changed_param)
self.assertEqual("26.37.0", config_change.current_value)

def test_compare_config_owlbot_cli_update(self):
self.baseline_config.owlbot_cli_image = "image_version_123"
self.current_config.owlbot_cli_image = "image_version_456"
Expand Down
Loading