diff --git a/gapic/ads-templates/%namespace/%name/%version/__init__.py.j2 b/gapic/ads-templates/%namespace/%name/%version/__init__.py.j2 index 749a408c42..aa12751852 100644 --- a/gapic/ads-templates/%namespace/%name/%version/__init__.py.j2 +++ b/gapic/ads-templates/%namespace/%name/%version/__init__.py.j2 @@ -20,6 +20,7 @@ _lazy_type_to_package_map = { '{{ enum.name }}': '{{ enum.ident.package|join('.') }}.types.{{enum.ident.module }}', {%- endfor %} + {# TODO(yon-mg): add rest transport service once I know what this is #} # Client classes and transports {%- for service in api.services.values() %} '{{ service.client_name }}': '{{ service.meta.address.package|join('.') }}.services.{{ service.meta.address.module }}', diff --git a/gapic/ads-templates/%namespace/%name/__init__.py.j2 b/gapic/ads-templates/%namespace/%name/__init__.py.j2 index 749a408c42..aa12751852 100644 --- a/gapic/ads-templates/%namespace/%name/__init__.py.j2 +++ b/gapic/ads-templates/%namespace/%name/__init__.py.j2 @@ -20,6 +20,7 @@ _lazy_type_to_package_map = { '{{ enum.name }}': '{{ enum.ident.package|join('.') }}.types.{{enum.ident.module }}', {%- endfor %} + {# TODO(yon-mg): add rest transport service once I know what this is #} # Client classes and transports {%- for service in api.services.values() %} '{{ service.client_name }}': '{{ service.meta.address.package|join('.') }}.services.{{ service.meta.address.module }}', diff --git a/gapic/generator/generator.py b/gapic/generator/generator.py index 5bd8a4f3c9..45b204ce83 100644 --- a/gapic/generator/generator.py +++ b/gapic/generator/generator.py @@ -272,8 +272,11 @@ def _render_template( if "%service" in template_name: for service in api_schema.services.values(): if ( - skip_subpackages - and service.meta.address.subpackage != api_schema.subpackage_view + (skip_subpackages + and service.meta.address.subpackage != api_schema.subpackage_view) + or + ('transport' in template_name + and not self._is_desired_transport(template_name, opts)) ): continue @@ -293,6 +296,11 @@ def _render_template( template_name, api_schema=api_schema, opts=opts)) return answer + def _is_desired_transport(self, template_name: str, opts: Options) -> bool: + """Returns true if template name contains a desired transport""" + desired_transports = ['__init__', 'base'] + opts.transport + return any(transport in template_name for transport in desired_transports) + def _get_file( self, template_name: str, diff --git a/gapic/schema/wrappers.py b/gapic/schema/wrappers.py index b5ae6fd0e9..664f240ffa 100644 --- a/gapic/schema/wrappers.py +++ b/gapic/schema/wrappers.py @@ -719,6 +719,8 @@ def _client_output(self, enable_asyncio: bool): # Return the usual output. return self.output + # TODO(yon-mg): remove or rewrite: don't think it performs as intended + # e.g. doesn't work with basic case of gRPC transcoding @property def field_headers(self) -> Sequence[str]: """Return the field headers defined for this method.""" @@ -737,6 +739,35 @@ def field_headers(self) -> Sequence[str]: return next((tuple(pattern.findall(verb)) for verb in potential_verbs if verb), ()) + @property + def http_opt(self) -> Optional[Dict[str, str]]: + """Return the http option for this method. + + e.g. {'verb': 'post' + 'url': '/some/path' + 'body': '*'} + + """ + http: List[Tuple[descriptor_pb2.FieldDescriptorProto, str]] + http = self.options.Extensions[annotations_pb2.http].ListFields() + + if len(http) < 1: + return None + + http_method = http[0] + answer: Dict[str, str] = { + 'verb': http_method[0].name, + 'url': http_method[1], + } + if len(http) > 1: + body_spec = http[1] + answer[body_spec[0].name] = body_spec[1] + + # TODO(yon-mg): handle nested fields & fields past body i.e. 'additional bindings' + # TODO(yon-mg): enums for http verbs? + return answer + + # TODO(yon-mg): refactor as there may be more than one method signature @utils.cached_property def flattened_fields(self) -> Mapping[str, Field]: """Return the signature defined for this method.""" @@ -786,6 +817,7 @@ def grpc_stub_type(self) -> str: server='stream' if self.server_streaming else 'unary', ) + # TODO(yon-mg): figure out why idempotent is reliant on http annotation @utils.cached_property def idempotent(self) -> bool: """Return True if we know this method is idempotent, False otherwise. @@ -980,6 +1012,10 @@ def grpc_transport_name(self): def grpc_asyncio_transport_name(self): return self.name + "GrpcAsyncIOTransport" + @property + def rest_transport_name(self): + return self.name + "RestTransport" + @property def has_lro(self) -> bool: """Return whether the service has a long-running method.""" diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 index 9d5150d869..c6320144ef 100644 --- a/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 +++ b/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 @@ -30,6 +30,7 @@ from .transports.grpc_asyncio import {{ service.grpc_asyncio_transport_name }} from .client import {{ service.client_name }} +{# TODO(yon-mg): handle rest transport async client interaction #} class {{ service.async_client_name }}: """{{ service.meta.doc|rst(width=72, indent=4) }}""" diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 index c3093aa1cf..6ec3d5d879 100644 --- a/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 +++ b/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 @@ -30,8 +30,13 @@ from google.iam.v1 import policy_pb2 as policy # type: ignore {% endif %} {% endfilter %} from .transports.base import {{ service.name }}Transport, DEFAULT_CLIENT_INFO +{%- if 'grpc' in opts.transport %} from .transports.grpc import {{ service.grpc_transport_name }} from .transports.grpc_asyncio import {{ service.grpc_asyncio_transport_name }} +{%- endif %} +{%- if 'rest' in opts.transport %} +from .transports.rest import {{ service.name }}RestTransport +{%- endif %} class {{ service.client_name }}Meta(type): @@ -42,8 +47,13 @@ class {{ service.client_name }}Meta(type): objects. """ _transport_registry = OrderedDict() # type: Dict[str, Type[{{ service.name }}Transport]] + {%- if 'grpc' in opts.transport %} _transport_registry['grpc'] = {{ service.grpc_transport_name }} _transport_registry['grpc_asyncio'] = {{ service.grpc_asyncio_transport_name }} + {%- endif %} + {%- if 'rest' in opts.transport %} + _transport_registry['rest'] = {{ service.name }}RestTransport + {%- endif %} def get_transport_class(cls, label: str = None, diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/__init__.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/__init__.py.j2 index fa97f46164..bd7981387f 100644 --- a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/__init__.py.j2 +++ b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/__init__.py.j2 @@ -5,19 +5,34 @@ from collections import OrderedDict from typing import Dict, Type from .base import {{ service.name }}Transport +{%- if 'grpc' in opts.transport %} from .grpc import {{ service.name }}GrpcTransport from .grpc_asyncio import {{ service.name }}GrpcAsyncIOTransport +{%- endif %} +{%- if 'rest' in opts.transport %} +from .rest import {{ service.name }}RestTransport +{%- endif %} + # Compile a registry of transports. _transport_registry = OrderedDict() # type: Dict[str, Type[{{ service.name }}Transport]] +{%- if 'grpc' in opts.transport %} _transport_registry['grpc'] = {{ service.name }}GrpcTransport _transport_registry['grpc_asyncio'] = {{ service.name }}GrpcAsyncIOTransport - +{%- endif %} +{%- if 'rest' in opts.transport %} +_transport_registry['rest'] = {{ service.name }}RestTransport +{%- endif %} __all__ = ( '{{ service.name }}Transport', + {%- if 'grpc' in opts.transport %} '{{ service.name }}GrpcTransport', '{{ service.name }}GrpcAsyncIOTransport', + {%- endif %} + {%- if 'rest' in opts.transport %} + '{{ service.name }}RestTransport', + {%- endif %} ) {% endblock %} diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 index e2c68c483d..3d1f5ca9b2 100644 --- a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 +++ b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 @@ -151,6 +151,9 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport): ) self._stubs = {} # type: Dict[str, Callable] + {%- if service.has_lro %} + self._operations_client = None + {%- endif %} # Run the base constructor. super().__init__( @@ -172,7 +175,7 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport): **kwargs) -> grpc.Channel: """Create and return a gRPC channel object. Args: - address (Optionsl[str]): The host for the channel to use. + address (Optional[str]): The host for the channel to use. credentials (Optional[~.Credentials]): The authorization credentials to attach to requests. These credentials identify this application to the service. If @@ -220,13 +223,13 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport): client. """ # Sanity check: Only create a new client if we do not already have one. - if 'operations_client' not in self.__dict__: - self.__dict__['operations_client'] = operations_v1.OperationsClient( + if self._operations_client is None: + self._operations_client = operations_v1.OperationsClient( self.grpc_channel ) # Return the client from cache. - return self.__dict__['operations_client'] + return self._operations_client {%- endif %} {%- for method in service.methods.values() %} diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2 index 6399f1f1cd..5ea7031162 100644 --- a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2 +++ b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2 @@ -205,6 +205,9 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport): ) self._stubs = {} + {%- if service.has_lro %} + self._operations_client = None + {%- endif %} @property def grpc_channel(self) -> aio.Channel: @@ -225,13 +228,13 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport): client. """ # Sanity check: Only create a new client if we do not already have one. - if 'operations_client' not in self.__dict__: - self.__dict__['operations_client'] = operations_v1.OperationsAsyncClient( + if self._operations_client is None: + self._operations_client = operations_v1.OperationsAsyncClient( self.grpc_channel ) # Return the client from cache. - return self.__dict__['operations_client'] + return self._operations_client {%- endif %} {%- for method in service.methods.values() %} diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 new file mode 100644 index 0000000000..d26856dd2c --- /dev/null +++ b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 @@ -0,0 +1,175 @@ +{% extends '_base.py.j2' %} + +{% block content %} +import warnings +from typing import Callable, Dict, Optional, Sequence, Tuple + +{% if service.has_lro %} +from google.api_core import operations_v1 +{%- endif %} +from google.api_core import gapic_v1 # type: ignore +from google import auth # type: ignore +from google.auth import credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore + +from google.auth.transport.requests import AuthorizedSession + +{# TODO(yon-mg): re-add python_import/ python_modules from removed diff/current grpc template code #} +{% filter sort_lines -%} +{% for method in service.methods.values() -%} +{{ method.input.ident.python_import }} +{{ method.output.ident.python_import }} +{% endfor -%} +{% if opts.add_iam_methods %} +from google.iam.v1 import iam_policy_pb2 as iam_policy # type: ignore +from google.iam.v1 import policy_pb2 as policy # type: ignore +{% endif %} +{% endfilter %} + +from .base import {{ service.name }}Transport, DEFAULT_CLIENT_INFO + + +class {{ service.name }}RestTransport({{ service.name }}Transport): + """REST backend transport for {{ service.name }}. + + {{ service.meta.doc|rst(width=72, indent=4) }} + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends JSON representations of protocol buffers over HTTP/1.1 + """ + {# TODO(yon-mg): handle mtls stuff if that's relevant for rest transport #} + def __init__(self, *, + host: str{% if service.host %} = '{{ service.host }}'{% endif %}, + credentials: credentials.Credentials = None, + credentials_file: str = None, + scopes: Sequence[str] = None, + ssl_channel_credentials: grpc.ChannelCredentials = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiate the transport. + + Args: + host ({% if service.host %}Optional[str]{% else %}str{% endif %}): + {{- ' ' }}The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if ``channel`` is provided. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for grpc channel. It is ignored if ``channel`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + """ + super().__init__(host=host, credentials=credentials) + self._session = AuthorizedSession(self._credentials) + {%- if service.has_lro %} + self._operations_client = None + {%- endif %} + + {%- if service.has_lro %} + + @property + def operations_client(self) -> operations_v1.OperationsClient: + """Create the client designed to process long-running operations. + + This property caches on the instance; repeated calls return the same + client. + """ + # Sanity check: Only create a new client if we do not already have one. + if self._operations_client is None: + from google.api_core import grpc_helpers + self._operations_client = operations_v1.OperationsClient( + grpc_helpers.create_channel( + self._host, + credentials=self._credentials, + scopes=self.AUTH_SCOPES, + ) + ) + + # Return the client from cache. + return self._operations_client + {%- endif %} + {%- for method in service.methods.values() %} + {%- if method.http_opt %} + + def {{ method.name|snake_case }}(self, + request: {{ method.input.ident }}, *, + metadata: Sequence[Tuple[str, str]] = (), + ) -> {{ method.output.ident }}: + r"""Call the {{- ' ' -}} + {{ (method.name|snake_case).replace('_',' ')|wrap( + width=70, offset=45, indent=8) }} + {{- ' ' -}} method over HTTP. + + Args: + request (~.{{ method.input.ident }}): + The request object. + {{ method.input.meta.doc|rst(width=72, indent=16) }} + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + {%- if not method.void %} + + Returns: + ~.{{ method.output.ident }}: + {{ method.output.meta.doc|rst(width=72, indent=16) }} + {%- endif %} + """ + + {%- if 'body' in method.http_opt.keys() %} + # Jsonify the input + data = {{ method.output.ident }}.to_json( + {%- if method.http_opt['body'] == '*' %} + request + {%- else %} + request.body + {%- endif %} + ) + {%- endif %} + + {# TODO(yon-mg): Write helper method for handling grpc transcoding url #} + # TODO(yon-mg): need to handle grpc transcoding and parse url correctly + # current impl assumes simpler version of grpc transcoding + # Send the request + url = 'https://{host}{{ method.http_opt['url'] }}'.format( + host=self._host, + {%- for field in method.input.fields.keys() %} + {{ field }}=request.{{ field }}, + {%- endfor %} + ) + {% if not method.void %}response = {% endif %}self._session.{{ method.http_opt['verb'] }}( + url, + {%- if 'body' in method.http_opt.keys() %} + json=data, + {%- endif %} + ) + {%- if not method.void %} + + # Return the response + return {{ method.output.ident }}.from_json(response.content) + {%- endif %} + {%- endif %} + {%- endfor %} + + +__all__ = ( + '{{ service.name }}RestTransport', +) +{% endblock %} diff --git a/gapic/utils/options.py b/gapic/utils/options.py index c3a1ef322e..d99e34c631 100644 --- a/gapic/utils/options.py +++ b/gapic/utils/options.py @@ -40,6 +40,8 @@ class Options: lazy_import: bool = False old_naming: bool = False add_iam_methods: bool = False + # TODO(yon-mg): should there be an enum for transport type? + transport: List[str] = dataclasses.field(default_factory=lambda: []) # Class constants PYTHON_GAPIC_PREFIX: str = 'python-gapic-' @@ -49,6 +51,8 @@ class Options: 'samples', # output dir 'lazy-import', # requires >= 3.7 'add-iam-methods', # microgenerator implementation for `reroute_to_grpc_interface` + # transport type(s) delineated by '+' (i.e. grpc, rest, custom.[something], etc?) + 'transport', )) @classmethod @@ -121,6 +125,7 @@ def tweak_path(p): # Build the options instance. sample_paths = opts.pop('samples', []) + answer = Options( name=opts.pop('name', ['']).pop(), namespace=tuple(opts.pop('namespace', [])), @@ -134,6 +139,8 @@ def tweak_path(p): lazy_import=bool(opts.pop('lazy-import', False)), old_naming=bool(opts.pop('old-naming', False)), add_iam_methods=bool(opts.pop('add-iam-methods', False)), + # transport should include desired transports delimited by '+', e.g. transport='grpc+rest' + transport=opts.pop('transport', ['grpc'])[0].split('+') ) # Note: if we ever need to recursively check directories for sample diff --git a/tests/unit/generator/test_generator.py b/tests/unit/generator/test_generator.py index a258c294cf..97793e4433 100644 --- a/tests/unit/generator/test_generator.py +++ b/tests/unit/generator/test_generator.py @@ -116,6 +116,43 @@ def test_get_response_fails_invalid_file_paths(): assert "%proto" in ex_str and "%service" in ex_str +def test_get_response_ignores_unwanted_transports(): + g = make_generator() + with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as lt: + lt.return_value = [ + "foo/%service/transports/river.py.j2", + "foo/%service/transports/car.py.j2", + "foo/%service/transports/grpc.py.j2", + "foo/%service/transports/__init__.py.j2", + "foo/%service/transports/base.py.j2", + "mollusks/squid/sample.py.j2", + ] + + with mock.patch.object(jinja2.Environment, "get_template") as gt: + gt.return_value = jinja2.Template("Service: {{ service.name }}") + cgr = g.get_response( + api_schema=make_api( + make_proto( + descriptor_pb2.FileDescriptorProto( + service=[ + descriptor_pb2.ServiceDescriptorProto( + name="SomeService"), + ] + ), + ) + ), + opts=Options.build("transport=river+car") + ) + + assert len(cgr.file) == 4 + assert {i.name for i in cgr.file} == { + "foo/some_service/transports/river.py", + "foo/some_service/transports/car.py", + "foo/some_service/transports/__init__.py", + "foo/some_service/transports/base.py", + } + + def test_get_response_enumerates_services(): g = make_generator() with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as lt: diff --git a/tests/unit/schema/wrappers/test_method.py b/tests/unit/schema/wrappers/test_method.py index 8b551df560..f6db3c044b 100644 --- a/tests/unit/schema/wrappers/test_method.py +++ b/tests/unit/schema/wrappers/test_method.py @@ -249,6 +249,36 @@ def test_method_field_headers_present(): assert method.field_headers == ('parent',) +def test_method_http_opt(): + http_rule = http_pb2.HttpRule( + post='/v1/{parent=projects/*}/topics', + body='*' + ) + method = make_method('DoSomething', http_rule=http_rule) + assert method.http_opt == { + 'verb': 'post', + 'url': '/v1/{parent=projects/*}/topics', + 'body': '*' + } +# TODO(yon-mg) to test: grpc transcoding, +# correct handling of path/query params +# correct handling of body & additional binding + + +def test_method_http_opt_no_body(): + http_rule = http_pb2.HttpRule(post='/v1/{parent=projects/*}/topics') + method = make_method('DoSomething', http_rule=http_rule) + assert method.http_opt == { + 'verb': 'post', + 'url': '/v1/{parent=projects/*}/topics' + } + + +def test_method_http_opt_no_http_rule(): + method = make_method('DoSomething') + assert method.http_opt == None + + def test_method_idempotent_yes(): http_rule = http_pb2.HttpRule(get='/v1/{parent=projects/*}/topics') method = make_method('DoSomething', http_rule=http_rule) diff --git a/tests/unit/schema/wrappers/test_service.py b/tests/unit/schema/wrappers/test_service.py index c4c8d9b838..ef14e27a6f 100644 --- a/tests/unit/schema/wrappers/test_service.py +++ b/tests/unit/schema/wrappers/test_service.py @@ -59,6 +59,7 @@ def test_service_properties(): assert service.transport_name == 'ThingDoerTransport' assert service.grpc_transport_name == 'ThingDoerGrpcTransport' assert service.grpc_asyncio_transport_name == 'ThingDoerGrpcAsyncIOTransport' + assert service.rest_transport_name == 'ThingDoerRestTransport' def test_service_host():